From 742f5defd98e316d04d1ffed1a05567ee0c2466a Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 25 Feb 2025 12:36:00 -0500 Subject: [PATCH 001/289] [BI-2539] - added GitHub OAuth support --- .env.template | 4 ++ pom.xml | 5 ++ .../api/auth/GithubApiClient.java | 32 ++++++++++ .../breedinginsight/api/auth/GithubUser.java | 37 +++++++++++ .../api/auth/GithubUserDetailsMapper.java | 61 +++++++++++++++++++ src/main/resources/application.yml | 11 ++++ 6 files changed, 150 insertions(+) create mode 100644 src/main/java/org/breedinginsight/api/auth/GithubApiClient.java create mode 100644 src/main/java/org/breedinginsight/api/auth/GithubUser.java create mode 100644 src/main/java/org/breedinginsight/api/auth/GithubUserDetailsMapper.java diff --git a/.env.template b/.env.template index 66a62fb5d..5d9e7b531 100644 --- a/.env.template +++ b/.env.template @@ -3,6 +3,10 @@ USER_ID= GROUP_ID= +# GitHub OAuth variables. Only required if using GitHub as an alternative to ORCID. +GITHUB_OAUTH_CLIENT_ID= +GITHUB_OAUTH_CLIENT_SECRET= + ORCID_SANDBOX_AUTHENTICATION=use the Sandbox Orcid, false=>use the Production Orcid. Defaults to false.> # Authentication variables diff --git a/pom.xml b/pom.xml index 4280fd0b1..28d93a5a8 100644 --- a/pom.xml +++ b/pom.xml @@ -190,6 +190,11 @@ micronaut-inject compile + + io.micronaut + micronaut-http-client + compile + io.micronaut micronaut-validation diff --git a/src/main/java/org/breedinginsight/api/auth/GithubApiClient.java b/src/main/java/org/breedinginsight/api/auth/GithubApiClient.java new file mode 100644 index 000000000..a25bb9f41 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/auth/GithubApiClient.java @@ -0,0 +1,32 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.auth; + +import io.micronaut.http.annotation.Get; +import io.micronaut.http.annotation.Header; +import io.micronaut.http.client.annotation.Client; +import io.reactivex.Flowable; + +@Header(name = "User-Agent", value = "Micronaut") +@Client("https://api.github.com") +public interface GithubApiClient { + + @Get("/user") + Flowable getUser(@Header("Authorization") String authorization); +} + diff --git a/src/main/java/org/breedinginsight/api/auth/GithubUser.java b/src/main/java/org/breedinginsight/api/auth/GithubUser.java new file mode 100644 index 000000000..a71dc64fc --- /dev/null +++ b/src/main/java/org/breedinginsight/api/auth/GithubUser.java @@ -0,0 +1,37 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.auth; + +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import io.micronaut.core.annotation.Introspected; +import lombok.Getter; + +@Introspected +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +@Getter +public class GithubUser { + + private String id; + // The login will be the unique GitHub username. + private String login; + private String name; + private String email; + +} + diff --git a/src/main/java/org/breedinginsight/api/auth/GithubUserDetailsMapper.java b/src/main/java/org/breedinginsight/api/auth/GithubUserDetailsMapper.java new file mode 100644 index 000000000..02515f766 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/auth/GithubUserDetailsMapper.java @@ -0,0 +1,61 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.auth; + +import io.micronaut.core.annotation.Nullable; +import io.micronaut.core.async.publisher.Publishers; +import io.micronaut.security.authentication.AuthenticationResponse; +import io.micronaut.security.authentication.UserDetails; +import io.micronaut.security.oauth2.endpoint.authorization.state.State; +import io.micronaut.security.oauth2.endpoint.token.response.OauthUserDetailsMapper; +import io.micronaut.security.oauth2.endpoint.token.response.TokenResponse; +import lombok.extern.slf4j.Slf4j; +import org.reactivestreams.Publisher; + + +import javax.inject.Named; +import javax.inject.Singleton; +import java.util.Collections; +import java.util.List; + +@Slf4j +@Named("github") +@Singleton +class GithubUserDetailsMapper implements OauthUserDetailsMapper { + + private final GithubApiClient apiClient; + + GithubUserDetailsMapper(GithubApiClient apiClient) { + this.apiClient = apiClient; + } + + @Override + public Publisher createUserDetails(TokenResponse tokenResponse) { + return Publishers.just(new UnsupportedOperationException()); + } + + @Override + public Publisher createAuthenticationResponse(TokenResponse tokenResponse, @Nullable State state) { + return apiClient.getUser("token " + tokenResponse.getAccessToken()) + .map(user -> { + List roles = Collections.singletonList("ROLE_GITHUB"); + return new UserDetails(user.getLogin(), roles); + }); + } +} + diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 85987ef9d..3550a56b9 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -59,6 +59,17 @@ micronaut: jwks-uri: ${OAUTH_OPENID_JWKSURI:`https://sandbox.orcid.org/oauth/jwks`} user-info: url: ${OAUTH_OPENID_USERINFOURL:`https://sandbox.orcid.org/oauth/userinfo`} + github: + client-id: ${GITHUB_OAUTH_CLIENT_ID} + client-secret: ${GITHUB_OAUTH_CLIENT_SECRET} + scopes: + - user:email + - read:user + authorization: + url: https://github.com/login/oauth/authorize + token: + url: https://github.com/login/oauth/access_token + auth-method: client-secret-basic state: cookie: cookie-max-age: 10m From 6d746e4f49b601ed8e820973f069e0f3f5a98a00 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 9 Apr 2025 15:25:38 -0400 Subject: [PATCH 002/289] [BI-2212] - updated postgres version --- docker-compose.yml | 2 +- src/test/java/org/breedinginsight/DatabaseTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index bb30e858e..2932c30fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,7 @@ services: networks: - backend bidb: - image: postgres:11.4 + image: postgres:17 container_name: bidb environment: - POSTGRES_DB=${DB_NAME} diff --git a/src/test/java/org/breedinginsight/DatabaseTest.java b/src/test/java/org/breedinginsight/DatabaseTest.java index b1213ee33..ddf3043f8 100644 --- a/src/test/java/org/breedinginsight/DatabaseTest.java +++ b/src/test/java/org/breedinginsight/DatabaseTest.java @@ -62,7 +62,7 @@ public DatabaseTest() { network = Network.newNetwork(); } if(dbContainer == null) { - dbContainer = new GenericContainer<>("postgres:11.4") + dbContainer = new GenericContainer<>("postgres:17") .withNetwork(network) .withNetworkAliases("testdb") .withImagePullPolicy(PullPolicy.defaultPolicy()) From cc67eb1017259a282f6725e0207dab109ee195a3 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 14 Apr 2025 15:49:09 +0000 Subject: [PATCH 003/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 40f2de21e..a05c6c713 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,6 +14,6 @@ # limitations under the License. # -version=v1.1.0+930 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/aeb24ccc78479a5ee665c50b7c4531a7698e1603 +version=v1.1.0+932 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/32ac1b2c0c35d34606919b3a11fd78b84958ba5e From 17051e4f9b708fe158f4ddd848f721aa69022747 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Mon, 21 Apr 2025 16:15:01 -0400 Subject: [PATCH 004/289] [BI-2009] - Dynamic Concatenation of Entity + ObsUnitID (Exp UI) --- .../brapi/v2/services/BrAPITrialService.java | 18 ++++++++++++++++++ .../java/org/breedinginsight/model/Column.java | 10 ++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 6b8a3f233..19588dd20 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -228,6 +228,11 @@ public DownloadFile exportObservations( } } + //dynamically append observation level to obsUnitID column header + //todo get obslvl + String observationLvl = "Pancake"; + columns = dynamicUpdateObsUnitIDLabel(columns, observationLvl); + log.debug(logHash + ": writing data to file for export"); // If one or more envs requested, create a separate file for each env, then zip if there are multiple. if (!requestedEnvIds.isEmpty()) { @@ -306,6 +311,19 @@ private StreamedFile zipFiles(List files) throws IOException { return new StreamedFile(in, new MediaType(MediaType.APPLICATION_OCTET_STREAM)); } + public List dynamicUpdateObsUnitIDLabel(List columns, String observationLvl){ + Column oldObsUnitIDCol = new Column(ExperimentObservation.Columns.OBS_UNIT_ID, Column.ColumnDataType.STRING); + String dynamicLabel = observationLvl + " " + ExperimentObservation.Columns.OBS_UNIT_ID; + Column dynamicLabelObsUnitIDCol = new Column(dynamicLabel, Column.ColumnDataType.STRING); + //need to check index of is valid + int index = columns.indexOf(oldObsUnitIDCol); + //find item in cols with val ExperimentObservation.Columns.OBS_UNIT_ID + if (index != -1) { + columns.set(index, dynamicLabelObsUnitIDCol); + } + return columns; + } + public Dataset getDatasetData(Program program, UUID experimentId, UUID datasetId, Boolean stats) throws ApiException, DoesNotExistException { log.debug("fetching dataset: " + datasetId + " for experiment: " + experimentId + ". including stats: " + stats); log.debug("fetching observationUnits for dataset: " + datasetId); diff --git a/src/main/java/org/breedinginsight/model/Column.java b/src/main/java/org/breedinginsight/model/Column.java index 054dcb435..65b1fd392 100644 --- a/src/main/java/org/breedinginsight/model/Column.java +++ b/src/main/java/org/breedinginsight/model/Column.java @@ -21,6 +21,8 @@ import lombok.experimental.Accessors; import lombok.experimental.SuperBuilder; +import java.util.Objects; + @Getter @Setter @Accessors(chain=true) @@ -46,4 +48,12 @@ public enum ColumnDataType { INTEGER, DOUBLE } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Column that = (Column) o; + return Objects.equals(getValue(), that.getValue()) && Objects.equals(getDataType(), that.getDataType()); + } } From c3475d04d04a93273fe6225167edba814aae5481 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 23 Apr 2025 19:21:37 +0000 Subject: [PATCH 005/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 84bd05d94..3ae6ea1cc 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+936 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/5159c9d5cdaf3adb87a8e24fe32702e4cee36cf7 +version=v1.2.0+938 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f2e205ba3cfd898a309a4b3aefb740870ceba5b0 From d355316fbb8597dff1a01a0564224515a762b9ce Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 23 Apr 2025 16:27:08 -0400 Subject: [PATCH 006/289] Add germplasm name validation --- .../germplasm/GermplasmProcessor.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index d4d2389a8..bdf1f8d69 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -91,6 +91,7 @@ public class GermplasmProcessor implements Processor { public static String missingParentalGIDsMsg = "The following parental GIDs were not found in the database: %s"; public static String missingParentalEntryNoMsg = "The following parental entry numbers were not found in the database: %s"; public static String badBreedMethodsMsg = "Invalid breeding method"; + public static String badGermplasmNameMsg = "Germplasm name cannot contain /"; public static String missingEntryNumbersMsg = "Either all or none of the germplasm must have entry numbers"; public static String duplicateEntryNoMsg = "Entry numbers must be unique. Duplicated entry numbers found: %s"; public static String circularDependency = "Circular dependency in the pedigree tree"; @@ -357,6 +358,7 @@ private void processNewGermplasm(Germplasm germplasm, ValidationErrors validatio } } + validateGermplasmName(germplasm, i+2, validationErrors); validatePedigree(germplasm, i + 2, validationErrors); BrAPIGermplasm newGermplasm = germplasm.constructBrAPIGermplasm(program, breedingMethod, user, commit, BRAPI_REFERENCE_SOURCE, nextVal, importListId); @@ -543,6 +545,31 @@ private Map getStatisticsMap(List } + /** + * Validates the name of the given Germplasm, ensuring it does not contain any slash ("/") characters. + *

+ * If the germplasm name contains a "/", a new {@link ValidationError} with status + * {@code 422 Unprocessable Entity} is created and added to the provided {@code ValidationErrors} object. + * This method does not throw an exception; instead, it records validation failures by mutating + * the {@code validationErrors} parameter. + *

+ * + * @param germplasm + * the {@link Germplasm} instance whose name is to be validated; must not be {@code null} + * @param rowNumber + * the row index (for example, in a spreadsheet or CSV file) corresponding to this + * germplasm entry; used when reporting errors + * @param validationErrors + * the {@link ValidationErrors} collector into which any detected errors will be added; + * this object is modified by this method to record validation issues; must not be {@code null} + */ + private void validateGermplasmName(Germplasm germplasm, Integer rowNumber, ValidationErrors validationErrors) { + if (germplasm.getGermplasmName().contains("/")) { + ValidationError error = new ValidationError("Germplasm Name", badGermplasmNameMsg, HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowNumber, error); + } + } + private void validatePedigree(Germplasm germplasm, Integer rowNumber, ValidationErrors validationErrors) { String femaleParentEntryNo = germplasm.getFemaleParentEntryNo(); String maleParentEntryNo = germplasm.getMaleParentEntryNo(); From c68b47f2d0f90490263e53402abeb708ccf57979 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 23 Apr 2025 16:27:47 -0400 Subject: [PATCH 007/289] Add test for germplasm name validation --- .../importer/GermplasmFileImportTest.java | 29 +++++++++++++++++++ .../germplasm_import/bad_germplasm_names.csv | 4 +++ 2 files changed, 33 insertions(+) create mode 100644 src/test/resources/files/germplasm_import/bad_germplasm_names.csv diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index 13d60a714..ac2860441 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -542,6 +542,35 @@ public void badBreedingMethods() { assertEquals("Breeding Method", firstError.get("field").getAsString()); } + /** + * Test for BI-2185 adding validation for germplasm names to not allow / characters + */ + @Test + @SneakyThrows + public void badGermplasmNames() { + File file = new File("src/test/resources/files/germplasm_import/bad_germplasm_names.csv"); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad Germplasm Names"), true, client, validProgram, germplasmMappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + + JsonArray errorList = result + .getAsJsonObject("progress") + .getAsJsonArray("rowErrors"); + + assertEquals(3, errorList.size()); + + JsonObject firstError = errorList + .get(0).getAsJsonObject() + .getAsJsonArray("errors").get(0).getAsJsonObject(); + assertEquals("Germplasm name cannot contain /", firstError.get("errorMessage").getAsString()); + assertEquals("Germplasm Name", firstError.get("field").getAsString()); + } + @Test @SneakyThrows public void someEntryNumbersError() { diff --git a/src/test/resources/files/germplasm_import/bad_germplasm_names.csv b/src/test/resources/files/germplasm_import/bad_germplasm_names.csv new file mode 100644 index 000000000..22f177757 --- /dev/null +++ b/src/test/resources/files/germplasm_import/bad_germplasm_names.csv @@ -0,0 +1,4 @@ +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1/Germplasm 2,BCR,Test,,,1,,,, +,Germplasm3 / Germaplasm4,BCR,Test,,,2,,,, +,Germplasm5/Germplasm6/Germplasm7,BCR,Test,,,3,1,2,, \ No newline at end of file From c493e9b74132f00809354f6d0c25912442e83dbf Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Mon, 28 Apr 2025 10:22:02 -0400 Subject: [PATCH 008/289] [BI-2009] - retrieve obslvl --- .../brapi/v2/services/BrAPITrialService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 19588dd20..7914be8a3 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -229,8 +229,10 @@ public DownloadFile exportObservations( } //dynamically append observation level to obsUnitID column header - //todo get obslvl - String observationLvl = "Pancake"; + //check if ous always limited to dataset ous + //check if ou will always have obslvl in additional info + //check if dataset name will always be obs lvl and if that is a better place to retrieve + String observationLvl = ous.get(0).getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); columns = dynamicUpdateObsUnitIDLabel(columns, observationLvl); log.debug(logHash + ": writing data to file for export"); From 635a20373cf02489d14316c6b7571756afa0e91c Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Tue, 29 Apr 2025 04:52:42 -0400 Subject: [PATCH 009/289] [BI-2009] - Unit test fix part 1 --- .../v1/controller/ExperimentControllerIntegrationTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index a3c47e573..9adb44425 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -987,10 +987,11 @@ private void checkDownloadTable( } assertEquals(requestedImportRows.size(),matchingImportRows.size()); + //Observation level for tests should be "Plot" // Observation units populated. - assertEquals(0, table.column("ObsUnitID").countMissing()); + assertEquals(0, table.column("Plot ObsUnitID").countMissing()); // Observation Unit IDs are assigned. - assertEquals(requestedImportRows.size(), table.column("ObsUnitID").countUnique()); + assertEquals(requestedImportRows.size(), table.column("Plot ObsUnitID").countUnique()); } private boolean isMatchedRow(Map importRow, Row downloadRow) { From 2248dd173b2c8e31d2c297ff75358a117f1ddc28 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Tue, 29 Apr 2025 05:27:20 -0400 Subject: [PATCH 010/289] [BI-2009] - Unit test matching labels update --- .../v1/controller/ExperimentControllerIntegrationTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 9adb44425..90eeb107d 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -967,15 +967,18 @@ private void checkDownloadTable( matchingImportRow = requestedImportRows.stream().filter(row -> { String gid = ExperimentObservation.Columns.GERMPLASM_GID; String env = ExperimentObservation.Columns.ENV; + //For now import and export ObsUnitId labels do not match due to BI-2009 changes + //This unit test will need to be updated once import is updated to append observation lvls to ObsUnitID label String expUnitId = ExperimentObservation.Columns.EXP_UNIT_ID; + String obsLvlexpUnitId = "Plot " + expUnitId; if (extension.equalsIgnoreCase(FileType.CSV.getName())) { return Integer.parseInt(row.get(gid).toString()) == downloadRow.getInt(gid) && row.get(env).equals(downloadRow.getString(env)) && - row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); + row.get(expUnitId).equals(downloadRow.getObject(obsLvlExpUnitId).toString()); } else { return row.get(gid).equals(downloadRow.getString(gid)) && row.get(env).equals(downloadRow.getString(env)) && - row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); + row.get(expUnitId).equals(downloadRow.getObject(obsLvlExpUnitId).toString()); } }).findAny(); assertTrue(matchingImportRow.isPresent() && !matchingImportRow.get().isEmpty()); From 0e91c05bfb9984edbb8ad69cf64d895fc2c12012 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Tue, 29 Apr 2025 05:32:32 -0400 Subject: [PATCH 011/289] [BI-2009] - always those one character typos --- .../api/v1/controller/ExperimentControllerIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 90eeb107d..99ebcb843 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -970,7 +970,7 @@ private void checkDownloadTable( //For now import and export ObsUnitId labels do not match due to BI-2009 changes //This unit test will need to be updated once import is updated to append observation lvls to ObsUnitID label String expUnitId = ExperimentObservation.Columns.EXP_UNIT_ID; - String obsLvlexpUnitId = "Plot " + expUnitId; + String obsLvlExpUnitId = "Plot " + expUnitId; if (extension.equalsIgnoreCase(FileType.CSV.getName())) { return Integer.parseInt(row.get(gid).toString()) == downloadRow.getInt(gid) && row.get(env).equals(downloadRow.getString(env)) && From 7a5306ce736ecc2b5bee667dc1199b3d92e13a86 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Tue, 29 Apr 2025 06:41:17 -0400 Subject: [PATCH 012/289] [BI-2009] - Fix row labels --- .../brapi/v2/services/BrAPITrialService.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 7914be8a3..34ec2b28f 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -229,9 +229,6 @@ public DownloadFile exportObservations( } //dynamically append observation level to obsUnitID column header - //check if ous always limited to dataset ous - //check if ou will always have obslvl in additional info - //check if dataset name will always be obs lvl and if that is a better place to retrieve String observationLvl = ous.get(0).getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); columns = dynamicUpdateObsUnitIDLabel(columns, observationLvl); @@ -255,7 +252,7 @@ public DownloadFile exportObservations( // Generate a file for each study. for (Map.Entry>> entry: rowsByStudyId.entrySet()) { List> rows = entry.getValue(); - sortDefaultForExportRows(rows); + sortDefaultForExportRows(rows, observationLvl); StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, rows, fileType, SHEET_NAME); // TODO: [BI-2183] remove hardcoded datasetName, use observation level. String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName(), "Observation Dataset") + fileType.getExtension(); @@ -274,7 +271,7 @@ public DownloadFile exportObservations( } } else { List> exportRows = new ArrayList<>(rowByOUId.values()); - sortDefaultForExportRows(exportRows); + sortDefaultForExportRows(exportRows, observationLvl); // write export data to requested file format StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, exportRows, fileType, SHEET_NAME); // Set filename. @@ -776,7 +773,10 @@ private Map createExportRow( BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); - row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + + String observationLvl = ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); + String expUnitIDLabel = observationLvl + " " + ExperimentObservation.Columns.EXP_UNIT_ID; + row.put(expUnitIDLabel, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); // get replicate number Optional repLevel = ou.getObservationUnitPosition() @@ -891,10 +891,11 @@ private void sortDefaultForObservationUnit(List ous) { ous.sort( (studyNameComparator).thenComparing(ouNameComparator)); } - private void sortDefaultForExportRows(@NotNull List> exportRows) { + private void sortDefaultForExportRows(@NotNull List> exportRows, String observationLvl) { Comparator> envComparator = Comparator.comparing(row -> (row.get(Columns.ENV).toString()), new IntOrderComparator()); + String obsUnitIDLabel = observationLvl + " " + Columns.EXP_UNIT_ID; Comparator> expUnitIdComparator = - Comparator.comparing(row -> (row.get(Columns.EXP_UNIT_ID).toString()), new IntOrderComparator()); + Comparator.comparing(row -> (row.get(obsUnitIDLabel).toString()), new IntOrderComparator()); exportRows.sort(envComparator.thenComparing(expUnitIdComparator)); } From a04cb40049689a274ad4c204faa4a4dcd569a641 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 30 Apr 2025 16:08:18 +0000 Subject: [PATCH 013/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 3ae6ea1cc..cb9ca73dd 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+938 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f2e205ba3cfd898a309a4b3aefb740870ceba5b0 +version=v1.2.0+953 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/60ad8ba9be853f68b625505b49897b47ef7a17af From 992cb70cdd4dca5966cc92b398dae1d43e31f84b Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Thu, 1 May 2025 01:42:46 -0400 Subject: [PATCH 014/289] [BI-2009] - fixed variable that gets appended --- .../brapi/v2/services/BrAPITrialService.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 34ec2b28f..1f4bacee7 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -252,7 +252,7 @@ public DownloadFile exportObservations( // Generate a file for each study. for (Map.Entry>> entry: rowsByStudyId.entrySet()) { List> rows = entry.getValue(); - sortDefaultForExportRows(rows, observationLvl); + sortDefaultForExportRows(rows); StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, rows, fileType, SHEET_NAME); // TODO: [BI-2183] remove hardcoded datasetName, use observation level. String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName(), "Observation Dataset") + fileType.getExtension(); @@ -271,7 +271,7 @@ public DownloadFile exportObservations( } } else { List> exportRows = new ArrayList<>(rowByOUId.values()); - sortDefaultForExportRows(exportRows, observationLvl); + sortDefaultForExportRows(exportRows); // write export data to requested file format StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, exportRows, fileType, SHEET_NAME); // Set filename. @@ -773,10 +773,7 @@ private Map createExportRow( BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); - - String observationLvl = ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); - String expUnitIDLabel = observationLvl + " " + ExperimentObservation.Columns.EXP_UNIT_ID; - row.put(expUnitIDLabel, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); // get replicate number Optional repLevel = ou.getObservationUnitPosition() @@ -805,7 +802,10 @@ private Map createExportRow( } else { row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, null); } - row.put(ExperimentObservation.Columns.OBS_UNIT_ID, ouId); + + //Append observation level to obsUnitID + String observationLvl = ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); + row.put(observationLvl + " " + ExperimentObservation.Columns.OBS_UNIT_ID, ouId); return row; } @@ -891,11 +891,10 @@ private void sortDefaultForObservationUnit(List ous) { ous.sort( (studyNameComparator).thenComparing(ouNameComparator)); } - private void sortDefaultForExportRows(@NotNull List> exportRows, String observationLvl) { + private void sortDefaultForExportRows(@NotNull List> exportRows) { Comparator> envComparator = Comparator.comparing(row -> (row.get(Columns.ENV).toString()), new IntOrderComparator()); - String obsUnitIDLabel = observationLvl + " " + Columns.EXP_UNIT_ID; Comparator> expUnitIdComparator = - Comparator.comparing(row -> (row.get(obsUnitIDLabel).toString()), new IntOrderComparator()); + Comparator.comparing(row -> (row.get(Columns.EXP_UNIT_ID).toString()), new IntOrderComparator()); exportRows.sort(envComparator.thenComparing(expUnitIdComparator)); } From fe7de2d04ceb7b77783cf927438da5a2a6a7bc84 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Thu, 1 May 2025 03:11:31 -0400 Subject: [PATCH 015/289] [BI-2009] - Unit test fixing --- .../controller/ExperimentControllerIntegrationTest.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 99ebcb843..461664ce4 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -967,18 +967,15 @@ private void checkDownloadTable( matchingImportRow = requestedImportRows.stream().filter(row -> { String gid = ExperimentObservation.Columns.GERMPLASM_GID; String env = ExperimentObservation.Columns.ENV; - //For now import and export ObsUnitId labels do not match due to BI-2009 changes - //This unit test will need to be updated once import is updated to append observation lvls to ObsUnitID label - String expUnitId = ExperimentObservation.Columns.EXP_UNIT_ID; - String obsLvlExpUnitId = "Plot " + expUnitId; + String expUnitId = ExperimentObservation.Columns.EXP_UNIT_ID; if (extension.equalsIgnoreCase(FileType.CSV.getName())) { return Integer.parseInt(row.get(gid).toString()) == downloadRow.getInt(gid) && row.get(env).equals(downloadRow.getString(env)) && - row.get(expUnitId).equals(downloadRow.getObject(obsLvlExpUnitId).toString()); + row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); } else { return row.get(gid).equals(downloadRow.getString(gid)) && row.get(env).equals(downloadRow.getString(env)) && - row.get(expUnitId).equals(downloadRow.getObject(obsLvlExpUnitId).toString()); + row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); } }).findAny(); assertTrue(matchingImportRow.isPresent() && !matchingImportRow.get().isEmpty()); From 3ebbe379c30f964f24a39378bd00f7f54ce0aa98 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Thu, 1 May 2025 06:39:07 -0400 Subject: [PATCH 016/289] [BI-2009] - Multi env case --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 1f4bacee7..a383a96a0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -237,9 +237,10 @@ public DownloadFile exportObservations( if (!requestedEnvIds.isEmpty()) { // This will hold a list of rows for each study, each list will become a separate file. Map>> rowsByStudyId = new HashMap<>(); + String obsUnitIDLabel = observationLvl + " " + ExperimentObservation.Columns.OBS_UNIT_ID; for (Map row: rowByOUId.values()) { - String studyId = studyDbIdByOUId.get((String)row.get(ExperimentObservation.Columns.OBS_UNIT_ID)); + String studyId = studyDbIdByOUId.get((String)row.get(obsUnitIDLabel)); // Initialize key with empty list if it is not present. if (!rowsByStudyId.containsKey(studyId)) { From 29b14cc6b7cf8c9b0ced8b1324b5de22f6868194 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Thu, 1 May 2025 07:11:48 -0400 Subject: [PATCH 017/289] [BI-2009] - Cleanup --- .../api/v1/controller/ExperimentControllerIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 461664ce4..9adb44425 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -967,7 +967,7 @@ private void checkDownloadTable( matchingImportRow = requestedImportRows.stream().filter(row -> { String gid = ExperimentObservation.Columns.GERMPLASM_GID; String env = ExperimentObservation.Columns.ENV; - String expUnitId = ExperimentObservation.Columns.EXP_UNIT_ID; + String expUnitId = ExperimentObservation.Columns.EXP_UNIT_ID; if (extension.equalsIgnoreCase(FileType.CSV.getName())) { return Integer.parseInt(row.get(gid).toString()) == downloadRow.getInt(gid) && row.get(env).equals(downloadRow.getString(env)) && From e5045ccc66b8a30dc1ce5d7491ec98833476016d Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Mon, 5 May 2025 11:42:16 -0400 Subject: [PATCH 018/289] [BI-2009] - Override hashCode --- src/main/java/org/breedinginsight/model/Column.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/breedinginsight/model/Column.java b/src/main/java/org/breedinginsight/model/Column.java index 65b1fd392..52b4a9de3 100644 --- a/src/main/java/org/breedinginsight/model/Column.java +++ b/src/main/java/org/breedinginsight/model/Column.java @@ -56,4 +56,9 @@ public boolean equals(Object o) { Column that = (Column) o; return Objects.equals(getValue(), that.getValue()) && Objects.equals(getDataType(), that.getDataType()); } + + @Override + public int hashCode() { + return Objects.hash(getValue(), getDataType()); + } } From b3aff546ed938fe645380abd13729b0edfaa0c88 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Mon, 5 May 2025 11:43:57 -0400 Subject: [PATCH 019/289] [BI-2009] - Another hashcode fix --- src/main/java/org/breedinginsight/model/Country.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/breedinginsight/model/Country.java b/src/main/java/org/breedinginsight/model/Country.java index e2d93dbfe..4cbe21997 100644 --- a/src/main/java/org/breedinginsight/model/Country.java +++ b/src/main/java/org/breedinginsight/model/Country.java @@ -81,4 +81,9 @@ public boolean equals(Object o) { Country that = (Country) o; return Objects.equals(getId(), that.getId()) && Objects.equals(getName(), that.getName()); } + + @Override + public int hashCode() { + return Objects.hash(getId(), getName()); + } } From ac7e1c4f659aabe2501ff4536e26cf7d6517de2f Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 6 May 2025 11:38:46 -0400 Subject: [PATCH 020/289] Update species migration to work with new uuids --- src/main/resources/brapi/sql/R__species.sql | 65 ++++++++++++--------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/src/main/resources/brapi/sql/R__species.sql b/src/main/resources/brapi/sql/R__species.sql index 00c1b7042..843ed32ce 100644 --- a/src/main/resources/brapi/sql/R__species.sql +++ b/src/main/resources/brapi/sql/R__species.sql @@ -13,32 +13,39 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '4', 'Blueberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '5', 'Salmon') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '6', 'Grape') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '7', 'Alfalfa') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '8', 'Sweet Potato') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '9', 'Trout') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '10', 'Soybean') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '11', 'Cranberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '12', 'Cucumber') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'Oat') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING; --- for the Honey Bee case, want to overwrite name, not preserve existing -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey Bee') ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '21', 'Sorghum') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '22', 'Hemp') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '23', 'Hop') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '24', 'Hydrangea') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '25', 'Red Clover') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '26', 'Potato') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '27', 'Blackberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '28', 'Raspberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '29', 'Sugar Beet') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '30', 'Strawberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '31', 'Coffee') ON CONFLICT DO NOTHING; \ No newline at end of file +-- for uuid_generate_v4() +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +DO $$ +DECLARE + v_auth_id constant uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; +BEGIN + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Blueberry') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Salmon') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Grape') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Alfalfa') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sweet Potato') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Trout') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Soybean') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Cranberry') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Cucumber') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Oat') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Citrus') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sugar Cane') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Strawberry') ON CONFLICT DO NOTHING; + -- for the Honey Bee case, want to overwrite name, not preserve existing + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Honey Bee') ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Pecan') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Lettuce') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Cotton') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sorghum') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Hemp') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Hop') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Hydrangea') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Red Clover') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Potato') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Blackberry') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Raspberry') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sugar Beet') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Coffee') ON CONFLICT DO NOTHING; +END $$; \ No newline at end of file From 8a285016ac06649fbd77d37a03140c6e985b568f Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 7 May 2025 15:35:59 +0000 Subject: [PATCH 021/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 5ac440627..be65b691c 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+959 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/738dee1bea47d07b5e24a37304dd83e6d3f29b65 \ No newline at end of file +version=v1.2.0+961 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/06df6ae9acb10d77620dc978d8f1c290dab21324 \ No newline at end of file From bf60e06899ab680689480807083372b7acc606dc Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 7 Apr 2025 17:00:27 -0400 Subject: [PATCH 022/289] Use appropriate ids for both preview and commit --- .../processors/GermplasmProcessor.java | 94 ++++++++++++++----- .../duplicate_names_circular_dependency.csv | 6 ++ 2 files changed, 78 insertions(+), 22 deletions(-) create mode 100644 src/test/resources/files/germplasm_import/duplicate_names_circular_dependency.csv diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index aad1720d9..b6987488c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapps.importer.services.processors; import com.google.gson.Gson; +import com.google.gson.JsonElement; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.HttpStatus; @@ -58,6 +59,9 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; + @Slf4j @Prototype public class GermplasmProcessor implements Processor { @@ -327,10 +331,7 @@ public Map process(ImportUpload upload, List
[ - ]) - // for !commit: Validate for circular pedigree dependencies. - createPostOrder(commit); + createPostOrder(); // Construct our response object return getStatisticsMap(importRows); @@ -428,11 +429,11 @@ private boolean canUpdatePedigree(BrAPIGermplasm existingGermplasm, Germplasm ge } private boolean hasPedigreeString(BrAPIGermplasm germplasm) { - return StringUtils.isNotBlank(germplasm.getPedigree()); + return isNotBlank(germplasm.getPedigree()); } private boolean hasPedigree(BrAPIGermplasm germplasm) { - return StringUtils.isNotBlank(germplasm.getPedigree()) + return isNotBlank(germplasm.getPedigree()) || germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_GID) || germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID) || (germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.FEMALE_PARENT_UNKNOWN) && @@ -458,9 +459,9 @@ private boolean arePedigreesEqual(BrAPIGermplasm existingGermplasm, Germplasm ge String existingMalePedigree = getParentId(existingGermplasm, existingPedigreeGIDString, BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID, BrAPIAdditionalInfoFields.MALE_PARENT_UNKNOWN); StringBuilder germplasmPedigreeGIDString = new StringBuilder(); - if (StringUtils.isNotBlank(germplasm.getFemaleParentAccessionNumber())) { + if (isNotBlank(germplasm.getFemaleParentAccessionNumber())) { germplasmPedigreeGIDString.append(germplasm.getFemaleParentAccessionNumber()); - } else if (StringUtils.isNotBlank(germplasm.getFemaleParentEntryNo())) { + } else if (isNotBlank(germplasm.getFemaleParentEntryNo())) { Integer femaleParentIdx = germplasmIndexByEntryNo.get(germplasm.getFemaleParentEntryNo()); BrAPIImport femaleParentRow = importRows.get(femaleParentIdx); BrAPIGermplasm femaleGerm = dbGermplasmByName.get(femaleParentRow.getGermplasm() @@ -474,9 +475,9 @@ private boolean arePedigreesEqual(BrAPIGermplasm existingGermplasm, Germplasm ge germplasmPedigreeGIDString.append(existingFemalePedigree); } germplasmPedigreeGIDString.append("/"); - if (StringUtils.isNotBlank(germplasm.getMaleParentAccessionNumber())) { + if (isNotBlank(germplasm.getMaleParentAccessionNumber())) { germplasmPedigreeGIDString.append(germplasm.getMaleParentAccessionNumber()); - } else if (StringUtils.isNotBlank(germplasm.getMaleParentEntryNo())) { + } else if (isNotBlank(germplasm.getMaleParentEntryNo())) { Integer maleParentIdx = germplasmIndexByEntryNo.get(germplasm.getMaleParentEntryNo()); BrAPIImport maleParentRow = importRows.get(maleParentIdx); BrAPIGermplasm maleGerm = dbGermplasmByName.get(maleParentRow.getGermplasm() @@ -518,7 +519,7 @@ private String getParentId(BrAPIGermplasm existingGermplasm, StringBuilder pedig private boolean canUpdatePedigreeNoEqualsCheck(BrAPIGermplasm existingGermplasm, Germplasm germplasm) { - return StringUtils.isBlank(existingGermplasm.getPedigree()) && + return isBlank(existingGermplasm.getPedigree()) && germplasm.pedigreeExists(); } @@ -550,25 +551,62 @@ private void validatePedigree(Germplasm germplasm, Integer rowNumber, Validation String femaleParentGID = germplasm.getFemaleParentAccessionNumber(); String maleParentGID = germplasm.getMaleParentAccessionNumber(); - if(StringUtils.isNotBlank(maleParentEntryNo) && StringUtils.isBlank(femaleParentEntryNo) && StringUtils.isBlank(femaleParentGID)) { + if(isNotBlank(maleParentEntryNo) && isBlank(femaleParentEntryNo) && isBlank(femaleParentGID)) { validationErrors.addError(rowNumber, new ValidationError("Male Parent Entry No", missingFemaleParent, HttpStatus.UNPROCESSABLE_ENTITY)); - } else if(StringUtils.isNotBlank(maleParentGID) && StringUtils.isBlank(femaleParentEntryNo) && StringUtils.isBlank(femaleParentGID)) { + } else if(isNotBlank(maleParentGID) && isBlank(femaleParentEntryNo) && isBlank(femaleParentGID)) { validationErrors.addError(rowNumber, new ValidationError("Male Parent GID", missingFemaleParent, HttpStatus.UNPROCESSABLE_ENTITY)); } } + private String getImportId(BrAPIGermplasm germplasm) { + String gid = germplasm.getAccessionNumber(); + String entryNo = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER).getAsString(); + return generateImportId(gid, entryNo); + } + + private String getMotherImportId(BrAPIGermplasm germplasm) { + JsonElement motherGidElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_GID); + JsonElement motherEntryNoElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_ENTRY_NO); + String motherGid = !motherGidElement.isJsonNull() ? motherGidElement.getAsString() : null; + String motherEntryNo = !motherEntryNoElement.isJsonNull() ? motherEntryNoElement.getAsString() : null; + return generateImportId(motherGid, motherEntryNo); + } + + private String getFatherImportId(BrAPIGermplasm germplasm) { + JsonElement fatherGidElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID); + JsonElement fatherEntryNoElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO); + String fatherGid = !fatherGidElement.isJsonNull() ? fatherGidElement.getAsString() : null; + String fatherEntryNo = !fatherEntryNoElement.isJsonNull() ? fatherEntryNoElement.getAsString() : null; + return generateImportId(fatherGid, fatherEntryNo); + } + + private String generateImportId(String gid, String entryNo) { + if (gid == null && entryNo == null) return null; + return isNotBlank(gid) ? "GID " + gid : "ENTRY NO " + entryNo; + } + + private boolean maleParentPresent(BrAPIGermplasm germplasm) { + boolean fatherGidNull = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID).isJsonNull(); + boolean fatherEntryNoNull = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO).isJsonNull(); + return !fatherGidNull || !fatherEntryNoNull; + } + + private boolean femaleParentUnknown(BrAPIGermplasm germplasm) { + return germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.FEMALE_PARENT_UNKNOWN).getAsBoolean(); + } + + private boolean maleParentUnknown(BrAPIGermplasm germplasm) { + return germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.MALE_PARENT_UNKNOWN).getAsBoolean(); + } + /* This will set the postOrder and validate for circular pedigree dependencies. */ - private void createPostOrder(boolean commit) { + private void createPostOrder() { + Set created = null; // Construct a dependency tree for POSTing order - if(commit){ - created = existingGermplasm.stream().map(BrAPIGermplasm::getGermplasmName).collect(Collectors.toSet()); - } - else { - created = existingGermplasm.stream().map(BrAPIGermplasm::getDefaultDisplayName).collect(Collectors.toSet()); - } + created = existingGermplasm.stream().map(this::getImportId).collect(Collectors.toSet()); //todo this gets messy @@ -579,7 +617,7 @@ private void createPostOrder(boolean commit) { for (BrAPIGermplasm germplasm : newGermplasmList) { // If we've already planned this germplasm, skip - if ( (commit && created.contains(germplasm.getGermplasmName())) || (!commit && created.contains(germplasm.getDefaultDisplayName())) ) { + if (created.contains(getImportId(germplasm))) { continue; } @@ -589,8 +627,19 @@ private void createPostOrder(boolean commit) { continue; } + String femaleImportId = getMotherImportId(germplasm); + String maleImportId = getFatherImportId(germplasm); + + if (created.contains(femaleImportId) || femaleParentUnknown(germplasm)) { + if (!maleParentPresent(germplasm) || created.contains(maleImportId) || maleParentUnknown(germplasm)) { + createList.add(germplasm); + } + } + + /* // If both parents have been created already, add it List pedigreeArray = List.of(germplasm.getPedigree().split("/")); + // name + gid or name + entry no if no gid String femaleParent = pedigreeArray.get(0); String maleParent = pedigreeArray.size() > 1 ? pedigreeArray.get(1) : null; if (created.contains(femaleParent) || germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.FEMALE_PARENT_UNKNOWN).getAsBoolean()) { @@ -598,11 +647,12 @@ private void createPostOrder(boolean commit) { createList.add(germplasm); } } + */ } totalRecorded += createList.size(); if (createList.size() > 0) { - created.addAll(createList.stream().map(BrAPIGermplasm::getGermplasmName).collect(Collectors.toList())); + created.addAll(createList.stream().map(this::getImportId).collect(Collectors.toList())); postOrder.add(createList); } else if (totalRecorded < newGermplasmList.size()) { // We ran into circular dependencies, throw an error diff --git a/src/test/resources/files/germplasm_import/duplicate_names_circular_dependency.csv b/src/test/resources/files/germplasm_import/duplicate_names_circular_dependency.csv new file mode 100644 index 000000000..a602f6233 --- /dev/null +++ b/src/test/resources/files/germplasm_import/duplicate_names_circular_dependency.csv @@ -0,0 +1,6 @@ +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,TestDup,BCR,Test,1,2,1,,,1, +,TestDup,BCR,Test,1,,2,,3,2, +,TestDup,BCR,Test,,,3,,,3, +,TestDup,BCR,Test,,,4,,,4, +,TestDup,BCR,Test,,,5,3,4,5, \ No newline at end of file From 274539418a6163560d7d6fd6f0189eccc4c5efbb Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 8 Apr 2025 11:02:39 -0400 Subject: [PATCH 023/289] Refactored to clean up a bit --- .../germplasm/GermplasmImportService.java | 7 +- .../germplasm/GermplasmImportIdUtils.java | 94 +++++++++++++++++++ .../{ => germplasm}/GermplasmProcessor.java | 91 ++++-------------- .../importer/GermplasmFileImportTest.java | 3 +- 4 files changed, 114 insertions(+), 81 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmImportIdUtils.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/{ => germplasm}/GermplasmProcessor.java (89%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index 0caebe65e..64d4c3b23 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -18,18 +18,13 @@ package org.breedinginsight.brapps.importer.model.imports.germplasm; import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; +import org.breedinginsight.brapps.importer.services.processors.germplasm.GermplasmProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import tech.tablesaw.api.Table; import javax.inject.Inject; import javax.inject.Provider; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmImportIdUtils.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmImportIdUtils.java new file mode 100644 index 000000000..4fe10e0c7 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmImportIdUtils.java @@ -0,0 +1,94 @@ +package org.breedinginsight.brapps.importer.services.processors.germplasm; + +import com.google.gson.JsonElement; +import org.apache.commons.lang3.StringUtils; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; + +/** + * Utility class for managing germplasm import identifiers and pedigree relationships. + */ +public class GermplasmImportIdUtils { + + private GermplasmImportIdUtils() { + // Private constructor to prevent instantiation + } + + /** + * Generates an import ID for a germplasm based on its GID or entry number. + * @param gid The germplasm ID + * @param entryNo The entry number + * @return The generated import ID or null if both parameters are null + */ + public static String generateImportId(String gid, String entryNo) { + if (gid == null && entryNo == null) return null; + return StringUtils.isNotBlank(gid) ? "GID " + gid : "ENTRY NO " + entryNo; + } + + /** + * Gets the import ID for a germplasm. + * @param germplasm The germplasm object + * @return The import ID + */ + public static String getImportId(BrAPIGermplasm germplasm) { + String gid = germplasm.getAccessionNumber(); + String entryNo = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER).getAsString(); + return generateImportId(gid, entryNo); + } + + /** + * Gets the import ID for the mother/female parent of a germplasm. + * @param germplasm The germplasm object + * @return The import ID of the mother/female parent + */ + public static String getMotherImportId(BrAPIGermplasm germplasm) { + JsonElement motherGidElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_GID); + JsonElement motherEntryNoElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_ENTRY_NO); + String motherGid = !motherGidElement.isJsonNull() ? motherGidElement.getAsString() : null; + String motherEntryNo = !motherEntryNoElement.isJsonNull() ? motherEntryNoElement.getAsString() : null; + return generateImportId(motherGid, motherEntryNo); + } + + /** + * Gets the import ID for the father/male parent of a germplasm. + * @param germplasm The germplasm object + * @return The import ID of the father/male parent + */ + public static String getFatherImportId(BrAPIGermplasm germplasm) { + JsonElement fatherGidElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID); + JsonElement fatherEntryNoElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO); + String fatherGid = !fatherGidElement.isJsonNull() ? fatherGidElement.getAsString() : null; + String fatherEntryNo = !fatherEntryNoElement.isJsonNull() ? fatherEntryNoElement.getAsString() : null; + return generateImportId(fatherGid, fatherEntryNo); + } + + /** + * Checks if a male parent is present for a germplasm. + * @param germplasm The germplasm object + * @return true if a male parent is present, false otherwise + */ + public static boolean maleParentPresent(BrAPIGermplasm germplasm) { + boolean fatherGidNull = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID).isJsonNull(); + boolean fatherEntryNoNull = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO).isJsonNull(); + return !fatherGidNull || !fatherEntryNoNull; + } + + /** + * Checks if the female parent is unknown for a germplasm. + * @param germplasm The germplasm object + * @return true if the female parent is unknown, false otherwise + */ + public static boolean femaleParentUnknown(BrAPIGermplasm germplasm) { + return germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.FEMALE_PARENT_UNKNOWN).getAsBoolean(); + } + + /** + * Checks if the male parent is unknown for a germplasm. + * @param germplasm The germplasm object + * @return true if the male parent is unknown, false otherwise + */ + public static boolean maleParentUnknown(BrAPIGermplasm germplasm) { + return germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.MALE_PARENT_UNKNOWN).getAsBoolean(); + } + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java similarity index 89% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index b6987488c..d4d2389a8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.breedinginsight.brapps.importer.services.processors; +package org.breedinginsight.brapps.importer.services.processors.germplasm; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -43,6 +43,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.dao.db.tables.pojos.ProgramBreedingMethodEntity; import org.breedinginsight.daos.BreedingMethodDAO; import org.breedinginsight.model.Program; @@ -59,9 +60,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.apache.commons.lang3.StringUtils.isBlank; -import static org.apache.commons.lang3.StringUtils.isNotBlank; - @Slf4j @Prototype public class GermplasmProcessor implements Processor { @@ -429,11 +427,11 @@ private boolean canUpdatePedigree(BrAPIGermplasm existingGermplasm, Germplasm ge } private boolean hasPedigreeString(BrAPIGermplasm germplasm) { - return isNotBlank(germplasm.getPedigree()); + return StringUtils.isNotBlank(germplasm.getPedigree()); } private boolean hasPedigree(BrAPIGermplasm germplasm) { - return isNotBlank(germplasm.getPedigree()) + return StringUtils.isNotBlank(germplasm.getPedigree()) || germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_GID) || germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID) || (germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.FEMALE_PARENT_UNKNOWN) && @@ -459,9 +457,9 @@ private boolean arePedigreesEqual(BrAPIGermplasm existingGermplasm, Germplasm ge String existingMalePedigree = getParentId(existingGermplasm, existingPedigreeGIDString, BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID, BrAPIAdditionalInfoFields.MALE_PARENT_UNKNOWN); StringBuilder germplasmPedigreeGIDString = new StringBuilder(); - if (isNotBlank(germplasm.getFemaleParentAccessionNumber())) { + if (StringUtils.isNotBlank(germplasm.getFemaleParentAccessionNumber())) { germplasmPedigreeGIDString.append(germplasm.getFemaleParentAccessionNumber()); - } else if (isNotBlank(germplasm.getFemaleParentEntryNo())) { + } else if (StringUtils.isNotBlank(germplasm.getFemaleParentEntryNo())) { Integer femaleParentIdx = germplasmIndexByEntryNo.get(germplasm.getFemaleParentEntryNo()); BrAPIImport femaleParentRow = importRows.get(femaleParentIdx); BrAPIGermplasm femaleGerm = dbGermplasmByName.get(femaleParentRow.getGermplasm() @@ -475,9 +473,9 @@ private boolean arePedigreesEqual(BrAPIGermplasm existingGermplasm, Germplasm ge germplasmPedigreeGIDString.append(existingFemalePedigree); } germplasmPedigreeGIDString.append("/"); - if (isNotBlank(germplasm.getMaleParentAccessionNumber())) { + if (StringUtils.isNotBlank(germplasm.getMaleParentAccessionNumber())) { germplasmPedigreeGIDString.append(germplasm.getMaleParentAccessionNumber()); - } else if (isNotBlank(germplasm.getMaleParentEntryNo())) { + } else if (StringUtils.isNotBlank(germplasm.getMaleParentEntryNo())) { Integer maleParentIdx = germplasmIndexByEntryNo.get(germplasm.getMaleParentEntryNo()); BrAPIImport maleParentRow = importRows.get(maleParentIdx); BrAPIGermplasm maleGerm = dbGermplasmByName.get(maleParentRow.getGermplasm() @@ -519,7 +517,7 @@ private String getParentId(BrAPIGermplasm existingGermplasm, StringBuilder pedig private boolean canUpdatePedigreeNoEqualsCheck(BrAPIGermplasm existingGermplasm, Germplasm germplasm) { - return isBlank(existingGermplasm.getPedigree()) && + return StringUtils.isBlank(existingGermplasm.getPedigree()) && germplasm.pedigreeExists(); } @@ -551,54 +549,13 @@ private void validatePedigree(Germplasm germplasm, Integer rowNumber, Validation String femaleParentGID = germplasm.getFemaleParentAccessionNumber(); String maleParentGID = germplasm.getMaleParentAccessionNumber(); - if(isNotBlank(maleParentEntryNo) && isBlank(femaleParentEntryNo) && isBlank(femaleParentGID)) { + if(StringUtils.isNotBlank(maleParentEntryNo) && StringUtils.isBlank(femaleParentEntryNo) && StringUtils.isBlank(femaleParentGID)) { validationErrors.addError(rowNumber, new ValidationError("Male Parent Entry No", missingFemaleParent, HttpStatus.UNPROCESSABLE_ENTITY)); - } else if(isNotBlank(maleParentGID) && isBlank(femaleParentEntryNo) && isBlank(femaleParentGID)) { + } else if(StringUtils.isNotBlank(maleParentGID) && StringUtils.isBlank(femaleParentEntryNo) && StringUtils.isBlank(femaleParentGID)) { validationErrors.addError(rowNumber, new ValidationError("Male Parent GID", missingFemaleParent, HttpStatus.UNPROCESSABLE_ENTITY)); } } - private String getImportId(BrAPIGermplasm germplasm) { - String gid = germplasm.getAccessionNumber(); - String entryNo = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER).getAsString(); - return generateImportId(gid, entryNo); - } - - private String getMotherImportId(BrAPIGermplasm germplasm) { - JsonElement motherGidElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_GID); - JsonElement motherEntryNoElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_ENTRY_NO); - String motherGid = !motherGidElement.isJsonNull() ? motherGidElement.getAsString() : null; - String motherEntryNo = !motherEntryNoElement.isJsonNull() ? motherEntryNoElement.getAsString() : null; - return generateImportId(motherGid, motherEntryNo); - } - - private String getFatherImportId(BrAPIGermplasm germplasm) { - JsonElement fatherGidElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID); - JsonElement fatherEntryNoElement = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO); - String fatherGid = !fatherGidElement.isJsonNull() ? fatherGidElement.getAsString() : null; - String fatherEntryNo = !fatherEntryNoElement.isJsonNull() ? fatherEntryNoElement.getAsString() : null; - return generateImportId(fatherGid, fatherEntryNo); - } - - private String generateImportId(String gid, String entryNo) { - if (gid == null && entryNo == null) return null; - return isNotBlank(gid) ? "GID " + gid : "ENTRY NO " + entryNo; - } - - private boolean maleParentPresent(BrAPIGermplasm germplasm) { - boolean fatherGidNull = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID).isJsonNull(); - boolean fatherEntryNoNull = germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO).isJsonNull(); - return !fatherGidNull || !fatherEntryNoNull; - } - - private boolean femaleParentUnknown(BrAPIGermplasm germplasm) { - return germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.FEMALE_PARENT_UNKNOWN).getAsBoolean(); - } - - private boolean maleParentUnknown(BrAPIGermplasm germplasm) { - return germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.MALE_PARENT_UNKNOWN).getAsBoolean(); - } - /* This will set the postOrder and validate for circular pedigree dependencies. */ @@ -606,7 +563,7 @@ private void createPostOrder() { Set created = null; // Construct a dependency tree for POSTing order - created = existingGermplasm.stream().map(this::getImportId).collect(Collectors.toSet()); + created = existingGermplasm.stream().map(GermplasmImportIdUtils::getImportId).collect(Collectors.toSet()); //todo this gets messy @@ -617,7 +574,7 @@ private void createPostOrder() { for (BrAPIGermplasm germplasm : newGermplasmList) { // If we've already planned this germplasm, skip - if (created.contains(getImportId(germplasm))) { + if (created.contains(GermplasmImportIdUtils.getImportId(germplasm))) { continue; } @@ -627,32 +584,20 @@ private void createPostOrder() { continue; } - String femaleImportId = getMotherImportId(germplasm); - String maleImportId = getFatherImportId(germplasm); + String femaleImportId = GermplasmImportIdUtils.getMotherImportId(germplasm); + String maleImportId = GermplasmImportIdUtils.getFatherImportId(germplasm); - if (created.contains(femaleImportId) || femaleParentUnknown(germplasm)) { - if (!maleParentPresent(germplasm) || created.contains(maleImportId) || maleParentUnknown(germplasm)) { + if (created.contains(femaleImportId) || GermplasmImportIdUtils.femaleParentUnknown(germplasm)) { + if (!GermplasmImportIdUtils.maleParentPresent(germplasm) || created.contains(maleImportId) || GermplasmImportIdUtils.maleParentUnknown(germplasm)) { createList.add(germplasm); } } - /* - // If both parents have been created already, add it - List pedigreeArray = List.of(germplasm.getPedigree().split("/")); - // name + gid or name + entry no if no gid - String femaleParent = pedigreeArray.get(0); - String maleParent = pedigreeArray.size() > 1 ? pedigreeArray.get(1) : null; - if (created.contains(femaleParent) || germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.FEMALE_PARENT_UNKNOWN).getAsBoolean()) { - if (maleParent == null || created.contains(maleParent) || germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.MALE_PARENT_UNKNOWN).getAsBoolean()) { - createList.add(germplasm); - } - } - */ } totalRecorded += createList.size(); if (createList.size() > 0) { - created.addAll(createList.stream().map(this::getImportId).collect(Collectors.toList())); + created.addAll(createList.stream().map(GermplasmImportIdUtils::getImportId).collect(Collectors.toList())); postOrder.add(createList); } else if (totalRecorded < newGermplasmList.size()) { // We ran into circular dependencies, throw an error diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index d248d42ad..a9985273b 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -20,7 +20,7 @@ import org.breedinginsight.brapps.importer.model.imports.germplasm.GermplasmImportService; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; +import org.breedinginsight.brapps.importer.services.processors.germplasm.GermplasmProcessor; import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; import org.breedinginsight.dao.db.tables.pojos.ProgramBreedingMethodEntity; import org.breedinginsight.daos.BreedingMethodDAO; @@ -32,7 +32,6 @@ import org.junit.jupiter.api.*; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import tech.tablesaw.api.Row; import tech.tablesaw.api.Table; import javax.inject.Inject; From 0345ce3f0ba533a845001609185bdc90d59ba8ce Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 8 Apr 2025 11:06:57 -0400 Subject: [PATCH 024/289] Added file header --- .../germplasm/GermplasmImportIdUtils.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmImportIdUtils.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmImportIdUtils.java index 4fe10e0c7..5c695aaee 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmImportIdUtils.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmImportIdUtils.java @@ -1,3 +1,19 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.breedinginsight.brapps.importer.services.processors.germplasm; import com.google.gson.JsonElement; From 927885fab7e18d17aa74b58b8b84b7d77d617f98 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 8 Apr 2025 11:45:37 -0400 Subject: [PATCH 025/289] Added test case --- .../importer/GermplasmFileImportTest.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index a9985273b..13d60a714 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -786,6 +786,32 @@ public void entryNoDescending(boolean commit) { checkEntryNoFields(fileData, previewRows, commit, listName, listDescription, "EntryNoDescGerm 2", "EntryNoDescGerm 1"); } + /** + * Prior to BI-2593 this file would result in a false positive circular dependency error. This was due to germplasm + * references not being unique in the preview and commit phases of the postOrder method. + * + * @param commit controls whether import is preview or commit + */ + @Order(7) // want some existing gids to reference + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @SneakyThrows + public void duplicateNames(boolean commit) { + String pathname = "src/test/resources/files/germplasm_import/duplicate_names_circular_dependency.csv"; + Table fileData = Table.read().file(pathname); + String listName = "DuplicateNames"; + String listDescription = "Duplicate names with pedigree"; + + JsonObject result = importGermplasm(pathname, listName, listDescription, commit); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + + // preview table is sorted by entry number + fileData = fileData.sortAscendingOn("Entry No"); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + checkEntryNoFields(fileData, previewRows, commit, listName, listDescription, "TestDup", "TestDup"); + } + /** * Shared method to perform entry number order tests for preview and commit * From 03783baa5cb0115e5af2dc6c335221c35ad2444b Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 9 Apr 2025 15:35:55 +0000 Subject: [PATCH 026/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index ec1e82717..5b40e8aee 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+920 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/83f392e18d60bba157751d18ae72699b68824f3a +version=v1.1.0+926 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/8610b9a5849e0ce306e07d30abc3fbc1ff0aeaac From f7251201a04272d33835da93958f95fe8e3625c2 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 8 Apr 2025 15:49:31 -0400 Subject: [PATCH 027/289] Switched to using gids to track germplasm --- .../v2/services/BrAPIGermplasmService.java | 11 ++++--- .../breedinginsight/utilities/Utilities.java | 32 +++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index 7272e2b90..e0602e046 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -187,13 +187,13 @@ public List getGermplasmByList(UUID programId, String listDbId) // get list BrAPI germplasm variables List germplasmNames = listResponse.getResult().getData(); List germplasm = germplasmDAO.getGermplasmByRawName(germplasmNames, programId); - Map germplasmByName = new HashMap<>(); + Map germplasmByGid = new HashMap<>(); for (BrAPIGermplasm g : germplasm) { // set the list ID in the germplasm additional info g.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ID, listId); // Add to map. - germplasmByName.put(g.getGermplasmName(), g); + germplasmByGid.put(g.getAccessionNumber(), g); } // Get the program key. @@ -201,12 +201,15 @@ public List getGermplasmByList(UUID programId, String listDbId) .orElseThrow(ApiException::new) .getKey(); + // Extract gids from list names + List gids = germplasmNames.stream().map(Utilities::extractGid).collect(Collectors.toList()); + // Build list from BrAPI list that preserves ordering and duplicates and assigns sequential entry numbers. List germplasmList = new ArrayList<>(); int entryNumber = 0; - for (String germplasmName : germplasmNames) { + for (String gid : gids) { ++entryNumber; - BrAPIGermplasm listEntry = cloneBrAPIGermplasm(germplasmByName.get(Utilities.removeProgramKeyAndUnknownAdditionalData(germplasmName, programKey))); + BrAPIGermplasm listEntry = cloneBrAPIGermplasm(germplasmByGid.get(gid)); // Set entry number. listEntry.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER, entryNumber); germplasmList.add(listEntry); diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 20f3254d6..31990928c 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -29,6 +29,7 @@ import java.sql.Statement; import java.util.*; import java.util.function.Function; +import java.util.regex.Matcher; import java.util.regex.Pattern; public class Utilities { @@ -183,6 +184,37 @@ public static String removeProgramKeyAndUnknownAdditionalData(String original, S return stripped; } + /** + * Extracts the germplasm identifier (GID) from a germplasm name string that contains + * a key in the format "[PROGKEY-NUMBER]". + * + *

This method searches for a pattern matching "[anything-digits]" in the input string + * and returns the numeric portion if found. The prefix before the hyphen can be any sequence + * of characters.

+ * + * @param germplasmNameWithKey The germplasm name string containing the identifier in the format + * "[PROGKEY-NUMBER]", e.g., "TestDup [DEMO-12]" + * @return The numeric portion after the hyphen as a String if the pattern is found, + * or null if the pattern is not found in the input string + * @throws NullPointerException If the input string is null + * + * @example + * 
+     * String gid = extractGid("TestDup [DEMO-12]"); // Returns "12"
+     * String gid = extractGid("Wheat [BRC-789]");   // Returns "789"
+     * String gid = extractGid("NoPattern");         // Returns null
+     *
+ */ + public static String extractGid(String germplasmNameWithKey) { + Pattern pattern = Pattern.compile("\\[(.*?)-(\\d+)\\]"); + Matcher matcher = pattern.matcher(germplasmNameWithKey); + + if (matcher.find()) { + return matcher.group(2); + } + return null; + } + public static String generateApiExceptionLogMessage(ApiException e) { return new StringBuilder("BrAPI Exception: \n\t").append("message: ") .append(e.getMessage()) From 3fb903932d755a71fd240b1ce25ac6970f207bbc Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 9 Apr 2025 15:50:57 -0400 Subject: [PATCH 028/289] Removed unused code --- .../brapi/v2/services/BrAPIGermplasmService.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index e0602e046..7e4a01e67 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -196,11 +196,6 @@ public List getGermplasmByList(UUID programId, String listDbId) germplasmByGid.put(g.getAccessionNumber(), g); } - // Get the program key. - String programKey = programService.getById(programId) - .orElseThrow(ApiException::new) - .getKey(); - // Extract gids from list names List gids = germplasmNames.stream().map(Utilities::extractGid).collect(Collectors.toList()); From 2d680d7df2247a4d97930c2bdd895e725ca6d903 Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Fri, 14 Mar 2025 14:14:34 -0400 Subject: [PATCH 029/289] [BI-2578] Support for unpaginated RQs to BrAPI server for cache loading --- .../brapi/v2/dao/BrAPIGermplasmDAO.java | 2 +- .../utilities/BrAPIDAOUtil.java | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 75226babf..9e83615a2 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -141,7 +141,7 @@ private Map fetchProgramGermplasm(UUID programId) throws BrAPIGermplasmSearchRequest germplasmSearch = new BrAPIGermplasmSearchRequest(); germplasmSearch.externalReferenceIDs(List.of(programId.toString())); germplasmSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); - return processGermplasmForDisplay(brAPIDAOUtil.search( + return processGermplasmForDisplay(brAPIDAOUtil.searchNoPaging( api::searchGermplasmPost, api::searchGermplasmSearchResultsDbIdGet, germplasmSearch diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index b730660e3..7fb115a53 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -79,26 +79,37 @@ public List search(Funct Function3, Optional>>> searchGetMethod, U searchBody ) throws ApiException { - return searchInternal(searchMethod, searchGetMethod, null, searchBody); + return searchInternal(searchMethod, searchGetMethod, null, searchBody, true); } public List search(Function, Optional>>> searchMethod, Function4, Optional>>> searchGetMethod, U searchBody ) throws ApiException { - return searchInternal(searchMethod, null, searchGetMethod, searchBody); + return searchInternal(searchMethod, null, searchGetMethod, searchBody, true); + } + + public List searchNoPaging(Function, Optional>>> searchMethod, + Function3, Optional>>> searchGetMethod, + U searchBody + ) throws ApiException { + return searchInternal(searchMethod, searchGetMethod, null, searchBody, false); } private List searchInternal(Function, Optional>>> searchMethod, Function3, Optional>>> searchGetMethod, Function4, Optional>>> searchGetMethodWithMimeType, - U searchBody) throws ApiException { + U searchBody, boolean sendPaging) throws ApiException { try { List listResult = new ArrayList<>(); //NOTE: Because of the way Breedbase implements BrAPI searches, the page size is initially set to an //arbitrary, large value to ensure that in the event that a 202 response is returned, the searchDbId //stored will refer to all records of the BrAPI variable. - searchBody.pageSize(10000000); + + if (sendPaging) { + searchBody.pageSize(65000); + } + ApiResponse, Optional>> response = searchMethod.apply(searchBody); if (response.getBody().getLeft().isPresent()) { BrAPIResponse listResponse = (BrAPIResponse) response.getBody().getLeft().get(); @@ -108,7 +119,7 @@ private List searchInter pagination params are handled for POST search endpoints or the corresponding endpoints in Breedbase are changed or updated */ - if(hasMorePages(listResponse)) { + if(sendPaging && hasMorePages(listResponse)) { int currentPage = listResponse.getMetadata().getPagination().getCurrentPage() + 1; int totalPages = listResponse.getMetadata().getPagination().getTotalPages(); @@ -137,7 +148,7 @@ private List searchInter BrAPIResponse listResponse = (BrAPIResponse) searchGetResponse.getBody().getLeft().get(); listResult = getListResult(searchGetResponse); - if(hasMorePages(listResponse)) { + if(sendPaging && hasMorePages(listResponse)) { currentPage++; int totalPages = listResponse.getMetadata() .getPagination() From cfcce994cbd10c6062bb78b3826d26852b493784 Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Mon, 17 Mar 2025 16:25:09 -0400 Subject: [PATCH 030/289] Add comment about configurable variable on brapi test server side --- .../java/org/breedinginsight/utilities/BrAPIDAOUtil.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index 7fb115a53..3d4925414 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -102,11 +102,11 @@ private List searchInter U searchBody, boolean sendPaging) throws ApiException { try { List listResult = new ArrayList<>(); - //NOTE: Because of the way Breedbase implements BrAPI searches, the page size is initially set to an - //arbitrary, large value to ensure that in the event that a 202 response is returned, the searchDbId - //stored will refer to all records of the BrAPI variable. if (sendPaging) { + // This should be set to whatever the maximum allowable value is configured in the brapi test server, + // perhaps it should be configurable on bi side as well. + // For reference, that prop name is paging.page-size.max-allowed searchBody.pageSize(65000); } From 4ea3793c976db57d7041b9096ea9cb820118f874 Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Wed, 19 Mar 2025 14:24:09 -0400 Subject: [PATCH 031/289] [BI-2578] Add config props for max rq page size and pagination flipper for germs --- .env.template | 4 ++++ .../brapi/v2/dao/BrAPIGermplasmDAO.java | 24 +++++++++++++++---- .../utilities/BrAPIDAOUtil.java | 5 +++- src/main/resources/application.yml | 13 ++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/.env.template b/.env.template index abe859391..1a481aae6 100644 --- a/.env.template +++ b/.env.template @@ -31,6 +31,10 @@ BRAPI_READ_TIMEOUT=60m # Max number of records to POST to the BrAPI service per request POST_CHUNK_SIZE=1000 +# Request cache records paginating through available records per program by CACHE_BRAPI_FETCH_PAGE_SIZE +CACHE_PAGINATE_GERMPLASM=false +CACHE_BRAPI_FETCH_PAGE_SIZE=65000 + # BrAPI Server Variables BRAPI_SERVER_PORT=8083 diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 9e83615a2..a8feaecf6 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -65,6 +65,9 @@ public class BrAPIGermplasmDAO { @Property(name = "micronaut.bi.api.run-scheduled-tasks") private boolean runScheduledTasks; + @Property(name = "brapi.paginate.germplasm") + private boolean paginateGermplasm; + private final ProgramCache programGermplasmCache; private final BrAPIEndpointProvider brAPIEndpointProvider; @@ -141,11 +144,22 @@ private Map fetchProgramGermplasm(UUID programId) throws BrAPIGermplasmSearchRequest germplasmSearch = new BrAPIGermplasmSearchRequest(); germplasmSearch.externalReferenceIDs(List.of(programId.toString())); germplasmSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); - return processGermplasmForDisplay(brAPIDAOUtil.searchNoPaging( - api::searchGermplasmPost, - api::searchGermplasmSearchResultsDbIdGet, - germplasmSearch - ), program.getKey()); + + if (paginateGermplasm) { + log.debug("Fetching germplasm with pagination to BrAPI"); + return processGermplasmForDisplay(brAPIDAOUtil.search( + api::searchGermplasmPost, + api::searchGermplasmSearchResultsDbIdGet, + germplasmSearch), + program.getKey()); + } else { + log.debug("Fetching germplasm without pagination to BrAPI"); + return processGermplasmForDisplay(brAPIDAOUtil.searchNoPaging( + api::searchGermplasmPost, + api::searchGermplasmSearchResultsDbIdGet, + germplasmSearch), + program.getKey()); + } } /** diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index 3d4925414..15dc57236 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -62,6 +62,9 @@ public class BrAPIDAOUtil { private final int postGroupSize; private final ProgramService programService; + @Property(name = "brapi.cache.fetch-page-size") + private int brapiFetchPageSize; + @Inject public BrAPIDAOUtil(@Property(name = "brapi.search.wait-time") int searchWaitTime, @Property(name = "brapi.read-timeout") Duration searchTimeout, @@ -107,7 +110,7 @@ private List searchInter // This should be set to whatever the maximum allowable value is configured in the brapi test server, // perhaps it should be configurable on bi side as well. // For reference, that prop name is paging.page-size.max-allowed - searchBody.pageSize(65000); + searchBody.pageSize(brapiFetchPageSize); } ApiResponse, Optional>> response = searchMethod.apply(searchBody); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 3a98c3fb6..820d0cbac 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -173,6 +173,19 @@ brapi: search: wait-time: 1000 post-group-size: ${POST_CHUNK_SIZE:1000} + paginate: + # These props are used as flippers for specific cache entities to be fetched all at once or paginated. + # If the props are set to true, there's a possibility the BrAPI server could run out of memory grabbing all the + # data and assigning it to entities in memory given a program that has a large amount of those types of entities. + # At that point these flippers should be set to true. + germplasm: ${CACHE_PAGINATE_GERMPLASM:false} + cache: + # This prop sets how many program cache entity records can be fetched at a time from BrAPI. Today, if over 65000 is used, + # a SQL error can happen for programs that have more than this page size, because to fetch these certain entities + # the same amount of IDs must be passed for any collections that need to be fetched in the entity, and SQL has a max of ~65355 params. + # The ProgramCache will iterate through all pages of an entity to retrieve all the data from BrAPI. + fetch-page-size: ${CACHE_BRAPI_FETCH_PAGE_SIZE:65000} + email: relay-server: From d92659cc18a77e0c2963351e54c2b2ea633565f4 Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Mon, 31 Mar 2025 12:25:35 -0400 Subject: [PATCH 032/289] Add test server template props to bi-api --- .../resources/brapi/properties/application.properties | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/resources/brapi/properties/application.properties b/src/main/resources/brapi/properties/application.properties index 28a097f3a..3733fcd37 100644 --- a/src/main/resources/brapi/properties/application.properties +++ b/src/main/resources/brapi/properties/application.properties @@ -36,3 +36,13 @@ spring.mvc.dispatch-options-request=true security.oidc_discovery_url=https://example.com/auth/.well-known/openid-configuration security.enabled=false +security.issuer_url=http://example.com/issuerurl + +# This should either be set in accordance with a maximum number of SQL parameters (on JOIN FETCHES of collections, +# if there is more than one collection the IDs of each entity need to be passed through as parameters, and there is a SQL +# maximum of 65535. See GermplasmService.findGermplasmEntities()), +# whatever returns in a reasonable amount of time, +# or if you want to limit for the sake of server efficiency. +paging.page-size.max-allowed=65000 + +paging.page-size.default=1000 From 9ac32af1f622889ec9045134dc022f012c4f832c Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Wed, 2 Apr 2025 16:40:43 -0400 Subject: [PATCH 033/289] Fix unit test errors related to new property not loading --- .../java/org/breedinginsight/utilities/BrAPIDAOUtil.java | 6 +++--- .../java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java | 2 +- .../geno/impl/GigwaGenotypeServiceImplIntegrationTest.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index 15dc57236..cab44d951 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -60,21 +60,21 @@ public class BrAPIDAOUtil { private final Duration searchTimeout; private final int pageSize; private final int postGroupSize; + private final int brapiFetchPageSize; private final ProgramService programService; - @Property(name = "brapi.cache.fetch-page-size") - private int brapiFetchPageSize; - @Inject public BrAPIDAOUtil(@Property(name = "brapi.search.wait-time") int searchWaitTime, @Property(name = "brapi.read-timeout") Duration searchTimeout, @Property(name = "brapi.page-size") int pageSize, @Property(name = "brapi.post-group-size") int postGroupSize, + @Property(name = "brapi.cache.fetch-page-size") int brapiFetchPageSize, ProgramService programService) { this.searchWaitTime = searchWaitTime; this.searchTimeout = searchTimeout; this.pageSize = pageSize; this.postGroupSize = postGroupSize; + this.brapiFetchPageSize = brapiFetchPageSize; this.programService = programService; } diff --git a/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java b/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java index 427ed6d88..d77971817 100644 --- a/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java +++ b/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java @@ -70,7 +70,7 @@ public ApiResponse, Optional Date: Wed, 2 Apr 2025 16:59:49 -0400 Subject: [PATCH 034/289] Fix unit test failure related to searchNoPaging function --- .../services/BrAPIGermplasmServiceUnitTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java index 8f906c169..9d5f7b792 100644 --- a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java +++ b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java @@ -141,9 +141,9 @@ public void getGermplasmListExport() { when(programDAO.getAll()).thenReturn(Arrays.asList(Program.builder().id(testProgramId).name("Test Program").active(true).build())); when(programDAO.fetchOneById(any(UUID.class))).thenReturn(testProgram); when(programDAO.get(any(UUID.class))).thenReturn(Arrays.asList(testProgram)); - when(brAPIDAOUtil.search(any(Function.class), - any(Function3.class), - any(BrAPIGermplasmSearchRequest.class))).thenReturn(germplasm); + when(brAPIDAOUtil.searchNoPaging(any(Function.class), + any(Function3.class), + any(BrAPIGermplasmSearchRequest.class))).thenReturn(germplasm); //Create germplasm cache of stub data Method setupMethod = BrAPIGermplasmDAO.class.getDeclaredMethod("setup"); From e1577803fd7860fc56ad31f50080a35e0ead28e9 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 23 Apr 2025 16:27:08 -0400 Subject: [PATCH 035/289] Add germplasm name validation --- .../germplasm/GermplasmProcessor.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index d4d2389a8..bdf1f8d69 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -91,6 +91,7 @@ public class GermplasmProcessor implements Processor { public static String missingParentalGIDsMsg = "The following parental GIDs were not found in the database: %s"; public static String missingParentalEntryNoMsg = "The following parental entry numbers were not found in the database: %s"; public static String badBreedMethodsMsg = "Invalid breeding method"; + public static String badGermplasmNameMsg = "Germplasm name cannot contain /"; public static String missingEntryNumbersMsg = "Either all or none of the germplasm must have entry numbers"; public static String duplicateEntryNoMsg = "Entry numbers must be unique. Duplicated entry numbers found: %s"; public static String circularDependency = "Circular dependency in the pedigree tree"; @@ -357,6 +358,7 @@ private void processNewGermplasm(Germplasm germplasm, ValidationErrors validatio } } + validateGermplasmName(germplasm, i+2, validationErrors); validatePedigree(germplasm, i + 2, validationErrors); BrAPIGermplasm newGermplasm = germplasm.constructBrAPIGermplasm(program, breedingMethod, user, commit, BRAPI_REFERENCE_SOURCE, nextVal, importListId); @@ -543,6 +545,31 @@ private Map getStatisticsMap(List } + /** + * Validates the name of the given Germplasm, ensuring it does not contain any slash ("/") characters. + *

+ * If the germplasm name contains a "/", a new {@link ValidationError} with status + * {@code 422 Unprocessable Entity} is created and added to the provided {@code ValidationErrors} object. + * This method does not throw an exception; instead, it records validation failures by mutating + * the {@code validationErrors} parameter. + *

+ * + * @param germplasm + * the {@link Germplasm} instance whose name is to be validated; must not be {@code null} + * @param rowNumber + * the row index (for example, in a spreadsheet or CSV file) corresponding to this + * germplasm entry; used when reporting errors + * @param validationErrors + * the {@link ValidationErrors} collector into which any detected errors will be added; + * this object is modified by this method to record validation issues; must not be {@code null} + */ + private void validateGermplasmName(Germplasm germplasm, Integer rowNumber, ValidationErrors validationErrors) { + if (germplasm.getGermplasmName().contains("/")) { + ValidationError error = new ValidationError("Germplasm Name", badGermplasmNameMsg, HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowNumber, error); + } + } + private void validatePedigree(Germplasm germplasm, Integer rowNumber, ValidationErrors validationErrors) { String femaleParentEntryNo = germplasm.getFemaleParentEntryNo(); String maleParentEntryNo = germplasm.getMaleParentEntryNo(); From 9195b923d182e8b13c389783292e44965518b81f Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 23 Apr 2025 16:27:47 -0400 Subject: [PATCH 036/289] Add test for germplasm name validation --- .../importer/GermplasmFileImportTest.java | 29 +++++++++++++++++++ .../germplasm_import/bad_germplasm_names.csv | 4 +++ 2 files changed, 33 insertions(+) create mode 100644 src/test/resources/files/germplasm_import/bad_germplasm_names.csv diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index 13d60a714..ac2860441 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -542,6 +542,35 @@ public void badBreedingMethods() { assertEquals("Breeding Method", firstError.get("field").getAsString()); } + /** + * Test for BI-2185 adding validation for germplasm names to not allow / characters + */ + @Test + @SneakyThrows + public void badGermplasmNames() { + File file = new File("src/test/resources/files/germplasm_import/bad_germplasm_names.csv"); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad Germplasm Names"), true, client, validProgram, germplasmMappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + + JsonArray errorList = result + .getAsJsonObject("progress") + .getAsJsonArray("rowErrors"); + + assertEquals(3, errorList.size()); + + JsonObject firstError = errorList + .get(0).getAsJsonObject() + .getAsJsonArray("errors").get(0).getAsJsonObject(); + assertEquals("Germplasm name cannot contain /", firstError.get("errorMessage").getAsString()); + assertEquals("Germplasm Name", firstError.get("field").getAsString()); + } + @Test @SneakyThrows public void someEntryNumbersError() { diff --git a/src/test/resources/files/germplasm_import/bad_germplasm_names.csv b/src/test/resources/files/germplasm_import/bad_germplasm_names.csv new file mode 100644 index 000000000..22f177757 --- /dev/null +++ b/src/test/resources/files/germplasm_import/bad_germplasm_names.csv @@ -0,0 +1,4 @@ +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1/Germplasm 2,BCR,Test,,,1,,,, +,Germplasm3 / Germaplasm4,BCR,Test,,,2,,,, +,Germplasm5/Germplasm6/Germplasm7,BCR,Test,,,3,1,2,, \ No newline at end of file From d1e0c23bc35b22ad0a60255c6ee5bcecd613817d Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 15 Apr 2025 13:11:31 -0400 Subject: [PATCH 037/289] [BI-2595] - added logging --- .../org/breedinginsight/services/UserService.java | 2 ++ .../breedinginsight/utilities/email/EmailUtil.java | 13 +++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/services/UserService.java b/src/main/java/org/breedinginsight/services/UserService.java index 7502ff8b2..b78fdf8a6 100644 --- a/src/main/java/org/breedinginsight/services/UserService.java +++ b/src/main/java/org/breedinginsight/services/UserService.java @@ -390,6 +390,8 @@ private void sendAccountSignUpEmail(BiUserEntity user, SignedJWT jwtToken) { String filledBody = emailTemplate.render(); String subject = "Activate DeltaBreed Account"; + log.debug(filledBody); + // Send email emailUtil.sendEmail(user.getEmail(), subject, filledBody); } diff --git a/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java b/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java index a839715a1..7aef12550 100644 --- a/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java +++ b/src/main/java/org/breedinginsight/utilities/email/EmailUtil.java @@ -19,6 +19,7 @@ import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.HttpServerException; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import javax.inject.Singleton; @@ -29,6 +30,7 @@ import java.util.Date; import java.util.Properties; +@Slf4j @Singleton public class EmailUtil { @@ -66,6 +68,8 @@ protected PasswordAuthentication getPasswordAuthentication() { public void sendEmail(String toEmail, String subject, String body){ try { + log.debug("Sending email to: " + toEmail + " from: " + fromEmail + " with subject: " + subject); + Session session = getSmtpHost(); MimeMessage msg = new MimeMessage(session); //set message headers @@ -83,11 +87,16 @@ public void sendEmail(String toEmail, String subject, String body){ msg.setRecipients(Message.RecipientType.TO, InternetAddress.parse(toEmail, false)); Transport.send(msg); + + log.debug("Email sent to: " + toEmail + " from: " + fromEmail + " with subject: " + subject); } - catch (UnsupportedEncodingException | MessagingException e) { + catch (UnsupportedEncodingException e) { + log.debug("UnsupportedEncodingException " + e.getMessage()); + throw new HttpServerException(e.getMessage()); + } catch (MessagingException e) { + log.debug("MessagingException " + e.getMessage()); throw new HttpServerException(e.getMessage()); } } - } From d9e5aa47283133722713a4f3f3e7ace7d9c5b018 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Fri, 18 Apr 2025 16:23:01 -0400 Subject: [PATCH 038/289] [BI-2595] - reverted dependency changes --- pom.xml | 222 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 122 insertions(+), 100 deletions(-) diff --git a/pom.xml b/pom.xml index 4280fd0b1..fe89f2ca6 100644 --- a/pom.xml +++ b/pom.xml @@ -90,9 +90,11 @@ 4.9.3 4.3.1 2.1-SNAPSHOT - 2.9.2 - - 1.26.1 + 2.11.0 + 2.2.1 + + 4.1.2 + 4.1.2 1.6.2 4.3.1 1.0.0-SNAPSHOT @@ -377,14 +379,34 @@ ${brapi-java-client.version} - org.apache.tika - tika-app - ${tika-app.version} + org.apache.commons + commons-csv + ${apache-commons-csv.version} + + + org.apache.poi + poi + ${apache-poi.version} + + + org.apache.poi + poi-ooxml + ${apache-poi-ooxml.version} org.apache.commons - commons-compress - ${commons-compress.version} + commons-lang3 + ${apache-commons-lang.version} + + + commons-io + commons-io + ${commons-io.version} + + + org.apache.tika + tika-app + ${tika-app.version} com.sun.mail @@ -468,7 +490,7 @@ shade - + *:* @@ -477,9 +499,9 @@ META-INF/*.RSA - - - + + + ${exec.mainClass} @@ -514,98 +536,98 @@ ${maven.compiler.target} - - org.jooq - jooq-codegen-maven - ${jooq.version} - - - - generate - - - + + org.jooq + jooq-codegen-maven + ${jooq.version} + + + + generate + + + + + + + org.postgresql.Driver + jdbc:postgresql://${DB_SERVER}/${DB_NAME} + ${DB_USER} + ${DB_PASSWORD} + + + org.breedinginsight.generation.JooqDaoGenerator + + org.jooq.meta.postgres.PostgresDatabase + .* + public + true + true + false + true + true + true + true + false + + flyway_schema_history|base_entity|base_track_edit_entity|spatial_ref_sys + + - - - org.postgresql.Driver + + + + + + PASCAL + $0_ENTITY + + + PASCAL + $0_TABLE + +
+ + + + + + org.breedinginsight.dao.db + target/generated-sources/jooq + + + true + true + false + true + + + + + + org.breedinginsight + bi-jooq-codegen + ${jooq.version} + + + + + org.flywaydb + flyway-maven-plugin + ${flyway.version} + jdbc:postgresql://${DB_SERVER}/${DB_NAME} ${DB_USER} ${DB_PASSWORD} - - - org.breedinginsight.generation.JooqDaoGenerator - - org.jooq.meta.postgres.PostgresDatabase - .* - public - true - true - false - true - true - true - true - false - - flyway_schema_history|base_entity|base_track_edit_entity|spatial_ref_sys - - - - - - - - - PASCAL - $0_ENTITY - - - PASCAL - $0_TABLE - -
- - - - - - org.breedinginsight.dao.db - target/generated-sources/jooq - - - true - true - false - true - - - - - - org.breedinginsight - bi-jooq-codegen - ${jooq.version} - - - - - org.flywaydb - flyway-maven-plugin - ${flyway.version} - - jdbc:postgresql://${DB_SERVER}/${DB_NAME} - ${DB_USER} - ${DB_PASSWORD} - - - - org.postgresql - postgresql - ${postgres.version} - - - + + + + org.postgresql + postgresql + ${postgres.version} + + + From f261e44f04b0e1a0ed1080f27cf07b70363ed109 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 24 Apr 2025 14:58:40 -0400 Subject: [PATCH 039/289] [REVERT ME] - temporary test configuration change --- src/test/java/org/breedinginsight/BrAPITest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/BrAPITest.java b/src/test/java/org/breedinginsight/BrAPITest.java index 8c5c35bad..e6e6072d6 100644 --- a/src/test/java/org/breedinginsight/BrAPITest.java +++ b/src/test/java/org/breedinginsight/BrAPITest.java @@ -49,7 +49,7 @@ public class BrAPITest extends DatabaseTest { public BrAPITest() { super(); - brapiContainer = new GenericContainer<>("breedinginsight/brapi-java-server:develop") + brapiContainer = new GenericContainer<>("breedinginsight/brapi-java-server:rc") .withNetwork(super.getNetwork()) .withImagePullPolicy(PullPolicy.ageBased(Duration.ofMinutes(60))) .withExposedPorts(8080) From 053e19c442a8591ed11e04573a63e77f321aea08 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 23 Apr 2025 16:00:22 -0400 Subject: [PATCH 040/289] [BI-2608] Changed column-name from 'Long' to 'Elevation' in error message --- .../create/workflow/steps/ValidatePendingImportObjectsStep.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java index b1391c483..ee570c238 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java @@ -363,7 +363,7 @@ private void validateGeoCoordinates(ValidationErrors validationErrors, int rowNu } if (elevationBadValue) { - ExperimentUtilities.addRowError(ExperimentObservation.Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); + ExperimentUtilities.addRowError(ExperimentObservation.Columns.ELEVATION, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); } } From 115efbee849a9cb3eb45170ae885295237b3c489 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 30 Apr 2025 16:07:32 -0400 Subject: [PATCH 041/289] Update src/test/java/org/breedinginsight/BrAPITest.java --- src/test/java/org/breedinginsight/BrAPITest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/BrAPITest.java b/src/test/java/org/breedinginsight/BrAPITest.java index e6e6072d6..8c5c35bad 100644 --- a/src/test/java/org/breedinginsight/BrAPITest.java +++ b/src/test/java/org/breedinginsight/BrAPITest.java @@ -49,7 +49,7 @@ public class BrAPITest extends DatabaseTest { public BrAPITest() { super(); - brapiContainer = new GenericContainer<>("breedinginsight/brapi-java-server:rc") + brapiContainer = new GenericContainer<>("breedinginsight/brapi-java-server:develop") .withNetwork(super.getNetwork()) .withImagePullPolicy(PullPolicy.ageBased(Duration.ofMinutes(60))) .withExposedPorts(8080) From 5e1d14cd17a70fcfe6dc3e7b4fa43c6869022855 Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Fri, 14 Mar 2025 14:21:45 -0400 Subject: [PATCH 042/289] [BI-2579] Forgo cache refresh on response from server during germ import chunks --- .../org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index a8feaecf6..2f756ba26 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -310,11 +310,9 @@ public List createBrAPIGermplasm(List postBrAPIG var program = programDAO.fetchOneById(programId); try { if (!postBrAPIGermplasmList.isEmpty()) { - Callable> postFunction = () -> { List postResponse = brAPIDAOUtil.post(postBrAPIGermplasmList, upload, api::germplasmPost, importDAO::update); - return processGermplasmForDisplay(postResponse, program.getKey()); - }; - return programGermplasmCache.post(programId, postFunction); + processGermplasmForDisplay(postResponse, program.getKey()); + return postResponse; } return new ArrayList<>(); } catch (Exception e) { From b4875b49a5700fe34d057f29f5fc7127fdbdf869 Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Thu, 20 Mar 2025 14:20:57 -0400 Subject: [PATCH 043/289] Fully populate program germplasm cache when upload completes --- .../org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java | 4 ++++ .../services/processors/germplasm/GermplasmProcessor.java | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 2f756ba26..c4090ef8e 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -162,6 +162,10 @@ private Map fetchProgramGermplasm(UUID programId) throws } } + public void repopulateGermplasmCacheForProgram(UUID programId) { + programGermplasmCache.populate(programId); + } + /** * Process germplasm into a format for display * @param programGermplasm diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index bdf1f8d69..e0adba213 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -662,6 +662,8 @@ public void postBrapiData(Map mappedBrAPIImport, Program try { // Create germplasm list brAPIListDAO.createBrAPILists(List.of(importList), program.getId(), upload); + // Now that we have finished uploading, fetch all the data posted to BrAPI to the cache so it is up-to-date. + brAPIGermplasmDAO.repopulateGermplasmCacheForProgram(program.getId()); } catch (ApiException e) { throw new InternalServerException(e.toString(), e); } From 1ae7ed451025193a858fd73d23bfede424af9f28 Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Mon, 31 Mar 2025 12:30:36 -0400 Subject: [PATCH 044/289] Add properties added to test server properties file --- .../brapi/properties/application.properties | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/resources/brapi/properties/application.properties b/src/main/resources/brapi/properties/application.properties index 3733fcd37..5ee3656c5 100644 --- a/src/main/resources/brapi/properties/application.properties +++ b/src/main/resources/brapi/properties/application.properties @@ -25,13 +25,25 @@ spring.datasource.password=${BRAPI_DB_PASSWORD} spring.datasource.driver-class-name=org.postgresql.Driver +# This property when set to true makes it so that a DB transaction is open through the body of a request, nullifying the use of @Transactional. +# It is generally recommended that this be set to false, and methods are properly annotated and release the transactions when complete. +# However, many of the endpoints already rely on this kind of connection infrastructure and changing is more trouble than it's worth. +spring.jpa.open-in-view=true + +spring.jpa.properties.hibernate.jdbc.batch_size=50 +spring.jpa.properties.hibernate.order_inserts=true +spring.jpa.properties.hibernate.order_updates=true +spring.jpa.hibernate.ddl-auto=validate + +# Use these to help debug queries. +# The stats will tell you how long hibernate transactions are taking, how many queries occur, how many entities are being flushed/accessed, etc. +spring.jpa.show-sql=false +spring.jpa.properties.hibernate.generate_statistics=false + spring.flyway.locations=classpath:db/migration,classpath:db/sql,classpath:org/brapi/test/BrAPITestServer/db/migration spring.flyway.schemas=public spring.flyway.baselineOnMigrate=true -spring.jpa.hibernate.ddl-auto=validate -spring.jpa.show-sql=false - spring.mvc.dispatch-options-request=true security.oidc_discovery_url=https://example.com/auth/.well-known/openid-configuration From 5f04fb6192f6d80e55ded9aeca284bf24be91f8f Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Mon, 31 Mar 2025 16:24:03 -0400 Subject: [PATCH 045/289] Restore functionality of using post return values --- .../org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index c4090ef8e..478c99bcf 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -315,8 +315,7 @@ public List createBrAPIGermplasm(List postBrAPIG try { if (!postBrAPIGermplasmList.isEmpty()) { List postResponse = brAPIDAOUtil.post(postBrAPIGermplasmList, upload, api::germplasmPost, importDAO::update); - processGermplasmForDisplay(postResponse, program.getKey()); - return postResponse; + return new ArrayList<>(processGermplasmForDisplay(postResponse, program.getKey()).values()); } return new ArrayList<>(); } catch (Exception e) { From 1f453d58287d0e0bc9eace790be279ed6920c0a5 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 7 May 2025 18:22:35 +0000 Subject: [PATCH 046/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 7d37e46f7..ed12a56b1 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+961 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/06df6ae9acb10d77620dc978d8f1c290dab21324 +version=v1.2.0+963 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/faa5e691f8b526a520e15c3f374520cc9d29b2d2 From 1d5fc1a95fd334c8af1f927fc126cd1aca9ffd54 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 15 May 2025 10:35:55 -0400 Subject: [PATCH 047/289] create observation unit dynamic column validator --- .../experiment/ExperimentUtilities.java | 42 ++++++++-- .../AppendOverwriteIDValidation.java | 21 +++-- .../model/AppendOverwriteWorkflowContext.java | 3 + .../DynamicColumnValidator.java | 27 +++++++ .../ObservationUnitDuplicateIDValidator.java | 65 +++++++++++++++ .../ObservationUnitIDBlankValidator.java | 59 ++++++++++++++ .../ObservationUnitIDColumnNameValidator.java | 80 +++++++++++++++++++ .../ObservationUnitIDFormatValidator.java | 66 +++++++++++++++ .../ObservationUnitIDValidator.java | 44 ++++++++++ .../model/ExpImportProcessConstants.java | 7 +- 10 files changed, 399 insertions(+), 15 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/DynamicColumnValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDValidator.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 4c79478e8..c3dd7b8ce 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -50,6 +50,7 @@ import org.breedinginsight.model.Scale; import org.breedinginsight.model.Trait; import tech.tablesaw.columns.Column; +import org.breedinginsight.services.exceptions.BadRequestException; import javax.inject.Singleton; import java.math.BigDecimal; @@ -59,6 +60,8 @@ import java.util.*; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.OZEX; + @Slf4j @Singleton public class ExperimentUtilities { @@ -299,7 +302,7 @@ public static void addYearToStudyAdditionalInfo(Program program, BrAPIStudy stud * This method iterates through all import rows in the given context and * extracts unique Observation Unit IDs (ObsUnit IDs) that are not null or blank. * - * @param context The AppendOverwriteMiddlewareContext containing the import data. + * @param ctx The AppendOverwriteMiddlewareContext containing the import data. * @return A Set of String containing all unique, non-null, non-blank Observation Unit IDs. * * @implNote The method performs the following steps: @@ -309,15 +312,19 @@ public static void addYearToStudyAdditionalInfo(Program program, BrAPIStudy stud * 4. If valid, adds the ObsUnit ID to the set. * 5. Returns the set of unique ObsUnit IDs. */ - public static Set collateUniqueOUIds(AppendOverwriteMiddlewareContext context) { - // Initialize variables to track the presence of ObsUnit IDs + public static Set collateUniqueOUIds(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() == null) { + throw new BadRequestException(OZEX.getValue()); + } + Set referenceOUIds = new HashSet<>(); + String idColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column idCol = ctx.getImportContext().getData().columns(idColName).get(0); - // Iterate through the import rows to process ObsUnit IDs - for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); - if (importRow.getObsUnitID() != null && !importRow.getObsUnitID().isBlank()) { - referenceOUIds.add(importRow.getObsUnitID()); + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + String id = idCol.getString(rowNum); + if (id != null && !id.isBlank()) { + referenceOUIds.add(id); } } return referenceOUIds; @@ -366,6 +373,25 @@ public static void validateReferenceOUIdValues(AppendOverwriteMiddlewareContext } } + public static boolean hasUniqueIds(AppendOverwriteMiddlewareContext ctx, String colName) throws IllegalStateException { + Set referenceOUIds = new HashSet<>(); + List> columns = ctx.getImportContext().getData().columns(colName); + if (columns.isEmpty()) { + throw new IllegalStateException("No columns found for: " + colName); + } + + Column col = columns.get(0); + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + if (referenceOUIds.contains(col.getString(rowNum))) { + return false; + } else { + referenceOUIds.add(col.getString(rowNum)); + } + } + + return true; + } + /** * Adds validation errors for observation units that were not found in the database. * diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java index a2edbdf65..053ee905a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java @@ -23,12 +23,15 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID.ObservationUnitIDValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIReadFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowReadInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.BadRequestException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; import javax.inject.Inject; @@ -40,26 +43,32 @@ public class AppendOverwriteIDValidation extends AppendOverwriteMiddleware { WorkflowReadInitialization brAPIObservationUnitReadWorkflowInitialization; BrAPIReadFactory brAPIReadFactory; + ObservationUnitIDValidator ouIdValidator; @Inject - public AppendOverwriteIDValidation(BrAPIReadFactory brAPIReadFactory) { + public AppendOverwriteIDValidation(BrAPIReadFactory brAPIReadFactory, ObservationUnitIDValidator ouIdValidator) { this.brAPIReadFactory = brAPIReadFactory; + this.ouIdValidator = ouIdValidator; } + @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { brAPIObservationUnitReadWorkflowInitialization = brAPIReadFactory.observationUnitWorkflowReadInitializationBean(context); - // Initialize the validation error collection + // Initialize the tabular error collection Optional.ofNullable(context.getAppendOverwriteWorkflowContext().getValidationErrors()).orElseGet(() -> { context.getAppendOverwriteWorkflowContext().setValidationErrors(new ValidationErrors()); return new ValidationErrors(); }); ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); - ExperimentUtilities.validateReferenceOUIdValues(context); // Check for missing or duplicate OU ids - Set uniqueOUIds = ExperimentUtilities.collateUniqueOUIds(context); - context.getAppendOverwriteWorkflowContext().setReferenceOUIds(uniqueOUIds); + try { + ouIdValidator.validateDynamicColumns(context); + Set uniqueOUIds = ExperimentUtilities.collateUniqueOUIds(context); + context.getAppendOverwriteWorkflowContext().setReferenceOUIds(uniqueOUIds); brAPIObservationUnitReadWorkflowInitialization.execute(); // Fetch the obs units from the BrAPi service + + // Check for tabular errors collected during validation if (validationErrors.hasErrors()) { throw new ValidatorException(validationErrors); } @@ -72,7 +81,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext ExperimentUtilities.addValidationErrorsForObsUnitsNotFound(e, context); context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(new ValidatorException(validationErrors))); return this.compensate(context); - } catch (ApiException | ValidatorException e) { + } catch (BadRequestException | ApiException | ValidatorException e) { /** * If OUs were fetched for all unique reference ids but some of the reference ids failed validation, * return an error response and a list of rows with duplicate or missing ids diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java index 13d892722..b1c7dfbc3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java @@ -39,6 +39,9 @@ @Getter @Setter public class AppendOverwriteWorkflowContext { + // Dynamic Columns + private String obsUnitColName; + // Cache maps keyed by existing observation unit ids private Set referenceOUIds = new HashSet<>(); private Map> pendingTrialByOUId = new HashMap<>(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/DynamicColumnValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/DynamicColumnValidator.java new file mode 100644 index 000000000..909f48b21 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/DynamicColumnValidator.java @@ -0,0 +1,27 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns; + +import io.micronaut.core.order.Ordered; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.services.exceptions.BadRequestException; + +@FunctionalInterface +public interface DynamicColumnValidator extends Ordered { + void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java new file mode 100644 index 000000000..3331503ba --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java @@ -0,0 +1,65 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; +import org.breedinginsight.services.exceptions.BadRequestException; +import tech.tablesaw.columns.Column; + +import javax.inject.Singleton; +import java.util.HashSet; +import java.util.Set; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.OZEX; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.VVCN; + +@Slf4j +@Singleton +public class ObservationUnitDuplicateIDValidator implements DynamicColumnValidator { + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() == null) { + throw new BadRequestException(OZEX.getValue()); + } + + ValidationErrors rowErrors = ctx.getAppendOverwriteWorkflowContext().getValidationErrors(); + Set referenceOUIds = new HashSet<>(); + String idColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column idCol = ctx.getImportContext().getData().columns(idColName).get(0); + + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + String id = idCol.get(rowNum).toString(); + if (referenceOUIds.contains(idCol.get(rowNum).toString())) { + // Check if ObsUnitID is duplicated + ExperimentUtilities.addRowError(idColName, VVCN.getValue(), rowErrors, rowNum); + } else { + // Add ObsUnitID to referenceOUIds + referenceOUIds.add(idCol.get(rowNum).toString()); + } + } + } + + @Override + public int getOrder() { + return 4; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java new file mode 100644 index 000000000..caf505fac --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java @@ -0,0 +1,59 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; +import org.breedinginsight.services.exceptions.BadRequestException; +import tech.tablesaw.columns.Column; + +import javax.inject.Singleton; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.BITB; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.OZEX; + +@Slf4j +@Singleton +public class ObservationUnitIDBlankValidator implements DynamicColumnValidator { + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() == null) { + throw new BadRequestException(OZEX.getValue()); + } + + ValidationErrors rowErrors = ctx.getAppendOverwriteWorkflowContext().getValidationErrors(); + String idColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column idCol = ctx.getImportContext().getData().columns(idColName).get(0); + + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + String id = idCol.get(rowNum).toString(); + if ( id == null || id.isBlank()) { + // Check if ObsUnitID is blank + ExperimentUtilities.addRowError(idColName, BITB.getValue(), rowErrors, rowNum); + } + } + } + + @Override + public int getOrder() { + return 2; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java new file mode 100644 index 000000000..32b2ccbc1 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java @@ -0,0 +1,80 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; +import org.breedinginsight.services.exceptions.BadRequestException; + +import javax.inject.Singleton; +import java.util.Arrays; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.OZEX; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.SUB_UNIT_NUMBER; + +@Slf4j +@Singleton +public class ObservationUnitIDColumnNameValidator implements DynamicColumnValidator { + + public ObservationUnitIDColumnNameValidator() {} + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + // Skip this validation if it has already been successfully completed + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() != null) return; + + // Get the names of all the dynamic columns with observation unit ids + String[] idColNames = Arrays.stream(ctx.getImportContext().getUpload().getDynamicColumnNames()) + .filter(name->name.endsWith(OBSERVATION_UNIT_ID_SUFFIX)).toArray(String[]::new); + + // throw an error if there are no obs unit id columns in the import + int idColCount = idColNames.length; + if (idColCount == 0) throw new BadRequestException(OZEX.getValue()); + + // count the number of columns and throw an error if the count is neither 1 and unique nor 2 with 1 unique and 1 non-unique + if (!(idColCount == 1 || idColCount == 2)) throw new BadRequestException(OZEX.getValue()); + + if (idColCount == 2) { + // if sub-entity ids in import then check for presence of sub-unit # column + Arrays.stream(ctx.getImportContext().getUpload().getDynamicColumnNames()) + .filter(name-> name.equals(SUB_UNIT_NUMBER)) + .findAny() + .orElseThrow(()->new BadRequestException(OZEX.getValue())); + + // check that if there is a column with unique ids, it is the right-most column + boolean leftIsUnique = ExperimentUtilities.hasUniqueIds(ctx, idColNames[0]); + boolean rightIsUnique = ExperimentUtilities.hasUniqueIds(ctx, idColNames[1]); + if (leftIsUnique && !rightIsUnique) throw new BadRequestException(OZEX.getValue()); + + // the right column should be the most nested level and this column is to be used for processing the import + ctx.getAppendOverwriteWorkflowContext().setObsUnitColName(idColNames[1]); + } else if (idColCount == 1) { + + // there is only one top level whose column is used for processing the import + ctx.getAppendOverwriteWorkflowContext().setObsUnitColName(idColNames[0]); + } + } + + @Override + public int getOrder() { + return 1; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java new file mode 100644 index 000000000..5b5585d0e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java @@ -0,0 +1,66 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; +import org.breedinginsight.services.exceptions.BadRequestException; +import tech.tablesaw.columns.Column; +import java.util.regex.Pattern; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.BITB; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.OZEX; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; + +import javax.inject.Singleton; + +@Slf4j +@Singleton +public class ObservationUnitIDFormatValidator implements DynamicColumnValidator { + private static final Pattern UUID_PATTERN = Pattern.compile( + "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" + ); + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() == null) { + throw new BadRequestException(OZEX.getValue()); + } + + ValidationErrors rowErrors = ctx.getAppendOverwriteWorkflowContext().getValidationErrors(); + String idColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column idCol = ctx.getImportContext().getData().columns(idColName).get(0); + + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + Object value = idCol.get(rowNum); + String id = value != null ? value.toString() : null; + + // Validate UUID format + if (id == null || !UUID_PATTERN.matcher(id).matches()) { + ExperimentUtilities.addRowError(idColName, BITB.getValue(), rowErrors, rowNum); + } + } + } + + @Override + public int getOrder() { + return 3; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDValidator.java new file mode 100644 index 000000000..46e76384a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDValidator.java @@ -0,0 +1,44 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; + +import io.micronaut.context.annotation.Primary; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; +import org.breedinginsight.services.exceptions.BadRequestException; + +import javax.inject.Singleton; +import java.util.List; + +@Primary +@Singleton +public class ObservationUnitIDValidator implements DynamicColumnValidator { + private final List validators; + + public ObservationUnitIDValidator(List validators) { + this.validators = validators; + } + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) + throws BadRequestException { + for (DynamicColumnValidator validator : validators) { + validator.validateDynamicColumns(ctx); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index 2c0ae2e53..32b880388 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -25,10 +25,12 @@ public class ExpImportProcessConstants { public static final CharSequence COMMA_DELIMITER = ","; + public static final String OBSERVATION_UNIT_ID_SUFFIX = "ObsUnitID"; public static final String TIMESTAMP_PREFIX = "TS:"; public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; public static String BRAPI_REFERENCE_SOURCE; public static final String MIDNIGHT = "T00:00:00-00:00"; + public static final String SUB_UNIT_NUMBER = "Sub-Unit #"; public enum ErrMessage { MULTIPLE_EXP_TITLES("File contains more than one Experiment Title"), @@ -36,7 +38,10 @@ public enum ErrMessage { PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"), UNMATCHED_COLUMN("Ontology term(s) not found: "), OBS_UNIT_NOT_FOUND("Invalid ObsUnitID"), - DUPLICATE_OBS_UNIT_ID("ObsUnitId is repeated"); + DUPLICATE_OBS_UNIT_ID("ObsUnitId is repeated"), + OZEX("Missing ObsUnitID column. Import cannot proceed"), + VVCN("ObsUnitID is duplicated"), + BITB("Invalid or missing ObsUnitID"); private String value; From 18fd6ca2782bdda2c5307aa3cbee8e97cfa7fc6e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 15 May 2025 11:53:23 -0400 Subject: [PATCH 048/289] remove static ou id references ValidatePendingInmportObjectsStep --- .../ValidatePendingImportObjectsStep.java | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java index ee570c238..b865d8437 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java @@ -228,18 +228,20 @@ private void validateConditionallyRequired(PendingData pendingData, ValidationEr validateRequiredCell(importRow.getExpReplicateNo(), ExperimentObservation.Columns.REP_NUM, errorMessage, validationErrors, rowNum); validateRequiredCell(importRow.getExpBlockNo(), ExperimentObservation.Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); - if(StringUtils.isNotBlank(importRow.getObsUnitID())) { - ExperimentUtilities.addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); - } + // TODO: replace validating each row for ObsUnitID with a single validation for the absence of the entire column " ObsUnitID" +// if(StringUtils.isNotBlank(importRow.getObsUnitID())) { +// ExperimentUtilities.addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); +// } } else { + // TODO: is this validation still needed? //Check if existing environment. If so, ObsUnitId must be assigned - validateRequiredCell( - importRow.getObsUnitID(), - ExperimentObservation.Columns.OBS_UNIT_ID, - ExperimentUtilities.MISSING_OBS_UNIT_ID_ERROR, - validationErrors, - rowNum - ); +// validateRequiredCell( +// importRow.getObsUnitID(), +// ExperimentObservation.Columns.OBS_UNIT_ID, +// ExperimentUtilities.MISSING_OBS_UNIT_ID_ERROR, +// validationErrors, +// rowNum +// ); } } @@ -263,9 +265,10 @@ private void validateObservationUnits( String key = ExperimentUtilities.createObservationUnitKey(importRow); PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); - if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { - ExperimentUtilities.addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); - } + // TODO: Is this check still needed for new observation units? +// if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { +// ExperimentUtilities.addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); +// } validateGeoCoordinates(validationErrors, rowNum, importRow); } @@ -397,7 +400,7 @@ private void validateObservations(PendingData pendingData, !existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { ExperimentUtilities.addRowError( phenoCol.name(), - String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), + String.format("Value already exists for Phenotype: %s", phenoCol.name()), validationErrors, rowNum ); From 011be9a99b4d86811dcbcc4bd27ff1ed1959f4cb Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 15 May 2025 12:19:56 -0400 Subject: [PATCH 049/289] update CreateNewExperimentWorkflow to check for dynamic ObsUnitID column --- .../workflow/CreateNewExperimentWorkflow.java | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index d0a4ca975..0a48da65b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -50,10 +50,7 @@ import javax.inject.Inject; import javax.inject.Named; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; +import java.util.*; import lombok.Getter; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; @@ -63,7 +60,8 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Singleton; -import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; @Slf4j @Getter @@ -216,12 +214,8 @@ private long getNewObjectCount(ImportPreviewResponse response) { } private boolean containsObsUnitIDs(ImportContext importContext) { - List importRows = importContext.getImportRows(); - return importRows.stream() - .anyMatch(row -> { - ExperimentObservation expRow = (ExperimentObservation) row; - return StringUtils.isNotBlank(expRow.getObsUnitID()); - }); + return Arrays.stream(importContext.getUpload().getDynamicColumnNames()) + .anyMatch(name->name.endsWith(OBSERVATION_UNIT_ID_SUFFIX)); } // TODO: move to shared area: experiment import service From 1009a5755e58e629c9c0c4d89c73b7179cf975c7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 15 May 2025 12:35:19 -0400 Subject: [PATCH 050/289] update PopulateNewPendingImportObjectsStep to remove refs to static ObsUnitID col --- .../workflow/steps/PopulateNewPendingImportObjectsStep.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index 0e2e465cf..ab4f41505 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -310,7 +310,7 @@ public PendingImportObject populateTrial(ImportContext importContext // creating new units for existing experiments and environments is not possible if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && - (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); } } else if (!trialByNameNoScope.isEmpty()) { From 5cb65f64f7958f82ed0aace6ebdd8ee4abd599d8 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 15 May 2025 13:37:11 -0400 Subject: [PATCH 051/289] short-circuit existing OU hashmap in populateExistingPIO step of create workflow --- .../workflow/CreateNewExperimentWorkflow.java | 1 + ...ulateExistingPendingImportObjectsStep.java | 91 ++++++++++--------- 2 files changed, 47 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 0a48da65b..010a14095 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -106,6 +106,7 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); + // TODO: eliminate or modify unnecessary populateExistingPIO step as it relies on the user supplying existing observation unit ids ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); populateNewPendingImportObjectsStep.process(processContext, phenotypeData); ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context, processContext.getPendingData(), phenotypeData, processedData); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index b6bca19d0..30f8fdaed 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -143,51 +143,52 @@ public ProcessContext process(ImportContext input, ProcessedPhenotypeData phenot */ private Map> initializeObservationUnits(Program program, List experimentImportRows) { Map> observationUnitByName = new HashMap<>(); - - Map rowByObsUnitId = new HashMap<>(); - experimentImportRows.forEach(row -> { - if (StringUtils.isNotBlank(row.getObsUnitID())) { - if(rowByObsUnitId.containsKey(row.getObsUnitID())) { - throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); - } - rowByObsUnitId.put(row.getObsUnitID(), row); - } - }); - - try { - List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); - - // TODO: grab from externalReferences - /* - observationUnitByObsUnitId = existingObsUnits.stream() - .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, - (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false))); - */ - - String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - if (existingObsUnits.size() == rowByObsUnitId.size()) { - existingObsUnits.forEach(brAPIObservationUnit -> { - processAndCacheObservationUnit(brAPIObservationUnit, refSource, program, observationUnitByName, rowByObsUnitId); - - BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) - .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); - row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); - row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); - row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); - }); - } else { - List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); - missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); - } - - return observationUnitByName; - } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } + return observationUnitByName; + // TODO: change how ProcessContext is generated so it does not rely on this unused method +// Map rowByObsUnitId = new HashMap<>(); +// experimentImportRows.forEach(row -> { +// if (StringUtils.isNotBlank(row.getObsUnitID())) { +// if(rowByObsUnitId.containsKey(row.getObsUnitID())) { +// throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); +// } +// rowByObsUnitId.put(row.getObsUnitID(), row); +// } +// }); +// +// try { +// List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); +// +// // TODO: grab from externalReferences +// /* +// observationUnitByObsUnitId = existingObsUnits.stream() +// .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, +// (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false))); +// */ +// +// String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); +// if (existingObsUnits.size() == rowByObsUnitId.size()) { +// existingObsUnits.forEach(brAPIObservationUnit -> { +// processAndCacheObservationUnit(brAPIObservationUnit, refSource, program, observationUnitByName, rowByObsUnitId); +// +// BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) +// .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); +// +// ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); +// row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); +// row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); +// row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); +// }); +// } else { +// List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); +// missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); +// throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); +// } +// +// return observationUnitByName; +// } catch (ApiException e) { +// log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); +// throw new InternalServerException(e.toString(), e); +// } } /** From 3bedcd9410a044626073ab4ef3b008475dc0d464 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 16 May 2025 10:13:57 -0400 Subject: [PATCH 052/289] delete unused TrialService#initializeTrialNoScope() --- .../process/ImportTableProcess.java | 8 +++-- ...ulateExistingPendingImportObjectsStep.java | 2 +- .../experiment/service/TrialService.java | 33 ------------------- 3 files changed, 6 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 770576b67..e06d773ab 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -233,7 +233,9 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext return new HashMap<>(); }); PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); - String unitId = row.getObsUnitID(); + String obsUnitIDColName = context.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column obsUnitIDCol = context.getImportContext().getData().column(obsUnitIDColName); + String unitId = obsUnitIDCol.getString(rowNum); String studyName = context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); mappedImportRow.setTrial(context.getAppendOverwriteWorkflowContext().getPendingTrialByOUId().get(unitId)); mappedImportRow.setLocation(context.getAppendOverwriteWorkflowContext().getPendingLocationByOUId().get(unitId)); @@ -319,7 +321,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext } else if (!cellData.isBlank()) { // Clone the observation unit and trait - BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(row.getObsUnitID()).getBrAPIObject()), BrAPIObservationUnit.class); + BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(obsUnitIDCol.getString(rowNum)).getBrAPIObject()), BrAPIObservationUnit.class); Trait initialTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); // create new instance of InitialData @@ -342,7 +344,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context.getImportContext().getProgram()); } else { // Clone the observation unit - BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(row.getObsUnitID()).getBrAPIObject()), BrAPIObservationUnit.class); + BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(obsUnitIDCol.getString(rowNum)).getBrAPIObject()), BrAPIObservationUnit.class); processedData = processedDataFactory.emptyDataBean(brapiReferenceSource, context.getImportContext().isCommit(), diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index 30f8fdaed..14a891482 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -280,7 +280,7 @@ private Map> initializeStudyByNameNoScop private Optional> getTrialPIO(List experimentImportRows, Map> trialByNameNoScope) { Optional expTitle = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID()) && StringUtils.isNotBlank(row.getExpTitle())) + .filter(row -> StringUtils.isNotBlank(row.getExpTitle())) .map(ExperimentObservation::getExpTitle) .findFirst(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index dcbc79159..bae599bfc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -306,39 +306,6 @@ public PendingImportObject constructPIOFromBrapiTrial(BrAPITrial tri return pio; } - /** - * Initializes trials by name without scope for the given program. - * - * @param program the program to initialize trials for - * @param observationUnitByNameNoScope a map of observation units by name without scope - * @param experimentImportRows a list of experiment observation rows - * @return a map of trials by name with pending import objects - * - * @throws InternalServerException - */ - private Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, - List experimentImportRows) { - Map> trialByName = new HashMap<>(); - - initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); - - List uniqueTrialNames = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID())) - .map(ExperimentObservation::getExpTitle) - .distinct() - .collect(Collectors.toList()); - try { - brAPITrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> - processAndCacheTrial(existingTrial, program, trialByName) - ); - } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - return trialByName; - } - private void initializeTrialsForExistingObservationUnits(Program program, Map> observationUnitByNameNoScope, Map> trialByName) { } From 0732da4484b09447d137518820fbf1c2560585bf Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 16 May 2025 10:15:49 -0400 Subject: [PATCH 053/289] delete empty and unused TrialService method --- .../services/processors/experiment/service/TrialService.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index bae599bfc..1a9a7f02f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -306,9 +306,6 @@ public PendingImportObject constructPIOFromBrapiTrial(BrAPITrial tri return pio; } - private void initializeTrialsForExistingObservationUnits(Program program, Map> observationUnitByNameNoScope, Map> trialByName) { - } - // TODO: used by expunit workflow public Map> mapPendingTrialByOUId( String unitId, From 021523aca3dc97d23afaf68bba523d57b89cc511 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 16 May 2025 10:19:29 -0400 Subject: [PATCH 054/289] delete unused method in ExperimentUtilities --- .../experiment/ExperimentUtilities.java | 43 ------------------- .../services/ExperimentTrialService.java | 1 - 2 files changed, 44 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index c3dd7b8ce..e5685ac9a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -330,49 +330,6 @@ public static Set collateUniqueOUIds(AppendOverwriteMiddlewareContext ct return referenceOUIds; } - /** - * Validates Observation Unit ID values in the import context. - * - * This method checks each import row for the validity of its Observation Unit ID (ObsUnitID). - * It performs the following validations: - * 1. Checks if the ObsUnitID is null or blank. - * 2. Checks if the ObsUnitID is a duplicate within the import data. - * - * @param context The AppendOverwriteMiddlewareContext containing import data and validation error storage. - * @throws HttpStatusException If there's an HTTP-related error during the validation process. - * @throws IllegalStateException If the system encounters an unexpected state during validation. - * - * @implNote The method performs the following steps: - * 1. Retrieves the ValidationErrors object from the context. - * 2. Initializes a HashSet to track unique ObsUnitIDs. - * 3. Iterates through each import row in the context. - * 4. For each row: - * - If ObsUnitID is null or blank, adds a "missing ObsUnitID" error. - * - If ObsUnitID is already in the set (duplicate), adds a "duplicate ObsUnitID" error. - * - Otherwise, adds the ObsUnitID to the set of unique IDs. - * 5. Errors are added using the addRowError method, specifying the OBS_UNIT_ID column and appropriate error messages. - */ - public static void validateReferenceOUIdValues(AppendOverwriteMiddlewareContext context) throws HttpStatusException, IllegalStateException { - ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); - Set referenceOUIds = new HashSet<>(); - - // Iterate through the import rows to process ObsUnit IDs - for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); - - if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { - // Check if ObsUnitID is blank - addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExpImportProcessConstants.ErrMessage.MISSING_OBS_UNIT_ID.getValue(), validationErrors, rowNum); - } else if (referenceOUIds.contains(importRow.getObsUnitID())) { - // Check if ObsUnitID is repeated - addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExpImportProcessConstants.ErrMessage.DUPLICATE_OBS_UNIT_ID.getValue(), validationErrors, rowNum); - } else { - // Add ObsUnitID to referenceOUIds - referenceOUIds.add(importRow.getObsUnitID()); - } - } - } - public static boolean hasUniqueIds(AppendOverwriteMiddlewareContext ctx, String colName) throws IllegalStateException { Set referenceOUIds = new HashSet<>(); List> columns = ctx.getImportContext().getData().columns(colName); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java index 9db927345..b94fb6789 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java @@ -177,7 +177,6 @@ public Map> initializeTrialByNameNoScope initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); List uniqueTrialNames = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID())) .map(ExperimentObservation::getExpTitle) .distinct() .collect(Collectors.toList()); From 43e1252c0b6085bdb8348e28da44ba4f7cce5fcd Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 16 May 2025 10:27:44 -0400 Subject: [PATCH 055/289] delete unused ExperimentProcessor --- .../processors/ExperimentProcessor.java | 2567 ----------------- 1 file changed, 2567 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java deleted file mode 100644 index 0666870c6..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ /dev/null @@ -1,2567 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.breedinginsight.brapps.importer.services.processors; - - -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import io.micronaut.context.annotation.Property; -import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.HttpStatus; -import io.micronaut.http.exceptions.HttpStatusException; -import io.micronaut.http.server.exceptions.InternalServerException; -import io.reactivex.functions.Function; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.collections4.map.CaseInsensitiveMap; -import org.apache.commons.lang3.StringUtils; -import org.brapi.client.v2.JSON; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.*; -import org.brapi.v2.model.core.request.BrAPIListNewRequest; -import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; -import org.breedinginsight.api.auth.AuthenticatedUser; -import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; -import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapi.v2.dao.*; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; -import org.breedinginsight.brapps.importer.model.imports.PendingImport; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.FileMappingUtil; -import org.breedinginsight.dao.db.tables.pojos.TraitEntity; -import org.breedinginsight.model.*; -import org.breedinginsight.services.OntologyService; -import org.breedinginsight.services.ProgramLocationService; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; -import org.breedinginsight.utilities.DatasetUtil; -import org.breedinginsight.utilities.Utilities; -import org.jooq.DSLContext; -import tech.tablesaw.api.Table; -import tech.tablesaw.columns.Column; - -import javax.inject.Inject; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.*; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -@Slf4j -@Prototype -public class ExperimentProcessor implements Processor { - - private static final String NAME = "Experiment"; - private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs"; - private static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists"; - private static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; - private static final String MIDNIGHT = "T00:00:00-00:00"; - private static final String TIMESTAMP_PREFIX = "TS:"; - private static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; - private static final String COMMA_DELIMITER = ","; - private static final String BLANK_FIELD_EXPERIMENT = "Field is blank when creating a new experiment"; - private static final String BLANK_FIELD_ENV = "Field is blank when creating a new environment"; - private static final String BLANK_FIELD_OBS = "Field is blank when creating new observations"; - private static final String ENV_LOCATION_MISMATCH = "All locations must be the same for a given environment"; - private static final String ENV_YEAR_MISMATCH = "All years must be the same for a given environment"; - - @Property(name = "brapi.server.reference-source") - private String BRAPI_REFERENCE_SOURCE; - - private final DSLContext dsl; - private final BrAPITrialDAO brapiTrialDAO; - private final ProgramLocationService locationService; - private final BrAPIStudyDAO brAPIStudyDAO; - private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; - private final BrAPIObservationDAO brAPIObservationDAO; - private final BrAPISeasonDAO brAPISeasonDAO; - private final BrAPIGermplasmDAO brAPIGermplasmDAO; - private final BrAPIListDAO brAPIListDAO; - private final OntologyService ontologyService; - private final FileMappingUtil fileMappingUtil; - - - // used to make the yearsToSeasonDbId() function more efficient - private final Map yearToSeasonDbIdCache = new HashMap<>(); - // used to make the seasonDbIdtoYear() function more efficient - private final Map seasonDbIdToYearCache = new HashMap<>(); - - //These BrapiData-objects are initially populated by the getExistingBrapiData() method, - // then updated by the initNewBrapiData() method. - private Map> trialByNameNoScope = new HashMap<>(); - private Map> pendingTrialByOUId = new HashMap<>(); - private Map> locationByName = null; - private Map> pendingLocationByOUId = new HashMap<>(); - private Map> studyByNameNoScope = new HashMap<>(); - private Map> pendingStudyByOUId = new HashMap<>(); - private Map> obsVarDatasetByName = null; - private Map> pendingObsDatasetByOUId = new HashMap<>(); - private Map> observationUnitByNameNoScope = null; - private Map> pendingObsUnitByOUId = new HashMap<>(); - - private final Map> observationByHash = new HashMap<>(); - private Map existingObsByObsHash = new HashMap<>(); - // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the initNewBrapiData() method - private Map> existingGermplasmByGID = new HashMap<>(); - private Map> pendingGermplasmByOUId = new HashMap<>(); - - // Associates timestamp columns to associated phenotype column name for ease of storage - private final Map> timeStampColByPheno = new HashMap<>(); - private final Gson gson; - private boolean hasAllReferenceUnitIds = true; - private boolean hasNoReferenceUnitIds = true; - private Set referenceOUIds = new HashSet<>(); - - @Inject - public ExperimentProcessor(DSLContext dsl, - BrAPITrialDAO brapiTrialDAO, - ProgramLocationService locationService, - BrAPIStudyDAO brAPIStudyDAO, - BrAPIObservationUnitDAO brAPIObservationUnitDAO, - BrAPIObservationDAO brAPIObservationDAO, - BrAPISeasonDAO brAPISeasonDAO, - BrAPIGermplasmDAO brAPIGermplasmDAO, - BrAPIListDAO brAPIListDAO, OntologyService ontologyService, - FileMappingUtil fileMappingUtil) { - this.dsl = dsl; - this.brapiTrialDAO = brapiTrialDAO; - this.locationService = locationService; - this.brAPIStudyDAO = brAPIStudyDAO; - this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; - this.brAPIObservationDAO = brAPIObservationDAO; - this.brAPISeasonDAO = brAPISeasonDAO; - this.brAPIGermplasmDAO = brAPIGermplasmDAO; - this.brAPIListDAO = brAPIListDAO; - this.ontologyService = ontologyService; - this.fileMappingUtil = fileMappingUtil; - this.gson = new JSON().getGson(); - } - - @Override - public String getName() { - return NAME; - } - - /** - * Initialize the Map objects with existing BrAPI Data. - * - * @param importRows - * @param program - */ - @Override - public void getExistingBrapiData(List importRows, Program program) { - - List experimentImportRows = importRows.stream() - .map(trialImport -> (ExperimentObservation) trialImport) - .collect(Collectors.toList()); - - // check for references to Deltabreed-generated observation units - referenceOUIds = collateReferenceOUIds(importRows); - - if (hasAllReferenceUnitIds) { - try { - - // get all prior units referenced in import - pendingObsUnitByOUId = fetchReferenceObservationUnits(referenceOUIds, program); - observationUnitByNameNoScope = mapPendingObservationUnitByName(pendingObsUnitByOUId, program); - initializeTrialsForExistingObservationUnits(program, trialByNameNoScope); - initializeStudiesForExistingObservationUnits(program, studyByNameNoScope); - locationByName = initializeLocationByName(program, studyByNameNoScope); - obsVarDatasetByName = initializeObsVarDatasetForExistingObservationUnits(trialByNameNoScope, program); - existingGermplasmByGID = initializeGermplasmByGIDForExistingObservationUnits(observationUnitByNameNoScope, program); - for (Map.Entry> unitEntry : pendingObsUnitByOUId.entrySet()) { - String unitId = unitEntry.getKey(); - BrAPIObservationUnit unit = unitEntry.getValue().getBrAPIObject(); - mapPendingTrialByOUId(unitId, unit, trialByNameNoScope, studyByNameNoScope, pendingTrialByOUId, program); - mapPendingStudyByOUId(unitId, unit, studyByNameNoScope, pendingStudyByOUId, program); - mapPendingLocationByOUId(unitId, unit, pendingStudyByOUId, locationByName, pendingLocationByOUId); - mapPendingObsDatasetByOUId(unitId, pendingTrialByOUId, obsVarDatasetByName, pendingObsDatasetByOUId); - mapGermplasmByOUId(unitId, unit, existingGermplasmByGID, pendingGermplasmByOUId); - } - - } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (Exception e) { - log.error("Error processing experiment with ", e); - throw new InternalServerException(e.toString(), e); - } - } else if (hasNoReferenceUnitIds) { - observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); - trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); - studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); - locationByName = initializeUniqueLocationNames(program, experimentImportRows); - obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); - existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); - - } else { - - // can't proceed if the import has a mix of ObsUnitId for some but not all rows - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, MISSING_OBS_UNIT_ID_ERROR); - } - } - - /** - * @param importRows - one element of the list for every row of the import file. - * @param mappedBrAPIImport - passed in by reference and modified within this program (this will later be passed to the front end for the preview) - * @param program - * @param user - * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) - * false when used for preview only - * @return Map - used to display the summary statistics. - * @throws ValidatorException - */ - @Override - public Map process( - ImportUpload upload, - List importRows, - Map mappedBrAPIImport, - Table data, - Program program, - User user, - boolean commit) throws ApiException, ValidatorException, MissingRequiredInfoException, UnprocessableEntityException { - log.debug("processing experiment import"); - - ValidationErrors validationErrors = new ValidationErrors(); - - // Get dynamic phenotype columns for processing - List> dynamicCols = data.columns(upload.getDynamicColumnNames()); - List> phenotypeCols = new ArrayList<>(); - List> timestampCols = new ArrayList<>(); - for (Column dynamicCol : dynamicCols) { - //Distinguish between phenotype and timestamp columns - if (dynamicCol.name().startsWith(TIMESTAMP_PREFIX)) { - timestampCols.add(dynamicCol); - } else { - phenotypeCols.add(dynamicCol); - } - } - - List referencedTraits = verifyTraits(program.getId(), phenotypeCols, timestampCols); - - //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval - for (Column tsColumn : timestampCols) { - timeStampColByPheno.put(tsColumn.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), tsColumn); - } - - // add "New" pending data to the BrapiData objects - initNewBrapiData(importRows, phenotypeCols, program, user, referencedTraits, commit); - - prepareDataForValidation(importRows, phenotypeCols, mappedBrAPIImport); - - validateFields(importRows, validationErrors, mappedBrAPIImport, referencedTraits, program, phenotypeCols, commit, user); - - if (validationErrors.hasErrors()) { - throw new ValidatorException(validationErrors); - } - - log.debug("done processing experiment import"); - // Construct our response object - return generateStatisticsMap(importRows); - } - - @Override - public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { - // TODO - } - - @Override - public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { - log.debug("starting post of experiment data to BrAPI server"); - - List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); - Map mutatedTrialsById = ProcessorData - .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); - - Map mutatedObservationByDbId = ProcessorData - .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); - - List newLocations = ProcessorData.getNewObjects(this.locationByName) - .stream() - .map(location -> ProgramLocationRequest.builder() - .name(location.getName()) - .build()) - .collect(Collectors.toList()); - List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); - - List newDatasetRequests = ProcessorData.getNewObjects(obsVarDatasetByName).stream().map(details -> { - BrAPIListNewRequest request = new BrAPIListNewRequest(); - request.setListName(details.getListName()); - request.setListType(details.getListType()); - request.setExternalReferences(details.getExternalReferences()); - request.setAdditionalInfo(details.getAdditionalInfo()); - request.data(details.getData()); - return request; - }).collect(Collectors.toList()); - Map datasetNewDataById = ProcessorData - .getMutationsByObjectId(obsVarDatasetByName, BrAPIListSummary::getListDbId); - - List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); - - // filter out observations with no 'value' so they will not be saved - List newObservations = ProcessorData.getNewObjects(this.observationByHash) - .stream() - .filter(obs -> !obs.getValue().isBlank()) - .collect(Collectors.toList()); - - AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); - - try { - List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); - createdDatasets.forEach(summary -> obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId())); - - List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); - // set the DbId to the for each newly created trial - for (BrAPITrial createdTrial : createdTrials) { - String createdTrialName = Utilities.removeProgramKey(createdTrial.getTrialName(), program.getKey()); - this.trialByNameNoScope.get(createdTrialName) - .getBrAPIObject() - .setTrialDbId(createdTrial.getTrialDbId()); - } - - List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); - // set the DbId to the for each newly created location - for (ProgramLocation createdLocation : createdLocations) { - String createdLocationName = createdLocation.getName(); - this.locationByName.get(createdLocationName) - .getBrAPIObject() - .setLocationDbId(createdLocation.getLocationDbId()); - } - - updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); - List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); - - // set the DbId to the for each newly created study - for (BrAPIStudy createdStudy : createdStudies) { - String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); - this.studyByNameNoScope.get(createdStudy_name_no_key) - .getBrAPIObject() - .setStudyDbId(createdStudy.getStudyDbId()); - } - - updateObsUnitDependencyValues(program.getKey()); - List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); - - // set the DbId to the for each newly created Observation Unit - for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { - // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope - String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); - String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); - String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); - this.observationUnitByNameNoScope.get(createdObsUnit_key) - .getBrAPIObject() - .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); - } - - updateObservationDependencyValues(program); - brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); - } catch (ApiException e) { - log.error("Error saving experiment import: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error saving experiment import", e); - throw new InternalServerException(e.getMessage(), e); - } - - mutatedTrialsById.forEach((id, trial) -> { - try { - brapiTrialDAO.updateBrAPITrial(id, trial, program.getId()); - } catch (ApiException e) { - log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error updating dataset observation variables: ", e); - throw new InternalServerException(e.getMessage(), e); - } - }); - - datasetNewDataById.forEach((id, dataset) -> { - try { - List existingObsVarIds = brAPIListDAO.getListById(id, program.getId()).getResult().getData(); - List newObsVarIds = dataset - .getData() - .stream() - .filter(obsVarId -> !existingObsVarIds.contains(obsVarId)).collect(Collectors.toList()); - List obsVarIds = new ArrayList<>(existingObsVarIds); - obsVarIds.addAll(newObsVarIds); - dataset.setData(obsVarIds); - brAPIListDAO.updateBrAPIList(id, dataset, program.getId()); - } catch (ApiException e) { - log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error updating dataset observation variables: ", e); - throw new InternalServerException(e.getMessage(), e); - } - }); - - mutatedObservationByDbId.forEach((id, observation) -> { - try { - if (observation == null) { - throw new Exception("Null observation"); - } - - BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); - - if (updatedObs == null) { - throw new Exception("Null updated observation"); - } - - if (!Objects.equals(observation.getValue(), updatedObs.getValue()) - || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { - String message; - if(!Objects.equals(observation.getValue(), updatedObs.getValue())) { - message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); - } else { - message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); - } - throw new Exception(message); - } - } catch (ApiException e) { - log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error updating observation: ", e); - throw new InternalServerException(e.getMessage(), e); - } - }); - log.debug("experiment import complete"); - } - - private void prepareDataForValidation(List importRows, List> phenotypeCols, Map mappedBrAPIImport) { - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); - List> observations = mappedImportRow.getObservations(); - String observationHash; - if (hasAllReferenceUnitIds) { - String refOUId = importRow.getObsUnitID(); - mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); - mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); - mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); - mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); - mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); - - // loop over phenotype column observation data for current row - for (Column column : phenotypeCols) { - observationHash = getObservationHash( - pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + - pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), - getVariableNameFromColumn(column), - pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() - ); - - // if value was blank won't be entry in map for this observation - observations.add(observationByHash.get(observationHash)); - } - - } else { - mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); - mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); - mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); - mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); - mappedImportRow.setGermplasm(getGidPIO(importRow)); - - // loop over phenotype column observation data for current row - for (Column column : phenotypeCols) { - - // if value was blank won't be entry in map for this observation - observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); - } - } - - mappedBrAPIImport.put(rowNum, mappedImportRow); - } - } - - private List verifyTraits(UUID programId, List> phenotypeCols, List> timestampCols) { - Set varNames = phenotypeCols.stream() - .map(Column::name) - .collect(Collectors.toSet()); - Set tsNames = timestampCols.stream() - .map(Column::name) - .collect(Collectors.toSet()); - - // filter out just traits specified in file - List filteredTraits = fetchFileTraits(programId, varNames); - - // check that all specified ontology terms were found - if (filteredTraits.size() != varNames.size()) { - Set returnedVarNames = filteredTraits.stream() - .map(TraitEntity::getObservationVariableName) - .collect(Collectors.toSet()); - List differences = varNames.stream() - .filter(var -> !Utilities.containsCaseInsensitive(var, returnedVarNames)) - .collect(Collectors.toList()); - //TODO convert this to a ValidationError - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, - "Ontology term(s) not found: " + String.join(COMMA_DELIMITER, differences)); - } - - // Check that each ts column corresponds to a phenotype column - List unmatchedTimestamps = tsNames.stream() - .filter(e -> !(varNames.contains(e.replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY)))) - .collect(Collectors.toList()); - if (unmatchedTimestamps.size() > 0) { - //TODO convert this to a ValidationError - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, - "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(COMMA_DELIMITER, unmatchedTimestamps)); - } - - // sort the verified traits to match the order of the trait columns - List phenotypeColNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList()); - return fileMappingUtil.sortByField(phenotypeColNames, filteredTraits, TraitEntity::getObservationVariableName); - } - - private List fetchFileTraits(UUID programId, Collection varNames) { - try { - Collection upperCaseVarNames = varNames.stream().map(String::toUpperCase).collect(Collectors.toList()); - List traits = ontologyService.getTraitsByProgramId(programId, true); - // filter out just traits specified in file - return traits.stream() - .filter(e -> upperCaseVarNames.contains(e.getObservationVariableName().toUpperCase())) - .collect(Collectors.toList()); - } catch (DoesNotExistException e) { - log.error(e.getMessage(), e); - throw new InternalServerException(e.toString(), e); - } - } - - private String getVariableNameFromColumn(Column column) { - // TODO: timestamp stripping? - return column.name(); - } - - private void initNewBrapiData( - List importRows, - List> phenotypeCols, - Program program, - User user, - List referencedTraits, - boolean commit - ) throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { - - String expSequenceName = program.getExpSequence(); - if (expSequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); - } - Supplier expNextVal = () -> dsl.nextval(expSequenceName.toLowerCase()); - - String envSequenceName = program.getEnvSequence(); - if (envSequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the env sequence column.", program.getName())); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for environment import"); - } - Supplier envNextVal = () -> dsl.nextval(envSequenceName.toLowerCase()); - existingObsByObsHash = fetchExistingObservations(referencedTraits, program); - - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - - PendingImportObject trialPIO = null; - try { - trialPIO = fetchOrCreateTrialPIO(program, user, commit, importRow, expNextVal); - } catch (UnprocessableEntityException e) { - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); - } - - String expSeqValue = null; - if (commit) { - expSeqValue = trialPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) - .getAsString(); - } - - if (commit) { - fetchOrCreateDatasetPIO(importRow, program, referencedTraits); - } - - fetchOrCreateLocationPIO(importRow); - - PendingImportObject studyPIO = fetchOrCreateStudyPIO(program, commit, expSeqValue, importRow, envNextVal); - - String envSeqValue = null; - if (commit) { - envSeqValue = studyPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER) - .getAsString(); - } - - PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); - - for (Column column : phenotypeCols) { - //If associated timestamp column, add - String dateTimeValue = null; - if (timeStampColByPheno.containsKey(column.name())) { - dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); - //If no timestamp, set to midnight - if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { - dateTimeValue += MIDNIGHT; - } - } - - // get the study year either referenced from the observation unit or listed explicitly on the import row - String studyYear = hasAllReferenceUnitIds ? studyPIO.getBrAPIObject().getSeasons().get(0) : importRow.getEnvYear(); - String seasonDbId = yearToSeasonDbId(studyYear, program.getId()); - fetchOrCreateObservationPIO( - program, - user, - importRow, - column, //column.name() gets phenotype name - rowNum, - dateTimeValue, - commit, - seasonDbId, - obsUnitPIO, - studyPIO, - referencedTraits - ); - } - } - } - - private String createObservationUnitKey(ExperimentObservation importRow) { - return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); - } - - private String createObservationUnitKey(String studyName, String obsUnitName) { - return studyName + obsUnitName; - } - - /** - * This method is responsible for generating a hash based on the import observation unit information. - * It takes the observation unit name, variable name, and study name as input parameters. - * The observation unit key is created using the study name and observation unit name. - * The hash is generated based on the observation unit key, variable name, and study name. - * - * @param obsUnitName The name of the observation unit being imported. - * @param variableName The name of the variable associated with the observation unit. - * @param studyName The name of the study associated with the observation unit. - * @return A string representing the hash of the import observation unit information. - */ - private String getImportObservationHash(String obsUnitName, String variableName, String studyName) { - return getObservationHash(createObservationUnitKey(studyName, obsUnitName), variableName, studyName); - } - - private String getImportObservationHash(ExperimentObservation importRow, String variableName) { - return getObservationHash(createObservationUnitKey(importRow), variableName, importRow.getEnv()); - } - - private String getObservationHash(String observationUnitName, String variableName, String studyName) { - String concat = DigestUtils.sha256Hex(observationUnitName) + - DigestUtils.sha256Hex(variableName) + - DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); - return DigestUtils.sha256Hex(concat); - } - - private void validateFields(List importRows, ValidationErrors validationErrors, Map mappedBrAPIImport, List referencedTraits, Program program, - List> phenotypeCols, boolean commit, User user) { - //fetching any existing observations for any OUs in the import - CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); - for ( Trait trait: referencedTraits) { - colVarMap.put(trait.getObservationVariableName(),trait); - } - Set uniqueStudyAndObsUnit = new HashSet<>(); - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); - if (hasAllReferenceUnitIds) { - validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); - } else { - if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. - validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); - } - validateTestOrCheck(importRow, validationErrors, rowNum); - validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); - validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); - } - } - } - - private void validateObservationUnits( - ValidationErrors validationErrors, - Set uniqueStudyAndObsUnit, - int rowNum, - ExperimentObservation importRow - ) { - validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - - String key = createObservationUnitKey(importRow); - PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); - if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { - addRowError(Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); - } - - validateGeoCoordinates(validationErrors, rowNum, importRow); - } - - private void validateGeoCoordinates(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) { - - String lat = importRow.getLatitude(); - String lon = importRow.getLongitude(); - String elevation = importRow.getElevation(); - - // If any of Lat, Long, or Elevation are provided, Lat and Long must both be provided. - if (StringUtils.isNotBlank(lat) || StringUtils.isNotBlank(lon) || StringUtils.isNotBlank(elevation)) { - if (StringUtils.isBlank(lat)) { - addRowError(Columns.LAT, "Latitude must be provided for complete coordinate specification", validationErrors, rowNum); - } - if (StringUtils.isBlank(lon)) { - addRowError(Columns.LONG, "Longitude must be provided for complete coordinate specification", validationErrors, rowNum); - } - } - - // Validate coordinate values - boolean latBadValue = false; - boolean lonBadValue = false; - boolean elevationBadValue = false; - double latDouble; - double lonDouble; - double elevationDouble; - - // Only check latitude format if not blank since already had previous error - if (StringUtils.isNotBlank(lat)) { - try { - latDouble = Double.parseDouble(lat); - if (latDouble < -90 || latDouble > 90) { - latBadValue = true; - } - } catch (NumberFormatException e) { - latBadValue = true; - } - } - - // Only check longitude format if not blank since already had previous error - if (StringUtils.isNotBlank(lon)) { - try { - lonDouble = Double.parseDouble(lon); - if (lonDouble < -180 || lonDouble > 180) { - lonBadValue = true; - } - } catch (NumberFormatException e) { - lonBadValue = true; - } - } - - if (StringUtils.isNotBlank(elevation)) { - try { - elevationDouble = Double.parseDouble(elevation); - } catch (NumberFormatException e) { - elevationBadValue = true; - } - } - - if (latBadValue) { - addRowError(Columns.LAT, "Invalid Lat value (expected range -90 to 90)", validationErrors, rowNum); - } - - if (lonBadValue) { - addRowError(Columns.LONG, "Invalid Long value (expected range -180 to 180)", validationErrors, rowNum); - } - - if (elevationBadValue) { - addRowError(Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); - } - - } - - private Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { - Set ouDbIds = new HashSet<>(); - Set variableDbIds = new HashSet<>(); - Map variableNameByDbId = new HashMap<>(); - Map ouNameByDbId = new HashMap<>(); - Map studyNameByDbId = studyByNameNoScope.values() - .stream() - .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) - .map(PendingImportObject::getBrAPIObject) - .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - - studyNameByDbId.keySet().forEach(studyDbId -> { - try { - brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { - if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { - ouDbIds.add(ou.getObservationUnitDbId()); - } - ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); - }); - } catch (ApiException e) { - throw new RuntimeException(e); - } - }); - - for (Trait referencedTrait : referencedTraits) { - variableDbIds.add(referencedTrait.getObservationVariableDbId()); - variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); - } - - List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); - - return existingObservations.stream() - .map(obs -> { - String studyName = studyNameByDbId.get(obs.getStudyDbId()); - String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); - String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); - - String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); - - return Map.entry(key, obs); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - private void validateObservations(ValidationErrors validationErrors, - int rowNum, - ExperimentObservation importRow, - List> phenotypeCols, - CaseInsensitiveMap colVarMap, - boolean commit, - User user) { - phenotypeCols.forEach(phenoCol -> { - String importHash; - String importObsValue = phenoCol.getString(rowNum); - - if (hasAllReferenceUnitIds) { - importHash = getImportObservationHash( - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), - getVariableNameFromColumn(phenoCol), - pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() - ); - - } else { - importHash = getImportObservationHash(importRow, phenoCol.name()); - } - - // error if import observation data already exists and user has not selected to overwrite - if(commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && - this.existingObsByObsHash.containsKey(importHash) && - StringUtils.isNotBlank(phenoCol.getString(rowNum)) && - !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { - addRowError( - phenoCol.name(), - String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), - validationErrors, rowNum - ); - - // preview case where observation has already been committed and the import row ObsVar data differs from what - // had been saved prior to import - } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { - - // different data means validations still need to happen - // TODO consider moving these two calls into a separate method since called twice together - validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); - - //Timestamp validation - if(timeStampColByPheno.containsKey(phenoCol.name())) { - Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); - validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); - } - - // add a change log entry when updating the value of an observation - // only will update and thereby need change log entry if no error - if (commit && (!validationErrors.hasErrors())) { - BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); - String timestamp = formatter.format(OffsetDateTime.now()); - String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; - String prior = ""; - if (isValueMatched(importHash, importObsValue)) { - prior.concat(existingObsByObsHash.get(importHash).getValue()); - } - if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { - prior = prior.isEmpty() ? prior : prior.concat(" "); - prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); - } - ChangeLogEntry change = new ChangeLogEntry(prior, - reason, - user.getId(), - timestamp - ); - - // create the changelog field in additional info if it does not already exist - if (pendingObservation.getAdditionalInfo().isJsonNull()) { - pendingObservation.setAdditionalInfo(new JsonObject()); - pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } - - if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { - pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } - - // add a new entry to the changelog - pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); - } - - // preview case where observation has already been committed and import ObsVar data is the - // same as has been committed prior to import - } else if(isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { - BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); - existingObs.setObservationVariableName(phenoCol.name()); - observationByHash.get(importHash).setState(ImportObjectState.EXISTING); - observationByHash.get(importHash).setBrAPIObject(existingObs); - - // preview case where observation has already been committed and import ObsVar data is empty prior to import - } else if(!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { - observationByHash.get(importHash).setState(ImportObjectState.EXISTING); - } else { - validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); - - //Timestamp validation - if(timeStampColByPheno.containsKey(phenoCol.name())) { - Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); - validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); - } - } - }); - } - - /** - * Validate that the observation unit is unique within a study. - *
- * SIDE EFFECTS: validationErrors and uniqueStudyAndObsUnit can be modified. - * - * @param validationErrors can be modified as a side effect. - * @param uniqueStudyAndObsUnit can be modified as a side effect. - * @param rowNum counter that is always two less the file row being validated - * @param importRow the data row being validated - */ - private void validateUniqueObsUnits( - ValidationErrors validationErrors, - Set uniqueStudyAndObsUnit, - int rowNum, - ExperimentObservation importRow) { - String envIdPlusStudyId = createObservationUnitKey(importRow); - if (uniqueStudyAndObsUnit.contains(envIdPlusStudyId)) { - String errorMessage = String.format("The ID (%s) is not unique within the environment(%s)", importRow.getExpUnitId(), importRow.getEnv()); - this.addRowError(Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); - } else { - //Only want to add valid unique study-obs unit combos - //To avoid situations like system counting a null value as a unique combo - if (!envIdPlusStudyId.isBlank()) { - uniqueStudyAndObsUnit.add(envIdPlusStudyId); - } - } - } - - private void validateConditionallyRequired(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, Program program, boolean commit) { - ImportObjectState expState = this.trialByNameNoScope.get(importRow.getExpTitle()) - .getState(); - ImportObjectState envState = this.studyByNameNoScope.get(importRow.getEnv()).getState(); - - String errorMessage = BLANK_FIELD_EXPERIMENT; - if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { - errorMessage = BLANK_FIELD_ENV; - } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { - errorMessage = BLANK_FIELD_OBS; - } - - if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { - validateRequiredCell(importRow.getGid(), Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpTitle(),Columns.EXP_TITLE,errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpUnit(), Columns.EXP_UNIT, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpType(), Columns.EXP_TYPE, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getEnv(), Columns.ENV, errorMessage, validationErrors, rowNum); - if(validateRequiredCell(importRow.getEnvLocation(), Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum)) { - if(!Utilities.removeProgramKeyAndUnknownAdditionalData(this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getLocationName(), program.getKey()).equals(importRow.getEnvLocation())) { - addRowError(Columns.ENV_LOCATION, ENV_LOCATION_MISMATCH, validationErrors, rowNum); - } - } - if(validateRequiredCell(importRow.getEnvYear(), Columns.ENV_YEAR, errorMessage, validationErrors, rowNum)) { - String studyYear = StringUtils.defaultString( this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getSeasons().get(0) ); - String rowYear = importRow.getEnvYear(); - if(commit) { - rowYear = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); - } - if(StringUtils.isNotBlank(studyYear) && !studyYear.equals(rowYear)) { - addRowError(Columns.ENV_YEAR, ENV_YEAR_MISMATCH, validationErrors, rowNum); - } - } - validateRequiredCell(importRow.getExpUnitId(), Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpReplicateNo(), Columns.REP_NUM, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpBlockNo(), Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); - - if(StringUtils.isNotBlank(importRow.getObsUnitID())) { - addRowError(Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); - } - } else { - //Check if existing environment. If so, ObsUnitId must be assigned - validateRequiredCell( - importRow.getObsUnitID(), - Columns.OBS_UNIT_ID, - MISSING_OBS_UNIT_ID_ERROR, - validationErrors, - rowNum - ); - } - } - - private boolean validateRequiredCell(String value, String columnHeader, String errorMessage, ValidationErrors validationErrors, int rowNum) { - if (StringUtils.isBlank(value)) { - addRowError(columnHeader, errorMessage, validationErrors, rowNum); - return false; - } - return true; - } - - private void addRowError(String field, String errorMessage, ValidationErrors validationErrors, int rowNum) { - ValidationError ve = new ValidationError(field, errorMessage, HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(rowNum + 2, ve); // +2 instead of +1 to account for the column header row. - } - - private void addIfNotNull(HashSet set, String setValue) { - if (setValue != null) { - set.add(setValue); - } - } - - private Map generateStatisticsMap(List importRows) { - // Data for stats. - HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names - HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's - HashSet gidCounter = new HashSet<>(); // set of unique GID's - - for (BrAPIImport row : importRows) { - ExperimentObservation importRow = (ExperimentObservation) row; - // Collect date for stats. - addIfNotNull(environmentNameCounter, importRow.getEnv()); - addIfNotNull(obsUnitsIDCounter, createObservationUnitKey(importRow)); - addIfNotNull(gidCounter, importRow.getGid()); - } - - int numNewObservations = Math.toIntExact( - observationByHash.values() - .stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && - !StringUtils.isBlank(preview.getBrAPIObject() - .getValue())) - .count() - ); - - int numExistingObservations = Math.toIntExact( - this.observationByHash.values() - .stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING && - !StringUtils.isBlank(preview.getBrAPIObject() - .getValue())) - .count() - ); - - int numMutatedObservations = Math.toIntExact( - this.observationByHash.values() - .stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && - !StringUtils.isBlank(preview.getBrAPIObject() - .getValue())) - .count() - ); - - - ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() - .newObjectCount(environmentNameCounter.size()) - .build(); - ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() - .newObjectCount(obsUnitsIDCounter.size()) - .build(); - ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() - .newObjectCount(gidCounter.size()) - .build(); - ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() - .newObjectCount(numNewObservations) - .build(); - ImportPreviewStatistics existingObservationStats = ImportPreviewStatistics.builder() - .newObjectCount(numExistingObservations) - .build(); - ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() - .newObjectCount(numMutatedObservations) - .build(); - - return Map.of( - "Environments", environmentStats, - "Observation_Units", obdUnitStats, - "GIDs", gidStats, - "Observations", observationStats, - "Existing_Observations", existingObservationStats, - "Mutated_Observations", mutatedObservationStats - ); - } - - private void validateGermplasm(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum, PendingImportObject germplasmPIO) { - // error if GID is not blank but GID does not already exist - if (StringUtils.isNotBlank(importRow.getGid()) && germplasmPIO == null) { - addRowError(Columns.GERMPLASM_GID, "A non-existing GID", validationErrors, rowNum); - } - } - - private void validateTestOrCheck(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum) { - String testOrCheck = importRow.getTestOrCheck(); - if ( ! ( testOrCheck==null || testOrCheck.isBlank() - || "C".equalsIgnoreCase(testOrCheck) || "CHECK".equalsIgnoreCase(testOrCheck) - || "T".equalsIgnoreCase(testOrCheck) || "TEST".equalsIgnoreCase(testOrCheck) ) - ){ - addRowError(Columns.TEST_CHECK, String.format("Invalid value (%s)", testOrCheck), validationErrors, rowNum) ; - } - } - - private PendingImportObject getGidPIO(ExperimentObservation importRow) { - if (this.existingGermplasmByGID.containsKey(importRow.getGid())) { - return existingGermplasmByGID.get(importRow.getGid()); - } - - return null; - } - - private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { - PendingImportObject pio; - String key = createObservationUnitKey(importRow); - if (hasAllReferenceUnitIds) { - pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); - } else if (observationUnitByNameNoScope.containsKey(key)) { - pio = observationUnitByNameNoScope.get(key); - } else { - String germplasmName = ""; - if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { - germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) - .getBrAPIObject() - .getGermplasmName(); - } - PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; - UUID trialID = trialPIO.getId(); - UUID datasetId = null; - if (commit) { - JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); - DatasetMetadata dataset = DatasetUtil.getDatasetByNameFromJson(datasetsJson, importRow.getExpUnit()); - if (dataset != null) { - datasetId = dataset.getId(); - } - } - PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); - UUID studyID = studyPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); - - // check for existing units if this is an existing study - if (studyPIO.getBrAPIObject().getStudyDbId() != null) { - List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); - List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); - if (matchingOU.isEmpty()) { - throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); - } else { - pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); - } - } else { - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); - } - this.observationUnitByNameNoScope.put(key, pio); - } - return pio; - } - - boolean isTimestampMatched(String observationHash, String timeStamp) { - OffsetDateTime priorStamp = null; - if(existingObsByObsHash.get(observationHash)!=null){ - priorStamp = existingObsByObsHash.get(observationHash).getObservationTimeStamp(); - } - if (priorStamp == null) { - return timeStamp == null; - } - boolean isMatched = false; - try { - isMatched = priorStamp.isEqual(OffsetDateTime.parse(timeStamp)); - } catch(DateTimeParseException e){ - //if timestamp is invalid DateTime not equal to validated priorStamp - log.error(e.getMessage(), e); - } - return isMatched; - } - - boolean isValueMatched(String observationHash, String value) { - if (!existingObsByObsHash.containsKey(observationHash) || existingObsByObsHash.get(observationHash).getValue() == null) { - return value == null; - } - return existingObsByObsHash.get(observationHash).getValue().equals(value); - } - - boolean isObservationMatched(String observationHash, String value, Column phenoCol, Integer rowNum) { - if (timeStampColByPheno.isEmpty() || !timeStampColByPheno.containsKey(phenoCol.name())) { - return isValueMatched(observationHash, value); - } else { - String importObsTimestamp = timeStampColByPheno.get(phenoCol.name()).getString(rowNum); - return isTimestampMatched(observationHash, importObsTimestamp) && isValueMatched(observationHash, value); - } - } - - private void fetchOrCreateObservationPIO(Program program, - User user, - ExperimentObservation importRow, - Column column, - Integer rowNum, - String timeStampValue, - boolean commit, - String seasonDbId, - PendingImportObject obsUnitPIO, - PendingImportObject studyPIO, - List referencedTraits) throws ApiException, UnprocessableEntityException { - PendingImportObject pio; - BrAPIObservation newObservation; - String variableName = column.name(); - String value = column.getString(rowNum); - String key; - if (hasAllReferenceUnitIds) { - String unitName = obsUnitPIO.getBrAPIObject().getObservationUnitName(); - String studyName = studyPIO.getBrAPIObject().getStudyName(); - key = getObservationHash(studyName + unitName, variableName, studyName); - } else { - key = getImportObservationHash(importRow, variableName); - } - - if (existingObsByObsHash.containsKey(key)) { - // Update observation value only if it is changed and new value is not blank. - if (!isObservationMatched(key, value, column, rowNum) && StringUtils.isNotBlank(value)){ - - // prior observation with updated value - newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); - if (!isValueMatched(key, value)){ - newObservation.setValue(value); - } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(key, timeStampValue)) { - DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; - String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); - newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); - } - pio = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); - } else { - - // prior observation - pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(existingObsByObsHash.get(key), BrAPIObservation.class, program)); - } - - observationByHash.put(key, pio); - } else if (!this.observationByHash.containsKey(key)){ - - // new observation - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - - UUID trialID = trialPIO.getId(); - UUID studyID = studyPIO.getId(); - UUID id = UUID.randomUUID(); - newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); - //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. - // Validation error should be thrown for offending value, but that doesn't happen until later downstream - if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) { - newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); - } - - newObservation.setStudyDbId(studyPIO.getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations - - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); - this.observationByHash.put(key, pio); - } - } - private void addObsVarsToDatasetDetails(PendingImportObject pio, List referencedTraits, Program program) { - BrAPIListDetails details = pio.getBrAPIObject(); - referencedTraits.forEach(trait -> { - String id = Utilities.appendProgramKey(trait.getObservationVariableName(), program.getKey()); - - // TODO - Don't append the key if connected to a brapi service operating with legacy data(no appended program key) - - if (!details.getData().contains(id) && ImportObjectState.EXISTING != pio.getState()) { - details.getData().add(id); - } - if (!details.getData().contains(id) && ImportObjectState.EXISTING == pio.getState()) { - details.getData().add(id); - pio.setState(ImportObjectState.MUTATED); - } - }); - } - - private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { - PendingImportObject pio; - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - String name = String.format("Observation Dataset [%s-%s]", - program.getKey(), - trialPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) - .getAsString()); - if (obsVarDatasetByName.containsKey(name)) { - pio = obsVarDatasetByName.get(name); - } else { - UUID id = UUID.randomUUID(); - BrAPIListDetails newDataset = importRow.constructDatasetDetails( - name, - id, - BRAPI_REFERENCE_SOURCE, - program, - trialPIO.getId().toString()); - pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); - - JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); - // If datasets property does not yet exist, create it - String datasetName = StringUtils.isNotBlank(importRow.getSubObsUnit()) ? importRow.getSubObsUnit() : importRow.getExpUnit(); - List datasets = DatasetUtil.datasetsFromJson(datasetsJson); - DatasetMetadata dataset = DatasetMetadata.builder() - .name(datasetName) - .id(id) - .level(StringUtils.isNotBlank(importRow.getSubObsUnit()) ? DatasetLevel.SUB_OBS_UNIT : DatasetLevel.EXP_UNIT) - .build(); - - log.debug(dataset.getName()); - datasets.add(dataset); - datasetsJson = DatasetUtil.jsonArrayFromDatasets(datasets); - trialPIO.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, datasetsJson); - - if (ImportObjectState.EXISTING == trialPIO.getState()) { - trialPIO.setState(ImportObjectState.MUTATED); - } - obsVarDatasetByName.put(name, pio); - } - addObsVarsToDatasetDetails(pio, referencedTraits, program); - } - - private PendingImportObject fetchOrCreateStudyPIO( - Program program, - boolean commit, - String expSequenceValue, - ExperimentObservation importRow, - Supplier envNextVal - ) throws UnprocessableEntityException { - PendingImportObject pio; - if (hasAllReferenceUnitIds) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), - program.getKey() - ); - pio = studyByNameNoScope.get(studyName); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { - pio = studyByNameNoScope.get(importRow.getEnv()); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else { - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - UUID trialID = trialPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); - newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies - - // It is assumed that the study has only one season, And that the Years and not - // the dbId's are stored in getSeason() list. - String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season - if (commit) { - if(StringUtils.isNotBlank(year)) { - String seasonID = this.yearToSeasonDbId(year, program.getId()); - newStudy.setSeasons(Collections.singletonList(seasonID)); - } - } else { - addYearToStudyAdditionalInfo(program, newStudy, year); - } - - pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); - this.studyByNameNoScope.put(importRow.getEnv(), pio); - } - return pio; - } - - - /* - * this finds the YEAR from the season list on the BrAPIStudy and then - * will add the year to the additionalInfo-field of the BrAPIStudy - * */ - private void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study) { - JsonObject additionalInfo = study.getAdditionalInfo(); - - //if it is already there, don't add it. - if(additionalInfo==null || additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { - String year = study.getSeasons().get(0); - addYearToStudyAdditionalInfo(program, study, year); - } - } - - - /* - * this will add the given year to the additionalInfo field of the BrAPIStudy (if it does not already exist) - * */ - private void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, String year) { - JsonObject additionalInfo = study.getAdditionalInfo(); - if (additionalInfo==null){ - additionalInfo = new JsonObject(); - study.setAdditionalInfo(additionalInfo); - } - if( additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { - additionalInfo.addProperty(BrAPIAdditionalInfoFields.ENV_YEAR, year); - } - } - - private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { - PendingImportObject pio; - String envLocationName = hasAllReferenceUnitIds ? - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName() : importRow.getEnvLocation(); - if (!locationByName.containsKey((importRow.getEnvLocation()))) { - ProgramLocation newLocation = new ProgramLocation(); - newLocation.setName(envLocationName); - pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); - this.locationByName.put(envLocationName, pio); - } - } - - private PendingImportObject fetchOrCreateTrialPIO( - Program program, - User user, - boolean commit, - ExperimentObservation importRow, - Supplier expNextVal - ) throws UnprocessableEntityException { - PendingImportObject trialPio; - - // use the prior trial if observation unit IDs are supplied - if (hasAllReferenceUnitIds) { - trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); - - // otherwise create a new trial, but there can be only one allowed - } else { - if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { - PendingImportObject envPio; - trialPio = trialByNameNoScope.get(importRow.getExpTitle()); - envPio = studyByNameNoScope.get(importRow.getEnv()); - - // creating new units for existing experiments and environments is not possible - if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && - (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ - throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); - } - } else if (!trialByNameNoScope.isEmpty()) { - throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); - } else { - UUID id = UUID.randomUUID(); - String expSeqValue = null; - if (commit) { - expSeqValue = expNextVal.get().toString(); - } - BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); - trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - trialByNameNoScope.put(importRow.getExpTitle(), trialPio); - } - } - return trialPio; - } - - private void updateObservationDependencyValues(Program program) { - String programKey = program.getKey(); - - // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds - this.observationUnitByNameNoScope.values().stream() - .map(PendingImportObject::getBrAPIObject) - .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); - - // Update ObservationVariable DbIds - List traits = getTraitList(program); - CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); - for ( Trait trait: traits) { - traitMap.put(trait.getObservationVariableName(),trait); - } - for (PendingImportObject observation : this.observationByHash.values()) { - String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); - if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { - String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); - observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); - } - } - } - - private List getTraitList(Program program) { - try { - return ontologyService.getTraitsByProgramId(program.getId(), true); - } catch (DoesNotExistException e) { - log.error(e.getMessage(), e); - throw new InternalServerException(e.toString(), e); - } - } - - // Update each ovservation's observationUnit DbId, study DbId, and germplasm DbId - private void updateObservationDbIds(BrAPIObservationUnit obsUnit, String programKey) { - // FILTER LOGIC: Match on Env and Exp Unit ID - this.observationByHash.values() - .stream() - .filter(obs -> obs.getBrAPIObject() - .getAdditionalInfo() != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) - .getAsString() - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getStudyName(), programKey)) - && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject().getObservationUnitName(), programKey) - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey)) - ) - .forEach(obs -> { - if (StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { - obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); - } - obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); - obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId()); - }); - } - - private void updateObsUnitDependencyValues(String programKey) { - - // update study DbIds - this.studyByNameNoScope.values() - .stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(study -> updateStudyDbId(study, programKey)); - - // update germplasm DbIds - this.existingGermplasmByGID.values() - .stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(this::updateGermplasmDbId); - } - - private void updateStudyDbId(BrAPIStudy study, String programKey) { - this.observationUnitByNameNoScope.values() - .stream() - .filter(obsUnit -> obsUnit.getBrAPIObject() - .getStudyName() - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey))) - .forEach(obsUnit -> { - obsUnit.getBrAPIObject() - .setStudyDbId(study.getStudyDbId()); - obsUnit.getBrAPIObject() - .setTrialDbId(study.getTrialDbId()); - }); - } - - private void updateGermplasmDbId(BrAPIGermplasm germplasm) { - this.observationUnitByNameNoScope.values() - .stream() - .filter(obsUnit -> germplasm.getAccessionNumber() != null && - germplasm.getAccessionNumber().equals(obsUnit - .getBrAPIObject() - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.GID).getAsString())) - .forEach(obsUnit -> obsUnit.getBrAPIObject() - .setGermplasmDbId(germplasm.getGermplasmDbId())); - } - - private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { - // update location DbIds in studies for all distinct locations - mappedBrAPIImport.values() - .stream() - .map(PendingImport::getLocation) - .forEach(this::updateStudyLocationDbId); - - // update trial DbIds in studies for all distinct trials - this.trialByNameNoScope.values() - .stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(trial -> this.updateTrialDbId(trial, programKey)); - } - - private void updateStudyLocationDbId(PendingImportObject location) { - this.studyByNameNoScope.values() - .stream() - .filter(study -> location.getId().toString() - .equals(study.getBrAPIObject() - .getLocationDbId())) - .forEach(study -> study.getBrAPIObject() - .setLocationDbId(location.getBrAPIObject().getLocationDbId())); - } - - private void updateTrialDbId(BrAPITrial trial, String programKey) { - this.studyByNameNoScope.values() - .stream() - .filter(study -> study.getBrAPIObject() - .getTrialName() - .equals(Utilities.removeProgramKey(trial.getTrialName(), programKey))) - .forEach(study -> study.getBrAPIObject() - .setTrialDbId(trial.getTrialDbId())); - } - - private ArrayList getGermplasmByAccessionNumber( - List germplasmAccessionNumbers, - UUID programId) throws ApiException { - List germplasmList = brAPIGermplasmDAO.getGermplasm(programId); - ArrayList resultGermplasm = new ArrayList<>(); - // Search for accession number matches - for (BrAPIGermplasm germplasm : germplasmList) { - for (String accessionNumber : germplasmAccessionNumbers) { - if (germplasm.getAccessionNumber() - .equals(accessionNumber)) { - resultGermplasm.add(germplasm); - break; - } - } - } - return resultGermplasm; - } - - private Map> initializeObservationUnits(Program program, List experimentImportRows) { - Map> ret = new HashMap<>(); - - Map rowByObsUnitId = new HashMap<>(); - experimentImportRows.forEach(row -> { - if (StringUtils.isNotBlank(row.getObsUnitID())) { - if(rowByObsUnitId.containsKey(row.getObsUnitID())) { - throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); - } - rowByObsUnitId.put(row.getObsUnitID(), row); - } - }); - - try { - List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); - - String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - if (existingObsUnits.size() == rowByObsUnitId.size()) { - existingObsUnits.forEach(brAPIObservationUnit -> { - processAndCacheObservationUnit(brAPIObservationUnit, refSource, program, ret, rowByObsUnitId); - - BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) - .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); - row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); - row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); - row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); - }); - } else { - List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); - missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(COMMA_DELIMITER, missingIds)); - } - - return ret; - } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - /** - * Maps pending locations by Observation Unit (OU) Id based on given parameters. - * - * This method takes in a unitId, BrAPIObservationUnit unit, Maps of studyByOUId, locationByName, - * and locationByOUId. It then associates the location of the observation unit with the respective OU Id. - * If the locationName is not null for the unit, it is directly added to locationByOUId. - * If the locationName is null, it checks the studyByOUId map for a location related to the unit. - * If a location related to the unit is found, it maps that location with the respective OU Id. - * If no location is found, it throws an IllegalStateException. - * - * @param unitId the Observation Unit Id - * @param unit the BrAPIObservationUnit object - * @param studyByOUId a Map of Study by Observation Unit Id - * @param locationByName a Map of Location by Name - * @param locationByOUId a Map of Location by Observation Unit Id - * @return the updated locationByOUId map after mapping the pending locations - * @throws IllegalStateException if the Observation Unit is missing a location - */ - private Map> mapPendingLocationByOUId( - String unitId, - BrAPIObservationUnit unit, - Map> studyByOUId, - Map> locationByName, - Map> locationByOUId - ) { - if (unit.getLocationName() != null) { - locationByOUId.put(unitId, locationByName.get(unit.getLocationName())); - } else if (studyByOUId.get(unitId) != null && studyByOUId.get(unitId).getBrAPIObject().getLocationName() != null) { - locationByOUId.put( - unitId, - locationByName.get(studyByOUId.get(unitId).getBrAPIObject().getLocationName()) - ); - } else { - throw new IllegalStateException("Observation unit missing location: " + unitId); - } - - return locationByOUId; - } - - private Map> mapPendingStudyByOUId( - String unitId, - BrAPIObservationUnit unit, - Map> studyByName, - Map> studyByOUId, - Program program - ) { - if (unit.getStudyName() != null) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); - studyByOUId.put(unitId, studyByName.get(studyName)); - } else { - throw new IllegalStateException("Observation unit missing study name: " + unitId); - } - - return studyByOUId; - } - private Map> mapPendingTrialByOUId( - String unitId, - BrAPIObservationUnit unit, - Map> trialByName, - Map> studyByName, - Map> trialByOUId, - Program program - ) { - String trialName; - if (unit.getTrialName() != null) { - trialName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getTrialName(), program.getKey()); - } else if (unit.getStudyName() != null) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); - trialName = Utilities.removeProgramKeyAndUnknownAdditionalData( - studyByName.get(studyName).getBrAPIObject().getTrialName(), - program.getKey() - ); - } else { - throw new IllegalStateException("Observation unit missing trial name and study name: " + unitId); - } - trialByOUId.put(unitId, trialByName.get(trialName)); - - return trialByOUId; - } - private Map> mapPendingObservationUnitByName( - Map> pendingUnitById, - Program program - ) { - Map> pendingUnitByName = new HashMap<>(); - for (Map.Entry> entry : pendingUnitById.entrySet()) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( - entry.getValue().getBrAPIObject().getStudyName(), - program.getKey() - ); - String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( - entry.getValue().getBrAPIObject().getObservationUnitName(), - program.getKey() - ); - pendingUnitByName.put(createObservationUnitKey(studyName, observationUnitName), entry.getValue()); - } - return pendingUnitByName; - } - - /** - * Retrieves reference Observation Units based on a set of reference Observation Unit IDs and a Program. - * Constructs DeltaBreed observation unit source for external references and sets up pending Observation Units. - * - * @param referenceOUIds A set of reference Observation Unit IDs to retrieve - * @param program The Program associated with the Observation Units - * @return A Map containing pending Observation Units by their ID - * @throws ApiException if an error occurs during the process - */ - private Map> fetchReferenceObservationUnits( - Set referenceOUIds, - Program program - ) throws ApiException { - Map> pendingUnitById = new HashMap<>(); - try { - // Retrieve reference Observation Units based on IDs - List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(referenceOUIds), - program - ); - - // Construct the DeltaBreed observation unit source for external references - String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - - if (referenceObsUnits.size() == referenceOUIds.size()) { - // Iterate through reference Observation Units - referenceObsUnits.forEach(unit -> { - // Get external reference for the Observation Unit - BrAPIExternalReference unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) - .orElseThrow(() -> new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID")); - - // Set pending Observation Unit by its ID - pendingUnitById.put( - unitXref.getReferenceId(), - new PendingImportObject( - ImportObjectState.EXISTING, unit, UUID.fromString(unitXref.getReferenceId())) - ); - }); - } else { - // Handle missing Observation Unit IDs - List missingIds = new ArrayList<>(referenceOUIds); - Set fetchedIds = referenceObsUnits.stream().map(unit -> - Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) - .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) - .getReferenceId()) - .collect(Collectors.toSet()); - missingIds.removeAll(fetchedIds); - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(COMMA_DELIMITER, missingIds)); - } - - return pendingUnitById; - } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new ApiException(e); - } - } - - private Map> initializeTrialByNameNoScope(Program program, List experimentImportRows) { - Map> trialByName = new HashMap<>(); - - initializeTrialsForExistingObservationUnits(program, trialByName); - - List uniqueTrialNames = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID())) - .map(ExperimentObservation::getExpTitle) - .distinct() - .collect(Collectors.toList()); - try { - brapiTrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> - processAndCacheTrial(existingTrial, program, trialByName) - ); - } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - return trialByName; - } - - private Map> initializeStudyByNameNoScope(Program program, List experimentImportRows) { - Map> studyByName = new HashMap<>(); - if (this.trialByNameNoScope.size() != 1) { - return studyByName; - } - - try { - initializeStudiesForExistingObservationUnits(program, studyByName); - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (Exception e) { - log.error("Error processing studies", e); - throw new InternalServerException(e.toString(), e); - } - - List existingStudies; - Optional> trial = getTrialPIO(experimentImportRows); - - try { - if (trial.isEmpty()) { - // TODO: throw ValidatorException and return 422 - } - UUID experimentId = trial.get().getId(); - existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); - for (BrAPIStudy existingStudy : existingStudies) { - processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); - } - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (Exception e) { - log.error("Error processing studies: ", e); - throw new InternalServerException(e.toString(), e); - } - - return studyByName; - } - - private void initializeStudiesForExistingObservationUnits( - Program program, - Map> studyByName - ) throws Exception { - Set studyDbIds = observationUnitByNameNoScope.values() - .stream() - .map(pio -> pio.getBrAPIObject() - .getStudyDbId()) - .collect(Collectors.toSet()); - - List studies = fetchStudiesByDbId(studyDbIds, program); - for (BrAPIStudy study : studies) { - processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); - } - } - - /** - * Fetches a list of BrAPI studies by their study database IDs for a given program. - * - * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. - * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. - * - * @param studyDbIds a Set of Strings representing the study database IDs to fetch - * @param program the Program object representing the program context in which to fetch studies - * @return a List of BrAPIStudy objects matching the provided study database IDs - * @throws ApiException if there is an issue fetching the studies - * @throws IllegalStateException if any requested study database IDs are not found in the result set - */ - private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { - List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); - if (studies.size() != studyDbIds.size()) { - List missingIds = new ArrayList<>(studyDbIds); - missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); - throw new IllegalStateException( - "Study not found for studyDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); - } - return studies; - } - - /** - * Initializes a map of ProgramLocation objects by their names using the given Program and a map of BrAPIStudy objects by their names. - * - * This method takes a Program object and a map of BrAPIStudy objects by their names, retrieves the location database IDs from the studies, - * and fetches existing ProgramLocation objects based on the database IDs. It then creates a map of ProgramLocation objects by their names - * with PendingImportObject wrappers that indicate the state of the object as existing. - * - * @param program the Program object to associate with the locations - * @param studyByName a map of BrAPIStudy objects by their names - * @return a map of ProgramLocation objects by their names with PendingImportObject wrappers - * @throws InternalServerException if an error occurs during the location retrieval process - */ - Map> initializeLocationByName( - Program program, - Map> studyByName) { - Map> locationByName = new HashMap<>(); - - List existingLocations = new ArrayList<>(); - if(studyByName.size() > 0) { - Set locationDbIds = studyByName.values() - .stream() - .map(study -> study.getBrAPIObject() - .getLocationDbId()) - .collect(Collectors.toSet()); - try { - existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - existingLocations.forEach(existingLocation -> locationByName.put( - existingLocation.getName(), - new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()) - ) - ); - return locationByName; - } - private Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { - Map> locationByName = new HashMap<>(); - - List existingLocations = new ArrayList<>(); - if(studyByNameNoScope.size() > 0) { - Set locationDbIds = studyByNameNoScope.values() - .stream() - .map(study -> study.getBrAPIObject() - .getLocationDbId()) - .collect(Collectors.toSet()); - try { - existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - List uniqueLocationNames = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) - .map(ExperimentObservation::getEnvLocation) - .distinct() - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - try { - existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); - } catch (ApiException e) { - log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); - return locationByName; - } - - /** - * Maps a given germplasm to an observation unit ID in a given map of germplasm by observation unit ID. - * - * This method retrieves the Global Identifier (GID) of the provided observation unit and uses it to lookup - * the corresponding PendingImportObject in the map of germplasm by name. The found germplasm - * object is then mapped to the observation unit ID in the provided map of germplasm by observation unit ID. - * The updated map is returned after the mapping operation has been performed. - * - * @param unitId The observation unit ID to which the germplasm should be mapped. - * @param unit The BrAPIObservationUnit object representing the observation unit. - * @param germplasmByName The map of germplasm objects by name used to lookup the desired germplasm. - * @param germplasmByOUId The map of germplasm objects by observation unit ID to update with the mapping result. - * @return The updated map of germplasm objects by observation unit ID after mapping the germplasm to the provided observation unit ID. - */ - private Map> mapGermplasmByOUId( - String unitId, - BrAPIObservationUnit unit, - Map> germplasmByName, - Map> germplasmByOUId) { - String gid = unit.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.GID).getAsString(); - germplasmByOUId.put(unitId, germplasmByName.get(gid)); - - return germplasmByOUId; - } - - /** - * Maps the pending observation dataset by OU Id based on the given inputs. - * This function checks if the trialByOUId map is not empty, the obsVarDatasetByName map is not empty, - * and if the first entry in the trialByOUId map contains observation dataset id in its additional info. - * If the conditions are met, it adds the pending import object from the obsVarDatasetByName map to the - * obsVarDatasetByOUId map using the unitId as the key. - * - * @param unitId the unit ID based on which the mapping is done - * @param trialByOUId a map containing pending import objects with BrAPITrial as the value, mapped by OU Id - * @param obsVarDatasetByName a map containing pending import objects with BrAPIListDetails as the value, mapped by dataset name - * @param obsVarDatasetByOUId a map containing pending import objects with BrAPIListDetails as the value, mapped by OU Id - * @return the updated obsVarDatasetByOUId map after potential addition of a pending import object - */ - private Map> mapPendingObsDatasetByOUId( - String unitId, - Map> trialByOUId, - Map> obsVarDatasetByName, - Map> obsVarDatasetByOUId) { - if (!trialByOUId.isEmpty() && - !obsVarDatasetByName.isEmpty() && - !trialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { - obsVarDatasetByOUId.put(unitId, obsVarDatasetByName.values().iterator().next()); - } - - return obsVarDatasetByOUId; - } - - /** - * Initializes observation variable dataset for existing observation units. This function retrieves existing datasets related to observation variables for the specified trial and program, processes the dataset details, and caches the data accordingly. - * - * @param trialByName A map containing trial information indexed by trial name. - * @param program The program to which the datasets are related. - * @return A map of observation variable dataset objects indexed by dataset name. - * @throws InternalServerException If the existing dataset summary is not retrieved from the BrAPI server, or an error occurs during API communication. - */ - // TODO: add dataset name or id parameter? This only supports top level for now. - private Map> initializeObsVarDatasetForExistingObservationUnits( - Map> trialByName, - Program program) { - Map> obsVarDatasetByName = new HashMap<>(); - - if (trialByName.size() > 0 && - !trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { - String datasetId = DatasetUtil.getDatasetIdByNameFromJson( - trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS), - trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString() - ); - - try { - List existingDatasets = brAPIListDAO - .getListsByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, - program.getId(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), - UUID.fromString(datasetId)); - if (existingDatasets == null || existingDatasets.isEmpty()) { - throw new InternalServerException("existing dataset summary not returned from brapi server"); - } - BrAPIListDetails dataSetDetails = brAPIListDAO - .getListById(existingDatasets.get(0).getListDbId(), program.getId()) - .getResult(); - processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - return obsVarDatasetByName; - } - - private Map> initializeObsVarDatasetByName(Program program, List experimentImportRows) { - Map> obsVarDatasetByName = new HashMap<>(); - - Optional> trialPIO = getTrialPIO(experimentImportRows); - - if (trialPIO.isPresent() && !trialPIO.get().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { - String datasetId = DatasetUtil.getTopLevelDatasetFromJson( - trialPIO.get().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS) - ).getId().toString(); - try { - List existingDatasets = brAPIListDAO - .getListsByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, - program.getId(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), - UUID.fromString(datasetId)); - if (existingDatasets == null || existingDatasets.isEmpty()) { - throw new InternalServerException("existing dataset summary not returned from brapi server"); - } - BrAPIListDetails dataSetDetails = brAPIListDAO - .getListById(existingDatasets.get(0).getListDbId(), program.getId()) - .getResult(); - processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - return obsVarDatasetByName; - } - - private Optional> getTrialPIO(List experimentImportRows) { - Optional expTitle = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID()) && StringUtils.isNotBlank(row.getExpTitle())) - .map(ExperimentObservation::getExpTitle) - .findFirst(); - - if (expTitle.isEmpty() && trialByNameNoScope.keySet().stream().findFirst().isEmpty()) { - return Optional.empty(); - } - if(expTitle.isEmpty()) { - expTitle = trialByNameNoScope.keySet().stream().findFirst(); - } - - return Optional.ofNullable(this.trialByNameNoScope.get(expTitle.get())); - } - - private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> obsVarDatasetByName) { - BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); - obsVarDatasetByName.put(existingList.getListName(), - new PendingImportObject(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); - } - - /** - * Initializes a mapping of BrAPI Germplasm objects by Germplasm ID for existing BrAPI Observation Units. - * This method retrieves existing Germplasms associated with the provided Observation Units and creates a mapping - * using their Accession Number as the key and a PendingImportObject containing the Germplasm object and a reference ID. - * If no existing Germplasms are found, an empty mapping is returned. - * - * @param unitByName A mapping of Observation Units by name. - * @param program The BrAPI Program object to which the Germplasms belong. - * @return A mapping of BrAPI Germplasm objects by Germplasm ID for existing Observation Units. - * @throws InternalServerException If an error occurs while fetching Germplasms from the database. - */ - private Map> initializeGermplasmByGIDForExistingObservationUnits( - Map> unitByName, - Program program) { - Map> existingGermplasmByGID = new HashMap<>(); - - List existingGermplasms = new ArrayList<>(); - if(unitByName.size() > 0) { - Set germplasmDbIds = unitByName.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); - try { - existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - existingGermplasms.forEach(existingGermplasm -> { - BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); - existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); - }); - return existingGermplasmByGID; - } - - - private Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { - Map> existingGermplasmByGID = new HashMap<>(); - - List existingGermplasms = new ArrayList<>(); - if(observationUnitByNameNoScope.size() > 0) { - Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); - try { - existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - List uniqueGermplasmGIDs = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) - .map(ExperimentObservation::getGid) - .distinct() - .collect(Collectors.toList()); - - try { - existingGermplasms.addAll(this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - existingGermplasms.forEach(existingGermplasm -> { - BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); - existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); - }); - return existingGermplasmByGID; - } - - private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservationUnit, String refSource, Program program, Map> observationUnitByName, - Map rowByObsUnitId) { - BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) - .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); - row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); - observationUnitByName.put(createObservationUnitKey(row), - new PendingImportObject<>(ImportObjectState.EXISTING, - brAPIObservationUnit, - UUID.fromString(idRef.getReferenceId()))); - } - private PendingImportObject processAndCacheStudy( - BrAPIStudy existingStudy, - Program program, - Function getterFunction, - Map> studyMap) throws Exception { - PendingImportObject pendingStudy; - BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); - // map season dbid to year - String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season - if(StringUtils.isNotBlank(seasonDbId)) { - String seasonYear = this.seasonDbIdToYear(seasonDbId, program.getId()); - existingStudy.setSeasons(Collections.singletonList(seasonYear)); - } - pendingStudy = new PendingImportObject<>( - ImportObjectState.EXISTING, - (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), - UUID.fromString(xref.getReferenceId()) - ); - studyMap.put( - Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), - pendingStudy - ); - return pendingStudy; - } - - private void initializeTrialsForExistingObservationUnits(Program program, Map> trialByName) { - if(observationUnitByNameNoScope.size() > 0) { - Set trialDbIds = new HashSet<>(); - Set studyDbIds = new HashSet<>(); - - observationUnitByNameNoScope.values() - .forEach(pio -> { - BrAPIObservationUnit existingOu = pio.getBrAPIObject(); - if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { - throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); - } - - if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { - trialDbIds.add(existingOu.getTrialDbId()); - } else { - studyDbIds.add(existingOu.getStudyDbId()); - } - }); - - //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId - if(!studyDbIds.isEmpty()) { - try { - trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - try { - List trials = brapiTrialDAO.getTrialsByDbIds(trialDbIds, program); - if (trials.size() != trialDbIds.size()) { - List missingIds = new ArrayList<>(trialDbIds); - missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); - throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); - } - - trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); - } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - } - - private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { - Set trialDbIds = new HashSet<>(); - List studies = fetchStudiesByDbId(studyDbIds, program); - studies.forEach(study -> { - if (StringUtils.isBlank(study.getTrialDbId())) { - throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); - } - trialDbIds.add(study.getTrialDbId()); - }); - - return trialDbIds; - } - - private void processAndCacheTrial( - BrAPITrial existingTrial, - Program program, - Map> trialByNameNoScope) { - - //get TrialId from existingTrial - BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) - .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); - UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); - - trialByNameNoScope.put( - Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), - new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); - } - - /** - * This function collates unique ObsUnitID values from a list of BrAPIImport objects. - * It iterates through the list and adds non-blank ObsUnitID values to a Set. - * It also checks for any repeated ObsUnitIDs. The instance variables hasAllReferenceUnitIds and - * hasNoReferenceUnitIds are updated. - * - * @param importRows a List of BrAPIImport objects containing ExperimentObservation data - * @return a Set of unique ObsUnitID strings - * @throws IllegalStateException if a repeated ObsUnitID is encountered - */ - private Set collateReferenceOUIds(List importRows) { - Set referenceOUIds = new HashSet<>(); - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - - // Check if ObsUnitID is blank - if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { - hasAllReferenceUnitIds = false; - } else if (referenceOUIds.contains(importRow.getObsUnitID())) { - // Throw exception if ObsUnitID is repeated - throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); - } else { - // Add ObsUnitID to referenceOUIds - referenceOUIds.add(importRow.getObsUnitID()); - hasNoReferenceUnitIds = false; - } - } - return referenceOUIds; - } - - private void validateTimeStampValue(String value, - String columnHeader, ValidationErrors validationErrors, int row) { - if (StringUtils.isBlank(value)) { - log.debug(String.format("skipping validation of observation timestamp because there is no value.\n\tvariable: %s\n\trow: %d", columnHeader, row)); - return; - } - if (!validDateValue(value) && !validDateTimeValue(value)) { - addRowError(columnHeader, "Incorrect datetime format detected. Expected YYYY-MM-DD or YYYY-MM-DDThh:mm:ss+hh:mm", validationErrors, row); - } - - } - - private boolean isNAObservation(String value){ - return value.equalsIgnoreCase("NA"); - } - - private void validateObservationValue(Trait variable, String value, - String columnHeader, ValidationErrors validationErrors, int row) { - if (StringUtils.isBlank(value)) { - log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row)); - return; - } - - if (isNAObservation(value)) { - log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row)); - return; - } - - switch (variable.getScale().getDataType()) { - case NUMERICAL: - Optional number = validNumericValue(value); - if (number.isEmpty()) { - addRowError(columnHeader, "Non-numeric text detected detected", validationErrors, row); - } else if (!validNumericRange(number.get(), variable.getScale())) { - addRowError(columnHeader, "Value outside of min/max range detected", validationErrors, row); - } - break; - case DATE: - if (!validDateValue(value)) { - addRowError(columnHeader, "Incorrect date format detected. Expected YYYY-MM-DD", validationErrors, row); - } - break; - case ORDINAL: - if (!validCategory(variable.getScale().getCategories(), value)) { - addRowError(columnHeader, "Undefined ordinal category detected", validationErrors, row); - } - break; - case NOMINAL: - if (!validCategory(variable.getScale().getCategories(), value)) { - addRowError(columnHeader, "Undefined nominal category detected", validationErrors, row); - } - break; - default: - break; - } - - } - - private Optional validNumericValue(String value) { - BigDecimal number; - try { - number = new BigDecimal(value); - } catch (NumberFormatException e) { - return Optional.empty(); - } - return Optional.of(number); - } - - private boolean validNumericRange(BigDecimal value, Scale validValues) { - // account for empty min or max in valid determination - return (validValues.getValidValueMin() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMin())) >= 0) && - (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0); - } - - private boolean validDateValue(String value) { - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE; - try { - formatter.parse(value); - } catch (DateTimeParseException e) { - return false; - } - return true; - } - - private boolean validDateTimeValue(String value) { - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; - try { - formatter.parse(value); - } catch (DateTimeParseException e) { - return false; - } - return true; - } - - private boolean validCategory(List categories, String value) { - Set categoryValues = categories.stream() - .map(category -> category.getValue().toLowerCase()) - .collect(Collectors.toSet()); - return categoryValues.contains(value.toLowerCase()); - } - - /** - * Converts year String to SeasonDbId - *
- * NOTE: This assumes that the only Season records of interest are ones - * with a blank name or a name that is the same as the year. - * - * @param year The year as a string - * @param programId the program ID. - * @return the DbId of the season-record associated with the year - */ - private String yearToSeasonDbId(String year, UUID programId) { - String dbID = null; - if (this.yearToSeasonDbIdCache.containsKey(year)) { // get it from cache if possible - dbID = this.yearToSeasonDbIdCache.get(year); - } else { - dbID = this.yearToSeasonDbIdFromDatabase(year, programId); - this.yearToSeasonDbIdCache.put(year, dbID); - } - return dbID; - } - - private String seasonDbIdToYear(String seasonDbId, UUID programId) { - String year = null; - if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible - year = this.seasonDbIdToYearCache.get(seasonDbId); - } else { - year = this.seasonDbIdToYearFromDatabase(seasonDbId, programId); - this.seasonDbIdToYearCache.put(seasonDbId, year); - } - return year; - } - - private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { - BrAPISeason targetSeason = null; - List seasons; - try { - seasons = this.brAPISeasonDAO.getSeasonsByYear(year, programId); - for (BrAPISeason season : seasons) { - if (null == season.getSeasonName() || season.getSeasonName().isBlank() || season.getSeasonName().equals(year)) { - targetSeason = season; - break; - } - } - if (targetSeason == null) { - BrAPISeason newSeason = new BrAPISeason(); - Integer intYear = null; - if( StringUtils.isNotBlank(year) ){ - intYear = Integer.parseInt(year); - } - newSeason.setYear(intYear); - newSeason.setSeasonName(year); - targetSeason = this.brAPISeasonDAO.addOneSeason(newSeason, programId); - } - - } catch (ApiException e) { - log.warn(Utilities.generateApiExceptionLogMessage(e)); - log.error(e.getResponseBody(), e); - } - - return (targetSeason == null) ? null : targetSeason.getSeasonDbId(); - } - - private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { - BrAPISeason season = null; - try { - season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e), e); - } - Integer yearInt = (season == null) ? null : season.getYear(); - return (yearInt == null) ? "" : yearInt.toString(); - } - - /** - * Returns the single value from the given map, throwing an UnprocessableEntityException if the map does not contain exactly one entry. - * - * @param map The map from which to retrieve the single value. - * @param message The error message to include in the UnprocessableEntityException if the map does not contain exactly one entry. - * @return The single value from the map. - * @throws UnprocessableEntityException if the map does not contain exactly one entry. - */ - private V getSingleEntryValue(Map map, String message) throws UnprocessableEntityException { - if (map.size() != 1) { - throw new UnprocessableEntityException(message); - } - return map.values().iterator().next(); - } -} From b6fa1a78b1918f89f6dff9158f763c400d0efeb1 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 16 May 2025 10:44:16 -0400 Subject: [PATCH 056/289] update ExperimentUtilities --- .../experiment/ExperimentUtilities.java | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index e5685ac9a..46769bb5b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -319,7 +319,7 @@ public static Set collateUniqueOUIds(AppendOverwriteMiddlewareContext ct Set referenceOUIds = new HashSet<>(); String idColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); - Column idCol = ctx.getImportContext().getData().columns(idColName).get(0); + Column idCol = ctx.getImportContext().getData().column(idColName); for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { String id = idCol.getString(rowNum); @@ -332,12 +332,9 @@ public static Set collateUniqueOUIds(AppendOverwriteMiddlewareContext ct public static boolean hasUniqueIds(AppendOverwriteMiddlewareContext ctx, String colName) throws IllegalStateException { Set referenceOUIds = new HashSet<>(); - List> columns = ctx.getImportContext().getData().columns(colName); - if (columns.isEmpty()) { - throw new IllegalStateException("No columns found for: " + colName); - } + Column col = Optional.ofNullable(ctx.getImportContext().getData().column(colName)) + .orElseThrow(()->new IllegalStateException("Column "+colName+" not found")); - Column col = columns.get(0); for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { if (referenceOUIds.contains(col.getString(rowNum))) { return false; @@ -356,7 +353,7 @@ public static boolean hasUniqueIds(AppendOverwriteMiddlewareContext ctx, String * to the context for each import row where the Observation Unit ID was not found. * * @param e The EntityNotFoundException containing information about missing Observation Unit IDs. - * @param context The AppendOverwriteMiddlewareContext containing import data and validation error storage. + * @param ctx The AppendOverwriteMiddlewareContext containing import data and validation error storage. * * @implNote The method performs the following steps: * 1. Retrieves the ValidationErrors object from the context. @@ -365,14 +362,15 @@ public static boolean hasUniqueIds(AppendOverwriteMiddlewareContext ctx, String * 4. If a match is found, adds a validation error for that row, indicating an invalid Observation Unit ID. * 5. The error is added using the addRowError method, specifying the OBS_UNIT_ID column and using a predefined error message. */ - public static void addValidationErrorsForObsUnitsNotFound(EntityNotFoundException e, AppendOverwriteMiddlewareContext context) { - ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + public static void addValidationErrorsForObsUnitsNotFound(EntityNotFoundException e, AppendOverwriteMiddlewareContext ctx) { + ValidationErrors validationErrors = ctx.getAppendOverwriteWorkflowContext().getValidationErrors(); List errors = new ArrayList<>(); - for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { - String rowObsUnitId = ((ExperimentObservation)context.getImportContext().getImportRows().get(rowNum)).getObsUnitID(); - if (e.getMissingEntityIds().contains(rowObsUnitId)) { - addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExperimentUtilities.INVALID_OBS_UNIT_ID_ERROR, validationErrors, rowNum); + String obsUnitIDColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column idCol = ctx.getImportContext().getData().column(obsUnitIDColName); + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + if (e.getMissingEntityIds().contains(idCol.getString(rowNum))) { + addRowError(obsUnitIDColName, ExperimentUtilities.INVALID_OBS_UNIT_ID_ERROR, validationErrors, rowNum); } } } From 1ef3115fbd462ce47d9ae3368c2b3a55dcd9bda6 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 16 May 2025 10:48:31 -0400 Subject: [PATCH 057/289] remove filters on blank obsunitid in PopulateExistingPendingImportObjectsStep --- .../steps/PopulateExistingPendingImportObjectsStep.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index 14a891482..b00ac9ffc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -342,7 +342,6 @@ private Map> initializeUniqueLocati } List uniqueLocationNames = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) .map(ExperimentObservation::getEnvLocation) .distinct() .filter(Objects::nonNull) @@ -446,7 +445,6 @@ private Map> initializeExistingGermp } List uniqueGermplasmGIDs = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) .map(ExperimentObservation::getGid) .distinct() .collect(Collectors.toList()); From 4112d297e2c95699a387d00c93851ee82e661f3c Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 16 May 2025 10:53:25 -0400 Subject: [PATCH 058/289] delete obsUnitID column from ImportTestUtils#writeExperimentDataToFile --- .../org/breedinginsight/brapps/importer/ImportTestUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 53abd8483..d246394b9 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -322,7 +322,6 @@ public File writeExperimentDataToFile(List> data, List { From 0389aa20c7ca62fbd267792ea1b5c28000028cc9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 16 May 2025 12:00:31 -0400 Subject: [PATCH 059/289] create migration to remove obsunitid col from experiment import template map --- ...periment_template_delete_obsUnitID_col.sql | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/main/resources/db/migration/V1.32.0__experiment_template_delete_obsUnitID_col.sql diff --git a/src/main/resources/db/migration/V1.32.0__experiment_template_delete_obsUnitID_col.sql b/src/main/resources/db/migration/V1.32.0__experiment_template_delete_obsUnitID_col.sql new file mode 100644 index 000000000..01bcbb1c5 --- /dev/null +++ b/src/main/resources/db/migration/V1.32.0__experiment_template_delete_obsUnitID_col.sql @@ -0,0 +1,27 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DO $$ + +BEGIN +-- fix file mapping for ObsUnitID +update importer_mapping +set + mapping = '[{"id": "726a9f10-4892-4204-9e52-bd2b1d735f65", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "98774e20-6f89-4d6a-a7c9-f88887228ed6", "value": {"fileFieldName": "Germplasm GID"}, "objectId": "gid"}, {"id": "880ef0c9-4e3e-42d4-9edc-667684a91889", "value": {"fileFieldName": "Test (T) or Check (C)"}, "objectId": "test_or_check"}, {"id": "b693eca7-efcd-4518-a9d3-db0b037a76ee", "value": {"fileFieldName": "Exp Title"}, "objectId": "exp_title"}, {"id": "df340215-db6e-4219-a3b7-119f297b81c3", "value": {"fileFieldName": "Exp Description"}, "objectId": "expDescription"}, {"id": "9ca7cc81-562c-43a7-989a-41da309f603d", "value": {"fileFieldName": "Exp Unit"}, "objectId": "expUnit"}, {"id": "27215777-c8f9-4fe7-a7ac-918d6168b0dd", "value": {"fileFieldName": "Exp Type"}, "objectId": "expType"}, {"id": "19d220e2-dff0-4a3a-ad6e-32f4d8602b5c", "value": {"fileFieldName": "Env"}, "objectId": "env"}, {"id": "861518b9-5c9e-4fe5-b31e-baf16e27155d", "value": {"fileFieldName": "Env Location"}, "objectId": "envLocation"}, {"id": "667355c3-dae1-4a64-94c8-ac2d543bd474", "value": {"fileFieldName": "Env Year"}, "objectId": "envYear"}, {"id": "ad11f2df-c5b4-4a05-8e52-c57625140061", "value": {"fileFieldName": "Exp Unit ID"}, "objectId": "expUnitId"}, {"id": "639b40ec-20f8-4659-8464-6a4be997ac7a", "value": {"fileFieldName": "Exp Replicate #"}, "objectId": "expReplicateNo"}, {"id": "2a62a80f-d8ba-42c4-9997-3b4ac8a965aa", "value": {"fileFieldName": "Exp Block #"}, "objectId": "expBlockNo"}, {"id": "f3e7de69-21ad-4cda-b1cc-a5e1987fb931", "value": {"fileFieldName": "Row"}, "objectId": "row"}, {"id": "251c5bbd-fc4d-4371-a4ce-4e2686f6837e", "value": {"fileFieldName": "Column"}, "objectId": "column"}, {"id": "a0edcc27-d423-478f-8eed-05b554805ec9", "value": {"fileFieldName": "Lat"}, "objectId": "lat"}, {"id": "a8fd2190-d277-4369-ae72-af32416f14ac", "value": {"fileFieldName": "Long"}, "objectId": "long"}, {"id": "95caefec-2b12-4728-9f2e-3bc17478b662", "value": {"fileFieldName": "Elevation"}, "objectId": "elevation"}, {"id": "e8f80336-0982-4a48-85ac-4b0278e28b70", "value": {"fileFieldName": "RTK"}, "objectId": "rtk"}, {"id": "ce5f61f2-f1de-45a4-8baf-e2471a5d863d", "value": {"fileFieldName": "Treatment Factors"}, "objectId": "treatmentFactors"}]', + file = '[{"Env": "New Study", "Lat": 42.4440, "RTK": "RTK description", "Row": 4, "Long": 76.5019, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "plot", "Elevation": 123, "Exp Title": "New Trial", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Treatment Factors": "Jam application", "Test (T) or Check (C)": true}]' +where import_type_id = 'ExperimentImport'; +END $$; \ No newline at end of file From 69ac5b960d174067807a734442e8f444712353b4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 16 May 2025 12:06:38 -0400 Subject: [PATCH 060/289] delete obsunitid field in ExperimentObservation --- .../experimentObservation/ExperimentObservation.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 8df8be09b..ace52b094 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -145,10 +145,6 @@ public class ExperimentObservation implements BrAPIImport { @ImportFieldMetadata(id = "treatmentFactors", name = Columns.TREATMENT_FACTORS, description = "Treatment factors in an experiment with applied variables, like fertilizer or water regimens.") private String treatmentFactors; - @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "ObsUnitID", name = Columns.OBS_UNIT_ID, description = "A database generated unique identifier for experimental observation units") - private String obsUnitID; - public BrAPITrial constructBrAPITrial(Program program, User user, boolean commit, String referenceSource, UUID id, String expSeqValue) { BrAPIProgram brapiProgram = program.getBrapiProgram(); BrAPITrial trial = new BrAPITrial(); @@ -363,10 +359,6 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( observationUnit.setTreatments(List.of(treatment)); } - if (getObsUnitID() != null) { - observationUnit.setObservationUnitDbId(getObsUnitID()); - } - return observationUnit; } @@ -478,7 +470,6 @@ public static final class Columns { public static final String ELEVATION = "Elevation"; public static final String RTK = "RTK"; public static final String TREATMENT_FACTORS = "Treatment Factors"; - public static final String OBS_UNIT_ID = "ObsUnitID"; } } From 45e47e4f614dbbc6171ba8b71592a72d3dd15ebd Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 19 May 2025 10:44:32 -0400 Subject: [PATCH 061/289] update ExperimentFileImportTest to use dynamic column names for ou id in append workflow tests --- .../importer/ExperimentFileImportTest.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 975c2b083..edfee7583 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -80,6 +80,7 @@ import java.util.stream.StreamSupport; import static io.micronaut.http.HttpRequest.GET; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.*; @@ -688,7 +689,7 @@ public void importNewObsVarExistingOu() { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, appendOverwriteWorkflowId); @@ -741,7 +742,7 @@ public void importNewObsVarByObsUnitId() { assertTrue(ouIdXref.isPresent()); Map newObsVar = new HashMap<>(); - newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, appendOverwriteWorkflowId); @@ -808,7 +809,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); @@ -886,7 +887,7 @@ public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); // Indicates this is an overwrite. + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); // Indicates this is an overwrite. newObsVar.put(traits.get(0).getObservationVariableName(), ""); // Empty string should be no op. Map requestBody = new HashMap<>(); @@ -954,7 +955,7 @@ public void importNewObsExistingOu(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); @@ -1022,7 +1023,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); uploadAndVerifyWorkflowFailureNonTabular(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); @@ -1151,7 +1152,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); @@ -1230,7 +1231,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), ""); // This blank value should not overwrite. newObservation.put(traits.get(1).getObservationVariableName(), "3"); // This valid value should overwrite. newObservation.put(traits.get(2).getObservationVariableName(), "4"); // This valid new observation should be appended. @@ -1264,7 +1265,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { private Map assertRowReferencedByOUIdSaved(Map expected, Program program, List traits) throws ApiException { Map ret = new HashMap<>(); - List units = ouDAO.getObservationUnitsById(List.of((String)expected.get(Columns.OBS_UNIT_ID)), program); + List units = ouDAO.getObservationUnitsById(List.of((String)expected.get("Plot "+OBSERVATION_UNIT_ID_SUFFIX)), program); assertFalse(units.isEmpty()); List observations = null; From 23453447a42550885acee53c1a66906516c54c0f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 19 May 2025 12:55:02 -0400 Subject: [PATCH 062/289] rename SQL migration to 1.33.0 to avoid conflict with java-based migration 1.32.0 --- ....sql => V1.33.0__experiment_template_delete_obsUnitID_col.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V1.32.0__experiment_template_delete_obsUnitID_col.sql => V1.33.0__experiment_template_delete_obsUnitID_col.sql} (100%) diff --git a/src/main/resources/db/migration/V1.32.0__experiment_template_delete_obsUnitID_col.sql b/src/main/resources/db/migration/V1.33.0__experiment_template_delete_obsUnitID_col.sql similarity index 100% rename from src/main/resources/db/migration/V1.32.0__experiment_template_delete_obsUnitID_col.sql rename to src/main/resources/db/migration/V1.33.0__experiment_template_delete_obsUnitID_col.sql From 03a717ac1c7d689a942d671c05e3e8f9a0070781 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 19 May 2025 19:00:48 +0000 Subject: [PATCH 063/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index ed12a56b1..1fecf2ae1 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+963 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/faa5e691f8b526a520e15c3f374520cc9d29b2d2 +version=v1.2.0+967 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1f453d58287d0e0bc9eace790be279ed6920c0a5 From 910beb2325b9a3b170fc085e32b2c62695d806f4 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 21 May 2025 10:48:15 -0400 Subject: [PATCH 064/289] Update species migration to better handle repeat runs --- src/main/resources/brapi/sql/R__species.sql | 74 ++++++++++++--------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/src/main/resources/brapi/sql/R__species.sql b/src/main/resources/brapi/sql/R__species.sql index 843ed32ce..993895398 100644 --- a/src/main/resources/brapi/sql/R__species.sql +++ b/src/main/resources/brapi/sql/R__species.sql @@ -13,39 +13,53 @@ -- See the License for the specific language governing permissions and -- limitations under the License. --- for uuid_generate_v4() +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; DO $$ DECLARE - v_auth_id constant uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; + v_auth_id CONSTANT uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; BEGIN - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Blueberry') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Salmon') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Grape') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Alfalfa') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sweet Potato') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Trout') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Soybean') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Cranberry') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Cucumber') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Oat') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Citrus') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sugar Cane') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Strawberry') ON CONFLICT DO NOTHING; - -- for the Honey Bee case, want to overwrite name, not preserve existing - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Honey Bee') ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Pecan') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Lettuce') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Cotton') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sorghum') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Hemp') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Hop') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Hydrangea') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Red Clover') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Potato') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Blackberry') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Raspberry') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sugar Beet') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Coffee') ON CONFLICT DO NOTHING; + /* ------------------------------------------------------------------------------------------ + • uuid_generate_v5(namespace, crop_name) → deterministic UUID can be used for idempotency + • Do it this way so no schema changes are required + • Removed the Honey Bee special case because all systems will be starting fresh + ------------------------------------------------------------------------------------------ */ + INSERT INTO crop (id, auth_user_id, crop_name) + SELECT + uuid_generate_v5('9a4deca9-4068-46a3-9efe-db0c181f491a'::uuid, + -- 1) lower‑case + -- 2) trim leading/trailing space + -- 3) REMOVE every space or tab + regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), + v_auth_id, + crop_name + FROM (VALUES + ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), + ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), + ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), + ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), + ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), + ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), + ('Sugar Beet'), ('Coffee') + ) AS src(crop_name) + ON CONFLICT (id) DO + -- want case changes or space changes to overwrite existing + -- Only rewrite the row if name changed + UPDATE SET crop_name = EXCLUDED.crop_name + WHERE crop.crop_name IS DISTINCT FROM EXCLUDED.crop_name; END $$; \ No newline at end of file From 938afe0fb480595de07c00386e6e788ad6b6dba7 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 22 May 2025 12:46:39 +0000 Subject: [PATCH 065/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1fecf2ae1..43f157033 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+967 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1f453d58287d0e0bc9eace790be279ed6920c0a5 +version=v1.2.0+971 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/222685405ca1e7fc3e00fa3420524c92f0699573 From fbdd019a7535e5089e2902ab9450f5ee7fcc7729 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 22 May 2025 10:35:05 -0400 Subject: [PATCH 066/289] Get brapi server running for tests --- .../java/org/breedinginsight/BrAPITest.java | 1 + src/test/resources/sql/brapi/species.sql | 44 +++++++++++-------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/test/java/org/breedinginsight/BrAPITest.java b/src/test/java/org/breedinginsight/BrAPITest.java index 8c5c35bad..bdf362979 100644 --- a/src/test/java/org/breedinginsight/BrAPITest.java +++ b/src/test/java/org/breedinginsight/BrAPITest.java @@ -60,6 +60,7 @@ public BrAPITest() { .withEnv("BRAPI_DB", "postgres") .withEnv("BRAPI_DB_USER", "postgres") .withEnv("BRAPI_DB_PASSWORD", "postgres") + .withEnv("SECURITY_ISSUER_URL", "http://example.com/issuerurl") .withClasspathResourceMapping("sql/brapi/mount", "/home/brapi/db/sql", BindMode.READ_WRITE) // HACK - READ_WRITE forces testcontainers to use a bind mount (which overwrites) instead of copying files. .withClasspathResourceMapping("brapi/properties/application.properties", "/home/brapi/properties/application.properties", BindMode.READ_ONLY) .waitingFor(Wait.forLogMessage(".*Started BrapiTestServer in \\d*.\\d* seconds.*", 1).withStartupTimeout(Duration.ofMinutes(1))); diff --git a/src/test/resources/sql/brapi/species.sql b/src/test/resources/sql/brapi/species.sql index 817a1c726..f62e69156 100644 --- a/src/test/resources/sql/brapi/species.sql +++ b/src/test/resources/sql/brapi/species.sql @@ -17,22 +17,28 @@ */ -- name: InsertSpecies -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '4', 'Blueberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '5', 'Salmon') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '6', 'Grape') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '7', 'Alfalfa') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '8', 'Sweet Potato') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '9', 'Trout') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '10', 'Soybean') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '11', 'Cranberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '12', 'Cucumber') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'Oat') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '21', 'Sorghum') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '22', 'Hemp') ON CONFLICT DO NOTHING; +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +DO $$ +DECLARE +v_auth_id CONSTANT uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; +BEGIN +INSERT INTO crop (id, auth_user_id, crop_name) +SELECT + uuid_generate_v5('9a4deca9-4068-46a3-9efe-db0c181f491a'::uuid, + regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), + v_auth_id, + crop_name +FROM (VALUES + ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), + ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), + ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), + ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), + ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), + ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), + ('Sugar Beet'), ('Coffee') + ) AS src(crop_name) + ON CONFLICT (id) DO +UPDATE SET crop_name = EXCLUDED.crop_name +WHERE crop.crop_name IS DISTINCT FROM EXCLUDED.crop_name; +END $$; \ No newline at end of file From 58a6990ed31cc28bb3bdb246cfa8a0ba880b40e8 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 22 May 2025 14:35:28 +0000 Subject: [PATCH 067/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 43f157033..b9d6e1d3f 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+971 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/222685405ca1e7fc3e00fa3420524c92f0699573 +version=v1.2.0+973 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fbdd019a7535e5089e2902ab9450f5ee7fcc7729 From 55a963274e5206eeecfa08cf969ef9d84d511917 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 22 May 2025 16:52:22 -0400 Subject: [PATCH 068/289] Temporarily disable some tests related to brapi server changes --- .../v1/controller/ExperimentControllerIntegrationTest.java | 2 ++ .../controller/SampleSubmissionControllerIntegrationTest.java | 4 ++++ .../brapi/v2/ListControllerIntegrationTest.java | 2 ++ 3 files changed, 8 insertions(+) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 9adb44425..1047ad412 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -748,9 +748,11 @@ public void deleteExperimentInvalid() { * 3. hard delete without obs - success * 4. soft delete without obs - success */ + // TODO: Re-enable after brapi server fixes @ParameterizedTest @CsvSource(value = {"true,true", "false,true", "true,false", "false,false"}) @SneakyThrows + @Disabled public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations) { // Set up a test trial and get the trialDbId. String trialDbId; diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java index ec28c689b..0fec47433 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -223,7 +223,9 @@ public void testManualUpdateSubmissions() throws IOException, InterruptedExcepti assertNull(retrievedSubmission.getVendorStatusLastCheck()); } + // TODO: Re-enable after brapi server fixes @Test + @Disabled public void testSubmitViaBrAPI() throws IOException, InterruptedException { Pair>> uploadedSubmission = createSubmission(program); @@ -245,7 +247,9 @@ public void testSubmitViaBrAPI() throws IOException, InterruptedException { assertNull(retrievedSubmission.getVendorStatusLastCheck()); } + // TODO: Re-enable after brapi server fixes @Test + @Disabled public void testCheckVendorStatus() throws IOException, InterruptedException { Pair>> uploadedSubmission = createSubmission(program); diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index 2d421d306..34b6723cc 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -199,7 +199,9 @@ public void getAllListsSuccess() { } + // TODO: Re-enable after brapi server fixes @Test + @Disabled @SneakyThrows @Order(2) public void deleteListSuccess() { From 1e8bc3e877c79311c5085e7ec226c4e489a4e719 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 22 May 2025 20:53:18 +0000 Subject: [PATCH 069/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index b9d6e1d3f..edcba6847 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+973 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fbdd019a7535e5089e2902ab9450f5ee7fcc7729 +version=v1.2.0+975 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a2d0506f23468593435c2e9a9d85aa62703113ba From 3522f4eb573564971d7dcced8c7daa97eb0e106a Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 28 May 2025 10:44:50 -0400 Subject: [PATCH 070/289] Change dbid to uuid to work with new brapi server changes --- .../brapi/v2/ListControllerIntegrationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index 34b6723cc..17b720d76 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -201,7 +201,7 @@ public void getAllListsSuccess() { // TODO: Re-enable after brapi server fixes @Test - @Disabled + //@Disabled @SneakyThrows @Order(2) public void deleteListSuccess() { @@ -233,7 +233,7 @@ public void deleteListSuccess() { // A DELETE request to the brapi/v2/lists/ endpoint with invalid dbId. Flowable> invalidDeleteCall = client.exchange( - DELETE(String.format("/programs/%s/brapi/v2/lists/%s", program.getId().toString(), "NOT-VALID-DBID")) + DELETE(String.format("/programs/%s/brapi/v2/lists/%s", program.getId().toString(), "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class ); From 61c036e092ec6826d6cb3a97cd6c7c4c71e6c21f Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 28 May 2025 10:59:01 -0400 Subject: [PATCH 071/289] Re-enable list delete test --- .../brapi/v2/ListControllerIntegrationTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index 17b720d76..dd0fbe69c 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -198,10 +198,7 @@ public void getAllListsSuccess() { } } - - // TODO: Re-enable after brapi server fixes @Test - //@Disabled @SneakyThrows @Order(2) public void deleteListSuccess() { From e5e5825d006cdad477735ee4ae5fd7c91caea283 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 28 May 2025 14:56:59 -0400 Subject: [PATCH 072/289] pin to 17.5 --- docker-compose.yml | 2 +- src/test/java/org/breedinginsight/DatabaseTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2932c30fe..efebf5fb6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,7 @@ services: networks: - backend bidb: - image: postgres:17 + image: postgres:17.5 container_name: bidb environment: - POSTGRES_DB=${DB_NAME} diff --git a/src/test/java/org/breedinginsight/DatabaseTest.java b/src/test/java/org/breedinginsight/DatabaseTest.java index ddf3043f8..5a1279e7c 100644 --- a/src/test/java/org/breedinginsight/DatabaseTest.java +++ b/src/test/java/org/breedinginsight/DatabaseTest.java @@ -62,7 +62,7 @@ public DatabaseTest() { network = Network.newNetwork(); } if(dbContainer == null) { - dbContainer = new GenericContainer<>("postgres:17") + dbContainer = new GenericContainer<>("postgres:17.5") .withNetwork(network) .withNetworkAliases("testdb") .withImagePullPolicy(PullPolicy.defaultPolicy()) From 18ee565c636cde94e1e699794380bc36cf949650 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 28 May 2025 18:57:44 +0000 Subject: [PATCH 073/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index edcba6847..4ad06bd37 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+975 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a2d0506f23468593435c2e9a9d85aa62703113ba +version=v1.2.0+977 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/9d584e4d8f3c381b2325ffff327a380b2a477f99 From 436b89f70fe7de41ce43bda64baae946db96c6ce Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 28 May 2025 16:05:03 -0400 Subject: [PATCH 074/289] Re-enable delete experiment test --- .../api/v1/controller/ExperimentControllerIntegrationTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 1047ad412..9adb44425 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -748,11 +748,9 @@ public void deleteExperimentInvalid() { * 3. hard delete without obs - success * 4. soft delete without obs - success */ - // TODO: Re-enable after brapi server fixes @ParameterizedTest @CsvSource(value = {"true,true", "false,true", "true,false", "false,false"}) @SneakyThrows - @Disabled public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations) { // Set up a test trial and get the trialDbId. String trialDbId; From 2ac3988f16b3772beaa1dbcdac70205567749aa7 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 29 May 2025 11:03:01 -0400 Subject: [PATCH 075/289] Re-enable sample submission tests --- .../controller/SampleSubmissionControllerIntegrationTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java index 0fec47433..ec28c689b 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -223,9 +223,7 @@ public void testManualUpdateSubmissions() throws IOException, InterruptedExcepti assertNull(retrievedSubmission.getVendorStatusLastCheck()); } - // TODO: Re-enable after brapi server fixes @Test - @Disabled public void testSubmitViaBrAPI() throws IOException, InterruptedException { Pair>> uploadedSubmission = createSubmission(program); @@ -247,9 +245,7 @@ public void testSubmitViaBrAPI() throws IOException, InterruptedException { assertNull(retrievedSubmission.getVendorStatusLastCheck()); } - // TODO: Re-enable after brapi server fixes @Test - @Disabled public void testCheckVendorStatus() throws IOException, InterruptedException { Pair>> uploadedSubmission = createSubmission(program); From f487faad198a23d5f496e141a3a3fd893fe9050a Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 29 May 2025 18:13:41 -0400 Subject: [PATCH 076/289] [BI-2654] - replaced reserved keyword with allowed value --- src/test/resources/sql/UserControllerIntegrationTest.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/resources/sql/UserControllerIntegrationTest.sql b/src/test/resources/sql/UserControllerIntegrationTest.sql index 55cafed8d..5dce6cca0 100644 --- a/src/test/resources/sql/UserControllerIntegrationTest.sql +++ b/src/test/resources/sql/UserControllerIntegrationTest.sql @@ -27,19 +27,19 @@ join bi_user on bi_user.name = 'system' limit 1; -- name: InsertUserProgramAssociations insert into program_user_role (program_id, user_id, role_id, created_by, updated_by) -select program.id, bi_user.id, role.id, system_user.id, system_user.id +select program.id, bi_user.id, role.id, bi_system_user.id, bi_system_user.id from program join bi_user on bi_user.name = 'Test User' or bi_user.name = 'Other Test User' join role on role.domain = 'Read Only' -join bi_user as system_user on system_user.name = 'system' +join bi_user as bi_system_user on bi_system_user.name = 'system' where program.name = 'Test Program'; insert into program_user_role (program_id, user_id, role_id, active, created_by, updated_by) -select program.id, bi_user.id, role.id, false, system_user.id, system_user.id +select program.id, bi_user.id, role.id, false, bi_system_user.id, bi_system_user.id from program join bi_user on bi_user.name = 'Test User' or bi_user.name = 'Other Test User' join role on role.domain = 'Read Only' -join bi_user as system_user on system_user.name = 'system' +join bi_user as bi_system_user on bi_system_user.name = 'system' where program.name = 'Test Program1'; -- name: DeactivateProgram From 695855b0b054f3fe13ead3550a45356bc2928d38 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 30 May 2025 14:40:16 +0000 Subject: [PATCH 077/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 4ad06bd37..c86934fa2 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+977 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/9d584e4d8f3c381b2325ffff327a380b2a477f99 +version=v1.2.0+981 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d8f5488486d96997c3c77b5f50365b9151b1fd72 From 6b146e120bce297407131d5dde30c9e82db84741 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 30 May 2025 18:41:49 +0000 Subject: [PATCH 078/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index c86934fa2..1fd09f4d5 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+981 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d8f5488486d96997c3c77b5f50365b9151b1fd72 +version=v1.2.0+983 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6a7275fe51dff62877e91eedacfc959e3e3e5096 From bee5117f0797c39619958d4a5f4079d7dab7f80a Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 9 Jun 2025 12:34:56 -0400 Subject: [PATCH 079/289] filter out obsunit comns from dynamic columns --- .../process/ImportTableProcess.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index e06d773ab..e655f2156 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -67,9 +67,8 @@ import java.util.*; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.*; import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_REGEX; @Slf4j @Prototype @@ -110,19 +109,21 @@ public ImportTableProcess(StudyService studyService, public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { log.debug("verifying traits listed in import"); - // Get all the dynamic columns of the import + // Get all the phenotypic columns of the import ImportUpload upload = context.getImportContext().getUpload(); Table data = context.getImportContext().getData(); - String[] dynamicColNames = upload.getDynamicColumnNames(); - - // don't allow periods (.) or square brackets in Dynamic Column Names - for (String dynamicColumnName: dynamicColNames) { - if(dynamicColumnName.contains(".") || dynamicColumnName.contains("[") || dynamicColumnName.contains("]")){ - String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", dynamicColumnName); + List phenotypeColNames = Arrays.stream(upload.getDynamicColumnNames()) + .filter(name -> !name.endsWith(OBSERVATION_UNIT_ID_SUFFIX)) + .collect(Collectors.toList()); + + // don't allow periods (.) or square brackets in Phenotype Column Names + for (String phenotypeColumnName: phenotypeColNames) { + if(phenotypeColumnName.contains(".") || phenotypeColumnName.contains("[") || phenotypeColumnName.contains("]")){ + String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", phenotypeColumnName); throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, errorMsg); } } - List> dynamicCols = data.columns(dynamicColNames); + List> dynamicCols = data.columns(phenotypeColNames.toArray(new String[0])); // Collect the columns for observation variable data List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); From 28a0fc3e16811043b4d5a4c0d6a9efbf952e080f Mon Sep 17 00:00:00 2001 From: HMS17 Date: Tue, 20 May 2025 11:57:05 -0400 Subject: [PATCH 080/289] [BI-2631] - Experiment Export: Turn ObsUnitID into a Dynamic Column --- .../brapi/v2/services/BrAPITrialService.java | 28 +++---- .../experiment/ExperimentFileColumns.java | 5 +- .../ExperimentControllerIntegrationTest.java | 2 +- ...leSubmissionControllerIntegrationTest.java | 2 +- .../v2/ListControllerIntegrationTest.java | 2 +- .../importer/ExperimentFileImportTest.java | 80 +++++++++---------- .../brapps/importer/ImportTestUtils.java | 7 +- .../SampleSubmissionFileImportTest.java | 2 +- 8 files changed, 66 insertions(+), 62 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index a383a96a0..6b7a01fec 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -55,6 +55,8 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; + @Slf4j @Singleton public class BrAPITrialService { @@ -185,6 +187,11 @@ public DownloadFile exportObservations( log.error(logHash + ": Error fetching observation units for a study by its DbId" + Utilities.generateApiExceptionLogMessage(err), err); } + + //add obsUnitID as dynamic column with observation level appended to header + String observationLvl = ous.get(0).getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); + columns = dynamicUpdateObsUnitIDLabel(columns, observationLvl); + if (params.getDatasetId() != null) { log.debug(logHash + ": fetching " + params.getDatasetId() + " dataset observation variables for export"); obsVars = getDatasetObsVars(params.getDatasetId(), program); @@ -228,16 +235,12 @@ public DownloadFile exportObservations( } } - //dynamically append observation level to obsUnitID column header - String observationLvl = ous.get(0).getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); - columns = dynamicUpdateObsUnitIDLabel(columns, observationLvl); - log.debug(logHash + ": writing data to file for export"); // If one or more envs requested, create a separate file for each env, then zip if there are multiple. if (!requestedEnvIds.isEmpty()) { // This will hold a list of rows for each study, each list will become a separate file. Map>> rowsByStudyId = new HashMap<>(); - String obsUnitIDLabel = observationLvl + " " + ExperimentObservation.Columns.OBS_UNIT_ID; + String obsUnitIDLabel = observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX; for (Map row: rowByOUId.values()) { String studyId = studyDbIdByOUId.get((String)row.get(obsUnitIDLabel)); @@ -312,15 +315,10 @@ private StreamedFile zipFiles(List files) throws IOException { } public List dynamicUpdateObsUnitIDLabel(List columns, String observationLvl){ - Column oldObsUnitIDCol = new Column(ExperimentObservation.Columns.OBS_UNIT_ID, Column.ColumnDataType.STRING); - String dynamicLabel = observationLvl + " " + ExperimentObservation.Columns.OBS_UNIT_ID; - Column dynamicLabelObsUnitIDCol = new Column(dynamicLabel, Column.ColumnDataType.STRING); - //need to check index of is valid - int index = columns.indexOf(oldObsUnitIDCol); - //find item in cols with val ExperimentObservation.Columns.OBS_UNIT_ID - if (index != -1) { - columns.set(index, dynamicLabelObsUnitIDCol); - } + String dynamicLabel = observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX; + Column ObsUnitIDCol = new Column(dynamicLabel, Column.ColumnDataType.STRING); + columns.add(ObsUnitIDCol); + return columns; } @@ -806,7 +804,7 @@ private Map createExportRow( //Append observation level to obsUnitID String observationLvl = ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); - row.put(observationLvl + " " + ExperimentObservation.Columns.OBS_UNIT_ID, ouId); + row.put(observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX, ouId); return row; } diff --git a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java index 3eedbaf11..39f7b22c0 100644 --- a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java +++ b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; + public enum ExperimentFileColumns { GERMPLASM_NAME(ExperimentObservation.Columns.GERMPLASM_NAME, Column.ColumnDataType.STRING), @@ -47,8 +49,7 @@ public enum ExperimentFileColumns { LONG(ExperimentObservation.Columns.LONG, Column.ColumnDataType.STRING), ELEVATION(ExperimentObservation.Columns.ELEVATION, Column.ColumnDataType.STRING), RTK(ExperimentObservation.Columns.RTK, Column.ColumnDataType.STRING), - TREATMENT_FACTORS(ExperimentObservation.Columns.TREATMENT_FACTORS, Column.ColumnDataType.STRING), - OBS_UNIT_ID(ExperimentObservation.Columns.OBS_UNIT_ID, Column.ColumnDataType.STRING); + TREATMENT_FACTORS(ExperimentObservation.Columns.TREATMENT_FACTORS, Column.ColumnDataType.STRING); private final Column column; diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 9adb44425..fd67cefab 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -265,7 +265,7 @@ private String uploadExperimentWithoutObs() throws Exception { @SneakyThrows void downloadDatasets(boolean includeTimestamps, String extension, int numberOfEnvsRequested) { // How many columns are expected in the output? - int expectedColNumber = columns.size(); + int expectedColNumber = columns.size() + 1; //Need to account for ObsUnitID column which is present in export but not import if (includeTimestamps) { expectedColNumber += traits.size(); } diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java index ec28c689b..978bb0aa8 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -407,7 +407,7 @@ private String createExperiment(Program program) throws IOException, Interrupted .get(0).getAsJsonObject().get("id").getAsString(); JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( - importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null), + importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null, false), null, true, client, diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index 2d421d306..92fdb6fd7 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -148,7 +148,7 @@ public void setup() { newExp.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetchWorkflow( - importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); } @Test diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index edfee7583..8275e1cea 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -202,7 +202,7 @@ public void importNewExpNewLocNoObsSuccess() { validRow.put(Columns.COLUMN, "1"); validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(validRow), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -254,7 +254,7 @@ public void importNewExpMultiNewEnvSuccess() { secondEnv.put(Columns.COLUMN, "1"); secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(2, previewRows.size()); @@ -295,7 +295,7 @@ public void importExistingExpAndEnvErrorMessage() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); Map dupExp = new HashMap<>(); dupExp.put(Columns.GERMPLASM_GID, "1"); @@ -312,7 +312,7 @@ public void importExistingExpAndEnvErrorMessage() { dupExp.put(Columns.ROW, "1"); dupExp.put(Columns.COLUMN, "1"); - JsonObject expResult = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject expResult = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); assertEquals(422, expResult.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + expResult); assertTrue(expResult.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); @@ -339,7 +339,7 @@ public void importNewEnvNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); - JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -375,52 +375,52 @@ public void verifyMissingDataThrowsError(boolean commit) { Map noGID = new HashMap<>(base); noGID.remove(Columns.GERMPLASM_GID); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null, false), Columns.GERMPLASM_GID, commit, newExperimentWorkflowId); Map noExpTitle = new HashMap<>(base); noExpTitle.remove(Columns.EXP_TITLE); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null, false), Columns.EXP_TITLE, commit, newExperimentWorkflowId); Map noExpUnit = new HashMap<>(base); noExpUnit.remove(Columns.EXP_UNIT); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null, false), Columns.EXP_UNIT, commit, newExperimentWorkflowId); Map noExpType = new HashMap<>(base); noExpType.remove(Columns.EXP_TYPE); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null, false), Columns.EXP_TYPE, commit, newExperimentWorkflowId); Map noEnv = new HashMap<>(base); noEnv.remove(Columns.ENV); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null, false), Columns.ENV, commit, newExperimentWorkflowId); Map noEnvLoc = new HashMap<>(base); noEnvLoc.remove(Columns.ENV_LOCATION); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null, false), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); Map noExpUnitId = new HashMap<>(base); noExpUnitId.remove(Columns.EXP_UNIT_ID); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null, false), Columns.EXP_UNIT_ID, commit, newExperimentWorkflowId); Map noExpRep = new HashMap<>(base); noExpRep.remove(Columns.REP_NUM); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null, false), Columns.REP_NUM, commit, newExperimentWorkflowId); Map noExpBlock = new HashMap<>(base); noExpBlock.remove(Columns.BLOCK_NUM); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null, false), Columns.BLOCK_NUM, commit, newExperimentWorkflowId); Map noEnvYear = new HashMap<>(base); noEnvYear.remove(Columns.ENV_YEAR); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null, false), Columns.ENV_YEAR, commit, newExperimentWorkflowId); } @Test @@ -445,7 +445,7 @@ public void importNewExpWithObsVar() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -497,7 +497,7 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null, false), Columns.ENV_YEAR, commit, newExperimentWorkflowId); } @@ -537,7 +537,7 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null, false), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); } @ParameterizedTest @@ -563,7 +563,7 @@ public void importNewExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -604,7 +604,7 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "Red"); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); } @@ -629,14 +629,14 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); Map newOU = new HashMap<>(newExp); newOU.put(Columns.EXP_UNIT_ID, "a-2"); newOU.put(Columns.ROW, "1"); newOU.put(Columns.COLUMN, "2"); - JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(newOU), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); @@ -664,7 +664,7 @@ public void importNewObsVarExistingOu() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -692,7 +692,7 @@ public void importNewObsVarExistingOu() { newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true), null, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -730,7 +730,7 @@ public void importNewObsVarByObsUnitId() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -745,7 +745,7 @@ public void importNewObsVarByObsUnitId() { newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true), null, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -784,7 +784,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); // empty dataset - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -812,7 +812,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -860,7 +860,7 @@ public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); // Valid observation value. - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); // Fetch the ObsUnitId to use in the overwrite upload. BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); @@ -894,7 +894,7 @@ public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { requestBody.put("overwrite", "true"); requestBody.put("overwriteReason", "testing"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), requestBody, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true), requestBody, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -930,7 +930,7 @@ public void importNewObsExistingOu(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -958,7 +958,7 @@ public void importNewObsExistingOu(boolean commit) { newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -998,7 +998,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1026,7 +1026,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); - uploadAndVerifyWorkflowFailureNonTabular(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailureNonTabular(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); } /* @@ -1057,7 +1057,7 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsA.size()); @@ -1085,7 +1085,7 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsB.size()); @@ -1127,7 +1127,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1156,7 +1156,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1204,7 +1204,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newExp.put(traits.get(0).getObservationVariableName(), originalValue); newExp.put(traits.get(1).getObservationVariableName(), "2"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1240,7 +1240,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { Map requestBody = new HashMap<>(); requestBody.put("overwrite", "true"); requestBody.put("overwriteReason", "testing"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), requestBody, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true), requestBody, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index d246394b9..eb4030150 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -52,6 +52,7 @@ import static io.micronaut.http.HttpRequest.*; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; /** * Intended to be a utility class, but methods being static was causing issues. @@ -296,7 +297,7 @@ public List createTraits(int numToCreate) { return traits; } - public File writeExperimentDataToFile(List> data, List traits) throws IOException { + public File writeExperimentDataToFile(List> data, List traits, boolean ObsUnitIDCol) throws IOException { File file = File.createTempFile("test", ".csv"); List columns = new ArrayList<>(); @@ -323,6 +324,10 @@ public File writeExperimentDataToFile(List> data, List { columns.add(Column.builder().value(trait.getObservationVariableName()).dataType(Column.ColumnDataType.STRING).build()); diff --git a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java index 71bf6197c..fb5f93ea3 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java @@ -568,7 +568,7 @@ private String createExperiment(Program program) throws IOException, Interrupted .get(0).getAsJsonObject().get("id").getAsString(); JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( - importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null), + importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null, false), null, true, client, From ead3784f618f5d2ab39455d249c8a7d7110d4a20 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 6 May 2025 11:38:46 -0400 Subject: [PATCH 081/289] Update species migration to work with new uuids --- src/main/resources/brapi/sql/R__species.sql | 65 ++++++++++++--------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/src/main/resources/brapi/sql/R__species.sql b/src/main/resources/brapi/sql/R__species.sql index 00c1b7042..843ed32ce 100644 --- a/src/main/resources/brapi/sql/R__species.sql +++ b/src/main/resources/brapi/sql/R__species.sql @@ -13,32 +13,39 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '4', 'Blueberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '5', 'Salmon') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '6', 'Grape') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '7', 'Alfalfa') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '8', 'Sweet Potato') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '9', 'Trout') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '10', 'Soybean') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '11', 'Cranberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '12', 'Cucumber') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'Oat') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING; --- for the Honey Bee case, want to overwrite name, not preserve existing -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey Bee') ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '21', 'Sorghum') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '22', 'Hemp') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '23', 'Hop') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '24', 'Hydrangea') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '25', 'Red Clover') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '26', 'Potato') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '27', 'Blackberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '28', 'Raspberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '29', 'Sugar Beet') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '30', 'Strawberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '31', 'Coffee') ON CONFLICT DO NOTHING; \ No newline at end of file +-- for uuid_generate_v4() +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +DO $$ +DECLARE + v_auth_id constant uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; +BEGIN + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Blueberry') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Salmon') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Grape') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Alfalfa') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sweet Potato') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Trout') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Soybean') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Cranberry') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Cucumber') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Oat') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Citrus') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sugar Cane') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Strawberry') ON CONFLICT DO NOTHING; + -- for the Honey Bee case, want to overwrite name, not preserve existing + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Honey Bee') ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Pecan') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Lettuce') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Cotton') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sorghum') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Hemp') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Hop') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Hydrangea') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Red Clover') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Potato') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Blackberry') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Raspberry') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sugar Beet') ON CONFLICT DO NOTHING; + INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Coffee') ON CONFLICT DO NOTHING; +END $$; \ No newline at end of file From 1e05a5630d2c72ef955e68da38a9beeba893f09a Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 21 May 2025 10:48:15 -0400 Subject: [PATCH 082/289] Update species migration to better handle repeat runs --- src/main/resources/brapi/sql/R__species.sql | 74 ++++++++++++--------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/src/main/resources/brapi/sql/R__species.sql b/src/main/resources/brapi/sql/R__species.sql index 843ed32ce..993895398 100644 --- a/src/main/resources/brapi/sql/R__species.sql +++ b/src/main/resources/brapi/sql/R__species.sql @@ -13,39 +13,53 @@ -- See the License for the specific language governing permissions and -- limitations under the License. --- for uuid_generate_v4() +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; DO $$ DECLARE - v_auth_id constant uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; + v_auth_id CONSTANT uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; BEGIN - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Blueberry') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Salmon') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Grape') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Alfalfa') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sweet Potato') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Trout') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Soybean') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Cranberry') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Cucumber') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Oat') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Citrus') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sugar Cane') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Strawberry') ON CONFLICT DO NOTHING; - -- for the Honey Bee case, want to overwrite name, not preserve existing - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Honey Bee') ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Pecan') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Lettuce') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Cotton') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sorghum') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Hemp') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Hop') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Hydrangea') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Red Clover') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Potato') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Blackberry') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Raspberry') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Sugar Beet') ON CONFLICT DO NOTHING; - INSERT INTO crop (id, auth_user_id, crop_name) VALUES (uuid_generate_v4(), v_auth_id, 'Coffee') ON CONFLICT DO NOTHING; + /* ------------------------------------------------------------------------------------------ + • uuid_generate_v5(namespace, crop_name) → deterministic UUID can be used for idempotency + • Do it this way so no schema changes are required + • Removed the Honey Bee special case because all systems will be starting fresh + ------------------------------------------------------------------------------------------ */ + INSERT INTO crop (id, auth_user_id, crop_name) + SELECT + uuid_generate_v5('9a4deca9-4068-46a3-9efe-db0c181f491a'::uuid, + -- 1) lower‑case + -- 2) trim leading/trailing space + -- 3) REMOVE every space or tab + regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), + v_auth_id, + crop_name + FROM (VALUES + ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), + ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), + ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), + ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), + ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), + ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), + ('Sugar Beet'), ('Coffee') + ) AS src(crop_name) + ON CONFLICT (id) DO + -- want case changes or space changes to overwrite existing + -- Only rewrite the row if name changed + UPDATE SET crop_name = EXCLUDED.crop_name + WHERE crop.crop_name IS DISTINCT FROM EXCLUDED.crop_name; END $$; \ No newline at end of file From 94976bdb0cb2c666acbc6c21c95e14bd59d16c49 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 19 May 2025 19:00:48 +0000 Subject: [PATCH 083/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index ed12a56b1..1fecf2ae1 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+963 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/faa5e691f8b526a520e15c3f374520cc9d29b2d2 +version=v1.2.0+967 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1f453d58287d0e0bc9eace790be279ed6920c0a5 From 3b0faaf123f9055e8cc2f118091dcff900987993 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 22 May 2025 12:46:39 +0000 Subject: [PATCH 084/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1fecf2ae1..43f157033 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+967 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1f453d58287d0e0bc9eace790be279ed6920c0a5 +version=v1.2.0+971 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/222685405ca1e7fc3e00fa3420524c92f0699573 From 4ca398f2c4ac07a2be5011c77e28980072a444bb Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 22 May 2025 10:35:05 -0400 Subject: [PATCH 085/289] Get brapi server running for tests --- .../java/org/breedinginsight/BrAPITest.java | 1 + src/test/resources/sql/brapi/species.sql | 44 +++++++++++-------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/test/java/org/breedinginsight/BrAPITest.java b/src/test/java/org/breedinginsight/BrAPITest.java index 8c5c35bad..bdf362979 100644 --- a/src/test/java/org/breedinginsight/BrAPITest.java +++ b/src/test/java/org/breedinginsight/BrAPITest.java @@ -60,6 +60,7 @@ public BrAPITest() { .withEnv("BRAPI_DB", "postgres") .withEnv("BRAPI_DB_USER", "postgres") .withEnv("BRAPI_DB_PASSWORD", "postgres") + .withEnv("SECURITY_ISSUER_URL", "http://example.com/issuerurl") .withClasspathResourceMapping("sql/brapi/mount", "/home/brapi/db/sql", BindMode.READ_WRITE) // HACK - READ_WRITE forces testcontainers to use a bind mount (which overwrites) instead of copying files. .withClasspathResourceMapping("brapi/properties/application.properties", "/home/brapi/properties/application.properties", BindMode.READ_ONLY) .waitingFor(Wait.forLogMessage(".*Started BrapiTestServer in \\d*.\\d* seconds.*", 1).withStartupTimeout(Duration.ofMinutes(1))); diff --git a/src/test/resources/sql/brapi/species.sql b/src/test/resources/sql/brapi/species.sql index 817a1c726..f62e69156 100644 --- a/src/test/resources/sql/brapi/species.sql +++ b/src/test/resources/sql/brapi/species.sql @@ -17,22 +17,28 @@ */ -- name: InsertSpecies -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '4', 'Blueberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '5', 'Salmon') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '6', 'Grape') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '7', 'Alfalfa') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '8', 'Sweet Potato') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '9', 'Trout') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '10', 'Soybean') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '11', 'Cranberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '12', 'Cucumber') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'Oat') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '21', 'Sorghum') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '22', 'Hemp') ON CONFLICT DO NOTHING; +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +DO $$ +DECLARE +v_auth_id CONSTANT uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; +BEGIN +INSERT INTO crop (id, auth_user_id, crop_name) +SELECT + uuid_generate_v5('9a4deca9-4068-46a3-9efe-db0c181f491a'::uuid, + regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), + v_auth_id, + crop_name +FROM (VALUES + ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), + ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), + ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), + ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), + ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), + ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), + ('Sugar Beet'), ('Coffee') + ) AS src(crop_name) + ON CONFLICT (id) DO +UPDATE SET crop_name = EXCLUDED.crop_name +WHERE crop.crop_name IS DISTINCT FROM EXCLUDED.crop_name; +END $$; \ No newline at end of file From 44567fb365fd38c0ddfc14c2e79bcb899bb24691 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 22 May 2025 16:52:22 -0400 Subject: [PATCH 086/289] Temporarily disable some tests related to brapi server changes --- .../v1/controller/ExperimentControllerIntegrationTest.java | 2 ++ .../controller/SampleSubmissionControllerIntegrationTest.java | 4 ++++ .../brapi/v2/ListControllerIntegrationTest.java | 2 ++ 3 files changed, 8 insertions(+) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index fd67cefab..1d3d4e90d 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -748,9 +748,11 @@ public void deleteExperimentInvalid() { * 3. hard delete without obs - success * 4. soft delete without obs - success */ + // TODO: Re-enable after brapi server fixes @ParameterizedTest @CsvSource(value = {"true,true", "false,true", "true,false", "false,false"}) @SneakyThrows + @Disabled public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations) { // Set up a test trial and get the trialDbId. String trialDbId; diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java index 978bb0aa8..20d21f75c 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -223,7 +223,9 @@ public void testManualUpdateSubmissions() throws IOException, InterruptedExcepti assertNull(retrievedSubmission.getVendorStatusLastCheck()); } + // TODO: Re-enable after brapi server fixes @Test + @Disabled public void testSubmitViaBrAPI() throws IOException, InterruptedException { Pair>> uploadedSubmission = createSubmission(program); @@ -245,7 +247,9 @@ public void testSubmitViaBrAPI() throws IOException, InterruptedException { assertNull(retrievedSubmission.getVendorStatusLastCheck()); } + // TODO: Re-enable after brapi server fixes @Test + @Disabled public void testCheckVendorStatus() throws IOException, InterruptedException { Pair>> uploadedSubmission = createSubmission(program); diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index 92fdb6fd7..5118d8d72 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -199,7 +199,9 @@ public void getAllListsSuccess() { } + // TODO: Re-enable after brapi server fixes @Test + @Disabled @SneakyThrows @Order(2) public void deleteListSuccess() { From 345920ebb8e11666b7837779267ee3d0e9819f10 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 22 May 2025 14:35:28 +0000 Subject: [PATCH 087/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 43f157033..b9d6e1d3f 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+971 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/222685405ca1e7fc3e00fa3420524c92f0699573 +version=v1.2.0+973 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fbdd019a7535e5089e2902ab9450f5ee7fcc7729 From 280161f6e8bd18e45efe038ee766028e5f7f21b9 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 28 May 2025 10:44:50 -0400 Subject: [PATCH 088/289] Change dbid to uuid to work with new brapi server changes --- .../brapi/v2/ListControllerIntegrationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index 5118d8d72..eb3f195b7 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -201,7 +201,7 @@ public void getAllListsSuccess() { // TODO: Re-enable after brapi server fixes @Test - @Disabled + //@Disabled @SneakyThrows @Order(2) public void deleteListSuccess() { @@ -233,7 +233,7 @@ public void deleteListSuccess() { // A DELETE request to the brapi/v2/lists/ endpoint with invalid dbId. Flowable> invalidDeleteCall = client.exchange( - DELETE(String.format("/programs/%s/brapi/v2/lists/%s", program.getId().toString(), "NOT-VALID-DBID")) + DELETE(String.format("/programs/%s/brapi/v2/lists/%s", program.getId().toString(), "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class ); From f58927ce0e2d823de910f56b4dc4915cd537851c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 28 May 2025 10:59:01 -0400 Subject: [PATCH 089/289] Re-enable list delete test --- .../brapi/v2/ListControllerIntegrationTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index eb3f195b7..ca6e5c2e9 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -198,10 +198,7 @@ public void getAllListsSuccess() { } } - - // TODO: Re-enable after brapi server fixes @Test - //@Disabled @SneakyThrows @Order(2) public void deleteListSuccess() { From 8b94baf08ce40892233353036e9250b33e41fe88 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 28 May 2025 16:05:03 -0400 Subject: [PATCH 090/289] Re-enable delete experiment test --- .../api/v1/controller/ExperimentControllerIntegrationTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 1d3d4e90d..fd67cefab 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -748,11 +748,9 @@ public void deleteExperimentInvalid() { * 3. hard delete without obs - success * 4. soft delete without obs - success */ - // TODO: Re-enable after brapi server fixes @ParameterizedTest @CsvSource(value = {"true,true", "false,true", "true,false", "false,false"}) @SneakyThrows - @Disabled public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations) { // Set up a test trial and get the trialDbId. String trialDbId; From 8787e7be4b4a57b45ada3d6cc16ea3f0be860a11 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 29 May 2025 11:03:01 -0400 Subject: [PATCH 091/289] Re-enable sample submission tests --- .../controller/SampleSubmissionControllerIntegrationTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java index 20d21f75c..978bb0aa8 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -223,9 +223,7 @@ public void testManualUpdateSubmissions() throws IOException, InterruptedExcepti assertNull(retrievedSubmission.getVendorStatusLastCheck()); } - // TODO: Re-enable after brapi server fixes @Test - @Disabled public void testSubmitViaBrAPI() throws IOException, InterruptedException { Pair>> uploadedSubmission = createSubmission(program); @@ -247,9 +245,7 @@ public void testSubmitViaBrAPI() throws IOException, InterruptedException { assertNull(retrievedSubmission.getVendorStatusLastCheck()); } - // TODO: Re-enable after brapi server fixes @Test - @Disabled public void testCheckVendorStatus() throws IOException, InterruptedException { Pair>> uploadedSubmission = createSubmission(program); From 0a56f5182987e8f84e4d11c84af3f66bfab05ddb Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 9 Apr 2025 15:25:38 -0400 Subject: [PATCH 092/289] [BI-2212] - updated postgres version --- docker-compose.yml | 2 +- src/test/java/org/breedinginsight/DatabaseTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index bb30e858e..2932c30fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,7 @@ services: networks: - backend bidb: - image: postgres:11.4 + image: postgres:17 container_name: bidb environment: - POSTGRES_DB=${DB_NAME} diff --git a/src/test/java/org/breedinginsight/DatabaseTest.java b/src/test/java/org/breedinginsight/DatabaseTest.java index b1213ee33..ddf3043f8 100644 --- a/src/test/java/org/breedinginsight/DatabaseTest.java +++ b/src/test/java/org/breedinginsight/DatabaseTest.java @@ -62,7 +62,7 @@ public DatabaseTest() { network = Network.newNetwork(); } if(dbContainer == null) { - dbContainer = new GenericContainer<>("postgres:11.4") + dbContainer = new GenericContainer<>("postgres:17") .withNetwork(network) .withNetworkAliases("testdb") .withImagePullPolicy(PullPolicy.defaultPolicy()) From 3c65bd948ed0ae7962bc10138f8b6ded53d7bec1 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 22 May 2025 20:53:18 +0000 Subject: [PATCH 093/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index b9d6e1d3f..edcba6847 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+973 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fbdd019a7535e5089e2902ab9450f5ee7fcc7729 +version=v1.2.0+975 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a2d0506f23468593435c2e9a9d85aa62703113ba From 2bce10ab1fc4f20a66d57384c3613e330e537eae Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 28 May 2025 14:56:59 -0400 Subject: [PATCH 094/289] pin to 17.5 --- docker-compose.yml | 2 +- src/test/java/org/breedinginsight/DatabaseTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2932c30fe..efebf5fb6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,7 @@ services: networks: - backend bidb: - image: postgres:17 + image: postgres:17.5 container_name: bidb environment: - POSTGRES_DB=${DB_NAME} diff --git a/src/test/java/org/breedinginsight/DatabaseTest.java b/src/test/java/org/breedinginsight/DatabaseTest.java index ddf3043f8..5a1279e7c 100644 --- a/src/test/java/org/breedinginsight/DatabaseTest.java +++ b/src/test/java/org/breedinginsight/DatabaseTest.java @@ -62,7 +62,7 @@ public DatabaseTest() { network = Network.newNetwork(); } if(dbContainer == null) { - dbContainer = new GenericContainer<>("postgres:17") + dbContainer = new GenericContainer<>("postgres:17.5") .withNetwork(network) .withNetworkAliases("testdb") .withImagePullPolicy(PullPolicy.defaultPolicy()) From bbcead178da8bbce930a90da09a685a75b3741b6 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 28 May 2025 18:57:44 +0000 Subject: [PATCH 095/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index edcba6847..4ad06bd37 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+975 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a2d0506f23468593435c2e9a9d85aa62703113ba +version=v1.2.0+977 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/9d584e4d8f3c381b2325ffff327a380b2a477f99 From 4fb4b49f2214f813b133df01985cbf3fba42b304 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 29 May 2025 18:13:41 -0400 Subject: [PATCH 096/289] [BI-2654] - replaced reserved keyword with allowed value --- src/test/resources/sql/UserControllerIntegrationTest.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/resources/sql/UserControllerIntegrationTest.sql b/src/test/resources/sql/UserControllerIntegrationTest.sql index 55cafed8d..5dce6cca0 100644 --- a/src/test/resources/sql/UserControllerIntegrationTest.sql +++ b/src/test/resources/sql/UserControllerIntegrationTest.sql @@ -27,19 +27,19 @@ join bi_user on bi_user.name = 'system' limit 1; -- name: InsertUserProgramAssociations insert into program_user_role (program_id, user_id, role_id, created_by, updated_by) -select program.id, bi_user.id, role.id, system_user.id, system_user.id +select program.id, bi_user.id, role.id, bi_system_user.id, bi_system_user.id from program join bi_user on bi_user.name = 'Test User' or bi_user.name = 'Other Test User' join role on role.domain = 'Read Only' -join bi_user as system_user on system_user.name = 'system' +join bi_user as bi_system_user on bi_system_user.name = 'system' where program.name = 'Test Program'; insert into program_user_role (program_id, user_id, role_id, active, created_by, updated_by) -select program.id, bi_user.id, role.id, false, system_user.id, system_user.id +select program.id, bi_user.id, role.id, false, bi_system_user.id, bi_system_user.id from program join bi_user on bi_user.name = 'Test User' or bi_user.name = 'Other Test User' join role on role.domain = 'Read Only' -join bi_user as system_user on system_user.name = 'system' +join bi_user as bi_system_user on bi_system_user.name = 'system' where program.name = 'Test Program1'; -- name: DeactivateProgram From 38e815b4f074894780cb5a146ac7aac1f5de1d7e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 30 May 2025 14:40:16 +0000 Subject: [PATCH 097/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 4ad06bd37..c86934fa2 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+977 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/9d584e4d8f3c381b2325ffff327a380b2a477f99 +version=v1.2.0+981 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d8f5488486d96997c3c77b5f50365b9151b1fd72 From 541a88d2fa739d455372d864b32047d1088d4775 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 30 May 2025 18:41:49 +0000 Subject: [PATCH 098/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index c86934fa2..1fd09f4d5 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+981 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d8f5488486d96997c3c77b5f50365b9151b1fd72 +version=v1.2.0+983 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6a7275fe51dff62877e91eedacfc959e3e3e5096 From 2b6b3a1d90232a1986cb8627e6ef8ddea3b5c8ff Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 8 May 2025 10:03:05 -0400 Subject: [PATCH 099/289] [BI-2632] restore the @Get("/programs/{programId}/germplasm/lists/{listDbId}/export{?fileExtension}") endpoint --- .../brapi/v2/BrAPIGermplasmController.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java index 84a6a17f0..8bf929855 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java @@ -221,6 +221,25 @@ public HttpResponse>>> getGermplasm( return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, "Error parsing requested date format"); } } + @Get("/programs/{programId}/germplasm/lists/{listDbId}/export{?fileExtension}") + @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) + public HttpResponse germplasmListExport( + @PathVariable("programId") UUID programId, @PathVariable("listDbId") String listDbId, @QueryValue(defaultValue = "XLSX") String fileExtension) { + String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; + try { + FileType extension = Enum.valueOf(FileType.class, fileExtension); + DownloadFile germplasmListFile = germplasmService.exportGermplasmList(programId, listDbId, extension); + HttpResponse germplasmListExport = HttpResponse.ok(germplasmListFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+germplasmListFile.getFileName()+extension.getExtension()); + return germplasmListExport; + } + catch (Exception e) { + log.info(e.getMessage(), e); + e.printStackTrace(); + HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); + return response; + } + } @Get("/programs/{programId}/germplasm/export{?fileExtension,list}") @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") From 3f37b4fcf1a862f46e578e6db4c3039b6f66aae9 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 22 May 2025 10:20:11 -0400 Subject: [PATCH 100/289] [BI-2632] Added test for exporting a Germplasm List --- .../GermplasmControllerIntegrationTest.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java index 32781dd8c..de42ac99f 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java @@ -25,16 +25,22 @@ import org.breedinginsight.model.Program; import org.breedinginsight.model.Species; import org.breedinginsight.services.SpeciesService; +import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.response.mappers.GermplasmQueryMapper; import org.jooq.DSLContext; import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import tech.tablesaw.api.Table; import javax.inject.Inject; +import java.io.ByteArrayInputStream; import java.io.File; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import static io.micronaut.http.HttpRequest.GET; import static io.micronaut.http.HttpRequest.POST; @@ -237,7 +243,38 @@ public void getAllGermplasmListsSuccess() { } } } + @ParameterizedTest + @CsvSource(value = {"CSV", "XLSX", "XLS"}) + @SneakyThrows + public void germplasmListExport(String extension) { + String programId = validProgram.getId().toString(); + String germplasmListDbId = fetchGermplasmListDbId(programId); + + // Build the endpoint to get germplasm by germplasm list. + String endpoint = String.format("/programs/%s/germplasm/lists/%s/export?fileExtension=%s", programId, germplasmListDbId, extension); + + // Get germplasm by list. + Flowable> call = client.exchange( + GET(endpoint).cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse response = call.blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(response.body())); + + Table download = Table.create(); + if (extension.equals("CSV")) { + download = FileUtil.parseTableFromCsv(bodyStream); + } + if (extension.equals("XLS") || extension.equals("XLSX")) { + download = FileUtil.parseTableFromExcel(bodyStream, 0); + } + int dataSize = download.rowCount(); + assertEquals(3, dataSize, "Wrong number of germplasm were returned"); + } @Test @SneakyThrows public void getAllGermplasmByListSuccess() { From 1747ca706a93290470c55a54d147f3b87a4be3f3 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 9 Jun 2025 19:16:46 +0000 Subject: [PATCH 101/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1fd09f4d5..d45717fbc 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+983 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6a7275fe51dff62877e91eedacfc959e3e3e5096 +version=v1.2.0+997 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a006f10aed0b1d64b4cb1a5dec3370e5963d04c3 From c48f57262dabe649a2b2cd57bae9186418d29309 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 9 Jun 2025 15:47:45 -0400 Subject: [PATCH 102/289] code improvements --- .../observationUnitID/ObservationUnitIDBlankValidator.java | 3 ++- .../observationUnitID/ObservationUnitIDFormatValidator.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java index caf505fac..97e49ad34 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java @@ -44,7 +44,8 @@ public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws Column idCol = ctx.getImportContext().getData().columns(idColName).get(0); for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { - String id = idCol.get(rowNum).toString(); + Object cellValue = idCol.get(rowNum); + String id = (cellValue != null) ? cellValue.toString() : null; if ( id == null || id.isBlank()) { // Check if ObsUnitID is blank ExperimentUtilities.addRowError(idColName, BITB.getValue(), rowErrors, rowNum); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java index 5b5585d0e..7ff63f36e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java @@ -46,7 +46,7 @@ public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws ValidationErrors rowErrors = ctx.getAppendOverwriteWorkflowContext().getValidationErrors(); String idColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); - Column idCol = ctx.getImportContext().getData().columns(idColName).get(0); + Column idCol = ctx.getImportContext().getData().column(idColName); for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { Object value = idCol.get(rowNum); From 41cd6a0f900f5f92412e94104f8cb50ae1371f36 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 11 Jun 2025 10:46:06 -0400 Subject: [PATCH 103/289] Add missing Honey Bee entry --- src/main/resources/brapi/sql/R__species.sql | 8 +-- src/test/resources/sql/brapi/species.sql | 65 ++++++++++++++------- 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/src/main/resources/brapi/sql/R__species.sql b/src/main/resources/brapi/sql/R__species.sql index 993895398..ed7667322 100644 --- a/src/main/resources/brapi/sql/R__species.sql +++ b/src/main/resources/brapi/sql/R__species.sql @@ -52,10 +52,10 @@ BEGIN ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), - ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), - ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), - ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), - ('Sugar Beet'), ('Coffee') + ('Strawberry'), ('Honey Bee'), ('Pecan'), ('Lettuce'), + ('Cotton'), ('Sorghum'), ('Hemp'), ('Hop'), + ('Hydrangea'), ('Red Clover'), ('Potato'), ('Blackberry'), + ('Raspberry'), ('Sugar Beet'), ('Coffee') ) AS src(crop_name) ON CONFLICT (id) DO -- want case changes or space changes to overwrite existing diff --git a/src/test/resources/sql/brapi/species.sql b/src/test/resources/sql/brapi/species.sql index f62e69156..69e3cf706 100644 --- a/src/test/resources/sql/brapi/species.sql +++ b/src/test/resources/sql/brapi/species.sql @@ -1,31 +1,50 @@ --- name: CopyrightNotice -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. --- name: InsertSpecies CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; DO $$ DECLARE v_auth_id CONSTANT uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; BEGIN + /* ------------------------------------------------------------------------------------------ + • uuid_generate_v5(namespace, crop_name) → deterministic UUID can be used for idempotency + • Do it this way so no schema changes are required + • Removed the Honey Bee special case because all systems will be starting fresh + ------------------------------------------------------------------------------------------ */ INSERT INTO crop (id, auth_user_id, crop_name) SELECT uuid_generate_v5('9a4deca9-4068-46a3-9efe-db0c181f491a'::uuid, + -- 1) lower‑case + -- 2) trim leading/trailing space + -- 3) REMOVE every space or tab regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), v_auth_id, crop_name @@ -33,12 +52,14 @@ FROM (VALUES ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), - ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), - ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), - ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), - ('Sugar Beet'), ('Coffee') + ('Strawberry'), ('Honey Bee'), ('Pecan'), ('Lettuce'), + ('Cotton'), ('Sorghum'), ('Hemp'), ('Hop'), + ('Hydrangea'), ('Red Clover'), ('Potato'), ('Blackberry'), + ('Raspberry'), ('Sugar Beet'), ('Coffee') ) AS src(crop_name) ON CONFLICT (id) DO +-- want case changes or space changes to overwrite existing +-- Only rewrite the row if name changed UPDATE SET crop_name = EXCLUDED.crop_name WHERE crop.crop_name IS DISTINCT FROM EXCLUDED.crop_name; END $$; \ No newline at end of file From a65bafa6e42d38000dd29433bc37549520574587 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 11 Jun 2025 10:50:00 -0400 Subject: [PATCH 104/289] Fix test headers --- src/test/resources/sql/brapi/species.sql | 47 +++++++++--------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/src/test/resources/sql/brapi/species.sql b/src/test/resources/sql/brapi/species.sql index 69e3cf706..258b5db72 100644 --- a/src/test/resources/sql/brapi/species.sql +++ b/src/test/resources/sql/brapi/species.sql @@ -1,33 +1,22 @@ --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. +-- name: CopyrightNotice +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- name: InsertSpecies CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; DO $$ From 8ebc08023fa521a20d8f3aeb56df20ef415f6e7f Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 11 Jun 2025 14:38:58 -0400 Subject: [PATCH 105/289] Remove comments in test script --- src/test/resources/sql/brapi/species.sql | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/test/resources/sql/brapi/species.sql b/src/test/resources/sql/brapi/species.sql index 258b5db72..2b8fcf26d 100644 --- a/src/test/resources/sql/brapi/species.sql +++ b/src/test/resources/sql/brapi/species.sql @@ -23,17 +23,9 @@ DO $$ DECLARE v_auth_id CONSTANT uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; BEGIN - /* ------------------------------------------------------------------------------------------ - • uuid_generate_v5(namespace, crop_name) → deterministic UUID can be used for idempotency - • Do it this way so no schema changes are required - • Removed the Honey Bee special case because all systems will be starting fresh - ------------------------------------------------------------------------------------------ */ INSERT INTO crop (id, auth_user_id, crop_name) SELECT uuid_generate_v5('9a4deca9-4068-46a3-9efe-db0c181f491a'::uuid, - -- 1) lower‑case - -- 2) trim leading/trailing space - -- 3) REMOVE every space or tab regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), v_auth_id, crop_name @@ -47,8 +39,6 @@ FROM (VALUES ('Raspberry'), ('Sugar Beet'), ('Coffee') ) AS src(crop_name) ON CONFLICT (id) DO --- want case changes or space changes to overwrite existing --- Only rewrite the row if name changed UPDATE SET crop_name = EXCLUDED.crop_name WHERE crop.crop_name IS DISTINCT FROM EXCLUDED.crop_name; END $$; \ No newline at end of file From 7752d3e22d0ff8b32cda1065d15e3a15adeba472 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 12 Jun 2025 14:13:40 +0000 Subject: [PATCH 106/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index d45717fbc..8439876d0 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0+997 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a006f10aed0b1d64b4cb1a5dec3370e5963d04c3 +version=v1.2.0+1001 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c39e002e66c44f749fccb35b7ac8e9bf3e2cd9dc From ddb3147f2615e4d14d9f084a9e20ebd372215e5d Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 23 Jun 2025 14:16:05 -0400 Subject: [PATCH 107/289] respond to comments and code clean up --- .../AppendOverwriteIDValidation.java | 1 - .../ObservationUnitDuplicateIDValidator.java | 1 - ...ulateExistingPendingImportObjectsStep.java | 28 ------------------- .../ValidatePendingImportObjectsStep.java | 10 ------- 4 files changed, 40 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java index 053ee905a..a3adec8f1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java @@ -31,7 +31,6 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; import org.breedinginsight.services.exceptions.BadRequestException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java index 3331503ba..d4621d058 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java @@ -47,7 +47,6 @@ public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws Column idCol = ctx.getImportContext().getData().columns(idColName).get(0); for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { - String id = idCol.get(rowNum).toString(); if (referenceOUIds.contains(idCol.get(rowNum).toString())) { // Check if ObsUnitID is duplicated ExperimentUtilities.addRowError(idColName, VVCN.getValue(), rowErrors, rowNum); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index b00ac9ffc..e611ec46f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -191,34 +191,6 @@ private Map> initializeObserva // } } - /** - * Adds a new map entry to observationUnitByName based on the brAPIObservationUnit passed in and sets the - * expUnitId in the rowsByObsUnitId map. - * - * @param brAPIObservationUnit the BrAPI observation unit object - * @param refSource the reference source - * @param program the program object - * @param observationUnitByName the map of observation units by name (will be modified in place) - * @param rowByObsUnitId the map of rows by observation unit ID (will be modified in place) - * - * @throws InternalServerException - */ - private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservationUnit, String refSource, Program program, - Map> observationUnitByName, - Map rowByObsUnitId) { - BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) - .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); - row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); - observationUnitByName.put(ExperimentUtilities.createObservationUnitKey(row), - new PendingImportObject<>(ImportObjectState.EXISTING, - brAPIObservationUnit, - UUID.fromString(idRef.getReferenceId()))); - } - - - /** * Initializes studies by name without scope. * diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java index b865d8437..1ed8f9f97 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java @@ -232,16 +232,6 @@ private void validateConditionallyRequired(PendingData pendingData, ValidationEr // if(StringUtils.isNotBlank(importRow.getObsUnitID())) { // ExperimentUtilities.addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); // } - } else { - // TODO: is this validation still needed? - //Check if existing environment. If so, ObsUnitId must be assigned -// validateRequiredCell( -// importRow.getObsUnitID(), -// ExperimentObservation.Columns.OBS_UNIT_ID, -// ExperimentUtilities.MISSING_OBS_UNIT_ID_ERROR, -// validationErrors, -// rowNum -// ); } } From a3b3f618dd9d87e2ae4529ee96f048dea2e1db6d Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 25 Jun 2025 16:34:15 -0400 Subject: [PATCH 108/289] [BI-2540] - renamed orcid, added oauth_provider column --- .../api/auth/AuthServiceLoginHandler.java | 11 +++++----- .../breedinginsight/daos/ProgramUserDAO.java | 4 ++-- .../org/breedinginsight/daos/UserDAO.java | 4 ++-- .../daos/impl/UserDAOImpl.java | 6 ++--- .../java/org/breedinginsight/model/User.java | 13 ++++++----- .../breedinginsight/services/UserService.java | 22 +++++++++---------- .../response/mappers/UserQueryMapper.java | 2 +- .../db/migration/V1.33.0__rename-orcid.sql | 22 +++++++++++++++++++ ...mSecuredAnnotationRuleIntegrationTest.java | 6 ++--- ...eedingMethodControllerIntegrationTest.java | 2 +- .../ExperimentControllerIntegrationTest.java | 4 ++-- .../ImportControllerIntegrationTest.java | 2 +- .../InternalServerErrorHandlerUnitTest.java | 2 +- .../JobControllerIntegrationTest.java | 3 +-- .../OntologyControllerIntegrationTest.java | 2 +- .../ProgramControllerIntegrationTest.java | 11 ++++------ ...leSubmissionControllerIntegrationTest.java | 3 +-- .../api/v1/controller/TestTokenValidator.java | 8 +++---- .../TraitControllerIntegrationTest.java | 2 +- .../UploadControllerIntegrationTest.java | 2 +- .../UserControllerIntegrationTest.java | 8 +++---- .../BrAPIServiceFilterIntegrationTest.java | 2 +- .../MetadataFilterIntegrationTest.java | 4 ++-- ...apiAuthorizeControllerIntegrationTest.java | 2 +- ...ionVariablesControllerIntegrationTest.java | 4 ++-- ...vationLevelsControllerIntegrationTest.java | 2 +- ...ervationUnitControllerIntegrationTest.java | 2 +- ...ObservationsControllerIntegrationTest.java | 2 +- .../brapi/v2/BrAPITestUtils.java | 6 ++--- .../v2/BrAPIV2ControllerIntegrationTest.java | 2 +- ...tionVariableControllerIntegrationTest.java | 2 +- .../GermplasmControllerIntegrationTest.java | 2 +- .../brapps/importer/ImportTestUtils.java | 2 +- .../daos/BrAPIObservationUnitDAOTest.java | 2 +- .../DSLTransactionResultIntegrationTest.java | 3 +-- .../TraitValidatorIntegrationTest.java | 2 +- .../ResponseUtilsIntegrationTest.java | 2 +- .../mappers/UserQueryMapperUnitTest.java | 4 ++-- 38 files changed, 100 insertions(+), 84 deletions(-) create mode 100644 src/main/resources/db/migration/V1.33.0__rename-orcid.sql diff --git a/src/main/java/org/breedinginsight/api/auth/AuthServiceLoginHandler.java b/src/main/java/org/breedinginsight/api/auth/AuthServiceLoginHandler.java index 3a25eae13..8e210f608 100644 --- a/src/main/java/org/breedinginsight/api/auth/AuthServiceLoginHandler.java +++ b/src/main/java/org/breedinginsight/api/auth/AuthServiceLoginHandler.java @@ -30,7 +30,6 @@ import io.micronaut.security.token.jwt.cookie.JwtCookieLoginHandler; import io.micronaut.security.token.jwt.generator.AccessRefreshTokenGenerator; import io.micronaut.security.token.jwt.generator.AccessTokenConfiguration; -import io.micronaut.security.token.jwt.generator.JwtGeneratorConfiguration; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.model.v1.auth.SignUpJWT; import org.breedinginsight.model.ProgramUser; @@ -83,7 +82,7 @@ public AuthServiceLoginHandler(JwtCookieConfiguration jwtCookieConfiguration, @Override public MutableHttpResponse loginSuccess(UserDetails userDetails, HttpRequest request) { - // Called when login to orcid is successful. + // Called when login to OAuth provider is successful. // Check if our login to our system is successful. if (request.getCookies().contains(accountTokenCookieName)) { Cookie accountTokenCookie = request.getCookies().get(accountTokenCookieName); @@ -124,7 +123,7 @@ public MutableHttpResponse loginSuccess(UserDetails userDetails, HttpRequest< private AuthenticatedUser getUserCredentials(UserDetails userDetails) throws AuthenticationException { - Optional user = userService.getByOrcid(userDetails.getUsername()); + Optional user = userService.getByOAuthId(userDetails.getUsername()); if (user.isPresent()) { if (user.get().getActive()) { @@ -161,7 +160,7 @@ public MutableHttpResponse loginFailed(AuthenticationResponse authenticationF private MutableHttpResponse newAccountCreationResponse(UserDetails userDetails, String accountToken, HttpRequest request) { - String orcid = userDetails.getUsername(); + String oAuthId = userDetails.getUsername(); SignUpJWT signUpJWT; try { signUpJWT = signUpJwtService.validateAndParseAccountSignUpJwt(accountToken); @@ -185,9 +184,9 @@ private MutableHttpResponse newAccountCreationResponse(UserDetails userDetails, } if (newUser.getAccountToken().equals(signUpJWT.getJwtId().toString())) { - // Assign orcid to that user + // Assign oAuthId to that user. try { - userService.updateOrcid(newUser.getId(), orcid); + userService.updateOAuthId(newUser.getId(), oAuthId); } catch (DoesNotExistException e) { MutableHttpResponse resp = HttpResponse.seeOther(URI.create(newAccountErrorUrl)); return resp; diff --git a/src/main/java/org/breedinginsight/daos/ProgramUserDAO.java b/src/main/java/org/breedinginsight/daos/ProgramUserDAO.java index 106a34688..94bb9b3a6 100644 --- a/src/main/java/org/breedinginsight/daos/ProgramUserDAO.java +++ b/src/main/java/org/breedinginsight/daos/ProgramUserDAO.java @@ -120,14 +120,14 @@ public List getProgramUsersByUserId(UUID userId) { return parseRecords(records, createdByUser, updatedByUser); } - public List getProgramUsersByOrcid(String orcid) { + public List getProgramUsersByOAuthId(String oAuthId) { BiUserTable createdByUser = BI_USER.as("createdByUser"); BiUserTable updatedByUser = BI_USER.as("updatedByUser"); // TODO: When we allow for pulling archived users, active condition won't be hardcoded. Result records = getProgramUsersQuery(createdByUser, updatedByUser) - .where(BI_USER.ORCID.eq(orcid)) + .where(BI_USER.OAUTH_ID.eq(oAuthId)) .and(PROGRAM.ACTIVE.eq(true)) .fetch(); diff --git a/src/main/java/org/breedinginsight/daos/UserDAO.java b/src/main/java/org/breedinginsight/daos/UserDAO.java index 562801735..a89376982 100644 --- a/src/main/java/org/breedinginsight/daos/UserDAO.java +++ b/src/main/java/org/breedinginsight/daos/UserDAO.java @@ -32,11 +32,11 @@ public interface UserDAO extends DAO { Optional getUser(UUID id); - Optional getUserByOrcId(String orcid); + Optional getUserByOAuthId(String oAuthId); BiUserEntity fetchOneById(UUID value); List fetchByEmail(String... values); - List fetchByOrcid(String... values); + List fetchByOauthId(String... values); } diff --git a/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java index e179bc42d..1e6e391b2 100644 --- a/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java @@ -67,11 +67,11 @@ public Optional getUser(UUID id) { return Utilities.getSingleOptional(users); } - public Optional getUserByOrcId(String orcid) { + public Optional getUserByOAuthId(String oAuthId) { List records = getUsersQuery() - .where(BI_USER.ORCID.eq(orcid)) + .where(BI_USER.OAUTH_ID.eq(oAuthId)) .fetch(); - List programUsers = programUserDAO.getProgramUsersByOrcid(orcid); + List programUsers = programUserDAO.getProgramUsersByOAuthId(oAuthId); List users = parseRecords(records, programUsers); return Utilities.getSingleOptional(users); diff --git a/src/main/java/org/breedinginsight/model/User.java b/src/main/java/org/breedinginsight/model/User.java index 30386b79d..2ff26f5f5 100644 --- a/src/main/java/org/breedinginsight/model/User.java +++ b/src/main/java/org/breedinginsight/model/User.java @@ -56,13 +56,14 @@ public class User extends BiUserEntity { public User(BiUserEntity biUser) { this.setId(biUser.getId()); - this.setOrcid(biUser.getOrcid()); + this.setOauthId(biUser.getOauthId()); this.setName(biUser.getName()); this.setEmail(biUser.getEmail()); this.setSystemRoles(new ArrayList<>()); this.setProgramRoles(new ArrayList<>()); this.setActive(biUser.getActive()); this.setAccountToken(biUser.getAccountToken()); + this.setOauthProvider(biUser.getOauthProvider()); } public User() { @@ -72,13 +73,14 @@ public User() { public static User parseSQLRecord(Record record, @NotNull BiUserTable tableName){ return User.builder() .id(record.getValue(tableName.ID)) - .orcid(record.getValue(tableName.ORCID)) + .oauthId(record.getValue(tableName.OAUTH_ID)) .name(record.getValue(tableName.NAME)) .email(record.getValue(tableName.EMAIL)) .systemRoles(new ArrayList<>()) .programRoles(new ArrayList<>()) .active(record.getValue(tableName.ACTIVE)) .accountToken(record.getValue(tableName.ACCOUNT_TOKEN)) + .oauthProvider(record.getValue(tableName.OAUTH_PROVIDER)) .build(); } @@ -98,7 +100,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; User user = (User) o; return Objects.equals(getId(), user.getId()) && - Objects.equals(getOrcid(), user.getOrcid()) && + Objects.equals(getOauthId(), user.getOauthId()) && Objects.equals(getName(), user.getName()) && Objects.equals(getEmail(), user.getEmail()) && Objects.equals(getCreatedAt(), user.getCreatedAt()) && @@ -106,11 +108,12 @@ public boolean equals(Object o) { Objects.equals(getCreatedBy(), user.getCreatedBy()) && Objects.equals(getUpdatedBy(), user.getUpdatedBy()) && Objects.equals(getActive(), user.getActive()) && - Objects.equals(getAccountToken(), user.getAccountToken()); + Objects.equals(getAccountToken(), user.getAccountToken()) && + Objects.equals(getOauthProvider(), user.getOauthProvider()); } @Override public int hashCode() { - return Objects.hash(getId(), getOrcid(), getName(), getEmail(), getCreatedAt(), getUpdatedAt(), getCreatedBy(), getUpdatedBy(), getActive(), getAccountToken()); + return Objects.hash(getId(), getOauthId(), getName(), getEmail(), getCreatedAt(), getUpdatedAt(), getCreatedBy(), getUpdatedBy(), getActive(), getAccountToken()); } } diff --git a/src/main/java/org/breedinginsight/services/UserService.java b/src/main/java/org/breedinginsight/services/UserService.java index b78fdf8a6..9355e2cf7 100644 --- a/src/main/java/org/breedinginsight/services/UserService.java +++ b/src/main/java/org/breedinginsight/services/UserService.java @@ -91,10 +91,10 @@ public UserService(UserDAO dao, SystemUserRoleDao systemUserRoleDao, SystemRoleD } - public Optional getByOrcid(String orcid) { + public Optional getByOAuthId(String oAuthId) { - // User has been authenticated against orcid, check they have a bi account. - Optional users = dao.getUserByOrcId(orcid); + // User has been authenticated against OAuth provider, check they have a bi account. + Optional users = dao.getUserByOAuthId(oAuthId); if (users.isEmpty()) { return Optional.empty(); @@ -113,7 +113,7 @@ public List getAll() { public Optional getById(UUID userId) { - // User has been authenticated against orcid, check they have a bi account. + // User has been authenticated against OAuth provider, check they have a bi account. Optional user = dao.getUser(userId); if (!user.isPresent()) { @@ -146,7 +146,7 @@ public User create(AuthenticatedUser actingUser, UserRequest userRequest, Config insertSystemRoles(actingUser, newUser.getId(), systemRoles); } - // Start OrcID association flow + // Start OAuth account association flow createAndSendAccountToken(newUser.getId()); return getById(newUser.getId()).get(); @@ -352,7 +352,7 @@ public void createAndSendAccountToken(UUID userId) throws DoesNotExistException sendAccountSignUpEmail(biUser, jwt.getSignedJWT()); } - public void updateOrcid(UUID userId, String orcid) throws DoesNotExistException, AlreadyExistsException { + public void updateOAuthId(UUID userId, String oAuthId) throws DoesNotExistException, AlreadyExistsException { BiUserEntity biUser = dao.fetchOneById(userId); @@ -360,14 +360,14 @@ public void updateOrcid(UUID userId, String orcid) throws DoesNotExistException, throw new DoesNotExistException("UUID for user does not exist"); } - List biUserWithOrcidList = dao.fetchByOrcid(orcid); - for (BiUserEntity biUserWithOrcid: biUserWithOrcidList){ - if (!biUserWithOrcid.getId().equals(userId)){ - throw new AlreadyExistsException("Orcid already in use"); + List biUserWithOAuthIdList = dao.fetchByOauthId(oAuthId); + for (BiUserEntity biUserWithOAuthId: biUserWithOAuthIdList){ + if (!biUserWithOAuthId.getId().equals(userId)){ + throw new AlreadyExistsException("OAuth Id already in use"); } } - biUser.setOrcid(orcid); + biUser.setOauthId(oAuthId); biUser.setAccountToken(null); dao.update(biUser); } diff --git a/src/main/java/org/breedinginsight/utilities/response/mappers/UserQueryMapper.java b/src/main/java/org/breedinginsight/utilities/response/mappers/UserQueryMapper.java index 2eab94b77..5e91ce61e 100644 --- a/src/main/java/org/breedinginsight/utilities/response/mappers/UserQueryMapper.java +++ b/src/main/java/org/breedinginsight/utilities/response/mappers/UserQueryMapper.java @@ -35,7 +35,7 @@ public UserQueryMapper() { fields = Map.ofEntries( Map.entry("name", User::getName), Map.entry("email", User::getEmail), - Map.entry("orcid", User::getOrcid), + Map.entry("orcid", User::getOauthId), Map.entry("systemRoles", user -> user.getSystemRoles() != null ? user.getSystemRoles().stream() .map(role -> role.getDomain()).collect(Collectors.toList()) : null), diff --git a/src/main/resources/db/migration/V1.33.0__rename-orcid.sql b/src/main/resources/db/migration/V1.33.0__rename-orcid.sql new file mode 100644 index 000000000..ac2ad3685 --- /dev/null +++ b/src/main/resources/db/migration/V1.33.0__rename-orcid.sql @@ -0,0 +1,22 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ALTER TABLE bi_user +RENAME COLUMN orcid TO oauth_id; + +ALTER TABLE bi_user +ADD COLUMN oauth_provider text; diff --git a/src/test/java/org/breedinginsight/api/auth/rules/ProgramSecuredAnnotationRuleIntegrationTest.java b/src/test/java/org/breedinginsight/api/auth/rules/ProgramSecuredAnnotationRuleIntegrationTest.java index 26a53b9ca..58a35c919 100644 --- a/src/test/java/org/breedinginsight/api/auth/rules/ProgramSecuredAnnotationRuleIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/auth/rules/ProgramSecuredAnnotationRuleIntegrationTest.java @@ -45,11 +45,9 @@ import java.time.OffsetDateTime; import java.util.List; -import java.util.UUID; import static io.micronaut.http.HttpRequest.GET; import static io.micronaut.http.HttpRequest.POST; -import static org.breedinginsight.TestUtils.insertAndFetchTestProgram; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -106,8 +104,8 @@ void setup() { dsl.execute(fp.get("InsertPrograms")); programs = programDAO.getAll(); - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); - otherTestUser = userDAO.getUserByOrcId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); + otherTestUser = userDAO.getUserByOAuthId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(); // Insert system roles dsl.execute(fp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java index 0cf098c04..c3ba499c4 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java @@ -90,7 +90,7 @@ void setup() throws Exception { securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 9adb44425..e89b0efda 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -113,8 +113,8 @@ void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); - otherTestUser = userDAO.getUserByOrcId(TestTokenValidator.OTHER_TEST_USER_ORCID).orElseThrow(Exception::new); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + otherTestUser = userDAO.getUserByOAuthId(TestTokenValidator.OTHER_TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ImportControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ImportControllerIntegrationTest.java index 378e99085..ebd602d3d 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ImportControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ImportControllerIntegrationTest.java @@ -109,7 +109,7 @@ public void setup() { validProgram = insertAndFetchTestProgram(program); // Set program user - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/InternalServerErrorHandlerUnitTest.java b/src/test/java/org/breedinginsight/api/v1/controller/InternalServerErrorHandlerUnitTest.java index 4fc66e151..42d5d976c 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/InternalServerErrorHandlerUnitTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/InternalServerErrorHandlerUnitTest.java @@ -112,7 +112,7 @@ void setup() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); } @BeforeEach diff --git a/src/test/java/org/breedinginsight/api/v1/controller/JobControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/JobControllerIntegrationTest.java index 6949499e9..e964cbb92 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/JobControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/JobControllerIntegrationTest.java @@ -39,7 +39,6 @@ import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.jooq.DSLContext; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -95,7 +94,7 @@ public void setup() { program = programs.get(0); // Insert system roles - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID) + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID) .get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId() diff --git a/src/test/java/org/breedinginsight/api/v1/controller/OntologyControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/OntologyControllerIntegrationTest.java index bf8dde632..a8293b5d6 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/OntologyControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/OntologyControllerIntegrationTest.java @@ -72,7 +72,7 @@ void setup() throws Exception { brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); brapiObservationFp = FannyPack.fill("src/test/resources/sql/brapi/BrAPIOntologyControllerIntegrationTest.sql"); - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ProgramControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ProgramControllerIntegrationTest.java index b3ef32a2b..32a044fae 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ProgramControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ProgramControllerIntegrationTest.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.*; -import com.google.gson.reflect.TypeToken; import io.kowalski.fannypack.FannyPack; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; @@ -42,7 +41,6 @@ import org.breedinginsight.api.model.v1.request.*; import org.breedinginsight.api.model.v1.request.query.FilterRequest; import org.breedinginsight.api.model.v1.request.query.SearchRequest; -import org.breedinginsight.api.model.v1.response.Response; import org.breedinginsight.api.v1.controller.metadata.SortOrder; import org.breedinginsight.dao.db.tables.daos.ProgramDao; import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; @@ -58,7 +56,6 @@ import javax.inject.Inject; import javax.inject.Named; -import java.lang.reflect.Type; import java.math.BigDecimal; import java.time.OffsetDateTime; import java.util.*; @@ -165,13 +162,13 @@ void setup() throws Exception { securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); - otherUser = userDAO.getUserByOrcId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); + otherUser = userDAO.getUserByOAuthId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); - Optional optionalUser = userService.getByOrcid(TestTokenValidator.TEST_USER_ORCID); + Optional optionalUser = userService.getByOAuthId(TestTokenValidator.TEST_USER_ORCID); testUser = optionalUser.get(); // Get species for tests @@ -275,7 +272,7 @@ public ProgramLocation insertAndFetchTestLocation() throws Exception { public User fetchTestUser() throws Exception{ - Optional user = userService.getByOrcid(TestTokenValidator.TEST_USER_ORCID); + Optional user = userService.getByOAuthId(TestTokenValidator.TEST_USER_ORCID); if (!user.isPresent()){ throw new Exception("Failed to insert test user"); } diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java index ec28c689b..091d836a3 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -48,7 +48,6 @@ import org.breedinginsight.model.*; import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.parsers.ParsingException; -import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.Utilities; @@ -102,7 +101,7 @@ void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TestTokenValidator.java b/src/test/java/org/breedinginsight/api/v1/controller/TestTokenValidator.java index 644ddc046..0e40555ab 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TestTokenValidator.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TestTokenValidator.java @@ -53,7 +53,7 @@ public TestTokenValidator(Collection signatureConfigurat public Publisher validateToken(String token) { if (token.equals("test-registered-user")) { - Optional testUser = userService.getByOrcid(TEST_USER_ORCID); + Optional testUser = userService.getByOAuthId(TEST_USER_ORCID); Map adminClaims = new HashMap<>(); List roles = new ArrayList<>(); roles.add("SYSTEM ADMINISTRATOR"); @@ -61,14 +61,14 @@ public Publisher validateToken(String token) { adminClaims.put("id", testUser.get().getId().toString()); return Flowable.just(new DefaultAuthentication(TEST_USER_ORCID, adminClaims)); } else if (token.equals("other-registered-user")) { - Optional otherTestUser = userService.getByOrcid(OTHER_TEST_USER_ORCID); + Optional otherTestUser = userService.getByOAuthId(OTHER_TEST_USER_ORCID); Map userClaims = new HashMap<>(); List roles = new ArrayList<>(); userClaims.put("roles", roles); userClaims.put("id", otherTestUser.get().getId().toString()); return Flowable.just(new DefaultAuthentication(OTHER_TEST_USER_ORCID, userClaims)); } else if (token.equals("another-registered-user")) { - Optional anotherTestUser = userService.getByOrcid(ANOTHER_TEST_USER_ORCID); + Optional anotherTestUser = userService.getByOAuthId(ANOTHER_TEST_USER_ORCID); Map userClaims = new HashMap<>(); List roles = new ArrayList<>(); userClaims.put("roles", roles); @@ -82,7 +82,7 @@ public Publisher validateToken(String token) { adminClaims.put("id", NON_EXISTENT_USER_ID); return Flowable.just(new DefaultAuthentication(NON_EXISTENT_USER_ID, adminClaims)); } else if (token.equals("inactive-user")) { - Optional inactiveUser = userService.getByOrcid(INACTIVE_USER_ORCID); + Optional inactiveUser = userService.getByOAuthId(INACTIVE_USER_ORCID); Map adminClaims = new HashMap<>(); List roles = new ArrayList<>(); roles.add("SYSTEM ADMINISTRATOR"); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java index ebfeec7ed..08c7edd37 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java @@ -160,7 +160,7 @@ public void setup() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Insert program diff --git a/src/test/java/org/breedinginsight/api/v1/controller/UploadControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/UploadControllerIntegrationTest.java index 6549586e0..b008ccf51 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/UploadControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/UploadControllerIntegrationTest.java @@ -107,7 +107,7 @@ public void setup() { validProgram = insertAndFetchTestProgram(program); // Set program user - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java index 23fcfa32c..45774b5ea 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java @@ -110,8 +110,8 @@ void setup() { dsl.execute(fp.get("InsertProgram")); dsl.execute(fp.get("InsertUserProgramAssociations")); - testUser = biUserDao.fetchByOrcid(TestTokenValidator.TEST_USER_ORCID).get(0); - otherTestUser = biUserDao.fetchByOrcid(TestTokenValidator.OTHER_TEST_USER_ORCID).get(0); + testUser = biUserDao.fetchByOauthId(TestTokenValidator.TEST_USER_ORCID).get(0); + otherTestUser = biUserDao.fetchByOauthId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(0); validSystemRole = systemRoleDao.findAll().get(0); validRole = roleDao.findAll().get(0); validPrograms = programDao.findAll(); @@ -144,7 +144,7 @@ public void getUsersExistingId() { JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(testUser.getId().toString(), result.get("id").getAsString(), "Wrong id"); assertEquals("Test User", result.get("name").getAsString(), "Wrong name"); - assertEquals(testUser.getOrcid(), result.get("orcid").getAsString(), "Wrong orcid"); + assertEquals(testUser.getOauthId(), result.get("orcid").getAsString(), "Wrong orcid"); assertEquals("test@test.com", result.get("email").getAsString(), "Wrong email"); JsonArray resultRoles = (JsonArray) result.get("systemRoles"); @@ -584,7 +584,7 @@ public void getUserInfoRegisteredUser() { JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(otherTestUser.getName(), result.get("name").getAsString(), "Wrong name"); - assertEquals(otherTestUser.getOrcid(), result.get("orcid").getAsString(), "Wrong orcid"); + assertEquals(otherTestUser.getOauthId(), result.get("orcid").getAsString(), "Wrong orcid"); assertEquals(otherTestUser.getEmail(), result.get("email").getAsString(), "Wrong email"); assertEquals(otherTestUser.getId().toString(), result.get("id").getAsString(), "Wrong id"); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java index bd565374e..0fbfe4035 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java @@ -181,7 +181,7 @@ public void insertTestData() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Insert program diff --git a/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java index 4257f90d6..4f04c3578 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java @@ -84,8 +84,8 @@ public void setup() { dsl.execute(fp.get("InsertUserProgramAssociations")); dsl.execute(fp.get("InsertManyUsers")); - testUser = biUserDao.fetchByOrcid(TestTokenValidator.TEST_USER_ORCID).get(0); - otherTestUser = biUserDao.fetchByOrcid(TestTokenValidator.OTHER_TEST_USER_ORCID).get(0); + testUser = biUserDao.fetchByOauthId(TestTokenValidator.TEST_USER_ORCID).get(0); + otherTestUser = biUserDao.fetchByOauthId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(0); dsl.execute(fp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); dsl.execute(fp.get("InsertSystemRoleAdmin"), otherTestUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiAuthorizeControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiAuthorizeControllerIntegrationTest.java index 861bf27db..5567f7db8 100644 --- a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiAuthorizeControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiAuthorizeControllerIntegrationTest.java @@ -67,7 +67,7 @@ public void setup() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Insert program diff --git a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java index 038a691ec..9173d3cdf 100644 --- a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java @@ -105,8 +105,8 @@ public void setup() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); - User otherTestUser = userDAO.getUserByOrcId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); + User otherTestUser = userDAO.getUserByOAuthId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Insert program diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java index 61ce61a57..4f15c553d 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java @@ -106,7 +106,7 @@ void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java index 6bea444f9..44792ea64 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java @@ -110,7 +110,7 @@ void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java index 27cf6defd..59cd33630 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java @@ -111,7 +111,7 @@ void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java index d0b440f79..936bd401c 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java @@ -96,7 +96,7 @@ public Tuple2> setupTestProgram(DSLContext brAPIDslContext FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species @@ -121,7 +121,7 @@ public Tuple2> setupTestProgram(DSLContext brAPIDslContext dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); // Add Experimental Collaborator user. - User collaborator = userDAO.getUserByOrcId(TestTokenValidator.OTHER_TEST_USER_ORCID).orElseThrow(Exception::new); + User collaborator = userDAO.getUserByOAuthId(TestTokenValidator.OTHER_TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertProgramRolesExperimentalCollaborator"), collaborator.getId().toString(), program.getId().toString()); // Get experiment import map @@ -208,7 +208,7 @@ public Tuple2> setupTestProgram(DSLContext brAPIDslContext .get("id").getAsString(); // Refetch collaborator, since we updated program_user_roles. - collaborator = userDAO.getUserByOrcId(TestTokenValidator.OTHER_TEST_USER_ORCID).orElseThrow(Exception::new); + collaborator = userDAO.getUserByOAuthId(TestTokenValidator.OTHER_TEST_USER_ORCID).orElseThrow(Exception::new); // Add collaborator to experiment. dsl.execute(securityFp.get("AddCollaborator"), experiment2Id, collaborator.getProgramRoles().get(0).getId()); diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java index 1e1cb922b..24868f3fe 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java @@ -89,7 +89,7 @@ public void setup() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID) + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID) .get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId() diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java index 42dfa47c2..9fc0aac90 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java @@ -108,7 +108,7 @@ void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species diff --git a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java index de42ac99f..9b9cda353 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java @@ -87,7 +87,7 @@ public void setup() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); var brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Set up BrAPI diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 53abd8483..2d89d5110 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -180,7 +180,7 @@ public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, .getAsJsonArray("data") .get(0).getAsJsonObject().get("id").getAsString(); - BiUserEntity testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + BiUserEntity testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java b/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java index c4c214682..f4375e590 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java @@ -92,7 +92,7 @@ public void setup() { // Insert system admin role so can create program FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); SpeciesRequest speciesRequest = SpeciesRequest.builder() diff --git a/src/test/java/org/breedinginsight/daos/DSLTransactionResultIntegrationTest.java b/src/test/java/org/breedinginsight/daos/DSLTransactionResultIntegrationTest.java index a298361fa..e0cd9bb31 100644 --- a/src/test/java/org/breedinginsight/daos/DSLTransactionResultIntegrationTest.java +++ b/src/test/java/org/breedinginsight/daos/DSLTransactionResultIntegrationTest.java @@ -28,7 +28,6 @@ import org.jooq.DSLContext; import org.jooq.exception.DataAccessException; import org.jooq.impl.DSL; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -60,7 +59,7 @@ public class DSLTransactionResultIntegrationTest extends DatabaseTest { @BeforeAll void setup() throws Exception { - Optional userOptional = userService.getByOrcid(TestTokenValidator.TEST_USER_ORCID); + Optional userOptional = userService.getByOAuthId(TestTokenValidator.TEST_USER_ORCID); actingUser = userOptional.get(); } diff --git a/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java b/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java index 3006bc7b5..b31839ff2 100644 --- a/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java +++ b/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java @@ -86,7 +86,7 @@ public void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species diff --git a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java index 7cd16622a..56f977a47 100644 --- a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java +++ b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java @@ -93,7 +93,7 @@ public void setup() throws MissingRequiredInfoException, UnprocessableEntityExce var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Insert program diff --git a/src/test/java/org/breedinginsight/utilities/response/mappers/UserQueryMapperUnitTest.java b/src/test/java/org/breedinginsight/utilities/response/mappers/UserQueryMapperUnitTest.java index 8e5be117d..daedfa20a 100644 --- a/src/test/java/org/breedinginsight/utilities/response/mappers/UserQueryMapperUnitTest.java +++ b/src/test/java/org/breedinginsight/utilities/response/mappers/UserQueryMapperUnitTest.java @@ -48,7 +48,7 @@ public void testMappings() { User user = User.builder() .name("Test User") .email("test@user.com") - .orcid("000000-000000-000000-00000") + .oauthId("000000-000000-000000-00000") .systemRoles(List.of(SystemRole.builder().domain("System Administrator").build())) .programRoles(List.of(ProgramUser.builder().program(Program.builder().name("Test program").build()).build())) .active(false) @@ -60,7 +60,7 @@ public void testMappings() { assertEquals(user.getName(), userQueryMapper.getField("name").apply(user), "Wrong getter"); assertEquals(user.getEmail(), userQueryMapper.getField("email").apply(user), "Wrong getter"); - assertEquals(user.getOrcid(), userQueryMapper.getField("orcid").apply(user), "Wrong getter"); + assertEquals(user.getOauthId(), userQueryMapper.getField("orcid").apply(user), "Wrong getter"); assertEquals(user.getSystemRoles().stream().map(role -> role.getDomain()).collect(Collectors.toList()), userQueryMapper.getField("systemRoles").apply(user), "Wrong getter"); assertEquals(user.getProgramRoles().stream().map(role -> role.getProgram().getName()).collect(Collectors.toList()), From 61dc2b1927aef4514b92ee9cebc05d832944ac93 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 25 Jun 2025 16:34:39 -0400 Subject: [PATCH 109/289] [BI-2540] - disabled flyway out-of-order --- src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 76cf54b28..616747556 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -122,7 +122,7 @@ flyway: default-url: ${brapi.server.default-url} brapi-reference-source: ${brapi.server.reference-source} orcid-sandbox-authentication: ${ORCID_SANDBOX_AUTHENTICATION:false} - out-of-order: true + out-of-order: false jooq: datasources: default: From d3c0508c51b3fdfbc5f374a6f2a9de516fd3d507 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 26 Jun 2025 11:51:45 -0400 Subject: [PATCH 110/289] [BI-2540] - utilized oauth_provider column --- .../api/auth/AuthServiceLoginHandler.java | 15 +++++++++++++-- src/main/java/org/breedinginsight/model/User.java | 2 +- .../org/breedinginsight/services/UserService.java | 3 ++- .../response/mappers/UserQueryMapper.java | 2 +- .../db/migration/V1.33.0__rename-orcid.sql | 4 +++- .../controller/UserControllerIntegrationTest.java | 8 ++++---- .../response/mappers/UserQueryMapperUnitTest.java | 2 +- 7 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/auth/AuthServiceLoginHandler.java b/src/main/java/org/breedinginsight/api/auth/AuthServiceLoginHandler.java index 8e210f608..a03b21641 100644 --- a/src/main/java/org/breedinginsight/api/auth/AuthServiceLoginHandler.java +++ b/src/main/java/org/breedinginsight/api/auth/AuthServiceLoginHandler.java @@ -158,9 +158,20 @@ public MutableHttpResponse loginFailed(AuthenticationResponse authenticationF } } + private String parseOAuthProvider(HttpRequest request) { + // The request path will be something like "/sso/success/github". + if (request.getPath().toLowerCase().contains("github")) { + return "github"; + } else { + // Default to ORCID. + return "orcid"; + } + } + private MutableHttpResponse newAccountCreationResponse(UserDetails userDetails, String accountToken, HttpRequest request) { String oAuthId = userDetails.getUsername(); + String oAuthProvider = parseOAuthProvider(request); SignUpJWT signUpJWT; try { signUpJWT = signUpJwtService.validateAndParseAccountSignUpJwt(accountToken); @@ -184,9 +195,9 @@ private MutableHttpResponse newAccountCreationResponse(UserDetails userDetails, } if (newUser.getAccountToken().equals(signUpJWT.getJwtId().toString())) { - // Assign oAuthId to that user. + // Assign OAuth Id and provider to that user. try { - userService.updateOAuthId(newUser.getId(), oAuthId); + userService.updateOAuthInfo(newUser.getId(), oAuthId, oAuthProvider); } catch (DoesNotExistException e) { MutableHttpResponse resp = HttpResponse.seeOther(URI.create(newAccountErrorUrl)); return resp; diff --git a/src/main/java/org/breedinginsight/model/User.java b/src/main/java/org/breedinginsight/model/User.java index 2ff26f5f5..9d72b1841 100644 --- a/src/main/java/org/breedinginsight/model/User.java +++ b/src/main/java/org/breedinginsight/model/User.java @@ -114,6 +114,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(getId(), getOauthId(), getName(), getEmail(), getCreatedAt(), getUpdatedAt(), getCreatedBy(), getUpdatedBy(), getActive(), getAccountToken()); + return Objects.hash(getId(), getOauthId(), getName(), getEmail(), getCreatedAt(), getUpdatedAt(), getCreatedBy(), getUpdatedBy(), getActive(), getAccountToken(), getOauthProvider()); } } diff --git a/src/main/java/org/breedinginsight/services/UserService.java b/src/main/java/org/breedinginsight/services/UserService.java index 9355e2cf7..33c459254 100644 --- a/src/main/java/org/breedinginsight/services/UserService.java +++ b/src/main/java/org/breedinginsight/services/UserService.java @@ -352,7 +352,7 @@ public void createAndSendAccountToken(UUID userId) throws DoesNotExistException sendAccountSignUpEmail(biUser, jwt.getSignedJWT()); } - public void updateOAuthId(UUID userId, String oAuthId) throws DoesNotExistException, AlreadyExistsException { + public void updateOAuthInfo(UUID userId, String oAuthId, String oAuthProvider) throws DoesNotExistException, AlreadyExistsException { BiUserEntity biUser = dao.fetchOneById(userId); @@ -368,6 +368,7 @@ public void updateOAuthId(UUID userId, String oAuthId) throws DoesNotExistExcept } biUser.setOauthId(oAuthId); + biUser.setOauthProvider(oAuthProvider); biUser.setAccountToken(null); dao.update(biUser); } diff --git a/src/main/java/org/breedinginsight/utilities/response/mappers/UserQueryMapper.java b/src/main/java/org/breedinginsight/utilities/response/mappers/UserQueryMapper.java index 5e91ce61e..c1ba7f0c6 100644 --- a/src/main/java/org/breedinginsight/utilities/response/mappers/UserQueryMapper.java +++ b/src/main/java/org/breedinginsight/utilities/response/mappers/UserQueryMapper.java @@ -35,7 +35,7 @@ public UserQueryMapper() { fields = Map.ofEntries( Map.entry("name", User::getName), Map.entry("email", User::getEmail), - Map.entry("orcid", User::getOauthId), + Map.entry("oauthId", User::getOauthId), Map.entry("systemRoles", user -> user.getSystemRoles() != null ? user.getSystemRoles().stream() .map(role -> role.getDomain()).collect(Collectors.toList()) : null), diff --git a/src/main/resources/db/migration/V1.33.0__rename-orcid.sql b/src/main/resources/db/migration/V1.33.0__rename-orcid.sql index ac2ad3685..928a426fa 100644 --- a/src/main/resources/db/migration/V1.33.0__rename-orcid.sql +++ b/src/main/resources/db/migration/V1.33.0__rename-orcid.sql @@ -15,8 +15,10 @@ * limitations under the License. */ +-- Rename orcid column to more generic oauth_id. ALTER TABLE bi_user RENAME COLUMN orcid TO oauth_id; +-- Add a column to store the OAuth provider with 'orcid' as the default value. ALTER TABLE bi_user -ADD COLUMN oauth_provider text; +ADD COLUMN oauth_provider text DEFAULT 'orcid'; diff --git a/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java index 45774b5ea..4914824f3 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java @@ -144,7 +144,7 @@ public void getUsersExistingId() { JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(testUser.getId().toString(), result.get("id").getAsString(), "Wrong id"); assertEquals("Test User", result.get("name").getAsString(), "Wrong name"); - assertEquals(testUser.getOauthId(), result.get("orcid").getAsString(), "Wrong orcid"); + assertEquals(testUser.getOauthId(), result.get("oauthId").getAsString(), "Wrong OAuthId"); assertEquals("test@test.com", result.get("email").getAsString(), "Wrong email"); JsonArray resultRoles = (JsonArray) result.get("systemRoles"); @@ -203,12 +203,12 @@ void postUsersRolesNotFound() { String name = "Test User4"; String email = "test4@test.com"; - String orcid = "0000-0000-0000-0000"; + String oAuthId = "0000-0000-0000-0000"; JsonObject requestBody = new JsonObject(); requestBody.addProperty("name", name); requestBody.addProperty("email", email); - requestBody.addProperty("orcid", orcid); + requestBody.addProperty("oauthId", oAuthId); JsonObject role = new JsonObject(); role.addProperty("id", invalidUUID); JsonArray roles = new JsonArray(); @@ -584,7 +584,7 @@ public void getUserInfoRegisteredUser() { JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(otherTestUser.getName(), result.get("name").getAsString(), "Wrong name"); - assertEquals(otherTestUser.getOauthId(), result.get("orcid").getAsString(), "Wrong orcid"); + assertEquals(otherTestUser.getOauthId(), result.get("oauthId").getAsString(), "Wrong orcid"); assertEquals(otherTestUser.getEmail(), result.get("email").getAsString(), "Wrong email"); assertEquals(otherTestUser.getId().toString(), result.get("id").getAsString(), "Wrong id"); diff --git a/src/test/java/org/breedinginsight/utilities/response/mappers/UserQueryMapperUnitTest.java b/src/test/java/org/breedinginsight/utilities/response/mappers/UserQueryMapperUnitTest.java index daedfa20a..50545644b 100644 --- a/src/test/java/org/breedinginsight/utilities/response/mappers/UserQueryMapperUnitTest.java +++ b/src/test/java/org/breedinginsight/utilities/response/mappers/UserQueryMapperUnitTest.java @@ -60,7 +60,7 @@ public void testMappings() { assertEquals(user.getName(), userQueryMapper.getField("name").apply(user), "Wrong getter"); assertEquals(user.getEmail(), userQueryMapper.getField("email").apply(user), "Wrong getter"); - assertEquals(user.getOauthId(), userQueryMapper.getField("orcid").apply(user), "Wrong getter"); + assertEquals(user.getOauthId(), userQueryMapper.getField("oauthId").apply(user), "Wrong getter"); assertEquals(user.getSystemRoles().stream().map(role -> role.getDomain()).collect(Collectors.toList()), userQueryMapper.getField("systemRoles").apply(user), "Wrong getter"); assertEquals(user.getProgramRoles().stream().map(role -> role.getProgram().getName()).collect(Collectors.toList()), From 05c3656eeba00a026f4f52682beb1968a85472e2 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 27 Jun 2025 12:21:04 -0400 Subject: [PATCH 111/289] fix validation error behavior --- .../api/model/v1/response/ValidationErrors.java | 12 ++++++++++++ .../brapps/importer/services/FileImportService.java | 7 +++++++ .../middleware/AppendOverwriteIDValidation.java | 5 ++++- .../middleware/process/ImportTableProcess.java | 1 + .../ObservationUnitIDFormatValidator.java | 5 ++++- .../experiment/model/ExpImportProcessConstants.java | 2 +- 6 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java index 7fdd58347..d809d72fc 100644 --- a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java +++ b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java @@ -49,6 +49,18 @@ public void addError(Integer rowNumber, ValidationError validationError){ rowErrors.add(newRow); } + public boolean hasErrorAtCell(int rowIndex, String field) { + for (RowValidationErrors row: rowErrors) { + if (row.getRowIndex() == rowIndex){ + return row.getErrors() + .stream() + .anyMatch(error -> error.getField().equals(field)); + } + } + + return false; + } + public void merge(ValidationErrors validationErrors){ for (RowValidationErrors rowValidationErrors: validationErrors.getRowErrors()){ for (ValidationError validationError: rowValidationErrors.getErrors()) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 039c0b58c..2af599402 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -458,6 +458,13 @@ private void processFile(String workflowId, List finalBrAPIImportLi progress.setBody(JSONB.valueOf(json)); progress.setUpdatedBy(actingUser.getId()); importDAO.update(upload); + } catch (BadRequestException e) { + log.error(e.getMessage(), e); + ImportProgress progress = upload.getProgress(); + progress.setStatuscode((short) HttpStatus.BAD_REQUEST.getCode()); + progress.setMessage(e.getMessage()); + progress.setUpdatedBy(actingUser.getId()); + importDAO.update(upload); } catch (Exception e) { if(e instanceof ApiException) { log.error("Error making BrAPI call: " + Utilities.generateApiExceptionLogMessage((ApiException) e), e); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java index a3adec8f1..2692a2eaa 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java @@ -65,12 +65,15 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext ouIdValidator.validateDynamicColumns(context); Set uniqueOUIds = ExperimentUtilities.collateUniqueOUIds(context); context.getAppendOverwriteWorkflowContext().setReferenceOUIds(uniqueOUIds); - brAPIObservationUnitReadWorkflowInitialization.execute(); // Fetch the obs units from the BrAPi service // Check for tabular errors collected during validation if (validationErrors.hasErrors()) { throw new ValidatorException(validationErrors); } + + // Fetch the obs units from the BrAPi service + brAPIObservationUnitReadWorkflowInitialization.execute(); + return processNext(context); } catch (EntityNotFoundException e) { /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index e655f2156..45eea09e1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -114,6 +114,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext Table data = context.getImportContext().getData(); List phenotypeColNames = Arrays.stream(upload.getDynamicColumnNames()) .filter(name -> !name.endsWith(OBSERVATION_UNIT_ID_SUFFIX)) + .filter(name -> !name.contains(SUB_UNIT_NUMBER)) .collect(Collectors.toList()); // don't allow periods (.) or square brackets in Phenotype Column Names diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java index 7ff63f36e..bf612a2a4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java @@ -54,7 +54,10 @@ public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws // Validate UUID format if (id == null || !UUID_PATTERN.matcher(id).matches()) { - ExperimentUtilities.addRowError(idColName, BITB.getValue(), rowErrors, rowNum); + if (!rowErrors.hasErrorAtCell(rowNum + 2, idColName)) { // take header row into account + // don't add another error for format if it already has an error for being blank + ExperimentUtilities.addRowError(idColName, BITB.getValue(), rowErrors, rowNum); + } } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index 32b880388..d4bc7f084 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -39,7 +39,7 @@ public enum ErrMessage { UNMATCHED_COLUMN("Ontology term(s) not found: "), OBS_UNIT_NOT_FOUND("Invalid ObsUnitID"), DUPLICATE_OBS_UNIT_ID("ObsUnitId is repeated"), - OZEX("Missing ObsUnitID column. Import cannot proceed"), + OZEX("Missing ObsUnitID column"), VVCN("ObsUnitID is duplicated"), BITB("Invalid or missing ObsUnitID"); From 1556ca36bfbd24a506b51ee1b413b87445a52a5e Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Fri, 27 Jun 2025 14:07:25 -0400 Subject: [PATCH 112/289] [BI-2540] - updated test SQL --- .../resources/sql/ExperimentalCollaboratorServiceTest.sql | 4 ++-- src/test/resources/sql/JobControllerIntegrationTest.sql | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/resources/sql/ExperimentalCollaboratorServiceTest.sql b/src/test/resources/sql/ExperimentalCollaboratorServiceTest.sql index b1b1a26c9..10f361cf4 100644 --- a/src/test/resources/sql/ExperimentalCollaboratorServiceTest.sql +++ b/src/test/resources/sql/ExperimentalCollaboratorServiceTest.sql @@ -17,8 +17,8 @@ */ -- name: CreateUser -INSERT INTO bi_user (id, orcid, name, email, created_by, updated_by, active) -VALUES ('594ec70e-0476-4c40-baf5-581ab0cfcd75', '0000-0001-2345-6789', 'Tester', 'tester@mailinator.com', '00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000000', true); +INSERT INTO bi_user (id, oauth_id, name, email, created_by, updated_by, active, oauth_provider) +VALUES ('594ec70e-0476-4c40-baf5-581ab0cfcd75', '0000-0001-2345-6789', 'Tester', 'tester@mailinator.com', '00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000000', true, 'orcid'); -- name: CreateProgram INSERT INTO program (id, species_id, name, created_by, updated_by, active, key, germplasm_sequence, exp_sequence, env_sequence) diff --git a/src/test/resources/sql/JobControllerIntegrationTest.sql b/src/test/resources/sql/JobControllerIntegrationTest.sql index 596c44bda..d238741eb 100644 --- a/src/test/resources/sql/JobControllerIntegrationTest.sql +++ b/src/test/resources/sql/JobControllerIntegrationTest.sql @@ -18,12 +18,12 @@ -- name: InsertJobs -INSERT INTO public.importer_progress (id, statuscode, message, body, total, finished, in_progress, created_at, updated_at, created_by, updated_by) VALUES ('d02090c3-e40d-45f4-9a63-b8c701c9531b', 202, 'Uploading to brapi service', null, 500, 100, 400, '2022-05-05 21:42:42 +00:00', '2022-05-05 21:42:42 +00:00', (select id from bi_user where orcid = '1111-2222-3333-4444'), (select id from bi_user where orcid = '1111-2222-3333-4444')); -INSERT INTO public.importer_progress (id, statuscode, message, body, total, finished, in_progress, created_at, updated_at, created_by, updated_by) VALUES ('066a37e4-c2e7-4d04-9175-7164c7dce70f', 200, 'Completed upload to brapi service', null, 17, 1, 0, '2022-05-17 14:19:33 +00:00', '2022-05-17 14:19:33 +00:00', (select id from bi_user where orcid = '1111-2222-3333-4444'), (select id from bi_user where orcid = '1111-2222-3333-4444')); +INSERT INTO public.importer_progress (id, statuscode, message, body, total, finished, in_progress, created_at, updated_at, created_by, updated_by) VALUES ('d02090c3-e40d-45f4-9a63-b8c701c9531b', 202, 'Uploading to brapi service', null, 500, 100, 400, '2022-05-05 21:42:42 +00:00', '2022-05-05 21:42:42 +00:00', (select id from bi_user where oauth_id = '1111-2222-3333-4444'), (select id from bi_user where oauth_id = '1111-2222-3333-4444')); +INSERT INTO public.importer_progress (id, statuscode, message, body, total, finished, in_progress, created_at, updated_at, created_by, updated_by) VALUES ('066a37e4-c2e7-4d04-9175-7164c7dce70f', 200, 'Completed upload to brapi service', null, 17, 1, 0, '2022-05-17 14:19:33 +00:00', '2022-05-17 14:19:33 +00:00', (select id from bi_user where oauth_id = '1111-2222-3333-4444'), (select id from bi_user where oauth_id = '1111-2222-3333-4444')); -INSERT INTO public.importer_import (id, program_id, user_id, importer_mapping_id, importer_progress_id, upload_file_name, file_data, modified_data, mapped_data, created_at, updated_at, created_by, updated_by) VALUES ('9aeeace2-7370-4b7d-a574-0ea799a342ef', ?::uuid, (select id from bi_user where orcid = '1111-2222-3333-4444'), (select id from importer_mapping where name = 'GermplasmTemplateMap'), '066a37e4-c2e7-4d04-9175-7164c7dce70f', 'germ import 2.xlsx', '[{"Name": "Gnome-1", "Source": "USDA", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-2", "Source": "USDA", "Entry No": "2", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3", "Source": "Breeding Insight", "Entry No": "3", "External UID": "", "Breeding Method": "BPC", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "2", "Female Parent Entry No": "1"}, {"Name": "Gnome-3-1", "Source": "Breeding Insight", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-2", "Source": "Breeding Insight", "Entry No": "5", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-3", "Source": "Breeding Insight", "Entry No": "6", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-4", "Source": "Breeding Insight", "Entry No": "7", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-5", "Source": "Breeding Insight", "Entry No": "8", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-6", "Source": "Breeding Insight", "Entry No": "9", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-7", "Source": "Breeding Insight", "Entry No": "10", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "11", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "9"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "12", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "3"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "13", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "7"}]', null, '{"rows": [{"germplasm": {"id": "1580b261-f504-4a7e-bbc7-e5599f0b2f4b", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-1 [MNTTIM-1]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "1"}, "commonCropName": "test", "accessionNumber": "1", "defaultDisplayName": "Gnome-1", "externalReferences": [{"referenceID": "6a1269cb-6e7a-4b87-a353-c681c214bd69", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "7e665eb0-2f7b-4b14-85a4-50f5508de2a8", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-2 [MNTTIM-2]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "2"}, "commonCropName": "test", "accessionNumber": "2", "defaultDisplayName": "Gnome-2", "externalReferences": [{"referenceID": "bcc3fab6-04ed-4e2f-b0b0-9e9b247462ef", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "eddb03ff-09c5-481d-9c1c-8fc609d432e5", "state": "NEW", "brAPIObject": {"pedigree": "Gnome-1 [MNTTIM-1]/Gnome-2 [MNTTIM-2]", "seedSource": "Breeding Insight", "germplasmName": "Gnome-3 [MNTTIM-3]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Biparental cross", "breedingMethodId": "aee4d3a3-0df8-422f-b052-d9f0b81e5b4d", "importEntryNumber": "3", "maleParentEntryNo": "2", "femaleParentEntryNo": "1"}, "commonCropName": "test", "accessionNumber": "3", "defaultDisplayName": "Gnome-3", "externalReferences": [{"referenceID": "24c3164e-0b37-44b4-8b18-9118ca9b2949", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "42ff9b3e-3f04-42f3-909b-2aaa158457e5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-1 [MNTTIM-4]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "4"}, "commonCropName": "test", "accessionNumber": "4", "defaultDisplayName": "Gnome-3-1", "externalReferences": [{"referenceID": "c271722b-488b-41c8-9671-73ac008be522", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "883cef8e-34c9-43f7-8062-d7d80c8a65fd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-2 [MNTTIM-5]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "5"}, "commonCropName": "test", "accessionNumber": "5", "defaultDisplayName": "Gnome-3-2", "externalReferences": [{"referenceID": "6bb014ce-0c5f-480c-a1e2-9b87a914a733", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "6215c013-2ed9-43c1-a964-15f87024da36", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-3 [MNTTIM-6]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "6"}, "commonCropName": "test", "accessionNumber": "6", "defaultDisplayName": "Gnome-3-3", "externalReferences": [{"referenceID": "988fe16a-6308-4146-9cae-be9d0cd9c295", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "bd62dec0-25d3-4bef-b9f7-5af634f78366", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-4 [MNTTIM-7]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "7"}, "commonCropName": "test", "accessionNumber": "7", "defaultDisplayName": "Gnome-3-4", "externalReferences": [{"referenceID": "28e6482a-2804-4aa8-b98d-6c4a56f8d8d2", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "c0067078-7a6d-4c99-b6ee-ab220feccddd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-5 [MNTTIM-8]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "8"}, "commonCropName": "test", "accessionNumber": "8", "defaultDisplayName": "Gnome-3-5", "externalReferences": [{"referenceID": "e7c25b55-61f5-4845-9080-d2243171f70b", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "cdb38aac-51fc-4ea7-9b38-8072258ceac5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-6 [MNTTIM-9]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "9"}, "commonCropName": "test", "accessionNumber": "9", "defaultDisplayName": "Gnome-3-6", "externalReferences": [{"referenceID": "ade6b8ca-b2e0-49cc-ac09-09476b1416e7", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "3f6c7d4b-fca2-4b22-aa6a-aadb347cd625", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-7 [MNTTIM-10]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "10"}, "commonCropName": "test", "accessionNumber": "10", "defaultDisplayName": "Gnome-3-7", "externalReferences": [{"referenceID": "6cd34ea3-86b0-41a2-aef0-57c101bd68fb", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "68a72f81-42b5-4425-8034-eef7d6921928", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-6 [MNTTIM-9]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-11]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "11", "maleParentEntryNo": "1", "femaleParentEntryNo": "9"}, "commonCropName": "test", "accessionNumber": "11", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "354faf95-682c-4a62-90f4-4f6c9c2d4191", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "25d84084-45a6-4d93-a683-004f95d3dc3e", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3 [MNTTIM-3]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-12]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "12", "maleParentEntryNo": "1", "femaleParentEntryNo": "3"}, "commonCropName": "test", "accessionNumber": "12", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "936a608f-0e2b-4cee-b348-dbbd57cbdf65", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "569fd3a0-1960-45e0-86d4-5157e21e8130", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-4 [MNTTIM-7]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-13]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "13", "maleParentEntryNo": "1", "femaleParentEntryNo": "7"}, "commonCropName": "test", "accessionNumber": "13", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "4e708554-e088-48ce-a16d-43f1778549fa", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}], "statistics": {"Germplasm": {"newObjectCount": 13}, "Pedigree Connections": {"newObjectCount": 4}}}', '2022-05-17 14:19:33 +00:00', '2022-05-17 14:19:33 +00:00', (select id from bi_user where orcid = '1111-2222-3333-4444'), (select id from bi_user where orcid = '1111-2222-3333-4444')); +INSERT INTO public.importer_import (id, program_id, user_id, importer_mapping_id, importer_progress_id, upload_file_name, file_data, modified_data, mapped_data, created_at, updated_at, created_by, updated_by) VALUES ('9aeeace2-7370-4b7d-a574-0ea799a342ef', ?::uuid, (select id from bi_user where oauth_id = '1111-2222-3333-4444'), (select id from importer_mapping where name = 'GermplasmTemplateMap'), '066a37e4-c2e7-4d04-9175-7164c7dce70f', 'germ import 2.xlsx', '[{"Name": "Gnome-1", "Source": "USDA", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-2", "Source": "USDA", "Entry No": "2", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3", "Source": "Breeding Insight", "Entry No": "3", "External UID": "", "Breeding Method": "BPC", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "2", "Female Parent Entry No": "1"}, {"Name": "Gnome-3-1", "Source": "Breeding Insight", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-2", "Source": "Breeding Insight", "Entry No": "5", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-3", "Source": "Breeding Insight", "Entry No": "6", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-4", "Source": "Breeding Insight", "Entry No": "7", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-5", "Source": "Breeding Insight", "Entry No": "8", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-6", "Source": "Breeding Insight", "Entry No": "9", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-7", "Source": "Breeding Insight", "Entry No": "10", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "11", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "9"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "12", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "3"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "13", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "7"}]', null, '{"rows": [{"germplasm": {"id": "1580b261-f504-4a7e-bbc7-e5599f0b2f4b", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-1 [MNTTIM-1]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "1"}, "commonCropName": "test", "accessionNumber": "1", "defaultDisplayName": "Gnome-1", "externalReferences": [{"referenceID": "6a1269cb-6e7a-4b87-a353-c681c214bd69", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "7e665eb0-2f7b-4b14-85a4-50f5508de2a8", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-2 [MNTTIM-2]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "2"}, "commonCropName": "test", "accessionNumber": "2", "defaultDisplayName": "Gnome-2", "externalReferences": [{"referenceID": "bcc3fab6-04ed-4e2f-b0b0-9e9b247462ef", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "eddb03ff-09c5-481d-9c1c-8fc609d432e5", "state": "NEW", "brAPIObject": {"pedigree": "Gnome-1 [MNTTIM-1]/Gnome-2 [MNTTIM-2]", "seedSource": "Breeding Insight", "germplasmName": "Gnome-3 [MNTTIM-3]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Biparental cross", "breedingMethodId": "aee4d3a3-0df8-422f-b052-d9f0b81e5b4d", "importEntryNumber": "3", "maleParentEntryNo": "2", "femaleParentEntryNo": "1"}, "commonCropName": "test", "accessionNumber": "3", "defaultDisplayName": "Gnome-3", "externalReferences": [{"referenceID": "24c3164e-0b37-44b4-8b18-9118ca9b2949", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "42ff9b3e-3f04-42f3-909b-2aaa158457e5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-1 [MNTTIM-4]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "4"}, "commonCropName": "test", "accessionNumber": "4", "defaultDisplayName": "Gnome-3-1", "externalReferences": [{"referenceID": "c271722b-488b-41c8-9671-73ac008be522", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "883cef8e-34c9-43f7-8062-d7d80c8a65fd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-2 [MNTTIM-5]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "5"}, "commonCropName": "test", "accessionNumber": "5", "defaultDisplayName": "Gnome-3-2", "externalReferences": [{"referenceID": "6bb014ce-0c5f-480c-a1e2-9b87a914a733", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "6215c013-2ed9-43c1-a964-15f87024da36", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-3 [MNTTIM-6]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "6"}, "commonCropName": "test", "accessionNumber": "6", "defaultDisplayName": "Gnome-3-3", "externalReferences": [{"referenceID": "988fe16a-6308-4146-9cae-be9d0cd9c295", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "bd62dec0-25d3-4bef-b9f7-5af634f78366", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-4 [MNTTIM-7]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "7"}, "commonCropName": "test", "accessionNumber": "7", "defaultDisplayName": "Gnome-3-4", "externalReferences": [{"referenceID": "28e6482a-2804-4aa8-b98d-6c4a56f8d8d2", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "c0067078-7a6d-4c99-b6ee-ab220feccddd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-5 [MNTTIM-8]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "8"}, "commonCropName": "test", "accessionNumber": "8", "defaultDisplayName": "Gnome-3-5", "externalReferences": [{"referenceID": "e7c25b55-61f5-4845-9080-d2243171f70b", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "cdb38aac-51fc-4ea7-9b38-8072258ceac5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-6 [MNTTIM-9]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "9"}, "commonCropName": "test", "accessionNumber": "9", "defaultDisplayName": "Gnome-3-6", "externalReferences": [{"referenceID": "ade6b8ca-b2e0-49cc-ac09-09476b1416e7", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "3f6c7d4b-fca2-4b22-aa6a-aadb347cd625", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-7 [MNTTIM-10]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "10"}, "commonCropName": "test", "accessionNumber": "10", "defaultDisplayName": "Gnome-3-7", "externalReferences": [{"referenceID": "6cd34ea3-86b0-41a2-aef0-57c101bd68fb", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "68a72f81-42b5-4425-8034-eef7d6921928", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-6 [MNTTIM-9]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-11]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "11", "maleParentEntryNo": "1", "femaleParentEntryNo": "9"}, "commonCropName": "test", "accessionNumber": "11", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "354faf95-682c-4a62-90f4-4f6c9c2d4191", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "25d84084-45a6-4d93-a683-004f95d3dc3e", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3 [MNTTIM-3]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-12]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "12", "maleParentEntryNo": "1", "femaleParentEntryNo": "3"}, "commonCropName": "test", "accessionNumber": "12", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "936a608f-0e2b-4cee-b348-dbbd57cbdf65", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "569fd3a0-1960-45e0-86d4-5157e21e8130", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-4 [MNTTIM-7]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-13]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "13", "maleParentEntryNo": "1", "femaleParentEntryNo": "7"}, "commonCropName": "test", "accessionNumber": "13", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "4e708554-e088-48ce-a16d-43f1778549fa", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}], "statistics": {"Germplasm": {"newObjectCount": 13}, "Pedigree Connections": {"newObjectCount": 4}}}', '2022-05-17 14:19:33 +00:00', '2022-05-17 14:19:33 +00:00', (select id from bi_user where oauth_id = '1111-2222-3333-4444'), (select id from bi_user where oauth_id = '1111-2222-3333-4444')); -INSERT INTO public.importer_import (id, program_id, user_id, importer_mapping_id, importer_progress_id, upload_file_name, file_data, modified_data, mapped_data, created_at, updated_at, created_by, updated_by) VALUES ('210c7175-9d7a-454c-b35f-c8aa0ee0a31e', ?::uuid, (select id from bi_user where orcid = '1111-2222-3333-4444'), (select id from importer_mapping where name = 'GermplasmTemplateMap'), 'd02090c3-e40d-45f4-9a63-b8c701c9531b', 'germ import 2.xlsx', '[{"Name": "Gnome-1", "Source": "USDA", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-2", "Source": "USDA", "Entry No": "2", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3", "Source": "Breeding Insight", "Entry No": "3", "External UID": "", "Breeding Method": "BPC", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "2", "Female Parent Entry No": "1"}, {"Name": "Gnome-3-1", "Source": "Breeding Insight", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-2", "Source": "Breeding Insight", "Entry No": "5", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-3", "Source": "Breeding Insight", "Entry No": "6", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-4", "Source": "Breeding Insight", "Entry No": "7", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-5", "Source": "Breeding Insight", "Entry No": "8", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-6", "Source": "Breeding Insight", "Entry No": "9", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-7", "Source": "Breeding Insight", "Entry No": "10", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "11", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "9"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "12", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "3"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "13", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "7"}]', null, '{"rows": [{"germplasm": {"id": "1580b261-f504-4a7e-bbc7-e5599f0b2f4b", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-1 [MNTTIM-1]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "1"}, "commonCropName": "test", "accessionNumber": "1", "defaultDisplayName": "Gnome-1", "externalReferences": [{"referenceID": "6a1269cb-6e7a-4b87-a353-c681c214bd69", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "7e665eb0-2f7b-4b14-85a4-50f5508de2a8", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-2 [MNTTIM-2]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "2"}, "commonCropName": "test", "accessionNumber": "2", "defaultDisplayName": "Gnome-2", "externalReferences": [{"referenceID": "bcc3fab6-04ed-4e2f-b0b0-9e9b247462ef", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "eddb03ff-09c5-481d-9c1c-8fc609d432e5", "state": "NEW", "brAPIObject": {"pedigree": "Gnome-1 [MNTTIM-1]/Gnome-2 [MNTTIM-2]", "seedSource": "Breeding Insight", "germplasmName": "Gnome-3 [MNTTIM-3]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Biparental cross", "breedingMethodId": "aee4d3a3-0df8-422f-b052-d9f0b81e5b4d", "importEntryNumber": "3", "maleParentEntryNo": "2", "femaleParentEntryNo": "1"}, "commonCropName": "test", "accessionNumber": "3", "defaultDisplayName": "Gnome-3", "externalReferences": [{"referenceID": "24c3164e-0b37-44b4-8b18-9118ca9b2949", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "42ff9b3e-3f04-42f3-909b-2aaa158457e5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-1 [MNTTIM-4]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "4"}, "commonCropName": "test", "accessionNumber": "4", "defaultDisplayName": "Gnome-3-1", "externalReferences": [{"referenceID": "c271722b-488b-41c8-9671-73ac008be522", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "883cef8e-34c9-43f7-8062-d7d80c8a65fd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-2 [MNTTIM-5]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "5"}, "commonCropName": "test", "accessionNumber": "5", "defaultDisplayName": "Gnome-3-2", "externalReferences": [{"referenceID": "6bb014ce-0c5f-480c-a1e2-9b87a914a733", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "6215c013-2ed9-43c1-a964-15f87024da36", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-3 [MNTTIM-6]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "6"}, "commonCropName": "test", "accessionNumber": "6", "defaultDisplayName": "Gnome-3-3", "externalReferences": [{"referenceID": "988fe16a-6308-4146-9cae-be9d0cd9c295", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "bd62dec0-25d3-4bef-b9f7-5af634f78366", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-4 [MNTTIM-7]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "7"}, "commonCropName": "test", "accessionNumber": "7", "defaultDisplayName": "Gnome-3-4", "externalReferences": [{"referenceID": "28e6482a-2804-4aa8-b98d-6c4a56f8d8d2", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "c0067078-7a6d-4c99-b6ee-ab220feccddd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-5 [MNTTIM-8]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "8"}, "commonCropName": "test", "accessionNumber": "8", "defaultDisplayName": "Gnome-3-5", "externalReferences": [{"referenceID": "e7c25b55-61f5-4845-9080-d2243171f70b", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "cdb38aac-51fc-4ea7-9b38-8072258ceac5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-6 [MNTTIM-9]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "9"}, "commonCropName": "test", "accessionNumber": "9", "defaultDisplayName": "Gnome-3-6", "externalReferences": [{"referenceID": "ade6b8ca-b2e0-49cc-ac09-09476b1416e7", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "3f6c7d4b-fca2-4b22-aa6a-aadb347cd625", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-7 [MNTTIM-10]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "10"}, "commonCropName": "test", "accessionNumber": "10", "defaultDisplayName": "Gnome-3-7", "externalReferences": [{"referenceID": "6cd34ea3-86b0-41a2-aef0-57c101bd68fb", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "68a72f81-42b5-4425-8034-eef7d6921928", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-6 [MNTTIM-9]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-11]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "11", "maleParentEntryNo": "1", "femaleParentEntryNo": "9"}, "commonCropName": "test", "accessionNumber": "11", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "354faf95-682c-4a62-90f4-4f6c9c2d4191", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "25d84084-45a6-4d93-a683-004f95d3dc3e", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3 [MNTTIM-3]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-12]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "12", "maleParentEntryNo": "1", "femaleParentEntryNo": "3"}, "commonCropName": "test", "accessionNumber": "12", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "936a608f-0e2b-4cee-b348-dbbd57cbdf65", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "569fd3a0-1960-45e0-86d4-5157e21e8130", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-4 [MNTTIM-7]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-13]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "13", "maleParentEntryNo": "1", "femaleParentEntryNo": "7"}, "commonCropName": "test", "accessionNumber": "13", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "4e708554-e088-48ce-a16d-43f1778549fa", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}], "statistics": {"Germplasm": {"newObjectCount": 13}, "Pedigree Connections": {"newObjectCount": 4}}}', '2022-05-18 14:19:33 +00:00', '2022-05-18 14:19:33 +00:00', (select id from bi_user where orcid = '1111-2222-3333-4444'), (select id from bi_user where orcid = '1111-2222-3333-4444')); +INSERT INTO public.importer_import (id, program_id, user_id, importer_mapping_id, importer_progress_id, upload_file_name, file_data, modified_data, mapped_data, created_at, updated_at, created_by, updated_by) VALUES ('210c7175-9d7a-454c-b35f-c8aa0ee0a31e', ?::uuid, (select id from bi_user where oauth_id = '1111-2222-3333-4444'), (select id from importer_mapping where name = 'GermplasmTemplateMap'), 'd02090c3-e40d-45f4-9a63-b8c701c9531b', 'germ import 2.xlsx', '[{"Name": "Gnome-1", "Source": "USDA", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-2", "Source": "USDA", "Entry No": "2", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3", "Source": "Breeding Insight", "Entry No": "3", "External UID": "", "Breeding Method": "BPC", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "2", "Female Parent Entry No": "1"}, {"Name": "Gnome-3-1", "Source": "Breeding Insight", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-2", "Source": "Breeding Insight", "Entry No": "5", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-3", "Source": "Breeding Insight", "Entry No": "6", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-4", "Source": "Breeding Insight", "Entry No": "7", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-5", "Source": "Breeding Insight", "Entry No": "8", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-6", "Source": "Breeding Insight", "Entry No": "9", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-7", "Source": "Breeding Insight", "Entry No": "10", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "11", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "9"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "12", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "3"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "13", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "7"}]', null, '{"rows": [{"germplasm": {"id": "1580b261-f504-4a7e-bbc7-e5599f0b2f4b", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-1 [MNTTIM-1]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "1"}, "commonCropName": "test", "accessionNumber": "1", "defaultDisplayName": "Gnome-1", "externalReferences": [{"referenceID": "6a1269cb-6e7a-4b87-a353-c681c214bd69", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "7e665eb0-2f7b-4b14-85a4-50f5508de2a8", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-2 [MNTTIM-2]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "2"}, "commonCropName": "test", "accessionNumber": "2", "defaultDisplayName": "Gnome-2", "externalReferences": [{"referenceID": "bcc3fab6-04ed-4e2f-b0b0-9e9b247462ef", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "eddb03ff-09c5-481d-9c1c-8fc609d432e5", "state": "NEW", "brAPIObject": {"pedigree": "Gnome-1 [MNTTIM-1]/Gnome-2 [MNTTIM-2]", "seedSource": "Breeding Insight", "germplasmName": "Gnome-3 [MNTTIM-3]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Biparental cross", "breedingMethodId": "aee4d3a3-0df8-422f-b052-d9f0b81e5b4d", "importEntryNumber": "3", "maleParentEntryNo": "2", "femaleParentEntryNo": "1"}, "commonCropName": "test", "accessionNumber": "3", "defaultDisplayName": "Gnome-3", "externalReferences": [{"referenceID": "24c3164e-0b37-44b4-8b18-9118ca9b2949", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "42ff9b3e-3f04-42f3-909b-2aaa158457e5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-1 [MNTTIM-4]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "4"}, "commonCropName": "test", "accessionNumber": "4", "defaultDisplayName": "Gnome-3-1", "externalReferences": [{"referenceID": "c271722b-488b-41c8-9671-73ac008be522", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "883cef8e-34c9-43f7-8062-d7d80c8a65fd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-2 [MNTTIM-5]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "5"}, "commonCropName": "test", "accessionNumber": "5", "defaultDisplayName": "Gnome-3-2", "externalReferences": [{"referenceID": "6bb014ce-0c5f-480c-a1e2-9b87a914a733", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "6215c013-2ed9-43c1-a964-15f87024da36", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-3 [MNTTIM-6]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "6"}, "commonCropName": "test", "accessionNumber": "6", "defaultDisplayName": "Gnome-3-3", "externalReferences": [{"referenceID": "988fe16a-6308-4146-9cae-be9d0cd9c295", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "bd62dec0-25d3-4bef-b9f7-5af634f78366", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-4 [MNTTIM-7]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "7"}, "commonCropName": "test", "accessionNumber": "7", "defaultDisplayName": "Gnome-3-4", "externalReferences": [{"referenceID": "28e6482a-2804-4aa8-b98d-6c4a56f8d8d2", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "c0067078-7a6d-4c99-b6ee-ab220feccddd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-5 [MNTTIM-8]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "8"}, "commonCropName": "test", "accessionNumber": "8", "defaultDisplayName": "Gnome-3-5", "externalReferences": [{"referenceID": "e7c25b55-61f5-4845-9080-d2243171f70b", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "cdb38aac-51fc-4ea7-9b38-8072258ceac5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-6 [MNTTIM-9]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "9"}, "commonCropName": "test", "accessionNumber": "9", "defaultDisplayName": "Gnome-3-6", "externalReferences": [{"referenceID": "ade6b8ca-b2e0-49cc-ac09-09476b1416e7", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "3f6c7d4b-fca2-4b22-aa6a-aadb347cd625", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-7 [MNTTIM-10]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "10"}, "commonCropName": "test", "accessionNumber": "10", "defaultDisplayName": "Gnome-3-7", "externalReferences": [{"referenceID": "6cd34ea3-86b0-41a2-aef0-57c101bd68fb", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "68a72f81-42b5-4425-8034-eef7d6921928", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-6 [MNTTIM-9]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-11]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "11", "maleParentEntryNo": "1", "femaleParentEntryNo": "9"}, "commonCropName": "test", "accessionNumber": "11", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "354faf95-682c-4a62-90f4-4f6c9c2d4191", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "25d84084-45a6-4d93-a683-004f95d3dc3e", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3 [MNTTIM-3]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-12]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "12", "maleParentEntryNo": "1", "femaleParentEntryNo": "3"}, "commonCropName": "test", "accessionNumber": "12", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "936a608f-0e2b-4cee-b348-dbbd57cbdf65", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "569fd3a0-1960-45e0-86d4-5157e21e8130", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-4 [MNTTIM-7]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-13]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "13", "maleParentEntryNo": "1", "femaleParentEntryNo": "7"}, "commonCropName": "test", "accessionNumber": "13", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "4e708554-e088-48ce-a16d-43f1778549fa", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}], "statistics": {"Germplasm": {"newObjectCount": 13}, "Pedigree Connections": {"newObjectCount": 4}}}', '2022-05-18 14:19:33 +00:00', '2022-05-18 14:19:33 +00:00', (select id from bi_user where oauth_id = '1111-2222-3333-4444'), (select id from bi_user where oauth_id = '1111-2222-3333-4444')); From 7305b8fbf746a7522d8d735b104f9214d86afcb6 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 27 Jun 2025 21:57:14 +0000 Subject: [PATCH 113/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 56c13f8c2..3a2ba301c 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1001 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c39e002e66c44f749fccb35b7ac8e9bf3e2cd9dc +version=v1.2.0+1011 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2700fde850e28247dffd20ea101ab6158415fe0e From 5af44b98bbb710550d9ba8b2beedee40acb3e1c1 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 30 Jun 2025 16:20:37 -0400 Subject: [PATCH 114/289] add test for appending obs data using prior variables --- ...leSubmissionControllerIntegrationTest.java | 2 +- .../v2/ListControllerIntegrationTest.java | 2 +- .../importer/ExperimentFileImportTest.java | 200 ++++++++++++++---- .../brapps/importer/ImportTestUtils.java | 43 +++- .../SampleSubmissionFileImportTest.java | 2 +- 5 files changed, 196 insertions(+), 53 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java index 978bb0aa8..44a6e3425 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -407,7 +407,7 @@ private String createExperiment(Program program) throws IOException, Interrupted .get(0).getAsJsonObject().get("id").getAsString(); JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( - importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null, false), + importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null, false, false, null), null, true, client, diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index ca6e5c2e9..b5ee8ce0d 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -148,7 +148,7 @@ public void setup() { newExp.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetchWorkflow( - importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); } @Test diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 8275e1cea..45631d558 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -48,6 +48,7 @@ import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; +import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; @@ -63,6 +64,8 @@ import org.breedinginsight.services.exceptions.BadRequestException; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.utilities.DatasetUtil; +import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import org.junit.jupiter.api.*; @@ -70,8 +73,10 @@ import org.junit.jupiter.params.provider.ValueSource; import org.junit.platform.commons.util.StringUtils; import org.opentest4j.AssertionFailedError; +import tech.tablesaw.api.Table; import javax.inject.Inject; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.time.OffsetDateTime; @@ -80,6 +85,7 @@ import java.util.stream.StreamSupport; import static io.micronaut.http.HttpRequest.GET; +import static io.micronaut.http.HttpRequest.POST; import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -93,6 +99,7 @@ public class ExperimentFileImportTest extends BrAPITest { private FannyPack securityFp; private String mappingId; private BiUserEntity testUser; + private Program program; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -136,6 +143,9 @@ public class ExperimentFileImportTest extends BrAPITest { @Inject private ProgramLocationService locationService; + @Inject + private BrAPITrialService experimentService; + @Inject private BrAPIGermplasmDAO germplasmDAO; @@ -160,6 +170,7 @@ public void setup() { mappingId = (String) setupObjects.get("mappingId"); testUser = (BiUserEntity) setupObjects.get("testUser"); securityFp = (FannyPack) setupObjects.get("securityFp"); + program = (Program) setupObjects.get("program"); newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); appendOverwriteWorkflowId = importTestUtils.getExperimentWorkflowId(client, 1); } @@ -180,6 +191,115 @@ public void setup() { - existing env that already has observation variables (existing dataset) */ + @Test + @SneakyThrows + //@Disabled + public void appendExperimentWithObsVarFromPriorDataset() { + log.debug("appendExperimentWithObsVarFromPriorDataset"); + + // Create a plot-level dataset that includes observation variable tt_test_1 + List traits = importTestUtils.createTraits(1); + Program program = createProgram("Append Exp with Prior Observations Vars", "EXPPRI", "EXPPRI", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), null); + + JsonObject importResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); + String expId = importResponse + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + + // Create sub-entity dataset that has two plant-level units but no obsvars and, therefore, no observations. + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", + program.getId().toString(), expId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class); + HttpResponse postResponse = postCall.blockingFirst(); + + // Assert 200 response + assertEquals(HttpStatus.OK, postResponse.getStatus()); + + // Grab the system ids for the sub-entity dataset units + JsonObject result = JsonParser.parseString(postResponse.body()).getAsJsonObject().getAsJsonObject("result"); + + // Export the plant-level sub-entity dataset + String extension = "CSV"; + BrAPITrial experiment = experimentService.getTrialDataByUUID(program.getId(), UUID.fromString(expId), false); + String plantDatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), "Plant"); + Flowable> plantExportCall = client.exchange( + GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", + program.getId().toString(), expId, extension, plantDatasetId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse plantResponse = plantExportCall.blockingFirst(); + + // Make sub-entity dataset export request. + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(plantResponse.body())); + Table exportTable = FileUtil.parseTableFromCsv(bodyStream); + + // Assert 200 response + assertEquals(HttpStatus.OK, plantResponse.getStatus()); + + // Build a request to append tt_test_1 observation data on the sub-entity dataset + String sub1ObsUnitId = exportTable.row(0).getString("Plant ObsUnitID"); + String sub2ObsUnitId = exportTable.row(1).getString("Plant ObsUnitID"); + + Map sub1 = new HashMap<>(); + sub1.put(Columns.GERMPLASM_GID, "1"); + sub1.put(Columns.TEST_CHECK, "T"); + sub1.put(Columns.EXP_TITLE, "Test Exp"); + sub1.put(Columns.EXP_UNIT, "Plot"); + sub1.put(Columns.EXP_TYPE, "Phenotyping"); + sub1.put(Columns.ENV, "New Env"); + sub1.put(Columns.ENV_LOCATION, "Location A"); + sub1.put(Columns.ENV_YEAR, "2023"); + sub1.put(Columns.EXP_UNIT_ID, "a-1"); + sub1.put(Columns.REP_NUM, "1"); + sub1.put(Columns.BLOCK_NUM, "1"); + sub1.put(Columns.ROW, "1"); + sub1.put(Columns.COLUMN, "1"); + sub1.put("Plant " + OBSERVATION_UNIT_ID_SUFFIX, sub1ObsUnitId); + sub1.put(traits.get(0).getObservationVariableName(), "1"); + + Map sub2 = new HashMap<>(); + sub2.put(Columns.GERMPLASM_GID, "1"); + sub2.put(Columns.TEST_CHECK, "T"); + sub2.put(Columns.EXP_TITLE, "Test Exp"); + sub2.put(Columns.EXP_UNIT, "Plot"); + sub2.put(Columns.EXP_TYPE, "Phenotyping"); + sub2.put(Columns.ENV, "New Env"); + sub2.put(Columns.ENV_LOCATION, "Location A"); + sub2.put(Columns.ENV_YEAR, "2023"); + sub2.put(Columns.EXP_UNIT_ID, "a-1"); + sub2.put(Columns.REP_NUM, "1"); + sub2.put(Columns.BLOCK_NUM, "1"); + sub2.put(Columns.ROW, "1"); + sub2.put(Columns.COLUMN, "1"); + sub2.put("Plant " + OBSERVATION_UNIT_ID_SUFFIX, sub2ObsUnitId); + sub2.put(traits.get(0).getObservationVariableName(), "2"); + + // Verify that the validation check returns a 400-level response since tt_test_1 is already used in the plot-level dataset + JsonObject previewResponse = importTestUtils.uploadAndFetchWorkflowPreview(importTestUtils.writeExperimentDataToFile(List.of(sub1, sub2), null, true, false, "Plant"), null, true, client, program, mappingId, appendOverwriteWorkflowId); + assertEquals(422, previewResponse.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + } + @Test @SneakyThrows public void importNewExpNewLocNoObsSuccess() { @@ -202,7 +322,7 @@ public void importNewExpNewLocNoObsSuccess() { validRow.put(Columns.COLUMN, "1"); validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(validRow), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(validRow), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -254,7 +374,7 @@ public void importNewExpMultiNewEnvSuccess() { secondEnv.put(Columns.COLUMN, "1"); secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(2, previewRows.size()); @@ -295,7 +415,7 @@ public void importExistingExpAndEnvErrorMessage() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); Map dupExp = new HashMap<>(); dupExp.put(Columns.GERMPLASM_GID, "1"); @@ -312,7 +432,7 @@ public void importExistingExpAndEnvErrorMessage() { dupExp.put(Columns.ROW, "1"); dupExp.put(Columns.COLUMN, "1"); - JsonObject expResult = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject expResult = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); assertEquals(422, expResult.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + expResult); assertTrue(expResult.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); @@ -339,7 +459,7 @@ public void importNewEnvNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); - JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -375,52 +495,52 @@ public void verifyMissingDataThrowsError(boolean commit) { Map noGID = new HashMap<>(base); noGID.remove(Columns.GERMPLASM_GID); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null, false), Columns.GERMPLASM_GID, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null, false, false, null), Columns.GERMPLASM_GID, commit, newExperimentWorkflowId); Map noExpTitle = new HashMap<>(base); noExpTitle.remove(Columns.EXP_TITLE); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null, false), Columns.EXP_TITLE, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null, false, false, null), Columns.EXP_TITLE, commit, newExperimentWorkflowId); Map noExpUnit = new HashMap<>(base); noExpUnit.remove(Columns.EXP_UNIT); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null, false), Columns.EXP_UNIT, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null, false, false, null), Columns.EXP_UNIT, commit, newExperimentWorkflowId); Map noExpType = new HashMap<>(base); noExpType.remove(Columns.EXP_TYPE); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null, false), Columns.EXP_TYPE, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null, false, false, null), Columns.EXP_TYPE, commit, newExperimentWorkflowId); Map noEnv = new HashMap<>(base); noEnv.remove(Columns.ENV); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null, false), Columns.ENV, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null, false, false, null), Columns.ENV, commit, newExperimentWorkflowId); Map noEnvLoc = new HashMap<>(base); noEnvLoc.remove(Columns.ENV_LOCATION); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null, false), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null, false, false, null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); Map noExpUnitId = new HashMap<>(base); noExpUnitId.remove(Columns.EXP_UNIT_ID); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null, false), Columns.EXP_UNIT_ID, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null, false, false, null), Columns.EXP_UNIT_ID, commit, newExperimentWorkflowId); Map noExpRep = new HashMap<>(base); noExpRep.remove(Columns.REP_NUM); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null, false), Columns.REP_NUM, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null, false, false, null), Columns.REP_NUM, commit, newExperimentWorkflowId); Map noExpBlock = new HashMap<>(base); noExpBlock.remove(Columns.BLOCK_NUM); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null, false), Columns.BLOCK_NUM, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null, false, false, null), Columns.BLOCK_NUM, commit, newExperimentWorkflowId); Map noEnvYear = new HashMap<>(base); noEnvYear.remove(Columns.ENV_YEAR); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null, false), Columns.ENV_YEAR, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null, false, false, null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); } @Test @@ -445,7 +565,7 @@ public void importNewExpWithObsVar() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -497,7 +617,7 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null, false), Columns.ENV_YEAR, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null, false, false, null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); } @@ -537,7 +657,7 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null, false), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null, false, false, null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); } @ParameterizedTest @@ -563,7 +683,7 @@ public void importNewExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -604,7 +724,7 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "Red"); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); } @@ -629,14 +749,14 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); Map newOU = new HashMap<>(newExp); newOU.put(Columns.EXP_UNIT_ID, "a-2"); newOU.put(Columns.ROW, "1"); newOU.put(Columns.COLUMN, "2"); - JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(newOU), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(newOU), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); @@ -664,7 +784,7 @@ public void importNewObsVarExistingOu() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -692,7 +812,7 @@ public void importNewObsVarExistingOu() { newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true), null, true, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), null, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -730,7 +850,7 @@ public void importNewObsVarByObsUnitId() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -745,7 +865,7 @@ public void importNewObsVarByObsUnitId() { newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true), null, true, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), null, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -784,7 +904,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); // empty dataset - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -812,7 +932,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true), null, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -860,7 +980,7 @@ public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); // Valid observation value. - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); // Fetch the ObsUnitId to use in the overwrite upload. BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); @@ -894,7 +1014,7 @@ public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { requestBody.put("overwrite", "true"); requestBody.put("overwriteReason", "testing"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true), requestBody, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), requestBody, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -930,7 +1050,7 @@ public void importNewObsExistingOu(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -958,7 +1078,7 @@ public void importNewObsExistingOu(boolean commit) { newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true), null, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -998,7 +1118,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1026,7 +1146,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); - uploadAndVerifyWorkflowFailureNonTabular(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailureNonTabular(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); } /* @@ -1057,7 +1177,7 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsA.size()); @@ -1085,7 +1205,7 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsB.size()); @@ -1127,7 +1247,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1156,7 +1276,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true), null, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1204,7 +1324,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newExp.put(traits.get(0).getObservationVariableName(), originalValue); newExp.put(traits.get(1).getObservationVariableName(), "2"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1240,7 +1360,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { Map requestBody = new HashMap<>(); requestBody.put("overwrite", "true"); requestBody.put("overwriteReason", "testing"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true), requestBody, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), requestBody, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index eb4030150..2562a9a66 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -240,16 +240,34 @@ public JsonObject uploadAndFetchWorkflow(File file, Program program, String mappingId, String workflowId) throws InterruptedException { + JsonObject result = generatePreview(file,userData, commit, client, program, mappingId, workflowId); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + return result; + } + + public JsonObject uploadAndFetchWorkflowPreview(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + return generatePreview(file, userData, commit, client, program, mappingId, workflowId); + } + + private JsonObject generatePreview(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - return result; + + return JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); } public JsonObject uploadAndFetchWorkflowNoStatusCheck(File file, @@ -297,7 +315,8 @@ public List createTraits(int numToCreate) { return traits; } - public File writeExperimentDataToFile(List> data, List traits, boolean ObsUnitIDCol) throws IOException { + public File writeExperimentDataToFile(List> data, List traits, boolean ObsUnitIDCol, boolean isSubEntity, String level) throws IOException { + String obsLevel = level == null ? "Plot" : level; File file = File.createTempFile("test", ".csv"); List columns = new ArrayList<>(); @@ -307,13 +326,17 @@ public File writeExperimentDataToFile(List> data, List> data, List Date: Tue, 1 Jul 2025 10:32:46 -0400 Subject: [PATCH 115/289] [BI-2540] - bumped migration version to avoid conflict --- .../{V1.33.0__rename-orcid.sql => V1.34.0__rename-orcid.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V1.33.0__rename-orcid.sql => V1.34.0__rename-orcid.sql} (100%) diff --git a/src/main/resources/db/migration/V1.33.0__rename-orcid.sql b/src/main/resources/db/migration/V1.34.0__rename-orcid.sql similarity index 100% rename from src/main/resources/db/migration/V1.33.0__rename-orcid.sql rename to src/main/resources/db/migration/V1.34.0__rename-orcid.sql From a8bd40d1f0ec0541fdf5da3e58a730c2d8ba2fa4 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 1 Jul 2025 14:39:19 -0400 Subject: [PATCH 116/289] [BI-2540] - added GitHub OAuth variables to docker-compose.yml --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index efebf5fb6..f45227ea7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,8 @@ services: - API_INTERNAL_PORT=${API_INTERNAL_PORT} - API_INTERNAL_TEST_PORT=${API_INTERNAL_TEST_PORT} - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID} + - GITHUB_OAUTH_CLIENT_ID=${GITHUB_OAUTH_CLIENT_ID} + - GITHUB_OAUTH_CLIENT_SECRET=${GITHUB_OAUTH_CLIENT_SECRET} - JWT_DOMAIN=${JWT_DOMAIN} - DB_SERVER=${DB_SERVER} - DB_NAME=${DB_NAME} From db262d3eb30d0600279aff11c6fde742ccd190b4 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 1 Jul 2025 14:42:18 -0400 Subject: [PATCH 117/289] [BI-2540] - added (placeholder) GitHub OAuth variables to build.yml --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0f77a41e0..0e1b29b66 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,4 +59,6 @@ jobs: JWT_SECRET: ${{ secrets.JWT_SECRET }} OAUTH_CLIENT_ID: 123abc OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf + GITHUB_OAUTH_CLIENT_ID: 12345678901234567890 + GITHUB_OAUTH_CLIENT_SECRET: 1234567890123456789012345678901234567890 BRAPI_REFERENCE_SOURCE: breedinginsight.org \ No newline at end of file From 995d57133690172fbf1a36f5ab7523e2489c1442 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 1 Jul 2025 16:00:49 -0400 Subject: [PATCH 118/289] [BI-2540] - renamed constraint --- src/main/resources/db/migration/V1.34.0__rename-orcid.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/resources/db/migration/V1.34.0__rename-orcid.sql b/src/main/resources/db/migration/V1.34.0__rename-orcid.sql index 928a426fa..5c559237c 100644 --- a/src/main/resources/db/migration/V1.34.0__rename-orcid.sql +++ b/src/main/resources/db/migration/V1.34.0__rename-orcid.sql @@ -19,6 +19,10 @@ ALTER TABLE bi_user RENAME COLUMN orcid TO oauth_id; +-- Rename unique constraint. +ALTER TABLE bi_user +RENAME CONSTRAINT orcid_unique TO oauth_id_unique; + -- Add a column to store the OAuth provider with 'orcid' as the default value. ALTER TABLE bi_user ADD COLUMN oauth_provider text DEFAULT 'orcid'; From b053edffa5f933cd4f9b0ebec8ed41daf27ea7b9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 2 Jul 2025 17:14:49 -0400 Subject: [PATCH 119/289] add test for appending data to multiple datasets --- .../importer/ExperimentFileImportTest.java | 138 +++++++++++++++++- 1 file changed, 131 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 45631d558..89203c53c 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -193,7 +193,6 @@ public void setup() { @Test @SneakyThrows - //@Disabled public void appendExperimentWithObsVarFromPriorDataset() { log.debug("appendExperimentWithObsVarFromPriorDataset"); @@ -235,8 +234,6 @@ public void appendExperimentWithObsVarFromPriorDataset() { // Assert 200 response assertEquals(HttpStatus.OK, postResponse.getStatus()); - - // Grab the system ids for the sub-entity dataset units JsonObject result = JsonParser.parseString(postResponse.body()).getAsJsonObject().getAsJsonObject("result"); // Export the plant-level sub-entity dataset @@ -250,13 +247,13 @@ public void appendExperimentWithObsVarFromPriorDataset() { ); HttpResponse plantResponse = plantExportCall.blockingFirst(); - // Make sub-entity dataset export request. - ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(plantResponse.body())); - Table exportTable = FileUtil.parseTableFromCsv(bodyStream); - // Assert 200 response assertEquals(HttpStatus.OK, plantResponse.getStatus()); + // Parse the export table + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(plantResponse.body())); + Table exportTable = FileUtil.parseTableFromCsv(bodyStream); + // Build a request to append tt_test_1 observation data on the sub-entity dataset String sub1ObsUnitId = exportTable.row(0).getString("Plant ObsUnitID"); String sub2ObsUnitId = exportTable.row(1).getString("Plant ObsUnitID"); @@ -300,6 +297,133 @@ public void appendExperimentWithObsVarFromPriorDataset() { assertEquals(422, previewResponse.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); } + @Test + @SneakyThrows + public void appendExperimentMultipleDatasets() { + log.debug("appendExperimentMultipleDatasets"); + + // Create a plot-level dataset + List traits = importTestUtils.createTraits(1); + Program program = createProgram("Append Exp with Multiple Datasets", "MULSET", "MULSET", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + //newExp.put(traits.get(0).getObservationVariableName(), null); + + JsonObject importResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); + String expId = importResponse + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + + // Create two sub-entity datasets that have two plant-level units + Flowable> sub1PostCall = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", + program.getId().toString(), expId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class); + HttpResponse sub1PostResponse = sub1PostCall.blockingFirst(); + Flowable> sub2PostCall = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", + program.getId().toString(), expId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class); + HttpResponse sub2PostResponse = sub2PostCall.blockingFirst(); + + // Assert 200 response + assertEquals(HttpStatus.OK, sub1PostResponse.getStatus()); + assertEquals(HttpStatus.OK, sub2PostResponse.getStatus()); + JsonObject sub1Result = JsonParser.parseString(sub1PostResponse.body()) + .getAsJsonObject() + .getAsJsonObject("result"); + JsonObject sub2Result = JsonParser.parseString(sub2PostResponse.body()) + .getAsJsonObject() + .getAsJsonObject("result"); + + // Export the plant-level sub-entity datasets + String extension = "CSV"; + BrAPITrial experiment = experimentService.getTrialDataByUUID(program.getId(), UUID.fromString(expId), false); + String plant1DatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), "Plant"); + Flowable> plant1ExportCall = client.exchange( + GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", + program.getId().toString(), expId, extension, plant1DatasetId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse plant1Response = plant1ExportCall.blockingFirst(); + + + String plant2DatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), "Plant"); + Flowable> plant2ExportCall = client.exchange( + GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", + program.getId().toString(), expId, extension, plant2DatasetId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse plant2Response = plant2ExportCall.blockingFirst(); + + // Parse the export tables + ByteArrayInputStream bodyStream1 = new ByteArrayInputStream(Objects.requireNonNull(plant1Response.body())); + Table exportTable1 = FileUtil.parseTableFromCsv(bodyStream1); + ByteArrayInputStream bodyStream2 = new ByteArrayInputStream(Objects.requireNonNull(plant2Response.body())); + Table exportTable2 = FileUtil.parseTableFromCsv(bodyStream2); + + // Build a request to append tt_test_1 observation data on observation units from two separate datasets + String sub1ObsUnitId = exportTable1.row(0).getString("Plant ObsUnitID"); + String sub2ObsUnitId = exportTable2.row(0).getString("Plant ObsUnitID"); + + Map sub1 = new HashMap<>(); + sub1.put(Columns.GERMPLASM_GID, "1"); + sub1.put(Columns.TEST_CHECK, "T"); + sub1.put(Columns.EXP_TITLE, "Test Exp"); + sub1.put(Columns.EXP_UNIT, "Plot"); + sub1.put(Columns.EXP_TYPE, "Phenotyping"); + sub1.put(Columns.ENV, "New Env"); + sub1.put(Columns.ENV_LOCATION, "Location A"); + sub1.put(Columns.ENV_YEAR, "2023"); + sub1.put(Columns.EXP_UNIT_ID, "a-1"); + sub1.put(Columns.REP_NUM, "1"); + sub1.put(Columns.BLOCK_NUM, "1"); + sub1.put(Columns.ROW, "1"); + sub1.put(Columns.COLUMN, "1"); + sub1.put("Plant " + OBSERVATION_UNIT_ID_SUFFIX, sub1ObsUnitId); + sub1.put(traits.get(0).getObservationVariableName(), "1"); + + Map sub2 = new HashMap<>(); + sub1.put(Columns.GERMPLASM_GID, "1"); + sub1.put(Columns.TEST_CHECK, "T"); + sub1.put(Columns.EXP_TITLE, "Test Exp"); + sub1.put(Columns.EXP_UNIT, "Plot"); + sub1.put(Columns.EXP_TYPE, "Phenotyping"); + sub1.put(Columns.ENV, "New Env"); + sub1.put(Columns.ENV_LOCATION, "Location A"); + sub1.put(Columns.ENV_YEAR, "2023"); + sub1.put(Columns.EXP_UNIT_ID, "a-1"); + sub1.put(Columns.REP_NUM, "1"); + sub1.put(Columns.BLOCK_NUM, "1"); + sub1.put(Columns.ROW, "1"); + sub1.put(Columns.COLUMN, "1"); + sub1.put("Plant " + OBSERVATION_UNIT_ID_SUFFIX, sub2ObsUnitId); + sub1.put(traits.get(0).getObservationVariableName(), "2"); + + // Verify that the validation check returns a 400-level response since obervation units belonging to different datasets are included + JsonObject previewResponse = importTestUtils.uploadAndFetchWorkflowPreview(importTestUtils.writeExperimentDataToFile(List.of(sub1, sub2), null, true, false, "Plant"), null, true, client, program, mappingId, appendOverwriteWorkflowId); + assertEquals(422, previewResponse.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + sub1Result + sub2Result); + } + @Test @SneakyThrows public void importNewExpNewLocNoObsSuccess() { From bb70d3d857e3b97c6dceebd3b5e8f3f7acdf5290 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 8 Jul 2025 15:14:33 -0400 Subject: [PATCH 120/289] create validation check for single dataset used in append workflow --- .../AppendOverwriteIDValidation.java | 3 + .../ObservationUnitDuplicateIDValidator.java | 3 + .../ObservationUnitIDBlankValidator.java | 3 + .../ObservationUnitIDColumnNameValidator.java | 3 + .../ObservationUnitIDFormatValidator.java | 3 + ...ObservationUnitSingleDatasetValidator.java | 91 +++++++++++++++++++ .../model/ExpImportProcessConstants.java | 3 +- .../importer/ExperimentFileImportTest.java | 51 +++++++---- 8 files changed, 140 insertions(+), 20 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitSingleDatasetValidator.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java index 2692a2eaa..42fe01230 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java @@ -74,6 +74,9 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext // Fetch the obs units from the BrAPi service brAPIObservationUnitReadWorkflowInitialization.execute(); + // Validate retrieved observation units + ouIdValidator.validateDynamicColumns(context); + return processNext(context); } catch (EntityNotFoundException e) { /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java index d4621d058..03ee5838c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java @@ -37,6 +37,9 @@ public class ObservationUnitDuplicateIDValidator implements DynamicColumnValidator { @Override public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + // Skip this validation if the observation units have already been fetched from the BrAPI service + if (!ctx.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().isEmpty()) return; + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() == null) { throw new BadRequestException(OZEX.getValue()); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java index 97e49ad34..0dddf51ca 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java @@ -35,6 +35,9 @@ public class ObservationUnitIDBlankValidator implements DynamicColumnValidator { @Override public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + // Skip this validation if the observation units have already been fetched from the BrAPI service + if (!ctx.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().isEmpty()) return; + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() == null) { throw new BadRequestException(OZEX.getValue()); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java index 32b2ccbc1..e5a257236 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java @@ -41,6 +41,9 @@ public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws // Skip this validation if it has already been successfully completed if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() != null) return; + // Skip this validation if the observation units have already been fetched from the BrAPI service + if (!ctx.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().isEmpty()) return; + // Get the names of all the dynamic columns with observation unit ids String[] idColNames = Arrays.stream(ctx.getImportContext().getUpload().getDynamicColumnNames()) .filter(name->name.endsWith(OBSERVATION_UNIT_ID_SUFFIX)).toArray(String[]::new); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java index bf612a2a4..93417b2c6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java @@ -40,6 +40,9 @@ public class ObservationUnitIDFormatValidator implements DynamicColumnValidator @Override public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + // Skip this validation if the observation units have already been fetched from the BrAPI service + if (!ctx.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().isEmpty()) return; + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() == null) { throw new BadRequestException(OZEX.getValue()); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitSingleDatasetValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitSingleDatasetValidator.java new file mode 100644 index 000000000..e33667c66 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitSingleDatasetValidator.java @@ -0,0 +1,91 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; + +import io.micronaut.context.annotation.Property; +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; +import org.breedinginsight.services.exceptions.BadRequestException; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.columns.Column; + +import javax.inject.Singleton; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.OZEX; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.PJZH; + +@Slf4j +@Singleton +public class ObservationUnitSingleDatasetValidator implements DynamicColumnValidator { + private final String referenceSourceBase; + + public ObservationUnitSingleDatasetValidator(@Property(name = "brapi.server.reference-source") String referenceSourceBase) { + this.referenceSourceBase = referenceSourceBase; + } + + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + // Skip this validation if the observation units have not been fetched from the BrAPI service + if (ctx.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().isEmpty()) return; + + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() == null) { + throw new BadRequestException(OZEX.getValue()); + } + + String idColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column idCol = ctx.getImportContext().getData().columns(idColName).get(0); + Map> unitPioCache = ctx.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId(); + String singleDatasetId = null; + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + // Get the external references for the BrAPI Observation Unit stored with the given id + List refs = unitPioCache + .get(idCol.get(rowNum).toString()) + .getBrAPIObject() + .getExternalReferences(); + + // Find the id of the dataset that owns the given observation unit + String datasetId = Utilities + .getExternalReference(refs, referenceSourceBase, ExternalReferenceSource.DATASET) + .map(BrAPIExternalReference::getReferenceId) + .orElseThrow(() -> new BadRequestException(PJZH.getValue())); + + // Make sure there is only a single unique dataset used in the import + if (singleDatasetId == null) { + singleDatasetId = datasetId; + } else if (!singleDatasetId.equals(datasetId)) { + throw new BadRequestException(PJZH.getValue()); + } + } + + } + + @Override + public int getOrder() { + return 5; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index d4bc7f084..4ab957534 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -41,7 +41,8 @@ public enum ErrMessage { DUPLICATE_OBS_UNIT_ID("ObsUnitId is repeated"), OZEX("Missing ObsUnitID column"), VVCN("ObsUnitID is duplicated"), - BITB("Invalid or missing ObsUnitID"); + BITB("Invalid or missing ObsUnitID"), + PJZH("Required field is blank"); private String value; diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 89203c53c..d7188b474 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -319,7 +319,6 @@ public void appendExperimentMultipleDatasets() { newExp.put(Columns.BLOCK_NUM, "1"); newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - //newExp.put(traits.get(0).getObservationVariableName(), null); JsonObject importResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); String expId = importResponse @@ -357,8 +356,22 @@ public void appendExperimentMultipleDatasets() { // Export the plant-level sub-entity datasets String extension = "CSV"; + + // Get the dataset ids from the experiment additional info BrAPITrial experiment = experimentService.getTrialDataByUUID(program.getId(), UUID.fromString(expId), false); - String plant1DatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), "Plant"); + JsonObject additionalInfo = experiment.getAdditionalInfo(); + JsonArray datasetsJsonArray = additionalInfo.getAsJsonArray("datasets"); + List subEntityDatasetIds = new ArrayList<>(); + for (JsonElement elem : datasetsJsonArray) { + JsonObject datasetObj = elem.getAsJsonObject(); + int level = datasetObj.get("level").getAsInt(); + if (level == 1) { + String id = datasetObj.get("id").getAsString(); + subEntityDatasetIds.add(id); + } + } + + String plant1DatasetId = subEntityDatasetIds.get(0); Flowable> plant1ExportCall = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", program.getId().toString(), expId, extension, plant1DatasetId)) @@ -367,7 +380,7 @@ public void appendExperimentMultipleDatasets() { HttpResponse plant1Response = plant1ExportCall.blockingFirst(); - String plant2DatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), "Plant"); + String plant2DatasetId = subEntityDatasetIds.get(1); Flowable> plant2ExportCall = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", program.getId().toString(), expId, extension, plant2DatasetId)) @@ -403,25 +416,25 @@ public void appendExperimentMultipleDatasets() { sub1.put(traits.get(0).getObservationVariableName(), "1"); Map sub2 = new HashMap<>(); - sub1.put(Columns.GERMPLASM_GID, "1"); - sub1.put(Columns.TEST_CHECK, "T"); - sub1.put(Columns.EXP_TITLE, "Test Exp"); - sub1.put(Columns.EXP_UNIT, "Plot"); - sub1.put(Columns.EXP_TYPE, "Phenotyping"); - sub1.put(Columns.ENV, "New Env"); - sub1.put(Columns.ENV_LOCATION, "Location A"); - sub1.put(Columns.ENV_YEAR, "2023"); - sub1.put(Columns.EXP_UNIT_ID, "a-1"); - sub1.put(Columns.REP_NUM, "1"); - sub1.put(Columns.BLOCK_NUM, "1"); - sub1.put(Columns.ROW, "1"); - sub1.put(Columns.COLUMN, "1"); - sub1.put("Plant " + OBSERVATION_UNIT_ID_SUFFIX, sub2ObsUnitId); - sub1.put(traits.get(0).getObservationVariableName(), "2"); + sub2.put(Columns.GERMPLASM_GID, "1"); + sub2.put(Columns.TEST_CHECK, "T"); + sub2.put(Columns.EXP_TITLE, "Test Exp"); + sub2.put(Columns.EXP_UNIT, "Plot"); + sub2.put(Columns.EXP_TYPE, "Phenotyping"); + sub2.put(Columns.ENV, "New Env"); + sub2.put(Columns.ENV_LOCATION, "Location A"); + sub2.put(Columns.ENV_YEAR, "2023"); + sub2.put(Columns.EXP_UNIT_ID, "a-1"); + sub2.put(Columns.REP_NUM, "1"); + sub2.put(Columns.BLOCK_NUM, "1"); + sub2.put(Columns.ROW, "1"); + sub2.put(Columns.COLUMN, "1"); + sub2.put("Plant " + OBSERVATION_UNIT_ID_SUFFIX, sub2ObsUnitId); + sub2.put(traits.get(0).getObservationVariableName(), "2"); // Verify that the validation check returns a 400-level response since obervation units belonging to different datasets are included JsonObject previewResponse = importTestUtils.uploadAndFetchWorkflowPreview(importTestUtils.writeExperimentDataToFile(List.of(sub1, sub2), null, true, false, "Plant"), null, true, client, program, mappingId, appendOverwriteWorkflowId); - assertEquals(422, previewResponse.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + sub1Result + sub2Result); + assertEquals(400, previewResponse.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + sub1Result + sub2Result); } @Test From 0cf53b8f2ef1e61df7bb5fa522740e04b2a89719 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 10 Jul 2025 14:45:09 -0400 Subject: [PATCH 121/289] [BI-2540] - added migration locations for flyway maven plugin --- .env.template | 5 ++++- pom.xml | 1 + src/main/resources/application.yml | 6 ++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.env.template b/.env.template index 40910d2dd..bedcdfd54 100644 --- a/.env.template +++ b/.env.template @@ -77,4 +77,7 @@ STUDY_START_DELAY=10s TRIAL_START_DELAY=15s TRAIT_START_DELAY=20s OBSERVATION_START_DELAY=25s -OBSERVATION_UNIT_START_DELAY=30s \ No newline at end of file +OBSERVATION_UNIT_START_DELAY=30s + +# Migration locations for both maven flyway plugin (configured in pom.xml) and micronaut flyway library (configured in application.yml). +FLYWAY_LOCATIONS=classpath:org/breedinginsight/db/migration,classpath:db/migration \ No newline at end of file diff --git a/pom.xml b/pom.xml index 52567cddb..edf2b0eb9 100644 --- a/pom.xml +++ b/pom.xml @@ -624,6 +624,7 @@ jdbc:postgresql://${DB_SERVER}/${DB_NAME} ${DB_USER} ${DB_PASSWORD} + ${FLYWAY_LOCATIONS} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 616747556..2aa779ac6 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -114,15 +114,13 @@ jackson: flyway: datasources: default: - locations: - - classpath:org/breedinginsight/db/migration - - classpath:db/migration + locations: ${FLYWAY_LOCATIONS:classpath:org/breedinginsight/db/migration,classpath:db/migration} enabled: true placeholders: default-url: ${brapi.server.default-url} brapi-reference-source: ${brapi.server.reference-source} orcid-sandbox-authentication: ${ORCID_SANDBOX_AUTHENTICATION:false} - out-of-order: false + out-of-order: true jooq: datasources: default: From 509fcd9bea31494aee6005e4e6d751f0a5f10a90 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 10 Jul 2025 15:01:51 -0400 Subject: [PATCH 122/289] [BI-2540] - added configuration to workflow files --- .github/workflows/build.yml | 3 ++- .github/workflows/release.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0e1b29b66..73693e353 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,4 +61,5 @@ jobs: OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf GITHUB_OAUTH_CLIENT_ID: 12345678901234567890 GITHUB_OAUTH_CLIENT_SECRET: 1234567890123456789012345678901234567890 - BRAPI_REFERENCE_SOURCE: breedinginsight.org \ No newline at end of file + BRAPI_REFERENCE_SOURCE: breedinginsight.org + FLYWAY_LOCATIONS: classpath:org/breedinginsight/db/migration,classpath:db/migration \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 31d1bd0e6..20f060783 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,6 +67,7 @@ jobs: OAUTH_CLIENT_ID: 123abc OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf BRAPI_REFERENCE_SOURCE: breedinginsight.org + FLYWAY_LOCATIONS: classpath:org/breedinginsight/db/migration,classpath:db/migration - name: Login to Docker Hub uses: docker/login-action@v1 with: From 60c379de9d9619df5827610ec60010aabb6166c2 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Thu, 10 Jul 2025 19:42:26 +0000 Subject: [PATCH 123/289] feat: Optimize observation dependency updates Optimized the `updateObservationDependencyValues` method in `CommitPendingImportObjectsStep.java` to improve performance during large data imports. The previous implementation used a nested loop (O(N*M) complexity) to update dependency DbIds on BrAPIObservation objects. This has been changed to use a lookup map, reducing the complexity to O(N+M). This change aims to address significant slowdowns experienced when importing large experiment files, such as the Mr. Bean dataset. --- .../steps/CommitPendingImportObjectsStep.java | 62 +++++++++++-------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index 12e6bcc99..b9f9419bb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -320,10 +320,24 @@ private void updateObservationDependencyValues(PendingData pendingData, Program Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); Map> observationByHash = pendingData.getObservationByHash(); + // Create a lookup map for observations + // Key: studyName_observationUnitName (composite key) + // Value: List of observations matching the key + Map>> observationsByStudyAndUnit = new java.util.HashMap<>(); + for (PendingImportObject obsPio : observationByHash.values()) { + BrAPIObservation obs = obsPio.getBrAPIObject(); + if (obs.getAdditionalInfo() != null && obs.getAdditionalInfo().get(BrAPIAdditionalInfoFields.STUDY_NAME) != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getAdditionalInfo().get(BrAPIAdditionalInfoFields.STUDY_NAME).getAsString(), programKey); + String obsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getObservationUnitName(), programKey); + String key = studyName + "_" + obsUnitName; + observationsByStudyAndUnit.computeIfAbsent(key, k -> new ArrayList<>()).add(obsPio); + } + } + // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds observationUnitByNameNoScope.values().stream() .map(PendingImportObject::getBrAPIObject) - .forEach(obsUnit -> updateObservationDbIds(pendingData, obsUnit, programKey)); + .forEach(obsUnit -> updateObservationDbIds(observationsByStudyAndUnit, obsUnit, programKey)); // Pass the new map // Update ObservationVariable DbIds List traits = getTraitList(program); @@ -340,33 +354,27 @@ private void updateObservationDependencyValues(PendingData pendingData, Program } } - // Update each ovservation's observationUnit DbId, study DbId, and germplasm DbId - private void updateObservationDbIds(PendingData pendingData, BrAPIObservationUnit obsUnit, String programKey) { - Map> observationByHash = pendingData.getObservationByHash(); + // Update each observation's observationUnit DbId, study DbId, and germplasm DbId + private void updateObservationDbIds(Map>> observationsByStudyAndUnit, BrAPIObservationUnit obsUnit, String programKey) { // Modified signature - // FILTER LOGIC: Match on Env and Exp Unit ID - observationByHash.values() - .stream() - .filter(obs -> obs.getBrAPIObject() - .getAdditionalInfo() != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) - .getAsString() - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getStudyName(), programKey)) - && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject().getObservationUnitName(), programKey) - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey)) - ) - .forEach(obs -> { - if (StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { - obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); - } - obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); - obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId()); - }); + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getStudyName(), programKey); + String obsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey); + String key = studyName + "_" + obsUnitName; + + List> matchingObservations = observationsByStudyAndUnit.get(key); + + if (matchingObservations != null) { + for (PendingImportObject obsPio : matchingObservations) { + BrAPIObservation obs = obsPio.getBrAPIObject(); + // FILTER LOGIC is now implicitly handled by the map structure and the key lookup + + if (StringUtils.isBlank(obs.getObservationUnitDbId())) { + obs.setObservationUnitDbId(obsUnit.getObservationUnitDbId()); + } + obs.setStudyDbId(obsUnit.getStudyDbId()); + obs.setGermplasmDbId(obsUnit.getGermplasmDbId()); + } + } } private List getTraitList(Program program) { From f5b26d14e1920ef1cb8f39f8876a4144eee427d1 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 11 Jul 2025 09:39:37 -0400 Subject: [PATCH 124/289] create validator for ensuring obs vars are not re-used from other datasets --- .../factory/entity/PendingDataset.java | 32 ++-- .../factory/entity/PendingEntityFactory.java | 13 +- .../AppendOverwriteVariableValidation.java | 84 ++++++++++ .../process/ImportTableProcess.java | 56 +------ .../model/AppendOverwriteWorkflowContext.java | 9 +- ...ervationVariablePriorDatasetValidator.java | 143 ++++++++++++++++++ ...ObservationVariableTimestampValidator.java | 110 ++++++++++++++ .../ObservationVariableValidator.java | 44 ++++++ .../model/ExpImportProcessConstants.java | 3 +- .../experiment/service/DatasetService.java | 14 +- .../importer/ExperimentFileImportTest.java | 1 - 11 files changed, 435 insertions(+), 74 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteVariableValidation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableValidator.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java index 216006f23..70460a6c7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java @@ -17,9 +17,11 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; +import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import com.google.gson.JsonArray; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; @@ -27,15 +29,14 @@ import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.DatasetUtil; +import org.breedinginsight.utilities.Utilities; import java.util.ArrayList; import java.util.List; @@ -49,16 +50,19 @@ public class PendingDataset implements ExperimentImportEntity BrAPIListDAO brAPIListDAO; DatasetService datasetService; ExperimentUtilities experimentUtilities; + String referenceSourceBase; public PendingDataset(AppendOverwriteMiddlewareContext context, BrAPIListDAO brAPIListDAO, DatasetService datasetService, - ExperimentUtilities experimentUtilities) { + ExperimentUtilities experimentUtilities, + String referenceSourceBase) { this.cache = context.getAppendOverwriteWorkflowContext(); this.importContext = context.getImportContext(); this.brAPIListDAO = brAPIListDAO; this.datasetService = datasetService; this.experimentUtilities = experimentUtilities; + this.referenceSourceBase = referenceSourceBase; } /** * Create new objects generated by the workflow in the BrAPI service. @@ -103,13 +107,21 @@ public List brapiPost(List members) throws A */ @Override public List brapiRead() throws ApiException { - // Get the id of the dataset belonging to the required exp units - JsonArray datasetsJson = cache.getTrialByNameNoScope().values().iterator().next().getBrAPIObject() - .getAdditionalInfo() - .getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); - String datasetId = DatasetUtil.getTopLevelDatasetFromJson(datasetsJson).getId().toString(); + // Get the dataset id from the xref of any observation unit in the import + List refs = cache + .getPendingObsUnitByOUId() + .values() + .stream() + .findFirst() + .get() + .getBrAPIObject() + .getExternalReferences(); + String datasetId = Utilities + .getExternalReference(refs, referenceSourceBase, ExternalReferenceSource.DATASET) + .map(BrAPIExternalReference::getReferenceId) + .get(); - // Get the dataset belonging to required exp units + // Get the dataset return List.of(datasetService.fetchDatasetById(datasetId, importContext.getProgram()).orElseThrow(ApiException::new)); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java index 20e827f2f..440d663cc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java @@ -19,6 +19,7 @@ import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; @@ -45,6 +46,7 @@ public class PendingEntityFactory { private final ProgramLocationService programLocationService; private final LocationService locationService; private final ExperimentUtilities experimentUtilities; + private final String referenceSourceBase; @Inject public PendingEntityFactory(TrialService trialService, @@ -58,7 +60,8 @@ public PendingEntityFactory(TrialService trialService, DatasetService datasetService, BrAPIObservationDAO brAPIObservationDAO, OntologyService ontologyService, ProgramLocationService programLocationService, LocationService locationService, - ExperimentUtilities experimentUtilities) { + ExperimentUtilities experimentUtilities, + @Property(name = "brapi.server.reference-source") String referenceSourceBase) { this.trialService = trialService; this.brapiTrialDAO = brapiTrialDAO; this.observationUnitDAO = observationUnitDAO; @@ -73,6 +76,7 @@ public PendingEntityFactory(TrialService trialService, this.programLocationService = programLocationService; this.locationService = locationService; this.experimentUtilities = experimentUtilities; + this.referenceSourceBase = referenceSourceBase; } public static PendingTrial pendingTrial(AppendOverwriteMiddlewareContext context, @@ -105,8 +109,9 @@ public static PendingGermplasm pendingGermplasm(AppendOverwriteMiddlewareContext public static PendingDataset pendingDataset(AppendOverwriteMiddlewareContext context, BrAPIListDAO brAPIListDAO, DatasetService datasetService, - ExperimentUtilities experimentUtilities) { - return new PendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); + ExperimentUtilities experimentUtilities, + String referenceSourceBase) { + return new PendingDataset(context, brAPIListDAO, datasetService, experimentUtilities, referenceSourceBase); } public static PendingObservation pendingObservation(AppendOverwriteMiddlewareContext context, @@ -150,7 +155,7 @@ public PendingGermplasm pendingGermplasmBean(AppendOverwriteMiddlewareContext co @Bean @Prototype public PendingDataset pendingDatasetBean(AppendOverwriteMiddlewareContext context) { - return pendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); + return pendingDataset(context, brAPIListDAO, datasetService, experimentUtilities, referenceSourceBase); } @Bean diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteVariableValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteVariableValidation.java new file mode 100644 index 000000000..381ba1113 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteVariableValidation.java @@ -0,0 +1,84 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIReadFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowReadInitialization; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationVariable.ObservationVariableValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; +import org.breedinginsight.services.exceptions.BadRequestException; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.api.model.v1.response.ValidationErrors; + +@Slf4j +@Prototype +public class AppendOverwriteVariableValidation extends AppendOverwriteMiddleware { + private final ObservationVariableValidator obsVarValidator; + private final BrAPIReadFactory brAPIReadFactory; + WorkflowReadInitialization brAPITrialReadWorkflowInitialization; + WorkflowReadInitialization brAPIDatasetReadWorkflowInitialization; + + public AppendOverwriteVariableValidation(ObservationVariableValidator obsVarValidator, + BrAPIReadFactory brAPIReadFactory) { + this.obsVarValidator = obsVarValidator; + this.brAPIReadFactory = brAPIReadFactory; + } + + @Override + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + brAPITrialReadWorkflowInitialization = brAPIReadFactory.trialWorkflowReadInitializationBean(context); + brAPIDatasetReadWorkflowInitialization = brAPIReadFactory.datasetWorkflowReadInitializationBean(context); + ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + + try { + // Validate any dynamic columns that are phenotype variables + obsVarValidator.validateDynamicColumns(context); + + // Check for tabular errors collected during validation + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); + } + + // Fetch the observation variables owned by all datasets belonging to the experiment involved in the import + brAPITrialReadWorkflowInitialization.execute(); + brAPIDatasetReadWorkflowInitialization.execute(); + + // Validate again to check that none of the phenotypes in the import belong to other datasets + obsVarValidator.validateDynamicColumns(context); + + return processNext(context); + } catch ( EntityNotFoundException e) { + // TODO: change method to handle errors with other entities besides obs units + ExperimentUtilities.addValidationErrorsForObsUnitsNotFound(e, context); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(new ValidatorException(validationErrors))); + return this.compensate(context); + } catch (BadRequestException | ApiException | ValidatorException e) { + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); + return this.compensate(context); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 45eea09e1..d799b0631 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -107,60 +107,14 @@ public ImportTableProcess(StudyService studyService, @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { - log.debug("verifying traits listed in import"); - - // Get all the phenotypic columns of the import - ImportUpload upload = context.getImportContext().getUpload(); - Table data = context.getImportContext().getData(); - List phenotypeColNames = Arrays.stream(upload.getDynamicColumnNames()) - .filter(name -> !name.endsWith(OBSERVATION_UNIT_ID_SUFFIX)) - .filter(name -> !name.contains(SUB_UNIT_NUMBER)) - .collect(Collectors.toList()); - - // don't allow periods (.) or square brackets in Phenotype Column Names - for (String phenotypeColumnName: phenotypeColNames) { - if(phenotypeColumnName.contains(".") || phenotypeColumnName.contains("[") || phenotypeColumnName.contains("]")){ - String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", phenotypeColumnName); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, errorMsg); - } - } - List> dynamicCols = data.columns(phenotypeColNames.toArray(new String[0])); - - // Collect the columns for observation variable data - List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); - List varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList()); - - // Collect the columns for observation timestamps - List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); - Set tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toSet()); - - // Construct validation errors for any timestamp columns that don't have a matching variable column - List importRows = context.getImportContext().getImportRows(); - Optional.ofNullable(context.getAppendOverwriteWorkflowContext().getValidationErrors()).orElseGet(() -> { - context.getAppendOverwriteWorkflowContext().setValidationErrors(new ValidationErrors()); - return new ValidationErrors(); - }); - ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); - List tsValErrs = observationVariableService.validateMatchedTimestamps(Set.copyOf(varNames), timestampCols).orElse(new ArrayList<>()); - for (int i = 0; i < importRows.size(); i++) { - int rowNum = i; - tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); - } - try { - // Stop processing the import if there are unmatched timestamp columns - if (tsValErrs.size() > 0) { - throw new UnprocessableEntityException("One or more timestamp columns do not have a matching observation variable"); - } - - //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval - Map> tsColByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); - - // Add the map to the context for use in processing import - context.getAppendOverwriteWorkflowContext().setTimeStampColByPheno(tsColByPheno); + ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + Map> tsColByPheno = context.getAppendOverwriteWorkflowContext().getTimeStampColByPheno(); + List> phenotypeCols = context.getAppendOverwriteWorkflowContext().getPhenotypeCols(); + List varNames = context.getAppendOverwriteWorkflowContext().getVarNames(); + Program program = context.getImportContext().getProgram(); // Fetch the traits named in the observation variable columns - Program program = context.getImportContext().getProgram(); List traits = observationVariableService.fetchTraitsByName(Set.copyOf(varNames), program); // Map trait by phenotype column name diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java index b1c7dfbc3..e42b6185a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java @@ -31,10 +31,7 @@ import org.breedinginsight.model.ProgramLocation; import tech.tablesaw.columns.Column; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; @Getter @Setter @@ -58,6 +55,10 @@ public class AppendOverwriteWorkflowContext { private MiddlewareException processError; private ValidationErrors validationErrors; + // Dynamic Columns + private List> phenotypeCols; + private List varNames; + // Cache maps keyed by name without program scope private Map> observationUnitByNameNoScope; private Map> trialByNameNoScope; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java new file mode 100644 index 000000000..b1822df20 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java @@ -0,0 +1,143 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationVariable; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import io.micronaut.context.annotation.Property; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; +import org.breedinginsight.services.exceptions.BadRequestException; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Singleton; +import java.util.*; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.JABH; + +@Slf4j +@Singleton +public class ObservationVariablePriorDatasetValidator implements DynamicColumnValidator { + private final String referenceSourceBase; + private final DatasetService datasetService; + + public ObservationVariablePriorDatasetValidator(@Property(name = "brapi.server.reference-source") String referenceSourceBase, DatasetService datasetService) { + this.referenceSourceBase = referenceSourceBase; + this.datasetService = datasetService; + } + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + // Skip the validation if the dependencies have not been fetched from the BrAPI service + if(noMappings(ctx)) return; + + // Get the dataset id used for this import + String datasetId = getImportDatasetId(ctx); + + // Get the ids for the other datasets in the same experiment + Set otherIds = getExperimentDatasetIds(ctx); + otherIds.remove(datasetId); + + // Get the names of any observation variables owned by the other datasets + Set forbiddenVariables = getDatasetVariables(ctx, otherIds); + + // Check that no phenotype name used in the import already belongs to another dataset + if (forbidden(ctx, forbiddenVariables)) { + throw new BadRequestException(JABH.getValue()); + } + } + + @Override + public int getOrder() { + return 2; + } + + private Set getDatasetVariables(AppendOverwriteMiddlewareContext ctx, Set ids) throws ApiException { + Set variables = new HashSet<>(); + List varListDetails = datasetService + .fetchDatasetsByIds(ids, ctx.getImportContext().getProgram()).orElse(new ArrayList<>()); + for (BrAPIListDetails brAPIListDetails : varListDetails) { + variables.addAll(brAPIListDetails.getData()); + } + + return variables; + } + + private boolean forbidden(AppendOverwriteMiddlewareContext ctx, Set forbiddenVariables) { + List importVarNames = ctx.getAppendOverwriteWorkflowContext().getVarNames(); + for (String importVarName : importVarNames) { + if (forbiddenVariables.contains(importVarName)) return true; + } + + return false; + } + + private Set getExperimentDatasetIds(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + JsonArray expDatasets = ctx.getAppendOverwriteWorkflowContext() + .getPendingTrialByOUId() + .values() + .stream() + .findFirst() + .orElseThrow(() -> new BadRequestException("No pending trial found")) + .getBrAPIObject() + .getAdditionalInfo() + .getAsJsonArray("datasets"); + Set datasetIds = new HashSet<>(); + for (JsonElement expDataset : expDatasets) { + String datasetId = expDataset.getAsJsonObject().get("id").getAsString(); + datasetIds.add(datasetId); + } + + return datasetIds; + } + + private String getImportDatasetId(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + List refs = ctx.getAppendOverwriteWorkflowContext() + .getPendingObsUnitByOUId() + .values() + .stream() + .findFirst() + .orElseThrow(()->new BadRequestException("No pending obs unit found")) + .getBrAPIObject() + .getExternalReferences(); + + return Utilities + .getExternalReference(refs, referenceSourceBase, ExternalReferenceSource.DATASET) + .map(BrAPIExternalReference::getReferenceId) + .orElseThrow(() -> new BadRequestException("No dataset associated with observation unit")); + } + + private boolean noMappings(AppendOverwriteMiddlewareContext context) { + Map> trialByOUId = context + .getAppendOverwriteWorkflowContext() + .getPendingTrialByOUId(); + Map> datasetByOUId = context + .getAppendOverwriteWorkflowContext() + .getPendingObsDatasetByOUId(); + + return !(trialByOUId == null || trialByOUId.isEmpty() || datasetByOUId == null || datasetByOUId.isEmpty()); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java new file mode 100644 index 000000000..538bf220b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java @@ -0,0 +1,110 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationVariable; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; +import org.breedinginsight.services.exceptions.BadRequestException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; + +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.*; + +@Slf4j +@Singleton +public class ObservationVariableTimestampValidator implements DynamicColumnValidator { + private final ObservationVariableService observationVariableService; + + public ObservationVariableTimestampValidator(ObservationVariableService observationVariableService) { + this.observationVariableService = observationVariableService; + } + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + log.debug("verifying traits listed in import"); + + // Get all the phenotypic columns of the import + ImportUpload upload = ctx.getImportContext().getUpload(); + Table data = ctx.getImportContext().getData(); + List phenotypeColNames = Arrays.stream(upload.getDynamicColumnNames()) + .filter(name -> !name.endsWith(OBSERVATION_UNIT_ID_SUFFIX)) + .filter(name -> !name.contains(SUB_UNIT_NUMBER)) + .collect(Collectors.toList()); + + // don't allow periods (.) or square brackets in Phenotype Column Names + for (String phenotypeColumnName: phenotypeColNames) { + if(phenotypeColumnName.contains(".") || phenotypeColumnName.contains("[") || phenotypeColumnName.contains("]")){ + String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", phenotypeColumnName); + throw new BadRequestException(errorMsg); + } + } + List> dynamicCols = data.columns(phenotypeColNames.toArray(new String[0])); + + // Collect the columns for observation variable data + List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); + List varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList()); + + // Add the phenotypes to the context for use in processing import + ctx.getAppendOverwriteWorkflowContext().setPhenotypeCols(phenotypeCols); + ctx.getAppendOverwriteWorkflowContext().setVarNames(varNames); + + // Collect the columns for observation timestamps + List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); + + // Construct validation errors for any timestamp columns that don't have a matching variable column + List importRows = ctx.getImportContext().getImportRows(); + ValidationErrors validationErrors = ctx.getAppendOverwriteWorkflowContext().getValidationErrors(); + List tsValErrs = observationVariableService + .validateMatchedTimestamps(Set.copyOf(varNames), timestampCols) + .orElse(new ArrayList<>()); + for (int i = 0; i < importRows.size(); i++) { + int rowNum = i; + tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); + } + + if (tsValErrs.isEmpty()) { + //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval + Map> tsColByPheno = timestampCols + .stream() + .collect(Collectors + .toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), + col -> col)); + + // Add the map to the context for use in processing import + ctx.getAppendOverwriteWorkflowContext().setTimeStampColByPheno(tsColByPheno); + } + } + + @Override + public int getOrder() { + return 1; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableValidator.java new file mode 100644 index 000000000..ea039024b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableValidator.java @@ -0,0 +1,44 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationVariable; + +import io.micronaut.context.annotation.Primary; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; +import org.breedinginsight.services.exceptions.BadRequestException; + +import javax.inject.Singleton; +import java.util.List; + +@Primary +@Singleton +public class ObservationVariableValidator implements DynamicColumnValidator { + private final List validators; + + public ObservationVariableValidator(List validators) { + this.validators = validators; + } + + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + for (DynamicColumnValidator validator : validators) { + validator.validateDynamicColumns(ctx); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index 4ab957534..d508efff0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -42,7 +42,8 @@ public enum ErrMessage { OZEX("Missing ObsUnitID column"), VVCN("ObsUnitID is duplicated"), BITB("Invalid or missing ObsUnitID"), - PJZH("Required field is blank"); + PJZH("Required field is blank"), + JABH("Observation variable(s) are already associated with another dataset(s) in this experiment"); private String value; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index 68aca9e2b..0cf7d815b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -33,9 +33,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.List; -import java.util.Optional; -import java.util.UUID; +import java.util.*; @Singleton public class DatasetService { @@ -88,6 +86,16 @@ public Optional fetchDatasetById(String id, Program program) t return dataSetDetails; } + public Optional> fetchDatasetsByIds(Set datasetIds, Program program) throws ApiException { + List datasets = new ArrayList<>(); + for (String datasetId : datasetIds) { + Optional dataSetDetailsOptional = fetchDatasetById(datasetId, program); + dataSetDetailsOptional.ifPresent(datasets::add); + } + + return datasets.isEmpty() ? Optional.empty() : Optional.of(datasets); + } + /** * Constructs a PendingImportObject for a BrAPIListDetails dataset. * This method retrieves the external reference for the dataset from the existing list diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index d7188b474..2ed39d1dd 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -379,7 +379,6 @@ public void appendExperimentMultipleDatasets() { ); HttpResponse plant1Response = plant1ExportCall.blockingFirst(); - String plant2DatasetId = subEntityDatasetIds.get(1); Flowable> plant2ExportCall = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", From 3ef17424f4d5a386ef032dda38d0751e584c4244 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 11 Jul 2025 11:45:54 -0400 Subject: [PATCH 125/289] create interface for obs var validator --- .../DynamicObsUnitValidator.java} | 7 +++-- .../ObservationUnitDuplicateIDValidator.java | 3 +- .../ObservationUnitIDBlankValidator.java | 3 +- .../ObservationUnitIDColumnNameValidator.java | 3 +- .../ObservationUnitIDFormatValidator.java | 3 +- .../ObservationUnitIDValidator.java | 12 ++++---- ...ObservationUnitSingleDatasetValidator.java | 4 +-- .../DynamicObsVarValidator.java | 28 +++++++++++++++++++ ...ervationVariablePriorDatasetValidator.java | 9 +++--- ...ObservationVariableTimestampValidator.java | 5 +--- .../ObservationVariableValidator.java | 14 +++++----- 11 files changed, 56 insertions(+), 35 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/{DynamicColumnValidator.java => observationUnitID/DynamicObsUnitValidator.java} (81%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/DynamicObsVarValidator.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/DynamicColumnValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/DynamicObsUnitValidator.java similarity index 81% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/DynamicColumnValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/DynamicObsUnitValidator.java index 909f48b21..166a6fd0e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/DynamicColumnValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/DynamicObsUnitValidator.java @@ -15,13 +15,14 @@ * limitations under the License. */ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; import io.micronaut.core.order.Ordered; +import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.services.exceptions.BadRequestException; @FunctionalInterface -public interface DynamicColumnValidator extends Ordered { - void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException; +public interface DynamicObsUnitValidator extends Ordered { + void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException, ApiException; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java index 03ee5838c..54a70cf57 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java @@ -21,7 +21,6 @@ import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; import org.breedinginsight.services.exceptions.BadRequestException; import tech.tablesaw.columns.Column; @@ -34,7 +33,7 @@ @Slf4j @Singleton -public class ObservationUnitDuplicateIDValidator implements DynamicColumnValidator { +public class ObservationUnitDuplicateIDValidator implements DynamicObsUnitValidator { @Override public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { // Skip this validation if the observation units have already been fetched from the BrAPI service diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java index 0dddf51ca..b69234a1c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java @@ -21,7 +21,6 @@ import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; import org.breedinginsight.services.exceptions.BadRequestException; import tech.tablesaw.columns.Column; @@ -32,7 +31,7 @@ @Slf4j @Singleton -public class ObservationUnitIDBlankValidator implements DynamicColumnValidator { +public class ObservationUnitIDBlankValidator implements DynamicObsUnitValidator { @Override public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { // Skip this validation if the observation units have already been fetched from the BrAPI service diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java index e5a257236..49d86340b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java @@ -20,7 +20,6 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; import org.breedinginsight.services.exceptions.BadRequestException; import javax.inject.Singleton; @@ -32,7 +31,7 @@ @Slf4j @Singleton -public class ObservationUnitIDColumnNameValidator implements DynamicColumnValidator { +public class ObservationUnitIDColumnNameValidator implements DynamicObsUnitValidator { public ObservationUnitIDColumnNameValidator() {} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java index 93417b2c6..e35ffd884 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java @@ -20,7 +20,6 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; import org.breedinginsight.services.exceptions.BadRequestException; import tech.tablesaw.columns.Column; import java.util.regex.Pattern; @@ -33,7 +32,7 @@ @Slf4j @Singleton -public class ObservationUnitIDFormatValidator implements DynamicColumnValidator { +public class ObservationUnitIDFormatValidator implements DynamicObsUnitValidator { private static final Pattern UUID_PATTERN = Pattern.compile( "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" ); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDValidator.java index 46e76384a..dc37d7e16 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDValidator.java @@ -18,8 +18,8 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; import io.micronaut.context.annotation.Primary; +import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; import org.breedinginsight.services.exceptions.BadRequestException; import javax.inject.Singleton; @@ -27,17 +27,17 @@ @Primary @Singleton -public class ObservationUnitIDValidator implements DynamicColumnValidator { - private final List validators; +public class ObservationUnitIDValidator implements DynamicObsUnitValidator { + private final List validators; - public ObservationUnitIDValidator(List validators) { + public ObservationUnitIDValidator(List validators) { this.validators = validators; } @Override public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) - throws BadRequestException { - for (DynamicColumnValidator validator : validators) { + throws BadRequestException, ApiException { + for (DynamicObsUnitValidator validator : validators) { validator.validateDynamicColumns(ctx); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitSingleDatasetValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitSingleDatasetValidator.java index e33667c66..ef01603ed 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitSingleDatasetValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitSingleDatasetValidator.java @@ -24,14 +24,12 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; import org.breedinginsight.services.exceptions.BadRequestException; import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; import javax.inject.Singleton; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,7 +38,7 @@ @Slf4j @Singleton -public class ObservationUnitSingleDatasetValidator implements DynamicColumnValidator { +public class ObservationUnitSingleDatasetValidator implements DynamicObsUnitValidator { private final String referenceSourceBase; public ObservationUnitSingleDatasetValidator(@Property(name = "brapi.server.reference-source") String referenceSourceBase) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/DynamicObsVarValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/DynamicObsVarValidator.java new file mode 100644 index 000000000..b42cbee7c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/DynamicObsVarValidator.java @@ -0,0 +1,28 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationVariable; + +import io.micronaut.core.order.Ordered; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.services.exceptions.BadRequestException; + +@FunctionalInterface +public interface DynamicObsVarValidator extends Ordered { + void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException, ApiException; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java index b1822df20..c7b6ba3da 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java @@ -28,7 +28,6 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; import org.breedinginsight.services.exceptions.BadRequestException; import org.breedinginsight.utilities.Utilities; @@ -40,17 +39,19 @@ @Slf4j @Singleton -public class ObservationVariablePriorDatasetValidator implements DynamicColumnValidator { +public class ObservationVariablePriorDatasetValidator implements DynamicObsVarValidator { private final String referenceSourceBase; private final DatasetService datasetService; - public ObservationVariablePriorDatasetValidator(@Property(name = "brapi.server.reference-source") String referenceSourceBase, DatasetService datasetService) { + public ObservationVariablePriorDatasetValidator( + @Property(name = "brapi.server.reference-source") String referenceSourceBase, + DatasetService datasetService) { this.referenceSourceBase = referenceSourceBase; this.datasetService = datasetService; } @Override - public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException, ApiException { // Skip the validation if the dependencies have not been fetched from the BrAPI service if(noMappings(ctx)) return; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java index 538bf220b..17858065d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java @@ -17,7 +17,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationVariable; -import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.breedinginsight.api.model.v1.response.ValidationError; @@ -25,10 +24,8 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; import org.breedinginsight.services.exceptions.BadRequestException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; @@ -40,7 +37,7 @@ @Slf4j @Singleton -public class ObservationVariableTimestampValidator implements DynamicColumnValidator { +public class ObservationVariableTimestampValidator implements DynamicObsVarValidator { private final ObservationVariableService observationVariableService; public ObservationVariableTimestampValidator(ObservationVariableService observationVariableService) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableValidator.java index ea039024b..83cd9cd35 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableValidator.java @@ -18,8 +18,8 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationVariable; import io.micronaut.context.annotation.Primary; +import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.DynamicColumnValidator; import org.breedinginsight.services.exceptions.BadRequestException; import javax.inject.Singleton; @@ -27,17 +27,17 @@ @Primary @Singleton -public class ObservationVariableValidator implements DynamicColumnValidator { - private final List validators; +public class ObservationVariableValidator implements DynamicObsVarValidator { + private final List validators; - public ObservationVariableValidator(List validators) { + public ObservationVariableValidator(List validators) { this.validators = validators; } - @Override - public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { - for (DynamicColumnValidator validator : validators) { + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) + throws BadRequestException, ApiException { + for (DynamicObsVarValidator validator : validators) { validator.validateDynamicColumns(ctx); } } From ec56da1a6f4cde3c6ea28d0d69df4f0df01d0678 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Fri, 11 Jul 2025 18:17:16 +0000 Subject: [PATCH 126/289] feat: Configure BrAPI docker image and update GH Action - I've allowed the BrAPITest Docker image to be configured via the BRAPI_DOCKER_IMAGE environment variable. - I've updated .env.template with BRAPI_DOCKER_IMAGE. - I've added workflow_dispatch to the build.yml GH Action. - This allows you to specify brapi_server_image and bi_api_branch inputs. - PR builds will use the default image and checkout the PR branch. - Workflow_dispatch builds will use the specified inputs or defaults. --- .env.template | 1 + .github/workflows/build.yml | 20 ++++++++++++++++--- .../java/org/breedinginsight/BrAPITest.java | 7 ++++++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.env.template b/.env.template index 1a481aae6..a39401c6f 100644 --- a/.env.template +++ b/.env.template @@ -37,6 +37,7 @@ CACHE_BRAPI_FETCH_PAGE_SIZE=65000 # BrAPI Server Variables BRAPI_SERVER_PORT=8083 +BRAPI_DOCKER_IMAGE=breedinginsight/brapi-java-server:develop # Brapi Server Variables BRAPI_DEFAULT_URL=http://localhost:8083 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0f77a41e0..d8c73ed2a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,17 @@ name: maven build on: pull_request: - type: [opened, edited] + types: [opened, edited] + workflow_dispatch: + inputs: + brapi_server_image: + description: 'BrAPI Server Docker Image' + required: false + default: 'breedinginsight/brapi-java-server:develop' + bi_api_branch: + description: 'BI API Branch' + required: false + default: 'develop' jobs: build: @@ -38,7 +48,10 @@ jobs: options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v2 + - name: Checkout BI API + uses: actions/checkout@v2 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.bi_api_branch || github.ref }} - name: Set up JDK 13 uses: actions/setup-java@v1.4.3 with: @@ -59,4 +72,5 @@ jobs: JWT_SECRET: ${{ secrets.JWT_SECRET }} OAUTH_CLIENT_ID: 123abc OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf - BRAPI_REFERENCE_SOURCE: breedinginsight.org \ No newline at end of file + BRAPI_REFERENCE_SOURCE: breedinginsight.org + BRAPI_DOCKER_IMAGE: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.brapi_server_image || 'breedinginsight/brapi-java-server:develop' }} \ No newline at end of file diff --git a/src/test/java/org/breedinginsight/BrAPITest.java b/src/test/java/org/breedinginsight/BrAPITest.java index bdf362979..37c94a61a 100644 --- a/src/test/java/org/breedinginsight/BrAPITest.java +++ b/src/test/java/org/breedinginsight/BrAPITest.java @@ -49,7 +49,12 @@ public class BrAPITest extends DatabaseTest { public BrAPITest() { super(); - brapiContainer = new GenericContainer<>("breedinginsight/brapi-java-server:develop") + String dockerImage = System.getenv("BRAPI_DOCKER_IMAGE"); + if (dockerImage == null || dockerImage.isEmpty()) { + dockerImage = "breedinginsight/brapi-java-server:develop"; + } + + brapiContainer = new GenericContainer<>(dockerImage) .withNetwork(super.getNetwork()) .withImagePullPolicy(PullPolicy.ageBased(Duration.ofMinutes(60))) .withExposedPorts(8080) From 5235abd7c7c34e3e532efc1663901caf84a7f1a0 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 11 Jul 2025 20:16:44 +0000 Subject: [PATCH 127/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 3a2ba301c..f017b0a9a 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1011 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2700fde850e28247dffd20ea101ab6158415fe0e +version=v1.2.0+1013 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f5b34e6db6547540d34110856a6b5c9e25683b51 From d21d4268caf283b987c48a91683009d35f0d3d92 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 14 Jul 2025 14:57:07 -0400 Subject: [PATCH 128/289] Update actions versions --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8c73ed2a..30bbcdb1c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,13 +49,14 @@ jobs: steps: - name: Checkout BI API - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.bi_api_branch || github.ref }} - name: Set up JDK 13 - uses: actions/setup-java@v1.4.3 + uses: actions/setup-java@v4 with: java-version: 13 + distribution: 'temurin' # It's good practice to specify a distribution - name: Build with Maven run: mvn validate -B flyway:migrate clean install --file pom.xml --settings settings.xml From 9b49b43096467dc445ccf8c96bb045cd69bfc5aa Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 14 Jul 2025 18:57:18 +0000 Subject: [PATCH 129/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index f017b0a9a..1eebd53bb 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1013 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f5b34e6db6547540d34110856a6b5c9e25683b51 +version=v1.2.0+1015 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d21d4268caf283b987c48a91683009d35f0d3d92 From 229857725413b75b8c0923bf283063d6026ac04e Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 14 Jul 2025 15:31:47 -0400 Subject: [PATCH 130/289] Change distribution --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 30bbcdb1c..6f1b27632 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,7 +56,7 @@ jobs: uses: actions/setup-java@v4 with: java-version: 13 - distribution: 'temurin' # It's good practice to specify a distribution + distribution: 'zulu' - name: Build with Maven run: mvn validate -B flyway:migrate clean install --file pom.xml --settings settings.xml From c8df9053c51a839763f4417c06fcaf0734ac856f Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 14 Jul 2025 19:32:00 +0000 Subject: [PATCH 131/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1eebd53bb..5095a193c 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1015 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d21d4268caf283b987c48a91683009d35f0d3d92 +version=v1.2.0+1017 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/229857725413b75b8c0923bf283063d6026ac04e From 2302e6da90199f259fa936c9b0beed898c8e3181 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 15 Jul 2025 21:34:55 -0400 Subject: [PATCH 132/289] fix bug in obsvar validator --- .../experiment/ExperimentUtilities.java | 2 +- .../AppendOverwritePhenotypesWorkflow.java | 15 ++++++-- .../factory/entity/PendingDataset.java | 35 +++++++++++-------- .../initialize/WorkflowInitialization.java | 6 ---- .../process/ImportTableProcess.java | 24 +++++++++---- .../ObservationUnitIDColumnNameValidator.java | 4 +-- ...ervationVariablePriorDatasetValidator.java | 14 ++++++-- ...ObservationVariableTimestampValidator.java | 7 ++-- .../model/ExpImportProcessConstants.java | 5 +-- .../experiment/service/DatasetService.java | 4 +-- .../importer/ExperimentFileImportTest.java | 8 ++--- 11 files changed, 78 insertions(+), 46 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 46769bb5b..2f22d7e9e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -92,7 +92,7 @@ public ExperimentUtilities() { */ public boolean isInvalidMemberListForClass(List list, Class clazz) { // Check if the input list is null, empty, or contains any member that is not an instance of the specified class - return list == null || list.isEmpty() || !list.stream().allMatch(clazz::isInstance); + return (list == null) || !list.stream().allMatch(clazz::isInstance); } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index fbbc0afbd..7743e08cd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -28,6 +28,7 @@ import org.breedinginsight.brapps.importer.services.ImportStatusService; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.AppendOverwriteIDValidation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.AppendOverwriteVariableValidation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPICommit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize.WorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.ImportTableProcess; @@ -46,20 +47,24 @@ @Singleton public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { private final ExperimentWorkflowNavigator.Workflow workflow; + private final AppendOverwriteMiddleware validationMiddleware; private final AppendOverwriteMiddleware importPreviewMiddleware; private final AppendOverwriteMiddleware brapiCommitMiddleware; private final ImportStatusService statusService; @Inject public AppendOverwritePhenotypesWorkflow(AppendOverwriteIDValidation expUnitIDValidation, + AppendOverwriteVariableValidation obsVariableValidation, WorkflowInitialization workflowInitialization, ImportTableProcess importTableProcess, BrAPICommit brAPICommit, ImportStatusService statusService){ this.statusService = statusService; this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; + this.validationMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link( + expUnitIDValidation, + obsVariableValidation); this.importPreviewMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link( - expUnitIDValidation, workflowInitialization, importTableProcess); this.brapiCommitMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link(brAPICommit); @@ -115,11 +120,15 @@ public Optional process(ImportServiceContext context) { .appendOverwriteWorkflowContext(new AppendOverwriteWorkflowContext()) .build(); + // Validate the import + AppendOverwriteMiddlewareContext validatedImportContext = this.validationMiddleware.process(workflowContext); + // Process the import preview - AppendOverwriteMiddlewareContext processedPreviewContext = this.importPreviewMiddleware.process(workflowContext); + AppendOverwriteMiddlewareContext processedPreviewContext = this.importPreviewMiddleware.process(validatedImportContext); // Stop and return any errors that occurred while processing - Optional previewException = Optional.ofNullable(processedPreviewContext.getAppendOverwriteWorkflowContext().getProcessError()); + Optional previewException = Optional + .ofNullable(processedPreviewContext.getAppendOverwriteWorkflowContext().getProcessError()); if (previewException.isPresent()) { log.debug(String.format("%s in %s", previewException.get().getException().getClass().getName(), previewException.get().getLocalTransactionName())); result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(previewException.get().getException()))); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java index 70460a6c7..0a831d56c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java @@ -38,9 +38,7 @@ import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.Utilities; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; @Prototype @@ -122,7 +120,10 @@ public List brapiRead() throws ApiException { .get(); // Get the dataset - return List.of(datasetService.fetchDatasetById(datasetId, importContext.getProgram()).orElseThrow(ApiException::new)); + return datasetService + .fetchDatasetById(datasetId, importContext.getProgram()) + .map(List::of) + .orElseGet(List::of); } /** @@ -242,18 +243,22 @@ public void initializeWorkflow(List members) { .collect(Collectors.toMap(pio -> pio.getBrAPIObject().getListName(),pio -> pio)); // Construct a hashmap to look up the pending dataset by the observation unit ID of a unit stored in the BrAPI service - Map> pendingObsDatasetByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - e -> { - if (cache.getPendingTrialByOUId().isEmpty() || - pendingDatasetByName.isEmpty() || - cache.getPendingTrialByOUId().values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { - throw new IllegalStateException("There is not an observation data set for this unit: " + e.getKey()); + Map> pendingObsDatasetByOUId; + if (pendingDatasetByName.isEmpty()) { + pendingObsDatasetByOUId = Collections.emptyMap(); + } else { + pendingObsDatasetByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> { + if (cache.getPendingTrialByOUId().isEmpty() || + cache.getPendingTrialByOUId().values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { + throw new IllegalStateException("There is not an observation data set for this unit: " + e.getKey()); + } + return pendingDatasetByName.values().iterator().next(); } - return pendingDatasetByName.values().iterator().next(); - } - )); + )); + } // Add the maps to the context for use in processing import cache.setObsVarDatasetByName(pendingDatasetByName); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index 2aa889fc2..decd495a2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -39,10 +39,8 @@ @Slf4j @Prototype public class WorkflowInitialization extends AppendOverwriteMiddleware { - WorkflowReadInitialization brAPITrialReadWorkflowInitialization; WorkflowReadInitialization brAPIStudyReadWorkflowInitialization; WorkflowReadInitialization locationReadWorkflowInitialization; - WorkflowReadInitialization brAPIDatasetReadWorkflowInitialization; WorkflowReadInitialization brAPIGermplasmReadWorkflowInitialization; BrAPIReadFactory brAPIReadFactory; @@ -52,18 +50,14 @@ public WorkflowInitialization(BrAPIReadFactory brAPIReadFactory) { } @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { - brAPITrialReadWorkflowInitialization = brAPIReadFactory.trialWorkflowReadInitializationBean(context); brAPIStudyReadWorkflowInitialization = brAPIReadFactory.studyWorkflowReadInitializationBean(context); locationReadWorkflowInitialization = brAPIReadFactory.locationWorkflowReadInitializationBean(context); - brAPIDatasetReadWorkflowInitialization = brAPIReadFactory.datasetWorkflowReadInitializationBean(context); brAPIGermplasmReadWorkflowInitialization = brAPIReadFactory.germplasmWorkflowReadInitializationBean(context); log.debug("reading required BrAPI data from BrAPI service"); try { - brAPITrialReadWorkflowInitialization.execute(); brAPIStudyReadWorkflowInitialization.execute(); locationReadWorkflowInitialization.execute(); - brAPIDatasetReadWorkflowInitialization.execute(); brAPIGermplasmReadWorkflowInitialization.execute(); } catch (ApiException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index d799b0631..48d4e5a76 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -68,6 +68,7 @@ import java.util.stream.Collectors; import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.*; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.DATASET_NOT_FOUND; import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; @Slf4j @@ -130,17 +131,28 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext // Sort the traits to match the order of the headers in the import file List sortedTraits = experimentUtil.sortByField(varNames, new ArrayList<>(traits), TraitEntity::getObservationVariableName); - // Get the pending observation dataset - PendingImportObject pendingTrial = ExperimentUtilities.getSingleEntryValue(context.getAppendOverwriteWorkflowContext().getTrialByNameNoScope()).orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); - String datasetName = String.format("Observation Dataset [%s-%s]", program.getKey(), pendingTrial.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString()); - PendingImportObject pendingDataset = context.getAppendOverwriteWorkflowContext().getObsVarDatasetByName().get(datasetName); + // Get the pending observation dataset; there should only be a single dataset used for the import + PendingImportObject pendingTrial = ExperimentUtilities + .getSingleEntryValue(context.getAppendOverwriteWorkflowContext().getTrialByNameNoScope()) + .orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); + PendingImportObject pendingDataset = context + .getAppendOverwriteWorkflowContext() + .getPendingObsDatasetByOUId() + .values() + .stream() + .findAny() + .orElseGet(()-> new PendingImportObject(ImportObjectState.NEW, new BrAPIListDetails())); // Add new phenotypes to the pending observation dataset list (NOTE: "obsVarName [programKey]" is used instead of obsVarDbId) // TODO: Change to using actual dbIds as per the BrAPI spec, instead of namespaced obsVar names (was necessary for Breedbase) - List datasetObsVarDbIds = pendingDataset.getBrAPIObject().getData().stream().collect(Collectors.toList()); + List datasetObsVarDbIds = Optional.ofNullable(pendingDataset.getBrAPIObject().getData()) + .map(ArrayList::new) + .orElseGet(ArrayList::new); List phenoDbIds = sortedTraits.stream().map(t->Utilities.appendProgramKey(t.getObservationVariableName(), program.getKey())).collect(Collectors.toList()); phenoDbIds.removeAll(datasetObsVarDbIds); - pendingDataset.getBrAPIObject().getData().addAll(phenoDbIds); + for (String phenoDbId : phenoDbIds) { + pendingDataset.getBrAPIObject().addDataItem(phenoDbId); + } // Update pending status if (ImportObjectState.EXISTING == pendingDataset.getState()) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java index 49d86340b..98a9cb8f1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java @@ -27,7 +27,7 @@ import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.OZEX; import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.SUB_UNIT_NUMBER; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.SUB_UNIT_ID; @Slf4j @Singleton @@ -57,7 +57,7 @@ public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws if (idColCount == 2) { // if sub-entity ids in import then check for presence of sub-unit # column Arrays.stream(ctx.getImportContext().getUpload().getDynamicColumnNames()) - .filter(name-> name.equals(SUB_UNIT_NUMBER)) + .filter(name-> name.equals(SUB_UNIT_ID)) .findAny() .orElseThrow(()->new BadRequestException(OZEX.getValue())); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java index c7b6ba3da..70ba472d9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java @@ -34,6 +34,7 @@ import javax.inject.Singleton; import java.util.*; +import java.util.stream.Collectors; import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.JABH; @@ -78,10 +79,16 @@ public int getOrder() { private Set getDatasetVariables(AppendOverwriteMiddlewareContext ctx, Set ids) throws ApiException { Set variables = new HashSet<>(); + String progKey = ctx.getImportContext().getProgram().getKey(); List varListDetails = datasetService .fetchDatasetsByIds(ids, ctx.getImportContext().getProgram()).orElse(new ArrayList<>()); for (BrAPIListDetails brAPIListDetails : varListDetails) { - variables.addAll(brAPIListDetails.getData()); + List priorVariablesNoScope = brAPIListDetails + .getData() + .stream() + .map((scopedVariable) -> Utilities.removeProgramKey(scopedVariable, progKey)) + .collect(Collectors.toList()); + variables.addAll(priorVariablesNoScope); } return variables; @@ -139,6 +146,9 @@ private boolean noMappings(AppendOverwriteMiddlewareContext context) { .getAppendOverwriteWorkflowContext() .getPendingObsDatasetByOUId(); - return !(trialByOUId == null || trialByOUId.isEmpty() || datasetByOUId == null || datasetByOUId.isEmpty()); + if (trialByOUId == null) return true; + if (trialByOUId.isEmpty()) return true; + if (datasetByOUId == null) return true; + return false; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java index 17858065d..cb9b4ac21 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java @@ -51,9 +51,12 @@ public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws // Get all the phenotypic columns of the import ImportUpload upload = ctx.getImportContext().getUpload(); Table data = ctx.getImportContext().getData(); - List phenotypeColNames = Arrays.stream(upload.getDynamicColumnNames()) + List phenotypeColNames = upload + .getDynamicColumnNamesList() + .stream() .filter(name -> !name.endsWith(OBSERVATION_UNIT_ID_SUFFIX)) - .filter(name -> !name.contains(SUB_UNIT_NUMBER)) + .filter(name -> !name.contains(SUB_UNIT_ID)) + .filter(name -> !name.contains(SUB_OBS_UNIT)) .collect(Collectors.toList()); // don't allow periods (.) or square brackets in Phenotype Column Names diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index d508efff0..b8ea2b6af 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -18,7 +18,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.model; import com.fasterxml.jackson.annotation.JsonValue; -import io.micronaut.context.annotation.Property; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -30,7 +29,8 @@ public class ExpImportProcessConstants { public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; public static String BRAPI_REFERENCE_SOURCE; public static final String MIDNIGHT = "T00:00:00-00:00"; - public static final String SUB_UNIT_NUMBER = "Sub-Unit #"; + public static final String SUB_UNIT_ID = "Sub Unit ID"; + public static final String SUB_OBS_UNIT = "Sub-Obs Unit"; public enum ErrMessage { MULTIPLE_EXP_TITLES("File contains more than one Experiment Title"), @@ -39,6 +39,7 @@ public enum ErrMessage { UNMATCHED_COLUMN("Ontology term(s) not found: "), OBS_UNIT_NOT_FOUND("Invalid ObsUnitID"), DUPLICATE_OBS_UNIT_ID("ObsUnitId is repeated"), + DATASET_NOT_FOUND("Dataset not found"), OZEX("Missing ObsUnitID column"), VVCN("ObsUnitID is duplicated"), BITB("Invalid or missing ObsUnitID"), diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index 0cf7d815b..d772466ca 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -72,10 +72,8 @@ public Optional fetchDatasetById(String id, Program program) t program.getId(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), UUID.fromString(id)); - - // Check if the existing dataset summaries are returned, throw exception if not if (existingDatasets == null || existingDatasets.isEmpty()) { - throw new InternalServerException("Existing dataset summary not returned from BrAPI server"); + return Optional.empty(); } // Retrieve dataset details using the list DB ID from the existing dataset summary diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 2ed39d1dd..06b899020 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -213,9 +213,9 @@ public void appendExperimentWithObsVarFromPriorDataset() { newExp.put(Columns.BLOCK_NUM, "1"); newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - newExp.put(traits.get(0).getObservationVariableName(), null); + newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject importResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject importResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); String expId = importResponse .get("preview").getAsJsonObject() .get("rows").getAsJsonArray() @@ -293,8 +293,8 @@ public void appendExperimentWithObsVarFromPriorDataset() { sub2.put(traits.get(0).getObservationVariableName(), "2"); // Verify that the validation check returns a 400-level response since tt_test_1 is already used in the plot-level dataset - JsonObject previewResponse = importTestUtils.uploadAndFetchWorkflowPreview(importTestUtils.writeExperimentDataToFile(List.of(sub1, sub2), null, true, false, "Plant"), null, true, client, program, mappingId, appendOverwriteWorkflowId); - assertEquals(422, previewResponse.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + JsonObject previewResponse = importTestUtils.uploadAndFetchWorkflowPreview(importTestUtils.writeExperimentDataToFile(List.of(sub1, sub2), traits, true, true, "Plant"), null, true, client, program, mappingId, appendOverwriteWorkflowId); + assertEquals(400, previewResponse.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); } @Test From 1771679e497a7a18c00184ae93f985f74908d8f8 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 16 Jul 2025 09:37:07 -0400 Subject: [PATCH 133/289] Add check right before commmit --- .../steps/CommitPendingImportObjectsStep.java | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index 12e6bcc99..66024be09 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -48,6 +48,7 @@ import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; @@ -58,6 +59,8 @@ import java.util.Objects; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.PREEXISTING_EXPERIMENT_TITLE; + @Singleton @Slf4j public class CommitPendingImportObjectsStep { @@ -88,7 +91,7 @@ public CommitPendingImportObjectsStep(BrAPIListDAO brAPIListDAO, } // TODO: some common code between workflows here that could be broken out, removed append/update specific code - public void process(ProcessContext processContext, ProcessedData processedData) { + public void process(ProcessContext processContext, ProcessedData processedData) throws UnprocessableEntityException { PendingData pendingData = processContext.getPendingData(); ImportContext importContext = processContext.getImportContext(); @@ -138,6 +141,24 @@ public void process(ProcessContext processContext, ProcessedData processedData) AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); + // TODO: Implement more robust solution either brapi server side or possibly redis SETNX client side + // Do this check here, directly before creating new trials instead of in earlier step to minimize time window of race condition + if (!newTrials.isEmpty()) { + try { + List cachedTrials = brapiTrialDAO.getTrials(program.getId()); + List existingTrialNames = cachedTrials.stream().map(BrAPITrial::getTrialName).collect(Collectors.toList()); + List newTrialNames = newTrials.stream().map(t -> Utilities.removeProgramKey(t.getTrialName(), program.getKey())).collect(Collectors.toList()); + log.debug("** Trials Duplicate Check: {} -> {}", existingTrialNames, newTrialNames); + if (newTrialNames.stream().anyMatch(existingTrialNames::contains)) { + log.debug("** New matches existing"); + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } catch (ApiException e) { + log.error("Error getting trials for duplicate name check", e); + throw new InternalServerException(e.getMessage(), e); + } + } + try { List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); createdDatasets.forEach(summary -> obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId())); From 6fa158e435d8be4e1f667454a4e71491415d2b6d Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 18 Jul 2025 17:04:34 -0400 Subject: [PATCH 134/289] Don't create empty treatments array in sub-entity obs units --- .../brapi/v2/services/BrAPITrialService.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 6b7a01fec..b11983b56 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -501,14 +501,16 @@ public BrAPIObservationUnit createSubObservationUnit( } // Set treatment factors. - List treatmentFactors = new ArrayList<>(); - for (BrAPIObservationTreatment t : expUnit.getTreatments()) { - BrAPIObservationTreatment treatment = new BrAPIObservationTreatment(); - treatment.setFactor(t.getFactor()); - treatment.setModality(t.getModality()); - treatmentFactors.add(treatment); + if (!expUnit.getTreatments().isEmpty()) { + List treatmentFactors = new ArrayList<>(); + for (BrAPIObservationTreatment t : expUnit.getTreatments()) { + BrAPIObservationTreatment treatment = new BrAPIObservationTreatment(); + treatment.setFactor(t.getFactor()); + treatment.setModality(t.getModality()); + treatmentFactors.add(treatment); + } + observationUnit.setTreatments(treatmentFactors); } - observationUnit.setTreatments(treatmentFactors); // Put level in additional info: keep this in case we decide to rename levels in future. observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, subEntityDatasetName); From c2eb985818e307411b9bfb33729a246bfa44b9a5 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 21 Jul 2025 15:56:14 -0400 Subject: [PATCH 135/289] [BI-2540] - updated flyway locations paths for CI --- .github/workflows/build.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 73693e353..db3a9a148 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -62,4 +62,4 @@ jobs: GITHUB_OAUTH_CLIENT_ID: 12345678901234567890 GITHUB_OAUTH_CLIENT_SECRET: 1234567890123456789012345678901234567890 BRAPI_REFERENCE_SOURCE: breedinginsight.org - FLYWAY_LOCATIONS: classpath:org/breedinginsight/db/migration,classpath:db/migration \ No newline at end of file + FLYWAY_LOCATIONS: filesystem:src/main/java/org/breedinginsight/db/migration,filesystem:src/main/resources/db/migration \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 20f060783..4a5be2c73 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,7 +67,7 @@ jobs: OAUTH_CLIENT_ID: 123abc OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf BRAPI_REFERENCE_SOURCE: breedinginsight.org - FLYWAY_LOCATIONS: classpath:org/breedinginsight/db/migration,classpath:db/migration + FLYWAY_LOCATIONS: filesystem:src/main/java/org/breedinginsight/db/migration,filesystem:src/main/resources/db/migration - name: Login to Docker Hub uses: docker/login-action@v1 with: From a8a0eb29c33f5bcb09444e3752bce721c8f4d5f6 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 30 Jul 2025 10:18:33 -0400 Subject: [PATCH 136/289] [BI-2540] - updated comment --- .env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.template b/.env.template index 75be429a5..711c01221 100644 --- a/.env.template +++ b/.env.template @@ -3,7 +3,7 @@ USER_ID= GROUP_ID= -# GitHub OAuth variables. Only required if using GitHub as an alternative to ORCID. +# GitHub OAuth variables. Only required if using GitHub as an OAuth provider. GITHUB_OAUTH_CLIENT_ID= GITHUB_OAUTH_CLIENT_SECRET= From 4d62efa6a601d16ef57579b6feecb9e119d7f64e Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 31 Jul 2025 16:31:38 -0400 Subject: [PATCH 137/289] Removed comment --- .../create/workflow/steps/CommitPendingImportObjectsStep.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index b9f9419bb..5c318d336 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -366,7 +366,6 @@ private void updateObservationDbIds(Map obsPio : matchingObservations) { BrAPIObservation obs = obsPio.getBrAPIObject(); - // FILTER LOGIC is now implicitly handled by the map structure and the key lookup if (StringUtils.isBlank(obs.getObservationUnitDbId())) { obs.setObservationUnitDbId(obsUnit.getObservationUnitDbId()); From 4986ceff6b6b95d86c852d57926ca887ed276f4f Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 1 Aug 2025 15:48:30 +0000 Subject: [PATCH 138/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 5095a193c..37ec4ef9f 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1017 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/229857725413b75b8c0923bf283063d6026ac04e +version=v1.2.0+1021 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f969a8912d7f8b7bcbcaa533eb8207d98588b591 From 788af9377cac994df4b40d74bb47ed34b91ad8c2 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 1 Aug 2025 19:11:06 +0000 Subject: [PATCH 139/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 37ec4ef9f..9f9c78387 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1021 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f969a8912d7f8b7bcbcaa533eb8207d98588b591 +version=v1.2.0+1023 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c58a7c3a5a7c61bda8d5f39422ce1a149cad1531 From 0a45488c94e45ddc932060a724dc94f8668e740e Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 13 Aug 2025 10:44:39 -0400 Subject: [PATCH 140/289] Add gemini-cli github actions --- .github/workflows/gemini-cli.yml | 304 ++++++++++++ .../gemini-issue-automated-triage.yml | 130 +++++ .../gemini-issue-scheduled-triage.yml | 123 +++++ .github/workflows/gemini-pr-review.yml | 456 ++++++++++++++++++ 4 files changed, 1013 insertions(+) create mode 100644 .github/workflows/gemini-cli.yml create mode 100644 .github/workflows/gemini-issue-automated-triage.yml create mode 100644 .github/workflows/gemini-issue-scheduled-triage.yml create mode 100644 .github/workflows/gemini-pr-review.yml diff --git a/.github/workflows/gemini-cli.yml b/.github/workflows/gemini-cli.yml new file mode 100644 index 000000000..3fbaedc70 --- /dev/null +++ b/.github/workflows/gemini-cli.yml @@ -0,0 +1,304 @@ +name: '💬 Gemini CLI' + +on: + pull_request_review_comment: + types: + - 'created' + pull_request_review: + types: + - 'submitted' + issue_comment: + types: + - 'created' + +concurrency: + group: '${{ github.workflow }}-${{ github.event.issue.number }}' + cancel-in-progress: |- + ${{ github.event.sender.type == 'User' && ( github.event.issue.author_association == 'OWNER' || github.event.issue.author_association == 'MEMBER' || github.event.issue.author_association == 'COLLABORATOR') }} + +defaults: + run: + shell: 'bash' + +permissions: + contents: 'write' + id-token: 'write' + pull-requests: 'write' + issues: 'write' + +jobs: + gemini-cli: + # This condition is complex to ensure we only run when explicitly invoked. + if: |- + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'issues' && github.event.action == 'opened' && + contains(github.event.issue.body, '@gemini-cli') && + !contains(github.event.issue.body, '@gemini-cli /review') && + !contains(github.event.issue.body, '@gemini-cli /triage') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.issue.author_association) + ) || + ( + ( + github.event_name == 'issue_comment' || + github.event_name == 'pull_request_review_comment' + ) && + contains(github.event.comment.body, '@gemini-cli') && + !contains(github.event.comment.body, '@gemini-cli /review') && + !contains(github.event.comment.body, '@gemini-cli /triage') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) + ) || + ( + github.event_name == 'pull_request_review' && + contains(github.event.review.body, '@gemini-cli') && + !contains(github.event.review.body, '@gemini-cli /review') && + !contains(github.event.review.body, '@gemini-cli /triage') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.review.author_association) + ) + timeout-minutes: 10 + runs-on: 'ubuntu-latest' + + steps: + - name: 'Generate GitHub App Token' + id: 'generate_token' + if: |- + ${{ vars.APP_ID }} + uses: 'actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e' # ratchet:actions/create-github-app-token@v2 + with: + app-id: '${{ vars.APP_ID }}' + private-key: '${{ secrets.APP_PRIVATE_KEY }}' + + - name: 'Get context from event' + id: 'get_context' + env: + EVENT_NAME: '${{ github.event_name }}' + EVENT_PAYLOAD: '${{ toJSON(github.event) }}' + run: |- + set -euo pipefail + + USER_REQUEST="" + ISSUE_NUMBER="" + IS_PR="false" + + if [[ "${EVENT_NAME}" == "issues" ]]; then + USER_REQUEST=$(echo "${EVENT_PAYLOAD}" | jq -r .issue.body) + ISSUE_NUMBER=$(echo "${EVENT_PAYLOAD}" | jq -r .issue.number) + elif [[ "${EVENT_NAME}" == "issue_comment" ]]; then + USER_REQUEST=$(echo "${EVENT_PAYLOAD}" | jq -r .comment.body) + ISSUE_NUMBER=$(echo "${EVENT_PAYLOAD}" | jq -r .issue.number) + if [[ $(echo "${EVENT_PAYLOAD}" | jq -r .issue.pull_request) != "null" ]]; then + IS_PR="true" + fi + elif [[ "${EVENT_NAME}" == "pull_request_review" ]]; then + USER_REQUEST=$(echo "${EVENT_PAYLOAD}" | jq -r .review.body) + ISSUE_NUMBER=$(echo "${EVENT_PAYLOAD}" | jq -r .pull_request.number) + IS_PR="true" + elif [[ "${EVENT_NAME}" == "pull_request_review_comment" ]]; then + USER_REQUEST=$(echo "${EVENT_PAYLOAD}" | jq -r .comment.body) + ISSUE_NUMBER=$(echo "${EVENT_PAYLOAD}" | jq -r .pull_request.number) + IS_PR="true" + fi + + # Clean up user request + USER_REQUEST=$(echo "${USER_REQUEST}" | sed 's/.*@gemini-cli//' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + { + echo "user_request=${USER_REQUEST}" + echo "issue_number=${ISSUE_NUMBER}" + echo "is_pr=${IS_PR}" + } >> "${GITHUB_OUTPUT}" + + - name: 'Set up git user for commits' + run: |- + git config --global user.name 'gemini-cli[bot]' + git config --global user.email 'gemini-cli[bot]@users.noreply.github.com' + + - name: 'Checkout PR branch' + if: |- + ${{ steps.get_context.outputs.is_pr == 'true' }} + uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 + with: + token: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + repository: '${{ github.repository }}' + ref: 'refs/pull/${{ steps.get_context.outputs.issue_number }}/head' + fetch-depth: 0 + + - name: 'Checkout main branch' + if: |- + ${{ steps.get_context.outputs.is_pr == 'false' }} + uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 + with: + token: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + repository: '${{ github.repository }}' + fetch-depth: 0 + + - name: 'Acknowledge request' + env: + GITHUB_ACTOR: '${{ github.actor }}' + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + ISSUE_NUMBER: '${{ steps.get_context.outputs.issue_number }}' + REPOSITORY: '${{ github.repository }}' + REQUEST_TYPE: '${{ steps.get_context.outputs.request_type }}' + run: |- + set -euo pipefail + MESSAGE="@${GITHUB_ACTOR} I've received your request and I'm working on it now! 🤖" + if [[ -n "${MESSAGE}" ]]; then + gh issue comment "${ISSUE_NUMBER}" \ + --body "${MESSAGE}" \ + --repo "${REPOSITORY}" + fi + + - name: 'Get description' + id: 'get_description' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + IS_PR: '${{ steps.get_context.outputs.is_pr }}' + ISSUE_NUMBER: '${{ steps.get_context.outputs.issue_number }}' + run: |- + set -euo pipefail + if [[ "${IS_PR}" == "true" ]]; then + DESCRIPTION=$(gh pr view "${ISSUE_NUMBER}" --json body --template '{{.body}}') + else + DESCRIPTION=$(gh issue view "${ISSUE_NUMBER}" --json body --template '{{.body}}') + fi + { + echo "description<> "${GITHUB_OUTPUT}" + + - name: 'Get comments' + id: 'get_comments' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + IS_PR: '${{ steps.get_context.outputs.is_pr }}' + ISSUE_NUMBER: '${{ steps.get_context.outputs.issue_number }}' + run: |- + set -euo pipefail + if [[ "${IS_PR}" == "true" ]]; then + COMMENTS=$(gh pr view "${ISSUE_NUMBER}" --json comments --template '{{range .comments}}{{.author.login}}: {{.body}}{{"\n"}}{{end}}') + else + COMMENTS=$(gh issue view "${ISSUE_NUMBER}" --json comments --template '{{range .comments}}{{.author.login}}: {{.body}}{{"\n"}}{{end}}') + fi + { + echo "comments<> "${GITHUB_OUTPUT}" + + - name: 'Run Gemini' + id: 'run_gemini' + uses: 'google-github-actions/run-gemini-cli@v0' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + REPOSITORY: '${{ github.repository }}' + USER_REQUEST: '${{ steps.get_context.outputs.user_request }}' + ISSUE_NUMBER: '${{ steps.get_context.outputs.issue_number }}' + IS_PR: '${{ steps.get_context.outputs.is_pr }}' + with: + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' + gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' + gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' + gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' + gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' + use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' + use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' + settings: |- + { + "maxSessionTurns": 50, + "telemetry": { + "enabled": false, + "target": "gcp" + } + } + prompt: |- + ## Role + + You are a helpful AI assistant invoked via a CLI interface in a GitHub workflow. You have access to tools to interact with the repository and respond to the user. + + ## Context + + - **Repository**: `${{ github.repository }}` + - **Triggering Event**: `${{ github.event_name }}` + - **Issue/PR Number**: `${{ steps.get_context.outputs.issue_number }}` + - **Is this a PR?**: `${{ steps.get_context.outputs.is_pr }}` + - **Issue/PR Description**: + `${{ steps.get_description.outputs.description }}` + - **Comments**: + `${{ steps.get_comments.outputs.comments }}` + + ## User Request + + The user has sent the following request: + `${{ steps.get_context.outputs.user_request }}` + + ## How to Respond to Issues, PR Comments, and Questions + + This workflow supports three main scenarios: + + 1. **Creating a Fix for an Issue** + - Carefully read the user request and the related issue or PR description. + - Use available tools to gather all relevant context (e.g., `gh issue view`, `gh pr view`, `gh pr diff`, `cat`, `head`, `tail`). + - Identify the root cause of the problem before proceeding. + - **Show and maintain a plan as a checklist**: + - At the very beginning, outline the steps needed to resolve the issue or address the request and post them as a checklist comment on the issue or PR (use GitHub markdown checkboxes: `- [ ] Task`). + - Example: + ``` + ### Plan + - [ ] Investigate the root cause + - [ ] Implement the fix in `file.py` + - [ ] Add/modify tests + - [ ] Update documentation + - [ ] Verify the fix and close the issue + ``` + - Use: `gh pr comment "${ISSUE_NUMBER}" --body ""` or `gh issue comment "${ISSUE_NUMBER}" --body ""` to post the initial plan. + - As you make progress, keep the checklist visible and up to date by editing the same comment (check off completed tasks with `- [x]`). + - To update the checklist: + 1. Find the comment ID for the checklist (use `gh pr comment list "${ISSUE_NUMBER}"` or `gh issue comment list "${ISSUE_NUMBER}"`). + 2. Edit the comment with the updated checklist: + - For PRs: `gh pr comment --edit --body ""` + - For Issues: `gh issue comment --edit --body ""` + 3. The checklist should only be maintained as a comment on the issue or PR. Do not track or update the checklist in code files. + - If the fix requires code changes, determine which files and lines are affected. If clarification is needed, note any questions for the user. + - Make the necessary code or documentation changes using the available tools (e.g., `write_file`). Ensure all changes follow project conventions and best practices. Reference all shell variables as `"${VAR}"` (with quotes and braces) to prevent errors. + - Run any relevant tests or checks to verify the fix works as intended. If possible, provide evidence (test output, screenshots, etc.) that the issue is resolved. + - **Branching and Committing**: + - **NEVER commit directly to the `main` branch.** + - If you are working on a **pull request** (`IS_PR` is `true`), the correct branch is already checked out. Simply commit and push to it. + - `git add .` + - `git commit -m "feat: "` + - `git push` + - If you are working on an **issue** (`IS_PR` is `false`), create a new branch for your changes. A good branch name would be `issue/${ISSUE_NUMBER}/`. + - `git checkout -b issue/${ISSUE_NUMBER}/my-fix` + - `git add .` + - `git commit -m "feat: "` + - `git push origin issue/${ISSUE_NUMBER}/my-fix` + - After pushing, you can create a pull request: `gh pr create --title "Fixes #${ISSUE_NUMBER}: " --body "This PR addresses issue #${ISSUE_NUMBER}."` + - Summarize what was changed and why in a markdown file: `write_file("response.md", "")` + - Post the response as a comment: + - For PRs: `gh pr comment "${ISSUE_NUMBER}" --body-file response.md` + - For Issues: `gh issue comment "${ISSUE_NUMBER}" --body-file response.md` + + 2. **Addressing Comments on a Pull Request** + - Read the specific comment and the context of the PR. + - Use tools like `gh pr view`, `gh pr diff`, and `cat` to understand the code and discussion. + - If the comment requests a change or clarification, follow the same process as for fixing an issue: create a checklist plan, implement, test, and commit any required changes, updating the checklist as you go. + - **Committing Changes**: The correct PR branch is already checked out. Simply add, commit, and push your changes. + - `git add .` + - `git commit -m "fix: address review comments"` + - `git push` + - If the comment is a question, answer it directly and clearly, referencing code or documentation as needed. + - Document your response in `response.md` and post it as a PR comment: `gh pr comment "${ISSUE_NUMBER}" --body-file response.md` + + 3. **Answering Any Question on an Issue** + - Read the question and the full issue context using `gh issue view` and related tools. + - Research or analyze the codebase as needed to provide an accurate answer. + - If the question requires code or documentation changes, follow the fix process above, including creating and updating a checklist plan and **creating a new branch for your changes as described in section 1.** + - Write a clear, concise answer in `response.md` and post it as an issue comment: `gh issue comment "${ISSUE_NUMBER}" --body-file response.md` + + ## Guidelines + + - **Be concise and actionable.** Focus on solving the user's problem efficiently. + - **Always commit and push your changes if you modify code or documentation.** + - **If you are unsure about the fix or answer, explain your reasoning and ask clarifying questions.** + - **Follow project conventions and best practices.** diff --git a/.github/workflows/gemini-issue-automated-triage.yml b/.github/workflows/gemini-issue-automated-triage.yml new file mode 100644 index 000000000..50a671018 --- /dev/null +++ b/.github/workflows/gemini-issue-automated-triage.yml @@ -0,0 +1,130 @@ +name: '🏷️ Gemini Automated Issue Triage' + +on: + issues: + types: + - 'opened' + - 'reopened' + issue_comment: + types: + - 'created' + workflow_dispatch: + inputs: + issue_number: + description: 'issue number to triage' + required: true + type: 'number' + +concurrency: + group: '${{ github.workflow }}-${{ github.event.issue.number }}' + cancel-in-progress: true + +defaults: + run: + shell: 'bash' + +permissions: + contents: 'read' + id-token: 'write' + issues: 'write' + statuses: 'write' + +jobs: + triage-issue: + if: |- + github.event_name == 'issues' || + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'issue_comment' && + contains(github.event.comment.body, '@gemini-cli /triage') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) + ) + timeout-minutes: 5 + runs-on: 'ubuntu-latest' + + steps: + - name: 'Checkout repository' + uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 + + - name: 'Generate GitHub App Token' + id: 'generate_token' + if: |- + ${{ vars.APP_ID }} + uses: 'actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e' # ratchet:actions/create-github-app-token@v2 + with: + app-id: '${{ vars.APP_ID }}' + private-key: '${{ secrets.APP_PRIVATE_KEY }}' + + - name: 'Run Gemini Issue Triage' + uses: 'google-github-actions/run-gemini-cli@v0' + id: 'gemini_issue_triage' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + ISSUE_TITLE: '${{ github.event.issue.title }}' + ISSUE_BODY: '${{ github.event.issue.body }}' + ISSUE_NUMBER: '${{ github.event.issue.number }}' + REPOSITORY: '${{ github.repository }}' + with: + gemini_cli_version: '${{ vars.GEMINI_CLI_VERSION }}' + gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' + gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' + gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' + gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' + use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' + use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' + settings: |- + { + "maxSessionTurns": 25, + "coreTools": [ + "run_shell_command(echo)", + "run_shell_command(gh label list)", + "run_shell_command(gh issue edit)" + ], + "telemetry": { + "enabled": false, + "target": "gcp" + } + } + prompt: |- + ## Role + + You are an issue triage assistant. Analyze the current GitHub issue + and apply the most appropriate existing labels. Use the available + tools to gather information; do not ask for information to be + provided. + + ## Steps + + 1. Run: `gh label list` to get all available labels. + 2. Review the issue title and body provided in the environment + variables: "${ISSUE_TITLE}" and "${ISSUE_BODY}". + 3. Classify issues by their kind (bug, enhancement, documentation, + cleanup, etc) and their priority (p0, p1, p2, p3). Set the + labels accoridng to the format `kind/*` and `priority/*` patterns. + 4. Apply the selected labels to this issue using: + `gh issue edit "${ISSUE_NUMBER}" --add-label "label1,label2"` + 5. If the "status/needs-triage" label is present, remove it using: + `gh issue edit "${ISSUE_NUMBER}" --remove-label "status/needs-triage"` + + ## Guidelines + + - Only use labels that already exist in the repository + - Do not add comments or modify the issue content + - Triage only the current issue + - Assign all applicable labels based on the issue content + - Reference all shell variables as "${VAR}" (with quotes and braces) + + - name: 'Post Issue Triage Failure Comment' + if: |- + ${{ failure() && steps.gemini_issue_triage.outcome == 'failure' }} + uses: 'actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea' + with: + github-token: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + script: |- + github.rest.issues.createComment({ + owner: '${{ github.repository }}'.split('/')[0], + repo: '${{ github.repository }}'.split('/')[1], + issue_number: '${{ github.event.issue.number }}', + body: 'There is a problem with the Gemini CLI issue triaging. Please check the [action logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.' + }) diff --git a/.github/workflows/gemini-issue-scheduled-triage.yml b/.github/workflows/gemini-issue-scheduled-triage.yml new file mode 100644 index 000000000..f44205973 --- /dev/null +++ b/.github/workflows/gemini-issue-scheduled-triage.yml @@ -0,0 +1,123 @@ +name: '📋 Gemini Scheduled Issue Triage' + +on: + schedule: + - cron: '0 * * * *' # Runs every hour + workflow_dispatch: + +concurrency: + group: '${{ github.workflow }}' + cancel-in-progress: true + +defaults: + run: + shell: 'bash' + +permissions: + contents: 'read' + id-token: 'write' + issues: 'write' + statuses: 'write' + +jobs: + triage-issues: + timeout-minutes: 5 + runs-on: 'ubuntu-latest' + + steps: + - name: 'Checkout repository' + uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 + + - name: 'Generate GitHub App Token' + id: 'generate_token' + if: |- + ${{ vars.APP_ID }} + uses: 'actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e' # ratchet:actions/create-github-app-token@v2 + with: + app-id: '${{ vars.APP_ID }}' + private-key: '${{ secrets.APP_PRIVATE_KEY }}' + + - name: 'Find untriaged issues' + id: 'find_issues' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + GITHUB_REPOSITORY: '${{ github.repository }}' + GITHUB_OUTPUT: '${{ github.output }}' + run: |- + set -euo pipefail + + echo '🔍 Finding issues without labels...' + NO_LABEL_ISSUES="$(gh issue list --repo "${GITHUB_REPOSITORY}" \ + --search 'is:open is:issue no:label' --json number,title,body)" + + echo '🏷️ Finding issues that need triage...' + NEED_TRIAGE_ISSUES="$(gh issue list --repo "${GITHUB_REPOSITORY}" \ + --search 'is:open is:issue label:"status/needs-triage"' --json number,title,body)" + + echo '🔄 Merging and deduplicating issues...' + ISSUES="$(echo "${NO_LABEL_ISSUES}" "${NEED_TRIAGE_ISSUES}" | jq -c -s 'add | unique_by(.number)')" + + echo '📝 Setting output for GitHub Actions...' + echo "issues_to_triage=${ISSUES}" >> "${GITHUB_OUTPUT}" + + ISSUE_COUNT="$(echo "${ISSUES}" | jq 'length')" + echo "✅ Found ${ISSUE_COUNT} issues to triage! 🎯" + + - name: 'Run Gemini Issue Triage' + if: |- + ${{ steps.find_issues.outputs.issues_to_triage != '[]' }} + uses: 'google-github-actions/run-gemini-cli@v0' + id: 'gemini_issue_triage' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + ISSUES_TO_TRIAGE: '${{ steps.find_issues.outputs.issues_to_triage }}' + REPOSITORY: '${{ github.repository }}' + with: + gemini_cli_version: '${{ vars.GEMINI_CLI_VERSION }}' + gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' + gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' + gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' + gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' + use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' + use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' + settings: |- + { + "maxSessionTurns": 25, + "coreTools": [ + "run_shell_command(echo)", + "run_shell_command(gh label list)", + "run_shell_command(gh issue edit)", + "run_shell_command(gh issue list)" + ], + "telemetry": { + "enabled": false, + "target": "gcp" + } + } + prompt: |- + ## Role + + You are an issue triage assistant. Analyze issues and apply + appropriate labels. Use the available tools to gather information; + do not ask for information to be provided. + + ## Steps + + 1. Run: `gh label list` + 2. Check environment variable: "${ISSUES_TO_TRIAGE}" (JSON array + of issues) + 3. For each issue, apply labels: + `gh issue edit "${ISSUE_NUMBER}" --add-label "label1,label2"`. + If available, set labels that follow the `kind/*`, `area/*`, + and `priority/*` patterns. + 4. For each issue, if the `status/needs-triage` label is present, + remove it using: + `gh issue edit "${ISSUE_NUMBER}" --remove-label "status/needs-triage"` + + ## Guidelines + + - Only use existing repository labels + - Do not add comments + - Triage each issue independently + - Reference all shell variables as "${VAR}" (with quotes and braces) diff --git a/.github/workflows/gemini-pr-review.yml b/.github/workflows/gemini-pr-review.yml new file mode 100644 index 000000000..297c4572d --- /dev/null +++ b/.github/workflows/gemini-pr-review.yml @@ -0,0 +1,456 @@ +name: '🧐 Gemini Pull Request Review' + +on: + pull_request: + types: + - 'opened' + - 'reopened' + issue_comment: + types: + - 'created' + pull_request_review_comment: + types: + - 'created' + pull_request_review: + types: + - 'submitted' + workflow_dispatch: + inputs: + pr_number: + description: 'PR number to review' + required: true + type: 'number' + +concurrency: + group: '${{ github.workflow }}-${{ github.head_ref || github.ref }}' + cancel-in-progress: true + +defaults: + run: + shell: 'bash' + +permissions: + contents: 'read' + id-token: 'write' + issues: 'write' + pull-requests: 'write' + statuses: 'write' + +jobs: + review-pr: + if: |- + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'pull_request' && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) + ) || + ( + ( + ( + github.event_name == 'issue_comment' && + github.event.issue.pull_request + ) || + github.event_name == 'pull_request_review_comment' + ) && + contains(github.event.comment.body, '@gemini-cli /review') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) + ) || + ( + github.event_name == 'pull_request_review' && + contains(github.event.review.body, '@gemini-cli /review') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.review.author_association) + ) + timeout-minutes: 5 + runs-on: 'ubuntu-latest' + + steps: + - name: 'Checkout PR code' + uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 + + - name: 'Generate GitHub App Token' + id: 'generate_token' + if: |- + ${{ vars.APP_ID }} + uses: 'actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e' # ratchet:actions/create-github-app-token@v2 + with: + app-id: '${{ vars.APP_ID }}' + private-key: '${{ secrets.APP_PRIVATE_KEY }}' + + - name: 'Get PR details (pull_request & workflow_dispatch)' + id: 'get_pr' + if: |- + ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + EVENT_NAME: '${{ github.event_name }}' + WORKFLOW_PR_NUMBER: '${{ github.event.inputs.pr_number }}' + PULL_REQUEST_NUMBER: '${{ github.event.pull_request.number }}' + run: |- + set -euo pipefail + + if [[ "${EVENT_NAME}" = "workflow_dispatch" ]]; then + PR_NUMBER="${WORKFLOW_PR_NUMBER}" + else + PR_NUMBER="${PULL_REQUEST_NUMBER}" + fi + + echo "pr_number=${PR_NUMBER}" >> "${GITHUB_OUTPUT}" + + # Get PR details + PR_DATA="$(gh pr view "${PR_NUMBER}" --json title,body,additions,deletions,changedFiles,baseRefName,headRefName)" + echo "pr_data=${PR_DATA}" >> "${GITHUB_OUTPUT}" + + # Get file changes + CHANGED_FILES="$(gh pr diff "${PR_NUMBER}" --name-only)" + { + echo "changed_files<> "${GITHUB_OUTPUT}" + + + - name: 'Get PR details (issue_comment)' + id: 'get_pr_comment' + if: |- + ${{ github.event_name == 'issue_comment' }} + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + COMMENT_BODY: '${{ github.event.comment.body }}' + PR_NUMBER: '${{ github.event.issue.number }}' + run: |- + set -euo pipefail + + echo "pr_number=${PR_NUMBER}" >> "${GITHUB_OUTPUT}" + + # Extract additional instructions from comment + ADDITIONAL_INSTRUCTIONS="$( + echo "${COMMENT_BODY}" | sed 's/.*@gemini-cli \/review//' | xargs + )" + echo "additional_instructions=${ADDITIONAL_INSTRUCTIONS}" >> "${GITHUB_OUTPUT}" + + # Get PR details + PR_DATA="$(gh pr view "${PR_NUMBER}" --json title,body,additions,deletions,changedFiles,baseRefName,headRefName)" + echo "pr_data=${PR_DATA}" >> "${GITHUB_OUTPUT}" + + # Get file changes + CHANGED_FILES="$(gh pr diff "${PR_NUMBER}" --name-only)" + { + echo "changed_files<> "${GITHUB_OUTPUT}" + + - name: 'Run Gemini PR Review' + uses: 'google-github-actions/run-gemini-cli@v0' + id: 'gemini_pr_review' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + PR_NUMBER: '${{ steps.get_pr.outputs.pr_number || steps.get_pr_comment.outputs.pr_number }}' + PR_DATA: '${{ steps.get_pr.outputs.pr_data || steps.get_pr_comment.outputs.pr_data }}' + CHANGED_FILES: '${{ steps.get_pr.outputs.changed_files || steps.get_pr_comment.outputs.changed_files }}' + ADDITIONAL_INSTRUCTIONS: '${{ steps.get_pr.outputs.additional_instructions || steps.get_pr_comment.outputs.additional_instructions }}' + REPOSITORY: '${{ github.repository }}' + with: + gemini_cli_version: '${{ vars.GEMINI_CLI_VERSION }}' + gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' + gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' + gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' + gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' + use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' + use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' + settings: |- + { + "maxSessionTurns": 20, + "mcpServers": { + "github": { + "command": "docker", + "args": [ + "run", + "-i", + "--rm", + "-e", + "GITHUB_PERSONAL_ACCESS_TOKEN", + "ghcr.io/github/github-mcp-server" + ], + "includeTools": [ + "create_pending_pull_request_review", + "add_comment_to_pending_review", + "submit_pending_pull_request_review" + ], + "env": { + "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_TOKEN}" + } + } + }, + "coreTools": [ + "run_shell_command(echo)", + "run_shell_command(gh pr view)", + "run_shell_command(gh pr diff)", + "run_shell_command(cat)", + "run_shell_command(head)", + "run_shell_command(tail)", + "run_shell_command(grep)" + ], + "telemetry": { + "enabled": false, + "target": "gcp" + } + } + prompt: |- + ## Role + + You are an expert code reviewer. You have access to tools to gather + PR information and perform the review on GitHub. Use the available tools to + gather information; do not ask for information to be provided. + + ## Requirements + 1. All feedback must be left on GitHub. + 2. Any output that is not left in GitHub will not be seen. + + ## Steps + + Start by running these commands to gather the required data: + 1. Run: echo "${REPOSITORY}" to get the github repository in / format + 2. Run: echo "${PR_DATA}" to get PR details (JSON format) + 3. Run: echo "${CHANGED_FILES}" to get the list of changed files + 4. Run: echo "${PR_NUMBER}" to get the PR number + 5. Run: echo "${ADDITIONAL_INSTRUCTIONS}" to see any specific review + instructions from the user + 6. Run: gh pr diff "${PR_NUMBER}" to see the full diff and reference + Context section to understand it + 7. For any specific files, use: cat filename, head -50 filename, or + tail -50 filename + 8. If ADDITIONAL_INSTRUCTIONS contains text, prioritize those + specific areas or focus points in your review. Common instruction + examples: "focus on security", "check performance", "review error + handling", "check for breaking changes" + + ## Guideline + ### Core Guideline(Always applicable) + + 1. Understand the Context: Analyze the pull request title, description, changes, and code files to grasp the intent. + 2. Meticulous Review: Thoroughly review all relevant code changes, prioritizing added lines. Consider the specified + focus areas and any provided style guide. + 3. Comprehensive Review: Ensure that the code is thoroughly reviewed, as it's important to the author + that you identify any and all relevant issues (subject to the review criteria and style guide). + Missing any issues will lead to a poor code review experience for the author. + 4. Constructive Feedback: + * Provide clear explanations for each concern. + * Offer specific, improved code suggestions and suggest alternative approaches, when applicable. + Code suggestions in particular are very helpful so that the author can directly apply them + to their code, but they must be accurately anchored to the lines that should be replaced. + 5. Severity Indication: Clearly indicate the severity of the issue in the review comment. + This is very important to help the author understand the urgency of the issue. + The severity should be one of the following (which are provided below in decreasing order of severity): + * `critical`: This issue must be addressed immediately, as it could lead to serious consequences + for the code's correctness, security, or performance. + * `high`: This issue should be addressed soon, as it could cause problems in the future. + * `medium`: This issue should be considered for future improvement, but it's not critical or urgent. + * `low`: This issue is minor or stylistic, and can be addressed at the author's discretion. + 6. Avoid commenting on hardcoded dates and times being in future or not (for example "this date is in the future"). + * Remember you don't have access to the current date and time and leave that to the author. + 7. Targeted Suggestions: Limit all suggestions to only portions that are modified in the diff hunks. + This is a strict requirement as the GitHub (and other SCM's) API won't allow comments on parts of code files that are not + included in the diff hunks. + 8. Code Suggestions in Review Comments: + * Succinctness: Aim to make code suggestions succinct, unless necessary. Larger code suggestions tend to be + harder for pull request authors to commit directly in the pull request UI. + * Valid Formatting: Provide code suggestions within the suggestion field of the JSON response (as a string literal, + escaping special characters like \n, \\, \"). Do not include markdown code blocks in the suggestion field. + Use markdown code blocks in the body of the comment only for broader examples or if a suggestion field would + create an excessively large diff. Prefer the suggestion field for specific, targeted code changes. + * Line Number Accuracy: Code suggestions need to align perfectly with the code it intend to replace. + Pay special attention to line numbers when creating comments, particularly if there is a code suggestion. + Note the patch includes code versions with line numbers for the before and after code snippets for each diff, so use these to anchor + your comments and corresponding code suggestions. + * Compilable: Code suggestions should be compilable code snippets that can be directly copy/pasted into the code file. + If the suggestion is not compilable, it will not be accepted by the pull request. Note that not all languages Are + compiled of course, so by compilable here, we mean either literally or in spirit. + * Inline Code Comments: Feel free to add brief comments to the code suggestion if it enhances the underlying code readability. + Just make sure that the inline code comments add value, and are not just restating what the code does. Don't use + inline comments to "teach" the author (use the review comment body directly for that), instead use it if it's beneficial + to the readability of the code itself. + 10. Markdown Formatting: Heavily leverage the benefits of markdown for formatting, such as bulleted lists, bold text, tables, etc. + 11. Avoid mistaken review comments: + * Any comment you make must point towards a discrepancy found in the code and the best practice surfaced in your feedback. + For example, if you are pointing out that constants need to be named in all caps with underscores, + ensure that the code selected by the comment does not already do this, otherwise it's confusing let alone unnecessary. + 12. Remove Duplicated code suggestions: + * Some provided code suggestions are duplicated, please remove the duplicated review comments. + 13. Don't Approve The Pull Request + 14. Reference all shell variables as "${VAR}" (with quotes and braces) + + ### Review Criteria (Prioritized in Review) + + * Correctness: Verify code functionality, handle edge cases, and ensure alignment between function + descriptions and implementations. Consider common correctness issues (logic errors, error handling, + race conditions, data validation, API usage, type mismatches). + * Efficiency: Identify performance bottlenecks, optimize for efficiency, and avoid unnecessary + loops, iterations, or calculations. Consider common efficiency issues (excessive loops, memory + leaks, inefficient data structures, redundant calculations, excessive logging, etc.). + * Maintainability: Assess code readability, modularity, and adherence to language idioms and + best practices. Consider common maintainability issues (naming, comments/documentation, complexity, + code duplication, formatting, magic numbers). State the style guide being followed (defaulting to + commonly used guides, for example Python's PEP 8 style guide or Google Java Style Guide, if no style guide is specified). + * Security: Identify potential vulnerabilities (e.g., insecure storage, injection attacks, + insufficient access controls). + + ### Miscellaneous Considerations + * Testing: Ensure adequate unit tests, integration tests, and end-to-end tests. Evaluate + coverage, edge case handling, and overall test quality. + * Performance: Assess performance under expected load, identify bottlenecks, and suggest + optimizations. + * Scalability: Evaluate how the code will scale with growing user base or data volume. + * Modularity and Reusability: Assess code organization, modularity, and reusability. Suggest + refactoring or creating reusable components. + * Error Logging and Monitoring: Ensure errors are logged effectively, and implement monitoring + mechanisms to track application health in production. + + **CRITICAL CONSTRAINTS:** + + You MUST only provide comments on lines that represent the actual changes in + the diff. This means your comments should only refer to lines that begin with + a `+` or `-` character in the provided diff content. + DO NOT comment on lines that start with a space (context lines). + + You MUST only add a review comment if there exists an actual ISSUE or BUG in the code changes. + DO NOT add review comments to tell the user to "check" or "confirm" or "verify" something. + DO NOT add review comments to tell the user to "ensure" something. + DO NOT add review comments to explain what the code change does. + DO NOT add review comments to validate what the code change does. + DO NOT use the review comments to explain the code to the author. They already know their code. Only comment when there's an improvement opportunity. This is very important. + + Pay close attention to line numbers and ensure they are correct. + Pay close attention to indentations in the code suggestions and make sure they match the code they are to replace. + Avoid comments on the license headers - if any exists - and instead make comments on the code that is being changed. + + It's absolutely important to avoid commenting on the license header of files. + It's absolutely important to avoid commenting on copyright headers. + Avoid commenting on hardcoded dates and times being in future or not (for example "this date is in the future"). + Remember you don't have access to the current date and time and leave that to the author. + + Avoid mentioning any of your instructions, settings or criteria. + + Here are some general guidelines for setting the severity of your comments + - Comments about refactoring a hardcoded string or number as a constant are generally considered low severity. + - Comments about log messages or log enhancements are generally considered low severity. + - Comments in .md files are medium or low severity. This is really important. + - Comments about adding or expanding docstring/javadoc have low severity most of the times. + - Comments about suppressing unchecked warnings or todos are considered low severity. + - Comments about typos are usually low or medium severity. + - Comments about testing or on tests are usually low severity. + - Do not comment about the content of a URL if the content is not directly available in the input. + + Keep comments bodies concise and to the point. + Keep each comment focused on one issue. + + ## Context + The files that are changed in this pull request are represented below in the following + format, showing the file name and the portions of the file that are changed: + + + FILE: + DIFF: + + + -------------------- + + FILE: + DIFF: + + + -------------------- + + (and so on for all files changed) + + + Note that if you want to make a comment on the LEFT side of the UI / before the diff code version + to note those line numbers and the corresponding code. Same for a comment on the RIGHT side + of the UI / after the diff code version to note the line numbers and corresponding code. + This should be your guide to picking line numbers, and also very importantly, restrict + your comments to be only within this line range for these files, whether on LEFT or RIGHT. + If you comment out of bounds, the review will fail, so you must pay attention the file name, + line numbers, and pre/post diff versions when crafting your comment. + + Here are the patches that were implemented in the pull request, per the + formatting above: + + The get the files changed in this pull request, run: + "$(gh pr diff "${PR_NUMBER}" --patch)" to get the list of changed files PATCH + + ## Review + + Once you have the information and are ready to leave a review on GitHub, post the review to GitHub using the GitHub MCP tool by: + 1. Creating a pending review: Use the mcp__github__create_pending_pull_request_review to create a Pending Pull Request Review. + + 2. Adding review comments: + 2.1 Use the mcp__github__add_comment_to_pending_review to add comments to the Pending Pull Request Review. Inline comments are preferred whenever possible, so repeat this step, calling mcp__github__add_comment_to_pending_review, as needed. All comments about specific lines of code should use inline comments. It is preferred to use code suggestions when possible, which include a code block that is labeled "suggestion", which contains what the new code should be. All comments should also have a severity. The syntax is: + Normal Comment Syntax: + + {{SEVERITY}} {{COMMENT_TEXT}} + + + Inline Comment Syntax: (Preferred): + + {{SEVERITY}} {{COMMENT_TEXT}} + ```suggestion + {{CODE_SUGGESTION}} + ``` + + + Prepend a severity emoji to each comment: + - 🟢 for low severity + - 🟡 for medium severity + - 🟠 for high severity + - 🔴 for critical severity + - 🔵 if severity is unclear + + Including all of this, an example inline comment would be: + + 🟢 Use camelCase for function names + ```suggestion + myFooBarFunction + ``` + + + A critical severity example would be: + + 🔴 Remove storage key from GitHub + ```suggestion + ``` + + 3. Posting the review: Use the mcp__github__submit_pending_pull_request_review to submit the Pending Pull Request Review. + + 3.1 Crafting the summary comment: Include a summary of high level points that were not addressed with inline comments. Be concise. Do not repeat details mentioned inline. + + Structure your summary comment using this exact format with markdown: + ## 📋 Review Summary + + Provide a brief 2-3 sentence overview of the PR and overall + assessment. + + ## 🔍 General Feedback + - List general observations about code quality + - Mention overall patterns or architectural decisions + - Highlight positive aspects of the implementation + - Note any recurring themes across files + + ## Final Instructions + + Remember, you are running in a VM and no one reviewing your output. Your review must be posted to GitHub using the MCP tools to create a pending review, add comments to the pending review, and submit the pending review. + + + - name: 'Post PR review failure comment' + if: |- + ${{ failure() && steps.gemini_pr_review.outcome == 'failure' }} + uses: 'actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea' + with: + github-token: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + script: |- + github.rest.issues.createComment({ + owner: '${{ github.repository }}'.split('/')[0], + repo: '${{ github.repository }}'.split('/')[1], + issue_number: '${{ steps.get_pr.outputs.pr_number || steps.get_pr_comment.outputs.pr_number }}', + body: 'There is a problem with the Gemini CLI PR review. Please check the [action logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.' + }) From 5ee551c602dc2e24cc89021a3b1d5592bfdbd3e7 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 13 Aug 2025 14:44:54 +0000 Subject: [PATCH 141/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 9f9c78387..b2a3290f9 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1023 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c58a7c3a5a7c61bda8d5f39422ce1a149cad1531 +version=v1.2.0+1029 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/0a45488c94e45ddc932060a724dc94f8668e740e From 9c6bdd6ef356938983f9b4ce098d9cd9fa6d9e4b Mon Sep 17 00:00:00 2001 From: HMS17 Date: Mon, 18 Aug 2025 20:18:54 -0400 Subject: [PATCH 142/289] [BI-2193] Improve/Correct Sub-Entity Tabular View --- .../brapi/v2/constants/BrAPIAdditionalInfoFields.java | 1 + .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 611a5e5fb..d14663cd2 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -57,4 +57,5 @@ public final class BrAPIAdditionalInfoFields { public static final String OBS_UNIT_ID = "obsUnitID"; public static final String GERMPLASM_NAME = "germplasmName"; public static final String SUBMISSION_NAME = "submissionName"; + public static final String EXP_UNIT_ID = "expUnitID"; } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index b11983b56..158ce952a 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -573,6 +573,8 @@ public BrAPIObservationUnit createSubObservationUnit( levelRelationships.add(expUnitLevel); position.setObservationLevelRelationships(levelRelationships); + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.EXP_UNIT_ID, expUnit.getObservationUnitName()); + // Set ObservationUnitPosition. observationUnit.setObservationUnitPosition(position); From 73aa94871ce06fd828f32f364ee2bf6c165f6620 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 20 Aug 2025 11:59:57 -0400 Subject: [PATCH 143/289] [BI-2540] - fixed GitHub Actions --- .env.template | 3 - .github/workflows/build.yml | 1 - .github/workflows/gemini-cli.yml | 304 ++++++++++++ .../gemini-issue-automated-triage.yml | 130 +++++ .../gemini-issue-scheduled-triage.yml | 123 +++++ .github/workflows/gemini-pr-review.yml | 456 ++++++++++++++++++ .github/workflows/release.yml | 1 - pom.xml | 5 +- .../brapi/v2/services/BrAPITrialService.java | 16 +- .../steps/CommitPendingImportObjectsStep.java | 61 +-- src/main/resources/application.yml | 4 +- src/main/resources/version.properties | 4 +- 12 files changed, 1065 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/gemini-cli.yml create mode 100644 .github/workflows/gemini-issue-automated-triage.yml create mode 100644 .github/workflows/gemini-issue-scheduled-triage.yml create mode 100644 .github/workflows/gemini-pr-review.yml diff --git a/.env.template b/.env.template index 711c01221..0e768adcd 100644 --- a/.env.template +++ b/.env.template @@ -79,6 +79,3 @@ TRIAL_START_DELAY=15s TRAIT_START_DELAY=20s OBSERVATION_START_DELAY=25s OBSERVATION_UNIT_START_DELAY=30s - -# Migration locations for both maven flyway plugin (configured in pom.xml) and micronaut flyway library (configured in application.yml). -FLYWAY_LOCATIONS=classpath:org/breedinginsight/db/migration,classpath:db/migration \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c220ab81b..1505d8111 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,5 +76,4 @@ jobs: GITHUB_OAUTH_CLIENT_ID: 12345678901234567890 GITHUB_OAUTH_CLIENT_SECRET: 1234567890123456789012345678901234567890 BRAPI_REFERENCE_SOURCE: breedinginsight.org - FLYWAY_LOCATIONS: filesystem:src/main/java/org/breedinginsight/db/migration,filesystem:src/main/resources/db/migration BRAPI_DOCKER_IMAGE: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.brapi_server_image || 'breedinginsight/brapi-java-server:develop' }} diff --git a/.github/workflows/gemini-cli.yml b/.github/workflows/gemini-cli.yml new file mode 100644 index 000000000..3fbaedc70 --- /dev/null +++ b/.github/workflows/gemini-cli.yml @@ -0,0 +1,304 @@ +name: '💬 Gemini CLI' + +on: + pull_request_review_comment: + types: + - 'created' + pull_request_review: + types: + - 'submitted' + issue_comment: + types: + - 'created' + +concurrency: + group: '${{ github.workflow }}-${{ github.event.issue.number }}' + cancel-in-progress: |- + ${{ github.event.sender.type == 'User' && ( github.event.issue.author_association == 'OWNER' || github.event.issue.author_association == 'MEMBER' || github.event.issue.author_association == 'COLLABORATOR') }} + +defaults: + run: + shell: 'bash' + +permissions: + contents: 'write' + id-token: 'write' + pull-requests: 'write' + issues: 'write' + +jobs: + gemini-cli: + # This condition is complex to ensure we only run when explicitly invoked. + if: |- + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'issues' && github.event.action == 'opened' && + contains(github.event.issue.body, '@gemini-cli') && + !contains(github.event.issue.body, '@gemini-cli /review') && + !contains(github.event.issue.body, '@gemini-cli /triage') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.issue.author_association) + ) || + ( + ( + github.event_name == 'issue_comment' || + github.event_name == 'pull_request_review_comment' + ) && + contains(github.event.comment.body, '@gemini-cli') && + !contains(github.event.comment.body, '@gemini-cli /review') && + !contains(github.event.comment.body, '@gemini-cli /triage') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) + ) || + ( + github.event_name == 'pull_request_review' && + contains(github.event.review.body, '@gemini-cli') && + !contains(github.event.review.body, '@gemini-cli /review') && + !contains(github.event.review.body, '@gemini-cli /triage') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.review.author_association) + ) + timeout-minutes: 10 + runs-on: 'ubuntu-latest' + + steps: + - name: 'Generate GitHub App Token' + id: 'generate_token' + if: |- + ${{ vars.APP_ID }} + uses: 'actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e' # ratchet:actions/create-github-app-token@v2 + with: + app-id: '${{ vars.APP_ID }}' + private-key: '${{ secrets.APP_PRIVATE_KEY }}' + + - name: 'Get context from event' + id: 'get_context' + env: + EVENT_NAME: '${{ github.event_name }}' + EVENT_PAYLOAD: '${{ toJSON(github.event) }}' + run: |- + set -euo pipefail + + USER_REQUEST="" + ISSUE_NUMBER="" + IS_PR="false" + + if [[ "${EVENT_NAME}" == "issues" ]]; then + USER_REQUEST=$(echo "${EVENT_PAYLOAD}" | jq -r .issue.body) + ISSUE_NUMBER=$(echo "${EVENT_PAYLOAD}" | jq -r .issue.number) + elif [[ "${EVENT_NAME}" == "issue_comment" ]]; then + USER_REQUEST=$(echo "${EVENT_PAYLOAD}" | jq -r .comment.body) + ISSUE_NUMBER=$(echo "${EVENT_PAYLOAD}" | jq -r .issue.number) + if [[ $(echo "${EVENT_PAYLOAD}" | jq -r .issue.pull_request) != "null" ]]; then + IS_PR="true" + fi + elif [[ "${EVENT_NAME}" == "pull_request_review" ]]; then + USER_REQUEST=$(echo "${EVENT_PAYLOAD}" | jq -r .review.body) + ISSUE_NUMBER=$(echo "${EVENT_PAYLOAD}" | jq -r .pull_request.number) + IS_PR="true" + elif [[ "${EVENT_NAME}" == "pull_request_review_comment" ]]; then + USER_REQUEST=$(echo "${EVENT_PAYLOAD}" | jq -r .comment.body) + ISSUE_NUMBER=$(echo "${EVENT_PAYLOAD}" | jq -r .pull_request.number) + IS_PR="true" + fi + + # Clean up user request + USER_REQUEST=$(echo "${USER_REQUEST}" | sed 's/.*@gemini-cli//' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + { + echo "user_request=${USER_REQUEST}" + echo "issue_number=${ISSUE_NUMBER}" + echo "is_pr=${IS_PR}" + } >> "${GITHUB_OUTPUT}" + + - name: 'Set up git user for commits' + run: |- + git config --global user.name 'gemini-cli[bot]' + git config --global user.email 'gemini-cli[bot]@users.noreply.github.com' + + - name: 'Checkout PR branch' + if: |- + ${{ steps.get_context.outputs.is_pr == 'true' }} + uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 + with: + token: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + repository: '${{ github.repository }}' + ref: 'refs/pull/${{ steps.get_context.outputs.issue_number }}/head' + fetch-depth: 0 + + - name: 'Checkout main branch' + if: |- + ${{ steps.get_context.outputs.is_pr == 'false' }} + uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 + with: + token: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + repository: '${{ github.repository }}' + fetch-depth: 0 + + - name: 'Acknowledge request' + env: + GITHUB_ACTOR: '${{ github.actor }}' + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + ISSUE_NUMBER: '${{ steps.get_context.outputs.issue_number }}' + REPOSITORY: '${{ github.repository }}' + REQUEST_TYPE: '${{ steps.get_context.outputs.request_type }}' + run: |- + set -euo pipefail + MESSAGE="@${GITHUB_ACTOR} I've received your request and I'm working on it now! 🤖" + if [[ -n "${MESSAGE}" ]]; then + gh issue comment "${ISSUE_NUMBER}" \ + --body "${MESSAGE}" \ + --repo "${REPOSITORY}" + fi + + - name: 'Get description' + id: 'get_description' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + IS_PR: '${{ steps.get_context.outputs.is_pr }}' + ISSUE_NUMBER: '${{ steps.get_context.outputs.issue_number }}' + run: |- + set -euo pipefail + if [[ "${IS_PR}" == "true" ]]; then + DESCRIPTION=$(gh pr view "${ISSUE_NUMBER}" --json body --template '{{.body}}') + else + DESCRIPTION=$(gh issue view "${ISSUE_NUMBER}" --json body --template '{{.body}}') + fi + { + echo "description<> "${GITHUB_OUTPUT}" + + - name: 'Get comments' + id: 'get_comments' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + IS_PR: '${{ steps.get_context.outputs.is_pr }}' + ISSUE_NUMBER: '${{ steps.get_context.outputs.issue_number }}' + run: |- + set -euo pipefail + if [[ "${IS_PR}" == "true" ]]; then + COMMENTS=$(gh pr view "${ISSUE_NUMBER}" --json comments --template '{{range .comments}}{{.author.login}}: {{.body}}{{"\n"}}{{end}}') + else + COMMENTS=$(gh issue view "${ISSUE_NUMBER}" --json comments --template '{{range .comments}}{{.author.login}}: {{.body}}{{"\n"}}{{end}}') + fi + { + echo "comments<> "${GITHUB_OUTPUT}" + + - name: 'Run Gemini' + id: 'run_gemini' + uses: 'google-github-actions/run-gemini-cli@v0' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + REPOSITORY: '${{ github.repository }}' + USER_REQUEST: '${{ steps.get_context.outputs.user_request }}' + ISSUE_NUMBER: '${{ steps.get_context.outputs.issue_number }}' + IS_PR: '${{ steps.get_context.outputs.is_pr }}' + with: + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' + gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' + gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' + gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' + gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' + use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' + use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' + settings: |- + { + "maxSessionTurns": 50, + "telemetry": { + "enabled": false, + "target": "gcp" + } + } + prompt: |- + ## Role + + You are a helpful AI assistant invoked via a CLI interface in a GitHub workflow. You have access to tools to interact with the repository and respond to the user. + + ## Context + + - **Repository**: `${{ github.repository }}` + - **Triggering Event**: `${{ github.event_name }}` + - **Issue/PR Number**: `${{ steps.get_context.outputs.issue_number }}` + - **Is this a PR?**: `${{ steps.get_context.outputs.is_pr }}` + - **Issue/PR Description**: + `${{ steps.get_description.outputs.description }}` + - **Comments**: + `${{ steps.get_comments.outputs.comments }}` + + ## User Request + + The user has sent the following request: + `${{ steps.get_context.outputs.user_request }}` + + ## How to Respond to Issues, PR Comments, and Questions + + This workflow supports three main scenarios: + + 1. **Creating a Fix for an Issue** + - Carefully read the user request and the related issue or PR description. + - Use available tools to gather all relevant context (e.g., `gh issue view`, `gh pr view`, `gh pr diff`, `cat`, `head`, `tail`). + - Identify the root cause of the problem before proceeding. + - **Show and maintain a plan as a checklist**: + - At the very beginning, outline the steps needed to resolve the issue or address the request and post them as a checklist comment on the issue or PR (use GitHub markdown checkboxes: `- [ ] Task`). + - Example: + ``` + ### Plan + - [ ] Investigate the root cause + - [ ] Implement the fix in `file.py` + - [ ] Add/modify tests + - [ ] Update documentation + - [ ] Verify the fix and close the issue + ``` + - Use: `gh pr comment "${ISSUE_NUMBER}" --body ""` or `gh issue comment "${ISSUE_NUMBER}" --body ""` to post the initial plan. + - As you make progress, keep the checklist visible and up to date by editing the same comment (check off completed tasks with `- [x]`). + - To update the checklist: + 1. Find the comment ID for the checklist (use `gh pr comment list "${ISSUE_NUMBER}"` or `gh issue comment list "${ISSUE_NUMBER}"`). + 2. Edit the comment with the updated checklist: + - For PRs: `gh pr comment --edit --body ""` + - For Issues: `gh issue comment --edit --body ""` + 3. The checklist should only be maintained as a comment on the issue or PR. Do not track or update the checklist in code files. + - If the fix requires code changes, determine which files and lines are affected. If clarification is needed, note any questions for the user. + - Make the necessary code or documentation changes using the available tools (e.g., `write_file`). Ensure all changes follow project conventions and best practices. Reference all shell variables as `"${VAR}"` (with quotes and braces) to prevent errors. + - Run any relevant tests or checks to verify the fix works as intended. If possible, provide evidence (test output, screenshots, etc.) that the issue is resolved. + - **Branching and Committing**: + - **NEVER commit directly to the `main` branch.** + - If you are working on a **pull request** (`IS_PR` is `true`), the correct branch is already checked out. Simply commit and push to it. + - `git add .` + - `git commit -m "feat: "` + - `git push` + - If you are working on an **issue** (`IS_PR` is `false`), create a new branch for your changes. A good branch name would be `issue/${ISSUE_NUMBER}/`. + - `git checkout -b issue/${ISSUE_NUMBER}/my-fix` + - `git add .` + - `git commit -m "feat: "` + - `git push origin issue/${ISSUE_NUMBER}/my-fix` + - After pushing, you can create a pull request: `gh pr create --title "Fixes #${ISSUE_NUMBER}: " --body "This PR addresses issue #${ISSUE_NUMBER}."` + - Summarize what was changed and why in a markdown file: `write_file("response.md", "")` + - Post the response as a comment: + - For PRs: `gh pr comment "${ISSUE_NUMBER}" --body-file response.md` + - For Issues: `gh issue comment "${ISSUE_NUMBER}" --body-file response.md` + + 2. **Addressing Comments on a Pull Request** + - Read the specific comment and the context of the PR. + - Use tools like `gh pr view`, `gh pr diff`, and `cat` to understand the code and discussion. + - If the comment requests a change or clarification, follow the same process as for fixing an issue: create a checklist plan, implement, test, and commit any required changes, updating the checklist as you go. + - **Committing Changes**: The correct PR branch is already checked out. Simply add, commit, and push your changes. + - `git add .` + - `git commit -m "fix: address review comments"` + - `git push` + - If the comment is a question, answer it directly and clearly, referencing code or documentation as needed. + - Document your response in `response.md` and post it as a PR comment: `gh pr comment "${ISSUE_NUMBER}" --body-file response.md` + + 3. **Answering Any Question on an Issue** + - Read the question and the full issue context using `gh issue view` and related tools. + - Research or analyze the codebase as needed to provide an accurate answer. + - If the question requires code or documentation changes, follow the fix process above, including creating and updating a checklist plan and **creating a new branch for your changes as described in section 1.** + - Write a clear, concise answer in `response.md` and post it as an issue comment: `gh issue comment "${ISSUE_NUMBER}" --body-file response.md` + + ## Guidelines + + - **Be concise and actionable.** Focus on solving the user's problem efficiently. + - **Always commit and push your changes if you modify code or documentation.** + - **If you are unsure about the fix or answer, explain your reasoning and ask clarifying questions.** + - **Follow project conventions and best practices.** diff --git a/.github/workflows/gemini-issue-automated-triage.yml b/.github/workflows/gemini-issue-automated-triage.yml new file mode 100644 index 000000000..50a671018 --- /dev/null +++ b/.github/workflows/gemini-issue-automated-triage.yml @@ -0,0 +1,130 @@ +name: '🏷️ Gemini Automated Issue Triage' + +on: + issues: + types: + - 'opened' + - 'reopened' + issue_comment: + types: + - 'created' + workflow_dispatch: + inputs: + issue_number: + description: 'issue number to triage' + required: true + type: 'number' + +concurrency: + group: '${{ github.workflow }}-${{ github.event.issue.number }}' + cancel-in-progress: true + +defaults: + run: + shell: 'bash' + +permissions: + contents: 'read' + id-token: 'write' + issues: 'write' + statuses: 'write' + +jobs: + triage-issue: + if: |- + github.event_name == 'issues' || + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'issue_comment' && + contains(github.event.comment.body, '@gemini-cli /triage') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) + ) + timeout-minutes: 5 + runs-on: 'ubuntu-latest' + + steps: + - name: 'Checkout repository' + uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 + + - name: 'Generate GitHub App Token' + id: 'generate_token' + if: |- + ${{ vars.APP_ID }} + uses: 'actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e' # ratchet:actions/create-github-app-token@v2 + with: + app-id: '${{ vars.APP_ID }}' + private-key: '${{ secrets.APP_PRIVATE_KEY }}' + + - name: 'Run Gemini Issue Triage' + uses: 'google-github-actions/run-gemini-cli@v0' + id: 'gemini_issue_triage' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + ISSUE_TITLE: '${{ github.event.issue.title }}' + ISSUE_BODY: '${{ github.event.issue.body }}' + ISSUE_NUMBER: '${{ github.event.issue.number }}' + REPOSITORY: '${{ github.repository }}' + with: + gemini_cli_version: '${{ vars.GEMINI_CLI_VERSION }}' + gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' + gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' + gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' + gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' + use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' + use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' + settings: |- + { + "maxSessionTurns": 25, + "coreTools": [ + "run_shell_command(echo)", + "run_shell_command(gh label list)", + "run_shell_command(gh issue edit)" + ], + "telemetry": { + "enabled": false, + "target": "gcp" + } + } + prompt: |- + ## Role + + You are an issue triage assistant. Analyze the current GitHub issue + and apply the most appropriate existing labels. Use the available + tools to gather information; do not ask for information to be + provided. + + ## Steps + + 1. Run: `gh label list` to get all available labels. + 2. Review the issue title and body provided in the environment + variables: "${ISSUE_TITLE}" and "${ISSUE_BODY}". + 3. Classify issues by their kind (bug, enhancement, documentation, + cleanup, etc) and their priority (p0, p1, p2, p3). Set the + labels accoridng to the format `kind/*` and `priority/*` patterns. + 4. Apply the selected labels to this issue using: + `gh issue edit "${ISSUE_NUMBER}" --add-label "label1,label2"` + 5. If the "status/needs-triage" label is present, remove it using: + `gh issue edit "${ISSUE_NUMBER}" --remove-label "status/needs-triage"` + + ## Guidelines + + - Only use labels that already exist in the repository + - Do not add comments or modify the issue content + - Triage only the current issue + - Assign all applicable labels based on the issue content + - Reference all shell variables as "${VAR}" (with quotes and braces) + + - name: 'Post Issue Triage Failure Comment' + if: |- + ${{ failure() && steps.gemini_issue_triage.outcome == 'failure' }} + uses: 'actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea' + with: + github-token: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + script: |- + github.rest.issues.createComment({ + owner: '${{ github.repository }}'.split('/')[0], + repo: '${{ github.repository }}'.split('/')[1], + issue_number: '${{ github.event.issue.number }}', + body: 'There is a problem with the Gemini CLI issue triaging. Please check the [action logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.' + }) diff --git a/.github/workflows/gemini-issue-scheduled-triage.yml b/.github/workflows/gemini-issue-scheduled-triage.yml new file mode 100644 index 000000000..f44205973 --- /dev/null +++ b/.github/workflows/gemini-issue-scheduled-triage.yml @@ -0,0 +1,123 @@ +name: '📋 Gemini Scheduled Issue Triage' + +on: + schedule: + - cron: '0 * * * *' # Runs every hour + workflow_dispatch: + +concurrency: + group: '${{ github.workflow }}' + cancel-in-progress: true + +defaults: + run: + shell: 'bash' + +permissions: + contents: 'read' + id-token: 'write' + issues: 'write' + statuses: 'write' + +jobs: + triage-issues: + timeout-minutes: 5 + runs-on: 'ubuntu-latest' + + steps: + - name: 'Checkout repository' + uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 + + - name: 'Generate GitHub App Token' + id: 'generate_token' + if: |- + ${{ vars.APP_ID }} + uses: 'actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e' # ratchet:actions/create-github-app-token@v2 + with: + app-id: '${{ vars.APP_ID }}' + private-key: '${{ secrets.APP_PRIVATE_KEY }}' + + - name: 'Find untriaged issues' + id: 'find_issues' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + GITHUB_REPOSITORY: '${{ github.repository }}' + GITHUB_OUTPUT: '${{ github.output }}' + run: |- + set -euo pipefail + + echo '🔍 Finding issues without labels...' + NO_LABEL_ISSUES="$(gh issue list --repo "${GITHUB_REPOSITORY}" \ + --search 'is:open is:issue no:label' --json number,title,body)" + + echo '🏷️ Finding issues that need triage...' + NEED_TRIAGE_ISSUES="$(gh issue list --repo "${GITHUB_REPOSITORY}" \ + --search 'is:open is:issue label:"status/needs-triage"' --json number,title,body)" + + echo '🔄 Merging and deduplicating issues...' + ISSUES="$(echo "${NO_LABEL_ISSUES}" "${NEED_TRIAGE_ISSUES}" | jq -c -s 'add | unique_by(.number)')" + + echo '📝 Setting output for GitHub Actions...' + echo "issues_to_triage=${ISSUES}" >> "${GITHUB_OUTPUT}" + + ISSUE_COUNT="$(echo "${ISSUES}" | jq 'length')" + echo "✅ Found ${ISSUE_COUNT} issues to triage! 🎯" + + - name: 'Run Gemini Issue Triage' + if: |- + ${{ steps.find_issues.outputs.issues_to_triage != '[]' }} + uses: 'google-github-actions/run-gemini-cli@v0' + id: 'gemini_issue_triage' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + ISSUES_TO_TRIAGE: '${{ steps.find_issues.outputs.issues_to_triage }}' + REPOSITORY: '${{ github.repository }}' + with: + gemini_cli_version: '${{ vars.GEMINI_CLI_VERSION }}' + gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' + gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' + gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' + gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' + use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' + use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' + settings: |- + { + "maxSessionTurns": 25, + "coreTools": [ + "run_shell_command(echo)", + "run_shell_command(gh label list)", + "run_shell_command(gh issue edit)", + "run_shell_command(gh issue list)" + ], + "telemetry": { + "enabled": false, + "target": "gcp" + } + } + prompt: |- + ## Role + + You are an issue triage assistant. Analyze issues and apply + appropriate labels. Use the available tools to gather information; + do not ask for information to be provided. + + ## Steps + + 1. Run: `gh label list` + 2. Check environment variable: "${ISSUES_TO_TRIAGE}" (JSON array + of issues) + 3. For each issue, apply labels: + `gh issue edit "${ISSUE_NUMBER}" --add-label "label1,label2"`. + If available, set labels that follow the `kind/*`, `area/*`, + and `priority/*` patterns. + 4. For each issue, if the `status/needs-triage` label is present, + remove it using: + `gh issue edit "${ISSUE_NUMBER}" --remove-label "status/needs-triage"` + + ## Guidelines + + - Only use existing repository labels + - Do not add comments + - Triage each issue independently + - Reference all shell variables as "${VAR}" (with quotes and braces) diff --git a/.github/workflows/gemini-pr-review.yml b/.github/workflows/gemini-pr-review.yml new file mode 100644 index 000000000..297c4572d --- /dev/null +++ b/.github/workflows/gemini-pr-review.yml @@ -0,0 +1,456 @@ +name: '🧐 Gemini Pull Request Review' + +on: + pull_request: + types: + - 'opened' + - 'reopened' + issue_comment: + types: + - 'created' + pull_request_review_comment: + types: + - 'created' + pull_request_review: + types: + - 'submitted' + workflow_dispatch: + inputs: + pr_number: + description: 'PR number to review' + required: true + type: 'number' + +concurrency: + group: '${{ github.workflow }}-${{ github.head_ref || github.ref }}' + cancel-in-progress: true + +defaults: + run: + shell: 'bash' + +permissions: + contents: 'read' + id-token: 'write' + issues: 'write' + pull-requests: 'write' + statuses: 'write' + +jobs: + review-pr: + if: |- + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'pull_request' && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) + ) || + ( + ( + ( + github.event_name == 'issue_comment' && + github.event.issue.pull_request + ) || + github.event_name == 'pull_request_review_comment' + ) && + contains(github.event.comment.body, '@gemini-cli /review') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) + ) || + ( + github.event_name == 'pull_request_review' && + contains(github.event.review.body, '@gemini-cli /review') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.review.author_association) + ) + timeout-minutes: 5 + runs-on: 'ubuntu-latest' + + steps: + - name: 'Checkout PR code' + uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 + + - name: 'Generate GitHub App Token' + id: 'generate_token' + if: |- + ${{ vars.APP_ID }} + uses: 'actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e' # ratchet:actions/create-github-app-token@v2 + with: + app-id: '${{ vars.APP_ID }}' + private-key: '${{ secrets.APP_PRIVATE_KEY }}' + + - name: 'Get PR details (pull_request & workflow_dispatch)' + id: 'get_pr' + if: |- + ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + EVENT_NAME: '${{ github.event_name }}' + WORKFLOW_PR_NUMBER: '${{ github.event.inputs.pr_number }}' + PULL_REQUEST_NUMBER: '${{ github.event.pull_request.number }}' + run: |- + set -euo pipefail + + if [[ "${EVENT_NAME}" = "workflow_dispatch" ]]; then + PR_NUMBER="${WORKFLOW_PR_NUMBER}" + else + PR_NUMBER="${PULL_REQUEST_NUMBER}" + fi + + echo "pr_number=${PR_NUMBER}" >> "${GITHUB_OUTPUT}" + + # Get PR details + PR_DATA="$(gh pr view "${PR_NUMBER}" --json title,body,additions,deletions,changedFiles,baseRefName,headRefName)" + echo "pr_data=${PR_DATA}" >> "${GITHUB_OUTPUT}" + + # Get file changes + CHANGED_FILES="$(gh pr diff "${PR_NUMBER}" --name-only)" + { + echo "changed_files<> "${GITHUB_OUTPUT}" + + + - name: 'Get PR details (issue_comment)' + id: 'get_pr_comment' + if: |- + ${{ github.event_name == 'issue_comment' }} + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + COMMENT_BODY: '${{ github.event.comment.body }}' + PR_NUMBER: '${{ github.event.issue.number }}' + run: |- + set -euo pipefail + + echo "pr_number=${PR_NUMBER}" >> "${GITHUB_OUTPUT}" + + # Extract additional instructions from comment + ADDITIONAL_INSTRUCTIONS="$( + echo "${COMMENT_BODY}" | sed 's/.*@gemini-cli \/review//' | xargs + )" + echo "additional_instructions=${ADDITIONAL_INSTRUCTIONS}" >> "${GITHUB_OUTPUT}" + + # Get PR details + PR_DATA="$(gh pr view "${PR_NUMBER}" --json title,body,additions,deletions,changedFiles,baseRefName,headRefName)" + echo "pr_data=${PR_DATA}" >> "${GITHUB_OUTPUT}" + + # Get file changes + CHANGED_FILES="$(gh pr diff "${PR_NUMBER}" --name-only)" + { + echo "changed_files<> "${GITHUB_OUTPUT}" + + - name: 'Run Gemini PR Review' + uses: 'google-github-actions/run-gemini-cli@v0' + id: 'gemini_pr_review' + env: + GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + PR_NUMBER: '${{ steps.get_pr.outputs.pr_number || steps.get_pr_comment.outputs.pr_number }}' + PR_DATA: '${{ steps.get_pr.outputs.pr_data || steps.get_pr_comment.outputs.pr_data }}' + CHANGED_FILES: '${{ steps.get_pr.outputs.changed_files || steps.get_pr_comment.outputs.changed_files }}' + ADDITIONAL_INSTRUCTIONS: '${{ steps.get_pr.outputs.additional_instructions || steps.get_pr_comment.outputs.additional_instructions }}' + REPOSITORY: '${{ github.repository }}' + with: + gemini_cli_version: '${{ vars.GEMINI_CLI_VERSION }}' + gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' + gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' + gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' + gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' + use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' + use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' + settings: |- + { + "maxSessionTurns": 20, + "mcpServers": { + "github": { + "command": "docker", + "args": [ + "run", + "-i", + "--rm", + "-e", + "GITHUB_PERSONAL_ACCESS_TOKEN", + "ghcr.io/github/github-mcp-server" + ], + "includeTools": [ + "create_pending_pull_request_review", + "add_comment_to_pending_review", + "submit_pending_pull_request_review" + ], + "env": { + "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_TOKEN}" + } + } + }, + "coreTools": [ + "run_shell_command(echo)", + "run_shell_command(gh pr view)", + "run_shell_command(gh pr diff)", + "run_shell_command(cat)", + "run_shell_command(head)", + "run_shell_command(tail)", + "run_shell_command(grep)" + ], + "telemetry": { + "enabled": false, + "target": "gcp" + } + } + prompt: |- + ## Role + + You are an expert code reviewer. You have access to tools to gather + PR information and perform the review on GitHub. Use the available tools to + gather information; do not ask for information to be provided. + + ## Requirements + 1. All feedback must be left on GitHub. + 2. Any output that is not left in GitHub will not be seen. + + ## Steps + + Start by running these commands to gather the required data: + 1. Run: echo "${REPOSITORY}" to get the github repository in / format + 2. Run: echo "${PR_DATA}" to get PR details (JSON format) + 3. Run: echo "${CHANGED_FILES}" to get the list of changed files + 4. Run: echo "${PR_NUMBER}" to get the PR number + 5. Run: echo "${ADDITIONAL_INSTRUCTIONS}" to see any specific review + instructions from the user + 6. Run: gh pr diff "${PR_NUMBER}" to see the full diff and reference + Context section to understand it + 7. For any specific files, use: cat filename, head -50 filename, or + tail -50 filename + 8. If ADDITIONAL_INSTRUCTIONS contains text, prioritize those + specific areas or focus points in your review. Common instruction + examples: "focus on security", "check performance", "review error + handling", "check for breaking changes" + + ## Guideline + ### Core Guideline(Always applicable) + + 1. Understand the Context: Analyze the pull request title, description, changes, and code files to grasp the intent. + 2. Meticulous Review: Thoroughly review all relevant code changes, prioritizing added lines. Consider the specified + focus areas and any provided style guide. + 3. Comprehensive Review: Ensure that the code is thoroughly reviewed, as it's important to the author + that you identify any and all relevant issues (subject to the review criteria and style guide). + Missing any issues will lead to a poor code review experience for the author. + 4. Constructive Feedback: + * Provide clear explanations for each concern. + * Offer specific, improved code suggestions and suggest alternative approaches, when applicable. + Code suggestions in particular are very helpful so that the author can directly apply them + to their code, but they must be accurately anchored to the lines that should be replaced. + 5. Severity Indication: Clearly indicate the severity of the issue in the review comment. + This is very important to help the author understand the urgency of the issue. + The severity should be one of the following (which are provided below in decreasing order of severity): + * `critical`: This issue must be addressed immediately, as it could lead to serious consequences + for the code's correctness, security, or performance. + * `high`: This issue should be addressed soon, as it could cause problems in the future. + * `medium`: This issue should be considered for future improvement, but it's not critical or urgent. + * `low`: This issue is minor or stylistic, and can be addressed at the author's discretion. + 6. Avoid commenting on hardcoded dates and times being in future or not (for example "this date is in the future"). + * Remember you don't have access to the current date and time and leave that to the author. + 7. Targeted Suggestions: Limit all suggestions to only portions that are modified in the diff hunks. + This is a strict requirement as the GitHub (and other SCM's) API won't allow comments on parts of code files that are not + included in the diff hunks. + 8. Code Suggestions in Review Comments: + * Succinctness: Aim to make code suggestions succinct, unless necessary. Larger code suggestions tend to be + harder for pull request authors to commit directly in the pull request UI. + * Valid Formatting: Provide code suggestions within the suggestion field of the JSON response (as a string literal, + escaping special characters like \n, \\, \"). Do not include markdown code blocks in the suggestion field. + Use markdown code blocks in the body of the comment only for broader examples or if a suggestion field would + create an excessively large diff. Prefer the suggestion field for specific, targeted code changes. + * Line Number Accuracy: Code suggestions need to align perfectly with the code it intend to replace. + Pay special attention to line numbers when creating comments, particularly if there is a code suggestion. + Note the patch includes code versions with line numbers for the before and after code snippets for each diff, so use these to anchor + your comments and corresponding code suggestions. + * Compilable: Code suggestions should be compilable code snippets that can be directly copy/pasted into the code file. + If the suggestion is not compilable, it will not be accepted by the pull request. Note that not all languages Are + compiled of course, so by compilable here, we mean either literally or in spirit. + * Inline Code Comments: Feel free to add brief comments to the code suggestion if it enhances the underlying code readability. + Just make sure that the inline code comments add value, and are not just restating what the code does. Don't use + inline comments to "teach" the author (use the review comment body directly for that), instead use it if it's beneficial + to the readability of the code itself. + 10. Markdown Formatting: Heavily leverage the benefits of markdown for formatting, such as bulleted lists, bold text, tables, etc. + 11. Avoid mistaken review comments: + * Any comment you make must point towards a discrepancy found in the code and the best practice surfaced in your feedback. + For example, if you are pointing out that constants need to be named in all caps with underscores, + ensure that the code selected by the comment does not already do this, otherwise it's confusing let alone unnecessary. + 12. Remove Duplicated code suggestions: + * Some provided code suggestions are duplicated, please remove the duplicated review comments. + 13. Don't Approve The Pull Request + 14. Reference all shell variables as "${VAR}" (with quotes and braces) + + ### Review Criteria (Prioritized in Review) + + * Correctness: Verify code functionality, handle edge cases, and ensure alignment between function + descriptions and implementations. Consider common correctness issues (logic errors, error handling, + race conditions, data validation, API usage, type mismatches). + * Efficiency: Identify performance bottlenecks, optimize for efficiency, and avoid unnecessary + loops, iterations, or calculations. Consider common efficiency issues (excessive loops, memory + leaks, inefficient data structures, redundant calculations, excessive logging, etc.). + * Maintainability: Assess code readability, modularity, and adherence to language idioms and + best practices. Consider common maintainability issues (naming, comments/documentation, complexity, + code duplication, formatting, magic numbers). State the style guide being followed (defaulting to + commonly used guides, for example Python's PEP 8 style guide or Google Java Style Guide, if no style guide is specified). + * Security: Identify potential vulnerabilities (e.g., insecure storage, injection attacks, + insufficient access controls). + + ### Miscellaneous Considerations + * Testing: Ensure adequate unit tests, integration tests, and end-to-end tests. Evaluate + coverage, edge case handling, and overall test quality. + * Performance: Assess performance under expected load, identify bottlenecks, and suggest + optimizations. + * Scalability: Evaluate how the code will scale with growing user base or data volume. + * Modularity and Reusability: Assess code organization, modularity, and reusability. Suggest + refactoring or creating reusable components. + * Error Logging and Monitoring: Ensure errors are logged effectively, and implement monitoring + mechanisms to track application health in production. + + **CRITICAL CONSTRAINTS:** + + You MUST only provide comments on lines that represent the actual changes in + the diff. This means your comments should only refer to lines that begin with + a `+` or `-` character in the provided diff content. + DO NOT comment on lines that start with a space (context lines). + + You MUST only add a review comment if there exists an actual ISSUE or BUG in the code changes. + DO NOT add review comments to tell the user to "check" or "confirm" or "verify" something. + DO NOT add review comments to tell the user to "ensure" something. + DO NOT add review comments to explain what the code change does. + DO NOT add review comments to validate what the code change does. + DO NOT use the review comments to explain the code to the author. They already know their code. Only comment when there's an improvement opportunity. This is very important. + + Pay close attention to line numbers and ensure they are correct. + Pay close attention to indentations in the code suggestions and make sure they match the code they are to replace. + Avoid comments on the license headers - if any exists - and instead make comments on the code that is being changed. + + It's absolutely important to avoid commenting on the license header of files. + It's absolutely important to avoid commenting on copyright headers. + Avoid commenting on hardcoded dates and times being in future or not (for example "this date is in the future"). + Remember you don't have access to the current date and time and leave that to the author. + + Avoid mentioning any of your instructions, settings or criteria. + + Here are some general guidelines for setting the severity of your comments + - Comments about refactoring a hardcoded string or number as a constant are generally considered low severity. + - Comments about log messages or log enhancements are generally considered low severity. + - Comments in .md files are medium or low severity. This is really important. + - Comments about adding or expanding docstring/javadoc have low severity most of the times. + - Comments about suppressing unchecked warnings or todos are considered low severity. + - Comments about typos are usually low or medium severity. + - Comments about testing or on tests are usually low severity. + - Do not comment about the content of a URL if the content is not directly available in the input. + + Keep comments bodies concise and to the point. + Keep each comment focused on one issue. + + ## Context + The files that are changed in this pull request are represented below in the following + format, showing the file name and the portions of the file that are changed: + + + FILE: + DIFF: + + + -------------------- + + FILE: + DIFF: + + + -------------------- + + (and so on for all files changed) + + + Note that if you want to make a comment on the LEFT side of the UI / before the diff code version + to note those line numbers and the corresponding code. Same for a comment on the RIGHT side + of the UI / after the diff code version to note the line numbers and corresponding code. + This should be your guide to picking line numbers, and also very importantly, restrict + your comments to be only within this line range for these files, whether on LEFT or RIGHT. + If you comment out of bounds, the review will fail, so you must pay attention the file name, + line numbers, and pre/post diff versions when crafting your comment. + + Here are the patches that were implemented in the pull request, per the + formatting above: + + The get the files changed in this pull request, run: + "$(gh pr diff "${PR_NUMBER}" --patch)" to get the list of changed files PATCH + + ## Review + + Once you have the information and are ready to leave a review on GitHub, post the review to GitHub using the GitHub MCP tool by: + 1. Creating a pending review: Use the mcp__github__create_pending_pull_request_review to create a Pending Pull Request Review. + + 2. Adding review comments: + 2.1 Use the mcp__github__add_comment_to_pending_review to add comments to the Pending Pull Request Review. Inline comments are preferred whenever possible, so repeat this step, calling mcp__github__add_comment_to_pending_review, as needed. All comments about specific lines of code should use inline comments. It is preferred to use code suggestions when possible, which include a code block that is labeled "suggestion", which contains what the new code should be. All comments should also have a severity. The syntax is: + Normal Comment Syntax: + + {{SEVERITY}} {{COMMENT_TEXT}} + + + Inline Comment Syntax: (Preferred): + + {{SEVERITY}} {{COMMENT_TEXT}} + ```suggestion + {{CODE_SUGGESTION}} + ``` + + + Prepend a severity emoji to each comment: + - 🟢 for low severity + - 🟡 for medium severity + - 🟠 for high severity + - 🔴 for critical severity + - 🔵 if severity is unclear + + Including all of this, an example inline comment would be: + + 🟢 Use camelCase for function names + ```suggestion + myFooBarFunction + ``` + + + A critical severity example would be: + + 🔴 Remove storage key from GitHub + ```suggestion + ``` + + 3. Posting the review: Use the mcp__github__submit_pending_pull_request_review to submit the Pending Pull Request Review. + + 3.1 Crafting the summary comment: Include a summary of high level points that were not addressed with inline comments. Be concise. Do not repeat details mentioned inline. + + Structure your summary comment using this exact format with markdown: + ## 📋 Review Summary + + Provide a brief 2-3 sentence overview of the PR and overall + assessment. + + ## 🔍 General Feedback + - List general observations about code quality + - Mention overall patterns or architectural decisions + - Highlight positive aspects of the implementation + - Note any recurring themes across files + + ## Final Instructions + + Remember, you are running in a VM and no one reviewing your output. Your review must be posted to GitHub using the MCP tools to create a pending review, add comments to the pending review, and submit the pending review. + + + - name: 'Post PR review failure comment' + if: |- + ${{ failure() && steps.gemini_pr_review.outcome == 'failure' }} + uses: 'actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea' + with: + github-token: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' + script: |- + github.rest.issues.createComment({ + owner: '${{ github.repository }}'.split('/')[0], + repo: '${{ github.repository }}'.split('/')[1], + issue_number: '${{ steps.get_pr.outputs.pr_number || steps.get_pr_comment.outputs.pr_number }}', + body: 'There is a problem with the Gemini CLI PR review. Please check the [action logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.' + }) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4a5be2c73..31d1bd0e6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,7 +67,6 @@ jobs: OAUTH_CLIENT_ID: 123abc OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf BRAPI_REFERENCE_SOURCE: breedinginsight.org - FLYWAY_LOCATIONS: filesystem:src/main/java/org/breedinginsight/db/migration,filesystem:src/main/resources/db/migration - name: Login to Docker Hub uses: docker/login-action@v1 with: diff --git a/pom.xml b/pom.xml index edf2b0eb9..402e23d44 100644 --- a/pom.xml +++ b/pom.xml @@ -624,7 +624,10 @@ jdbc:postgresql://${DB_SERVER}/${DB_NAME} ${DB_USER} ${DB_PASSWORD} - ${FLYWAY_LOCATIONS} + + filesystem:src/main/java/org/breedinginsight/db/migration + filesystem:src/main/resources/db/migration + diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 6b7a01fec..b11983b56 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -501,14 +501,16 @@ public BrAPIObservationUnit createSubObservationUnit( } // Set treatment factors. - List treatmentFactors = new ArrayList<>(); - for (BrAPIObservationTreatment t : expUnit.getTreatments()) { - BrAPIObservationTreatment treatment = new BrAPIObservationTreatment(); - treatment.setFactor(t.getFactor()); - treatment.setModality(t.getModality()); - treatmentFactors.add(treatment); + if (!expUnit.getTreatments().isEmpty()) { + List treatmentFactors = new ArrayList<>(); + for (BrAPIObservationTreatment t : expUnit.getTreatments()) { + BrAPIObservationTreatment treatment = new BrAPIObservationTreatment(); + treatment.setFactor(t.getFactor()); + treatment.setModality(t.getModality()); + treatmentFactors.add(treatment); + } + observationUnit.setTreatments(treatmentFactors); } - observationUnit.setTreatments(treatmentFactors); // Put level in additional info: keep this in case we decide to rename levels in future. observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, subEntityDatasetName); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index 12e6bcc99..5c318d336 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -320,10 +320,24 @@ private void updateObservationDependencyValues(PendingData pendingData, Program Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); Map> observationByHash = pendingData.getObservationByHash(); + // Create a lookup map for observations + // Key: studyName_observationUnitName (composite key) + // Value: List of observations matching the key + Map>> observationsByStudyAndUnit = new java.util.HashMap<>(); + for (PendingImportObject obsPio : observationByHash.values()) { + BrAPIObservation obs = obsPio.getBrAPIObject(); + if (obs.getAdditionalInfo() != null && obs.getAdditionalInfo().get(BrAPIAdditionalInfoFields.STUDY_NAME) != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getAdditionalInfo().get(BrAPIAdditionalInfoFields.STUDY_NAME).getAsString(), programKey); + String obsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getObservationUnitName(), programKey); + String key = studyName + "_" + obsUnitName; + observationsByStudyAndUnit.computeIfAbsent(key, k -> new ArrayList<>()).add(obsPio); + } + } + // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds observationUnitByNameNoScope.values().stream() .map(PendingImportObject::getBrAPIObject) - .forEach(obsUnit -> updateObservationDbIds(pendingData, obsUnit, programKey)); + .forEach(obsUnit -> updateObservationDbIds(observationsByStudyAndUnit, obsUnit, programKey)); // Pass the new map // Update ObservationVariable DbIds List traits = getTraitList(program); @@ -340,33 +354,26 @@ private void updateObservationDependencyValues(PendingData pendingData, Program } } - // Update each ovservation's observationUnit DbId, study DbId, and germplasm DbId - private void updateObservationDbIds(PendingData pendingData, BrAPIObservationUnit obsUnit, String programKey) { - Map> observationByHash = pendingData.getObservationByHash(); + // Update each observation's observationUnit DbId, study DbId, and germplasm DbId + private void updateObservationDbIds(Map>> observationsByStudyAndUnit, BrAPIObservationUnit obsUnit, String programKey) { // Modified signature - // FILTER LOGIC: Match on Env and Exp Unit ID - observationByHash.values() - .stream() - .filter(obs -> obs.getBrAPIObject() - .getAdditionalInfo() != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) - .getAsString() - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getStudyName(), programKey)) - && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject().getObservationUnitName(), programKey) - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey)) - ) - .forEach(obs -> { - if (StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { - obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); - } - obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); - obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId()); - }); + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getStudyName(), programKey); + String obsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey); + String key = studyName + "_" + obsUnitName; + + List> matchingObservations = observationsByStudyAndUnit.get(key); + + if (matchingObservations != null) { + for (PendingImportObject obsPio : matchingObservations) { + BrAPIObservation obs = obsPio.getBrAPIObject(); + + if (StringUtils.isBlank(obs.getObservationUnitDbId())) { + obs.setObservationUnitDbId(obsUnit.getObservationUnitDbId()); + } + obs.setStudyDbId(obsUnit.getStudyDbId()); + obs.setGermplasmDbId(obsUnit.getGermplasmDbId()); + } + } } private List getTraitList(Program program) { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 2aa779ac6..76cf54b28 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -114,7 +114,9 @@ jackson: flyway: datasources: default: - locations: ${FLYWAY_LOCATIONS:classpath:org/breedinginsight/db/migration,classpath:db/migration} + locations: + - classpath:org/breedinginsight/db/migration + - classpath:db/migration enabled: true placeholders: default-url: ${brapi.server.default-url} diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 5095a193c..b2a3290f9 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1017 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/229857725413b75b8c0923bf283063d6026ac04e +version=v1.2.0+1029 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/0a45488c94e45ddc932060a724dc94f8668e740e From 07518d91e1daee208415a7d6cb0c1ca8f3d9daa1 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 20 Aug 2025 16:51:13 +0000 Subject: [PATCH 144/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index b2a3290f9..74eae63dc 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1029 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/0a45488c94e45ddc932060a724dc94f8668e740e +version=v1.2.0+1033 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fdb0f0eddeeb0e4e9cb30db99a92c913b7d59434 From 9ff30b0438404922ce5a7442885209ceb243b298 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:35:35 -0400 Subject: [PATCH 145/289] [BI-2540] - fixed release action --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 31d1bd0e6..a526b060e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,6 +66,8 @@ jobs: JWT_SECRET: ${{ secrets.JWT_SECRET }} OAUTH_CLIENT_ID: 123abc OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf + GITHUB_OAUTH_CLIENT_ID: 12345678901234567890 + GITHUB_OAUTH_CLIENT_SECRET: 1234567890123456789012345678901234567890 BRAPI_REFERENCE_SOURCE: breedinginsight.org - name: Login to Docker Hub uses: docker/login-action@v1 From 1e0597d04a5f652772a1a7cf211dc5de4bdfcc7c Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 20 Aug 2025 18:35:50 +0000 Subject: [PATCH 146/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 74eae63dc..1eb031cba 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1033 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fdb0f0eddeeb0e4e9cb30db99a92c913b7d59434 +version=v1.2.0+1035 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/9ff30b0438404922ce5a7442885209ceb243b298 From 2d2f406b7f7bf3000240bde1bc2201cd81bb1f40 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 25 Aug 2025 15:00:32 +0000 Subject: [PATCH 147/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1eb031cba..3c1ba2484 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1035 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/9ff30b0438404922ce5a7442885209ceb243b298 +version=v1.2.0+1037 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1e0597d04a5f652772a1a7cf211dc5de4bdfcc7c From 498259006a9d04c95806e7fea8458f9223958840 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 25 Aug 2025 11:09:12 -0400 Subject: [PATCH 148/289] [BI-2540] - fixed versioner-develop.yml action --- .github/workflows/versioner-develop.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/versioner-develop.yml b/.github/workflows/versioner-develop.yml index 3d2dd8217..9822f045c 100644 --- a/.github/workflows/versioner-develop.yml +++ b/.github/workflows/versioner-develop.yml @@ -80,6 +80,8 @@ jobs: JWT_SECRET: ${{ secrets.JWT_SECRET }} OAUTH_CLIENT_ID: 123abc OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf + GITHUB_OAUTH_CLIENT_ID: 12345678901234567890 + GITHUB_OAUTH_CLIENT_SECRET: 1234567890123456789012345678901234567890 BRAPI_REFERENCE_SOURCE: breedinginsight.org - name: Login to Docker Hub uses: docker/login-action@v1 From 27c1dfcf4bb6c1705865c2e14c150b96ec3ef5d7 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 25 Aug 2025 15:10:22 +0000 Subject: [PATCH 149/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 3c1ba2484..dfad02a5d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1037 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1e0597d04a5f652772a1a7cf211dc5de4bdfcc7c +version=v1.2.0+1039 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/498259006a9d04c95806e7fea8458f9223958840 From 779766c8c92de09d38d7032e73d3fe8c54a87aad Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 25 Aug 2025 15:08:09 -0400 Subject: [PATCH 150/289] patched Java-based migration Java-based migrations run out-of-order and can't conflict with the lastest SQL-based migrations --- .../db/migration/V1_32_0__Set_Dev_Admin_Email.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java b/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java index 8ffcc02e7..6085698b8 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java @@ -78,8 +78,12 @@ private void addConstraint(Context context) throws SQLException { // Add new constraint try (Statement altTable = context.getConnection().createStatement()) { + // NOTE: because Java-based migrations don't run at build time, they end up running last (out of order) on + // application startup. If they haven't been applied to the database already. For that reason, the reference + // to column `orcid` was changed to `oauth_id` 2025-08-25 to allow this migration to run after + // V1.34.0__rename-orcid.sql. String sql = "ALTER TABLE bi_user\n" + - "ADD CONSTRAINT " +CONSTRAINT_NAME+ " CHECK ( (email IS NOT NULL ) OR (orcid IS NULL) ) ;"; + "ADD CONSTRAINT " +CONSTRAINT_NAME+ " CHECK ( (email IS NOT NULL ) OR (oauth_id IS NULL) ) ;"; log.debug(sql); altTable.executeUpdate(sql); } From 01bfe8c17895fc137dce36d93277eb6416a9a182 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 25 Aug 2025 19:08:23 +0000 Subject: [PATCH 151/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index dfad02a5d..ee89610b8 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1039 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/498259006a9d04c95806e7fea8458f9223958840 +version=v1.2.0+1041 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/779766c8c92de09d38d7032e73d3fe8c54a87aad From fcf0efad7dd7f2290fa0ae2f328a03c283b3d84c Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 25 Aug 2025 15:18:11 -0400 Subject: [PATCH 152/289] [BI-2540] - added placeholders for GitHub OAuth variables --- src/main/resources/application.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 76cf54b28..8ec7aa907 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -60,8 +60,8 @@ micronaut: user-info: url: ${OAUTH_OPENID_USERINFOURL:`https://sandbox.orcid.org/oauth/userinfo`} github: - client-id: ${GITHUB_OAUTH_CLIENT_ID} - client-secret: ${GITHUB_OAUTH_CLIENT_SECRET} + client-id: ${GITHUB_OAUTH_CLIENT_ID:`placeholder`} + client-secret: ${GITHUB_OAUTH_CLIENT_SECRET:`placeholder`} scopes: - user:email - read:user From 6a33d36213e95463a8aa558eec8cc1801c720e89 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 25 Aug 2025 19:18:26 +0000 Subject: [PATCH 153/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index ee89610b8..6dfa8f69a 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1041 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/779766c8c92de09d38d7032e73d3fe8c54a87aad +version=v1.2.0+1043 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fcf0efad7dd7f2290fa0ae2f328a03c283b3d84c From 85abdce79bee7d43816835a1750d4e6668f52d7b Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 26 Aug 2025 12:55:23 -0400 Subject: [PATCH 154/289] fix for migration handle both in-order and out-of-order execution --- .../V1_32_0__Set_Dev_Admin_Email.java | 57 +++++++++++++++---- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java b/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java index 6085698b8..36e3a6700 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java @@ -69,23 +69,56 @@ private void deleteUserChris(Context context) throws SQLException { private void addConstraint(Context context) throws SQLException { final String CONSTRAINT_NAME = "email_orcid"; - // First, drop the constraint if it already exist. + // First, drop the constraint if it already exists. try (Statement altTable = context.getConnection().createStatement()) { - String sql = "ALTER TABLE bi_user DROP CONSTRAINT IF EXISTS "+ CONSTRAINT_NAME; + String sql = "ALTER TABLE bi_user DROP CONSTRAINT IF EXISTS " + CONSTRAINT_NAME; log.debug(sql); altTable.executeUpdate(sql); } - // Add new constraint - try (Statement altTable = context.getConnection().createStatement()) { - // NOTE: because Java-based migrations don't run at build time, they end up running last (out of order) on - // application startup. If they haven't been applied to the database already. For that reason, the reference - // to column `orcid` was changed to `oauth_id` 2025-08-25 to allow this migration to run after - // V1.34.0__rename-orcid.sql. - String sql = "ALTER TABLE bi_user\n" + - "ADD CONSTRAINT " +CONSTRAINT_NAME+ " CHECK ( (email IS NOT NULL ) OR (oauth_id IS NULL) ) ;"; - log.debug(sql); - altTable.executeUpdate(sql); + /* NOTE: + * (1) Because Java-based migrations don't run at build time, they end up running last (out of order) on + * application startup. If they haven't been applied to the database already. For that reason, the reference + * to column `orcid` was changed to `oauth_id` 2025-08-25 to allow this migration to run after + * V1.34.0__rename-orcid.sql. + * + * (2) At test time, however, the build has already run, but a fresh database is set up at runtime. + * Migrations run in order in this context, meaning this Java-based migration runs before + * V1.34.0__rename-orcid.sql, hence the logic below to check for the existence of the old column. + */ + + // Determine if the old column, orcid, exists (true when migrations run in order). + String orcidExistsSQL = "SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='bi_user' AND column_name='orcid') AS exists;"; + Boolean orcidExists = false; + try (Statement existsQuery = context.getConnection().createStatement()) { + var resultSet = existsQuery.executeQuery(orcidExistsSQL); + if (resultSet.next()) { + orcidExists = resultSet.getBoolean("exists"); + } + } + + if (orcidExists) { + try (Statement altTable = context.getConnection().createStatement()) { + + String oldSql = "ALTER TABLE bi_user\n" + + "ADD CONSTRAINT " + CONSTRAINT_NAME + " CHECK ( (email IS NOT NULL ) OR (orcid IS NULL) ) ;"; + + log.debug("Attempting to reference `orcid` column."); + altTable.executeUpdate(oldSql); + log.debug(oldSql); + log.debug("Successfully referenced `orcid` column."); + } + } else { + try (Statement newAltTable = context.getConnection().createStatement()) { + String newSql = "ALTER TABLE bi_user\n" + + "ADD CONSTRAINT " +CONSTRAINT_NAME+ " CHECK ( (email IS NOT NULL ) OR (oauth_id IS NULL) ) ;"; + + log.debug("Referencing `orcid` column failed."); + log.debug("Attempting to reference `oauth_id` column."); + newAltTable.executeUpdate(newSql); + log.debug(newSql); + log.debug("Successfully referenced `oauth_id` column."); + } } } } \ No newline at end of file From cb7ed255fd9a670a4efc35a0b2ed83ec3d48f1ea Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 26 Aug 2025 16:55:44 +0000 Subject: [PATCH 155/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 6dfa8f69a..84ddc5ad1 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,6 +15,6 @@ # -version=v1.2.0+1043 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fcf0efad7dd7f2290fa0ae2f328a03c283b3d84c +version=v1.2.0+1045 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/85abdce79bee7d43816835a1750d4e6668f52d7b From 57aebce2e1feffbed558bbb7b79a7dab5fee04f8 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 28 Aug 2025 15:39:18 +0000 Subject: [PATCH 156/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index a018e5553..5e95f8de2 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1027 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/64c0cf856256620b0f2e9cabf962a18257730bca \ No newline at end of file +version=v1.2.0+1049 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/49d7ef65f83856883b40cd3182c14d388eed7859 \ No newline at end of file From c1ea5ac9187396731b0a79d53d1369f526022a26 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 28 Aug 2025 16:20:08 +0000 Subject: [PATCH 157/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 5e95f8de2..40e236110 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1049 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/49d7ef65f83856883b40cd3182c14d388eed7859 \ No newline at end of file +version=v1.2.0+1051 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a137a7c034e4309f1e0c235161831be78141fe52 \ No newline at end of file From 91d08483b2fdb844937e991145e88a7dfebd08f7 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 28 Aug 2025 15:52:41 -0400 Subject: [PATCH 158/289] fix merge issue --- .../brapi/v2/BrAPIGermplasmController.java | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java index 85646f3d8..850ed8138 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java @@ -221,25 +221,6 @@ public HttpResponse>>> getGermplasm( return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, "Error parsing requested date format"); } } - @Get("/programs/{programId}/germplasm/lists/{listDbId}/export{?fileExtension}") - @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") - @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) - public HttpResponse germplasmListExport( - @PathVariable("programId") UUID programId, @PathVariable("listDbId") String listDbId, @QueryValue(defaultValue = "XLSX") String fileExtension) { - String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; - try { - FileType extension = Enum.valueOf(FileType.class, fileExtension); - DownloadFile germplasmListFile = germplasmService.exportGermplasmList(programId, listDbId, extension); - HttpResponse germplasmListExport = HttpResponse.ok(germplasmListFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+germplasmListFile.getFileName()+extension.getExtension()); - return germplasmListExport; - } - catch (Exception e) { - log.info(e.getMessage(), e); - e.printStackTrace(); - HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); - return response; - } - } @Get("/programs/{programId}/germplasm/lists/{listDbId}/export{?fileExtension}") @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") From b769afce5fe9dae801c0469bf410d1d7c494420c Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 28 Aug 2025 19:53:33 +0000 Subject: [PATCH 159/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 40e236110..1770a8fce 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1051 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a137a7c034e4309f1e0c235161831be78141fe52 \ No newline at end of file +version=v1.2.0+1053 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/91d08483b2fdb844937e991145e88a7dfebd08f7 \ No newline at end of file From 0a030718b0b626689e4e19a7acf17be481ac6add Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 28 Aug 2025 16:36:16 -0400 Subject: [PATCH 160/289] fix test --- .../brapps/importer/ExperimentFileImportTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 86437bc5e..8d31d62b7 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1204,7 +1204,7 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { // In the first upload, only 1 trait should be present. List initialTraits = List.of(traits.get(0)); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), initialTraits), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), initialTraits, false), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1229,13 +1229,13 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "1"); // Send overwrite parameters in request body to allow the append workflow to work normally. Map userData = Map.of("overwrite", "true", "overwriteReason", "testing"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), userData, true, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true), userData, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); From 0c250e557f83b92914e9042e88bc4c7242c13426 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 28 Aug 2025 20:36:36 +0000 Subject: [PATCH 161/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1770a8fce..56ce1b70c 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1053 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/91d08483b2fdb844937e991145e88a7dfebd08f7 \ No newline at end of file +version=v1.2.0+1055 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/0a030718b0b626689e4e19a7acf17be481ac6add \ No newline at end of file From be92ef72e1e0409fc425ff4746ceaefe19774407 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 28 Aug 2025 21:29:16 +0000 Subject: [PATCH 162/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 56ce1b70c..2a01d0796 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1055 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/0a030718b0b626689e4e19a7acf17be481ac6add \ No newline at end of file +version=v1.2.0+1057 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/38f69eef2a5b12c8f19b8f7fcf4c10250ca664d5 \ No newline at end of file From f13bbe720b36a7461b0d687b23fb4f82821b3deb Mon Sep 17 00:00:00 2001 From: HMS17 Date: Tue, 2 Sep 2025 11:15:39 -0400 Subject: [PATCH 163/289] [BI-2110] - Unit Test Fix --- .../brapps/importer/ExperimentFileImportTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index ab3823856..0e0fa4af0 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1460,7 +1460,7 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { // In the first upload, only 1 trait should be present. List initialTraits = List.of(traits.get(0)); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), initialTraits, false), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), initialTraits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1491,7 +1491,7 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { // Send overwrite parameters in request body to allow the append workflow to work normally. Map userData = Map.of("overwrite", "true", "overwriteReason", "testing"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true), userData, true, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), userData, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); From 6f760ae7a02f0d3910d2bb0ddf140de89d28548d Mon Sep 17 00:00:00 2001 From: HMS17 Date: Wed, 10 Sep 2025 19:07:22 -0400 Subject: [PATCH 164/289] [BI-2110] - Validation Fix, Enabled Append when more than one germplasm --- .../experiment/ExperimentUtilities.java | 20 ++++++++++--------- .../steps/CommitPendingImportObjectsStep.java | 3 ++- ...ulateExistingPendingImportObjectsStep.java | 3 ++- .../service/ObservationUnitService.java | 3 ++- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 2f22d7e9e..cf6ff0cf2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -162,26 +162,28 @@ public static List importRowsToExperimentObservations(Lis * @return a String representing the unique key for the observation unit */ public static String createObservationUnitKey(ExperimentObservation importRow) { - // Extract the environment and experimental unit ID from the ExperimentObservation object + // Extract the environment and experimental unit ID and germplasm GID from the ExperimentObservation object // and pass them to the createObservationUnitKey method - return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId(), importRow.getGermplasm().getAccessionNumber()); //todo check right one } /** * Create Observation Unit Key * - * This method takes in the name of a study and the name of an observation unit and concatenates them to create a unique key. + * This method takes in the name of a study and the name of an observation unit and the germplasm GID and concatenates them to create a unique key. + * Germplasm GID needed due to how repeated measures are created and named * - * If one or both of the inputs is null, returns an empty string since not a valid combination + * If any of the inputs are null, returns an empty string since not a valid combination * * @param studyName The name of the study * @param obsUnitName The name of the observation unit - * @return A string representing the unique key formed by concatenating the study name and observation unit name + * @param germplasmGID The GID of the germplasm + * @return A string representing the unique key formed by concatenating the study name and observation unit name and germplasm GID */ - public static String createObservationUnitKey(String studyName, String obsUnitName) { - // Concatenate the study name and observation unit name to create the unique key - if (studyName != null && obsUnitName != null) { - return studyName + obsUnitName; + public static String createObservationUnitKey(String studyName, String obsUnitName, String germplasmGID) { + // Concatenate the study name and observation unit name to create the unique key //todo needs to take in more because repeated measures in append + if (studyName != null && obsUnitName != null && germplasmGID != null) { + return studyName + obsUnitName + germplasmGID; } else { return ""; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index 5c318d336..0fa6a0146 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -179,7 +179,8 @@ public void process(ProcessContext processContext, ProcessedData processedData) // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); - String createdObsUnit_key = ExperimentUtilities.createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); + String createdObservationUnit_GermplasmGID = createdObservationUnit.getGermplasmDbId(); + String createdObsUnit_key = ExperimentUtilities.createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName, createdObservationUnit_GermplasmGID); observationUnitByNameNoScope.get(createdObsUnit_key) .getBrAPIObject() .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index e611ec46f..7d6b33a3b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -513,8 +513,9 @@ private Map fetchExistingObservations(List refe String studyName = studyNameByDbId.get(obs.getStudyDbId()); String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); + String germplasmGID = obs.getGermplasmDbId(); - String key = ExperimentUtilities.getObservationHash(ExperimentUtilities.createObservationUnitKey(studyName, ouName), variableName, studyName); + String key = ExperimentUtilities.getObservationHash(ExperimentUtilities.createObservationUnitKey(studyName, ouName, germplasmGID), variableName, studyName); return Map.entry(key, obs); }) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index d66507558..44f91b5d8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -160,7 +160,8 @@ public Map> mapPendingUnitByNa pio.getBrAPIObject().getObservationUnitName(), program.getKey() ); - pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), pio); + String germplasmGID = pio.getBrAPIObject().getGermplasmDbId(); + pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName, germplasmGID), pio); //todo try here dangerous } return pendingUnitByNameNoScope; From 9616246898a8ce66e0414dc60e17a745afbd9370 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Mon, 15 Sep 2025 10:36:37 -0400 Subject: [PATCH 165/289] [BI-2110] - Remove unneeded todos --- .../services/processors/experiment/ExperimentUtilities.java | 4 ++-- .../processors/experiment/service/ObservationUnitService.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index cf6ff0cf2..7aa306ecd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -164,7 +164,7 @@ public static List importRowsToExperimentObservations(Lis public static String createObservationUnitKey(ExperimentObservation importRow) { // Extract the environment and experimental unit ID and germplasm GID from the ExperimentObservation object // and pass them to the createObservationUnitKey method - return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId(), importRow.getGermplasm().getAccessionNumber()); //todo check right one + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId(), importRow.getGermplasm().getAccessionNumber()); } /** @@ -181,7 +181,7 @@ public static String createObservationUnitKey(ExperimentObservation importRow) { * @return A string representing the unique key formed by concatenating the study name and observation unit name and germplasm GID */ public static String createObservationUnitKey(String studyName, String obsUnitName, String germplasmGID) { - // Concatenate the study name and observation unit name to create the unique key //todo needs to take in more because repeated measures in append + // Concatenate the study name and observation unit name and germplasm GID to create the unique key if (studyName != null && obsUnitName != null && germplasmGID != null) { return studyName + obsUnitName + germplasmGID; } else { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index 44f91b5d8..e4d306e6d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -161,7 +161,7 @@ public Map> mapPendingUnitByNa program.getKey() ); String germplasmGID = pio.getBrAPIObject().getGermplasmDbId(); - pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName, germplasmGID), pio); //todo try here dangerous + pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName, germplasmGID), pio); } return pendingUnitByNameNoScope; From 88f5c9393bbebac7052dd457ff033a81d1833bb4 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 21 Oct 2025 18:20:29 +0000 Subject: [PATCH 166/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 2a01d0796..89cd0012c 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1057 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/38f69eef2a5b12c8f19b8f7fcf4c10250ca664d5 \ No newline at end of file +version=v1.2.0+1059 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/174f9dc14cef3ba7076e7cd4c0b3ec84d8162984 \ No newline at end of file From de0a598e1fd87957cc83418ed0dd67f17dbf989d Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Thu, 6 Nov 2025 11:23:47 -0500 Subject: [PATCH 167/289] [BI-2110] Missing check on validation --- .../AppendOverwritePhenotypesWorkflow.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 7743e08cd..67811154b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -123,6 +123,14 @@ public Optional process(ImportServiceContext context) { // Validate the import AppendOverwriteMiddlewareContext validatedImportContext = this.validationMiddleware.process(workflowContext); + //Stop and return any validation errors, needs to be done before processing to avoid null pointer exceptions + Optional validationErrorOptional = Optional + .ofNullable(validatedImportContext.getAppendOverwriteWorkflowContext().getValidationErrors()); + if (validationErrorOptional.isPresent() && validationErrorOptional.get().hasErrors()){ + result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.of(new ValidatorException(validationErrorOptional.get())))); + return result; + } + // Process the import preview AppendOverwriteMiddlewareContext processedPreviewContext = this.importPreviewMiddleware.process(validatedImportContext); From d63ca33589be16b76323b563d1e0e5c3b5008288 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Thu, 6 Nov 2025 11:32:05 -0500 Subject: [PATCH 168/289] [BI-2110] Added missing imports --- .../appendoverwrite/AppendOverwritePhenotypesWorkflow.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 67811154b..ab1990ed2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -19,6 +19,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; @@ -37,6 +38,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.services.exceptions.ValidatorException; import javax.inject.Inject; import javax.inject.Singleton; From a4bf18a4fbbd85cafbafe2ffd9f15c83fe0ae40c Mon Sep 17 00:00:00 2001 From: HMS17 Date: Tue, 18 Nov 2025 09:24:48 -0500 Subject: [PATCH 169/289] [BI-2110] - Adding GID to key generation --- .../experiment/ExperimentUtilities.java | 19 ++++++++++--------- .../steps/CommitPendingImportObjectsStep.java | 4 ++-- ...ulateExistingPendingImportObjectsStep.java | 4 ++-- .../service/ObservationUnitService.java | 3 ++- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 7aa306ecd..1cdfcfdaf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -156,34 +156,35 @@ public static List importRowsToExperimentObservations(Lis } /** - * This method generates a unique key for an observation unit based on the environment and experimental unit ID. + * This method generates a unique key for an observation unit based on the environment and experimental unit ID and germplasm GID. * - * @param importRow the ExperimentObservation object containing the environment and experimental unit ID + * @param importRow the ExperimentObservation object containing the environment and experimental unit ID and germplasm GID * @return a String representing the unique key for the observation unit */ public static String createObservationUnitKey(ExperimentObservation importRow) { // Extract the environment and experimental unit ID and germplasm GID from the ExperimentObservation object // and pass them to the createObservationUnitKey method - return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId(), importRow.getGermplasm().getAccessionNumber()); + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId(), importRow.getGid()); } /** * Create Observation Unit Key * - * This method takes in the name of a study and the name of an observation unit and the germplasm GID and concatenates them to create a unique key. - * Germplasm GID needed due to how repeated measures are created and named + * This method takes in the name of a study and the name of an observation unit and the germplasm name and concatenates them to create a unique key. + * Sub-observation unit name needed when repeated measures due to how they are created and named * * If any of the inputs are null, returns an empty string since not a valid combination * * @param studyName The name of the study * @param obsUnitName The name of the observation unit - * @param germplasmGID The GID of the germplasm - * @return A string representing the unique key formed by concatenating the study name and observation unit name and germplasm GID + * @param germplasmGID The germplasm gid + * @return A string representing the unique key formed by concatenating the study name and observation unit name and germplasm gid */ public static String createObservationUnitKey(String studyName, String obsUnitName, String germplasmGID) { - // Concatenate the study name and observation unit name and germplasm GID to create the unique key + // Concatenate the study name and observation unit name and germplasm gid to create the unique key if (studyName != null && obsUnitName != null && germplasmGID != null) { - return studyName + obsUnitName + germplasmGID; + String keyDelim = "@*"; + return studyName + keyDelim + obsUnitName + keyDelim + germplasmGID; } else { return ""; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index 0fa6a0146..4e0663066 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -179,8 +179,8 @@ public void process(ProcessContext processContext, ProcessedData processedData) // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); - String createdObservationUnit_GermplasmGID = createdObservationUnit.getGermplasmDbId(); - String createdObsUnit_key = ExperimentUtilities.createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName, createdObservationUnit_GermplasmGID); + String createdObservationUnit_GID = createdObservationUnit.getAdditionalInfo().get("gid").getAsString(); + String createdObsUnit_key = ExperimentUtilities.createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName, createdObservationUnit_GID); observationUnitByNameNoScope.get(createdObsUnit_key) .getBrAPIObject() .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index 7d6b33a3b..11f83bbec 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -513,9 +513,9 @@ private Map fetchExistingObservations(List refe String studyName = studyNameByDbId.get(obs.getStudyDbId()); String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); - String germplasmGID = obs.getGermplasmDbId(); + String germplasmGID = obs.getAdditionalInfo().get("gid").toString(); - String key = ExperimentUtilities.getObservationHash(ExperimentUtilities.createObservationUnitKey(studyName, ouName, germplasmGID), variableName, studyName); + String key = ExperimentUtilities.getObservationHash(ExperimentUtilities.createObservationUnitKey(studyName, ouName, germplasmGID), variableName, studyName); return Map.entry(key, obs); }) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index e4d306e6d..3c387f061 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -160,7 +160,8 @@ public Map> mapPendingUnitByNa pio.getBrAPIObject().getObservationUnitName(), program.getKey() ); - String germplasmGID = pio.getBrAPIObject().getGermplasmDbId(); + String germplasmGID = pio.getBrAPIObject().getAdditionalInfo().get("gid").getAsString(); + pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName, germplasmGID), pio); } From adce131e87df572de6c3f732ed89bdd34bd7fe68 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Wed, 19 Nov 2025 10:34:24 -0500 Subject: [PATCH 170/289] [BI-2110] - Changed validation handling to avoid nullpointerexception --- .../AppendOverwritePhenotypesWorkflow.java | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index ab1990ed2..1f815c80d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -19,6 +19,8 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; @@ -72,6 +74,33 @@ public AppendOverwritePhenotypesWorkflow(AppendOverwriteIDValidation expUnitIDVa this.brapiCommitMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link(brAPICommit); } + /** + * Determines if any validation or process errors are present in the context and handles result accordingly + * + * @param context The import service context containing upload, data, program, user, commit flag, and workflow information. + * @param result the ImportWorkflowResult to be modified in response to any errors found in context + * @return Pair containing a Boolean indicating whether any errors were found, and the potentially modified ImportWorkflowResult + */ + public Pair> checkForExistingErrors(AppendOverwriteMiddlewareContext context, Optional result){ + // Stop and return any validation errors + Optional validationErrorOptional = Optional + .ofNullable(context.getAppendOverwriteWorkflowContext().getValidationErrors()); + if (validationErrorOptional.isPresent() && validationErrorOptional.get().hasErrors()){ + result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.of(new ValidatorException(validationErrorOptional.get())))); + return new ImmutablePair<>(true, result); + } + + // Stop and return any errors that occurred while processing + Optional previewException = Optional + .ofNullable(context.getAppendOverwriteWorkflowContext().getProcessError()); + if (previewException.isPresent()) { + log.debug(String.format("%s in %s", previewException.get().getException().getClass().getName(), previewException.get().getLocalTransactionName())); + result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(previewException.get().getException()))); + return new ImmutablePair<>(true, result); + } + return new ImmutablePair<>(false, result); + } + /** * Processes the import workflow based on the provided import service context. * If the provided context is not valid or if the workflow is not equal to the context workflow, returns an empty Optional. @@ -108,6 +137,8 @@ public Optional process(ImportServiceContext context) { return result; } + Pair> errors; + // Build the workflow context for processing the import ImportContext importContext = ImportContext.builder() .upload(context.getUpload()) @@ -125,25 +156,16 @@ public Optional process(ImportServiceContext context) { // Validate the import AppendOverwriteMiddlewareContext validatedImportContext = this.validationMiddleware.process(workflowContext); - //Stop and return any validation errors, needs to be done before processing to avoid null pointer exceptions - Optional validationErrorOptional = Optional - .ofNullable(validatedImportContext.getAppendOverwriteWorkflowContext().getValidationErrors()); - if (validationErrorOptional.isPresent() && validationErrorOptional.get().hasErrors()){ - result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.of(new ValidatorException(validationErrorOptional.get())))); - return result; - } + //Stop and return any validation or process errors, needs to be done before processing import preview to avoid null pointer exceptions + errors = checkForExistingErrors(validatedImportContext, result); + if (errors.getLeft()) return errors.getRight(); // Process the import preview AppendOverwriteMiddlewareContext processedPreviewContext = this.importPreviewMiddleware.process(validatedImportContext); - // Stop and return any errors that occurred while processing - Optional previewException = Optional - .ofNullable(processedPreviewContext.getAppendOverwriteWorkflowContext().getProcessError()); - if (previewException.isPresent()) { - log.debug(String.format("%s in %s", previewException.get().getException().getClass().getName(), previewException.get().getLocalTransactionName())); - result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(previewException.get().getException()))); - return result; - } + // Stop and return any validation or process errors + errors = checkForExistingErrors(validatedImportContext, result); + if (errors.getLeft()) return errors.getRight(); // Build and return the preview response ImportPreviewResponse response = new ImportPreviewResponse(); From 6fea369871dbc16ea90afaa24dbe80b85b3d74c3 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 25 Nov 2025 14:48:20 +0000 Subject: [PATCH 171/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 89cd0012c..dbd376385 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1059 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/174f9dc14cef3ba7076e7cd4c0b3ec84d8162984 \ No newline at end of file +version=v1.2.0+1061 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/0b5b6025560ace8801f21b527088b98013de5bd1 \ No newline at end of file From f232934123de642a3a79537d4c0052fb819565ad Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 25 Nov 2025 19:47:54 +0000 Subject: [PATCH 172/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index dbd376385..312995081 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1061 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/0b5b6025560ace8801f21b527088b98013de5bd1 \ No newline at end of file +version=v1.2.0+1063 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a6382550f40df7ce78889dae23964d0b979cac7c \ No newline at end of file From ba9cc28b1485abd82d592fed1aad32c3485c29ad Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Fri, 5 Dec 2025 16:28:49 -0500 Subject: [PATCH 173/289] codex first pass --- .../v1/controller/ExperimentController.java | 4 + .../v2/dao/BrAPIObservationLevelDAO.java | 90 ++++++++ .../brapi/v2/dao/BrAPIObservationUnitDAO.java | 30 +++ .../brapi/v2/services/BrAPITrialService.java | 207 +++++++++++++----- .../services/lock/DistributedLockService.java | 61 ++++++ .../SubEntityDatasetLockIntegrationTest.java | 108 +++++++++ .../lock/DistributedLockServiceTest.java | 57 +++++ 7 files changed, 499 insertions(+), 58 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java create mode 100644 src/main/java/org/breedinginsight/services/lock/DistributedLockService.java create mode 100644 src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java create mode 100644 src/test/java/org/breedinginsight/services/lock/DistributedLockServiceTest.java diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index 8619a140a..fe5436cd6 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -5,6 +5,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.*; +import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.server.types.files.StreamedFile; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; @@ -134,6 +135,9 @@ public HttpResponse> createSubEntityDataset( Response response = new Response(experimentService.createSubEntityDataset(programOptional.get(), experimentId, datasetRequest)); return HttpResponse.ok(response); + } catch (HttpStatusException e) { + log.info(e.getMessage()); + return HttpResponse.status(e.getStatus(), e.getMessage()); } catch (Exception e){ log.info(e.getMessage()); return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java new file mode 100644 index 000000000..1af5aa33a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java @@ -0,0 +1,90 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2.dao; + +import com.google.gson.Gson; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.Request; +import okhttp3.RequestBody; +import org.brapi.client.v2.JSON; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; +import org.breedinginsight.model.DatasetLevel; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.BrAPIDAOUtil; + +import javax.inject.Inject; +import javax.inject.Singleton; + +@Slf4j +@Singleton +public class BrAPIObservationLevelDAO { + + private static final MediaType JSON_MEDIA_TYPE = MediaType.get("application/json"); + private final BrAPIDAOUtil brAPIDAOUtil; + private final Gson gson = new JSON().getGson(); + + @Inject + public BrAPIObservationLevelDAO(BrAPIDAOUtil brAPIDAOUtil) { + this.brAPIDAOUtil = brAPIDAOUtil; + } + + public HttpResponse createObservationLevelName(Program program, String levelName, DatasetLevel levelOrder) throws ApiException { + HttpUrl url = HttpUrl.parse(brAPIDAOUtil.getProgramBrAPIBaseUrl(program.getId())) + .newBuilder() + .addPathSegment("observationlevelnames") + .build(); + BrAPIObservationUnitHierarchyLevel level = new BrAPIObservationUnitHierarchyLevel() + .levelName(levelName); + if (levelOrder != null) { + level.setLevelOrder(levelOrder.getValue()); + } + RequestBody body = RequestBody.create(gson.toJson(level), JSON_MEDIA_TYPE); + var request = new Request.Builder() + .url(url) + .post(body) + .addHeader("Content-Type", "application/json") + .build(); + return brAPIDAOUtil.makeCall(request); + } + + public void deleteObservationLevelName(Program program, String levelName) { + HttpUrl url = HttpUrl.parse(brAPIDAOUtil.getProgramBrAPIBaseUrl(program.getId())) + .newBuilder() + .addPathSegment("observationlevelnames") + .addPathSegment(levelName) + .build(); + var request = new Request.Builder() + .url(url) + .delete() + .addHeader("Content-Type", "application/json") + .build(); + try { + HttpResponse response = brAPIDAOUtil.makeCall(request); + if (response.getStatus() != HttpStatus.OK && response.getStatus() != HttpStatus.NO_CONTENT && response.getStatus() != HttpStatus.ACCEPTED) { + log.warn("Observation level delete returned status {} for {}", response.getStatus(), levelName); + } + } catch (Exception e) { + log.warn("Failed to delete observation level {} during rollback", levelName, e); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index 9749bf093..5a86c2337 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -26,6 +26,8 @@ import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import okhttp3.HttpUrl; +import okhttp3.Request; import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; @@ -442,4 +444,32 @@ private void preprocessObservationUnits(List brapiObservat } } } + + public void deleteObservationUnits(Collection observationUnitDbIds, UUID programId) { + if (observationUnitDbIds == null || observationUnitDbIds.isEmpty()) { + return; + } + String baseUrl = brAPIDAOUtil.getProgramBrAPIBaseUrl(programId); + for (String ouDbId : observationUnitDbIds) { + if (StringUtils.isBlank(ouDbId)) { + continue; + } + HttpUrl url = HttpUrl.parse(baseUrl) + .newBuilder() + .addPathSegment("observationunits") + .addPathSegment(ouDbId) + .build(); + Request request = new Request.Builder() + .url(url) + .delete() + .addHeader("Content-Type", "application/json") + .build(); + try { + brAPIDAOUtil.makeCall(request); + } catch (Exception e) { + log.warn("Failed to delete observation unit {} during rollback", ouDbId, e); + } + } + repopulateCache(programId); + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 158ce952a..96337c026 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -6,7 +6,10 @@ import com.github.filosganga.geogson.model.positions.SinglePosition; import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; +import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.http.server.types.files.StreamedFile; import lombok.extern.slf4j.Slf4j; @@ -41,12 +44,14 @@ import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; +import org.breedinginsight.services.lock.DistributedLockService; import javax.inject.Inject; import javax.inject.Singleton; import java.io.IOException; import java.io.PipedInputStream; import java.io.PipedOutputStream; +import java.time.Duration; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.*; @@ -54,6 +59,7 @@ import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import java.util.concurrent.TimeoutException; import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; @@ -70,8 +76,10 @@ public class BrAPITrialService { private final BrAPIStudyDAO studyDAO; private final BrAPISeasonDAO seasonDAO; private final BrAPIObservationUnitDAO ouDAO; + private final BrAPIObservationLevelDAO observationLevelDAO; private final BrAPIGermplasmDAO germplasmDAO; private final FileMappingUtil fileMappingUtil; + private final DistributedLockService lockService; private static final String SHEET_NAME = "Data"; @Inject @@ -84,8 +92,10 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin BrAPIStudyDAO studyDAO, BrAPISeasonDAO seasonDAO, BrAPIObservationUnitDAO ouDAO, + BrAPIObservationLevelDAO observationLevelDAO, BrAPIGermplasmDAO germplasmDAO, - FileMappingUtil fileMappingUtil) { + FileMappingUtil fileMappingUtil, + DistributedLockService lockService) { this.referenceSource = referenceSource; this.trialDAO = trialDAO; @@ -96,8 +106,10 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin this.studyDAO = studyDAO; this.seasonDAO = seasonDAO; this.ouDAO = ouDAO; + this.observationLevelDAO = observationLevelDAO; this.germplasmDAO = germplasmDAO; this.fileMappingUtil = fileMappingUtil; + this.lockService = lockService; } public List getExperiments(UUID programId) throws ApiException, DoesNotExistException { @@ -189,7 +201,7 @@ public DownloadFile exportObservations( } //add obsUnitID as dynamic column with observation level appended to header - String observationLvl = ous.get(0).getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); + String observationLvl = requireObservationLevelName(ous.get(0)); columns = dynamicUpdateObsUnitIDLabel(columns, observationLvl); if (params.getDatasetId() != null) { @@ -394,57 +406,116 @@ public List getDatasetsMetadata(Program program, UUID experimen } public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEntityDatasetRequest request) throws ApiException, DoesNotExistException { - log.debug("creating sub-entity dataset: \"" + request.getName() + "\" for experiment: \"" + experimentId + "\" with: \"" + request.getRepeatedMeasures() + "\" repeated measures."); - UUID subEntityDatasetId = UUID.randomUUID(); - List subObsUnits = new ArrayList<>(); - BrAPITrial experiment = getExperiment(program, experimentId); - // Get top level dataset ObservationUnits. - DatasetMetadata topLevelDataset = DatasetUtil.getTopLevelDataset(experiment); - if (topLevelDataset == null) { - log.error("Experiment {} has no top level dataset.", experiment.getTrialDbId()); - throw new RuntimeException("Cannot create sub-entity dataset for experiment without top level dataset."); - } - - List expOUs = ouDAO.getObservationUnitsForDataset(topLevelDataset.getId().toString(), program); - for (BrAPIObservationUnit expUnit : expOUs) { - - // Get environment number from study. - String envSeqValue = studyDAO.getStudyByDbId(expUnit.getStudyDbId(), program).orElseThrow() - .getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER).getAsString(); - - for (int i=1; i<=request.getRepeatedMeasures(); i++) { - // Create subObsUnit and add to list. - subObsUnits.add( - createSubObservationUnit( - request.getName(), - Integer.toString(i), - program, - envSeqValue, - expUnit, - this.referenceSource, - subEntityDatasetId, - UUID.randomUUID() - ) - ); - } - } + final String datasetName = request.getName().trim(); + String lockKey = String.format("sub-entity-dataset:%s", experimentId); + try { + return lockService.withLock(lockKey, Duration.ofSeconds(30), Duration.ofMinutes(5), () -> { + log.debug("creating sub-entity dataset: \"{}\" for experiment: \"{}\" with: \"{}\" repeated measures.", datasetName, experimentId, request.getRepeatedMeasures()); + UUID subEntityDatasetId = UUID.randomUUID(); + List subObsUnits = new ArrayList<>(); + List createdObservationUnits = new ArrayList<>(); + boolean createdObservationLevel = false; + BrAPITrial experiment = getExperiment(program, experimentId); + DatasetMetadata topLevelDataset = DatasetUtil.getTopLevelDataset(experiment); + if (topLevelDataset == null) { + log.error("Experiment {} has no top level dataset.", experiment.getTrialDbId()); + throw new RuntimeException("Cannot create sub-entity dataset for experiment without top level dataset."); + } - List createdObservationUnits = observationUnitDAO.createBrAPIObservationUnits(subObsUnits, program.getId()); + List existingDatasets = DatasetUtil.datasetsFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); + if (existingDatasets.stream().anyMatch(dataset -> dataset.getName().equalsIgnoreCase(datasetName))) { + throw new HttpStatusException(HttpStatus.CONFLICT, "Dataset name already exists in this experiment"); + } - // Add the new dataset metadata to the datasets array in the trial's additionalInfo. - DatasetMetadata subEntityDatasetMetadata = DatasetMetadata.builder() - .id(subEntityDatasetId) - .name(request.getName()) - .level(DatasetLevel.SUB_OBS_UNIT) - .build(); - List datasets = DatasetUtil.datasetsFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); - datasets.add(subEntityDatasetMetadata); - experiment.getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, DatasetUtil.jsonArrayFromDatasets(datasets)); - // Ask the DAO to persist the updated trial. - trialDAO.updateBrAPITrial(experiment.getTrialDbId(), experiment, program.getId()); + HttpResponse levelResponse = observationLevelDAO.createObservationLevelName(program, datasetName, DatasetLevel.SUB_OBS_UNIT); + if (levelResponse.getStatus() == HttpStatus.CONFLICT) { + throw new HttpStatusException(HttpStatus.CONFLICT, "Dataset name already exists in this experiment"); + } else if (levelResponse.getStatus().getCode() < 200 || levelResponse.getStatus().getCode() >= 300) { + throw new ApiException(levelResponse.getStatus().getCode(), "Unable to create observation level: " + levelResponse.getStatus().getReason()); + } + createdObservationLevel = true; + + try { + List expOUs = ouDAO.getObservationUnitsForDataset(topLevelDataset.getId().toString(), program); + for (BrAPIObservationUnit expUnit : expOUs) { + + String envSeqValue = studyDAO.getStudyByDbId(expUnit.getStudyDbId(), program).orElseThrow() + .getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER).getAsString(); + + for (int i=1; i<=request.getRepeatedMeasures(); i++) { + subObsUnits.add( + createSubObservationUnit( + datasetName, + Integer.toString(i), + program, + envSeqValue, + expUnit, + this.referenceSource, + subEntityDatasetId, + UUID.randomUUID() + ) + ); + } + } + + createdObservationUnits = observationUnitDAO.createBrAPIObservationUnits(subObsUnits, program.getId()); + + DatasetMetadata subEntityDatasetMetadata = DatasetMetadata.builder() + .id(subEntityDatasetId) + .name(datasetName) + .level(DatasetLevel.SUB_OBS_UNIT) + .build(); + + // Refresh experiment so we merge with the latest dataset metadata and avoid clobbering concurrent updates. + BrAPITrial latestExperiment = getExperiment(program, experimentId); + List datasets = DatasetUtil.datasetsFromJson(latestExperiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); + if (datasets.stream().anyMatch(dataset -> dataset.getName().equalsIgnoreCase(datasetName))) { + throw new HttpStatusException(HttpStatus.CONFLICT, "Dataset name already exists in this experiment"); + } + datasets.add(subEntityDatasetMetadata); + latestExperiment.getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, DatasetUtil.jsonArrayFromDatasets(datasets)); + trialDAO.updateBrAPITrial(latestExperiment.getTrialDbId(), latestExperiment, program.getId()); + + return getDatasetData(program, experimentId, subEntityDatasetId, false); + } catch (Exception e) { + rollbackSubEntityDataset(program, datasetName, createdObservationUnits, createdObservationLevel); + throw e; + } + }); + } catch (TimeoutException e) { + throw new HttpStatusException(HttpStatus.SERVICE_UNAVAILABLE, "Dataset creation is busy, please retry"); + } catch (HttpStatusException e) { + throw e; + } catch (Exception e) { + if (e instanceof ApiException) { + throw (ApiException) e; + } + if (e instanceof DoesNotExistException) { + throw (DoesNotExistException) e; + } + throw new RuntimeException("Unexpected error creating sub-entity dataset", e); + } + } - // Return the new dataset. - return getDatasetData(program, experimentId, subEntityDatasetId, false); + private void rollbackSubEntityDataset(Program program, String datasetName, List createdObservationUnits, boolean createdObservationLevel) { + if (createdObservationUnits != null && !createdObservationUnits.isEmpty()) { + try { + List observationUnitDbIds = createdObservationUnits.stream() + .map(BrAPIObservationUnit::getObservationUnitDbId) + .filter(StringUtils::isNotBlank) + .collect(Collectors.toList()); + observationUnitDAO.deleteObservationUnits(observationUnitDbIds, program.getId()); + } catch (Exception err) { + log.warn("Failed to delete observation units for dataset {} during rollback", datasetName, err); + } + } + if (createdObservationLevel) { + try { + observationLevelDAO.deleteObservationLevelName(program, datasetName); + } catch (Exception err) { + log.warn("Failed to delete observation level {} during rollback", datasetName, err); + } + } } public BrAPIObservationUnit createSubObservationUnit( @@ -512,8 +583,6 @@ public BrAPIObservationUnit createSubObservationUnit( observationUnit.setTreatments(treatmentFactors); } - // Put level in additional info: keep this in case we decide to rename levels in future. - observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, subEntityDatasetName); // Put RTK in additional info. JsonElement rtk = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.RTK); if (rtk != null) { @@ -533,14 +602,14 @@ public BrAPIObservationUnit createSubObservationUnit( // ObservationLevel entry for Sub-Obs Unit. BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); - // TODO: consider removing toLowerCase() after BI-2219 is implemented. - level.setLevelName(subEntityDatasetName.toLowerCase()); + level.setLevelName(subEntityDatasetName); level.setLevelCode(Utilities.appendProgramKey(subUnitId, program.getKey(), seqVal)); level.setLevelOrder(DatasetLevel.SUB_OBS_UNIT.getValue()); position.setObservationLevel(level); // ObservationLevelRelationships. List levelRelationships = new ArrayList<>(); + levelRelationships.add(level); // ObservationLevelRelationships for block. BrAPIObservationUnitLevelRelationship expBlockLevel = expUnit.getObservationUnitPosition() .getObservationLevelRelationships().stream() @@ -565,8 +634,7 @@ public BrAPIObservationUnit createSubObservationUnit( } // ObservationLevelRelationships for top-level Exp Unit linking. BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); - // TODO: consider removing toLowerCase() after BI-2219 is implemented. - expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString().toLowerCase()); + expUnitLevel.setLevelName(requireObservationLevelName(expUnit)); String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId(); expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnitUUID, program.getKey(), seqVal)); expUnitLevel.setLevelOrder(DatasetLevel.EXP_UNIT.getValue()); @@ -581,6 +649,29 @@ public BrAPIObservationUnit createSubObservationUnit( return observationUnit; } + private String getObservationLevelName(BrAPIObservationUnit observationUnit) { + if (observationUnit.getObservationUnitPosition() != null + && observationUnit.getObservationUnitPosition().getObservationLevel() != null + && StringUtils.isNotBlank(observationUnit.getObservationUnitPosition().getObservationLevel().getLevelName())) { + return observationUnit.getObservationUnitPosition().getObservationLevel().getLevelName(); + } + JsonObject additionalInfo = observationUnit.getAdditionalInfo(); + if (additionalInfo != null + && additionalInfo.has(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL) + && !additionalInfo.get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).isJsonNull()) { + return additionalInfo.get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); + } + return null; + } + + private String requireObservationLevelName(BrAPIObservationUnit observationUnit) { + String levelName = getObservationLevelName(observationUnit); + if (StringUtils.isBlank(levelName)) { + throw new RuntimeException("Observation level not found for observation unit " + observationUnit.getObservationUnitDbId()); + } + return levelName; + } + private void addBrAPIObsToRecords( List dataset, BrAPITrial experiment, @@ -750,7 +841,7 @@ private Map createExportRow( row.put(ExperimentObservation.Columns.TEST_CHECK, testCheck); row.put(ExperimentObservation.Columns.EXP_TITLE, Utilities.removeProgramKey(experiment.getTrialName(), program.getKey())); row.put(ExperimentObservation.Columns.EXP_DESCRIPTION, experiment.getTrialDescription()); - row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + row.put(ExperimentObservation.Columns.EXP_UNIT, requireObservationLevelName(ou)); row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); row.put(ExperimentObservation.Columns.ENV, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); @@ -807,7 +898,7 @@ private Map createExportRow( } //Append observation level to obsUnitID - String observationLvl = ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); + String observationLvl = requireObservationLevelName(ou); row.put(observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX, ouId); return row; diff --git a/src/main/java/org/breedinginsight/services/lock/DistributedLockService.java b/src/main/java/org/breedinginsight/services/lock/DistributedLockService.java new file mode 100644 index 000000000..06f141f33 --- /dev/null +++ b/src/main/java/org/breedinginsight/services/lock/DistributedLockService.java @@ -0,0 +1,61 @@ +package org.breedinginsight.services.lock; + +import lombok.extern.slf4j.Slf4j; +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.time.Duration; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Small helper to provide a consistent pattern for distributed locks across the service layer. + */ +@Slf4j +@Singleton +public class DistributedLockService { + + private final RedissonClient redissonClient; + + @Inject + public DistributedLockService(RedissonClient redissonClient) { + this.redissonClient = redissonClient; + } + + /** + * Execute the given callback guarded by a distributed lock. + * + * @param lockKey the key for the distributed lock + * @param waitTime how long to wait to acquire the lock + * @param leaseTime how long before the lock automatically releases + * @param action the work to run while holding the lock + * @return result of the callback + * @throws TimeoutException if the lock cannot be acquired within the wait time + * @throws Exception bubbled up from the callback + */ + public T withLock(String lockKey, Duration waitTime, Duration leaseTime, Callable action) throws Exception { + RLock lock = redissonClient.getLock(lockKey); + boolean acquired = false; + try { + acquired = lock.tryLock(waitTime.toMillis(), leaseTime.toMillis(), TimeUnit.MILLISECONDS); + if (!acquired) { + throw new TimeoutException("Unable to acquire lock " + lockKey); + } + return action.call(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TimeoutException("Interrupted while acquiring lock " + lockKey); + } finally { + if (acquired && lock.isHeldByCurrentThread()) { + try { + lock.unlock(); + } catch (Exception e) { + log.warn("Failed to release lock {}", lockKey, e); + } + } + } + } +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java new file mode 100644 index 000000000..de48032bc --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java @@ -0,0 +1,108 @@ +package org.breedinginsight.brapi.v2; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpRequest; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.model.Program; +import org.junit.jupiter.api.*; + +import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.*; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class SubEntityDatasetLockIntegrationTest extends BrAPITest { + + private Program program; + private String experimentId; + + @Inject + private BrAPITestUtils brAPITestUtils; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (json, type, context) -> OffsetDateTime.parse(json.getAsString())).create(); + + @BeforeAll + void setup() throws Exception { + var setup = brAPITestUtils.setupTestProgram(super.getBrapiDsl(), gson); + program = setup.getV1(); + experimentId = setup.getV2().get(0); + } + + @Test + void concurrentDatasetCreateReturnsSingleSuccessAndConflict() throws Exception { + // Use a fresh name to avoid interference with other runs + String datasetName = "LockTest-" + UUID.randomUUID(); + JsonObject request = new JsonObject(); + request.addProperty("name", datasetName); + request.addProperty("repeatedMeasures", 1); + + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch start = new CountDownLatch(1); + + Callable call = () -> { + start.await(1, TimeUnit.SECONDS); + Flowable> response = client.exchange( + HttpRequest.POST(String.format("/programs/%s/experiments/%s/dataset", program.getId(), experimentId), request.toString()) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), + String.class + ); + return response.blockingFirst().getStatus(); + }; + + Future first = executor.submit(call); + Future second = executor.submit(call); + start.countDown(); + + HttpStatus status1 = first.get(10, TimeUnit.SECONDS); + HttpStatus status2 = second.get(10, TimeUnit.SECONDS); + executor.shutdownNow(); + + List statuses = List.of(status1, status2); + assertTrue(statuses.contains(HttpStatus.OK)); + assertTrue(statuses.contains(HttpStatus.CONFLICT)); + + // Confirm only one dataset with that name exists + Flowable> datasetsCall = client.exchange( + HttpRequest.GET(String.format("/programs/%s/experiments/%s/datasets", program.getId(), experimentId)) + .bearerAuth("test-registered-user"), + String.class + ); + HttpResponse datasetsResponse = datasetsCall.blockingFirst(); + assertEquals(HttpStatus.OK, datasetsResponse.getStatus()); + var datasetsJson = JsonParser.parseString(Objects.requireNonNull(datasetsResponse.body())).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data"); + long matching = 0; + for (int i = 0; i < datasetsJson.size(); i++) { + String name = datasetsJson.get(i).getAsJsonObject().get("name").getAsString(); + if (name.equalsIgnoreCase(datasetName)) { + matching++; + } + } + assertEquals(1, matching); + } +} diff --git a/src/test/java/org/breedinginsight/services/lock/DistributedLockServiceTest.java b/src/test/java/org/breedinginsight/services/lock/DistributedLockServiceTest.java new file mode 100644 index 000000000..4a0522c70 --- /dev/null +++ b/src/test/java/org/breedinginsight/services/lock/DistributedLockServiceTest.java @@ -0,0 +1,57 @@ +package org.breedinginsight.services.lock; + +import org.breedinginsight.DatabaseTest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class DistributedLockServiceTest extends DatabaseTest { + + private final ExecutorService executor = Executors.newFixedThreadPool(2); + + @AfterEach + void cleanup() { + executor.shutdownNow(); + } + + @Test + void secondLockAttemptTimesOutWhileFirstHolds() throws Exception { + DistributedLockService lockService = new DistributedLockService(super.getRedisConnection()); + String lockKey = "test-lock-key"; + + CountDownLatch firstAcquired = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + + Future firstCall = executor.submit(() -> + lockService.withLock(lockKey, Duration.ofMillis(500), Duration.ofSeconds(5), () -> { + firstAcquired.countDown(); + // keep the lock held until signaled + releaseFirst.await(2, TimeUnit.SECONDS); + return "first"; + }) + ); + + assertTrue(firstAcquired.await(1, TimeUnit.SECONDS), "First lock holder did not start in time"); + + assertThrows(TimeoutException.class, () -> + lockService.withLock(lockKey, Duration.ofMillis(100), Duration.ofSeconds(2), () -> "second") + ); + + releaseFirst.countDown(); + assertEquals("first", firstCall.get(2, TimeUnit.SECONDS)); + + String afterRelease = lockService.withLock(lockKey, Duration.ofMillis(500), Duration.ofSeconds(2), () -> "after-release"); + assertEquals("after-release", afterRelease); + } +} From d008c1599890884e61bc105d1c40b2178dad95fa Mon Sep 17 00:00:00 2001 From: HMS17 Date: Fri, 12 Dec 2025 08:49:07 -0500 Subject: [PATCH 174/289] [BI-2234] - Improve exp dataset export file --- .../brapi/v2/services/BrAPITrialService.java | 76 +++++++++++++++---- .../experiment/ExperimentFileColumns.java | 15 +++- 2 files changed, 72 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 158ce952a..b38af3de2 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -36,10 +36,7 @@ import org.breedinginsight.services.TraitService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; -import org.breedinginsight.utilities.DatasetUtil; -import org.breedinginsight.utilities.IntOrderComparator; -import org.breedinginsight.utilities.FileUtil; -import org.breedinginsight.utilities.Utilities; +import org.breedinginsight.utilities.*; import org.jetbrains.annotations.NotNull; import javax.inject.Inject; @@ -154,9 +151,6 @@ public DownloadFile exportObservations( new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))) : new ArrayList<>(); FileType fileType = params.getFileExtension(); - // make columns present in all exports - List columns = ExperimentFileColumns.getOrderedColumns(); - // add columns for requested dataset obsvars and timestamps log.debug(logHash + ": fetching experiment for export"); BrAPITrial experiment = getExperiment(program, experimentId); @@ -188,8 +182,29 @@ public DownloadFile exportObservations( Utilities.generateApiExceptionLogMessage(err), err); } + boolean isSubObs = isSubEntityDataset(ous); + + List columns; + + //make columns present in all exports + if (isSubObs){ + columns = ExperimentFileColumns.getOrderedColumnsSubEntity(); + } else { + columns = ExperimentFileColumns.getOrderedColumns(); + } + //add obsUnitID as dynamic column with observation level appended to header - String observationLvl = ous.get(0).getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); + if (isSubObs) { + //need to add top level obs unit ids as well + BrAPIObservationUnitLevelRelationship topLevel = ous.get(0).getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(x -> (x.getLevelOrder() != null && x.getLevelOrder().equals(0))).findFirst().orElse(null); + if (topLevel != null) { + String topObservationLvl = StringUtils.capitalize(topLevel.getLevelName()); + columns = dynamicUpdateObsUnitIDLabel(columns, topObservationLvl); + } + } + String observationLvl = ous.get(0).getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); columns = dynamicUpdateObsUnitIDLabel(columns, observationLvl); if (params.getDatasetId() != null) { @@ -220,7 +235,8 @@ public DownloadFile exportObservations( params.isIncludeTimestamps(), obsVars, studyDbIdByOUId, - programGermplasmByDbId + programGermplasmByDbId, + isSubObs ); // make export rows for OUs without observations @@ -230,7 +246,7 @@ public DownloadFile exportObservations( // Map Observation Unit to the Study it belongs to. studyDbIdByOUId.put(ouId, ou.getStudyDbId()); if (!rowByOUId.containsKey(ouId)) { - rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId)); + rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId, isSubObs)); } } } @@ -591,7 +607,8 @@ private void addBrAPIObsToRecords( boolean includeTimestamp, List obsVars, Map studyDbIdByOUId, - Map programGermplasmByDbId) throws ApiException, DoesNotExistException { + Map programGermplasmByDbId, + boolean isSubObs) throws ApiException, DoesNotExistException { Map varByDbId = new HashMap<>(); obsVars.forEach(var -> varByDbId.put(var.getObservationVariableDbId(), var)); for (BrAPIObservation obs: dataset) { @@ -609,7 +626,7 @@ private void addBrAPIObsToRecords( } else { // otherwise make a new row - Map row = createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId); + Map row = createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId, isSubObs); addObsVarDataToRow(row, obs, includeTimestamp, var, program); rowByOUId.put(ouId, row); } @@ -727,7 +744,8 @@ private Map createExportRow( Program program, BrAPIObservationUnit ou, Map studyByDbId, - Map programGermplasmByDbId) throws ApiException, DoesNotExistException { + Map programGermplasmByDbId, + boolean isSubEntity) throws ApiException, DoesNotExistException { HashMap row = new HashMap<>(); // get OU id, germplasm, and study @@ -750,7 +768,6 @@ private Map createExportRow( row.put(ExperimentObservation.Columns.TEST_CHECK, testCheck); row.put(ExperimentObservation.Columns.EXP_TITLE, Utilities.removeProgramKey(experiment.getTrialName(), program.getKey())); row.put(ExperimentObservation.Columns.EXP_DESCRIPTION, experiment.getTrialDescription()); - row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); row.put(ExperimentObservation.Columns.ENV, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); @@ -776,7 +793,6 @@ private Map createExportRow( BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); - row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); // get replicate number Optional repLevel = ou.getObservationUnitPosition() @@ -810,6 +826,28 @@ private Map createExportRow( String observationLvl = ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); row.put(observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX, ouId); + if (isSubEntity) { + BrAPIObservationUnitLevelRelationship topLevel = ou.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(x -> (x.getLevelOrder() != null && x.getLevelOrder().equals(0))).findFirst().orElse(null); + + if (topLevel != null) { + String topLvlName = StringUtils.capitalize(topLevel.getLevelName()); + row.put(ExperimentObservation.Columns.EXP_UNIT, topLvlName); + + String topLvlOuId = Utilities.removeProgramKeyAndUnknownAdditionalData(topLevel.getLevelCode(), program.getKey()); + row.put(topLvlName + " " + OBSERVATION_UNIT_ID_SUFFIX, topLvlOuId); + } + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXP_UNIT_ID).getAsString()); + + row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + row.put(ExperimentObservation.Columns.SUB_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + + } else { + row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + } + return row; } @@ -888,6 +926,10 @@ private List filterDatasetByEnvironment( .collect(Collectors.toList()); } + private boolean isSubEntityDataset(List ous){ + return (ous.get(0).getObservationUnitPosition().getObservationLevelRelationships().size() > 2); + } + private void sortDefaultForObservationUnit(List ous) { Comparator studyNameComparator = Comparator.comparing(BrAPIObservationUnit::getStudyName, new IntOrderComparator()); Comparator ouNameComparator = Comparator.comparing(BrAPIObservationUnit::getObservationUnitName, new IntOrderComparator()); @@ -899,7 +941,9 @@ private void sortDefaultForExportRows(@NotNull List> exportR Comparator> expUnitIdComparator = Comparator.comparing(row -> (row.get(Columns.EXP_UNIT_ID).toString()), new IntOrderComparator()); - exportRows.sort(envComparator.thenComparing(expUnitIdComparator)); + Comparator> subUnitIdComparator = Comparator.comparing(row -> (row.get(Columns.SUB_UNIT_ID).toString()), new IntOrderComparator()); + + exportRows.sort(envComparator.thenComparing(expUnitIdComparator).thenComparing(subUnitIdComparator)); } public BrAPIStudy getEnvironment(Program program, UUID envId) throws ApiException { diff --git a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java index 39f7b22c0..dd2e4789f 100644 --- a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java +++ b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java @@ -24,7 +24,7 @@ import java.util.List; import java.util.stream.Collectors; -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.*; public enum ExperimentFileColumns { @@ -34,13 +34,13 @@ public enum ExperimentFileColumns { EXP_TITLE(ExperimentObservation.Columns.EXP_TITLE, Column.ColumnDataType.STRING), EXP_DESCRIPTION(ExperimentObservation.Columns.EXP_DESCRIPTION, Column.ColumnDataType.STRING), EXP_UNIT(ExperimentObservation.Columns.EXP_UNIT, Column.ColumnDataType.STRING), - //SUB_OBS_UNIT(ExperimentObservation.Columns.SUB_OBS_UNIT, Column.ColumnDataType.STRING), + SUB_OBS_UNIT(ExperimentObservation.Columns.SUB_OBS_UNIT, Column.ColumnDataType.STRING), EXP_TYPE(ExperimentObservation.Columns.EXP_TYPE, Column.ColumnDataType.STRING), ENV(ExperimentObservation.Columns.ENV, Column.ColumnDataType.STRING), ENV_LOCATION(ExperimentObservation.Columns.ENV_LOCATION, Column.ColumnDataType.STRING), ENV_YEAR(ExperimentObservation.Columns.ENV_YEAR, Column.ColumnDataType.INTEGER), EXP_UNIT_ID(ExperimentObservation.Columns.EXP_UNIT_ID, Column.ColumnDataType.STRING), - //SUB_UNIT_ID(ExperimentObservation.Columns.SUB_UNIT_ID, Column.ColumnDataType.STRING), + SUB_UNIT_ID(ExperimentObservation.Columns.SUB_UNIT_ID, Column.ColumnDataType.STRING), REP_NUM(ExperimentObservation.Columns.REP_NUM, Column.ColumnDataType.INTEGER), BLOCK_NUM(ExperimentObservation.Columns.BLOCK_NUM, Column.ColumnDataType.INTEGER), ROW(ExperimentObservation.Columns.ROW, Column.ColumnDataType.STRING), @@ -52,6 +52,7 @@ public enum ExperimentFileColumns { TREATMENT_FACTORS(ExperimentObservation.Columns.TREATMENT_FACTORS, Column.ColumnDataType.STRING); private final Column column; + private static List subEntityOnlyColumns = Arrays.asList(SUB_OBS_UNIT, SUB_UNIT_ID); ExperimentFileColumns(String value, Column.ColumnDataType dataType) { this.column = new Column(value, dataType); @@ -63,6 +64,14 @@ public String toString() { } public static List getOrderedColumns() { + //Don't include subentity columns + return Arrays.stream(ExperimentFileColumns.values()) + .filter(val -> !(subEntityOnlyColumns.contains(val))) + .map(value -> value.column) + .collect(Collectors.toList()); + } + + public static List getOrderedColumnsSubEntity() { return Arrays.stream(ExperimentFileColumns.values()) .map(value -> value.column) .collect(Collectors.toList()); From 104127bf6b57a30f84e2d60fa0e86e13942eaa2a Mon Sep 17 00:00:00 2001 From: HMS17 Date: Mon, 15 Dec 2025 10:14:43 -0500 Subject: [PATCH 175/289] [BI-2234] - Code review helper method --- .../brapi/v2/services/BrAPITrialService.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index b38af3de2..9ca961920 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -135,6 +135,13 @@ private long countGermplasm(UUID programId, String trialDbId) throws ApiExceptio return obUnits.stream().map(BrAPIObservationUnit::getGermplasmDbId).distinct().count(); } + private BrAPIObservationUnitLevelRelationship getTopLevel(BrAPIObservationUnit ou) { + BrAPIObservationUnitLevelRelationship topLevel = ou.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(x -> (x.getLevelOrder() != null && x.getLevelOrder().equals(0))).findFirst().orElse(null); + return topLevel; + } + public DownloadFile exportObservations( Program program, UUID experimentId, @@ -196,9 +203,7 @@ public DownloadFile exportObservations( //add obsUnitID as dynamic column with observation level appended to header if (isSubObs) { //need to add top level obs unit ids as well - BrAPIObservationUnitLevelRelationship topLevel = ous.get(0).getObservationUnitPosition() - .getObservationLevelRelationships().stream() - .filter(x -> (x.getLevelOrder() != null && x.getLevelOrder().equals(0))).findFirst().orElse(null); + BrAPIObservationUnitLevelRelationship topLevel = getTopLevel(ous.get(0)); if (topLevel != null) { String topObservationLvl = StringUtils.capitalize(topLevel.getLevelName()); columns = dynamicUpdateObsUnitIDLabel(columns, topObservationLvl); @@ -827,9 +832,7 @@ private Map createExportRow( row.put(observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX, ouId); if (isSubEntity) { - BrAPIObservationUnitLevelRelationship topLevel = ou.getObservationUnitPosition() - .getObservationLevelRelationships().stream() - .filter(x -> (x.getLevelOrder() != null && x.getLevelOrder().equals(0))).findFirst().orElse(null); + BrAPIObservationUnitLevelRelationship topLevel = getTopLevel(ou); if (topLevel != null) { String topLvlName = StringUtils.capitalize(topLevel.getLevelName()); From 6460a2cd9e8233d95e922ffe1da2da96580dc260 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 15 Dec 2025 21:17:20 +0000 Subject: [PATCH 176/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 312995081..eda5aba8e 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1063 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a6382550f40df7ce78889dae23964d0b979cac7c \ No newline at end of file +version=v1.2.0+1065 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/05a5636ca285c99989e2c2c4fee0671e9f99e9f9 \ No newline at end of file From eb6f205698ffa22be404a87e31be0f72fedf3332 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Tue, 16 Dec 2025 08:41:04 -0500 Subject: [PATCH 177/289] [BI-2755] - improve default sub-unit sort --- .../brapi/v2/services/BrAPITrialService.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 9ca961920..9f5a93410 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -935,8 +935,16 @@ private boolean isSubEntityDataset(List ous){ private void sortDefaultForObservationUnit(List ous) { Comparator studyNameComparator = Comparator.comparing(BrAPIObservationUnit::getStudyName, new IntOrderComparator()); - Comparator ouNameComparator = Comparator.comparing(BrAPIObservationUnit::getObservationUnitName, new IntOrderComparator()); - ous.sort( (studyNameComparator).thenComparing(ouNameComparator)); + + if (isSubEntityDataset(ous)) { + Comparator subUnitComparator = Comparator.comparing(BrAPIObservationUnit::getObservationUnitName, new IntOrderComparator()); + Comparator ouNameComparator = Comparator.comparing(row -> (row.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXP_UNIT_ID).toString()), new IntOrderComparator()); + ous.sort((studyNameComparator).thenComparing(ouNameComparator).thenComparing(subUnitComparator)); + } + else { + Comparator ouNameComparator = Comparator.comparing(BrAPIObservationUnit::getObservationUnitName, new IntOrderComparator()); + ous.sort((studyNameComparator).thenComparing(ouNameComparator)); + } } private void sortDefaultForExportRows(@NotNull List> exportRows) { From 1fe79020cde8cdf2e767d43d67c4d14b0d904235 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 17 Dec 2025 18:26:14 +0000 Subject: [PATCH 178/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index eda5aba8e..db1bf9eaf 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1065 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/05a5636ca285c99989e2c2c4fee0671e9f99e9f9 \ No newline at end of file +version=v1.2.0+1067 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ec139fb01eff74b9322b96e8bf3537c11a81b2b9 \ No newline at end of file From b2785aef376aae487f391b834f4ab34ebfcb6bca Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Wed, 7 Jan 2026 14:35:59 -0500 Subject: [PATCH 179/289] Moved httpstatus codes to controller and fixed post /observationlevelnames --- .../v1/controller/ExperimentController.java | 10 ++- .../v2/dao/BrAPIObservationLevelDAO.java | 67 ++++++++++++++++--- .../brapi/v2/services/BrAPITrialService.java | 39 ++++++----- .../exceptions/CreationBusyException.java | 7 ++ 4 files changed, 92 insertions(+), 31 deletions(-) create mode 100644 src/main/java/org/breedinginsight/services/exceptions/CreationBusyException.java diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index fe5436cd6..5741e1313 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -5,7 +5,6 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.*; -import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.server.types.files.StreamedFile; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; @@ -29,7 +28,9 @@ import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.ProgramUserService; import org.breedinginsight.services.RoleService; +import org.breedinginsight.services.exceptions.AlreadyExistsException; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.CreationBusyException; import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; import javax.inject.Inject; @@ -135,9 +136,12 @@ public HttpResponse> createSubEntityDataset( Response response = new Response(experimentService.createSubEntityDataset(programOptional.get(), experimentId, datasetRequest)); return HttpResponse.ok(response); - } catch (HttpStatusException e) { + } catch (AlreadyExistsException e) { log.info(e.getMessage()); - return HttpResponse.status(e.getStatus(), e.getMessage()); + return HttpResponse.status(HttpStatus.CONFLICT, e.getMessage()); + } catch (CreationBusyException e) { + log.info(e.getMessage()); + return HttpResponse.status(HttpStatus.SERVICE_UNAVAILABLE, e.getMessage()); } catch (Exception e){ log.info(e.getMessage()); return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java index 1af5aa33a..5fad8981d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java @@ -27,10 +27,13 @@ import okhttp3.RequestBody; import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; import org.breedinginsight.model.DatasetLevel; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.BrAPIDAOUtil; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import javax.inject.Inject; import javax.inject.Singleton; @@ -48,17 +51,22 @@ public BrAPIObservationLevelDAO(BrAPIDAOUtil brAPIDAOUtil) { this.brAPIDAOUtil = brAPIDAOUtil; } - public HttpResponse createObservationLevelName(Program program, String levelName, DatasetLevel levelOrder) throws ApiException { + public HttpResponse createObservationLevelName(Program program, String levelName, DatasetLevel levelOrder, String programDbId) throws ApiException { HttpUrl url = HttpUrl.parse(brAPIDAOUtil.getProgramBrAPIBaseUrl(program.getId())) .newBuilder() .addPathSegment("observationlevelnames") .build(); - BrAPIObservationUnitHierarchyLevel level = new BrAPIObservationUnitHierarchyLevel() - .levelName(levelName); + JsonObject levelJson = new JsonObject(); + levelJson.addProperty("levelName", levelName); if (levelOrder != null) { - level.setLevelOrder(levelOrder.getValue()); + levelJson.addProperty("levelOrder", levelOrder.getValue()); } - RequestBody body = RequestBody.create(gson.toJson(level), JSON_MEDIA_TYPE); + if (programDbId != null) { + levelJson.addProperty("programDbId", programDbId); + } + JsonArray bodyArray = new JsonArray(); + bodyArray.add(levelJson); + RequestBody body = RequestBody.create(gson.toJson(bodyArray), JSON_MEDIA_TYPE); var request = new Request.Builder() .url(url) .post(body) @@ -67,11 +75,11 @@ public HttpResponse createObservationLevelName(Program program, String l return brAPIDAOUtil.makeCall(request); } - public void deleteObservationLevelName(Program program, String levelName) { + public void deleteObservationLevelName(Program program, String levelDbId) { HttpUrl url = HttpUrl.parse(brAPIDAOUtil.getProgramBrAPIBaseUrl(program.getId())) .newBuilder() .addPathSegment("observationlevelnames") - .addPathSegment(levelName) + .addPathSegment(levelDbId) .build(); var request = new Request.Builder() .url(url) @@ -81,10 +89,49 @@ public void deleteObservationLevelName(Program program, String levelName) { try { HttpResponse response = brAPIDAOUtil.makeCall(request); if (response.getStatus() != HttpStatus.OK && response.getStatus() != HttpStatus.NO_CONTENT && response.getStatus() != HttpStatus.ACCEPTED) { - log.warn("Observation level delete returned status {} for {}", response.getStatus(), levelName); + log.warn("Observation level delete returned status {} for {}", response.getStatus(), levelDbId); + } + } catch (Exception e) { + log.warn("Failed to delete observation level {}", levelDbId, e); + } + } + + public String extractObservationLevelDbId(HttpResponse response) { + try { + String body = response.getBody().orElse(null); + if (body == null || body.isBlank()) { + return null; + } + JsonElement root = JsonParser.parseString(body); + JsonArray dataArray = null; + if (root.isJsonArray()) { + dataArray = root.getAsJsonArray(); + } else if (root.isJsonObject()) { + JsonObject rootObj = root.getAsJsonObject(); + if (rootObj.has("result") && rootObj.get("result").isJsonObject()) { + JsonObject resultObj = rootObj.getAsJsonObject("result"); + if (resultObj.has("data") && resultObj.get("data").isJsonArray()) { + dataArray = resultObj.getAsJsonArray("data"); + } + } else if (rootObj.has("data") && rootObj.get("data").isJsonArray()) { + dataArray = rootObj.getAsJsonArray("data"); + } + } + if (dataArray == null || dataArray.size() == 0) { + return null; + } + for (JsonElement element : dataArray) { + if (!element.isJsonObject()) { + continue; + } + JsonObject obj = element.getAsJsonObject(); + if (obj.has("levelNameDbId")) { + return obj.get("levelNameDbId").getAsString(); + } } } catch (Exception e) { - log.warn("Failed to delete observation level {} during rollback", levelName, e); + log.warn("Failed to parse level name id from response", e); } + return null; } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 96337c026..90cbb1118 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -9,7 +9,6 @@ import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; -import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.http.server.types.files.StreamedFile; import lombok.extern.slf4j.Slf4j; @@ -37,7 +36,9 @@ import org.breedinginsight.model.Program; import org.breedinginsight.model.*; import org.breedinginsight.services.TraitService; +import org.breedinginsight.services.exceptions.AlreadyExistsException; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.CreationBusyException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.IntOrderComparator; @@ -405,7 +406,8 @@ public List getDatasetsMetadata(Program program, UUID experimen return datasets; } - public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEntityDatasetRequest request) throws ApiException, DoesNotExistException { + public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEntityDatasetRequest request) + throws ApiException, DoesNotExistException, AlreadyExistsException, CreationBusyException { final String datasetName = request.getName().trim(); String lockKey = String.format("sub-entity-dataset:%s", experimentId); try { @@ -415,6 +417,7 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt List subObsUnits = new ArrayList<>(); List createdObservationUnits = new ArrayList<>(); boolean createdObservationLevel = false; + String createdObservationLevelDbId = null; BrAPITrial experiment = getExperiment(program, experimentId); DatasetMetadata topLevelDataset = DatasetUtil.getTopLevelDataset(experiment); if (topLevelDataset == null) { @@ -424,16 +427,18 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt List existingDatasets = DatasetUtil.datasetsFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); if (existingDatasets.stream().anyMatch(dataset -> dataset.getName().equalsIgnoreCase(datasetName))) { - throw new HttpStatusException(HttpStatus.CONFLICT, "Dataset name already exists in this experiment"); + throw new AlreadyExistsException("Dataset name already exists in this experiment"); } - HttpResponse levelResponse = observationLevelDAO.createObservationLevelName(program, datasetName, DatasetLevel.SUB_OBS_UNIT); + String programDbId = program.getBrapiProgram() != null ? program.getBrapiProgram().getProgramDbId() : null; + HttpResponse levelResponse = observationLevelDAO.createObservationLevelName(program, datasetName, DatasetLevel.SUB_OBS_UNIT, programDbId); if (levelResponse.getStatus() == HttpStatus.CONFLICT) { - throw new HttpStatusException(HttpStatus.CONFLICT, "Dataset name already exists in this experiment"); + throw new AlreadyExistsException("Dataset name already exists in this experiment"); } else if (levelResponse.getStatus().getCode() < 200 || levelResponse.getStatus().getCode() >= 300) { throw new ApiException(levelResponse.getStatus().getCode(), "Unable to create observation level: " + levelResponse.getStatus().getReason()); } createdObservationLevel = true; + createdObservationLevelDbId = observationLevelDAO.extractObservationLevelDbId(levelResponse); try { List expOUs = ouDAO.getObservationUnitsForDataset(topLevelDataset.getId().toString(), program); @@ -470,7 +475,7 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt BrAPITrial latestExperiment = getExperiment(program, experimentId); List datasets = DatasetUtil.datasetsFromJson(latestExperiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); if (datasets.stream().anyMatch(dataset -> dataset.getName().equalsIgnoreCase(datasetName))) { - throw new HttpStatusException(HttpStatus.CONFLICT, "Dataset name already exists in this experiment"); + throw new AlreadyExistsException("Dataset name already exists in this experiment"); } datasets.add(subEntityDatasetMetadata); latestExperiment.getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, DatasetUtil.jsonArrayFromDatasets(datasets)); @@ -478,26 +483,20 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt return getDatasetData(program, experimentId, subEntityDatasetId, false); } catch (Exception e) { - rollbackSubEntityDataset(program, datasetName, createdObservationUnits, createdObservationLevel); + rollbackSubEntityDataset(program, datasetName, createdObservationUnits, createdObservationLevel, createdObservationLevelDbId); throw e; } }); } catch (TimeoutException e) { - throw new HttpStatusException(HttpStatus.SERVICE_UNAVAILABLE, "Dataset creation is busy, please retry"); - } catch (HttpStatusException e) { + throw new CreationBusyException("Dataset creation is busy, please retry"); + } catch (ApiException | DoesNotExistException | AlreadyExistsException | CreationBusyException e) { throw e; } catch (Exception e) { - if (e instanceof ApiException) { - throw (ApiException) e; - } - if (e instanceof DoesNotExistException) { - throw (DoesNotExistException) e; - } throw new RuntimeException("Unexpected error creating sub-entity dataset", e); } } - private void rollbackSubEntityDataset(Program program, String datasetName, List createdObservationUnits, boolean createdObservationLevel) { + private void rollbackSubEntityDataset(Program program, String datasetName, List createdObservationUnits, boolean createdObservationLevel, String createdObservationLevelDbId) { if (createdObservationUnits != null && !createdObservationUnits.isEmpty()) { try { List observationUnitDbIds = createdObservationUnits.stream() @@ -511,9 +510,13 @@ private void rollbackSubEntityDataset(Program program, String datasetName, List< } if (createdObservationLevel) { try { - observationLevelDAO.deleteObservationLevelName(program, datasetName); + if (StringUtils.isNotBlank(createdObservationLevelDbId)) { + observationLevelDAO.deleteObservationLevelName(program, createdObservationLevelDbId); + } else { + log.warn("Observation level id missing for dataset {} rollback; skipping level delete", datasetName); + } } catch (Exception err) { - log.warn("Failed to delete observation level {} during rollback", datasetName, err); + log.warn("Failed to delete observation level {} during rollback", createdObservationLevelDbId, err); } } } diff --git a/src/main/java/org/breedinginsight/services/exceptions/CreationBusyException.java b/src/main/java/org/breedinginsight/services/exceptions/CreationBusyException.java new file mode 100644 index 000000000..d32536159 --- /dev/null +++ b/src/main/java/org/breedinginsight/services/exceptions/CreationBusyException.java @@ -0,0 +1,7 @@ +package org.breedinginsight.services.exceptions; + +public class CreationBusyException extends Exception { + public CreationBusyException(String message) { + super(message); + } +} From 36b67b99ff6432d8bf244d98903f26ca4be1dc73 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Thu, 8 Jan 2026 10:59:44 -0500 Subject: [PATCH 180/289] Removed rollback code until brapi server supports observation unit deletes --- .../brapi/v2/dao/BrAPIObservationUnitDAO.java | 28 ---- .../brapi/v2/services/BrAPITrialService.java | 124 ++++++++---------- 2 files changed, 52 insertions(+), 100 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index 5a86c2337..4b761654a 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -444,32 +444,4 @@ private void preprocessObservationUnits(List brapiObservat } } } - - public void deleteObservationUnits(Collection observationUnitDbIds, UUID programId) { - if (observationUnitDbIds == null || observationUnitDbIds.isEmpty()) { - return; - } - String baseUrl = brAPIDAOUtil.getProgramBrAPIBaseUrl(programId); - for (String ouDbId : observationUnitDbIds) { - if (StringUtils.isBlank(ouDbId)) { - continue; - } - HttpUrl url = HttpUrl.parse(baseUrl) - .newBuilder() - .addPathSegment("observationunits") - .addPathSegment(ouDbId) - .build(); - Request request = new Request.Builder() - .url(url) - .delete() - .addHeader("Content-Type", "application/json") - .build(); - try { - brAPIDAOUtil.makeCall(request); - } catch (Exception e) { - log.warn("Failed to delete observation unit {} during rollback", ouDbId, e); - } - } - repopulateCache(programId); - } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 90cbb1118..7669e56fc 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -406,6 +406,21 @@ public List getDatasetsMetadata(Program program, UUID experimen return datasets; } + /** + * Creates sub-entity dataset + * TODO: Handle compensating transactions in event of failure. Currently brapi server does not support + * deleting observation units. Will need to add batch delete support for observation units before this + * can be done. + * + * @param program + * @param experimentId + * @param request + * @return + * @throws ApiException + * @throws DoesNotExistException + * @throws AlreadyExistsException + * @throws CreationBusyException + */ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEntityDatasetRequest request) throws ApiException, DoesNotExistException, AlreadyExistsException, CreationBusyException { final String datasetName = request.getName().trim(); @@ -415,9 +430,6 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt log.debug("creating sub-entity dataset: \"{}\" for experiment: \"{}\" with: \"{}\" repeated measures.", datasetName, experimentId, request.getRepeatedMeasures()); UUID subEntityDatasetId = UUID.randomUUID(); List subObsUnits = new ArrayList<>(); - List createdObservationUnits = new ArrayList<>(); - boolean createdObservationLevel = false; - String createdObservationLevelDbId = null; BrAPITrial experiment = getExperiment(program, experimentId); DatasetMetadata topLevelDataset = DatasetUtil.getTopLevelDataset(experiment); if (topLevelDataset == null) { @@ -437,55 +449,48 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt } else if (levelResponse.getStatus().getCode() < 200 || levelResponse.getStatus().getCode() >= 300) { throw new ApiException(levelResponse.getStatus().getCode(), "Unable to create observation level: " + levelResponse.getStatus().getReason()); } - createdObservationLevel = true; - createdObservationLevelDbId = observationLevelDAO.extractObservationLevelDbId(levelResponse); - - try { - List expOUs = ouDAO.getObservationUnitsForDataset(topLevelDataset.getId().toString(), program); - for (BrAPIObservationUnit expUnit : expOUs) { - - String envSeqValue = studyDAO.getStudyByDbId(expUnit.getStudyDbId(), program).orElseThrow() - .getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER).getAsString(); - - for (int i=1; i<=request.getRepeatedMeasures(); i++) { - subObsUnits.add( - createSubObservationUnit( - datasetName, - Integer.toString(i), - program, - envSeqValue, - expUnit, - this.referenceSource, - subEntityDatasetId, - UUID.randomUUID() - ) - ); - } + + List expOUs = ouDAO.getObservationUnitsForDataset(topLevelDataset.getId().toString(), program); + for (BrAPIObservationUnit expUnit : expOUs) { + + String envSeqValue = studyDAO.getStudyByDbId(expUnit.getStudyDbId(), program).orElseThrow() + .getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER).getAsString(); + + for (int i=1; i<=request.getRepeatedMeasures(); i++) { + subObsUnits.add( + createSubObservationUnit( + datasetName, + Integer.toString(i), + program, + envSeqValue, + expUnit, + this.referenceSource, + subEntityDatasetId, + UUID.randomUUID() + ) + ); } + } - createdObservationUnits = observationUnitDAO.createBrAPIObservationUnits(subObsUnits, program.getId()); + observationUnitDAO.createBrAPIObservationUnits(subObsUnits, program.getId()); - DatasetMetadata subEntityDatasetMetadata = DatasetMetadata.builder() - .id(subEntityDatasetId) - .name(datasetName) - .level(DatasetLevel.SUB_OBS_UNIT) - .build(); + DatasetMetadata subEntityDatasetMetadata = DatasetMetadata.builder() + .id(subEntityDatasetId) + .name(datasetName) + .level(DatasetLevel.SUB_OBS_UNIT) + .build(); - // Refresh experiment so we merge with the latest dataset metadata and avoid clobbering concurrent updates. - BrAPITrial latestExperiment = getExperiment(program, experimentId); - List datasets = DatasetUtil.datasetsFromJson(latestExperiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); - if (datasets.stream().anyMatch(dataset -> dataset.getName().equalsIgnoreCase(datasetName))) { - throw new AlreadyExistsException("Dataset name already exists in this experiment"); - } - datasets.add(subEntityDatasetMetadata); - latestExperiment.getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, DatasetUtil.jsonArrayFromDatasets(datasets)); - trialDAO.updateBrAPITrial(latestExperiment.getTrialDbId(), latestExperiment, program.getId()); - - return getDatasetData(program, experimentId, subEntityDatasetId, false); - } catch (Exception e) { - rollbackSubEntityDataset(program, datasetName, createdObservationUnits, createdObservationLevel, createdObservationLevelDbId); - throw e; + // Refresh experiment so we merge with the latest dataset metadata and avoid clobbering concurrent updates. + BrAPITrial latestExperiment = getExperiment(program, experimentId); + List datasets = DatasetUtil.datasetsFromJson(latestExperiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); + if (datasets.stream().anyMatch(dataset -> dataset.getName().equalsIgnoreCase(datasetName))) { + throw new AlreadyExistsException("Dataset name already exists in this experiment"); } + datasets.add(subEntityDatasetMetadata); + latestExperiment.getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, DatasetUtil.jsonArrayFromDatasets(datasets)); + trialDAO.updateBrAPITrial(latestExperiment.getTrialDbId(), latestExperiment, program.getId()); + + return getDatasetData(program, experimentId, subEntityDatasetId, false); }); } catch (TimeoutException e) { throw new CreationBusyException("Dataset creation is busy, please retry"); @@ -496,31 +501,6 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt } } - private void rollbackSubEntityDataset(Program program, String datasetName, List createdObservationUnits, boolean createdObservationLevel, String createdObservationLevelDbId) { - if (createdObservationUnits != null && !createdObservationUnits.isEmpty()) { - try { - List observationUnitDbIds = createdObservationUnits.stream() - .map(BrAPIObservationUnit::getObservationUnitDbId) - .filter(StringUtils::isNotBlank) - .collect(Collectors.toList()); - observationUnitDAO.deleteObservationUnits(observationUnitDbIds, program.getId()); - } catch (Exception err) { - log.warn("Failed to delete observation units for dataset {} during rollback", datasetName, err); - } - } - if (createdObservationLevel) { - try { - if (StringUtils.isNotBlank(createdObservationLevelDbId)) { - observationLevelDAO.deleteObservationLevelName(program, createdObservationLevelDbId); - } else { - log.warn("Observation level id missing for dataset {} rollback; skipping level delete", datasetName); - } - } catch (Exception err) { - log.warn("Failed to delete observation level {} during rollback", createdObservationLevelDbId, err); - } - } - } - public BrAPIObservationUnit createSubObservationUnit( String subEntityDatasetName, String subUnitId, From fdf5f2e24c43aa67e19780553e5e895dbdd28fde Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Thu, 8 Jan 2026 14:17:21 -0500 Subject: [PATCH 181/289] Removed unused code --- .../v2/dao/BrAPIObservationLevelDAO.java | 38 ------------------- .../brapi/v2/dao/BrAPIObservationUnitDAO.java | 2 - .../brapi/v2/services/BrAPITrialService.java | 2 +- 3 files changed, 1 insertion(+), 41 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java index 5fad8981d..e37e23dbe 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java @@ -96,42 +96,4 @@ public void deleteObservationLevelName(Program program, String levelDbId) { } } - public String extractObservationLevelDbId(HttpResponse response) { - try { - String body = response.getBody().orElse(null); - if (body == null || body.isBlank()) { - return null; - } - JsonElement root = JsonParser.parseString(body); - JsonArray dataArray = null; - if (root.isJsonArray()) { - dataArray = root.getAsJsonArray(); - } else if (root.isJsonObject()) { - JsonObject rootObj = root.getAsJsonObject(); - if (rootObj.has("result") && rootObj.get("result").isJsonObject()) { - JsonObject resultObj = rootObj.getAsJsonObject("result"); - if (resultObj.has("data") && resultObj.get("data").isJsonArray()) { - dataArray = resultObj.getAsJsonArray("data"); - } - } else if (rootObj.has("data") && rootObj.get("data").isJsonArray()) { - dataArray = rootObj.getAsJsonArray("data"); - } - } - if (dataArray == null || dataArray.size() == 0) { - return null; - } - for (JsonElement element : dataArray) { - if (!element.isJsonObject()) { - continue; - } - JsonObject obj = element.getAsJsonObject(); - if (obj.has("levelNameDbId")) { - return obj.get("levelNameDbId").getAsString(); - } - } - } catch (Exception e) { - log.warn("Failed to parse level name id from response", e); - } - return null; - } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index 4b761654a..9749bf093 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -26,8 +26,6 @@ import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import okhttp3.HttpUrl; -import okhttp3.Request; import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 7669e56fc..3648e2505 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -423,7 +423,7 @@ public List getDatasetsMetadata(Program program, UUID experimen */ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEntityDatasetRequest request) throws ApiException, DoesNotExistException, AlreadyExistsException, CreationBusyException { - final String datasetName = request.getName().trim(); + final String datasetName = request.getName().trim().toLowerCase(); String lockKey = String.format("sub-entity-dataset:%s", experimentId); try { return lockService.withLock(lockKey, Duration.ofSeconds(30), Duration.ofMinutes(5), () -> { From 10423985a5b15d6ead58de51fd6a595afddb0184 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Fri, 9 Jan 2026 14:10:53 -0500 Subject: [PATCH 182/289] Fix missing observationLevel in additionalInfo --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index df7570c64..4c217ca66 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -587,6 +587,9 @@ public BrAPIObservationUnit createSubObservationUnit( observationUnit.setTreatments(treatmentFactors); } + // Put level in additional info: keep this in case we decide to rename levels in future. + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, subEntityDatasetName); + // Put RTK in additional info. JsonElement rtk = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.RTK); if (rtk != null) { @@ -638,7 +641,7 @@ public BrAPIObservationUnit createSubObservationUnit( } // ObservationLevelRelationships for top-level Exp Unit linking. BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); - expUnitLevel.setLevelName(requireObservationLevelName(expUnit)); + expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId(); expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnitUUID, program.getKey(), seqVal)); expUnitLevel.setLevelOrder(DatasetLevel.EXP_UNIT.getValue()); From 50146d7bd0af247ffceda058f84c5d5939b7047d Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Fri, 9 Jan 2026 17:18:33 -0500 Subject: [PATCH 183/289] Fix test --- .../services/lock/DistributedLockServiceTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/org/breedinginsight/services/lock/DistributedLockServiceTest.java b/src/test/java/org/breedinginsight/services/lock/DistributedLockServiceTest.java index 4a0522c70..bb3275beb 100644 --- a/src/test/java/org/breedinginsight/services/lock/DistributedLockServiceTest.java +++ b/src/test/java/org/breedinginsight/services/lock/DistributedLockServiceTest.java @@ -3,6 +3,7 @@ import org.breedinginsight.DatabaseTest; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import java.time.Duration; import java.util.concurrent.CountDownLatch; @@ -16,6 +17,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +@TestInstance(TestInstance.Lifecycle.PER_CLASS) public class DistributedLockServiceTest extends DatabaseTest { private final ExecutorService executor = Executors.newFixedThreadPool(2); From 0d0f374114c77efa26e4bcd9cc595f5619114234 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Tue, 13 Jan 2026 16:05:17 -0500 Subject: [PATCH 184/289] Try changing fannypack url --- settings.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.xml b/settings.xml index b5281bb65..9b5fb7aa5 100644 --- a/settings.xml +++ b/settings.xml @@ -52,7 +52,7 @@ github-fannypack FannyPack github repository - https://maven.pkg.github.com/Kowalski-IO/fannypack + https://maven.pkg.github.com/BrandonKowalski/fannypack true true From 382e10f14e1303f84a3a7ae29a3245ed10bec728 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Tue, 13 Jan 2026 16:19:24 -0500 Subject: [PATCH 185/289] Update fannypack github url --- settings.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.xml b/settings.xml index b5281bb65..9b5fb7aa5 100644 --- a/settings.xml +++ b/settings.xml @@ -52,7 +52,7 @@ github-fannypack FannyPack github repository - https://maven.pkg.github.com/Kowalski-IO/fannypack + https://maven.pkg.github.com/BrandonKowalski/fannypack true true From 6a78d80ee6aea5f6ecf5b19f9bd18f7770d59fe4 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 13 Jan 2026 21:19:48 +0000 Subject: [PATCH 186/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index db1bf9eaf..7ebefd909 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1067 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ec139fb01eff74b9322b96e8bf3537c11a81b2b9 \ No newline at end of file +version=v1.2.0+1069 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/382e10f14e1303f84a3a7ae29a3245ed10bec728 \ No newline at end of file From d4b85b0ad15e9cb48221cf4db1847316cc42fd6e Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Thu, 15 Jan 2026 11:22:45 -0500 Subject: [PATCH 187/289] Fix test --- .../brapi/v2/BrAPITestUtils.java | 2 +- .../SubEntityDatasetLockIntegrationTest.java | 39 ++++++++++--------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java index 936bd401c..5f41de572 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java @@ -308,7 +308,7 @@ public Map makeExpImportRow(String environment, String expTitle) row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); row.put(ExperimentObservation.Columns.EXP_TITLE, expTitle); - row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "plot"); row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); row.put(ExperimentObservation.Columns.ENV, environment); row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); diff --git a/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java index c3fcbd1c9..d958d0b44 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java @@ -1,10 +1,6 @@ package org.breedinginsight.brapi.v2; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; +import com.google.gson.*; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpRequest; import io.micronaut.http.HttpResponse; @@ -26,8 +22,7 @@ import java.util.concurrent.*; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -67,13 +62,17 @@ void concurrentDatasetCreateReturnsSingleSuccessAndConflict() throws Exception { Callable call = () -> { start.await(1, TimeUnit.SECONDS); - Flowable> response = client.exchange( - HttpRequest.POST(String.format("/programs/%s/experiments/%s/dataset", program.getId(), experimentId), request.toString()) - .contentType(MediaType.APPLICATION_JSON) - .bearerAuth("test-registered-user"), - String.class - ); - return response.blockingFirst().getStatus(); + try { + Flowable> response = client.exchange( + HttpRequest.POST(String.format("/programs/%s/experiments/%s/dataset", program.getId(), experimentId), request.toString()) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), + String.class + ); + return response.blockingFirst().getStatus(); + } catch (io.micronaut.http.client.exceptions.HttpClientResponseException e) { + return e.getStatus(); + } }; Future first = executor.submit(call); @@ -96,12 +95,14 @@ void concurrentDatasetCreateReturnsSingleSuccessAndConflict() throws Exception { ); HttpResponse datasetsResponse = datasetsCall.blockingFirst(); assertEquals(HttpStatus.OK, datasetsResponse.getStatus()); - var datasetsJson = JsonParser.parseString(Objects.requireNonNull(datasetsResponse.body())).getAsJsonObject() - .getAsJsonObject("result") - .getAsJsonArray("data"); + JsonObject parsed = JsonParser.parseString(Objects.requireNonNull(datasetsResponse.body())).getAsJsonObject(); + JsonArray resultArray = parsed.has("result") && parsed.get("result").isJsonArray() + ? parsed.getAsJsonArray("result") + : null; long matching = 0; - for (int i = 0; i < datasetsJson.size(); i++) { - String name = datasetsJson.get(i).getAsJsonObject().get("name").getAsString(); + assertNotEquals(null, resultArray); + for (int i = 0; i < resultArray.size(); i++) { + String name = resultArray.get(i).getAsJsonObject().get("name").getAsString(); if (name.equalsIgnoreCase(datasetName)) { matching++; } From a59803e60da916ee7099fb8e2141bb802f3f27a4 Mon Sep 17 00:00:00 2001 From: "dr.phillips" Date: Tue, 20 Jan 2026 10:45:02 -0500 Subject: [PATCH 188/289] [BI-2641] WIP --- .../workflow/CreateNewExperimentWorkflow.java | 7 +- ...ulateExistingPendingImportObjectsStep.java | 305 ++++-------------- .../PopulateNewPendingImportObjectsStep.java | 7 +- .../services/ExperimentTrialService.java | 8 +- 4 files changed, 66 insertions(+), 261 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 010a14095..a6b6ea8b9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -17,11 +17,9 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; -import io.micronaut.context.annotation.Prototype; import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; -import lombok.val; import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.api.model.v1.response.ValidationErrors; @@ -35,7 +33,6 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.ImportStatusService; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; @@ -49,7 +46,6 @@ import org.breedinginsight.services.exceptions.ValidatorException; import javax.inject.Inject; -import javax.inject.Named; import java.util.*; import lombok.Getter; @@ -106,8 +102,7 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); - // TODO: eliminate or modify unnecessary populateExistingPIO step as it relies on the user supplying existing observation unit ids - ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); + ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context); populateNewPendingImportObjectsStep.process(processContext, phenotypeData); ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context, processContext.getPendingData(), phenotypeData, processedData); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index 11f83bbec..16453788a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -41,12 +41,10 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentStudyService; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentTrialService; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.model.Trait; import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.Utilities; @@ -65,13 +63,10 @@ @Slf4j public class PopulateExistingPendingImportObjectsStep { - private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; - private final BrAPITrialDAO brAPITrialDAO; private final BrAPIStudyDAO brAPIStudyDAO; private final ProgramLocationService locationService; private final BrAPIListDAO brAPIListDAO; private final BrAPIGermplasmDAO brAPIGermplasmDAO; - private final BrAPIObservationDAO brAPIObservationDAO; private final ExperimentStudyService experimentStudyService; private final ExperimentTrialService experimentTrialService; @@ -80,39 +75,34 @@ public class PopulateExistingPendingImportObjectsStep { @Inject public PopulateExistingPendingImportObjectsStep(BrAPIObservationUnitDAO brAPIObservationUnitDAO, - BrAPITrialDAO brAPITrialDAO, BrAPIStudyDAO brAPIStudyDAO, ProgramLocationService locationService, BrAPIListDAO brAPIListDAO, BrAPIGermplasmDAO brAPIGermplasmDAO, - BrAPIObservationDAO brAPIObservationDAO, ExperimentStudyService experimentStudyService, ExperimentTrialService experimentTrialService) { - this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; - this.brAPITrialDAO = brAPITrialDAO; this.brAPIStudyDAO = brAPIStudyDAO; this.locationService = locationService; this.brAPIListDAO = brAPIListDAO; this.brAPIGermplasmDAO = brAPIGermplasmDAO; - this.brAPIObservationDAO = brAPIObservationDAO; this.experimentStudyService = experimentStudyService; this.experimentTrialService = experimentTrialService; } - public ProcessContext process(ImportContext input, ProcessedPhenotypeData phenotypeData) { + public ProcessContext process(ImportContext input) { List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); Program program = input.getProgram(); // Populate pending objects with existing status - Map> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); - Map> trialByNameNoScope = experimentTrialService.initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); - Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); + Map> observationUnitByNameNoScope = new HashMap<>(); + Map> trialByNameNoScope = experimentTrialService.initializeTrialByNameNoScope(program, experimentImportRows); + Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, experimentImportRows); // interesting we're using our data model instead of brapi for locations - Map> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); - Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, trialByNameNoScope, experimentImportRows); - Map> existingGermplasmByGID = initializeExistingGermplasmByGID(program, observationUnitByNameNoScope, experimentImportRows); - Map existingObsByObsHash = fetchExistingObservations(phenotypeData.getReferencedTraits(), studyByNameNoScope, program); + Map> locationByName = initializeUniqueLocationNames(program, experimentImportRows); + Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); + Map> existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); + Map existingObsByObsHash = new HashMap<>(); PendingData existing = PendingData.builder() .observationUnitByNameNoScope(observationUnitByNameNoScope) @@ -131,117 +121,6 @@ public ProcessContext process(ImportContext input, ProcessedPhenotypeData phenot .build(); } - /** - * Initializes the observation units for the given program and experimentImportRows. - * - * @param program The program object - * @param experimentImportRows A list of ExperimentObservation objects - * @return A map of Observation Unit IDs to PendingImportObject objects - * - * @throws InternalServerException - * @throws IllegalStateException - */ - private Map> initializeObservationUnits(Program program, List experimentImportRows) { - Map> observationUnitByName = new HashMap<>(); - return observationUnitByName; - // TODO: change how ProcessContext is generated so it does not rely on this unused method -// Map rowByObsUnitId = new HashMap<>(); -// experimentImportRows.forEach(row -> { -// if (StringUtils.isNotBlank(row.getObsUnitID())) { -// if(rowByObsUnitId.containsKey(row.getObsUnitID())) { -// throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); -// } -// rowByObsUnitId.put(row.getObsUnitID(), row); -// } -// }); -// -// try { -// List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); -// -// // TODO: grab from externalReferences -// /* -// observationUnitByObsUnitId = existingObsUnits.stream() -// .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, -// (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false))); -// */ -// -// String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); -// if (existingObsUnits.size() == rowByObsUnitId.size()) { -// existingObsUnits.forEach(brAPIObservationUnit -> { -// processAndCacheObservationUnit(brAPIObservationUnit, refSource, program, observationUnitByName, rowByObsUnitId); -// -// BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) -// .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); -// -// ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); -// row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); -// row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); -// row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); -// }); -// } else { -// List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); -// missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); -// throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); -// } -// -// return observationUnitByName; -// } catch (ApiException e) { -// log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); -// throw new InternalServerException(e.toString(), e); -// } - } - - /** - * Initializes studies by name without scope. - * - * @param program The program object. - * @param trialByNameNoScope A map of trial names with their corresponding pending import objects. - * @param experimentImportRows A list of experiment observation objects. - * @return A map of study names with their corresponding pending import objects. - * @throws InternalServerException If there is an error while processing the method. - */ - private Map> initializeStudyByNameNoScope(Program program, - Map> trialByNameNoScope, - Map> observationUnitByNameNoScope, - List experimentImportRows) { - Map> studyByName = new HashMap<>(); - if (trialByNameNoScope.size() != 1) { - return studyByName; - } - - try { - initializeStudiesForExistingObservationUnits(program, studyByName, observationUnitByNameNoScope); - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (Exception e) { - log.error("Error processing studies", e); - throw new InternalServerException(e.toString(), e); - } - - List existingStudies; - Optional> trial = getTrialPIO(experimentImportRows, trialByNameNoScope); - - try { - if (trial.isEmpty()) { - // TODO: throw ValidatorException and return 422 - } - UUID experimentId = trial.get().getId(); - existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); - for (BrAPIStudy existingStudy : existingStudies) { - experimentStudyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); - } - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (Exception e) { - log.error("Error processing studies: ", e); - throw new InternalServerException(e.toString(), e); - } - - return studyByName; - } - /** * Retrieves the PendingImportObject of a BrAPITrial based on the given list of ExperimentObservation and trialByNameNoScope map. * @@ -266,53 +145,19 @@ private Optional> getTrialPIO(List> studyByName, - Map> observationUnitByNameNoScope - ) throws Exception { - Set studyDbIds = observationUnitByNameNoScope.values() - .stream() - .map(pio -> pio.getBrAPIObject() - .getStudyDbId()) - .collect(Collectors.toSet()); - - List studies = experimentStudyService.fetchStudiesByDbId(studyDbIds, program); - for (BrAPIStudy study : studies) { - experimentStudyService.processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); - } - } - /** * Initializes unique location names for a program. * * @param program The program object. - * @param studyByNameNoScope A map of study names and corresponding BrAPI study objects. * @param experimentImportRows A list of experiment observation objects for import. * @return A map of location names and their corresponding pending import objects. * @throws InternalServerException If there is an error fetching locations. */ private Map> initializeUniqueLocationNames(Program program, - Map> studyByNameNoScope, List experimentImportRows) { Map> locationByName = new HashMap<>(); - List existingLocations = new ArrayList<>(); - if(studyByNameNoScope.size() > 0) { - Set locationDbIds = studyByNameNoScope.values() - .stream() - .map(study -> study.getBrAPIObject() - .getLocationDbId()) - .collect(Collectors.toSet()); - try { - existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - + List existingLocations; List uniqueLocationNames = experimentImportRows.stream() .map(ExperimentObservation::getEnvLocation) .distinct() @@ -320,7 +165,7 @@ private Map> initializeUniqueLocati .collect(Collectors.toList()); try { - existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); + existingLocations = new ArrayList<>(locationService.getLocationsByName(uniqueLocationNames, program.getId())); } catch (ApiException e) { log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -331,19 +176,57 @@ private Map> initializeUniqueLocati } /** - * Initializes observation variable dataset by name. - * - * @param program The program associated with the dataset. - * @param trialByNameNoScope The map of trials identified by name without scope. - * @param experimentImportRows The list of experiment observation rows. - * @return The map of observation variable dataset indexed by name. + * Initializes studies by name without scope. * - * @throws InternalServerException + * @param program The program object. + * @param trialByNameNoScope A map of trial names with their corresponding pending import objects. + * @param experimentImportRows A list of experiment observation objects. + * @return A map of study names with their corresponding pending import objects. + * @throws InternalServerException If there is an error while processing the method. */ + private Map> initializeStudyByNameNoScope(Program program, + Map> trialByNameNoScope, + List experimentImportRows) { + Map> studyByName = new HashMap<>(); + if (trialByNameNoScope.size() != 1) { + return studyByName; + } + + + List existingStudies; + Optional> trial = getTrialPIO(experimentImportRows, trialByNameNoScope); + + try { + // the 'trial' variable will never be "null". + UUID experimentId = trial.get().getId(); + existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); + for (BrAPIStudy existingStudy : existingStudies) { + experimentStudyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); + } + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies: ", e); + throw new InternalServerException(e.toString(), e); + } + + return studyByName; + } + + /** + * Initializes observation variable dataset by name. + * + * @param program The program associated with the dataset. + * @param experimentImportRows The list of experiment observation rows. + * @return The map of observation variable dataset indexed by name. + * + * @throws InternalServerException + */ private Map> initializeObsVarDatasetByName(Program program, - Map> trialByNameNoScope, List experimentImportRows) { Map> obsVarDatasetByName = new HashMap<>(); + Map> trialByNameNoScope = new HashMap<>(); Optional> trialPIO = getTrialPIO(experimentImportRows, trialByNameNoScope); @@ -394,27 +277,16 @@ private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> initializeExistingGermplasmByGID(Program program, - Map> observationUnitByNameNoScope, List experimentImportRows) { Map> existingGermplasmByGID = new HashMap<>(); - List existingGermplasms = new ArrayList<>(); - if(observationUnitByNameNoScope.size() > 0) { - Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); - try { - existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } + List existingGermplasms; List uniqueGermplasmGIDs = experimentImportRows.stream() .map(ExperimentObservation::getGid) @@ -422,7 +294,7 @@ private Map> initializeExistingGermp .collect(Collectors.toList()); try { - existingGermplasms.addAll(getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); + existingGermplasms = new ArrayList<>(getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); } catch (ApiException e) { log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -461,65 +333,4 @@ private ArrayList getGermplasmByAccessionNumber( } return resultGermplasm; } - - /** - * Fetches existing observations based on the given referenced traits, studyByNameNoScope map, and program. - * - * @param referencedTraits The list of referenced traits. - * @param studyByNameNoScope The map of studies by name without scope. - * @param program The program. - * @return A map of existing observations with their unique keys. - */ - private Map fetchExistingObservations(List referencedTraits, - Map> studyByNameNoScope, - Program program) { - Set ouDbIds = new HashSet<>(); - Set variableDbIds = new HashSet<>(); - Map variableNameByDbId = new HashMap<>(); - Map ouNameByDbId = new HashMap<>(); - Map studyNameByDbId = studyByNameNoScope.values() - .stream() - .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) - .map(PendingImportObject::getBrAPIObject) - .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - - studyNameByDbId.keySet().forEach(studyDbId -> { - try { - brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { - if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { - ouDbIds.add(ou.getObservationUnitDbId()); - } - ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); - }); - } catch (ApiException e) { - throw new RuntimeException(e); - } - }); - - for (Trait referencedTrait : referencedTraits) { - variableDbIds.add(referencedTrait.getObservationVariableDbId()); - variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); - } - - List existingObservations = new ArrayList<>(); - try { - existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); - } catch (ApiException e) { - throw new RuntimeException(e); - } - - return existingObservations.stream() - .map(obs -> { - String studyName = studyNameByDbId.get(obs.getStudyDbId()); - String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); - String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); - String germplasmGID = obs.getAdditionalInfo().get("gid").toString(); - - String key = ExperimentUtilities.getObservationHash(ExperimentUtilities.createObservationUnitKey(studyName, ouName, germplasmGID), variableName, studyName); - - return Map.entry(key, obs); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index ab4f41505..57d62f492 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -1,4 +1,4 @@ -/* + /* * See the NOTICE file distributed with this work for additional information * regarding copyright ownership. * @@ -517,7 +517,7 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Import String key = ExperimentUtilities.createObservationUnitKey(importRow); // NOTE: removed other workflow - if (observationUnitByNameNoScope.containsKey(key)) { + if (observationUnitByNameNoScope!=null && observationUnitByNameNoScope.containsKey(key)) { pio = observationUnitByNameNoScope.get(key); } else { String germplasmName = ""; @@ -552,6 +552,9 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Import } else { pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); } + if(observationUnitByNameNoScope==null){ + observationUnitByNameNoScope = new HashMap<>(); + } observationUnitByNameNoScope.put(key, pio); } return pio; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java index b94fb6789..2e3a2ec33 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java @@ -164,18 +164,14 @@ private void processAndCacheTrial( * Initializes trials by name without scope for the given program. * * @param program the program to initialize trials for - * @param observationUnitByNameNoScope a map of observation units by name without scope * @param experimentImportRows a list of experiment observation rows * @return a map of trials by name with pending import objects * - * @throws InternalServerException */ - public Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, - List experimentImportRows) { + public Map> initializeTrialByNameNoScope(Program program, + List experimentImportRows) { Map> trialByName = new HashMap<>(); - initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); - List uniqueTrialNames = experimentImportRows.stream() .map(ExperimentObservation::getExpTitle) .distinct() From 14993fa70722c80cc315d4bb0e1a3b47429c8245 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 22 Jan 2026 16:21:23 +0000 Subject: [PATCH 189/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 7ebefd909..2c4bf852b 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1069 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/382e10f14e1303f84a3a7ae29a3245ed10bec728 \ No newline at end of file +version=v1.2.0+1071 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/9c44131bd738069fb4e699afb9a55d73355938d9 \ No newline at end of file From b36060263c6570fab29dc8307e5544e5e2162377 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Fri, 23 Jan 2026 15:41:26 -0500 Subject: [PATCH 190/289] Revert observation level change --- .../brapi/v2/services/BrAPITrialService.java | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 4c217ca66..e4309b26d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -656,29 +656,6 @@ public BrAPIObservationUnit createSubObservationUnit( return observationUnit; } - private String getObservationLevelName(BrAPIObservationUnit observationUnit) { - if (observationUnit.getObservationUnitPosition() != null - && observationUnit.getObservationUnitPosition().getObservationLevel() != null - && StringUtils.isNotBlank(observationUnit.getObservationUnitPosition().getObservationLevel().getLevelName())) { - return observationUnit.getObservationUnitPosition().getObservationLevel().getLevelName(); - } - JsonObject additionalInfo = observationUnit.getAdditionalInfo(); - if (additionalInfo != null - && additionalInfo.has(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL) - && !additionalInfo.get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).isJsonNull()) { - return additionalInfo.get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); - } - return null; - } - - private String requireObservationLevelName(BrAPIObservationUnit observationUnit) { - String levelName = getObservationLevelName(observationUnit); - if (StringUtils.isBlank(levelName)) { - throw new RuntimeException("Observation level not found for observation unit " + observationUnit.getObservationUnitDbId()); - } - return levelName; - } - private void addBrAPIObsToRecords( List dataset, BrAPITrial experiment, @@ -906,7 +883,7 @@ private Map createExportRow( } //Append observation level to obsUnitID - String observationLvl = requireObservationLevelName(ou); + String observationLvl = ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); row.put(observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX, ouId); if (isSubEntity) { From 2b2867ddefd7a238e06b0e1aa3639a13789a06c5 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Fri, 23 Jan 2026 15:47:12 -0500 Subject: [PATCH 191/289] Revert plot case change --- src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java index 5f41de572..936bd401c 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java @@ -308,7 +308,7 @@ public Map makeExpImportRow(String environment, String expTitle) row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); row.put(ExperimentObservation.Columns.EXP_TITLE, expTitle); - row.put(ExperimentObservation.Columns.EXP_UNIT, "plot"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); row.put(ExperimentObservation.Columns.ENV, environment); row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); From b2bfedb28ae83be4f06c023570c817c8f5db6208 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Tue, 27 Jan 2026 16:36:56 -0500 Subject: [PATCH 192/289] Revert removal of lowercase conversion and add todo --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index e4309b26d..193147834 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -641,7 +641,8 @@ public BrAPIObservationUnit createSubObservationUnit( } // ObservationLevelRelationships for top-level Exp Unit linking. BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); - expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + // TODO: switch to using level name in main osb unit properties once dynamic obs unit support is complete + expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString().toLowerCase()); String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId(); expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnitUUID, program.getKey(), seqVal)); expUnitLevel.setLevelOrder(DatasetLevel.EXP_UNIT.getValue()); From d70c7bb0ea9ef7f45c0ad6cdd51a2a501b6df858 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Tue, 27 Jan 2026 16:39:49 -0500 Subject: [PATCH 193/289] Fix comment typo --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 193147834..5d917bf5b 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -641,7 +641,7 @@ public BrAPIObservationUnit createSubObservationUnit( } // ObservationLevelRelationships for top-level Exp Unit linking. BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); - // TODO: switch to using level name in main osb unit properties once dynamic obs unit support is complete + // TODO: switch to using level name in main obs unit properties once dynamic obs unit support is complete expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString().toLowerCase()); String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId(); expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnitUUID, program.getKey(), seqVal)); From 236c009a943d1db0ff406d3bb6e1b2ff12010776 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Tue, 27 Jan 2026 16:53:40 -0500 Subject: [PATCH 194/289] Update javadocs --- .../brapi/v2/services/BrAPITrialService.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 5d917bf5b..f34b5542b 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -433,14 +433,14 @@ public List getDatasetsMetadata(Program program, UUID experimen * deleting observation units. Will need to add batch delete support for observation units before this * can be done. * - * @param program - * @param experimentId - * @param request - * @return - * @throws ApiException - * @throws DoesNotExistException - * @throws AlreadyExistsException - * @throws CreationBusyException + * @param program program object representing the program that the datasets belong to + * @param experimentId id of the experiment that the datasets are associated with + * @param request request body containing dataset name and repeated-measure count + * @return dataset metadata and data for the newly created sub-entity dataset + * @throws ApiException if the BrAPI server rejects a create request + * @throws DoesNotExistException if the experiment or required BrAPI entities are missing + * @throws AlreadyExistsException if a dataset with the requested name already exists in the experiment + * @throws CreationBusyException if the dataset creation lock cannot be acquired in time */ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEntityDatasetRequest request) throws ApiException, DoesNotExistException, AlreadyExistsException, CreationBusyException { From 1620a76ee2b51f9cc5220b1bc014a56ae59d270a Mon Sep 17 00:00:00 2001 From: HMS17 Date: Tue, 3 Feb 2026 10:11:41 -0500 Subject: [PATCH 195/289] [BI-2751] - Germplasm entry number weirdness --- .../germplasm/GermplasmProcessor.java | 86 ++++++++++--------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index 50ab5bafd..bfb2ceaa9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -91,7 +91,7 @@ public class GermplasmProcessor implements Processor { public static String missingGIDsMsg = "The following GIDs were not found in the database: %s"; public static String missingParentalGIDsMsg = "The following parental GIDs were not found in the database: %s"; - public static String missingParentalEntryNoMsg = "The following parental entry numbers were not found in the database: %s"; + public static String missingParentalEntryNoMsg = "The following parental entry numbers were not found in the file: %s"; public static String badBreedMethodsMsg = "Invalid breeding method"; public static String badGermplasmNameMsg = "Germplasm name cannot contain /"; public static String missingEntryNumbersMsg = "Either all or none of the germplasm must have entry numbers"; @@ -137,14 +137,17 @@ public void getExistingBrapiData(List importRows, Program program) BrAPIImport germplasmImport = importRows.get(i); Germplasm germplasm = germplasmImport.getGermplasm(); if (germplasm != null) { - + //Ignore this if germplasm already has a pedigree in the database // Retrieve parent accession numbers to assess if already in db - if (germplasm.getFemaleParentAccessionNumber() != null) { - germplasmAccessionNumbers.put(germplasm.getFemaleParentAccessionNumber(), true); - } - if (germplasm.getMaleParentAccessionNumber() != null) { - germplasmAccessionNumbers.put(germplasm.getMaleParentAccessionNumber(), true); + if (!databaseGermplasmHasPedigree(germplasm)) { + if (germplasm.getFemaleParentAccessionNumber() != null) { + germplasmAccessionNumbers.put(germplasm.getFemaleParentAccessionNumber(), true); + } + if (germplasm.getMaleParentAccessionNumber() != null) { + germplasmAccessionNumbers.put(germplasm.getMaleParentAccessionNumber(), true); + } } + if (germplasm.getAccessionNumber() != null) { germplasmAccessionNumbers.put(germplasm.getAccessionNumber(), false); } @@ -223,27 +226,31 @@ public void getExistingBrapiData(List importRows, Program program) arrayOfStringFormatter.apply(missingAccessionNumbers))); } - List missingEntryNumbers = new ArrayList<>(); - for (BrAPIImport importRow: importRows) { - Germplasm germplasm = importRow.getGermplasm(); - // Check Female Parent - if (germplasm.getFemaleParentEntryNo() != null) { - if ((!germplasmIndexByEntryNo.containsKey(germplasm.getFemaleParentEntryNo())) && !(germplasm.getFemaleParentEntryNo().equals("0"))) { - missingEntryNumbers.add(germplasm.getFemaleParentEntryNo()); + List missingEntryNumbers = new ArrayList<>(); + for (BrAPIImport importRow : importRows) { + Germplasm germplasm = importRow.getGermplasm(); + + //If germplasm already has a pedigree, pedigree cannot be overwritten and file values for pedigree will be ignored + boolean pedigreeExists = databaseGermplasmHasPedigree(germplasm); + + // Check Female Parent + if (germplasm.getFemaleParentEntryNo() != null && !pedigreeExists) { + if ((!germplasmIndexByEntryNo.containsKey(germplasm.getFemaleParentEntryNo())) && !(germplasm.getFemaleParentEntryNo().equals("0"))) { + missingEntryNumbers.add(germplasm.getFemaleParentEntryNo()); + } } - } - // Check Male Parent - if (germplasm.getMaleParentEntryNo() != null) { - if ((!germplasmIndexByEntryNo.containsKey(germplasm.getMaleParentEntryNo())) && !(germplasm.getMaleParentEntryNo().equals("0"))) { - missingEntryNumbers.add(germplasm.getMaleParentEntryNo()); + // Check Male Parent + if (germplasm.getMaleParentEntryNo() != null && !pedigreeExists) { + if ((!germplasmIndexByEntryNo.containsKey(germplasm.getMaleParentEntryNo())) && !(germplasm.getMaleParentEntryNo().equals("0"))) { + missingEntryNumbers.add(germplasm.getMaleParentEntryNo()); + } } } - } - if (missingEntryNumbers.size() > 0) { - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, - String.format(missingParentalEntryNoMsg, - arrayOfStringFormatter.apply(missingEntryNumbers))); - } + if (missingEntryNumbers.size() > 0) { + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, + String.format(missingParentalEntryNoMsg, + arrayOfStringFormatter.apply(missingEntryNumbers))); + } if (listNameDup) { throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, listNameAlreadyExists); @@ -405,21 +412,8 @@ private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors v return false; } - // Error conditions: - // has existing pedigree and file pedigree is different and not empty - // Valid conditions: - // no existing pedigree and file different pedigree - // existing pedigree and file pedigree same - // existing pedigree and file pedigree empty - if(hasPedigree(existingGermplasm) && germplasm.pedigreeExists()) { - if(!arePedigreesEqual(existingGermplasm, germplasm, importRows)) { - ValidationError ve = new ValidationError("Pedigree", pedigreeAlreadyExists, HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(rowIndex + 2, ve); // +2 instead of +1 to account for the column header row. - return false; - } - } - // if no existing pedigree and file has pedigree then validate and update + // if pedigree exists, file pedigree information should be ignored if(germplasm.pedigreeExists() && !hasPedigree(existingGermplasm)) { validatePedigree(germplasm, rowIndex + 2, validationErrors); updatePedigree = true; @@ -460,6 +454,16 @@ private boolean hasPedigree(BrAPIGermplasm germplasm) { germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.MALE_PARENT_UNKNOWN).getAsBoolean()); } + //Used to check if germplasm already has a pedigree in the database, if so, pedigree information in file should be ignored + private boolean databaseGermplasmHasPedigree(Germplasm germplasm) { + if (germplasm.getAccessionNumber() == null || dbGermplasmByAccessionNo.get(germplasm.getAccessionNumber()) == null) { + return false; + } else { + BrAPIGermplasm dbGermplasm = dbGermplasmByAccessionNo.get(germplasm.getAccessionNumber()); + return hasPedigree(dbGermplasm); + } + } + /** * Compare an existing germplasm's pedigree to the incoming germplasm's pedigree to ensure they are the same.

* Assumes that an empty value for a given parent in the incoming germplasm is equal to the existing germplasm.

@@ -793,6 +797,8 @@ else if (germplasmIndexByEntryNo.containsKey(germplasm.getFemaleParentEntryNo()) if (commit) { if (femaleParentFound) { brAPIGermplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_GID, femaleParent.getAccessionNumber()); + //entry number no longer needed for figuring out parentage, can remove (since same germplasm can have different entry numbers across multiple lists) + brAPIGermplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_ENTRY_NO, null); // Add femaleParentUUID to additionalInfo. Optional femaleParentUUID = Utilities.getExternalReference(femaleParent.getExternalReferences(), BRAPI_REFERENCE_SOURCE); if (femaleParentUUID.isPresent()) { @@ -802,6 +808,8 @@ else if (germplasmIndexByEntryNo.containsKey(germplasm.getFemaleParentEntryNo()) if (maleParent != null) { brAPIGermplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID, maleParent.getAccessionNumber()); + //entry number no longer needed for figuring out parentage, can remove (since same germplasm can have different entry numbers across multiple lists) + brAPIGermplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO, null); // Add maleParentUUID to additionalInfo. Optional maleParentUUID = Utilities.getExternalReference(maleParent.getExternalReferences(), BRAPI_REFERENCE_SOURCE); if (maleParentUUID.isPresent()) { From f03d4d44766b59d536975f80b34d0bb32406ba07 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Fri, 6 Feb 2026 10:44:07 -0500 Subject: [PATCH 196/289] Set version for brapi ossrh packages --- pom.xml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 402e23d44..3c36a3482 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,8 @@ 31.0.1-jre 4.9.3 4.3.1 - 2.1-SNAPSHOT + 2.1-20250225.145602-11 + 2.1-20250225.145602-11 2.11.0 2.2.1 @@ -383,6 +384,11 @@ brapi-java-client ${brapi-java-client.version} + + org.brapi + brapi-java-model + ${brapi-java-model.version} + org.apache.commons commons-csv From 01bcda0ac75787b1e9f51a33ba2515039eeff69f Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 6 Feb 2026 15:44:26 +0000 Subject: [PATCH 197/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 2c4bf852b..8615b8cfc 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1071 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/9c44131bd738069fb4e699afb9a55d73355938d9 \ No newline at end of file +version=v1.2.0+1073 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f03d4d44766b59d536975f80b34d0bb32406ba07 \ No newline at end of file From 0317e6d61ebb9e75668867ba5d8a790ac0c6224b Mon Sep 17 00:00:00 2001 From: HMS17 Date: Mon, 9 Feb 2026 10:02:47 -0500 Subject: [PATCH 198/289] [BI-2751] - fix parental accession no check --- .../germplasm/GermplasmProcessor.java | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index bfb2ceaa9..3747ecfbb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -137,17 +137,6 @@ public void getExistingBrapiData(List importRows, Program program) BrAPIImport germplasmImport = importRows.get(i); Germplasm germplasm = germplasmImport.getGermplasm(); if (germplasm != null) { - //Ignore this if germplasm already has a pedigree in the database - // Retrieve parent accession numbers to assess if already in db - if (!databaseGermplasmHasPedigree(germplasm)) { - if (germplasm.getFemaleParentAccessionNumber() != null) { - germplasmAccessionNumbers.put(germplasm.getFemaleParentAccessionNumber(), true); - } - if (germplasm.getMaleParentAccessionNumber() != null) { - germplasmAccessionNumbers.put(germplasm.getMaleParentAccessionNumber(), true); - } - } - if (germplasm.getAccessionNumber() != null) { germplasmAccessionNumbers.put(germplasm.getAccessionNumber(), false); } @@ -162,6 +151,31 @@ public void getExistingBrapiData(List importRows, Program program) } } + // Get existing germplasm names + List dbGermplasm = brAPIGermplasmService.getGermplasmByDisplayName(new ArrayList<>(fileGermplasmByName.keySet()), program.getId()); + dbGermplasm.forEach(germplasm -> { + dbGermplasmByName.put(germplasm.getDefaultDisplayName(), germplasm); + dbGermplasmByAccessionNo.put(germplasm.getAccessionNumber(), germplasm); + }); + + // Get parental accession nos in file + for (int i = 0; i < importRows.size(); i++) { + BrAPIImport germplasmImport = importRows.get(i); + Germplasm germplasm = germplasmImport.getGermplasm(); + if (germplasm != null) { + //Ignore this if germplasm already has a pedigree in the database + // Retrieve parent accession numbers to assess if already in db + if (!databaseGermplasmHasPedigree(germplasm)) { + if (germplasm.getFemaleParentAccessionNumber() != null) { + germplasmAccessionNumbers.put(germplasm.getFemaleParentAccessionNumber(), true); + } + if (germplasm.getMaleParentAccessionNumber() != null) { + germplasmAccessionNumbers.put(germplasm.getMaleParentAccessionNumber(), true); + } + } + } + } + // If a parental accession number is present, it should exist in the database. existingGermplasm = new ArrayList<>(); List missingParentalAccessionNumbers = germplasmAccessionNumbers.entrySet().stream().filter(Map.Entry::getValue).map(Map.Entry::getKey).collect(Collectors.toList()); @@ -184,13 +198,6 @@ public void getExistingBrapiData(List importRows, Program program) } } - // Get existing germplasm names - List dbGermplasm = brAPIGermplasmService.getGermplasmByDisplayName(new ArrayList<>(fileGermplasmByName.keySet()), program.getId()); - dbGermplasm.forEach(germplasm -> { - dbGermplasmByName.put(germplasm.getDefaultDisplayName(), germplasm); - dbGermplasmByAccessionNo.put(germplasm.getAccessionNumber(), germplasm); - }); - // Check for existing germplasm lists Boolean listNameDup = false; if (importRows.size() > 0 && importRows.get(0).getGermplasm().getListName() != null) { From 11bae4849b842f4c3fa9df806e73fdbc2de5c576 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Mon, 9 Feb 2026 10:30:37 -0500 Subject: [PATCH 199/289] Switch back to snapshot --- pom.xml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 3c36a3482..402e23d44 100644 --- a/pom.xml +++ b/pom.xml @@ -89,8 +89,7 @@ 31.0.1-jre 4.9.3 4.3.1 - 2.1-20250225.145602-11 - 2.1-20250225.145602-11 + 2.1-SNAPSHOT 2.11.0 2.2.1 @@ -384,11 +383,6 @@ brapi-java-client ${brapi-java-client.version} - - org.brapi - brapi-java-model - ${brapi-java-model.version} - org.apache.commons commons-csv From 5ee17ebf6216f090dd6b6cad335c5af9cb83c14d Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 9 Feb 2026 15:32:13 +0000 Subject: [PATCH 200/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 8615b8cfc..24863996f 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1073 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f03d4d44766b59d536975f80b34d0bb32406ba07 \ No newline at end of file +version=v1.2.0+1075 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/785f86ea2a728c0944663c6304a878789600ae1c \ No newline at end of file From b39ea9bb2f5db3066372280ed94024f9f1396714 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Wed, 11 Feb 2026 16:34:00 -0500 Subject: [PATCH 201/289] Changes for test failures --- .../importer/ExperimentFileImportTest.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 0e0fa4af0..f85a98dc5 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -193,6 +193,12 @@ public void setup() { @Test @SneakyThrows + @Disabled + /** + * TODO: Re-enable this test after dynamic observation level name migration + * Currently TrialService::isSubEntityDataset has an index out of bounds exception because + * plot has a level order of 6 in the new brapi server but the code is expecting level order 0 + */ public void appendExperimentWithObsVarFromPriorDataset() { log.debug("appendExperimentWithObsVarFromPriorDataset"); @@ -328,7 +334,7 @@ public void appendExperimentMultipleDatasets() { .get("trial").getAsJsonObject() .get("id").getAsString(); - // Create two sub-entity datasets that have two plant-level units + // Create two sub-entity datasets that have two different sub entity units Flowable> sub1PostCall = client.exchange( POST(String.format("/programs/%s/experiments/%s/dataset", program.getId().toString(), expId), @@ -339,7 +345,7 @@ public void appendExperimentMultipleDatasets() { Flowable> sub2PostCall = client.exchange( POST(String.format("/programs/%s/experiments/%s/dataset", program.getId().toString(), expId), - "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + "{\"name\":\"Tree\",\"repeatedMeasures\":2}") .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class); HttpResponse sub2PostResponse = sub2PostCall.blockingFirst(); @@ -372,30 +378,30 @@ public void appendExperimentMultipleDatasets() { } String plant1DatasetId = subEntityDatasetIds.get(0); - Flowable> plant1ExportCall = client.exchange( + Flowable> plantExportCall = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", program.getId().toString(), expId, extension, plant1DatasetId)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class ); - HttpResponse plant1Response = plant1ExportCall.blockingFirst(); + HttpResponse plantResponse = plantExportCall.blockingFirst(); String plant2DatasetId = subEntityDatasetIds.get(1); - Flowable> plant2ExportCall = client.exchange( + Flowable> treeExportCall = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", program.getId().toString(), expId, extension, plant2DatasetId)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class ); - HttpResponse plant2Response = plant2ExportCall.blockingFirst(); + HttpResponse treeResponse = treeExportCall.blockingFirst(); // Parse the export tables - ByteArrayInputStream bodyStream1 = new ByteArrayInputStream(Objects.requireNonNull(plant1Response.body())); + ByteArrayInputStream bodyStream1 = new ByteArrayInputStream(Objects.requireNonNull(plantResponse.body())); Table exportTable1 = FileUtil.parseTableFromCsv(bodyStream1); - ByteArrayInputStream bodyStream2 = new ByteArrayInputStream(Objects.requireNonNull(plant2Response.body())); + ByteArrayInputStream bodyStream2 = new ByteArrayInputStream(Objects.requireNonNull(treeResponse.body())); Table exportTable2 = FileUtil.parseTableFromCsv(bodyStream2); // Build a request to append tt_test_1 observation data on observation units from two separate datasets String sub1ObsUnitId = exportTable1.row(0).getString("Plant ObsUnitID"); - String sub2ObsUnitId = exportTable2.row(0).getString("Plant ObsUnitID"); + String sub2ObsUnitId = exportTable2.row(0).getString("Tree ObsUnitID"); Map sub1 = new HashMap<>(); sub1.put(Columns.GERMPLASM_GID, "1"); From 0565f4f9a3a24856e311538dea6fcdf9dc54d34a Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 12 Feb 2026 15:04:09 +0000 Subject: [PATCH 202/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 24863996f..6c723e33a 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1075 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/785f86ea2a728c0944663c6304a878789600ae1c \ No newline at end of file +version=v1.2.0+1077 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a115d6ee09b11d5a0aa9fbb35272e0dec3324d80 \ No newline at end of file From b85a0e4d4d0c4fe1bf86c6579573f42f0db22b58 Mon Sep 17 00:00:00 2001 From: Jason Loux Date: Fri, 13 Feb 2026 17:48:05 -0500 Subject: [PATCH 203/289] [BI-2786] Remove code to add treatments to additional info --- .../brapi/v2/dao/BrAPIObservationUnitDAO.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index 9749bf093..1dc00b2db 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -176,7 +176,6 @@ public List createBrAPIObservationUnits(List> postFunction = () -> { - preprocessObservationUnits(brAPIObservationUnitList); List ous = brAPIDAOUtil.post(brAPIObservationUnitList, upload, api::observationunitsPost, importDAO::update); return processObservationUnitsForCache(ous, program, false); }; @@ -198,7 +197,6 @@ public List createBrAPIObservationUnits(List> postFunction = () -> { - preprocessObservationUnits(brAPIObservationUnitList); List ous = brAPIDAOUtil.post(brAPIObservationUnitList, api::observationunitsPost); return processObservationUnitsForCache(ous, program, false); }; @@ -432,14 +430,4 @@ private void processObservationUnits(Program program, List } } } - - private void preprocessObservationUnits(List brapiObservationUnits) { - // add treatments to additional info - for (BrAPIObservationUnit obsUnit : brapiObservationUnits) { - List treatments = obsUnit.getTreatments(); - if (treatments != null) { - obsUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.TREATMENTS, treatments); - } - } - } } From ef2105c945c0b878534dc7bab58e09bc56798426 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Mon, 16 Feb 2026 11:21:33 -0500 Subject: [PATCH 204/289] Remove gemini github actions --- .github/workflows/gemini-cli.yml | 304 ------------ .../gemini-issue-automated-triage.yml | 130 ----- .../gemini-issue-scheduled-triage.yml | 123 ----- .github/workflows/gemini-pr-review.yml | 456 ------------------ 4 files changed, 1013 deletions(-) delete mode 100644 .github/workflows/gemini-cli.yml delete mode 100644 .github/workflows/gemini-issue-automated-triage.yml delete mode 100644 .github/workflows/gemini-issue-scheduled-triage.yml delete mode 100644 .github/workflows/gemini-pr-review.yml diff --git a/.github/workflows/gemini-cli.yml b/.github/workflows/gemini-cli.yml deleted file mode 100644 index 3fbaedc70..000000000 --- a/.github/workflows/gemini-cli.yml +++ /dev/null @@ -1,304 +0,0 @@ -name: '💬 Gemini CLI' - -on: - pull_request_review_comment: - types: - - 'created' - pull_request_review: - types: - - 'submitted' - issue_comment: - types: - - 'created' - -concurrency: - group: '${{ github.workflow }}-${{ github.event.issue.number }}' - cancel-in-progress: |- - ${{ github.event.sender.type == 'User' && ( github.event.issue.author_association == 'OWNER' || github.event.issue.author_association == 'MEMBER' || github.event.issue.author_association == 'COLLABORATOR') }} - -defaults: - run: - shell: 'bash' - -permissions: - contents: 'write' - id-token: 'write' - pull-requests: 'write' - issues: 'write' - -jobs: - gemini-cli: - # This condition is complex to ensure we only run when explicitly invoked. - if: |- - github.event_name == 'workflow_dispatch' || - ( - github.event_name == 'issues' && github.event.action == 'opened' && - contains(github.event.issue.body, '@gemini-cli') && - !contains(github.event.issue.body, '@gemini-cli /review') && - !contains(github.event.issue.body, '@gemini-cli /triage') && - contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.issue.author_association) - ) || - ( - ( - github.event_name == 'issue_comment' || - github.event_name == 'pull_request_review_comment' - ) && - contains(github.event.comment.body, '@gemini-cli') && - !contains(github.event.comment.body, '@gemini-cli /review') && - !contains(github.event.comment.body, '@gemini-cli /triage') && - contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) - ) || - ( - github.event_name == 'pull_request_review' && - contains(github.event.review.body, '@gemini-cli') && - !contains(github.event.review.body, '@gemini-cli /review') && - !contains(github.event.review.body, '@gemini-cli /triage') && - contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.review.author_association) - ) - timeout-minutes: 10 - runs-on: 'ubuntu-latest' - - steps: - - name: 'Generate GitHub App Token' - id: 'generate_token' - if: |- - ${{ vars.APP_ID }} - uses: 'actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e' # ratchet:actions/create-github-app-token@v2 - with: - app-id: '${{ vars.APP_ID }}' - private-key: '${{ secrets.APP_PRIVATE_KEY }}' - - - name: 'Get context from event' - id: 'get_context' - env: - EVENT_NAME: '${{ github.event_name }}' - EVENT_PAYLOAD: '${{ toJSON(github.event) }}' - run: |- - set -euo pipefail - - USER_REQUEST="" - ISSUE_NUMBER="" - IS_PR="false" - - if [[ "${EVENT_NAME}" == "issues" ]]; then - USER_REQUEST=$(echo "${EVENT_PAYLOAD}" | jq -r .issue.body) - ISSUE_NUMBER=$(echo "${EVENT_PAYLOAD}" | jq -r .issue.number) - elif [[ "${EVENT_NAME}" == "issue_comment" ]]; then - USER_REQUEST=$(echo "${EVENT_PAYLOAD}" | jq -r .comment.body) - ISSUE_NUMBER=$(echo "${EVENT_PAYLOAD}" | jq -r .issue.number) - if [[ $(echo "${EVENT_PAYLOAD}" | jq -r .issue.pull_request) != "null" ]]; then - IS_PR="true" - fi - elif [[ "${EVENT_NAME}" == "pull_request_review" ]]; then - USER_REQUEST=$(echo "${EVENT_PAYLOAD}" | jq -r .review.body) - ISSUE_NUMBER=$(echo "${EVENT_PAYLOAD}" | jq -r .pull_request.number) - IS_PR="true" - elif [[ "${EVENT_NAME}" == "pull_request_review_comment" ]]; then - USER_REQUEST=$(echo "${EVENT_PAYLOAD}" | jq -r .comment.body) - ISSUE_NUMBER=$(echo "${EVENT_PAYLOAD}" | jq -r .pull_request.number) - IS_PR="true" - fi - - # Clean up user request - USER_REQUEST=$(echo "${USER_REQUEST}" | sed 's/.*@gemini-cli//' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - - { - echo "user_request=${USER_REQUEST}" - echo "issue_number=${ISSUE_NUMBER}" - echo "is_pr=${IS_PR}" - } >> "${GITHUB_OUTPUT}" - - - name: 'Set up git user for commits' - run: |- - git config --global user.name 'gemini-cli[bot]' - git config --global user.email 'gemini-cli[bot]@users.noreply.github.com' - - - name: 'Checkout PR branch' - if: |- - ${{ steps.get_context.outputs.is_pr == 'true' }} - uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 - with: - token: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - repository: '${{ github.repository }}' - ref: 'refs/pull/${{ steps.get_context.outputs.issue_number }}/head' - fetch-depth: 0 - - - name: 'Checkout main branch' - if: |- - ${{ steps.get_context.outputs.is_pr == 'false' }} - uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 - with: - token: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - repository: '${{ github.repository }}' - fetch-depth: 0 - - - name: 'Acknowledge request' - env: - GITHUB_ACTOR: '${{ github.actor }}' - GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - ISSUE_NUMBER: '${{ steps.get_context.outputs.issue_number }}' - REPOSITORY: '${{ github.repository }}' - REQUEST_TYPE: '${{ steps.get_context.outputs.request_type }}' - run: |- - set -euo pipefail - MESSAGE="@${GITHUB_ACTOR} I've received your request and I'm working on it now! 🤖" - if [[ -n "${MESSAGE}" ]]; then - gh issue comment "${ISSUE_NUMBER}" \ - --body "${MESSAGE}" \ - --repo "${REPOSITORY}" - fi - - - name: 'Get description' - id: 'get_description' - env: - GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - IS_PR: '${{ steps.get_context.outputs.is_pr }}' - ISSUE_NUMBER: '${{ steps.get_context.outputs.issue_number }}' - run: |- - set -euo pipefail - if [[ "${IS_PR}" == "true" ]]; then - DESCRIPTION=$(gh pr view "${ISSUE_NUMBER}" --json body --template '{{.body}}') - else - DESCRIPTION=$(gh issue view "${ISSUE_NUMBER}" --json body --template '{{.body}}') - fi - { - echo "description<> "${GITHUB_OUTPUT}" - - - name: 'Get comments' - id: 'get_comments' - env: - GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - IS_PR: '${{ steps.get_context.outputs.is_pr }}' - ISSUE_NUMBER: '${{ steps.get_context.outputs.issue_number }}' - run: |- - set -euo pipefail - if [[ "${IS_PR}" == "true" ]]; then - COMMENTS=$(gh pr view "${ISSUE_NUMBER}" --json comments --template '{{range .comments}}{{.author.login}}: {{.body}}{{"\n"}}{{end}}') - else - COMMENTS=$(gh issue view "${ISSUE_NUMBER}" --json comments --template '{{range .comments}}{{.author.login}}: {{.body}}{{"\n"}}{{end}}') - fi - { - echo "comments<> "${GITHUB_OUTPUT}" - - - name: 'Run Gemini' - id: 'run_gemini' - uses: 'google-github-actions/run-gemini-cli@v0' - env: - GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - REPOSITORY: '${{ github.repository }}' - USER_REQUEST: '${{ steps.get_context.outputs.user_request }}' - ISSUE_NUMBER: '${{ steps.get_context.outputs.issue_number }}' - IS_PR: '${{ steps.get_context.outputs.is_pr }}' - with: - gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' - gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' - gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' - gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' - gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' - use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' - use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' - settings: |- - { - "maxSessionTurns": 50, - "telemetry": { - "enabled": false, - "target": "gcp" - } - } - prompt: |- - ## Role - - You are a helpful AI assistant invoked via a CLI interface in a GitHub workflow. You have access to tools to interact with the repository and respond to the user. - - ## Context - - - **Repository**: `${{ github.repository }}` - - **Triggering Event**: `${{ github.event_name }}` - - **Issue/PR Number**: `${{ steps.get_context.outputs.issue_number }}` - - **Is this a PR?**: `${{ steps.get_context.outputs.is_pr }}` - - **Issue/PR Description**: - `${{ steps.get_description.outputs.description }}` - - **Comments**: - `${{ steps.get_comments.outputs.comments }}` - - ## User Request - - The user has sent the following request: - `${{ steps.get_context.outputs.user_request }}` - - ## How to Respond to Issues, PR Comments, and Questions - - This workflow supports three main scenarios: - - 1. **Creating a Fix for an Issue** - - Carefully read the user request and the related issue or PR description. - - Use available tools to gather all relevant context (e.g., `gh issue view`, `gh pr view`, `gh pr diff`, `cat`, `head`, `tail`). - - Identify the root cause of the problem before proceeding. - - **Show and maintain a plan as a checklist**: - - At the very beginning, outline the steps needed to resolve the issue or address the request and post them as a checklist comment on the issue or PR (use GitHub markdown checkboxes: `- [ ] Task`). - - Example: - ``` - ### Plan - - [ ] Investigate the root cause - - [ ] Implement the fix in `file.py` - - [ ] Add/modify tests - - [ ] Update documentation - - [ ] Verify the fix and close the issue - ``` - - Use: `gh pr comment "${ISSUE_NUMBER}" --body ""` or `gh issue comment "${ISSUE_NUMBER}" --body ""` to post the initial plan. - - As you make progress, keep the checklist visible and up to date by editing the same comment (check off completed tasks with `- [x]`). - - To update the checklist: - 1. Find the comment ID for the checklist (use `gh pr comment list "${ISSUE_NUMBER}"` or `gh issue comment list "${ISSUE_NUMBER}"`). - 2. Edit the comment with the updated checklist: - - For PRs: `gh pr comment --edit --body ""` - - For Issues: `gh issue comment --edit --body ""` - 3. The checklist should only be maintained as a comment on the issue or PR. Do not track or update the checklist in code files. - - If the fix requires code changes, determine which files and lines are affected. If clarification is needed, note any questions for the user. - - Make the necessary code or documentation changes using the available tools (e.g., `write_file`). Ensure all changes follow project conventions and best practices. Reference all shell variables as `"${VAR}"` (with quotes and braces) to prevent errors. - - Run any relevant tests or checks to verify the fix works as intended. If possible, provide evidence (test output, screenshots, etc.) that the issue is resolved. - - **Branching and Committing**: - - **NEVER commit directly to the `main` branch.** - - If you are working on a **pull request** (`IS_PR` is `true`), the correct branch is already checked out. Simply commit and push to it. - - `git add .` - - `git commit -m "feat: "` - - `git push` - - If you are working on an **issue** (`IS_PR` is `false`), create a new branch for your changes. A good branch name would be `issue/${ISSUE_NUMBER}/`. - - `git checkout -b issue/${ISSUE_NUMBER}/my-fix` - - `git add .` - - `git commit -m "feat: "` - - `git push origin issue/${ISSUE_NUMBER}/my-fix` - - After pushing, you can create a pull request: `gh pr create --title "Fixes #${ISSUE_NUMBER}: " --body "This PR addresses issue #${ISSUE_NUMBER}."` - - Summarize what was changed and why in a markdown file: `write_file("response.md", "")` - - Post the response as a comment: - - For PRs: `gh pr comment "${ISSUE_NUMBER}" --body-file response.md` - - For Issues: `gh issue comment "${ISSUE_NUMBER}" --body-file response.md` - - 2. **Addressing Comments on a Pull Request** - - Read the specific comment and the context of the PR. - - Use tools like `gh pr view`, `gh pr diff`, and `cat` to understand the code and discussion. - - If the comment requests a change or clarification, follow the same process as for fixing an issue: create a checklist plan, implement, test, and commit any required changes, updating the checklist as you go. - - **Committing Changes**: The correct PR branch is already checked out. Simply add, commit, and push your changes. - - `git add .` - - `git commit -m "fix: address review comments"` - - `git push` - - If the comment is a question, answer it directly and clearly, referencing code or documentation as needed. - - Document your response in `response.md` and post it as a PR comment: `gh pr comment "${ISSUE_NUMBER}" --body-file response.md` - - 3. **Answering Any Question on an Issue** - - Read the question and the full issue context using `gh issue view` and related tools. - - Research or analyze the codebase as needed to provide an accurate answer. - - If the question requires code or documentation changes, follow the fix process above, including creating and updating a checklist plan and **creating a new branch for your changes as described in section 1.** - - Write a clear, concise answer in `response.md` and post it as an issue comment: `gh issue comment "${ISSUE_NUMBER}" --body-file response.md` - - ## Guidelines - - - **Be concise and actionable.** Focus on solving the user's problem efficiently. - - **Always commit and push your changes if you modify code or documentation.** - - **If you are unsure about the fix or answer, explain your reasoning and ask clarifying questions.** - - **Follow project conventions and best practices.** diff --git a/.github/workflows/gemini-issue-automated-triage.yml b/.github/workflows/gemini-issue-automated-triage.yml deleted file mode 100644 index 50a671018..000000000 --- a/.github/workflows/gemini-issue-automated-triage.yml +++ /dev/null @@ -1,130 +0,0 @@ -name: '🏷️ Gemini Automated Issue Triage' - -on: - issues: - types: - - 'opened' - - 'reopened' - issue_comment: - types: - - 'created' - workflow_dispatch: - inputs: - issue_number: - description: 'issue number to triage' - required: true - type: 'number' - -concurrency: - group: '${{ github.workflow }}-${{ github.event.issue.number }}' - cancel-in-progress: true - -defaults: - run: - shell: 'bash' - -permissions: - contents: 'read' - id-token: 'write' - issues: 'write' - statuses: 'write' - -jobs: - triage-issue: - if: |- - github.event_name == 'issues' || - github.event_name == 'workflow_dispatch' || - ( - github.event_name == 'issue_comment' && - contains(github.event.comment.body, '@gemini-cli /triage') && - contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) - ) - timeout-minutes: 5 - runs-on: 'ubuntu-latest' - - steps: - - name: 'Checkout repository' - uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 - - - name: 'Generate GitHub App Token' - id: 'generate_token' - if: |- - ${{ vars.APP_ID }} - uses: 'actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e' # ratchet:actions/create-github-app-token@v2 - with: - app-id: '${{ vars.APP_ID }}' - private-key: '${{ secrets.APP_PRIVATE_KEY }}' - - - name: 'Run Gemini Issue Triage' - uses: 'google-github-actions/run-gemini-cli@v0' - id: 'gemini_issue_triage' - env: - GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - ISSUE_TITLE: '${{ github.event.issue.title }}' - ISSUE_BODY: '${{ github.event.issue.body }}' - ISSUE_NUMBER: '${{ github.event.issue.number }}' - REPOSITORY: '${{ github.repository }}' - with: - gemini_cli_version: '${{ vars.GEMINI_CLI_VERSION }}' - gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' - gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' - gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' - gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' - gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' - use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' - use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' - settings: |- - { - "maxSessionTurns": 25, - "coreTools": [ - "run_shell_command(echo)", - "run_shell_command(gh label list)", - "run_shell_command(gh issue edit)" - ], - "telemetry": { - "enabled": false, - "target": "gcp" - } - } - prompt: |- - ## Role - - You are an issue triage assistant. Analyze the current GitHub issue - and apply the most appropriate existing labels. Use the available - tools to gather information; do not ask for information to be - provided. - - ## Steps - - 1. Run: `gh label list` to get all available labels. - 2. Review the issue title and body provided in the environment - variables: "${ISSUE_TITLE}" and "${ISSUE_BODY}". - 3. Classify issues by their kind (bug, enhancement, documentation, - cleanup, etc) and their priority (p0, p1, p2, p3). Set the - labels accoridng to the format `kind/*` and `priority/*` patterns. - 4. Apply the selected labels to this issue using: - `gh issue edit "${ISSUE_NUMBER}" --add-label "label1,label2"` - 5. If the "status/needs-triage" label is present, remove it using: - `gh issue edit "${ISSUE_NUMBER}" --remove-label "status/needs-triage"` - - ## Guidelines - - - Only use labels that already exist in the repository - - Do not add comments or modify the issue content - - Triage only the current issue - - Assign all applicable labels based on the issue content - - Reference all shell variables as "${VAR}" (with quotes and braces) - - - name: 'Post Issue Triage Failure Comment' - if: |- - ${{ failure() && steps.gemini_issue_triage.outcome == 'failure' }} - uses: 'actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea' - with: - github-token: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - script: |- - github.rest.issues.createComment({ - owner: '${{ github.repository }}'.split('/')[0], - repo: '${{ github.repository }}'.split('/')[1], - issue_number: '${{ github.event.issue.number }}', - body: 'There is a problem with the Gemini CLI issue triaging. Please check the [action logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.' - }) diff --git a/.github/workflows/gemini-issue-scheduled-triage.yml b/.github/workflows/gemini-issue-scheduled-triage.yml deleted file mode 100644 index f44205973..000000000 --- a/.github/workflows/gemini-issue-scheduled-triage.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: '📋 Gemini Scheduled Issue Triage' - -on: - schedule: - - cron: '0 * * * *' # Runs every hour - workflow_dispatch: - -concurrency: - group: '${{ github.workflow }}' - cancel-in-progress: true - -defaults: - run: - shell: 'bash' - -permissions: - contents: 'read' - id-token: 'write' - issues: 'write' - statuses: 'write' - -jobs: - triage-issues: - timeout-minutes: 5 - runs-on: 'ubuntu-latest' - - steps: - - name: 'Checkout repository' - uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 - - - name: 'Generate GitHub App Token' - id: 'generate_token' - if: |- - ${{ vars.APP_ID }} - uses: 'actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e' # ratchet:actions/create-github-app-token@v2 - with: - app-id: '${{ vars.APP_ID }}' - private-key: '${{ secrets.APP_PRIVATE_KEY }}' - - - name: 'Find untriaged issues' - id: 'find_issues' - env: - GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - GITHUB_REPOSITORY: '${{ github.repository }}' - GITHUB_OUTPUT: '${{ github.output }}' - run: |- - set -euo pipefail - - echo '🔍 Finding issues without labels...' - NO_LABEL_ISSUES="$(gh issue list --repo "${GITHUB_REPOSITORY}" \ - --search 'is:open is:issue no:label' --json number,title,body)" - - echo '🏷️ Finding issues that need triage...' - NEED_TRIAGE_ISSUES="$(gh issue list --repo "${GITHUB_REPOSITORY}" \ - --search 'is:open is:issue label:"status/needs-triage"' --json number,title,body)" - - echo '🔄 Merging and deduplicating issues...' - ISSUES="$(echo "${NO_LABEL_ISSUES}" "${NEED_TRIAGE_ISSUES}" | jq -c -s 'add | unique_by(.number)')" - - echo '📝 Setting output for GitHub Actions...' - echo "issues_to_triage=${ISSUES}" >> "${GITHUB_OUTPUT}" - - ISSUE_COUNT="$(echo "${ISSUES}" | jq 'length')" - echo "✅ Found ${ISSUE_COUNT} issues to triage! 🎯" - - - name: 'Run Gemini Issue Triage' - if: |- - ${{ steps.find_issues.outputs.issues_to_triage != '[]' }} - uses: 'google-github-actions/run-gemini-cli@v0' - id: 'gemini_issue_triage' - env: - GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - ISSUES_TO_TRIAGE: '${{ steps.find_issues.outputs.issues_to_triage }}' - REPOSITORY: '${{ github.repository }}' - with: - gemini_cli_version: '${{ vars.GEMINI_CLI_VERSION }}' - gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' - gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' - gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' - gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' - gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' - use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' - use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' - settings: |- - { - "maxSessionTurns": 25, - "coreTools": [ - "run_shell_command(echo)", - "run_shell_command(gh label list)", - "run_shell_command(gh issue edit)", - "run_shell_command(gh issue list)" - ], - "telemetry": { - "enabled": false, - "target": "gcp" - } - } - prompt: |- - ## Role - - You are an issue triage assistant. Analyze issues and apply - appropriate labels. Use the available tools to gather information; - do not ask for information to be provided. - - ## Steps - - 1. Run: `gh label list` - 2. Check environment variable: "${ISSUES_TO_TRIAGE}" (JSON array - of issues) - 3. For each issue, apply labels: - `gh issue edit "${ISSUE_NUMBER}" --add-label "label1,label2"`. - If available, set labels that follow the `kind/*`, `area/*`, - and `priority/*` patterns. - 4. For each issue, if the `status/needs-triage` label is present, - remove it using: - `gh issue edit "${ISSUE_NUMBER}" --remove-label "status/needs-triage"` - - ## Guidelines - - - Only use existing repository labels - - Do not add comments - - Triage each issue independently - - Reference all shell variables as "${VAR}" (with quotes and braces) diff --git a/.github/workflows/gemini-pr-review.yml b/.github/workflows/gemini-pr-review.yml deleted file mode 100644 index 297c4572d..000000000 --- a/.github/workflows/gemini-pr-review.yml +++ /dev/null @@ -1,456 +0,0 @@ -name: '🧐 Gemini Pull Request Review' - -on: - pull_request: - types: - - 'opened' - - 'reopened' - issue_comment: - types: - - 'created' - pull_request_review_comment: - types: - - 'created' - pull_request_review: - types: - - 'submitted' - workflow_dispatch: - inputs: - pr_number: - description: 'PR number to review' - required: true - type: 'number' - -concurrency: - group: '${{ github.workflow }}-${{ github.head_ref || github.ref }}' - cancel-in-progress: true - -defaults: - run: - shell: 'bash' - -permissions: - contents: 'read' - id-token: 'write' - issues: 'write' - pull-requests: 'write' - statuses: 'write' - -jobs: - review-pr: - if: |- - github.event_name == 'workflow_dispatch' || - ( - github.event_name == 'pull_request' && - contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) - ) || - ( - ( - ( - github.event_name == 'issue_comment' && - github.event.issue.pull_request - ) || - github.event_name == 'pull_request_review_comment' - ) && - contains(github.event.comment.body, '@gemini-cli /review') && - contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) - ) || - ( - github.event_name == 'pull_request_review' && - contains(github.event.review.body, '@gemini-cli /review') && - contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.review.author_association) - ) - timeout-minutes: 5 - runs-on: 'ubuntu-latest' - - steps: - - name: 'Checkout PR code' - uses: 'actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683' # ratchet:actions/checkout@v4 - - - name: 'Generate GitHub App Token' - id: 'generate_token' - if: |- - ${{ vars.APP_ID }} - uses: 'actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e' # ratchet:actions/create-github-app-token@v2 - with: - app-id: '${{ vars.APP_ID }}' - private-key: '${{ secrets.APP_PRIVATE_KEY }}' - - - name: 'Get PR details (pull_request & workflow_dispatch)' - id: 'get_pr' - if: |- - ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} - env: - GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - EVENT_NAME: '${{ github.event_name }}' - WORKFLOW_PR_NUMBER: '${{ github.event.inputs.pr_number }}' - PULL_REQUEST_NUMBER: '${{ github.event.pull_request.number }}' - run: |- - set -euo pipefail - - if [[ "${EVENT_NAME}" = "workflow_dispatch" ]]; then - PR_NUMBER="${WORKFLOW_PR_NUMBER}" - else - PR_NUMBER="${PULL_REQUEST_NUMBER}" - fi - - echo "pr_number=${PR_NUMBER}" >> "${GITHUB_OUTPUT}" - - # Get PR details - PR_DATA="$(gh pr view "${PR_NUMBER}" --json title,body,additions,deletions,changedFiles,baseRefName,headRefName)" - echo "pr_data=${PR_DATA}" >> "${GITHUB_OUTPUT}" - - # Get file changes - CHANGED_FILES="$(gh pr diff "${PR_NUMBER}" --name-only)" - { - echo "changed_files<> "${GITHUB_OUTPUT}" - - - - name: 'Get PR details (issue_comment)' - id: 'get_pr_comment' - if: |- - ${{ github.event_name == 'issue_comment' }} - env: - GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - COMMENT_BODY: '${{ github.event.comment.body }}' - PR_NUMBER: '${{ github.event.issue.number }}' - run: |- - set -euo pipefail - - echo "pr_number=${PR_NUMBER}" >> "${GITHUB_OUTPUT}" - - # Extract additional instructions from comment - ADDITIONAL_INSTRUCTIONS="$( - echo "${COMMENT_BODY}" | sed 's/.*@gemini-cli \/review//' | xargs - )" - echo "additional_instructions=${ADDITIONAL_INSTRUCTIONS}" >> "${GITHUB_OUTPUT}" - - # Get PR details - PR_DATA="$(gh pr view "${PR_NUMBER}" --json title,body,additions,deletions,changedFiles,baseRefName,headRefName)" - echo "pr_data=${PR_DATA}" >> "${GITHUB_OUTPUT}" - - # Get file changes - CHANGED_FILES="$(gh pr diff "${PR_NUMBER}" --name-only)" - { - echo "changed_files<> "${GITHUB_OUTPUT}" - - - name: 'Run Gemini PR Review' - uses: 'google-github-actions/run-gemini-cli@v0' - id: 'gemini_pr_review' - env: - GITHUB_TOKEN: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - PR_NUMBER: '${{ steps.get_pr.outputs.pr_number || steps.get_pr_comment.outputs.pr_number }}' - PR_DATA: '${{ steps.get_pr.outputs.pr_data || steps.get_pr_comment.outputs.pr_data }}' - CHANGED_FILES: '${{ steps.get_pr.outputs.changed_files || steps.get_pr_comment.outputs.changed_files }}' - ADDITIONAL_INSTRUCTIONS: '${{ steps.get_pr.outputs.additional_instructions || steps.get_pr_comment.outputs.additional_instructions }}' - REPOSITORY: '${{ github.repository }}' - with: - gemini_cli_version: '${{ vars.GEMINI_CLI_VERSION }}' - gcp_workload_identity_provider: '${{ vars.GCP_WIF_PROVIDER }}' - gcp_project_id: '${{ vars.GOOGLE_CLOUD_PROJECT }}' - gcp_location: '${{ vars.GOOGLE_CLOUD_LOCATION }}' - gcp_service_account: '${{ vars.SERVICE_ACCOUNT_EMAIL }}' - gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' - use_vertex_ai: '${{ vars.GOOGLE_GENAI_USE_VERTEXAI }}' - use_gemini_code_assist: '${{ vars.GOOGLE_GENAI_USE_GCA }}' - settings: |- - { - "maxSessionTurns": 20, - "mcpServers": { - "github": { - "command": "docker", - "args": [ - "run", - "-i", - "--rm", - "-e", - "GITHUB_PERSONAL_ACCESS_TOKEN", - "ghcr.io/github/github-mcp-server" - ], - "includeTools": [ - "create_pending_pull_request_review", - "add_comment_to_pending_review", - "submit_pending_pull_request_review" - ], - "env": { - "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_TOKEN}" - } - } - }, - "coreTools": [ - "run_shell_command(echo)", - "run_shell_command(gh pr view)", - "run_shell_command(gh pr diff)", - "run_shell_command(cat)", - "run_shell_command(head)", - "run_shell_command(tail)", - "run_shell_command(grep)" - ], - "telemetry": { - "enabled": false, - "target": "gcp" - } - } - prompt: |- - ## Role - - You are an expert code reviewer. You have access to tools to gather - PR information and perform the review on GitHub. Use the available tools to - gather information; do not ask for information to be provided. - - ## Requirements - 1. All feedback must be left on GitHub. - 2. Any output that is not left in GitHub will not be seen. - - ## Steps - - Start by running these commands to gather the required data: - 1. Run: echo "${REPOSITORY}" to get the github repository in / format - 2. Run: echo "${PR_DATA}" to get PR details (JSON format) - 3. Run: echo "${CHANGED_FILES}" to get the list of changed files - 4. Run: echo "${PR_NUMBER}" to get the PR number - 5. Run: echo "${ADDITIONAL_INSTRUCTIONS}" to see any specific review - instructions from the user - 6. Run: gh pr diff "${PR_NUMBER}" to see the full diff and reference - Context section to understand it - 7. For any specific files, use: cat filename, head -50 filename, or - tail -50 filename - 8. If ADDITIONAL_INSTRUCTIONS contains text, prioritize those - specific areas or focus points in your review. Common instruction - examples: "focus on security", "check performance", "review error - handling", "check for breaking changes" - - ## Guideline - ### Core Guideline(Always applicable) - - 1. Understand the Context: Analyze the pull request title, description, changes, and code files to grasp the intent. - 2. Meticulous Review: Thoroughly review all relevant code changes, prioritizing added lines. Consider the specified - focus areas and any provided style guide. - 3. Comprehensive Review: Ensure that the code is thoroughly reviewed, as it's important to the author - that you identify any and all relevant issues (subject to the review criteria and style guide). - Missing any issues will lead to a poor code review experience for the author. - 4. Constructive Feedback: - * Provide clear explanations for each concern. - * Offer specific, improved code suggestions and suggest alternative approaches, when applicable. - Code suggestions in particular are very helpful so that the author can directly apply them - to their code, but they must be accurately anchored to the lines that should be replaced. - 5. Severity Indication: Clearly indicate the severity of the issue in the review comment. - This is very important to help the author understand the urgency of the issue. - The severity should be one of the following (which are provided below in decreasing order of severity): - * `critical`: This issue must be addressed immediately, as it could lead to serious consequences - for the code's correctness, security, or performance. - * `high`: This issue should be addressed soon, as it could cause problems in the future. - * `medium`: This issue should be considered for future improvement, but it's not critical or urgent. - * `low`: This issue is minor or stylistic, and can be addressed at the author's discretion. - 6. Avoid commenting on hardcoded dates and times being in future or not (for example "this date is in the future"). - * Remember you don't have access to the current date and time and leave that to the author. - 7. Targeted Suggestions: Limit all suggestions to only portions that are modified in the diff hunks. - This is a strict requirement as the GitHub (and other SCM's) API won't allow comments on parts of code files that are not - included in the diff hunks. - 8. Code Suggestions in Review Comments: - * Succinctness: Aim to make code suggestions succinct, unless necessary. Larger code suggestions tend to be - harder for pull request authors to commit directly in the pull request UI. - * Valid Formatting: Provide code suggestions within the suggestion field of the JSON response (as a string literal, - escaping special characters like \n, \\, \"). Do not include markdown code blocks in the suggestion field. - Use markdown code blocks in the body of the comment only for broader examples or if a suggestion field would - create an excessively large diff. Prefer the suggestion field for specific, targeted code changes. - * Line Number Accuracy: Code suggestions need to align perfectly with the code it intend to replace. - Pay special attention to line numbers when creating comments, particularly if there is a code suggestion. - Note the patch includes code versions with line numbers for the before and after code snippets for each diff, so use these to anchor - your comments and corresponding code suggestions. - * Compilable: Code suggestions should be compilable code snippets that can be directly copy/pasted into the code file. - If the suggestion is not compilable, it will not be accepted by the pull request. Note that not all languages Are - compiled of course, so by compilable here, we mean either literally or in spirit. - * Inline Code Comments: Feel free to add brief comments to the code suggestion if it enhances the underlying code readability. - Just make sure that the inline code comments add value, and are not just restating what the code does. Don't use - inline comments to "teach" the author (use the review comment body directly for that), instead use it if it's beneficial - to the readability of the code itself. - 10. Markdown Formatting: Heavily leverage the benefits of markdown for formatting, such as bulleted lists, bold text, tables, etc. - 11. Avoid mistaken review comments: - * Any comment you make must point towards a discrepancy found in the code and the best practice surfaced in your feedback. - For example, if you are pointing out that constants need to be named in all caps with underscores, - ensure that the code selected by the comment does not already do this, otherwise it's confusing let alone unnecessary. - 12. Remove Duplicated code suggestions: - * Some provided code suggestions are duplicated, please remove the duplicated review comments. - 13. Don't Approve The Pull Request - 14. Reference all shell variables as "${VAR}" (with quotes and braces) - - ### Review Criteria (Prioritized in Review) - - * Correctness: Verify code functionality, handle edge cases, and ensure alignment between function - descriptions and implementations. Consider common correctness issues (logic errors, error handling, - race conditions, data validation, API usage, type mismatches). - * Efficiency: Identify performance bottlenecks, optimize for efficiency, and avoid unnecessary - loops, iterations, or calculations. Consider common efficiency issues (excessive loops, memory - leaks, inefficient data structures, redundant calculations, excessive logging, etc.). - * Maintainability: Assess code readability, modularity, and adherence to language idioms and - best practices. Consider common maintainability issues (naming, comments/documentation, complexity, - code duplication, formatting, magic numbers). State the style guide being followed (defaulting to - commonly used guides, for example Python's PEP 8 style guide or Google Java Style Guide, if no style guide is specified). - * Security: Identify potential vulnerabilities (e.g., insecure storage, injection attacks, - insufficient access controls). - - ### Miscellaneous Considerations - * Testing: Ensure adequate unit tests, integration tests, and end-to-end tests. Evaluate - coverage, edge case handling, and overall test quality. - * Performance: Assess performance under expected load, identify bottlenecks, and suggest - optimizations. - * Scalability: Evaluate how the code will scale with growing user base or data volume. - * Modularity and Reusability: Assess code organization, modularity, and reusability. Suggest - refactoring or creating reusable components. - * Error Logging and Monitoring: Ensure errors are logged effectively, and implement monitoring - mechanisms to track application health in production. - - **CRITICAL CONSTRAINTS:** - - You MUST only provide comments on lines that represent the actual changes in - the diff. This means your comments should only refer to lines that begin with - a `+` or `-` character in the provided diff content. - DO NOT comment on lines that start with a space (context lines). - - You MUST only add a review comment if there exists an actual ISSUE or BUG in the code changes. - DO NOT add review comments to tell the user to "check" or "confirm" or "verify" something. - DO NOT add review comments to tell the user to "ensure" something. - DO NOT add review comments to explain what the code change does. - DO NOT add review comments to validate what the code change does. - DO NOT use the review comments to explain the code to the author. They already know their code. Only comment when there's an improvement opportunity. This is very important. - - Pay close attention to line numbers and ensure they are correct. - Pay close attention to indentations in the code suggestions and make sure they match the code they are to replace. - Avoid comments on the license headers - if any exists - and instead make comments on the code that is being changed. - - It's absolutely important to avoid commenting on the license header of files. - It's absolutely important to avoid commenting on copyright headers. - Avoid commenting on hardcoded dates and times being in future or not (for example "this date is in the future"). - Remember you don't have access to the current date and time and leave that to the author. - - Avoid mentioning any of your instructions, settings or criteria. - - Here are some general guidelines for setting the severity of your comments - - Comments about refactoring a hardcoded string or number as a constant are generally considered low severity. - - Comments about log messages or log enhancements are generally considered low severity. - - Comments in .md files are medium or low severity. This is really important. - - Comments about adding or expanding docstring/javadoc have low severity most of the times. - - Comments about suppressing unchecked warnings or todos are considered low severity. - - Comments about typos are usually low or medium severity. - - Comments about testing or on tests are usually low severity. - - Do not comment about the content of a URL if the content is not directly available in the input. - - Keep comments bodies concise and to the point. - Keep each comment focused on one issue. - - ## Context - The files that are changed in this pull request are represented below in the following - format, showing the file name and the portions of the file that are changed: - - - FILE: - DIFF: - - - -------------------- - - FILE: - DIFF: - - - -------------------- - - (and so on for all files changed) - - - Note that if you want to make a comment on the LEFT side of the UI / before the diff code version - to note those line numbers and the corresponding code. Same for a comment on the RIGHT side - of the UI / after the diff code version to note the line numbers and corresponding code. - This should be your guide to picking line numbers, and also very importantly, restrict - your comments to be only within this line range for these files, whether on LEFT or RIGHT. - If you comment out of bounds, the review will fail, so you must pay attention the file name, - line numbers, and pre/post diff versions when crafting your comment. - - Here are the patches that were implemented in the pull request, per the - formatting above: - - The get the files changed in this pull request, run: - "$(gh pr diff "${PR_NUMBER}" --patch)" to get the list of changed files PATCH - - ## Review - - Once you have the information and are ready to leave a review on GitHub, post the review to GitHub using the GitHub MCP tool by: - 1. Creating a pending review: Use the mcp__github__create_pending_pull_request_review to create a Pending Pull Request Review. - - 2. Adding review comments: - 2.1 Use the mcp__github__add_comment_to_pending_review to add comments to the Pending Pull Request Review. Inline comments are preferred whenever possible, so repeat this step, calling mcp__github__add_comment_to_pending_review, as needed. All comments about specific lines of code should use inline comments. It is preferred to use code suggestions when possible, which include a code block that is labeled "suggestion", which contains what the new code should be. All comments should also have a severity. The syntax is: - Normal Comment Syntax: - - {{SEVERITY}} {{COMMENT_TEXT}} - - - Inline Comment Syntax: (Preferred): - - {{SEVERITY}} {{COMMENT_TEXT}} - ```suggestion - {{CODE_SUGGESTION}} - ``` - - - Prepend a severity emoji to each comment: - - 🟢 for low severity - - 🟡 for medium severity - - 🟠 for high severity - - 🔴 for critical severity - - 🔵 if severity is unclear - - Including all of this, an example inline comment would be: - - 🟢 Use camelCase for function names - ```suggestion - myFooBarFunction - ``` - - - A critical severity example would be: - - 🔴 Remove storage key from GitHub - ```suggestion - ``` - - 3. Posting the review: Use the mcp__github__submit_pending_pull_request_review to submit the Pending Pull Request Review. - - 3.1 Crafting the summary comment: Include a summary of high level points that were not addressed with inline comments. Be concise. Do not repeat details mentioned inline. - - Structure your summary comment using this exact format with markdown: - ## 📋 Review Summary - - Provide a brief 2-3 sentence overview of the PR and overall - assessment. - - ## 🔍 General Feedback - - List general observations about code quality - - Mention overall patterns or architectural decisions - - Highlight positive aspects of the implementation - - Note any recurring themes across files - - ## Final Instructions - - Remember, you are running in a VM and no one reviewing your output. Your review must be posted to GitHub using the MCP tools to create a pending review, add comments to the pending review, and submit the pending review. - - - - name: 'Post PR review failure comment' - if: |- - ${{ failure() && steps.gemini_pr_review.outcome == 'failure' }} - uses: 'actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea' - with: - github-token: '${{ steps.generate_token.outputs.token || secrets.GITHUB_TOKEN }}' - script: |- - github.rest.issues.createComment({ - owner: '${{ github.repository }}'.split('/')[0], - repo: '${{ github.repository }}'.split('/')[1], - issue_number: '${{ steps.get_pr.outputs.pr_number || steps.get_pr_comment.outputs.pr_number }}', - body: 'There is a problem with the Gemini CLI PR review. Please check the [action logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.' - }) From b3a1c3a6afbe9b1984f2708d4d4924e62db56fb3 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 16 Feb 2026 16:21:49 +0000 Subject: [PATCH 205/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 6c723e33a..1e2ac0e7c 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1077 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a115d6ee09b11d5a0aa9fbb35272e0dec3324d80 \ No newline at end of file +version=v1.2.0+1079 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ef2105c945c0b878534dc7bab58e09bc56798426 \ No newline at end of file From 1084dddc506d773dca2215195c720e5b415a5950 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Mon, 16 Feb 2026 11:37:07 -0500 Subject: [PATCH 206/289] Upgrade testcontainers version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 402e23d44..d1ea4efb7 100644 --- a/pom.xml +++ b/pom.xml @@ -83,7 +83,7 @@ 1.14 1.9.0 3.12.0 - 1.16.3 + 1.21.4 7.7.3 31.0.1-jre From ea256d3532e101bb60cc94444cb8b3d069889d98 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 16 Feb 2026 16:37:45 +0000 Subject: [PATCH 207/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1e2ac0e7c..55e62ba18 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1079 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ef2105c945c0b878534dc7bab58e09bc56798426 \ No newline at end of file +version=v1.2.0+1081 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fb34b5347087aefe71f173585350c873aa57b56f \ No newline at end of file From c4539f6dfdf73b35e21e99b1147af6997fec6b8b Mon Sep 17 00:00:00 2001 From: Jason Loux Date: Mon, 16 Feb 2026 15:43:58 -0500 Subject: [PATCH 208/289] Remove treatments from BrAPIAdditionalFields, update unit test --- .../brapi/v2/constants/BrAPIAdditionalInfoFields.java | 1 - .../brapi/v2/dao/BrAPIObservationUnitDAO.java | 6 ------ .../importer/daos/BrAPIObservationUnitDAOTest.java | 10 ++++++++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index d14663cd2..a69a7b983 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -47,7 +47,6 @@ public final class BrAPIAdditionalInfoFields { public static final String DATASETS = "datasets"; public static final String FEMALE_PARENT_UNKNOWN = "femaleParentUnknown"; public static final String MALE_PARENT_UNKNOWN = "maleParentUnknown"; - public static final String TREATMENTS = "treatments"; public static final String GID = "gid"; public static final String CHANGELOG = "changeLog"; public static final String ENV_YEAR = "envYear"; diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index 1dc00b2db..76acb8076 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -385,15 +385,9 @@ private void processObservationUnits(Program program, List this.germplasmService.getGermplasm(program.getId()).forEach((germplasm -> germplasmByDbId.put(germplasm.getGermplasmDbId(), germplasm))); } - // if has treatments in additionalInfo, copy to treatments property for (BrAPIObservationUnit ou : brapiObservationUnits) { JsonObject additionalInfo = ou.getAdditionalInfo(); if (additionalInfo != null) { - JsonElement treatmentsElement = additionalInfo.get(BrAPIAdditionalInfoFields.TREATMENTS); - if (treatmentsElement != null) { - List treatments = gson.fromJson(treatmentsElement, treatmentlistType); - ou.setTreatments(treatments); - } if( withGID ){ BrAPIGermplasm germplasm = germplasmByDbId.get(ou.getGermplasmDbId()); ou.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, germplasm.getAccessionNumber()); diff --git a/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java b/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java index f4375e590..bd2048b61 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java @@ -36,6 +36,7 @@ import static org.breedinginsight.TestUtils.insertAndFetchTestProgram; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -119,12 +120,14 @@ public void setup() { @SneakyThrows @Order(1) public void testCreateObservationUnitAdditionalInfoSingleTreatmentFactor() { - // create observation unit with treatments only in additional info to simulate breedbase not populating // treatments field BrAPIObservationUnit ou1 = new BrAPIObservationUnit(); ou1.setObservationUnitName("test1"); - ou1.putAdditionalInfoItem(BrAPIAdditionalInfoFields.TREATMENTS, List.of(testTreatment)); ou1.setProgramDbId(validProgram.getBrapiProgram().getProgramDbId()); + + var treatment = new BrAPIObservationTreatment(); + treatment.setFactor("ou1 treatment"); + ou1.setTreatments(List.of(treatment)); // Set xref. BrAPIExternalReference xref = new BrAPIExternalReference(); xref.setReferenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)); @@ -153,6 +156,9 @@ private void singleTreatmentAsserts(List obsUnits, BrAPIOb BrAPIObservationTreatment treatment = treatments.get(0); assertEquals(expectedTreatment, treatment, "Expected treatments to be same"); + + // Storing treatments in additionalInfo is no longer necessary since BrAPI server stores treatments in observation_unit_treatment + assertNull(ou.getAdditionalInfo(), "Expected OU additionalInfo to be null"); } } From b4b338f49c514084e3db35028406a5484136a569 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 17 Feb 2026 19:01:28 +0000 Subject: [PATCH 209/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 55e62ba18..c4d6105af 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1081 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fb34b5347087aefe71f173585350c873aa57b56f \ No newline at end of file +version=v1.2.0+1083 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/83c0a46a735c6b9216a629f8f2dea022ba2dc9e1 \ No newline at end of file From 1041fb53515027b2c186a0e6aca07908d5b3c919 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Wed, 18 Feb 2026 10:23:49 -0500 Subject: [PATCH 210/289] Try pinning older ubuntu version for gigwa issue --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1505d8111..2c4959b12 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 services: postgres: From d1203ba2fcd1574463df7b10463d5aeb8a685c6e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 18 Feb 2026 15:24:23 +0000 Subject: [PATCH 211/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index c4d6105af..abb572efa 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1083 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/83c0a46a735c6b9216a629f8f2dea022ba2dc9e1 \ No newline at end of file +version=v1.2.0+1085 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/3441dda2e5b002ad5b4314610e0d94d79889f9d3 \ No newline at end of file From a312d699c03d5c2c8c49e7d3e6c2bf9a97cfa3e9 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Wed, 18 Feb 2026 10:49:27 -0500 Subject: [PATCH 212/289] [BI-2751] - code review fix --- .../processors/germplasm/GermplasmProcessor.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index 3747ecfbb..3d93d3d07 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -419,8 +419,21 @@ private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors v return false; } + // Error conditions: + // has existing pedigree and file pedigree is different and not empty + // Valid conditions: + // no existing pedigree and file different pedigree + // existing pedigree and file pedigree same + // existing pedigree and file pedigree empty + if(hasPedigree(existingGermplasm) && germplasm.pedigreeExists()) { + if(!arePedigreesEqual(existingGermplasm, germplasm, importRows)) { + ValidationError ve = new ValidationError("Pedigree", pedigreeAlreadyExists, HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowIndex + 2, ve); // +2 instead of +1 to account for the column header row. + return false; + } + } + // if no existing pedigree and file has pedigree then validate and update - // if pedigree exists, file pedigree information should be ignored if(germplasm.pedigreeExists() && !hasPedigree(existingGermplasm)) { validatePedigree(germplasm, rowIndex + 2, validationErrors); updatePedigree = true; From 5fe7666c9a0b30e7dccda5cbc3449051414c97da Mon Sep 17 00:00:00 2001 From: HMS17 Date: Wed, 18 Feb 2026 11:33:53 -0500 Subject: [PATCH 213/289] [BI-2751] - code review fix more fixing pedigree logic --- .../processors/germplasm/GermplasmProcessor.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index 3d93d3d07..1cb15a4b3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -163,16 +163,13 @@ public void getExistingBrapiData(List importRows, Program program) BrAPIImport germplasmImport = importRows.get(i); Germplasm germplasm = germplasmImport.getGermplasm(); if (germplasm != null) { - //Ignore this if germplasm already has a pedigree in the database // Retrieve parent accession numbers to assess if already in db - if (!databaseGermplasmHasPedigree(germplasm)) { if (germplasm.getFemaleParentAccessionNumber() != null) { germplasmAccessionNumbers.put(germplasm.getFemaleParentAccessionNumber(), true); } if (germplasm.getMaleParentAccessionNumber() != null) { germplasmAccessionNumbers.put(germplasm.getMaleParentAccessionNumber(), true); } - } } } @@ -237,17 +234,14 @@ public void getExistingBrapiData(List importRows, Program program) for (BrAPIImport importRow : importRows) { Germplasm germplasm = importRow.getGermplasm(); - //If germplasm already has a pedigree, pedigree cannot be overwritten and file values for pedigree will be ignored - boolean pedigreeExists = databaseGermplasmHasPedigree(germplasm); - // Check Female Parent - if (germplasm.getFemaleParentEntryNo() != null && !pedigreeExists) { + if (germplasm.getFemaleParentEntryNo() != null) { if ((!germplasmIndexByEntryNo.containsKey(germplasm.getFemaleParentEntryNo())) && !(germplasm.getFemaleParentEntryNo().equals("0"))) { missingEntryNumbers.add(germplasm.getFemaleParentEntryNo()); } } // Check Male Parent - if (germplasm.getMaleParentEntryNo() != null && !pedigreeExists) { + if (germplasm.getMaleParentEntryNo() != null) { if ((!germplasmIndexByEntryNo.containsKey(germplasm.getMaleParentEntryNo())) && !(germplasm.getMaleParentEntryNo().equals("0"))) { missingEntryNumbers.add(germplasm.getMaleParentEntryNo()); } @@ -474,7 +468,7 @@ private boolean hasPedigree(BrAPIGermplasm germplasm) { germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.MALE_PARENT_UNKNOWN).getAsBoolean()); } - //Used to check if germplasm already has a pedigree in the database, if so, pedigree information in file should be ignored + //Used to check if germplasm already has a pedigree in the database private boolean databaseGermplasmHasPedigree(Germplasm germplasm) { if (germplasm.getAccessionNumber() == null || dbGermplasmByAccessionNo.get(germplasm.getAccessionNumber()) == null) { return false; From 5583a4236fc0886f0b5dd92008d22b5e905fe76c Mon Sep 17 00:00:00 2001 From: sb2597 <62260853+sb2597@users.noreply.github.com> Date: Wed, 18 Feb 2026 14:39:54 -0500 Subject: [PATCH 214/289] Update pull_request_template.md updated PR template --- .github/pull_request_template.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9fcdc371a..f46d280ff 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -16,8 +16,6 @@ _Please include any details needed for reviewers to test this code_ - [ ] I have performed a self-review of my own code - [ ] I have tested my code and ensured it meets the acceptance criteria of the story -- [ ] I have tested that my code works with both the brapi-java-server and BreedBase -- [ ] I have create/modified unit tests to cover this change +- [ ] I have create/modified unit and/or integration tests to cover this change or tests are not applicable - [ ] I have commented my code, particularly in hard-to-understand areas -- [ ] I have made corresponding changes to documentation -- [ ] I have run TAF: _\_ +- [ ] I have either updated the source of truth or arranged for update with product owner if needed https://breedinginsight.atlassian.net/wiki/spaces/BI/pages/1559953409/Source+of+Trut From 13ab6df13feafc4046c14fda99bf0e0b74b70161 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 18 Feb 2026 19:40:06 +0000 Subject: [PATCH 215/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index abb572efa..7a27bf3fb 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1085 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/3441dda2e5b002ad5b4314610e0d94d79889f9d3 \ No newline at end of file +version=v1.2.0+1087 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/5583a4236fc0886f0b5dd92008d22b5e905fe76c \ No newline at end of file From 7a2d21f9651472e3e7866b11f83b296f237b7df3 Mon Sep 17 00:00:00 2001 From: sb2597 <62260853+sb2597@users.noreply.github.com> Date: Wed, 18 Feb 2026 14:40:37 -0500 Subject: [PATCH 216/289] Update pull_request_template.md --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f46d280ff..2a083113d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -18,4 +18,4 @@ _Please include any details needed for reviewers to test this code_ - [ ] I have tested my code and ensured it meets the acceptance criteria of the story - [ ] I have create/modified unit and/or integration tests to cover this change or tests are not applicable - [ ] I have commented my code, particularly in hard-to-understand areas -- [ ] I have either updated the source of truth or arranged for update with product owner if needed https://breedinginsight.atlassian.net/wiki/spaces/BI/pages/1559953409/Source+of+Trut +- [ ] I have either updated the source of truth or arranged for update with product owner if needed https://breedinginsight.atlassian.net/wiki/spaces/BI/pages/1559953409/Source+of+Truth From 480e65c9452c2ab3779509e6c5e409cf5d056aa4 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 18 Feb 2026 19:40:49 +0000 Subject: [PATCH 217/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 7a27bf3fb..680ce0822 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1087 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/5583a4236fc0886f0b5dd92008d22b5e905fe76c \ No newline at end of file +version=v1.2.0+1089 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/7a2d21f9651472e3e7866b11f83b296f237b7df3 \ No newline at end of file From 33edf1fca20b7edb9dbc14b4be41d16b6b878aa1 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Wed, 18 Feb 2026 17:17:15 -0500 Subject: [PATCH 218/289] Exclude gigwa tests temporarily --- pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pom.xml b/pom.xml index d1ea4efb7..95d75a0d1 100644 --- a/pom.xml +++ b/pom.xml @@ -541,6 +541,16 @@ ${maven.compiler.target} + + org.apache.maven.plugins + maven-surefire-plugin + + + + **/GigwaGenotypeServiceImplIntegrationTest.java + + + org.jooq jooq-codegen-maven From 7d78445664ffe4babbcd27d1aadd56b194cdcdb6 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 18 Feb 2026 22:17:51 +0000 Subject: [PATCH 219/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 680ce0822..262913053 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1089 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/7a2d21f9651472e3e7866b11f83b296f237b7df3 \ No newline at end of file +version=v1.2.0+1091 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1efaf4e90b5499e1949cf7c8b94e6c8bbbeda6ac \ No newline at end of file From 8f6f047f1e1fef25dac0b561255284f835d3e0f5 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Thu, 19 Feb 2026 16:54:01 -0500 Subject: [PATCH 220/289] Fix status code check --- .../brapi/v2/services/BrAPITrialService.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index f34b5542b..5d8788beb 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -465,9 +465,11 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt String programDbId = program.getBrapiProgram() != null ? program.getBrapiProgram().getProgramDbId() : null; HttpResponse levelResponse = observationLevelDAO.createObservationLevelName(program, datasetName, DatasetLevel.SUB_OBS_UNIT, programDbId); - if (levelResponse.getStatus() == HttpStatus.CONFLICT) { - throw new AlreadyExistsException("Dataset name already exists in this experiment"); - } else if (levelResponse.getStatus().getCode() < 200 || levelResponse.getStatus().getCode() >= 300) { + + // 409 and 200 are expected response codes, anything else error out + // 409 means level already exists so we just use the name in OUs + // 200 means level was created successfully and can use the name in OUs + if (levelResponse.getStatus() != HttpStatus.CONFLICT && levelResponse.getStatus() != HttpStatus.OK) { throw new ApiException(levelResponse.getStatus().getCode(), "Unable to create observation level: " + levelResponse.getStatus().getReason()); } From 6a0651da1a44230b4e85b82720ec99fd610a1bb4 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 20 Feb 2026 16:09:28 +0000 Subject: [PATCH 221/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 262913053..cace2f05b 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1091 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1efaf4e90b5499e1949cf7c8b94e6c8bbbeda6ac \ No newline at end of file +version=v1.2.0+1093 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/386c889ea0658648e7a462cbe01402138a878609 \ No newline at end of file From 53b198c39504c681ea3be3f2f055b74d8126f908 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 20 Feb 2026 22:42:05 +0000 Subject: [PATCH 222/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index cace2f05b..19f29ba5a 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1093 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/386c889ea0658648e7a462cbe01402138a878609 \ No newline at end of file +version=v1.2.0+1095 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/51e40b2ea9984153fa6e0ff6704eb4802f02d644 \ No newline at end of file From a71fd52296ea2bad8c06df153b60fd0fde2cf174 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Mon, 23 Feb 2026 14:42:04 -0500 Subject: [PATCH 223/289] Update version --- src/main/resources/version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 19f29ba5a..d84e94091 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1095 +version=v1.3.0+1095 versionInfo=https://github.com/Breeding-Insight/bi-api/commit/51e40b2ea9984153fa6e0ff6704eb4802f02d644 \ No newline at end of file From 7d7201def64fbb83c88cd22911f6c01261aac4c5 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 23 Feb 2026 19:42:25 +0000 Subject: [PATCH 224/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index d84e94091..3f573296e 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1095 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/51e40b2ea9984153fa6e0ff6704eb4802f02d644 \ No newline at end of file +version=v1.3.0+1097 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a71fd52296ea2bad8c06df153b60fd0fde2cf174 \ No newline at end of file From 4f9a56b2593a2b1f548676c9c9a67e27c958dd18 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Tue, 24 Feb 2026 08:16:44 -0500 Subject: [PATCH 225/289] [BI-2744] DeltaBreed:Create exp, create obs level --- .../experiment/ExperimentUtilities.java | 1 + .../experiment/create/model/PendingData.java | 2 +- .../steps/CommitPendingImportObjectsStep.java | 26 ++++++++++++++++++- ...ulateExistingPendingImportObjectsStep.java | 2 ++ .../PopulateNewPendingImportObjectsStep.java | 9 +++++++ 5 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 1cdfcfdaf..44383d2ba 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -71,6 +71,7 @@ public class ExperimentUtilities { public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; public static final String MIDNIGHT = "T00:00:00-00:00"; public static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; + public static final String MULTIPLE_EXP_UNITS = "File contains more than one Exp Unit"; public static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists"; public static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs"; public static final String UNMATCHED_COLUMN = "Ontology term(s) not found: "; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index de3f60844..fe71cd39f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -45,7 +45,7 @@ public class PendingData { private Map> locationByName; private Map> obsVarDatasetByName; private Map> existingGermplasmByGID; - + private Map expUnitByTrialName; private Map> pendingObservationByHash; private Map> timeStampColByPheno; private ValidationErrors validationErrors; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index 9f75bae4d..77c5f54b1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -16,6 +16,7 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import io.micronaut.http.HttpResponse; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.map.CaseInsensitiveMap; @@ -42,6 +43,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; +import org.breedinginsight.model.DatasetLevel; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.model.Trait; @@ -72,6 +74,7 @@ public class CommitPendingImportObjectsStep { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final ProgramLocationService locationService; private final OntologyService ontologyService; + private final BrAPIObservationLevelDAO brAPIObservationLevelDAO; @Inject public CommitPendingImportObjectsStep(BrAPIListDAO brAPIListDAO, @@ -80,7 +83,8 @@ public CommitPendingImportObjectsStep(BrAPIListDAO brAPIListDAO, BrAPIObservationDAO brAPIObservationDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO, ProgramLocationService locationService, - OntologyService ontologyService) { + OntologyService ontologyService, + BrAPIObservationLevelDAO brAPIObservationLevelDAO) { this.brAPIListDAO = brAPIListDAO; this.brapiTrialDAO = brapiTrialDAO; this.brAPIStudyDAO = brAPIStudyDAO; @@ -88,6 +92,7 @@ public CommitPendingImportObjectsStep(BrAPIListDAO brAPIListDAO, this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.locationService = locationService; this.ontologyService = ontologyService; + this.brAPIObservationLevelDAO = brAPIObservationLevelDAO; } // TODO: some common code between workflows here that could be broken out, removed append/update specific code @@ -106,6 +111,7 @@ public void process(ProcessContext processContext, ProcessedData processedData) Map> locationByName = pendingData.getLocationByName(); Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); Map> observationByHash = pendingData.getObservationByHash(); + Map expUnitbyTrialName = pendingData.getExpUnitByTrialName(); List newTrials = ProcessorData.getNewObjects(pendingData.getTrialByNameNoScope()); @@ -170,6 +176,8 @@ public void process(ProcessContext processContext, ProcessedData processedData) trialByNameNoScope.get(createdTrialName) .getBrAPIObject() .setTrialDbId(createdTrial.getTrialDbId()); + + createObservationLevel(createdTrialName, expUnitbyTrialName, program); } List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); @@ -243,6 +251,22 @@ public void process(ProcessContext processContext, ProcessedData processedData) } + //Check if the experimental unit associated with the trial does not exist in the system and if so create a new observation level + private void createObservationLevel(String trialName, Map expUnitByTrialName, Program program) throws ApiException, InternalServerException { + String expUnit = expUnitByTrialName.get(trialName).toLowerCase(); + String programDbId = program.getBrapiProgram() != null ? program.getBrapiProgram().getProgramDbId() : null; + HttpResponse levelResponse = brAPIObservationLevelDAO.createObservationLevelName(program, expUnit, DatasetLevel.EXP_UNIT, programDbId); + + if (levelResponse.getStatus().getCode() == 409) { + log.info(String.format("Level %s already exists in database", expUnit)); + } else if (levelResponse.getStatus().getCode() == 200) { + log.info(String.format("Level %s created in database", expUnit)); + } else { + log.error("Error saving experiment import: " + levelResponse.getStatus().getReason()); + throw new InternalServerException("Unable to create observation level: " + levelResponse.getStatus().getReason()); + } + } + private void updateStudyDependencyValues(PendingData pendingData, Map mappedBrAPIImport, String programKey) { // update location DbIds in studies for all distinct locations Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index 16453788a..f30512d2b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -103,6 +103,7 @@ public ProcessContext process(ImportContext input) { Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); Map> existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); Map existingObsByObsHash = new HashMap<>(); + Map expUnitByTrialName = new HashMap<>(); PendingData existing = PendingData.builder() .observationUnitByNameNoScope(observationUnitByNameNoScope) @@ -113,6 +114,7 @@ public ProcessContext process(ImportContext input) { .existingGermplasmByGID(existingGermplasmByGID) .existingObsByObsHash(existingObsByObsHash) .observationByHash(new HashMap<>()) + .expUnitByTrialName(expUnitByTrialName) .build(); return ProcessContext.builder() diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index 57d62f492..d2a834bee 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -302,7 +302,9 @@ public PendingImportObject populateTrial(ImportContext importContext boolean commit = importContext.isCommit(); Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + Map expUnitByTrialName = pendingData.getExpUnitByTrialName(); + //Experiment titles and environment checks if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { PendingImportObject envPio; trialPio = trialByNameNoScope.get(importRow.getExpTitle()); @@ -327,6 +329,13 @@ public PendingImportObject populateTrial(ImportContext importContext //trialByNameNoScope.put(importRow.getExpTitle(), trialPio); } + //Experiment unit duplicate check + if (expUnitByTrialName.get(importRow.getExpTitle()) == null) { + expUnitByTrialName.put(importRow.getExpTitle(), importRow.getExpUnit()); + } else if (!expUnitByTrialName.get(importRow.getExpTitle()).equalsIgnoreCase(importRow.getExpUnit())) { + throw new UnprocessableEntityException(MULTIPLE_EXP_UNITS); + } + return trialPio; } From 08a4fbb4478aef58d956b05f9c006512257c903d Mon Sep 17 00:00:00 2001 From: HMS17 Date: Wed, 25 Feb 2026 19:03:13 -0500 Subject: [PATCH 226/289] [BI-2791] Exp File Name Improvement --- .../brapi/v2/services/BrAPITrialService.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 5d8788beb..116ff7173 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -292,8 +292,7 @@ public DownloadFile exportObservations( List> rows = entry.getValue(); sortDefaultForExportRows(rows); StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, rows, fileType, SHEET_NAME); - // TODO: [BI-2183] remove hardcoded datasetName, use observation level. - String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName(), "Observation Dataset") + fileType.getExtension(); + String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName(), StringUtils.capitalize(observationLvl.toLowerCase())) + fileType.getExtension(); // Add to file list. files.add(new DownloadFile(name, streamedFile)); } @@ -313,9 +312,8 @@ public DownloadFile exportObservations( // write export data to requested file format StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, exportRows, fileType, SHEET_NAME); // Set filename. - String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); - // TODO: [BI-2183] remove hardcoded datasetName, use observation level. - String fileName = makeFileName(experiment, program, envFilenameFragment, "Observation Dataset") + fileType.getExtension(); + String envFilenameFragment = params.getEnvironments() == null ? "All Env" : params.getEnvironments(); + String fileName = makeFileName(experiment, program, envFilenameFragment, StringUtils.capitalize(observationLvl.toLowerCase())) + fileType.getExtension(); downloadFile = new DownloadFile(fileName, streamedFile); } From 6e1c2fc3d0813e62e676c6785d0babcf62d12e3d Mon Sep 17 00:00:00 2001 From: HMS17 Date: Thu, 26 Feb 2026 10:35:33 -0500 Subject: [PATCH 227/289] [BI-2791] Zip file name improvement --- .../brapi/v2/services/BrAPITrialService.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 116ff7173..df3e3c01d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -304,7 +304,7 @@ public DownloadFile exportObservations( log.debug(logHash + ": zipping files for export"); // Zip, as there are multiple files. StreamedFile zipFile = zipFiles(files); - downloadFile = new DownloadFile(makeZipFileName(experiment, program), zipFile); + downloadFile = new DownloadFile(makeZipFileName(experiment, program, StringUtils.capitalize(observationLvl.toLowerCase())), zipFile); } } else { List> exportRows = new ArrayList<>(rowByOUId.values()); @@ -964,12 +964,13 @@ private String makeFileName(BrAPITrial experiment, Program program, String envNa return Utilities.makePortableFilename(unsafeName); } - private String makeZipFileName(BrAPITrial experiment, Program program) { + private String makeZipFileName(BrAPITrial experiment, Program program, String datasetName) { // .zip DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_hh-mm-ssZ"); String timestamp = formatter.format(OffsetDateTime.now()); - String unsafeName = String.format("%s_%s.zip", + String unsafeName = String.format("%s_%s_%s.zip", Utilities.removeProgramKey(experiment.getTrialName(), program.getKey()), + datasetName, timestamp); // Make file name safe for all platforms. return Utilities.makePortableFilename(unsafeName); From 75ed32fa158e4bfeb8d0b9d88075779b7af6cd16 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 2 Mar 2026 23:00:41 +0000 Subject: [PATCH 228/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 3f573296e..0a1edad4a 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1097 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a71fd52296ea2bad8c06df153b60fd0fde2cf174 \ No newline at end of file +version=v1.3.0+1099 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/cad6123bdc282a316ff3436b2a8f77290edc7e44 \ No newline at end of file From fa960fa5105de9fb665d59b472dc2848657d6fb0 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Wed, 4 Mar 2026 14:19:55 -0500 Subject: [PATCH 229/289] [BI-2744] - Code review logs suggestion Co-authored-by: jloux-brapi --- .../create/workflow/steps/CommitPendingImportObjectsStep.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index 77c5f54b1..a015b5a19 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -258,7 +258,7 @@ private void createObservationLevel(String trialName, Map expUni HttpResponse levelResponse = brAPIObservationLevelDAO.createObservationLevelName(program, expUnit, DatasetLevel.EXP_UNIT, programDbId); if (levelResponse.getStatus().getCode() == 409) { - log.info(String.format("Level %s already exists in database", expUnit)); + log.info(String.format("Level with name=%s, order=%s, programDbId=%s already exists in database", expUnit, DatasetLevel.EXP_UNIT, programDbId)); } else if (levelResponse.getStatus().getCode() == 200) { log.info(String.format("Level %s created in database", expUnit)); } else { From 56ce24a1758dcfaa7f6e9c6937ed1f5cfb582e36 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Wed, 4 Mar 2026 14:22:56 -0500 Subject: [PATCH 230/289] [BI-2744] - Code review logs suggestion 2 Co-authored-by: jloux-brapi --- .../create/workflow/steps/CommitPendingImportObjectsStep.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index a015b5a19..ef9dc9ef5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -260,7 +260,7 @@ private void createObservationLevel(String trialName, Map expUni if (levelResponse.getStatus().getCode() == 409) { log.info(String.format("Level with name=%s, order=%s, programDbId=%s already exists in database", expUnit, DatasetLevel.EXP_UNIT, programDbId)); } else if (levelResponse.getStatus().getCode() == 200) { - log.info(String.format("Level %s created in database", expUnit)); + log.info(String.format("Level with name=%s, order=%s, programDbId=%s created in database", expUnit, DatasetLevel.EXP_UNIT, programDbId)); } else { log.error("Error saving experiment import: " + levelResponse.getStatus().getReason()); throw new InternalServerException("Unable to create observation level: " + levelResponse.getStatus().getReason()); From 02d9a563406b902831d58f00b5136a2a4ff1ff34 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Thu, 5 Mar 2026 12:07:58 -0500 Subject: [PATCH 231/289] Add synchronize trigger for PRs --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c4959b12..3d848d159 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ name: maven build on: pull_request: - types: [opened, edited] + types: [opened, synchronize, edited] workflow_dispatch: inputs: brapi_server_image: From f367c5d486b745c490d547ad6e50b2efe14a8094 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Thu, 5 Mar 2026 12:14:54 -0500 Subject: [PATCH 232/289] Trigger CI From ad6934b795f2ffead3f4bee892fd76897071cd8d Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 5 Mar 2026 17:27:55 +0000 Subject: [PATCH 233/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 0a1edad4a..7378a4d1d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1099 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/cad6123bdc282a316ff3436b2a8f77290edc7e44 \ No newline at end of file +version=v1.3.0+1101 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/02d9a563406b902831d58f00b5136a2a4ff1ff34 \ No newline at end of file From d1ecd1e907bce43912ef403d3888061183bb371a Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 5 Mar 2026 21:30:34 +0000 Subject: [PATCH 234/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 7378a4d1d..f57fe5eb1 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1101 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/02d9a563406b902831d58f00b5136a2a4ff1ff34 \ No newline at end of file +version=v1.3.0+1103 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c2a827ca981dcc9327de0b1cfd06fd574118442d \ No newline at end of file From ddaf750e0569f70cd14fced1c7a5f2ff320b27a9 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Fri, 6 Mar 2026 09:22:36 -0500 Subject: [PATCH 235/289] [BI-2778] Deprecate additionalinfo usage for level and use obs unit level --- .../brapi/v2/services/BrAPITrialService.java | 20 +++++++------------ .../ExperimentObservation.java | 3 --- .../importer/ExperimentFileImportTest.java | 6 ++---- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index df3e3c01d..87a2fa8d5 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -222,7 +222,7 @@ public DownloadFile exportObservations( columns = dynamicUpdateObsUnitIDLabel(columns, topObservationLvl); } } - String observationLvl = ous.get(0).getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); + String observationLvl = StringUtils.capitalize(ous.get(0).getObservationUnitPosition().getObservationLevel().getLevelName()); columns = dynamicUpdateObsUnitIDLabel(columns, observationLvl); if (params.getDatasetId() != null) { @@ -587,9 +587,6 @@ public BrAPIObservationUnit createSubObservationUnit( observationUnit.setTreatments(treatmentFactors); } - // Put level in additional info: keep this in case we decide to rename levels in future. - observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, subEntityDatasetName); - // Put RTK in additional info. JsonElement rtk = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.RTK); if (rtk != null) { @@ -641,16 +638,13 @@ public BrAPIObservationUnit createSubObservationUnit( } // ObservationLevelRelationships for top-level Exp Unit linking. BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); - // TODO: switch to using level name in main obs unit properties once dynamic obs unit support is complete - expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString().toLowerCase()); + expUnitLevel.setLevelName(expUnit.getObservationUnitPosition().getObservationLevel().getLevelName().toLowerCase()); String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId(); expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnitUUID, program.getKey(), seqVal)); expUnitLevel.setLevelOrder(DatasetLevel.EXP_UNIT.getValue()); levelRelationships.add(expUnitLevel); position.setObservationLevelRelationships(levelRelationships); - observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.EXP_UNIT_ID, expUnit.getObservationUnitName()); - // Set ObservationUnitPosition. observationUnit.setObservationUnitPosition(position); @@ -884,7 +878,7 @@ private Map createExportRow( } //Append observation level to obsUnitID - String observationLvl = ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); + String observationLvl = ou.getObservationUnitPosition().getObservationLevel().getLevelName(); row.put(observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX, ouId); if (isSubEntity) { @@ -897,13 +891,13 @@ private Map createExportRow( String topLvlOuId = Utilities.removeProgramKeyAndUnknownAdditionalData(topLevel.getLevelCode(), program.getKey()); row.put(topLvlName + " " + OBSERVATION_UNIT_ID_SUFFIX, topLvlOuId); } - row.put(ExperimentObservation.Columns.EXP_UNIT_ID, ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXP_UNIT_ID).getAsString()); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, getTopLevel(ou).getLevelName()); - row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, ou.getObservationUnitPosition().getObservationLevel().getLevelName()); row.put(ExperimentObservation.Columns.SUB_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); } else { - row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getObservationUnitPosition().getObservationLevel().getLevelName()); row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); } @@ -995,7 +989,7 @@ private void sortDefaultForObservationUnit(List ous) { if (isSubEntityDataset(ous)) { Comparator subUnitComparator = Comparator.comparing(BrAPIObservationUnit::getObservationUnitName, new IntOrderComparator()); - Comparator ouNameComparator = Comparator.comparing(row -> (row.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXP_UNIT_ID).toString()), new IntOrderComparator()); + Comparator ouNameComparator = Comparator.comparing(row -> (getTopLevel(row).getLevelName()), new IntOrderComparator()); ous.sort((studyNameComparator).thenComparing(ouNameComparator).thenComparing(subUnitComparator)); } else { diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index ace52b094..740590410 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -164,7 +164,6 @@ public BrAPITrial constructBrAPITrial(Program program, User user, boolean commit .toString()); createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_NAME, user.getName()); trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_BY, createdBy); - trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL, getExpUnit()); trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE, getExpType()); trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER, expSeqValue); @@ -280,12 +279,10 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( // If expUnit is null, a validation error will be produced later on. if (getExpUnit() != null) { - // TODO: [BI-2219] BJTS only accepts hardcoded levels, need to handle dynamic levels. level.setLevelName(getExpUnit().toLowerCase()); // HACK: toLowerCase() is needed to match BJTS hardcoded levels. } level.setLevelCode(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); position.setObservationLevel(level); - observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getExpUnit()); // Exp Unit List levelRelationships = new ArrayList<>(); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index f85a98dc5..271d337b1 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1700,8 +1700,7 @@ private Map assertRowSaved(Map expected, Program assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_DESCRIPTION), trial.getTrialDescription()); - assertEquals(expected.get(Columns.EXP_UNIT), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString()); - assertEquals(expected.get(Columns.EXP_UNIT), ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + assertEquals(expected.get(Columns.EXP_UNIT), ou.getObservationUnitPosition().getObservationLevel().getLevelName()); assertEquals(expected.get(Columns.EXP_TYPE), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); assertEquals(expected.get(Columns.EXP_TYPE), study.getStudyType()); assertEquals(expected.get(Columns.ENV), Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); @@ -1807,8 +1806,7 @@ private Map assertValidPreviewRow(Map expected, assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_DESCRIPTION), trial.getTrialDescription()); - assertEquals(expected.get(Columns.EXP_UNIT), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString()); - assertEquals(expected.get(Columns.EXP_UNIT), ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + assertEquals(expected.get(Columns.EXP_UNIT), ou.getObservationUnitPosition().getObservationLevel().getLevelName()); assertEquals(expected.get(Columns.EXP_TYPE), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); assertEquals(expected.get(Columns.EXP_TYPE), study.getStudyType()); assertEquals(expected.get(Columns.ENV), Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); From 5379e4a734fea0381e7877655a112c114d1474ec Mon Sep 17 00:00:00 2001 From: HMS17 Date: Fri, 6 Mar 2026 10:30:49 -0500 Subject: [PATCH 236/289] [BI-2778] Tests casing fix --- .../brapps/importer/ExperimentFileImportTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 271d337b1..c3cf7ed27 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1700,7 +1700,7 @@ private Map assertRowSaved(Map expected, Program assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_DESCRIPTION), trial.getTrialDescription()); - assertEquals(expected.get(Columns.EXP_UNIT), ou.getObservationUnitPosition().getObservationLevel().getLevelName()); + assertEquals(expected.get(Columns.EXP_UNIT).toString().toLowerCase(), ou.getObservationUnitPosition().getObservationLevel().getLevelName().toLowerCase()); assertEquals(expected.get(Columns.EXP_TYPE), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); assertEquals(expected.get(Columns.EXP_TYPE), study.getStudyType()); assertEquals(expected.get(Columns.ENV), Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); @@ -1806,7 +1806,7 @@ private Map assertValidPreviewRow(Map expected, assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_DESCRIPTION), trial.getTrialDescription()); - assertEquals(expected.get(Columns.EXP_UNIT), ou.getObservationUnitPosition().getObservationLevel().getLevelName()); + assertEquals(expected.get(Columns.EXP_UNIT).toString().toLowerCase(), ou.getObservationUnitPosition().getObservationLevel().getLevelName().toLowerCase()); assertEquals(expected.get(Columns.EXP_TYPE), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); assertEquals(expected.get(Columns.EXP_TYPE), study.getStudyType()); assertEquals(expected.get(Columns.ENV), Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); From 70742470eda7989c441ed16824ffa37a28240dff Mon Sep 17 00:00:00 2001 From: HMS17 Date: Fri, 6 Mar 2026 14:19:36 -0500 Subject: [PATCH 237/289] [BI-2778] Export file casing fix --- .../brapi/v2/services/BrAPITrialService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 87a2fa8d5..a12a1ce71 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -891,13 +891,13 @@ private Map createExportRow( String topLvlOuId = Utilities.removeProgramKeyAndUnknownAdditionalData(topLevel.getLevelCode(), program.getKey()); row.put(topLvlName + " " + OBSERVATION_UNIT_ID_SUFFIX, topLvlOuId); } - row.put(ExperimentObservation.Columns.EXP_UNIT_ID, getTopLevel(ou).getLevelName()); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, StringUtils.capitalize(getTopLevel(ou).getLevelName())); - row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, ou.getObservationUnitPosition().getObservationLevel().getLevelName()); + row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, StringUtils.capitalize(ou.getObservationUnitPosition().getObservationLevel().getLevelName())); row.put(ExperimentObservation.Columns.SUB_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); } else { - row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getObservationUnitPosition().getObservationLevel().getLevelName()); + row.put(ExperimentObservation.Columns.EXP_UNIT, StringUtils.capitalize(ou.getObservationUnitPosition().getObservationLevel().getLevelName())); row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); } From d5e532cd217c61969ee44037f941da8d7a164f9a Mon Sep 17 00:00:00 2001 From: HMS17 Date: Fri, 6 Mar 2026 14:53:04 -0500 Subject: [PATCH 238/289] [BI-2778] Fixes --- .../brapi/v2/constants/BrAPIAdditionalInfoFields.java | 1 - .../imports/experimentObservation/ExperimentObservation.java | 1 + .../brapps/importer/ExperimentFileImportTest.java | 2 ++ 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index a69a7b983..010bb15e9 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -36,7 +36,6 @@ public final class BrAPIAdditionalInfoFields { public static final String GERMPLASM_BREEDING_METHOD = "breedingMethod"; public static final String CREATED_DATE = "createdDate"; public static final String DEFAULT_OBSERVATION_LEVEL = "defaultObservationLevel"; - public static final String OBSERVATION_LEVEL = "observationLevel"; public static final String RTK = "rtk"; public static final String EXPERIMENT_TYPE = "experimentType"; public static final String EXPERIMENT_NUMBER = "experimentNumber"; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 740590410..abe1d2ebe 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -164,6 +164,7 @@ public BrAPITrial constructBrAPITrial(Program program, User user, boolean commit .toString()); createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_NAME, user.getName()); trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_BY, createdBy); + trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL, getExpUnit()); trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE, getExpType()); trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER, expSeqValue); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index c3cf7ed27..802bf6889 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1700,6 +1700,7 @@ private Map assertRowSaved(Map expected, Program assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_DESCRIPTION), trial.getTrialDescription()); + assertEquals(expected.get(Columns.EXP_UNIT), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString()); assertEquals(expected.get(Columns.EXP_UNIT).toString().toLowerCase(), ou.getObservationUnitPosition().getObservationLevel().getLevelName().toLowerCase()); assertEquals(expected.get(Columns.EXP_TYPE), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); assertEquals(expected.get(Columns.EXP_TYPE), study.getStudyType()); @@ -1806,6 +1807,7 @@ private Map assertValidPreviewRow(Map expected, assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_DESCRIPTION), trial.getTrialDescription()); + assertEquals(expected.get(Columns.EXP_UNIT), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString()); assertEquals(expected.get(Columns.EXP_UNIT).toString().toLowerCase(), ou.getObservationUnitPosition().getObservationLevel().getLevelName().toLowerCase()); assertEquals(expected.get(Columns.EXP_TYPE), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); assertEquals(expected.get(Columns.EXP_TYPE), study.getStudyType()); From 81a7a856797ae62c3d8c709679644fa19977c590 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Fri, 6 Mar 2026 15:40:57 -0500 Subject: [PATCH 239/289] [BI-2778] Casing test --- .../v1/controller/ExperimentControllerIntegrationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 21358e69d..1b2b5ecc2 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -989,9 +989,9 @@ private void checkDownloadTable( //Observation level for tests should be "Plot" // Observation units populated. - assertEquals(0, table.column("Plot ObsUnitID").countMissing()); + assertEquals(0, table.column("plot ObsUnitID").countMissing()); // Observation Unit IDs are assigned. - assertEquals(requestedImportRows.size(), table.column("Plot ObsUnitID").countUnique()); + assertEquals(requestedImportRows.size(), table.column("plot ObsUnitID").countUnique()); } private boolean isMatchedRow(Map importRow, Row downloadRow) { From e4cae769915bab0368025dbf56dd66cf17522507 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Fri, 6 Mar 2026 16:45:05 -0500 Subject: [PATCH 240/289] [BI-2778] Another casing fix --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index a12a1ce71..69d58da75 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -878,7 +878,7 @@ private Map createExportRow( } //Append observation level to obsUnitID - String observationLvl = ou.getObservationUnitPosition().getObservationLevel().getLevelName(); + String observationLvl = StringUtils.capitalize(ou.getObservationUnitPosition().getObservationLevel().getLevelName()); row.put(observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX, ouId); if (isSubEntity) { From 341c310c89a8aa506d0c245f9711f911738c9665 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Fri, 6 Mar 2026 18:13:50 -0500 Subject: [PATCH 241/289] [BI-2778] Null pointer fix re treatment factors --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 69d58da75..e6699030a 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -576,7 +576,7 @@ public BrAPIObservationUnit createSubObservationUnit( } // Set treatment factors. - if (!expUnit.getTreatments().isEmpty()) { + if (!(expUnit.getTreatments() == null || expUnit.getTreatments().isEmpty())) { List treatmentFactors = new ArrayList<>(); for (BrAPIObservationTreatment t : expUnit.getTreatments()) { BrAPIObservationTreatment treatment = new BrAPIObservationTreatment(); From 7db547088cc0ab6df7a11b7dfbb405a799d7b5fa Mon Sep 17 00:00:00 2001 From: HMS17 Date: Mon, 9 Mar 2026 09:54:28 -0400 Subject: [PATCH 242/289] [BI-2778] Test fix --- .../v1/controller/ExperimentControllerIntegrationTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 1b2b5ecc2..ec6df569a 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -850,7 +850,7 @@ private Map makeExpImportRow(String title, String environment) { row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); row.put(ExperimentObservation.Columns.EXP_TITLE, title); - row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "PlotA"); //row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, ""); row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); row.put(ExperimentObservation.Columns.ENV, environment); @@ -989,9 +989,9 @@ private void checkDownloadTable( //Observation level for tests should be "Plot" // Observation units populated. - assertEquals(0, table.column("plot ObsUnitID").countMissing()); + assertEquals(0, table.column("Plota ObsUnitID").countMissing()); // Observation Unit IDs are assigned. - assertEquals(requestedImportRows.size(), table.column("plot ObsUnitID").countUnique()); + assertEquals(requestedImportRows.size(), table.column("Plota ObsUnitID").countUnique()); } private boolean isMatchedRow(Map importRow, Row downloadRow) { From a80a3810b66a496b9cdfc96047f097ed1bda0e9a Mon Sep 17 00:00:00 2001 From: HMS17 Date: Mon, 9 Mar 2026 10:19:54 -0400 Subject: [PATCH 243/289] [BI-2778] Unit test experiment --- .../v1/controller/ExperimentControllerIntegrationTest.java | 6 +++--- src/test/resources/sql/ImportControllerIntegrationTest.sql | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index ec6df569a..21358e69d 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -850,7 +850,7 @@ private Map makeExpImportRow(String title, String environment) { row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); row.put(ExperimentObservation.Columns.EXP_TITLE, title); - row.put(ExperimentObservation.Columns.EXP_UNIT, "PlotA"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); //row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, ""); row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); row.put(ExperimentObservation.Columns.ENV, environment); @@ -989,9 +989,9 @@ private void checkDownloadTable( //Observation level for tests should be "Plot" // Observation units populated. - assertEquals(0, table.column("Plota ObsUnitID").countMissing()); + assertEquals(0, table.column("Plot ObsUnitID").countMissing()); // Observation Unit IDs are assigned. - assertEquals(requestedImportRows.size(), table.column("Plota ObsUnitID").countUnique()); + assertEquals(requestedImportRows.size(), table.column("Plot ObsUnitID").countUnique()); } private boolean isMatchedRow(Map importRow, Row downloadRow) { diff --git a/src/test/resources/sql/ImportControllerIntegrationTest.sql b/src/test/resources/sql/ImportControllerIntegrationTest.sql index 7bab169b8..34afd6e2b 100644 --- a/src/test/resources/sql/ImportControllerIntegrationTest.sql +++ b/src/test/resources/sql/ImportControllerIntegrationTest.sql @@ -33,7 +33,7 @@ INSERT INTO public.importer_mapping (name,import_type_id,mapping,file,draft,crea INSERT INTO public.importer_mapping (name,import_type_id,mapping,file,draft,created_by,updated_by) values ('ExperimentsTemplateMap','ExperimentImport', '[{"id": "726a9f10-4892-4204-9e52-bd2b1d735f65", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "98774e20-6f89-4d6a-a7c9-f88887228ed6", "value": {"fileFieldName": "Germplasm GID"}, "objectId": "gid"}, {"id": "880ef0c9-4e3e-42d4-9edc-667684a91889", "value": {"fileFieldName": "Test (T) or Check (C)"}, "objectId": "test_or_check"}, {"id": "b693eca7-efcd-4518-a9d3-db0b037a76ee", "value": {"fileFieldName": "Exp Title"}, "objectId": "exp_title"}, {"id": "df340215-db6e-4219-a3b7-119f297b81c3", "value": {"fileFieldName": "Exp Description"}, "objectId": "expDescription"}, {"id": "9ca7cc81-562c-43a7-989a-41da309f603d", "value": {"fileFieldName": "Exp Unit"}, "objectId": "expUnit"}, {"id": "27215777-c8f9-4fe7-a7ac-918d6168b0dd", "value": {"fileFieldName": "Exp Type"}, "objectId": "expType"}, {"id": "19d220e2-dff0-4a3a-ad6e-32f4d8602b5c", "value": {"fileFieldName": "Env"}, "objectId": "env"}, {"id": "861518b9-5c9e-4fe5-b31e-baf16e27155d", "value": {"fileFieldName": "Env Location"}, "objectId": "envLocation"}, {"id": "667355c3-dae1-4a64-94c8-ac2d543bd474", "value": {"fileFieldName": "Env Year"}, "objectId": "envYear"}, {"id": "ad11f2df-c5b4-4a05-8e52-c57625140061", "value": {"fileFieldName": "Exp Unit ID"}, "objectId": "expUnitId"}, {"id": "639b40ec-20f8-4659-8464-6a4be997ac7a", "value": {"fileFieldName": "Exp Replicate #"}, "objectId": "expReplicateNo"}, {"id": "2a62a80f-d8ba-42c4-9997-3b4ac8a965aa", "value": {"fileFieldName": "Exp Block #"}, "objectId": "expBlockNo"}, {"id": "f3e7de69-21ad-4cda-b1cc-a5e1987fb931", "value": {"fileFieldName": "Row"}, "objectId": "row"}, {"id": "251c5bbd-fc4d-4371-a4ce-4e2686f6837e", "value": {"fileFieldName": "Column"}, "objectId": "column"}, {"id": "ce5f61f2-f1de-45a4-8baf-e2471a5d863d", "value": {"fileFieldName": "Treatment Factors"}, "objectId": "treatmentFactors"}, {"id": "c5b8276f-e777-4385-a80f-5199abe63aac", "value": {"fileFieldName": "ObsUnitID"}, "objectId": "ObsUnitID"}]', - '[{"Env": "New Study", "Row": 4, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "plot", "Exp Title": "New Trial", "ObsUnitID": "", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Treatment Factors": "Jam application", "Test (T) or Check (C)": true}]', + '[{"Env": "New Study", "Row": 4, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "Plot", "Exp Title": "New Trial", "ObsUnitID": "", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Treatment Factors": "Jam application", "Test (T) or Check (C)": true}]', false,user_id,user_id); END $$; From 6e601f8d78a369803811ce0152e4dbdfc2aa4b96 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Tue, 10 Mar 2026 16:59:16 -0400 Subject: [PATCH 244/289] [BI-2778] Revert erroneous EXP_UNIT_ID changes --- .../brapi/v2/services/BrAPITrialService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index e6699030a..d67aad8d6 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -891,8 +891,8 @@ private Map createExportRow( String topLvlOuId = Utilities.removeProgramKeyAndUnknownAdditionalData(topLevel.getLevelCode(), program.getKey()); row.put(topLvlName + " " + OBSERVATION_UNIT_ID_SUFFIX, topLvlOuId); } - row.put(ExperimentObservation.Columns.EXP_UNIT_ID, StringUtils.capitalize(getTopLevel(ou).getLevelName())); - + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXP_UNIT_ID).getAsString()); + row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, StringUtils.capitalize(ou.getObservationUnitPosition().getObservationLevel().getLevelName())); row.put(ExperimentObservation.Columns.SUB_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); @@ -989,7 +989,7 @@ private void sortDefaultForObservationUnit(List ous) { if (isSubEntityDataset(ous)) { Comparator subUnitComparator = Comparator.comparing(BrAPIObservationUnit::getObservationUnitName, new IntOrderComparator()); - Comparator ouNameComparator = Comparator.comparing(row -> (getTopLevel(row).getLevelName()), new IntOrderComparator()); + Comparator ouNameComparator = Comparator.comparing(row -> (row.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXP_UNIT_ID).toString()), new IntOrderComparator()); ous.sort((studyNameComparator).thenComparing(ouNameComparator).thenComparing(subUnitComparator)); } else { From e6895f42574b014f8c87705baf949fda47bc397d Mon Sep 17 00:00:00 2001 From: Jason Loux Date: Tue, 10 Mar 2026 15:03:58 -0400 Subject: [PATCH 245/289] Fix issues with sub-entity obs hash, list obs var creation --- .../brapi/v2/services/BrAPITrialService.java | 54 +++++++++++++++++++ .../process/ImportTableProcess.java | 31 ++++++----- .../service/ObservationService.java | 4 +- 3 files changed, 71 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index df3e3c01d..e31017754 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -16,6 +16,8 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.*; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; +import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.brapi.v2.model.germ.BrAPIGermplasm; @@ -27,6 +29,8 @@ import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.dao.db.enums.DataType; @@ -511,6 +515,17 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt latestExperiment.getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, DatasetUtil.jsonArrayFromDatasets(datasets)); trialDAO.updateBrAPITrial(latestExperiment.getTrialDbId(), latestExperiment, program.getId()); + BrAPIListDetails subEntityObsVarsList = createSubEntityObsVarList(programDbId, latestExperiment, subEntityDatasetMetadata); + + BrAPIListNewRequest listRq = new BrAPIListNewRequest(); + listRq.setListName(subEntityObsVarsList.getListName()); + listRq.setListType(subEntityObsVarsList.getListType()); + listRq.setExternalReferences(subEntityObsVarsList.getExternalReferences()); + listRq.setAdditionalInfo(subEntityObsVarsList.getAdditionalInfo()); + listRq.data(subEntityObsVarsList.getData()); + + var newList = listDAO.createBrAPILists(List.of(listRq), program.getId(), null); + return getDatasetData(program, experimentId, subEntityDatasetId, false); }); } catch (TimeoutException e) { @@ -522,6 +537,45 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt } } + public BrAPIListDetails createSubEntityObsVarList(String programDbId, + BrAPITrial trial, + DatasetMetadata subEntityDatasetMetadata) { + + // TODO: this is common to both workflows + String name = String.format("Observation Dataset [%s-%s-%s]", + programDbId, + trial.getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString(), + subEntityDatasetMetadata.getName()); + + return constructDatasetDetails( + name, + subEntityDatasetMetadata.getId(), + referenceSource, + programDbId, + trial.getTrialDbId()); + + } + + public BrAPIListDetails constructDatasetDetails( + String name, + UUID datasetId, + String referenceSourceBase, + String programDbId, String trialId) { + BrAPIListDetails dataSetDetails = new BrAPIListDetails(); + dataSetDetails.setListName(name); + dataSetDetails.setListType(BrAPIListTypes.OBSERVATIONVARIABLES); + dataSetDetails.setData(new ArrayList<>()); + dataSetDetails.putAdditionalInfoItem("datasetType", "observationDataset"); + List refs = new ArrayList<>(); + Utilities.addReference(refs, UUID.fromString(programDbId), referenceSourceBase, ExternalReferenceSource.PROGRAMS); + Utilities.addReference(refs, UUID.fromString(trialId), referenceSourceBase, ExternalReferenceSource.TRIALS); + Utilities.addReference(refs, datasetId, referenceSourceBase, ExternalReferenceSource.DATASET); + dataSetDetails.setExternalReferences(refs); + return dataSetDetails; + } + public BrAPIObservationUnit createSubObservationUnit( String subEntityDatasetName, String subUnitId, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 4f4cbe9e0..b9815ccfd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -22,27 +22,23 @@ import com.google.gson.GsonBuilder; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.HttpStatus; -import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.ProcessedDataFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; @@ -60,15 +56,12 @@ import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; import org.breedinginsight.utilities.Utilities; -import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; import javax.inject.Inject; import java.util.*; import java.util.stream.Collectors; -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.*; -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.DATASET_NOT_FOUND; import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; @Slf4j @@ -161,12 +154,19 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext // Read any observation data stored for these traits log.debug("fetching observation data stored for traits"); - Set ouDbIds = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); - Set varDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); - List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, varDbIds, program); + Set ouBrapiDbIds = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); + Set varBrapiDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); + List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouBrapiDbIds, varBrapiDbIds, program); + + // OU Exref Ids are what are displayed to users in the Obs Unit Id columns in experiments in DeltaBreed. + Map OuExRefIdByObsBrAPIDbId = new HashMap<>(); + + for (BrAPIObservation observation : observations) { + Optional ouExref = Utilities.getExternalReference(observation.getExternalReferences(), brapiReferenceSource, ExternalReferenceSource.OBSERVATION_UNITS); + ouExref.ifPresent(exRef -> OuExRefIdByObsBrAPIDbId.put(observation.getObservationDbId(), exRef.getReferenceId().toString())); + } // Construct helper lookup tables to use for hashing stored observation data - Map unitNameByDbId = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, BrAPIObservationUnit::getObservationUnitName)); Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); Map studyNameByDbId = context.getAppendOverwriteWorkflowContext().getStudyByNameNoScope().values().stream() .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) @@ -175,7 +175,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext // Hash stored observation data using a signature of unit, variable, and study names Map observationByObsHash = observations.stream().collect(Collectors.toMap(o->{ - return observationService.getObservationHash(unitNameByDbId.get(o.getObservationUnitDbId()), + return observationService.getObservationHash(OuExRefIdByObsBrAPIDbId.get(o.getObservationDbId()), variableNameByDbId.get(o.getObservationVariableDbId()), studyNameByDbId.get(o.getStudyDbId())); }, o->o)); @@ -226,9 +226,8 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext String cellData = column.getString(rowNum); // Generate hash for looking up prior observation data - String unitName = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); String phenoColumnName = column.name(); - String observationHash = observationService.getObservationHash(unitName, phenoColumnName, studyName); + String observationHash = observationService.getObservationHash(unitId, phenoColumnName, studyName); // Get timestamp if associated column var cell = new Object() { // mutable reference object to make timestamp accessible in anonymous methods diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index de98202fa..ed432fb7e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -99,8 +99,8 @@ public boolean validDateValue(String value) { } return true; } - public String getObservationHash(String observationUnitName, String variableName, String studyName) { - String concat = DigestUtils.sha256Hex(observationUnitName) + + public String getObservationHash(String observationUnitId, String variableName, String studyName) { + String concat = DigestUtils.sha256Hex(observationUnitId) + DigestUtils.sha256Hex(variableName) + DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); return DigestUtils.sha256Hex(concat); From 7451bacdb3cb60cb36a48dc33061bb58b634c3f8 Mon Sep 17 00:00:00 2001 From: Jason Loux Date: Thu, 12 Mar 2026 17:40:10 -0400 Subject: [PATCH 246/289] Refactor re-used code into DatasetService --- .../brapi/v2/dao/BrAPIListDAO.java | 2 + .../brapi/v2/services/BrAPITrialService.java | 64 +++---------------- .../ExperimentObservation.java | 17 ----- .../PopulateNewPendingImportObjectsStep.java | 7 +- .../experiment/service/DatasetService.java | 50 ++++++++++++++- 5 files changed, 64 insertions(+), 76 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java index 5ed73df60..4f6cfdec7 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java @@ -43,6 +43,7 @@ import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; +import javax.inject.Singleton; import javax.validation.constraints.NotNull; import java.util.ArrayList; import java.util.Collections; @@ -50,6 +51,7 @@ import java.util.UUID; @Slf4j +@Singleton public class BrAPIListDAO { private ProgramDAO programDAO; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index e31017754..989b9172d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -16,8 +16,6 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.*; -import org.brapi.v2.model.core.request.BrAPIListNewRequest; -import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.brapi.v2.model.germ.BrAPIGermplasm; @@ -29,10 +27,9 @@ import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.FileMappingUtil; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.model.BrAPIConstants; import org.breedinginsight.model.Column; @@ -83,6 +80,7 @@ public class BrAPITrialService { private final FileMappingUtil fileMappingUtil; private final DistributedLockService lockService; private static final String SHEET_NAME = "Data"; + private final DatasetService datasetService; @Inject public BrAPITrialService(@Property(name = "brapi.server.reference-source") String referenceSource, @@ -97,7 +95,8 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin BrAPIObservationLevelDAO observationLevelDAO, BrAPIGermplasmDAO germplasmDAO, FileMappingUtil fileMappingUtil, - DistributedLockService lockService) { + DistributedLockService lockService, + DatasetService datasetService) { this.referenceSource = referenceSource; this.trialDAO = trialDAO; @@ -112,6 +111,7 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin this.germplasmDAO = germplasmDAO; this.fileMappingUtil = fileMappingUtil; this.lockService = lockService; + this.datasetService = datasetService; } public List getExperiments(UUID programId) throws ApiException, DoesNotExistException { @@ -465,8 +465,8 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt throw new AlreadyExistsException("Dataset name already exists in this experiment"); } - String programDbId = program.getBrapiProgram() != null ? program.getBrapiProgram().getProgramDbId() : null; - HttpResponse levelResponse = observationLevelDAO.createObservationLevelName(program, datasetName, DatasetLevel.SUB_OBS_UNIT, programDbId); + String programBrapiDbId = program.getBrapiProgram() != null ? program.getBrapiProgram().getProgramDbId() : null; + HttpResponse levelResponse = observationLevelDAO.createObservationLevelName(program, datasetName, DatasetLevel.SUB_OBS_UNIT, programBrapiDbId); // 409 and 200 are expected response codes, anything else error out // 409 means level already exists so we just use the name in OUs @@ -515,16 +515,7 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt latestExperiment.getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, DatasetUtil.jsonArrayFromDatasets(datasets)); trialDAO.updateBrAPITrial(latestExperiment.getTrialDbId(), latestExperiment, program.getId()); - BrAPIListDetails subEntityObsVarsList = createSubEntityObsVarList(programDbId, latestExperiment, subEntityDatasetMetadata); - - BrAPIListNewRequest listRq = new BrAPIListNewRequest(); - listRq.setListName(subEntityObsVarsList.getListName()); - listRq.setListType(subEntityObsVarsList.getListType()); - listRq.setExternalReferences(subEntityObsVarsList.getExternalReferences()); - listRq.setAdditionalInfo(subEntityObsVarsList.getAdditionalInfo()); - listRq.data(subEntityObsVarsList.getData()); - - var newList = listDAO.createBrAPILists(List.of(listRq), program.getId(), null); + datasetService.createBrAPIObsVarListForDataset(program, latestExperiment, subEntityDatasetMetadata); return getDatasetData(program, experimentId, subEntityDatasetId, false); }); @@ -537,45 +528,6 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt } } - public BrAPIListDetails createSubEntityObsVarList(String programDbId, - BrAPITrial trial, - DatasetMetadata subEntityDatasetMetadata) { - - // TODO: this is common to both workflows - String name = String.format("Observation Dataset [%s-%s-%s]", - programDbId, - trial.getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) - .getAsString(), - subEntityDatasetMetadata.getName()); - - return constructDatasetDetails( - name, - subEntityDatasetMetadata.getId(), - referenceSource, - programDbId, - trial.getTrialDbId()); - - } - - public BrAPIListDetails constructDatasetDetails( - String name, - UUID datasetId, - String referenceSourceBase, - String programDbId, String trialId) { - BrAPIListDetails dataSetDetails = new BrAPIListDetails(); - dataSetDetails.setListName(name); - dataSetDetails.setListType(BrAPIListTypes.OBSERVATIONVARIABLES); - dataSetDetails.setData(new ArrayList<>()); - dataSetDetails.putAdditionalInfoItem("datasetType", "observationDataset"); - List refs = new ArrayList<>(); - Utilities.addReference(refs, UUID.fromString(programDbId), referenceSourceBase, ExternalReferenceSource.PROGRAMS); - Utilities.addReference(refs, UUID.fromString(trialId), referenceSourceBase, ExternalReferenceSource.TRIALS); - Utilities.addReference(refs, datasetId, referenceSourceBase, ExternalReferenceSource.DATASET); - dataSetDetails.setExternalReferences(refs); - return dataSetDetails; - } - public BrAPIObservationUnit createSubObservationUnit( String subEntityDatasetName, String subUnitId, diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index ace52b094..89596eabd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -228,23 +228,6 @@ public BrAPIStudy constructBrAPIStudy( return study; } - public BrAPIListDetails constructDatasetDetails( - String name, - UUID datasetId, - String referenceSourceBase, - Program program, String trialId) { - BrAPIListDetails dataSetDetails = new BrAPIListDetails(); - dataSetDetails.setListName(name); - dataSetDetails.setListType(BrAPIListTypes.OBSERVATIONVARIABLES); - dataSetDetails.setData(new ArrayList<>()); - dataSetDetails.putAdditionalInfoItem("datasetType", "observationDataset"); - List refs = new ArrayList<>(); - Utilities.addReference(refs, program.getId(), referenceSourceBase, ExternalReferenceSource.PROGRAMS); - Utilities.addReference(refs, UUID.fromString(trialId), referenceSourceBase, ExternalReferenceSource.TRIALS); - Utilities.addReference(refs, datasetId, referenceSourceBase, ExternalReferenceSource.DATASET); - dataSetDetails.setExternalReferences(refs); - return dataSetDetails; - } public BrAPIObservationUnit constructBrAPIObservationUnit( Program program, String seqVal, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index d2a834bee..f2b6ba133 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -43,6 +43,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingImportObjectData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; @@ -76,6 +77,7 @@ public class PopulateNewPendingImportObjectsStep { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final DSLContext dsl; private final Gson gson; + private final DatasetService datasetService; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -83,11 +85,12 @@ public class PopulateNewPendingImportObjectsStep { @Inject public PopulateNewPendingImportObjectsStep(ExperimentSeasonService experimentSeasonService, BrAPIObservationUnitDAO brAPIObservationUnitDAO, - DSLContext dsl) { + DSLContext dsl, DatasetService datasetService) { this.experimentSeasonService = experimentSeasonService; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.dsl = dsl; this.gson = new JSON().getGson(); + this.datasetService = datasetService; } /** @@ -370,7 +373,7 @@ public void fetchOrCreateDatasetPIO(ImportContext importContext, pio = obsVarDatasetByName.get(name); } else { UUID id = UUID.randomUUID(); - BrAPIListDetails newDataset = importRow.constructDatasetDetails( + BrAPIListDetails newDataset = datasetService.constructDatasetDetails( name, id, BRAPI_REFERENCE_SOURCE, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index d772466ca..fbc5186db 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -18,16 +18,19 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.context.annotation.Property; -import io.micronaut.http.server.exceptions.InternalServerException; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIListTypes; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.DatasetMetadata; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -114,4 +117,49 @@ public PendingImportObject constructPIOFromDataset(BrAPIListDe // Create a PendingImportObject for the dataset with the existing list and reference ID return new PendingImportObject(ImportObjectState.EXISTING, dataset, UUID.fromString(xref.getReferenceId())); } + + public void createBrAPIObsVarListForDataset(Program program, + BrAPITrial trial, + DatasetMetadata subEntityDatasetMetadata) throws ApiException { + + String name = String.format("Observation Dataset [%s-%s-%s]", + program.getKey(), + trial.getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString(), + subEntityDatasetMetadata.getName()); + + BrAPIListDetails subEntityObsVarsList = constructDatasetDetails(name, + subEntityDatasetMetadata.getId(), + BRAPI_REFERENCE_SOURCE, + program, + trial.getTrialDbId()); + + BrAPIListNewRequest listRq = new BrAPIListNewRequest(); + listRq.setListName(subEntityObsVarsList.getListName()); + listRq.setListType(subEntityObsVarsList.getListType()); + listRq.setExternalReferences(subEntityObsVarsList.getExternalReferences()); + listRq.setAdditionalInfo(subEntityObsVarsList.getAdditionalInfo()); + listRq.data(subEntityObsVarsList.getData()); + + brAPIListDAO.createBrAPILists(List.of(listRq), program.getId(), null); + } + + public BrAPIListDetails constructDatasetDetails( + String name, + UUID datasetId, + String referenceSourceBase, + Program program, String trialId) { + BrAPIListDetails dataSetDetails = new BrAPIListDetails(); + dataSetDetails.setListName(name); + dataSetDetails.setListType(BrAPIListTypes.OBSERVATIONVARIABLES); + dataSetDetails.setData(new ArrayList<>()); + dataSetDetails.putAdditionalInfoItem("datasetType", "observationDataset"); + List refs = new ArrayList<>(); + Utilities.addReference(refs, program.getId(), referenceSourceBase, ExternalReferenceSource.PROGRAMS); + Utilities.addReference(refs, UUID.fromString(trialId), referenceSourceBase, ExternalReferenceSource.TRIALS); + Utilities.addReference(refs, datasetId, referenceSourceBase, ExternalReferenceSource.DATASET); + dataSetDetails.setExternalReferences(refs); + return dataSetDetails; + } } From 4cee26f76f6e7f088dbaf58baa267efc166305c5 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 16 Mar 2026 19:29:24 +0000 Subject: [PATCH 247/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index f57fe5eb1..d29a4b3c1 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1103 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c2a827ca981dcc9327de0b1cfd06fd574118442d \ No newline at end of file +version=v1.3.0+1105 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/de810ccdf37f71ad5891de65bde50aec034dcc16 \ No newline at end of file From fb51f14cdc4bc27e273f80b1fa83db465d378196 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Tue, 17 Mar 2026 10:13:39 -0400 Subject: [PATCH 248/289] [BI-2778] Restored adding EXP_UNIT_ID --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index d67aad8d6..2ff1fbbfa 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -645,6 +645,8 @@ public BrAPIObservationUnit createSubObservationUnit( levelRelationships.add(expUnitLevel); position.setObservationLevelRelationships(levelRelationships); + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.EXP_UNIT_ID, expUnit.getObservationUnitName()); + // Set ObservationUnitPosition. observationUnit.setObservationUnitPosition(position); From e954f3bf9fcd344b9ddec6c69210207462536bd7 Mon Sep 17 00:00:00 2001 From: Jason Loux Date: Wed, 18 Mar 2026 09:48:41 -0400 Subject: [PATCH 249/289] Add new obsVarCount stat to create and append workflows --- .../middleware/process/AppendStatistic.java | 8 +++++++- .../middleware/process/ImportTableProcess.java | 8 +++++--- .../workflow/CreateNewExperimentWorkflow.java | 15 ++++++++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java index 0893df9f2..58381bcbe 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java @@ -18,6 +18,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; import io.micronaut.context.annotation.Prototype; +import lombok.Setter; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import java.util.HashSet; @@ -32,6 +33,8 @@ public class AppendStatistic { private int newCount; private int existingCount; private int mutatedCount; + @Setter + private int obsVarCount; public AppendStatistic() { this.clearData(); @@ -44,6 +47,7 @@ public void clearData() { this.newCount = 0; this.existingCount = 0; this.mutatedCount = 0; + this.obsVarCount = 0; } public int incrementNewCount(Integer value) { int increment = 0; @@ -94,6 +98,7 @@ public Map constructPreviewMap() { ImportPreviewStatistics newStats = ImportPreviewStatistics.builder().newObjectCount(newCount).build(); ImportPreviewStatistics existingStats = ImportPreviewStatistics.builder().newObjectCount(existingCount).build(); ImportPreviewStatistics mutatedStats = ImportPreviewStatistics.builder().newObjectCount(mutatedCount).build(); + ImportPreviewStatistics obsVarStats = ImportPreviewStatistics.builder().newObjectCount(obsVarCount).build(); return Map.of( "Environments", environmentStats, @@ -101,7 +106,8 @@ public Map constructPreviewMap() { "GIDs", gidStats, "Observations", newStats, "Existing_Observations", existingStats, - "Mutated_Observations", mutatedStats + "Mutated_Observations", mutatedStats, + "Observation_Variables", obsVarStats ); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index b9815ccfd..9aa92fca7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -218,7 +218,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext */ if (phenotypeCols.isEmpty()) { processedData = processedDataFactory.undefinedDatasetBean(); - updatePreviewStatistics(processedData, context, studyName, unitId); + updatePreviewStatistics(processedData, context, studyName, unitId, phenotypeCols.size()); } // Assemble the pending observation data for all phenotypes @@ -332,7 +332,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e -> validationErrors.addError(rowNum + 2, e))); // +2 to account for header row and excel file 1-based row index // Update import preview statistics and set in the context - updatePreviewStatistics(processedData, context, studyName, unitId); + updatePreviewStatistics(processedData, context, studyName, unitId, varNames.size()); // Construct a pending observation Optional> pendingProcessedData = Optional.ofNullable(processedData.constructPendingObservation()); @@ -395,12 +395,14 @@ private boolean isChanged(String cellData, BrAPIObservation observation, String private void updatePreviewStatistics(VisitedObservationData processedData, AppendOverwriteMiddlewareContext context, String studyName, - String unitId) { + String unitId, + int obsVarCount) { // Update import preview statistics and set in the context processedData.updateTally(statistic); statistic.addEnvironmentName(studyName); statistic.addObservationUnitId(unitId); statistic.addGid(context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getAccessionNumber()); + statistic.setObsVarCount(obsVarCount); context.getAppendOverwriteWorkflowContext().setStatistic(statistic); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index a6b6ea8b9..36bfb772d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -258,6 +258,15 @@ private Map generateStatisticsMap(PendingData p .count() ); + // Assume observationVariableName is set when pending observations are created. + int obsVarCount = Math.toIntExact( + observationByHash.values() + .stream() + .map(o -> o.getBrAPIObject().getObservationVariableName()) + .distinct() + .count() + ); + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() .newObjectCount(environmentNameCounter.size()) .build(); @@ -276,6 +285,9 @@ private Map generateStatisticsMap(PendingData p ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() .newObjectCount(numMutatedObservations) .build(); + ImportPreviewStatistics obsVarStats = ImportPreviewStatistics.builder() + .newObjectCount(obsVarCount) + .build(); return Map.of( "Environments", environmentStats, @@ -283,7 +295,8 @@ private Map generateStatisticsMap(PendingData p "GIDs", gidStats, "Observations", observationStats, "Existing_Observations", existingObservationStats, - "Mutated_Observations", mutatedObservationStats + "Mutated_Observations", mutatedObservationStats, + "Observation_Variables", obsVarStats ); } From dd0f51b18343f0cb3f8bf874f911707ce5ba6158 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 18 Mar 2026 16:52:44 +0000 Subject: [PATCH 250/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index d29a4b3c1..131b05630 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1105 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/de810ccdf37f71ad5891de65bde50aec034dcc16 \ No newline at end of file +version=v1.3.0+1107 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/7cdbd83148768ce9f5da4e16e61ecafbd294819e \ No newline at end of file From cb8d8b7ae88895727935aa52dd192dc671f28069 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 23 Mar 2026 15:16:54 +0000 Subject: [PATCH 251/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 131b05630..bb4fd1ed9 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1107 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/7cdbd83148768ce9f5da4e16e61ecafbd294819e \ No newline at end of file +version=v1.3.0+1109 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f163ebfe304c326b15808e337014bca158dcb18c \ No newline at end of file From c4e83aa443ccbd6a6c02dfb6882269cf1a29b93f Mon Sep 17 00:00:00 2001 From: HMS17 Date: Tue, 24 Mar 2026 10:22:18 -0400 Subject: [PATCH 252/289] [BI-2816] Unknown error when creating new experiment with file with more than one experiment title and at least one title is that of an existing experiment --- .../create/workflow/CreateNewExperimentWorkflow.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 36bfb772d..a22cb98bf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -43,6 +43,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateNewPendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.ValidatePendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentPhenotypeService; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; import javax.inject.Inject; @@ -57,6 +58,7 @@ import javax.inject.Singleton; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; @Slf4j @@ -99,6 +101,12 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "ObsUnitIDs are detected"); } + //Make sure file only contains one exp title, check early cause avoids issues with titles corresponding to existing and new trials + List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(importRows); + if (experimentImportRows.stream().map(ExperimentObservation::getExpTitle).distinct().count() > 1) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue()); + } + statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); From 6501476ce9c60b25f8164cee0b8705de613ac8f6 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Tue, 24 Mar 2026 14:22:58 -0400 Subject: [PATCH 253/289] Initial codex pass --- .../v1/controller/ExperimentController.java | 23 ++++ .../v2/dao/BrAPIObservationLevelDAO.java | 67 ++++++++++- .../brapi/v2/services/BrAPITrialService.java | 40 ++++++- .../ExperimentControllerIntegrationTest.java | 104 +++++++++++++++++- 4 files changed, 226 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index 5741e1313..5ee72f3a2 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -175,6 +175,29 @@ public HttpResponse>> getDatasets( } + @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/recommended-sub-entity-dataset-names") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) + @Produces(MediaType.APPLICATION_JSON) + public HttpResponse>> getRecommendedSubEntityDatasetNames( + @PathVariable("programId") UUID programId, + @PathVariable("experimentId") UUID experimentId) { + try { + Optional programOptional = programService.getById(programId); + if (programOptional.isEmpty()) { + return HttpResponse.status(HttpStatus.NOT_FOUND, "Program does not exist"); + } + + Response> response = new Response<>(experimentService.getRecommendedSubEntityDatasetNames(programOptional.get(), experimentId)); + return HttpResponse.ok(response); + } catch (DoesNotExistException e) { + log.info(e.getMessage()); + return HttpResponse.status(HttpStatus.NOT_FOUND, e.getMessage()); + } catch (Exception e) { + log.error("Error finding recommended sub-entity dataset names", e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error finding recommended sub-entity dataset names"); + } + } + /** * Adds a record to the experiment_program_user_role table * @param programId The UUID of the program diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java index e37e23dbe..95854caeb 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java @@ -18,6 +18,10 @@ package org.breedinginsight.brapi.v2.dao; import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; @@ -25,18 +29,17 @@ import okhttp3.MediaType; import okhttp3.Request; import okhttp3.RequestBody; +import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.model.DatasetLevel; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.BrAPIDAOUtil; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import javax.inject.Inject; import javax.inject.Singleton; +import java.util.ArrayList; +import java.util.List; @Slf4j @Singleton @@ -96,4 +99,60 @@ public void deleteObservationLevelName(Program program, String levelDbId) { } } + public List getObservationLevelNames(Program program, String programDbId) throws ApiException { + List levelNames = new ArrayList<>(); + int currentPage = 0; + int totalPages = 1; + + do { + HttpUrl.Builder urlBuilder = HttpUrl.parse(brAPIDAOUtil.getProgramBrAPIBaseUrl(program.getId())) + .newBuilder() + .addPathSegment("observationlevelnames") + .addQueryParameter("page", Integer.toString(currentPage)) + .addQueryParameter("pageSize", "1000"); + if (StringUtils.isNotBlank(programDbId)) { + urlBuilder.addQueryParameter("programDbId", programDbId); + } + + Request request = new Request.Builder() + .url(urlBuilder.build()) + .get() + .addHeader("Content-Type", "application/json") + .build(); + + HttpResponse response = brAPIDAOUtil.makeCall(request); + if (response.getStatus() != HttpStatus.OK) { + throw new ApiException(response.getStatus().getCode(), "Unable to fetch observation level names"); + } + + String responseBody = response.body(); + if (StringUtils.isBlank(responseBody)) { + return levelNames; + } + + JsonObject responseJson = JsonParser.parseString(responseBody).getAsJsonObject(); + JsonObject resultJson = responseJson.getAsJsonObject("result"); + if (resultJson != null) { + JsonArray data = resultJson.getAsJsonArray("data"); + if (data != null) { + for (JsonElement level : data) { + if (level.isJsonObject()) { + JsonElement levelName = level.getAsJsonObject().get("levelName"); + if (levelName != null && !levelName.isJsonNull()) { + levelNames.add(levelName.getAsString()); + } + } + } + } + } + + JsonObject metadata = responseJson.getAsJsonObject("metadata"); + JsonObject pagination = metadata != null ? metadata.getAsJsonObject("pagination") : null; + totalPages = pagination != null && pagination.has("totalPages") ? pagination.get("totalPages").getAsInt() : currentPage + 1; + currentPage++; + } while (currentPage < totalPages); + + return levelNames; + } + } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 92a7ac2db..6c567f990 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -36,6 +36,8 @@ import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; import org.breedinginsight.model.*; +import org.breedinginsight.model.delta.DeltaEntityFactory; +import org.breedinginsight.model.delta.Experiment; import org.breedinginsight.services.TraitService; import org.breedinginsight.services.exceptions.AlreadyExistsException; import org.breedinginsight.services.exceptions.DoesNotExistException; @@ -81,6 +83,7 @@ public class BrAPITrialService { private final DistributedLockService lockService; private static final String SHEET_NAME = "Data"; private final DatasetService datasetService; + private final DeltaEntityFactory deltaEntityFactory; @Inject public BrAPITrialService(@Property(name = "brapi.server.reference-source") String referenceSource, @@ -96,7 +99,8 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin BrAPIGermplasmDAO germplasmDAO, FileMappingUtil fileMappingUtil, DistributedLockService lockService, - DatasetService datasetService) { + DatasetService datasetService, + DeltaEntityFactory deltaEntityFactory) { this.referenceSource = referenceSource; this.trialDAO = trialDAO; @@ -112,6 +116,7 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin this.fileMappingUtil = fileMappingUtil; this.lockService = lockService; this.datasetService = datasetService; + this.deltaEntityFactory = deltaEntityFactory; } public List getExperiments(UUID programId) throws ApiException, DoesNotExistException { @@ -429,6 +434,34 @@ public List getDatasetsMetadata(Program program, UUID experimen return datasets; } + /** + * Assumptions: + * @param program + * @param experimentId + * @return + * @throws DoesNotExistException + * @throws ApiException + */ + public List getRecommendedSubEntityDatasetNames(Program program, UUID experimentId) throws DoesNotExistException, ApiException { + BrAPITrial experiment = trialDAO.getTrialById(program.getId(), experimentId).orElseThrow(() -> new DoesNotExistException("Trial does not exist")); + Experiment deltaExperiment = deltaEntityFactory.makeExperimentBean(experiment); + // set to eliminate possible duplicates like plant for exp unit and sub unit + Set currentExperimentDatasetNames = deltaExperiment.getDatasetsMetadata() + .stream() + .map(DatasetMetadata::getName) + //.filter(StringUtils::isNotBlank) + .collect(Collectors.toSet()); + + return getProgramObservationLevelNames(program).stream() + //.filter(StringUtils::isNotBlank) + //.filter(name -> !BrAPIConstants.REPLICATE.getValue().equalsIgnoreCase(name)) + //.filter(name -> !BrAPIConstants.BLOCK.getValue().equalsIgnoreCase(name)) + .filter(name -> !currentExperimentDatasetNames.contains(name)) + .distinct() + .sorted() + .collect(Collectors.toList()); + } + /** * Creates sub-entity dataset * TODO: Handle compensating transactions in event of failure. Currently brapi server does not support @@ -801,6 +834,11 @@ public int deleteExperiment(Program program, UUID experimentId, boolean hard) th return existingObservations.size(); } + private List getProgramObservationLevelNames(Program program) throws ApiException { + String programDbId = program.getBrapiProgram() != null ? program.getBrapiProgram().getProgramDbId() : null; + return observationLevelDAO.getObservationLevelNames(program, programDbId); + } + private Map createExportRow( BrAPITrial experiment, Program program, diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 21358e69d..be4895c26 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -213,12 +213,17 @@ void setup() throws Exception { // Create an experiment with no observations. private String uploadExperimentWithoutObs() throws Exception { + return uploadExperimentWithoutObs("Without Obs", "Plot"); + } + + private String uploadExperimentWithoutObs(String title, String expUnit) throws Exception { ImportTestUtils importTestUtils = new ImportTestUtils(); List> expRows = new ArrayList<>(); // Make test experiment import. - Map row1 = makeExpImportRow("Without Obs", "NewEnv1"); - Map row2 = makeExpImportRow("Without Obs", "NewEnv2"); + String envBase = title.replaceAll("\\s+", ""); + Map row1 = makeExpImportRow(title, envBase + "1", expUnit); + Map row2 = makeExpImportRow(title, envBase + "2", expUnit); expRows.add(row1); expRows.add(row2); @@ -396,6 +401,75 @@ void downloadSubEntityDataset(String extension) { parseAndCheck(plantBodyStream, extension, false, plantRows, false, 23); } + @Test + @Order(1) + public void createSubEntityDatasetRejectsExpUnitNameAlreadyUsedInSameExperiment() throws Exception { + String plantExperimentId = uploadExperimentWithoutObs("Plant Same Experiment", "Plant"); + + Flowable> call = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", program.getId(), plantExperimentId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + + HttpClientResponseException e = assertThrows(HttpClientResponseException.class, call::blockingFirst); + assertEquals(HttpStatus.CONFLICT, e.getStatus()); + } + + @Test + @Order(2) + public void createSubEntityDatasetAllowsExpUnitNameUsedInOtherExperiment() throws Exception { + uploadExperimentWithoutObs("Plant Source Experiment", "Plant"); + String recipientExperimentId = uploadExperimentWithoutObs("Plot Recipient Experiment", "Plot"); + + Flowable> call = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", program.getId(), recipientExperimentId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + } + + @Test + @Order(3) + public void recommendedSubEntityDatasetNamesIncludeExpUnitNamesFromOtherExperiments() throws Exception { + uploadExperimentWithoutObs("Plant Autocomplete Source", "Plant"); + String recipientExperimentId = uploadExperimentWithoutObs("Autocomplete Recipient", "Plot"); + + List recommendedNames = getRecommendedSubEntityDatasetNames(recipientExperimentId); + + assertTrue(recommendedNames.stream().anyMatch(name -> name.equalsIgnoreCase("plant"))); + assertFalse(recommendedNames.stream().anyMatch(name -> name.equalsIgnoreCase("plot"))); + } + + @Test + @Order(4) + public void recommendedSubEntityDatasetNamesDeDuplicateExpUnitAndSubUnitNamesAcrossExperiments() throws Exception { + uploadExperimentWithoutObs("Plant Exp Unit Source", "Plant"); + String subEntitySourceExperimentId = uploadExperimentWithoutObs("Plant Sub Unit Source", "Plot"); + String recipientExperimentId = uploadExperimentWithoutObs("Plant Unique Recipient", "Plot"); + + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", program.getId(), subEntitySourceExperimentId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + HttpResponse postResponse = postCall.blockingFirst(); + assertEquals(HttpStatus.OK, postResponse.getStatus()); + + List recommendedNames = getRecommendedSubEntityDatasetNames(recipientExperimentId); + + assertEquals(1L, recommendedNames.stream().filter(name -> name.equalsIgnoreCase("plant")).count()); + } + /** * Tests for Experimental Collaborator endpoints */ @@ -845,12 +919,36 @@ private File writeDataToFile(List> data, List traits) return file; } + private List getRecommendedSubEntityDatasetNames(String targetExperimentId) { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/experiments/%s/recommended-sub-entity-dataset-names", + program.getId(), targetExperimentId)) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonArray result = JsonParser.parseString(Objects.requireNonNull(response.body())) + .getAsJsonObject() + .getAsJsonArray("result"); + + List recommendedNames = new ArrayList<>(); + result.forEach(name -> recommendedNames.add(name.getAsString())); + return recommendedNames; + } + private Map makeExpImportRow(String title, String environment) { + return makeExpImportRow(title, environment, "Plot"); + } + + private Map makeExpImportRow(String title, String environment, String expUnit) { Map row = new HashMap<>(); row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); row.put(ExperimentObservation.Columns.EXP_TITLE, title); - row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_UNIT, expUnit); //row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, ""); row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); row.put(ExperimentObservation.Columns.ENV, environment); From 34bf6a096376ba6cf371215e69530dcdc7729a44 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Mon, 30 Mar 2026 10:22:00 -0400 Subject: [PATCH 254/289] Switch to makeCallWithResponse --- .../brapi/v2/dao/BrAPIObservationLevelDAO.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java index 95854caeb..979480f40 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java @@ -120,12 +120,7 @@ public List getObservationLevelNames(Program program, String programDbId .addHeader("Content-Type", "application/json") .build(); - HttpResponse response = brAPIDAOUtil.makeCall(request); - if (response.getStatus() != HttpStatus.OK) { - throw new ApiException(response.getStatus().getCode(), "Unable to fetch observation level names"); - } - - String responseBody = response.body(); + String responseBody = brAPIDAOUtil.makeCallWithResponse(request); if (StringUtils.isBlank(responseBody)) { return levelNames; } From f6ad85390c89c03b2df9dfe271a53912c966cfe7 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 31 Mar 2026 13:07:49 +0000 Subject: [PATCH 255/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index bb4fd1ed9..16662af79 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1109 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f163ebfe304c326b15808e337014bca158dcb18c \ No newline at end of file +version=v1.3.0+1111 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/4b84d09b1474bea1b37469b3f8751b7aab614fe8 \ No newline at end of file From c735d10e67c9df3b6ac4fec0965239b6761a3aa1 Mon Sep 17 00:00:00 2001 From: Jason Loux Date: Tue, 31 Mar 2026 18:02:52 -0400 Subject: [PATCH 256/289] Support for level order uniqueness use case for sub entity dataset creation - Consolidated relevant code into DatasetService for use in creation experiment use case - Updated to BrAPI Java Client 2.2 - Added methods to get data from the updated client in BrAPIObservationLevelDAO --- pom.xml | 2 +- .../v2/dao/BrAPIObservationLevelDAO.java | 123 +++++++++++++++--- .../BrAPIObservationLevelService.java | 60 +++++++++ .../brapi/v2/services/BrAPITrialService.java | 61 +++++---- .../experiment/service/DatasetService.java | 56 +++++++- 5 files changed, 250 insertions(+), 52 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationLevelService.java diff --git a/pom.xml b/pom.xml index 95d75a0d1..674e7ffea 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ 31.0.1-jre 4.9.3 4.3.1 - 2.1-SNAPSHOT + 2.2-SNAPSHOT 2.11.0 2.2.1 diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java index e37e23dbe..258ffbaf6 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java @@ -20,23 +20,38 @@ import com.google.gson.Gson; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; +import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import okhttp3.HttpUrl; import okhttp3.MediaType; import okhttp3.Request; import okhttp3.RequestBody; +import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.phenotype.ObservationLevelNamesApi; +import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; +import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; +import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; +import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponse; +import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponseResult; +import org.brapi.v2.model.pheno.response.BrAPIObservationLevelSingleResponse; +import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.DatasetLevel; import org.breedinginsight.model.Program; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.BrAPIDAOUtil; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; @Slf4j @Singleton @@ -46,12 +61,19 @@ public class BrAPIObservationLevelDAO { private final BrAPIDAOUtil brAPIDAOUtil; private final Gson gson = new JSON().getGson(); + private final BrAPIEndpointProvider brAPIEndpointProvider; + private final ProgramDAO programDAO; + @Inject - public BrAPIObservationLevelDAO(BrAPIDAOUtil brAPIDAOUtil) { + public BrAPIObservationLevelDAO(BrAPIDAOUtil brAPIDAOUtil, + BrAPIEndpointProvider brAPIEndpointProvider, + ProgramDAO programDAO) { this.brAPIDAOUtil = brAPIDAOUtil; + this.brAPIEndpointProvider = brAPIEndpointProvider; + this.programDAO = programDAO; } - public HttpResponse createObservationLevelName(Program program, String levelName, DatasetLevel levelOrder, String programDbId) throws ApiException { + public HttpResponse createObservationLevelName(Program program, String levelName, DatasetLevel levelOrder, String programDbId) { HttpUrl url = HttpUrl.parse(brAPIDAOUtil.getProgramBrAPIBaseUrl(program.getId())) .newBuilder() .addPathSegment("observationlevelnames") @@ -75,25 +97,90 @@ public HttpResponse createObservationLevelName(Program program, String l return brAPIDAOUtil.makeCall(request); } + public BrAPIObservationUnitHierarchyLevel createLevelName(Program program, + String programDbId, + String levelName, + DatasetLevel levelOrder) throws ApiException { + ObservationLevelNamesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationLevelNamesApi.class); + + ApiResponse response; + + + BrAPIObservationUnitHierarchyLevel level = new BrAPIObservationUnitHierarchyLevel(); + + level.setLevelName(levelName.toLowerCase()); + level.setLevelOrder(levelOrder.getValue()); + level.setProgramDbId(programDbId); + + try { + response = api.observationLevelNamesPost(List.of(level)); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); + } + + return Optional.of(response) + .map(ApiResponse::getBody) + .map(BrAPIObservationLevelListResponse::getResult) + .map(BrAPIObservationLevelListResponseResult::getData) + .flatMap(data -> data.stream().findFirst()) + .orElseThrow(() -> new ApiException(String.format("BrAPI indicated level name [%s] was created but no levelNameDbId was returned upon its creation", levelName))); + } + + public List getObservationLevelNamesByProgramId(Program program, String programDbId) { + ObservationLevelNamesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationLevelNamesApi.class); + + ApiResponse response; + + int pageSize = 100; + + try { + response = api.observationLevelNamesGet(programDbId, + false, + 0, + pageSize); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); + } + + if (response.getBody().getMetadata().getPagination().getTotalCount() > 100) { + throw new InternalServerException(String.format("More level names exist than requested [%s]", pageSize)); + } + + return response.getBody().getResult().getData(); + } + + public List getGlobalObservationLevelNames(Program program) { + return getObservationLevelNamesByProgramId(program, null); + } + public void deleteObservationLevelName(Program program, String levelDbId) { - HttpUrl url = HttpUrl.parse(brAPIDAOUtil.getProgramBrAPIBaseUrl(program.getId())) - .newBuilder() - .addPathSegment("observationlevelnames") - .addPathSegment(levelDbId) - .build(); - var request = new Request.Builder() - .url(url) - .delete() - .addHeader("Content-Type", "application/json") - .build(); + ObservationLevelNamesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationLevelNamesApi.class); + try { - HttpResponse response = brAPIDAOUtil.makeCall(request); - if (response.getStatus() != HttpStatus.OK && response.getStatus() != HttpStatus.NO_CONTENT && response.getStatus() != HttpStatus.ACCEPTED) { - log.warn("Observation level delete returned status {} for {}", response.getStatus(), levelDbId); - } - } catch (Exception e) { - log.warn("Failed to delete observation level {}", levelDbId, e); + api.observationLevelNameDbIdDelete(levelDbId); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); } } + public BrAPIObservationUnitHierarchyLevel updateObservationLevelName(Program program, + String levelNameDbId, + BrAPIObservationUnitHierarchyLevel level) { + ObservationLevelNamesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationLevelNamesApi.class); + + ApiResponse response; + + try { + response = api.observationLevelNameDbIdPut(levelNameDbId, level); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); + } + + return response.getBody().getResult(); + } + } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationLevelService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationLevelService.java new file mode 100644 index 000000000..8d48979cb --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationLevelService.java @@ -0,0 +1,60 @@ +package org.breedinginsight.brapi.v2.services; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; +import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; +import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; +import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponse; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationLevelDAO; +import org.breedinginsight.model.DatasetLevel; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.exceptions.DoesNotExistException; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Slf4j +@Singleton +public class BrAPIObservationLevelService { + private final BrAPIObservationLevelDAO brAPIObservationLevelDAO; + private final ProgramService programService; + + + @Inject + public BrAPIObservationLevelService(BrAPIObservationLevelDAO brAPIObservationLevelDAO, + ProgramService programService) { + this.brAPIObservationLevelDAO = brAPIObservationLevelDAO; + this.programService = programService; + } + + /** + * @return Pair[GlobalLevelNames, ProgrammaticLevelNames] + */ + public Pair, List> getGlobalAndProgrammaticLevelNames(Program program, String brapiProgramDbId) { + List globalLevels = brAPIObservationLevelDAO.getGlobalObservationLevelNames(program); + List programmaticLevels = brAPIObservationLevelDAO.getObservationLevelNamesByProgramId(program, brapiProgramDbId); + + + return new ImmutablePair<>(globalLevels, programmaticLevels); + } + + public List getProgrammaticLevelNames(Program program, String brapiProgramDbId) { + return brAPIObservationLevelDAO.getObservationLevelNamesByProgramId(program, brapiProgramDbId); + } + + public BrAPIObservationUnitHierarchyLevel createObservationLevel(Program program, + String brapiProgramDbId, + String levelName, + DatasetLevel levelOrder) throws ApiException { + return brAPIObservationLevelDAO.createLevelName(program, brapiProgramDbId, levelName, levelOrder); + } + +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 92a7ac2db..d59b0928c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -6,8 +6,6 @@ import com.github.filosganga.geogson.model.positions.SinglePosition; import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; -import io.micronaut.http.HttpResponse; -import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.http.server.types.files.StreamedFile; @@ -81,6 +79,7 @@ public class BrAPITrialService { private final DistributedLockService lockService; private static final String SHEET_NAME = "Data"; private final DatasetService datasetService; + private final BrAPIObservationLevelService observationLevelService; @Inject public BrAPITrialService(@Property(name = "brapi.server.reference-source") String referenceSource, @@ -96,7 +95,8 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin BrAPIGermplasmDAO germplasmDAO, FileMappingUtil fileMappingUtil, DistributedLockService lockService, - DatasetService datasetService) { + DatasetService datasetService, + BrAPIObservationLevelService observationLevelService) { this.referenceSource = referenceSource; this.trialDAO = trialDAO; @@ -112,6 +112,7 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin this.fileMappingUtil = fileMappingUtil; this.lockService = lockService; this.datasetService = datasetService; + this.observationLevelService = observationLevelService; } public List getExperiments(UUID programId) throws ApiException, DoesNotExistException { @@ -466,14 +467,13 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt } String programBrapiDbId = program.getBrapiProgram() != null ? program.getBrapiProgram().getProgramDbId() : null; - HttpResponse levelResponse = observationLevelDAO.createObservationLevelName(program, datasetName, DatasetLevel.SUB_OBS_UNIT, programBrapiDbId); - // 409 and 200 are expected response codes, anything else error out - // 409 means level already exists so we just use the name in OUs - // 200 means level was created successfully and can use the name in OUs - if (levelResponse.getStatus() != HttpStatus.CONFLICT && levelResponse.getStatus() != HttpStatus.OK) { - throw new ApiException(levelResponse.getStatus().getCode(), "Unable to create observation level: " + levelResponse.getStatus().getReason()); - } + String subEntityLevelNameDbId = datasetService.getOrCreateLevelNameForDataset(program, + programBrapiDbId, + datasetName, + DatasetLevel.SUB_OBS_UNIT + ); + List expOUs = ouDAO.getObservationUnitsForDataset(topLevelDataset.getId().toString(), program); for (BrAPIObservationUnit expUnit : expOUs) { @@ -484,14 +484,14 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt for (int i=1; i<=request.getRepeatedMeasures(); i++) { subObsUnits.add( createSubObservationUnit( - datasetName, Integer.toString(i), program, envSeqValue, expUnit, this.referenceSource, subEntityDatasetId, - UUID.randomUUID() + UUID.randomUUID(), + subEntityLevelNameDbId ) ); } @@ -529,14 +529,14 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt } public BrAPIObservationUnit createSubObservationUnit( - String subEntityDatasetName, String subUnitId, Program program, String seqVal, BrAPIObservationUnit expUnit, String referenceSource, UUID datasetId, - UUID id + UUID id, + String subEntityLevelNameDbId ) { BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); @@ -612,42 +612,39 @@ public BrAPIObservationUnit createSubObservationUnit( // ObservationLevel entry for Sub-Obs Unit. BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); - level.setLevelName(subEntityDatasetName); + level.setLevelNameDbId(subEntityLevelNameDbId); level.setLevelCode(Utilities.appendProgramKey(subUnitId, program.getKey(), seqVal)); - level.setLevelOrder(DatasetLevel.SUB_OBS_UNIT.getValue()); position.setObservationLevel(level); // ObservationLevelRelationships. List levelRelationships = new ArrayList<>(); + // TODO: Figure out if we actually need to add the sub entity level to the level relationships BI-2823 levelRelationships.add(level); - // ObservationLevelRelationships for block. - BrAPIObservationUnitLevelRelationship expBlockLevel = expUnit.getObservationUnitPosition() - .getObservationLevelRelationships().stream() - .filter(x -> x.getLevelName().equals(BrAPIConstants.REPLICATE.getValue())).findFirst().orElse(null); - if (expBlockLevel != null) { - BrAPIObservationUnitLevelRelationship blockLevel = new BrAPIObservationUnitLevelRelationship(); - blockLevel.setLevelName(expBlockLevel.getLevelName()); - blockLevel.setLevelCode(expBlockLevel.getLevelCode()); - blockLevel.setLevelOrder(expBlockLevel.getLevelOrder()); - levelRelationships.add(blockLevel); - } // ObservationLevelRelationships for rep. BrAPIObservationUnitLevelRelationship expRepLevel = expUnit.getObservationUnitPosition() .getObservationLevelRelationships().stream() - .filter(x -> x.getLevelName().equals(BrAPIConstants.BLOCK.getValue())).findFirst().orElse(null); + .filter(x -> x.getLevelName().equals(BrAPIConstants.REPLICATE.getValue())).findFirst().orElse(null); if (expRepLevel != null) { BrAPIObservationUnitLevelRelationship repLevel = new BrAPIObservationUnitLevelRelationship(); - repLevel.setLevelName(expRepLevel.getLevelName()); + repLevel.setLevelNameDbId(expRepLevel.getLevelNameDbId()); repLevel.setLevelCode(expRepLevel.getLevelCode()); - repLevel.setLevelOrder(expRepLevel.getLevelOrder()); levelRelationships.add(repLevel); } + // ObservationLevelRelationships for block. + BrAPIObservationUnitLevelRelationship expBlockLevel = expUnit.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(x -> x.getLevelName().equals(BrAPIConstants.BLOCK.getValue())).findFirst().orElse(null); + if (expBlockLevel != null) { + BrAPIObservationUnitLevelRelationship blockLevel = new BrAPIObservationUnitLevelRelationship(); + blockLevel.setLevelNameDbId(expBlockLevel.getLevelNameDbId()); + blockLevel.setLevelCode(expBlockLevel.getLevelCode()); + levelRelationships.add(blockLevel); + } // ObservationLevelRelationships for top-level Exp Unit linking. BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); - expUnitLevel.setLevelName(expUnit.getObservationUnitPosition().getObservationLevel().getLevelName().toLowerCase()); + expUnitLevel.setLevelNameDbId(expUnit.getObservationUnitPosition().getObservationLevel().getLevelNameDbId()); String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId(); expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnitUUID, program.getKey(), seqVal)); - expUnitLevel.setLevelOrder(DatasetLevel.EXP_UNIT.getValue()); levelRelationships.add(expUnitLevel); position.setObservationLevelRelationships(levelRelationships); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index fbc5186db..18157fd11 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -18,6 +18,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.context.annotation.Property; +import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIListSummary; @@ -25,11 +26,15 @@ import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationLevelDAO; +import org.breedinginsight.brapi.v2.services.BrAPIObservationLevelService; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.DatasetLevel; import org.breedinginsight.model.DatasetMetadata; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -37,16 +42,20 @@ import javax.inject.Inject; import javax.inject.Singleton; import java.util.*; +import java.util.stream.Collectors; @Singleton public class DatasetService { private final BrAPIListDAO brAPIListDAO; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; + private final BrAPIObservationLevelService observationLevelService; @Inject - public DatasetService(BrAPIListDAO brapiListDAO) { + public DatasetService(BrAPIListDAO brapiListDAO, + BrAPIObservationLevelService brAPIObservationLevelService) { this.brAPIListDAO = brapiListDAO; + this.observationLevelService = brAPIObservationLevelService; } /** * Module: Dataset Utility @@ -162,4 +171,49 @@ public BrAPIListDetails constructDatasetDetails( dataSetDetails.setExternalReferences(refs); return dataSetDetails; } + + /** + * @return brapiLevelNameDbId of found or created record + */ + public String getOrCreateLevelNameForDataset(Program program, + String brapiProgramDbId, + String levelName, + DatasetLevel levelOrder) throws ApiException { + + String existingLevelNameDbId = findLevelNameByNameAndOrder(program, brapiProgramDbId, levelName, levelOrder); + + if (StringUtils.isNotBlank(existingLevelNameDbId)) { + return existingLevelNameDbId; + } + + // Level name does not exist and needs to be created. + BrAPIObservationUnitHierarchyLevel createdLevelName = observationLevelService.createObservationLevel(program, brapiProgramDbId, levelName, levelOrder); + + return createdLevelName.getLevelNameDbId(); + } + + /** + * This method retrieves the programmatic level names and then matches the level names on the submitted + * level name and order. + * + * @return levelNameDbId of the matched level name + */ + private String findLevelNameByNameAndOrder(Program program, + String brapiProgramDbId, + String levelName, + DatasetLevel levelOrder) { + var programmaticLevelNames = observationLevelService.getProgrammaticLevelNames(program, brapiProgramDbId); + + List levelNameStreamResult + = programmaticLevelNames.stream() + .filter(ouln -> ouln.getLevelName().equals(levelName.toLowerCase()) && ouln.getLevelOrder() == levelOrder.getValue()) + .limit(1) + .collect(Collectors.toList()); + + if (levelNameStreamResult.isEmpty()) { + return null; + } + + return levelNameStreamResult.get(0).getLevelNameDbId(); + } } From af8c36aedf57354f21ac127676a8e3d2594a3b2c Mon Sep 17 00:00:00 2001 From: Jason Loux Date: Thu, 2 Apr 2026 15:39:23 -0400 Subject: [PATCH 257/289] [BI-2813] Set levelNameDbIds in OUs in experiment creation use case --- .../BrAPIObservationLevelService.java | 27 +------- .../steps/CommitPendingImportObjectsStep.java | 69 ++++++++++++------- .../experiment/service/DatasetService.java | 43 +++++++++++- 3 files changed, 88 insertions(+), 51 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationLevelService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationLevelService.java index 8d48979cb..e1f3d9b98 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationLevelService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationLevelService.java @@ -1,49 +1,28 @@ package org.breedinginsight.brapi.v2.services; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; -import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; -import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; -import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponse; import org.breedinginsight.brapi.v2.dao.BrAPIObservationLevelDAO; import org.breedinginsight.model.DatasetLevel; import org.breedinginsight.model.Program; -import org.breedinginsight.services.ProgramService; -import org.breedinginsight.services.exceptions.DoesNotExistException; import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; import java.util.List; -import java.util.Map; @Slf4j @Singleton public class BrAPIObservationLevelService { private final BrAPIObservationLevelDAO brAPIObservationLevelDAO; - private final ProgramService programService; - @Inject - public BrAPIObservationLevelService(BrAPIObservationLevelDAO brAPIObservationLevelDAO, - ProgramService programService) { + public BrAPIObservationLevelService(BrAPIObservationLevelDAO brAPIObservationLevelDAO) { this.brAPIObservationLevelDAO = brAPIObservationLevelDAO; - this.programService = programService; } - /** - * @return Pair[GlobalLevelNames, ProgrammaticLevelNames] - */ - public Pair, List> getGlobalAndProgrammaticLevelNames(Program program, String brapiProgramDbId) { - List globalLevels = brAPIObservationLevelDAO.getGlobalObservationLevelNames(program); - List programmaticLevels = brAPIObservationLevelDAO.getObservationLevelNamesByProgramId(program, brapiProgramDbId); - - - return new ImmutablePair<>(globalLevels, programmaticLevels); + public List getGlobalLevelNames(Program program) { + return brAPIObservationLevelDAO.getGlobalObservationLevelNames(program); } public List getProgrammaticLevelNames(Program program, String brapiProgramDbId) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index ef9dc9ef5..bdcc8956f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPIProgram; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPIListNewRequest; @@ -43,6 +44,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; import org.breedinginsight.model.DatasetLevel; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; @@ -55,10 +57,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.PREEXISTING_EXPERIMENT_TITLE; @@ -74,7 +73,7 @@ public class CommitPendingImportObjectsStep { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final ProgramLocationService locationService; private final OntologyService ontologyService; - private final BrAPIObservationLevelDAO brAPIObservationLevelDAO; + private final DatasetService datasetService; @Inject public CommitPendingImportObjectsStep(BrAPIListDAO brAPIListDAO, @@ -84,7 +83,7 @@ public CommitPendingImportObjectsStep(BrAPIListDAO brAPIListDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO, ProgramLocationService locationService, OntologyService ontologyService, - BrAPIObservationLevelDAO brAPIObservationLevelDAO) { + DatasetService datasetService) { this.brAPIListDAO = brAPIListDAO; this.brapiTrialDAO = brapiTrialDAO; this.brAPIStudyDAO = brAPIStudyDAO; @@ -92,11 +91,13 @@ public CommitPendingImportObjectsStep(BrAPIListDAO brAPIListDAO, this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.locationService = locationService; this.ontologyService = ontologyService; - this.brAPIObservationLevelDAO = brAPIObservationLevelDAO; + this.datasetService = datasetService; } // TODO: some common code between workflows here that could be broken out, removed append/update specific code - public void process(ProcessContext processContext, ProcessedData processedData) throws UnprocessableEntityException { + // TODO: For instance, multiple trials don't really exist in the use case which this code is used for: creating an experiment. + // TODO: This means having a Map of trials doesn't really make sense anymore, and should be replaced with a singular PendingImportObject + public void process(ProcessContext processContext, ProcessedData processedData) throws UnprocessableEntityException, ApiException { PendingData pendingData = processContext.getPendingData(); ImportContext importContext = processContext.getImportContext(); @@ -113,6 +114,32 @@ public void process(ProcessContext processContext, ProcessedData processedData) Map> observationByHash = pendingData.getObservationByHash(); Map expUnitbyTrialName = pendingData.getExpUnitByTrialName(); + // Assume that there's only one expUnit per trial in this workflow. + // This should be a safe assumption to make considering this workflow is for the creation of a single experiment. + // If that changes in the future, this code will break and need to be dealt with. + + if (expUnitbyTrialName.size() > 1) { + throw new InternalServerException("Multiple Experiment names exist during commit stage"); + } + + String expUnitName = expUnitbyTrialName.values() + .stream() + .findFirst() + .orElse(null); + + if (expUnitName == null) { + throw new InternalServerException("Experiment unit name not found during commit stage"); + } + + String brapiProgramId = Optional.of(program) + .map(Program::getBrapiProgram) + .map(BrAPIProgram::getProgramDbId) + .orElse(null); + + if (brapiProgramId == null) { + throw new InternalServerException("BrAPI program not found during commit stage"); + } + List newTrials = ProcessorData.getNewObjects(pendingData.getTrialByNameNoScope()); List newLocations = ProcessorData.getNewObjects(pendingData.getLocationByName()) @@ -139,6 +166,14 @@ public void process(ProcessContext processContext, ProcessedData processedData) List newObservationUnits = ProcessorData.getNewObjects(pendingData.getObservationUnitByNameNoScope()); + // Inject level names here + datasetService.updateObservationUnitsWithLevelNameDbIds(newObservationUnits, + program, + brapiProgramId, + expUnitName, + DatasetLevel.EXP_UNIT + ); + // filter out observations with no 'value' so they will not be saved List newObservations = ProcessorData.getNewObjects(observationByHash) .stream() @@ -176,8 +211,6 @@ public void process(ProcessContext processContext, ProcessedData processedData) trialByNameNoScope.get(createdTrialName) .getBrAPIObject() .setTrialDbId(createdTrial.getTrialDbId()); - - createObservationLevel(createdTrialName, expUnitbyTrialName, program); } List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); @@ -251,22 +284,6 @@ public void process(ProcessContext processContext, ProcessedData processedData) } - //Check if the experimental unit associated with the trial does not exist in the system and if so create a new observation level - private void createObservationLevel(String trialName, Map expUnitByTrialName, Program program) throws ApiException, InternalServerException { - String expUnit = expUnitByTrialName.get(trialName).toLowerCase(); - String programDbId = program.getBrapiProgram() != null ? program.getBrapiProgram().getProgramDbId() : null; - HttpResponse levelResponse = brAPIObservationLevelDAO.createObservationLevelName(program, expUnit, DatasetLevel.EXP_UNIT, programDbId); - - if (levelResponse.getStatus().getCode() == 409) { - log.info(String.format("Level with name=%s, order=%s, programDbId=%s already exists in database", expUnit, DatasetLevel.EXP_UNIT, programDbId)); - } else if (levelResponse.getStatus().getCode() == 200) { - log.info(String.format("Level with name=%s, order=%s, programDbId=%s created in database", expUnit, DatasetLevel.EXP_UNIT, programDbId)); - } else { - log.error("Error saving experiment import: " + levelResponse.getStatus().getReason()); - throw new InternalServerException("Unable to create observation level: " + levelResponse.getStatus().getReason()); - } - } - private void updateStudyDependencyValues(PendingData pendingData, Map mappedBrAPIImport, String programKey) { // update location DbIds in studies for all distinct locations Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index 18157fd11..0bba7f48a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -18,6 +18,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; @@ -26,14 +27,16 @@ import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; +import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationLevelDAO; import org.breedinginsight.brapi.v2.services.BrAPIObservationLevelService; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.BrAPIConstants; import org.breedinginsight.model.DatasetLevel; import org.breedinginsight.model.DatasetMetadata; import org.breedinginsight.model.Program; @@ -216,4 +219,42 @@ private String findLevelNameByNameAndOrder(Program program, return levelNameStreamResult.get(0).getLevelNameDbId(); } + + public void updateObservationUnitsWithLevelNameDbIds(List observationUnits, + Program program, + String brapiProgramDbId, + String expUnitName, + DatasetLevel levelOrder) throws ApiException { + Map levelNameDbIdByName = new HashMap<>(); + + String expLevelName = expUnitName.toLowerCase(); + + String existingLevelNameDbId = getOrCreateLevelNameForDataset(program, brapiProgramDbId, expLevelName, levelOrder); + levelNameDbIdByName.put(expLevelName, existingLevelNameDbId); + + List globalLevelNames = observationLevelService.getGlobalLevelNames(program); + + globalLevelNames.forEach(ouln -> levelNameDbIdByName.put(ouln.getLevelName(), ouln.getLevelNameDbId())); + + for (BrAPIObservationUnit observationUnit : observationUnits) { + + String positionLevelName = observationUnit.getObservationUnitPosition().getObservationLevel().getLevelName().toLowerCase(); + + observationUnit.getObservationUnitPosition().getObservationLevel().setLevelNameDbId(levelNameDbIdByName.get(positionLevelName)); + + for (BrAPIObservationUnitLevelRelationship lvlRelationship : observationUnit.getObservationUnitPosition().getObservationLevelRelationships()) { + if (lvlRelationship.getLevelName().equals(BrAPIConstants.BLOCK.getValue())) { + lvlRelationship.setLevelNameDbId(levelNameDbIdByName.get(lvlRelationship.getLevelName())); + } else if (lvlRelationship.getLevelName().equals(BrAPIConstants.REPLICATE.getValue())) { + lvlRelationship.setLevelNameDbId(levelNameDbIdByName.get(lvlRelationship.getLevelName())); + } else { + throw new InternalServerException(String.format("Level name [%s] detected in OU Level Relationship " + + "for experiment with Exp Unit name [%s]. This is unexpected and the new level " + + "name must be retrieved properly from BrAPI to insert its DbId into BrAPI request for proper creation and assignment.", + lvlRelationship.getLevelName(), expUnitName)); + } + } + } + + } } From 4fcc1ff94fb5014347880bf28103d16573624bb2 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 7 Apr 2026 13:10:50 +0000 Subject: [PATCH 258/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 16662af79..888b9a1e5 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1111 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/4b84d09b1474bea1b37469b3f8751b7aab614fe8 \ No newline at end of file +version=v1.3.0+1113 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6661de1e1ed1295c6271e24fca9b9a64c1fbc7c9 \ No newline at end of file From 5d469a686d0c856ac398b3a1677aabbd5d84d523 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Tue, 7 Apr 2026 11:07:07 -0400 Subject: [PATCH 259/289] Refinement and test simplification --- AGENTS.md | 54 +++ .../brapi/v2/services/BrAPITrialService.java | 121 +++-- .../services/BrAPITrialServiceUnitTest.java | 420 ++++++++++++++++++ 3 files changed, 566 insertions(+), 29 deletions(-) create mode 100644 AGENTS.md create mode 100644 src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..6a7b49b5a --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,54 @@ +# AGENTS.md + +## Scope +- These instructions apply to the whole repository. +- If a subdirectory contains its own `AGENTS.md`, the more specific file wins for that subtree. + +## Repository Overview +- Java 13 Maven backend using Micronaut. +- Main application code: `src/main/java/org/breedinginsight` +- Tests: `src/test/java/org/breedinginsight` +- DB migrations: `src/main/resources/db/migration` +- API docs and related materials: `docs` +- Build config: `pom.xml` +- Docker and local setup: `README.md`, `docker-compose.yml` + +## Working Rules +- Keep changes narrow and task-focused. +- Prefer fixing root causes over adding one-off workarounds. +- Do not edit `.env`, `.env.test`, `.env.template`, or other secret-bearing config files unless explicitly asked. +- Do not modify `target/`, `.idea/`, or the large SQL dump files unless the task is specifically about them. +- Scope searches to relevant directories instead of scanning the entire repository when possible. +- Do not edit historical Flyway migrations unless explicitly requested. Add a new migration instead. + +## Files To Ignore By Default +- `target/` +- `.idea/` + +## Validation +- Use Maven for validation. +- Source envs from .env when running mvn commands +- Preferred targeted test command: + `mvn -Dtest=ClassName test --settings settings.xml` +- Full test command: + `mvn test --settings settings.xml` +- Full build without tests: + `mvn clean validate install -D maven.test.skip=true --settings settings.xml` +- Tests may require Docker, Testcontainers, and local services. If validation cannot run, say exactly what blocked it. + +## API Change Rules +- If endpoint behavior changes, do not update the relevant API docs or spec files in `docs`, they are no longer being maintained. +- If API code changes, add or update endpoint or integration tests. + +## Database Rules +- Schema changes belong in a new Flyway migration under `src/main/resources/db/migration`. +- If schema changes affect generated jOOQ code, run the documented generation flow. + +## Testing Guidance +- Prefer targeted tests for the changed area before suggesting a full test run. +- For controller or API changes, look first under `src/test/java/org/breedinginsight/api` and `src/test/java/org/breedinginsight/brapi`. +- For importer work, check `src/test/java/org/breedinginsight/brapps/importer`. + +## Response Expectations +- Summarize changed files, validation performed, and any remaining risks. +- If tests or docs updates were skipped, explain why. diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 92a7ac2db..70c79afa8 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -170,6 +170,7 @@ public DownloadFile exportObservations( List obsVars = new ArrayList<>(); Map> rowByOUId = new HashMap<>(); Map studyByDbId = new HashMap<>(); + Map yearByStudyDbId = new HashMap<>(); Map studyDbIdByOUId = new HashMap<>(); List requestedEnvIds = StringUtils.isNotBlank(params.getEnvironments()) ? new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))) : new ArrayList<>(); @@ -205,6 +206,7 @@ public DownloadFile exportObservations( log.error(logHash + ": Error fetching observation units for a study by its DbId" + Utilities.generateApiExceptionLogMessage(err), err); } + yearByStudyDbId.putAll(getYearByStudyDbId(expStudies, program.getId())); boolean isSubObs = isSubEntityDataset(ous); @@ -258,6 +260,7 @@ public DownloadFile exportObservations( obsVars, studyDbIdByOUId, programGermplasmByDbId, + yearByStudyDbId, isSubObs ); @@ -268,7 +271,7 @@ public DownloadFile exportObservations( // Map Observation Unit to the Study it belongs to. studyDbIdByOUId.put(ouId, ou.getStudyDbId()); if (!rowByOUId.containsKey(ouId)) { - rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId, isSubObs)); + rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId, yearByStudyDbId, isSubObs)); } } } @@ -363,30 +366,12 @@ public Dataset getDatasetData(Program program, UUID experimentId, UUID datasetId log.debug("fetching observationUnits for dataset: " + datasetId); List datasetOUs = ouDAO.getObservationUnitsForDataset(datasetId.toString(), program); - //Add years to the addition_info elements - //TODO yearByStudyDbId will no longer be needed, and should be removed, once the seasonDAO uses the redis cache (BI-2261). - Map yearByStudyDbId = new HashMap<>(); // used to prevent the same season from being fetched repeatedly. - for ( BrAPIObservationUnit ou: datasetOUs ) { - String environmentId = Utilities.getExternalReference(ou.getExternalReferences(), this.referenceSource, ExternalReferenceSource.STUDIES) - .orElseThrow( ()-> new DoesNotExistException("No BI external reference for STUDIES was found")) - .getReferenceId(); - if( !yearByStudyDbId.containsKey( environmentId )) { - // Get the Study and extract the year from its Season - BrAPIStudy study = studyDAO.getStudyByEnvironmentId(UUID.fromString(environmentId), program).orElseThrow( () -> new DoesNotExistException(String.format("Study Id '%s' not found.", environmentId)) ); - if(study.getSeasons().isEmpty()){ - throw new DoesNotExistException(String.format("No Seasons found in Study Id = '%s'.", environmentId)); - } - String seasonId = study.getSeasons().get(0); - BrAPISeason season = seasonDAO.getSeasonById(seasonId, program.getId()); - if(season==null){ - throw new DoesNotExistException(String.format("Seasons not found for Id = '%s'.", seasonId)); - } - Integer year = season.getYear(); - yearByStudyDbId.put(environmentId, year); - } - - ou.putAdditionalInfoItem(BrAPIAdditionalInfoFields.ENV_YEAR, yearByStudyDbId.get(environmentId)); - } + Map yearByStudyDbId = getYearByStudyDbIds( + datasetOUs.stream() + .map(BrAPIObservationUnit::getStudyDbId) + .collect(Collectors.toSet()), + program); + addEnvYearToObservationUnits(datasetOUs, yearByStudyDbId); log.debug("fetching dataset variables dataset: " + datasetId); List datasetObsVars = getDatasetObsVars(datasetId.toString(), program); @@ -670,6 +655,7 @@ private void addBrAPIObsToRecords( List obsVars, Map studyDbIdByOUId, Map programGermplasmByDbId, + Map yearByStudyDbId, boolean isSubObs) throws ApiException, DoesNotExistException { Map varByDbId = new HashMap<>(); obsVars.forEach(var -> varByDbId.put(var.getObservationVariableDbId(), var)); @@ -688,7 +674,7 @@ private void addBrAPIObsToRecords( } else { // otherwise make a new row - Map row = createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId, isSubObs); + Map row = createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId, yearByStudyDbId, isSubObs); addObsVarDataToRow(row, obs, includeTimestamp, var, program); rowByOUId.put(ouId, row); } @@ -801,12 +787,19 @@ public int deleteExperiment(Program program, UUID experimentId, boolean hard) th return existingObservations.size(); } + /** + * Builds the static export columns for a single observation unit. + * + * Env Year is resolved ahead of time and passed in so export generation and + * dataset retrieval share the same study/season lookup path + */ private Map createExportRow( BrAPITrial experiment, Program program, BrAPIObservationUnit ou, Map studyByDbId, Map programGermplasmByDbId, + Map yearByStudyDbId, boolean isSubEntity) throws ApiException, DoesNotExistException { HashMap row = new HashMap<>(); @@ -818,7 +811,8 @@ private Map createExportRow( String ouId = ouXref.getReferenceID(); BrAPIGermplasm germplasm = Optional.ofNullable(programGermplasmByDbId.get(ou.getGermplasmDbId())) .orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service")); - BrAPIStudy study = studyByDbId.get(ou.getStudyDbId()); + BrAPIStudy study = Optional.ofNullable(studyByDbId.get(ou.getStudyDbId())) + .orElseThrow(() -> new DoesNotExistException(String.format("Study DbId '%s' not found.", ou.getStudyDbId()))); // make export row from BrAPI objects row.put(ExperimentObservation.Columns.GERMPLASM_NAME, Utilities.removeProgramKey(ou.getGermplasmName(), program.getKey(), germplasm.getAccessionNumber())); @@ -854,8 +848,11 @@ private Map createExportRow( : additionalInfo.get(BrAPIAdditionalInfoFields.RTK).getAsString(); row.put(ExperimentObservation.Columns.RTK, rtk); - BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); - row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); + // Treat a null season year as missing data. Experiment import requires Env Year, + // so a missing year here indicates unsupported upstream BrAPI data. + Integer year = Optional.ofNullable(yearByStudyDbId.get(study.getStudyDbId())) + .orElseThrow(() -> new DoesNotExistException(String.format("Env Year not found for Study DbId = '%s'.", study.getStudyDbId()))); + row.put(ExperimentObservation.Columns.ENV_YEAR, year); // get replicate number Optional repLevel = ou.getObservationUnitPosition() @@ -912,6 +909,72 @@ private Map createExportRow( return row; } + /** + * Resolves Env Year once per study while caching repeated season lookups by season DbId. + * + * This keeps export and dataset retrieval on the same code path and avoids refetching + * a season for every observation unit in the same study. + */ + private Map getYearByStudyDbId(Collection studies, UUID programId) throws ApiException, DoesNotExistException { + Map yearByStudyDbId = new HashMap<>(); + Map yearBySeasonDbId = new HashMap<>(); + for (BrAPIStudy study : studies) { + yearByStudyDbId.put(study.getStudyDbId(), getYearForStudy(study, programId, yearBySeasonDbId)); + } + + return yearByStudyDbId; + } + + /** + * Bulk-loads studies by studyDbId so dataset retrieval can avoid per-observation-unit + * environment lookups and reuse the shared year resolution flow. + */ + private Map getYearByStudyDbIds(Collection studyDbIds, Program program) throws ApiException, DoesNotExistException { + return getYearByStudyDbId( + studyDAO.getStudiesByStudyDbId(studyDbIds, program), + program.getId()); + } + + /** + * Adds the resolved Env Year to each observation unit for the dataset response. + */ + private void addEnvYearToObservationUnits(Collection observationUnits, Map yearByStudyDbId) throws DoesNotExistException { + // additionalinfo.envYear is for the frontend dataset view + for (BrAPIObservationUnit observationUnit : observationUnits) { + Integer year = yearByStudyDbId.get(observationUnit.getStudyDbId()); + if (year == null) { + throw new DoesNotExistException(String.format("Study DbId '%s' not found.", observationUnit.getStudyDbId())); + } + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.ENV_YEAR, year); + } + } + + /** + * Resolves the year for the study's first season reference. + * + * A season that exists but does not expose a numeric year is treated as invalid data. + * The experiment import workflow requires Env Year for created experiments/environments, + * so the stricter failure here is intentional + */ + private Integer getYearForStudy(BrAPIStudy study, UUID programId, Map yearBySeasonDbId) throws ApiException, DoesNotExistException { + if (study.getSeasons() == null || study.getSeasons().isEmpty()) { + throw new DoesNotExistException(String.format("No Seasons found in Study DbId = '%s'.", study.getStudyDbId())); + } + + String seasonId = study.getSeasons().get(0); + Integer year = yearBySeasonDbId.get(seasonId); + if (year == null) { + BrAPISeason season = seasonDAO.getSeasonById(seasonId, programId); + if (season == null) { + throw new DoesNotExistException(String.format("Seasons not found for Id = '%s'.", seasonId)); + } + year = season.getYear(); + yearBySeasonDbId.put(seasonId, year); + } + + return year; + } + private String doubleToString(double val){ return Double.isNaN(val) ? null : String.valueOf( val ); } diff --git a/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java b/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java new file mode 100644 index 000000000..0554319ec --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java @@ -0,0 +1,420 @@ +package org.breedinginsight.brapi.v2.services; + +import com.google.gson.JsonObject; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPISeason; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; +import org.brapi.v2.model.pheno.BrAPIObservationUnitPosition; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationLevelDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; +import org.breedinginsight.brapps.importer.model.exports.FileType; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.FileMappingUtil; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; +import org.breedinginsight.model.BrAPIConstants; +import org.breedinginsight.model.Dataset; +import org.breedinginsight.model.DownloadFile; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.TraitService; +import org.breedinginsight.services.lock.DistributedLockService; +import org.breedinginsight.utilities.FileUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.tablesaw.api.Table; + +import java.io.ByteArrayInputStream; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class BrAPITrialServiceUnitTest { + private static final String REFERENCE_SOURCE = "breedinginsight.org"; + private static final String EXPORT_DATASET_ID = "33333333-3333-3333-3333-333333333333"; + private static final String DATASET_ID = "22222222-2222-2222-2222-222222222222"; + private static final String ENVIRONMENT_ID = "44444444-4444-4444-4444-444444444444"; + private static final String SECOND_ENVIRONMENT_ID = "55555555-5555-5555-5555-555555555555"; + + private final BrAPITrialDAO trialDAO = mock(BrAPITrialDAO.class); + private final BrAPIObservationDAO observationDAO = mock(BrAPIObservationDAO.class); + private final BrAPIObservationUnitDAO observationUnitDAO = mock(BrAPIObservationUnitDAO.class); + private final BrAPIListDAO listDAO = mock(BrAPIListDAO.class); + private final TraitService traitService = mock(TraitService.class); + private final BrAPIStudyDAO studyDAO = mock(BrAPIStudyDAO.class); + private final BrAPISeasonDAO seasonDAO = mock(BrAPISeasonDAO.class); + private final BrAPIObservationLevelDAO observationLevelDAO = mock(BrAPIObservationLevelDAO.class); + private final BrAPIGermplasmDAO germplasmDAO = mock(BrAPIGermplasmDAO.class); + private final FileMappingUtil fileMappingUtil = mock(FileMappingUtil.class); + private final DistributedLockService lockService = mock(DistributedLockService.class); + private final DatasetService datasetService = mock(DatasetService.class); + + private BrAPITrialService service; + private Program program; + private BrAPITrial experiment; + private BrAPIStudy study; + private BrAPIGermplasm germplasm; + private BrAPISeason season; + + @BeforeEach + void setup() { + service = new BrAPITrialService( + REFERENCE_SOURCE, + trialDAO, + observationDAO, + observationUnitDAO, + listDAO, + traitService, + studyDAO, + seasonDAO, + observationUnitDAO, + observationLevelDAO, + germplasmDAO, + fileMappingUtil, + lockService, + datasetService + ); + + program = new Program(); + program.setId(UUID.randomUUID()); + program.setKey("TEST"); + + experiment = new BrAPITrial(); + experiment.setTrialDbId("trial-db-id"); + experiment.setTrialName("Unit Test Experiment"); + experiment.setTrialDescription("Trial description"); + JsonObject experimentInfo = new JsonObject(); + experimentInfo.addProperty(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE, "Phenotyping"); + experiment.setAdditionalInfo(experimentInfo); + + study = new BrAPIStudy(); + study.setStudyDbId("study-1"); + study.setStudyName("Environment 1"); + study.setLocationName("Location 1"); + study.setSeasons(List.of("season-1")); + study.setExternalReferences(List.of(createExternalReference( + String.format("%s/%s", REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName()), + ENVIRONMENT_ID + ))); + + germplasm = new BrAPIGermplasm(); + germplasm.setGermplasmDbId("germ-1"); + germplasm.setAccessionNumber("G-1"); + + season = new BrAPISeason(); + season.setSeasonDbId("season-1"); + season.setYear(2023); + } + + @Test + void exportObservationsFetchesSeasonOncePerSeasonAndWritesEnvYear() throws Exception { + ExperimentExportQuery params = exportQuery(EXPORT_DATASET_ID); + List observationUnits = new ArrayList<>(List.of( + createObservationUnit("ou-db-1", "plot-1"), + createObservationUnit("ou-db-2", "plot-2"), + createObservationUnit("ou-db-3", "plot-3") + )); + + when(trialDAO.getTrialsByExperimentIds(eq(List.of(UUID.fromString("11111111-1111-1111-1111-111111111111"))), eq(program))) + .thenReturn(List.of(experiment)); + when(studyDAO.getStudiesByExperimentID(eq(UUID.fromString("11111111-1111-1111-1111-111111111111")), eq(program))) + .thenReturn(List.of(study)); + when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); + when(observationUnitDAO.getObservationUnitsForDataset(EXPORT_DATASET_ID, program)).thenReturn(observationUnits); + when(listDAO.getListsByTypeAndExternalRef(any(), eq(program.getId()), any(), any())).thenReturn(Collections.emptyList()); + when(observationDAO.getObservationsByObservationUnits(anyCollection(), eq(program))).thenReturn(Collections.emptyList()); + when(germplasmDAO.getGermplasmsByDBID(anyList(), eq(program.getId()))).thenReturn(List.of(germplasm)); + + DownloadFile downloadFile = service.exportObservations(program, UUID.fromString("11111111-1111-1111-1111-111111111111"), params); + + Table exportTable = FileUtil.parseTableFromCsv(new ByteArrayInputStream(downloadFile.getStreamedFile().getInputStream().readAllBytes())); + assertEquals(3, exportTable.rowCount()); + assertEquals(List.of(2023, 2023, 2023), exportTable.intColumn(Columns.ENV_YEAR).asList()); + verify(seasonDAO, times(1)).getSeasonById("season-1", program.getId()); + } + + @Test + void exportObservationsFetchesYearsOnlyForRequestedEnvironments() throws Exception { + ExperimentExportQuery params = exportQuery(EXPORT_DATASET_ID); + setField(params, "environments", ENVIRONMENT_ID); + List observationUnits = List.of(createObservationUnit("ou-db-1", "plot-1")); + BrAPIStudy unrelatedStudy = new BrAPIStudy(); + unrelatedStudy.setStudyDbId("study-2"); + unrelatedStudy.setStudyName("Environment 2"); + unrelatedStudy.setLocationName("Location 2"); + unrelatedStudy.setSeasons(Collections.emptyList()); + unrelatedStudy.setExternalReferences(List.of(createExternalReference( + String.format("%s/%s", REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName()), + SECOND_ENVIRONMENT_ID + ))); + + when(trialDAO.getTrialsByExperimentIds(eq(List.of(UUID.fromString("11111111-1111-1111-1111-111111111111"))), eq(program))) + .thenReturn(List.of(experiment)); + when(studyDAO.getStudiesByExperimentID(eq(UUID.fromString("11111111-1111-1111-1111-111111111111")), eq(program))) + .thenReturn(List.of(study, unrelatedStudy)); + when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); + when(observationUnitDAO.getObservationUnitsForDatasetAndEnvs(EXPORT_DATASET_ID, List.of(ENVIRONMENT_ID), program)).thenReturn(observationUnits); + when(listDAO.getListsByTypeAndExternalRef(any(), eq(program.getId()), any(), any())).thenReturn(Collections.emptyList()); + when(observationDAO.getObservationsByObservationUnits(anyCollection(), eq(program))).thenReturn(Collections.emptyList()); + when(germplasmDAO.getGermplasmsByDBID(anyList(), eq(program.getId()))).thenReturn(List.of(germplasm)); + + DownloadFile downloadFile = service.exportObservations(program, UUID.fromString("11111111-1111-1111-1111-111111111111"), params); + + Table exportTable = FileUtil.parseTableFromCsv(new ByteArrayInputStream(downloadFile.getStreamedFile().getInputStream().readAllBytes())); + assertEquals(1, exportTable.rowCount()); + assertEquals(List.of(2023), exportTable.intColumn(Columns.ENV_YEAR).asList()); + verify(seasonDAO, times(1)).getSeasonById("season-1", program.getId()); + } + + @Test + void exportObservationsWritesDistinctYearsForMultipleEnvironments() throws Exception { + ExperimentExportQuery params = exportQuery(EXPORT_DATASET_ID); + BrAPIStudy secondStudy = new BrAPIStudy(); + secondStudy.setStudyDbId("study-2"); + secondStudy.setStudyName("Environment 2"); + secondStudy.setLocationName("Location 2"); + secondStudy.setSeasons(List.of("season-2")); + secondStudy.setExternalReferences(List.of(createExternalReference( + String.format("%s/%s", REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName()), + SECOND_ENVIRONMENT_ID + ))); + + BrAPISeason secondSeason = new BrAPISeason(); + secondSeason.setSeasonDbId("season-2"); + secondSeason.setYear(2024); + + List observationUnits = new ArrayList<>(List.of( + createObservationUnit("ou-db-1", "plot-1"), + createObservationUnit("ou-db-2", "plot-2", "study-2", "Environment 2", SECOND_ENVIRONMENT_ID) + )); + + when(trialDAO.getTrialsByExperimentIds(eq(List.of(UUID.fromString("11111111-1111-1111-1111-111111111111"))), eq(program))) + .thenReturn(List.of(experiment)); + when(studyDAO.getStudiesByExperimentID(eq(UUID.fromString("11111111-1111-1111-1111-111111111111")), eq(program))) + .thenReturn(List.of(study, secondStudy)); + when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); + when(seasonDAO.getSeasonById("season-2", program.getId())).thenReturn(secondSeason); + when(observationUnitDAO.getObservationUnitsForDataset(EXPORT_DATASET_ID, program)).thenReturn(observationUnits); + when(listDAO.getListsByTypeAndExternalRef(any(), eq(program.getId()), any(), any())).thenReturn(Collections.emptyList()); + when(observationDAO.getObservationsByObservationUnits(anyCollection(), eq(program))).thenReturn(Collections.emptyList()); + when(germplasmDAO.getGermplasmsByDBID(anyList(), eq(program.getId()))).thenReturn(List.of(germplasm)); + + DownloadFile downloadFile = service.exportObservations(program, UUID.fromString("11111111-1111-1111-1111-111111111111"), params); + + Table exportTable = FileUtil.parseTableFromCsv(new ByteArrayInputStream(downloadFile.getStreamedFile().getInputStream().readAllBytes())); + assertEquals(2, exportTable.rowCount()); + assertEquals(List.of("Environment 1", "Environment 2"), exportTable.stringColumn(Columns.ENV).asList()); + assertEquals(List.of(2023, 2024), exportTable.intColumn(Columns.ENV_YEAR).asList()); + verify(seasonDAO, times(1)).getSeasonById("season-1", program.getId()); + verify(seasonDAO, times(1)).getSeasonById("season-2", program.getId()); + } + + @Test + void exportObservationsThrowsWhenSeasonYearIsNull() throws Exception { + // This PR intentionally hardens null season years because experiment import + // requires Env Year when creating experiments/environments. + ExperimentExportQuery params = exportQuery(EXPORT_DATASET_ID); + List observationUnits = List.of(createObservationUnit("ou-db-1", "plot-1")); + season.setYear(null); + + when(trialDAO.getTrialsByExperimentIds(eq(List.of(UUID.fromString("11111111-1111-1111-1111-111111111111"))), eq(program))) + .thenReturn(List.of(experiment)); + when(studyDAO.getStudiesByExperimentID(eq(UUID.fromString("11111111-1111-1111-1111-111111111111")), eq(program))) + .thenReturn(List.of(study)); + when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); + when(observationUnitDAO.getObservationUnitsForDataset(EXPORT_DATASET_ID, program)).thenReturn(observationUnits); + when(listDAO.getListsByTypeAndExternalRef(any(), eq(program.getId()), any(), any())).thenReturn(Collections.emptyList()); + when(observationDAO.getObservationsByObservationUnits(anyCollection(), eq(program))).thenReturn(Collections.emptyList()); + when(germplasmDAO.getGermplasmsByDBID(anyList(), eq(program.getId()))).thenReturn(List.of(germplasm)); + + DoesNotExistException exception = assertThrows(DoesNotExistException.class, + () -> service.exportObservations(program, UUID.fromString("11111111-1111-1111-1111-111111111111"), params)); + + assertEquals("Env Year not found for Study DbId = 'study-1'.", exception.getMessage()); + } + + @Test + void getDatasetDataUsesStudyDbIdsAndCachesSeasonByStudy() throws Exception { + List observationUnits = new ArrayList<>(List.of( + createObservationUnit("ou-db-1", "plot-1"), + createObservationUnit("ou-db-2", "plot-2") + )); + + when(observationUnitDAO.getObservationUnitsForDataset(DATASET_ID, program)).thenReturn(observationUnits); + when(studyDAO.getStudiesByStudyDbId(eq(Set.of("study-1")), eq(program))).thenReturn(List.of(study)); + when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); + when(listDAO.getListsByTypeAndExternalRef(any(), eq(program.getId()), any(), any())).thenReturn(Collections.emptyList()); + when(observationDAO.getObservationsByObservationUnitsAndVariables(anyList(), eq(Collections.emptyList()), eq(program))) + .thenReturn(Collections.emptyList()); + + Dataset dataset = service.getDatasetData(program, UUID.randomUUID(), UUID.fromString(DATASET_ID), false); + + assertEquals(2, dataset.observationUnits.size()); + dataset.observationUnits.forEach(ou -> + assertEquals(2023, ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENV_YEAR).getAsInt())); + verify(studyDAO, times(1)).getStudiesByStudyDbId(eq(Set.of("study-1")), eq(program)); + verify(studyDAO, never()).getStudyByEnvironmentId(eq(UUID.fromString(ENVIRONMENT_ID)), eq(program)); + verify(seasonDAO, times(1)).getSeasonById("season-1", program.getId()); + } + + @Test + void getDatasetDataWritesDistinctYearsForMultipleStudies() throws Exception { + BrAPIStudy secondStudy = new BrAPIStudy(); + secondStudy.setStudyDbId("study-2"); + secondStudy.setStudyName("Environment 2"); + secondStudy.setLocationName("Location 2"); + secondStudy.setSeasons(List.of("season-2")); + secondStudy.setExternalReferences(List.of(createExternalReference( + String.format("%s/%s", REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName()), + SECOND_ENVIRONMENT_ID + ))); + + BrAPISeason secondSeason = new BrAPISeason(); + secondSeason.setSeasonDbId("season-2"); + secondSeason.setYear(2024); + + List observationUnits = new ArrayList<>(List.of( + createObservationUnit("ou-db-1", "plot-1"), + createObservationUnit("ou-db-2", "plot-2", "study-2", "Environment 2", SECOND_ENVIRONMENT_ID) + )); + + when(observationUnitDAO.getObservationUnitsForDataset(DATASET_ID, program)).thenReturn(observationUnits); + when(studyDAO.getStudiesByStudyDbId(eq(Set.of("study-1", "study-2")), eq(program))).thenReturn(List.of(study, secondStudy)); + when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); + when(seasonDAO.getSeasonById("season-2", program.getId())).thenReturn(secondSeason); + when(listDAO.getListsByTypeAndExternalRef(any(), eq(program.getId()), any(), any())).thenReturn(Collections.emptyList()); + when(observationDAO.getObservationsByObservationUnitsAndVariables(anyList(), eq(Collections.emptyList()), eq(program))) + .thenReturn(Collections.emptyList()); + + Dataset dataset = service.getDatasetData(program, UUID.randomUUID(), UUID.fromString(DATASET_ID), false); + + assertEquals(2, dataset.observationUnits.size()); + assertEquals(List.of(2023, 2024), dataset.observationUnits.stream() + .map(ou -> ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENV_YEAR).getAsInt()) + .collect(Collectors.toList())); + verify(studyDAO, times(1)).getStudiesByStudyDbId(eq(Set.of("study-1", "study-2")), eq(program)); + verify(seasonDAO, times(1)).getSeasonById("season-1", program.getId()); + verify(seasonDAO, times(1)).getSeasonById("season-2", program.getId()); + } + + @Test + void getDatasetDataThrowsWhenSeasonYearIsNull() throws Exception { + // Dataset retrieval now shares the same year resolution path as export, so + // unsupported null season years fail consistently across both entry points. + List observationUnits = List.of(createObservationUnit("ou-db-1", "plot-1")); + season.setYear(null); + + when(observationUnitDAO.getObservationUnitsForDataset(DATASET_ID, program)).thenReturn(observationUnits); + when(studyDAO.getStudiesByStudyDbId(eq(Set.of("study-1")), eq(program))).thenReturn(List.of(study)); + when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); + + DoesNotExistException exception = assertThrows(DoesNotExistException.class, + () -> service.getDatasetData(program, UUID.randomUUID(), UUID.fromString(DATASET_ID), false)); + + assertEquals("Study DbId 'study-1' not found.", exception.getMessage()); + } + + private ExperimentExportQuery exportQuery(String datasetId) throws Exception { + ExperimentExportQuery params = new ExperimentExportQuery(); + setField(params, "datasetId", datasetId); + setField(params, "fileExtension", FileType.CSV); + setField(params, "includeTimestamps", false); + return params; + } + + private BrAPIObservationUnit createObservationUnit(String observationUnitDbId, String observationUnitName) { + return createObservationUnit(observationUnitDbId, observationUnitName, "study-1", "Environment 1", ENVIRONMENT_ID); + } + + private BrAPIObservationUnit createObservationUnit( + String observationUnitDbId, + String observationUnitName, + String studyDbId, + String studyName, + String environmentId) { + BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); + observationUnit.setObservationUnitDbId(observationUnitDbId); + observationUnit.setObservationUnitName(observationUnitName); + observationUnit.setStudyDbId(studyDbId); + observationUnit.setStudyName(studyName); + observationUnit.setGermplasmDbId("germ-1"); + observationUnit.setGermplasmName("Germplasm 1"); + observationUnit.setExternalReferences(List.of( + createExternalReference( + String.format("%s/%s", REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()), + observationUnitDbId + ), + createExternalReference( + String.format("%s/%s", REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName()), + environmentId + ))); + observationUnit.setObservationUnitPosition(createObservationUnitPosition()); + return observationUnit; + } + + private BrAPIObservationUnitPosition createObservationUnitPosition() { + BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); + BrAPIObservationUnitLevelRelationship observationLevel = new BrAPIObservationUnitLevelRelationship(); + observationLevel.setLevelName("plot"); + observationLevel.setLevelCode("plot"); + observationLevel.setLevelOrder(1); + position.setObservationLevel(observationLevel); + + BrAPIObservationUnitLevelRelationship repLevel = new BrAPIObservationUnitLevelRelationship(); + repLevel.setLevelName(BrAPIConstants.REPLICATE.getValue()); + repLevel.setLevelCode("1"); + repLevel.setLevelOrder(2); + + BrAPIObservationUnitLevelRelationship blockLevel = new BrAPIObservationUnitLevelRelationship(); + blockLevel.setLevelName(BrAPIConstants.BLOCK.getValue()); + blockLevel.setLevelCode("1"); + blockLevel.setLevelOrder(3); + + position.setObservationLevelRelationships(new ArrayList<>(List.of(repLevel, blockLevel))); + return position; + } + + private BrAPIExternalReference createExternalReference(String source, String id) { + BrAPIExternalReference externalReference = new BrAPIExternalReference(); + externalReference.setReferenceSource(source); + externalReference.setReferenceId(id); + externalReference.setReferenceID(id); + return externalReference; + } + + private void setField(Object target, String fieldName, Object value) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } +} From 01f18bb45e7f2dc350a10c2226dd37427552d68e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 7 Apr 2026 20:44:04 +0000 Subject: [PATCH 260/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 888b9a1e5..23cf3879f 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1113 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6661de1e1ed1295c6271e24fca9b9a64c1fbc7c9 \ No newline at end of file +version=v1.3.0+1115 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/b4f02f3c4a91185281e5d23759d93de70a2bd144 \ No newline at end of file From ce31e054f069740930ae22625b013b1b6ac3ee05 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Fri, 10 Apr 2026 17:07:44 -0400 Subject: [PATCH 261/289] Cleanup comments --- .../brapi/v2/services/BrAPITrialService.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index ab81fdcda..68d59d526 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -436,12 +436,13 @@ public List getDatasetsMetadata(Program program, UUID experimen } /** - * Assumptions: - * @param program - * @param experimentId - * @return - * @throws DoesNotExistException - * @throws ApiException + * Returns list of recommended sub entity names based on observation levels for the program that exclude + * level names already used in the experiment and is deduplicated for same name at multiple levels + * @param program Program + * @param experimentId Experiment Id + * @return list of dataset name recommendations + * @throws DoesNotExistException If trial does not exist + * @throws ApiException If BrAPI trial retrieval fails */ public List getRecommendedSubEntityDatasetNames(Program program, UUID experimentId) throws DoesNotExistException, ApiException { BrAPITrial experiment = trialDAO.getTrialById(program.getId(), experimentId).orElseThrow(() -> new DoesNotExistException("Trial does not exist")); @@ -450,13 +451,9 @@ public List getRecommendedSubEntityDatasetNames(Program program, UUID ex Set currentExperimentDatasetNames = deltaExperiment.getDatasetsMetadata() .stream() .map(DatasetMetadata::getName) - //.filter(StringUtils::isNotBlank) .collect(Collectors.toSet()); return getProgramObservationLevelNames(program).stream() - //.filter(StringUtils::isNotBlank) - //.filter(name -> !BrAPIConstants.REPLICATE.getValue().equalsIgnoreCase(name)) - //.filter(name -> !BrAPIConstants.BLOCK.getValue().equalsIgnoreCase(name)) .filter(name -> !currentExperimentDatasetNames.contains(name)) .distinct() .sorted() From c9db13927c157e468951ac45bd144ee45461445b Mon Sep 17 00:00:00 2001 From: "dr.phillips" Date: Mon, 13 Apr 2026 12:19:41 -0400 Subject: [PATCH 262/289] [BI-2783] added comment --- .../services/processors/germplasm/GermplasmProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index 1cb15a4b3..5defe7d10 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -151,7 +151,8 @@ public void getExistingBrapiData(List importRows, Program program) } } - // Get existing germplasm names + // Get existing germplasm names. This should be used ONLY to identify duplicate germplasm names. + // In general you should look up germplasm by GID. List dbGermplasm = brAPIGermplasmService.getGermplasmByDisplayName(new ArrayList<>(fileGermplasmByName.keySet()), program.getId()); dbGermplasm.forEach(germplasm -> { dbGermplasmByName.put(germplasm.getDefaultDisplayName(), germplasm); From a642ee971ac64d0e2c0037de6042e6e97dcd6bea Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 13 Apr 2026 18:30:28 +0000 Subject: [PATCH 263/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 23cf3879f..d9ada084e 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1115 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/b4f02f3c4a91185281e5d23759d93de70a2bd144 \ No newline at end of file +version=v1.3.0+1117 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/bf85d8a29a7aafaf72569c5fced1c2c79b4dc2b3 \ No newline at end of file From 3c8d86a2dddb52b911cf6b48f43f89d936241dea Mon Sep 17 00:00:00 2001 From: "dr.phillips" Date: Thu, 9 Apr 2026 14:29:37 -0400 Subject: [PATCH 264/289] [BI-2823] removed sub entity level from the level relationships --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index d59b0928c..3c115afd9 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -618,8 +618,6 @@ public BrAPIObservationUnit createSubObservationUnit( // ObservationLevelRelationships. List levelRelationships = new ArrayList<>(); - // TODO: Figure out if we actually need to add the sub entity level to the level relationships BI-2823 - levelRelationships.add(level); // ObservationLevelRelationships for rep. BrAPIObservationUnitLevelRelationship expRepLevel = expUnit.getObservationUnitPosition() .getObservationLevelRelationships().stream() From 56f7db7d80ff09866c05557ba5f4e193d10ccff3 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 15 Apr 2026 17:53:27 +0000 Subject: [PATCH 265/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index d9ada084e..16b422639 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1117 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/bf85d8a29a7aafaf72569c5fced1c2c79b4dc2b3 \ No newline at end of file +version=v1.3.0+1119 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/9ddc5aa23c5a033232467f0bd934f96427dec959 \ No newline at end of file From 8df73c0eb07d5976862550dee469eca29a476ebf Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Thu, 16 Apr 2026 09:45:13 -0400 Subject: [PATCH 266/289] Update tests --- .../brapi/v2/services/BrAPITrialService.java | 4 + .../ExperimentControllerIntegrationTest.java | 332 +++++++++++++----- 2 files changed, 245 insertions(+), 91 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 68d59d526..0b2591d1d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -451,9 +451,13 @@ public List getRecommendedSubEntityDatasetNames(Program program, UUID ex Set currentExperimentDatasetNames = deltaExperiment.getDatasetsMetadata() .stream() .map(DatasetMetadata::getName) + .filter(Objects::nonNull) + .map(String::toLowerCase) .collect(Collectors.toSet()); return getProgramObservationLevelNames(program).stream() + .filter(Objects::nonNull) + .map(String::toLowerCase) .filter(name -> !currentExperimentDatasetNames.contains(name)) .distinct() .sorted() diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index be4895c26..5677dc0b0 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -14,6 +14,7 @@ import io.reactivex.Flowable; import lombok.SneakyThrows; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; @@ -53,6 +54,7 @@ import tech.tablesaw.api.Table; import javax.inject.Inject; import java.io.*; +import java.math.BigDecimal; import java.time.OffsetDateTime; import java.util.*; import java.util.stream.Collectors; @@ -62,7 +64,6 @@ @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ExperimentControllerIntegrationTest extends BrAPITest { private Program program; @@ -217,6 +218,10 @@ private String uploadExperimentWithoutObs() throws Exception { } private String uploadExperimentWithoutObs(String title, String expUnit) throws Exception { + return uploadExperimentWithoutObs(program, title, expUnit); + } + + private String uploadExperimentWithoutObs(Program targetProgram, String title, String expUnit) throws Exception { ImportTestUtils importTestUtils = new ImportTestUtils(); List> expRows = new ArrayList<>(); @@ -234,7 +239,7 @@ private String uploadExperimentWithoutObs(String title, String expUnit) throws E null, true, client, - program, + targetProgram, mappingId, newExperimentWorkflowId); String expId = importResult @@ -318,14 +323,14 @@ void downloadDatasets(boolean includeTimestamps, String extension, int numberOfE List> filteredRows = rows.stream() .filter(row -> file.getName().contains(row.get(ExperimentObservation.Columns.ENV).toString())) .collect(Collectors.toList()); - parseAndCheck(fileStream, extension, true, filteredRows, includeTimestamps, expectedColNumber); + parseAndCheck(fileStream, extension, true, filteredRows, includeTimestamps, expectedColNumber, "Plot ObsUnitID", filteredRows.size()); } } else { assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); // All (both) rows when 0 or 2 envs sent, first row when 1 env sent as query param. List> filteredRows = numberOfEnvsRequested == 1 ? List.of(rows.get(0)) : rows; - parseAndCheck(bodyStream, extension, numberOfEnvsRequested > 0, filteredRows, includeTimestamps, expectedColNumber); + parseAndCheck(bodyStream, extension, numberOfEnvsRequested > 0, filteredRows, includeTimestamps, expectedColNumber, "Plot ObsUnitID", filteredRows.size()); } // Remove temp directory after each test run. FileUtils.deleteDirectory(new File(tempDir)); @@ -338,14 +343,19 @@ void downloadDatasets(boolean includeTimestamps, String extension, int numberOfE @ParameterizedTest @CsvSource(value = {"CSV", "XLSX", "XLS"}) @SneakyThrows - @Disabled // disabled for now until we re-enable subentity support void downloadSubEntityDataset(String extension) { + Program subEntityProgram = createSeededProgram("SubEntity Download"); + String subEntityExperimentTitle = "SubEntity Download " + extension; + List> topLevelRows = buildObservedRows(subEntityExperimentTitle); + String subEntityExperimentId = uploadExperimentWithObs(subEntityProgram, subEntityExperimentTitle, topLevelRows); + String plantDatasetName = "plant" + extension.toLowerCase(Locale.ROOT); + String plantObservationLevel = StringUtils.capitalize(plantDatasetName); // Create sub-entity dataset. Flowable> postCall = client.exchange( POST(String.format("/programs/%s/experiments/%s/dataset", - program.getId().toString(), experimentId), - "{\"name\":\"Plant\",\"repeatedMeasures\":3}") + subEntityProgram.getId().toString(), subEntityExperimentId), + String.format("{\"name\":\"%s\",\"repeatedMeasures\":3}", plantDatasetName)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class); HttpResponse postResponse = postCall.blockingFirst(); @@ -354,11 +364,11 @@ void downloadSubEntityDataset(String extension) { assertEquals(HttpStatus.OK, postResponse.getStatus()); // Get top-level datasetId to include in export request. - BrAPITrial experiment = experimentService.getTrialDataByUUID(program.getId(), UUID.fromString(experimentId), false); + BrAPITrial experiment = experimentService.getTrialDataByUUID(subEntityProgram.getId(), UUID.fromString(subEntityExperimentId), false); String topLevelDatasetId = DatasetUtil.getTopLevelDataset(experiment).getId().toString(); Flowable> topLevelExportCall = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", - program.getId().toString(), experimentId, extension, topLevelDatasetId)) + subEntityProgram.getId().toString(), subEntityExperimentId, extension, topLevelDatasetId)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class ); HttpResponse topLevelResponse = topLevelExportCall.blockingFirst(); @@ -376,13 +386,22 @@ void downloadSubEntityDataset(String extension) { // Check file contents. ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(topLevelResponse.body())); - parseAndCheck(bodyStream, extension, false, rows, false, 25); + parseAndCheck( + bodyStream, + extension, + false, + topLevelRows, + false, + getExpectedExportColumnCount(false, traits.size(), false), + "Plot ObsUnitID", + topLevelRows.size() + ); // Make sub-entity dataset export request. - String plantDatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), "Plant"); + String plantDatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), plantDatasetName); Flowable> plantExportCall = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", - program.getId().toString(), experimentId, extension, plantDatasetId)) + subEntityProgram.getId().toString(), subEntityExperimentId, extension, plantDatasetId)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class ); HttpResponse plantResponse = plantExportCall.blockingFirst(); @@ -394,20 +413,29 @@ void downloadSubEntityDataset(String extension) { assertEquals(mediaTypeByExtension.get(extension), plantResponse.getHeaders().getContentType().orElseThrow(Exception::new)); // The expected contents of the exported Plant dataset (3 sub-obs units for each top-level unit were requested). - List> plantRows = buildSubEntityRows(rows, "Plant", 3); + List> plantRows = buildSubEntityRows(topLevelRows, plantObservationLevel, 3); // Check file contents. ByteArrayInputStream plantBodyStream = new ByteArrayInputStream(Objects.requireNonNull(plantResponse.body())); - parseAndCheck(plantBodyStream, extension, false, plantRows, false, 23); + parseAndCheck( + plantBodyStream, + extension, + false, + plantRows, + false, + getExpectedExportColumnCount(true, experimentService.getDatasetObsVars(plantDatasetId, subEntityProgram).size(), false), + plantObservationLevel + " ObsUnitID", + plantRows.size() + ); } @Test - @Order(1) public void createSubEntityDatasetRejectsExpUnitNameAlreadyUsedInSameExperiment() throws Exception { - String plantExperimentId = uploadExperimentWithoutObs("Plant Same Experiment", "Plant"); + Program testProgram = createSeededProgram("Reject Already Used"); + String plantExperimentId = uploadExperimentWithoutObs(testProgram, "Plant Same Experiment", "Plant"); Flowable> call = client.exchange( - POST(String.format("/programs/%s/experiments/%s/dataset", program.getId(), plantExperimentId), + POST(String.format("/programs/%s/experiments/%s/dataset", testProgram.getId(), plantExperimentId), "{\"name\":\"Plant\",\"repeatedMeasures\":2}") .contentType(MediaType.APPLICATION_JSON) .cookie(new NettyCookie("phylo-token", "test-registered-user")), @@ -419,13 +447,13 @@ public void createSubEntityDatasetRejectsExpUnitNameAlreadyUsedInSameExperiment( } @Test - @Order(2) public void createSubEntityDatasetAllowsExpUnitNameUsedInOtherExperiment() throws Exception { - uploadExperimentWithoutObs("Plant Source Experiment", "Plant"); - String recipientExperimentId = uploadExperimentWithoutObs("Plot Recipient Experiment", "Plot"); + Program testProgram = createSeededProgram("Allow Other Experiment"); + uploadExperimentWithoutObs(testProgram, "Plant Source Experiment", "Plant"); + String recipientExperimentId = uploadExperimentWithoutObs(testProgram, "Plot Recipient Experiment", "Plot"); Flowable> call = client.exchange( - POST(String.format("/programs/%s/experiments/%s/dataset", program.getId(), recipientExperimentId), + POST(String.format("/programs/%s/experiments/%s/dataset", testProgram.getId(), recipientExperimentId), "{\"name\":\"Plant\",\"repeatedMeasures\":2}") .contentType(MediaType.APPLICATION_JSON) .cookie(new NettyCookie("phylo-token", "test-registered-user")), @@ -437,26 +465,26 @@ public void createSubEntityDatasetAllowsExpUnitNameUsedInOtherExperiment() throw } @Test - @Order(3) public void recommendedSubEntityDatasetNamesIncludeExpUnitNamesFromOtherExperiments() throws Exception { - uploadExperimentWithoutObs("Plant Autocomplete Source", "Plant"); - String recipientExperimentId = uploadExperimentWithoutObs("Autocomplete Recipient", "Plot"); + Program testProgram = createSeededProgram("Recommended Names"); + uploadExperimentWithoutObs(testProgram, "Plant Autocomplete Source", "Plant"); + String recipientExperimentId = uploadExperimentWithoutObs(testProgram, "Autocomplete Recipient", "Plot"); - List recommendedNames = getRecommendedSubEntityDatasetNames(recipientExperimentId); + List recommendedNames = getRecommendedSubEntityDatasetNames(testProgram, recipientExperimentId); assertTrue(recommendedNames.stream().anyMatch(name -> name.equalsIgnoreCase("plant"))); assertFalse(recommendedNames.stream().anyMatch(name -> name.equalsIgnoreCase("plot"))); } @Test - @Order(4) public void recommendedSubEntityDatasetNamesDeDuplicateExpUnitAndSubUnitNamesAcrossExperiments() throws Exception { - uploadExperimentWithoutObs("Plant Exp Unit Source", "Plant"); - String subEntitySourceExperimentId = uploadExperimentWithoutObs("Plant Sub Unit Source", "Plot"); - String recipientExperimentId = uploadExperimentWithoutObs("Plant Unique Recipient", "Plot"); + Program testProgram = createSeededProgram("Dedup Across Experiments"); + uploadExperimentWithoutObs(testProgram, "Plant Exp Unit Source", "Plant"); + String subEntitySourceExperimentId = uploadExperimentWithoutObs(testProgram, "Plant Sub Unit Source", "Plot"); + String recipientExperimentId = uploadExperimentWithoutObs(testProgram, "Plant Unique Recipient", "Plot"); Flowable> postCall = client.exchange( - POST(String.format("/programs/%s/experiments/%s/dataset", program.getId(), subEntitySourceExperimentId), + POST(String.format("/programs/%s/experiments/%s/dataset", testProgram.getId(), subEntitySourceExperimentId), "{\"name\":\"Plant\",\"repeatedMeasures\":2}") .contentType(MediaType.APPLICATION_JSON) .cookie(new NettyCookie("phylo-token", "test-registered-user")), @@ -465,7 +493,7 @@ public void recommendedSubEntityDatasetNamesDeDuplicateExpUnitAndSubUnitNamesAcr HttpResponse postResponse = postCall.blockingFirst(); assertEquals(HttpStatus.OK, postResponse.getStatus()); - List recommendedNames = getRecommendedSubEntityDatasetNames(recipientExperimentId); + List recommendedNames = getRecommendedSubEntityDatasetNames(testProgram, recipientExperimentId); assertEquals(1L, recommendedNames.stream().filter(name -> name.equalsIgnoreCase("plant")).count()); } @@ -826,21 +854,20 @@ public void deleteExperimentInvalid() { @CsvSource(value = {"true,true", "false,true", "true,false", "false,false"}) @SneakyThrows public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations) { + Program deleteProgram = createSeededProgram("Delete Experiment"); + String deleteExperimentTitle = "Delete Experiment " + UUID.randomUUID(); // Set up a test trial and get the trialDbId. String trialDbId; if (withObservations) { - JsonArray beforeData = getProgramTrials(program.getId().toString()); - - // The trial created by setup has observations. - trialDbId = beforeData.get(0).getAsJsonObject().get("trialDbId").getAsString(); + trialDbId = uploadExperimentWithObs(deleteProgram, deleteExperimentTitle, buildObservedRows(deleteExperimentTitle)); } else { // Create a trial without observations. - trialDbId = uploadExperimentWithoutObs(); + trialDbId = uploadExperimentWithoutObs(deleteProgram, deleteExperimentTitle, "Plot"); } // A DELETE request should delete an experiment with observations unless there are observations and hardDelete = true. Flowable> deleteCall = client.exchange( - DELETE(String.format("/programs/%s/experiments/%s?hard=%s", program.getId().toString(), trialDbId, hardDelete)) + DELETE(String.format("/programs/%s/experiments/%s?hard=%s", deleteProgram.getId().toString(), trialDbId, hardDelete)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class ); @@ -854,15 +881,15 @@ public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations assertEquals(HttpStatus.CONFLICT, e.getStatus()); // Check that the trial was not deleted. - JsonArray trials = getProgramTrials(program.getId().toString()); + JsonArray trials = getProgramTrials(deleteProgram.getId().toString()); assertEquals(1, trials.size()); // Check that the studies were not deleted. - JsonArray studies = getProgramStudies(program.getId().toString()); + JsonArray studies = getProgramStudies(deleteProgram.getId().toString()); assertEquals(2, studies.size()); // Check that lists were not deleted. - JsonArray lists = getProgramObsVarLists(program.getId().toString()); + JsonArray lists = getProgramObsVarLists(deleteProgram.getId().toString()); assertEquals(1, lists.size()); } else { HttpResponse deleteResponse = deleteCall.blockingFirst(); @@ -870,20 +897,55 @@ public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations assertEquals(HttpStatus.NO_CONTENT, deleteResponse.getStatus()); // Check that the trial was deleted. - JsonArray trials = getProgramTrials(program.getId().toString()); + JsonArray trials = getProgramTrials(deleteProgram.getId().toString()); assertEquals(0, trials.size()); // Check that the studies were deleted. - JsonArray studies = getProgramStudies(program.getId().toString()); + JsonArray studies = getProgramStudies(deleteProgram.getId().toString()); assertEquals(0, studies.size()); // Check that the BrAPI lists were deleted. - JsonArray lists = getProgramObsVarLists(program.getId().toString()); + JsonArray lists = getProgramObsVarLists(deleteProgram.getId().toString()); assertEquals(0, lists.size()); } } + private List> buildObservedRows(String title) { + List> observedRows = new ArrayList<>(); + String envBase = title.replaceAll("\\s+", ""); + Map row1 = makeExpImportRow(title, envBase + "1"); + Map row2 = makeExpImportRow(title, envBase + "2"); + + for (int i = 0; i < traits.size(); i++) { + row1.put(traits.get(i).getObservationVariableName(), (float) (i + 1)); + } + + observedRows.add(row1); + observedRows.add(row2); + return observedRows; + } + + private String uploadExperimentWithObs(Program targetProgram, String title, List> expRows) throws Exception { + ImportTestUtils importTestUtils = new ImportTestUtils(); + + JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( + importTestUtils.writeExperimentDataToFile(expRows, traits, false, false, null), + null, + true, + client, + targetProgram, + mappingId, + newExperimentWorkflowId); + + return importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + } + private List> buildSubEntityRows(List> topLevelRows, String entityName, int repeatedMeasures) { List> plantRows = new ArrayList<>(); for (Map row : topLevelRows) { @@ -891,8 +953,8 @@ private List> buildSubEntityRows(List> t // Deep copy map entries. Map plantRow = new HashMap<>(row); - plantRow.put("Exp Unit", entityName); - plantRow.put("Exp Unit ID", i.toString()); + plantRow.put(ExperimentObservation.Columns.SUB_OBS_UNIT, entityName); + plantRow.put(ExperimentObservation.Columns.SUB_UNIT_ID, i.toString()); plantRow.remove("tt_test_1"); plantRow.remove("tt_test_2"); plantRows.add(plantRow); @@ -901,6 +963,14 @@ private List> buildSubEntityRows(List> t return plantRows; } + private int getExpectedExportColumnCount(boolean isSubEntity, int obsVarCount, boolean includeTimestamps) { + int baseColumnCount = isSubEntity + ? ExperimentFileColumns.getOrderedColumnsSubEntity().size() + 2 + : ExperimentFileColumns.getOrderedColumns().size() + 1; + int timestampColumnCount = includeTimestamps ? obsVarCount : 0; + return baseColumnCount + obsVarCount + timestampColumnCount; + } + private File writeDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); @@ -920,9 +990,13 @@ private File writeDataToFile(List> data, List traits) } private List getRecommendedSubEntityDatasetNames(String targetExperimentId) { + return getRecommendedSubEntityDatasetNames(program, targetExperimentId); + } + + private List getRecommendedSubEntityDatasetNames(Program targetProgram, String targetExperimentId) { Flowable> call = client.exchange( GET(String.format("/programs/%s/experiments/%s/recommended-sub-entity-dataset-names", - program.getId(), targetExperimentId)) + targetProgram.getId(), targetExperimentId)) .contentType(MediaType.APPLICATION_JSON) .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class @@ -939,6 +1013,42 @@ private List getRecommendedSubEntityDatasetNames(String targetExperiment return recommendedNames; } + private Program createSeededProgram(String prefix) throws Exception { + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + String suffix = UUID.randomUUID().toString().replaceAll("[^a-fA-F]", "").toUpperCase(Locale.ROOT); + while (suffix.length() < 4) { + suffix += UUID.randomUUID().toString().replaceAll("[^a-fA-F]", "").toUpperCase(Locale.ROOT); + } + suffix = suffix.substring(0, 4); + ProgramRequest programRequest = ProgramRequest.builder() + .name(prefix + " " + suffix) + .abbreviation("OT" + suffix) + .documentationUrl("localhost:8080") + .objective("To test ordered experiment scenarios") + .species(SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build()) + .key("OT" + suffix) + .build(); + Program seededProgram = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), seededProgram.getId()); + + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + ontologyService.createTraits(seededProgram.getId(), createTraits(2), user, false); + + List germplasm = createGermplasm(1); + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + newReference.setReferenceID(seededProgram.getId().toString()); + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + germplasmDAO.createBrAPIGermplasm(germplasm, seededProgram.getId(), null); + + return seededProgram; + } + private Map makeExpImportRow(String title, String environment) { return makeExpImportRow(title, environment, "Plot"); } @@ -1025,7 +1135,9 @@ private void parseAndCheck(InputStream stream, boolean requestEnv, List> rows, boolean includeTimestamps, - Integer expectedColNumber) throws ParsingException { + Integer expectedColNumber, + String expectedObsUnitIdColumn, + Integer expectedObsUnitIdUniqueCount) throws ParsingException { Table download = Table.create(); if (extension.equals("CSV")) { download = FileUtil.parseTableFromCsv(stream); @@ -1035,7 +1147,16 @@ private void parseAndCheck(InputStream stream, } // Assert import/export fidelity and presence of observation units in export - checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension, expectedColNumber); + checkDownloadTable( + requestEnv, + rows, + download, + includeTimestamps, + extension, + expectedColNumber, + expectedObsUnitIdColumn, + expectedObsUnitIdUniqueCount + ); } private void checkDownloadTable( @@ -1044,12 +1165,15 @@ private void checkDownloadTable( Table table, boolean includeTimestamps, String extension, - Integer expectedColNumber) { + Integer expectedColNumber, + String expectedObsUnitIdColumn, + Integer expectedObsUnitIdUniqueCount) { // Filename is correct: _Observation Dataset [-]__ List expectedEnvNames = requestedImportRows.stream() .map(row -> row.get(ExperimentObservation.Columns.ENV).toString()).collect(Collectors.toList()); assertEquals(expectedColNumber, table.columnCount()); + assertEquals(requestedImportRows.size(), table.rowCount()); // Check that requested envs are present. expectedEnvNames.forEach(envName -> assertTrue(table.stringColumn("Env").contains(envName))); @@ -1066,14 +1190,24 @@ private void checkDownloadTable( String gid = ExperimentObservation.Columns.GERMPLASM_GID; String env = ExperimentObservation.Columns.ENV; String expUnitId = ExperimentObservation.Columns.EXP_UNIT_ID; + String subObsUnit = ExperimentObservation.Columns.SUB_OBS_UNIT; + String subUnitId = ExperimentObservation.Columns.SUB_UNIT_ID; + boolean subObsUnitMatches = !row.containsKey(subObsUnit) || + row.get(subObsUnit).equals(downloadRow.getString(subObsUnit)); + boolean subUnitIdMatches = !row.containsKey(subUnitId) || + row.get(subUnitId).equals(downloadRow.getObject(subUnitId).toString()); if (extension.equalsIgnoreCase(FileType.CSV.getName())) { return Integer.parseInt(row.get(gid).toString()) == downloadRow.getInt(gid) && row.get(env).equals(downloadRow.getString(env)) && - row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); + row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()) && + subObsUnitMatches && + subUnitIdMatches; } else { return row.get(gid).equals(downloadRow.getString(gid)) && row.get(env).equals(downloadRow.getString(env)) && - row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); + row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()) && + subObsUnitMatches && + subUnitIdMatches; } }).findAny(); assertTrue(matchingImportRow.isPresent() && !matchingImportRow.get().isEmpty()); @@ -1085,58 +1219,74 @@ private void checkDownloadTable( } assertEquals(requestedImportRows.size(),matchingImportRows.size()); - //Observation level for tests should be "Plot" // Observation units populated. - assertEquals(0, table.column("Plot ObsUnitID").countMissing()); + assertTrue(table.columnNames().contains(expectedObsUnitIdColumn)); + assertEquals(0, table.column(expectedObsUnitIdColumn).countMissing()); // Observation Unit IDs are assigned. - assertEquals(requestedImportRows.size(), table.column("Plot ObsUnitID").countUnique()); + assertEquals(expectedObsUnitIdUniqueCount, table.column(expectedObsUnitIdColumn).countUnique()); } private boolean isMatchedRow(Map importRow, Row downloadRow) { - System.out.println("Validating row: " + downloadRow.getRowNumber()); - System.out.println("import columns: " + importRow.size()); return importRow.entrySet().stream().filter(e -> { String header = e.getKey(); - List importColumns = columns - .stream() - .filter(col -> header.equals(col.getValue())).collect(Collectors.toList()); - if (importColumns.size() != 1) { - return false; - } - Object expectedVal = null; - Object downloadedVal = null; - boolean doCompare = false; - - if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.STRING)) { - expectedVal = e.getValue().toString(); - downloadedVal = downloadRow.getString(e.getKey()); - doCompare = true; - } - if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.INTEGER)) { - expectedVal = Integer.parseInt(e.getValue().toString()); - downloadedVal = downloadRow.getInt(e.getKey()); - doCompare = true; - } - if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.DOUBLE)) { - expectedVal = Double.parseDouble(e.getValue().toString()); - downloadedVal = downloadRow.getDouble(e.getKey()); - doCompare = true; - } - if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.FLOAT)) { - expectedVal = e.getValue(); - downloadedVal = downloadRow.getFloat(e.getKey()); - doCompare = true; - } - System.out.println("Column: "+e.getKey()+", Expected: '"+ expectedVal +"', Received: '" + downloadedVal+"'"); - if(doCompare) { - assertEquals(expectedVal, downloadedVal); - return expectedVal.equals(downloadedVal); - } else { + if (!downloadRow.columnNames().contains(header)) { return false; } + + Object expectedVal = e.getValue(); + Object downloadedVal = getDownloadedValue(downloadRow, header); + assertValuesMatch(header, expectedVal, downloadedVal); + return true; }).count() == importRow.size(); } + private Object getDownloadedValue(Row downloadRow, String header) { + ColumnType columnType = downloadRow.getColumnType(header); + if (columnType.equals(ColumnType.STRING)) { + return downloadRow.getString(header); + } + if (columnType.equals(ColumnType.INTEGER)) { + return downloadRow.getInt(header); + } + if (columnType.equals(ColumnType.DOUBLE)) { + return downloadRow.getDouble(header); + } + if (columnType.equals(ColumnType.FLOAT)) { + return downloadRow.getFloat(header); + } + return downloadRow.getObject(header); + } + + private void assertValuesMatch(String header, Object expectedVal, Object downloadedVal) { + if (isNumeric(expectedVal) && isNumeric(downloadedVal)) { + assertEquals( + 0, + toBigDecimal(expectedVal).compareTo(toBigDecimal(downloadedVal)), + "Column " + header + " mismatch"); + return; + } + assertEquals( + Objects.toString(expectedVal, null), + Objects.toString(downloadedVal, null), + "Column " + header + " mismatch"); + } + + private boolean isNumeric(Object value) { + if (value == null) { + return false; + } + try { + new BigDecimal(value.toString()); + return true; + } catch (NumberFormatException e) { + return false; + } + } + + private BigDecimal toBigDecimal(Object value) { + return new BigDecimal(value.toString()); + } + private String getEnvId(JsonObject result, int index) { return result .get("preview").getAsJsonObject() From b742cfe4fdf5eded5041d6c79d1e7e94651feff9 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 16 Apr 2026 19:50:52 +0000 Subject: [PATCH 267/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 16b422639..1dcd18a63 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1119 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/9ddc5aa23c5a033232467f0bd934f96427dec959 \ No newline at end of file +version=v1.3.0+1121 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/4d0add514566148e3062708dd15f12eb424b38e7 \ No newline at end of file From 893394a022703998aafe410c39a82274883fe0d6 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Tue, 21 Apr 2026 11:48:17 -0400 Subject: [PATCH 268/289] Fix test setup and update mvn test command instructions --- AGENTS.md | 23 +++++++++++++++---- .../services/BrAPITrialServiceUnitTest.java | 7 +++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6a7b49b5a..ea7a5070e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,13 +27,28 @@ ## Validation - Use Maven for validation. -- Source envs from .env when running mvn commands +- Source envs from `.env` when running `mvn` commands by using `set -a`, `source .env`, and `set +a`. - Preferred targeted test command: - `mvn -Dtest=ClassName test --settings settings.xml` + ```sh + set -a + source .env + set +a + mvn -Dtest=ClassName test --settings settings.xml + ``` - Full test command: - `mvn test --settings settings.xml` + ```sh + set -a + source .env + set +a + mvn test --settings settings.xml + ``` - Full build without tests: - `mvn clean validate install -D maven.test.skip=true --settings settings.xml` + ```sh + set -a + source .env + set +a + mvn clean validate install -D maven.test.skip=true --settings settings.xml + ``` - Tests may require Docker, Testcontainers, and local services. If validation cannot run, say exactly what blocked it. ## API Change Rules diff --git a/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java b/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java index 0554319ec..0f6041874 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java @@ -30,6 +30,7 @@ import org.breedinginsight.model.Dataset; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; +import org.breedinginsight.model.delta.DeltaEntityFactory; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.TraitService; import org.breedinginsight.services.lock.DistributedLockService; @@ -78,6 +79,8 @@ class BrAPITrialServiceUnitTest { private final FileMappingUtil fileMappingUtil = mock(FileMappingUtil.class); private final DistributedLockService lockService = mock(DistributedLockService.class); private final DatasetService datasetService = mock(DatasetService.class); + private final DeltaEntityFactory deltaEntityFactory = mock(DeltaEntityFactory.class); + private final BrAPIObservationLevelService observationLevelService = mock(BrAPIObservationLevelService.class); private BrAPITrialService service; private Program program; @@ -102,7 +105,9 @@ void setup() { germplasmDAO, fileMappingUtil, lockService, - datasetService + datasetService, + deltaEntityFactory, + observationLevelService ); program = new Program(); From 5c00b21569947df68b3c798b28e94b73e28e74d0 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 21 Apr 2026 17:44:35 +0000 Subject: [PATCH 269/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1dcd18a63..a18488e0c 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1121 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/4d0add514566148e3062708dd15f12eb424b38e7 \ No newline at end of file +version=v1.3.0+1123 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/b742cfe4fdf5eded5041d6c79d1e7e94651feff9 \ No newline at end of file From 43ad57212a4e7fbb09252cf73c5bb6a8993c3f6e Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Wed, 22 Apr 2026 13:33:29 -0400 Subject: [PATCH 270/289] Changed to proper bi-api data response format --- .../api/v1/controller/ExperimentController.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index 5ee72f3a2..c846fb17d 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -178,7 +178,7 @@ public HttpResponse>> getDatasets( @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/recommended-sub-entity-dataset-names") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) @Produces(MediaType.APPLICATION_JSON) - public HttpResponse>> getRecommendedSubEntityDatasetNames( + public HttpResponse>> getRecommendedSubEntityDatasetNames( @PathVariable("programId") UUID programId, @PathVariable("experimentId") UUID experimentId) { try { @@ -187,7 +187,15 @@ public HttpResponse>> getRecommendedSubEntityDatasetNames( return HttpResponse.status(HttpStatus.NOT_FOUND, "Program does not exist"); } - Response> response = new Response<>(experimentService.getRecommendedSubEntityDatasetNames(programOptional.get(), experimentId)); + List recommendedNames = experimentService.getRecommendedSubEntityDatasetNames(programOptional.get(), experimentId); + + List metadataStatus = new ArrayList<>(); + metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); + //TODO: paging if needed, unlikely to get very large + Pagination pagination = new Pagination(recommendedNames.size(), recommendedNames.size(), 1, 0); + Metadata metadata = new Metadata(pagination, metadataStatus); + + Response> response = new Response<>(metadata, new DataResponse<>(recommendedNames)); return HttpResponse.ok(response); } catch (DoesNotExistException e) { log.info(e.getMessage()); From 8edf445893ec39c8f0ccb40cb06a7c8b6d60429b Mon Sep 17 00:00:00 2001 From: "dr.phillips" Date: Wed, 22 Apr 2026 13:59:23 -0400 Subject: [PATCH 271/289] [BI-2825] added 'sunflower' to species and crop tables --- src/main/resources/brapi/sql/R__species.sql | 2 +- .../migration/V1.35.0__add_sunflower_to_species.sql | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 src/main/resources/db/migration/V1.35.0__add_sunflower_to_species.sql diff --git a/src/main/resources/brapi/sql/R__species.sql b/src/main/resources/brapi/sql/R__species.sql index d5d3d62ab..150c74f7e 100644 --- a/src/main/resources/brapi/sql/R__species.sql +++ b/src/main/resources/brapi/sql/R__species.sql @@ -41,7 +41,7 @@ BEGIN ('Strawberry'), ('Honey Bee'), ('Pecan'), ('Lettuce'), ('Cotton'), ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), ('Red Clover'), ('Potato'), ('Blackberry'), - ('Raspberry'), ('Sugar Beet'), ('Coffee') + ('Raspberry'), ('Sugar Beet'), ('Coffee'), ('Sunflower') ) AS src(crop_name) ON CONFLICT (id) DO -- want case changes or space changes to overwrite existing diff --git a/src/main/resources/db/migration/V1.35.0__add_sunflower_to_species.sql b/src/main/resources/db/migration/V1.35.0__add_sunflower_to_species.sql new file mode 100644 index 000000000..7f456bf03 --- /dev/null +++ b/src/main/resources/db/migration/V1.35.0__add_sunflower_to_species.sql @@ -0,0 +1,12 @@ +DO $$ +DECLARE +user_id UUID; +BEGIN + +user_id := (SELECT id FROM bi_user WHERE name = 'system'); + +-- just putting blank strings in for descriptions until later date, can update if needed +INSERT INTO species (common_name, description, created_by, updated_by) +VALUES + ('Sunflower', '', user_id, user_id) ON CONFLICT DO NOTHING; +END $$; From 24bed04e26733bfc40ec5f8c7c8b8898388981aa Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Wed, 22 Apr 2026 15:19:00 -0400 Subject: [PATCH 272/289] Update tests for change in endpoint response --- .../api/v1/controller/ExperimentControllerIntegrationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 5677dc0b0..7c5c2a872 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -1006,7 +1006,8 @@ private List getRecommendedSubEntityDatasetNames(Program targetProgram, JsonArray result = JsonParser.parseString(Objects.requireNonNull(response.body())) .getAsJsonObject() - .getAsJsonArray("result"); + .getAsJsonObject("result") + .getAsJsonArray("data"); List recommendedNames = new ArrayList<>(); result.forEach(name -> recommendedNames.add(name.getAsString())); From a6e9e0f6224a9ef452420ee27fec5b61320bcda6 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 22 Apr 2026 21:03:43 +0000 Subject: [PATCH 273/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index a18488e0c..5c6798799 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1123 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/b742cfe4fdf5eded5041d6c79d1e7e94651feff9 \ No newline at end of file +version=v1.3.0+1125 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/379623e26f2efe18de83e90f5f40f2ec90f08c70 \ No newline at end of file From c0319b463f08e1b88509692029cd3db381a1e583 Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Thu, 23 Apr 2026 10:57:47 -0400 Subject: [PATCH 274/289] Updated test species.sql --- src/test/resources/sql/brapi/species.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/sql/brapi/species.sql b/src/test/resources/sql/brapi/species.sql index 2b8fcf26d..acb173fb2 100644 --- a/src/test/resources/sql/brapi/species.sql +++ b/src/test/resources/sql/brapi/species.sql @@ -36,7 +36,7 @@ FROM (VALUES ('Strawberry'), ('Honey Bee'), ('Pecan'), ('Lettuce'), ('Cotton'), ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), ('Red Clover'), ('Potato'), ('Blackberry'), - ('Raspberry'), ('Sugar Beet'), ('Coffee') + ('Raspberry'), ('Sugar Beet'), ('Coffee'), ('Sunflower') ) AS src(crop_name) ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name From 510b3703b84ff9b9cbefe4bdfec5dd9c4e31ab99 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 23 Apr 2026 15:17:16 +0000 Subject: [PATCH 275/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 5c6798799..37bf4112d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1125 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/379623e26f2efe18de83e90f5f40f2ec90f08c70 \ No newline at end of file +version=v1.3.0+1129 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/78441e199e08a0f02012108b0fa04548bad530f9 \ No newline at end of file From 5acfaa66e607c44c2962063ad89f35b16591ec6a Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Fri, 24 Apr 2026 17:09:29 -0400 Subject: [PATCH 276/289] Added more detailed testing guidance --- AGENTS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index ea7a5070e..aefcee485 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,6 +61,12 @@ ## Testing Guidance - Prefer targeted tests for the changed area before suggesting a full test run. +- Prefer high-signal tests over broad or repetitive test coverage. +- Do not add unit tests by default for straightforward changes. +- Add unit tests selectively for hard-to-reproduce bugs, complex logic regressions, or behavior that is difficult or expensive to validate through integration tests. +- Prefer integration or endpoint tests for user-facing and API behavior changes. +- Avoid tests that mainly lock in implementation details or create disproportionate maintenance burden. +- When adding regression coverage, prefer the smallest number of tests that gives confidence in the fix. - For controller or API changes, look first under `src/test/java/org/breedinginsight/api` and `src/test/java/org/breedinginsight/brapi`. - For importer work, check `src/test/java/org/breedinginsight/brapps/importer`. From 5383d0481de8d1718366c4d7f39a76bd6c29c1bc Mon Sep 17 00:00:00 2001 From: nickpalladino Date: Mon, 27 Apr 2026 16:09:03 -0400 Subject: [PATCH 277/289] Clean up tests --- .../brapi/v2/services/BrAPITrialService.java | 22 +++- .../services/BrAPITrialServiceUnitTest.java | 115 +----------------- 2 files changed, 21 insertions(+), 116 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index f991b0e0d..bc1ce1ce0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -963,11 +963,22 @@ private Map getYearByStudyDbId(Collection studies, /** * Bulk-loads studies by studyDbId so dataset retrieval can avoid per-observation-unit * environment lookups and reuse the shared year resolution flow. + * + * Missing studies are rejected here so later Env Year writes can reserve their + * failures for actual year resolution problems. */ private Map getYearByStudyDbIds(Collection studyDbIds, Program program) throws ApiException, DoesNotExistException { - return getYearByStudyDbId( - studyDAO.getStudiesByStudyDbId(studyDbIds, program), - program.getId()); + List studies = studyDAO.getStudiesByStudyDbId(studyDbIds, program); + Set resolvedStudyDbIds = studies.stream() + .map(BrAPIStudy::getStudyDbId) + .collect(Collectors.toSet()); + for (String studyDbId : studyDbIds) { + if (!resolvedStudyDbIds.contains(studyDbId)) { + throw new DoesNotExistException(String.format("Study DbId '%s' not found.", studyDbId)); + } + } + + return getYearByStudyDbId(studies, program.getId()); } /** @@ -978,7 +989,7 @@ private void addEnvYearToObservationUnits(Collection obser for (BrAPIObservationUnit observationUnit : observationUnits) { Integer year = yearByStudyDbId.get(observationUnit.getStudyDbId()); if (year == null) { - throw new DoesNotExistException(String.format("Study DbId '%s' not found.", observationUnit.getStudyDbId())); + throw new DoesNotExistException(String.format("Env Year not found for Study DbId = '%s'.", observationUnit.getStudyDbId())); } observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.ENV_YEAR, year); } @@ -1004,6 +1015,9 @@ private Integer getYearForStudy(BrAPIStudy study, UUID programId, Map observationUnits = new ArrayList<>(List.of( - createObservationUnit("ou-db-1", "plot-1"), - createObservationUnit("ou-db-2", "plot-2"), - createObservationUnit("ou-db-3", "plot-3") - )); - - when(trialDAO.getTrialsByExperimentIds(eq(List.of(UUID.fromString("11111111-1111-1111-1111-111111111111"))), eq(program))) - .thenReturn(List.of(experiment)); - when(studyDAO.getStudiesByExperimentID(eq(UUID.fromString("11111111-1111-1111-1111-111111111111")), eq(program))) - .thenReturn(List.of(study)); - when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); - when(observationUnitDAO.getObservationUnitsForDataset(EXPORT_DATASET_ID, program)).thenReturn(observationUnits); - when(listDAO.getListsByTypeAndExternalRef(any(), eq(program.getId()), any(), any())).thenReturn(Collections.emptyList()); - when(observationDAO.getObservationsByObservationUnits(anyCollection(), eq(program))).thenReturn(Collections.emptyList()); - when(germplasmDAO.getGermplasmsByDBID(anyList(), eq(program.getId()))).thenReturn(List.of(germplasm)); - - DownloadFile downloadFile = service.exportObservations(program, UUID.fromString("11111111-1111-1111-1111-111111111111"), params); - - Table exportTable = FileUtil.parseTableFromCsv(new ByteArrayInputStream(downloadFile.getStreamedFile().getInputStream().readAllBytes())); - assertEquals(3, exportTable.rowCount()); - assertEquals(List.of(2023, 2023, 2023), exportTable.intColumn(Columns.ENV_YEAR).asList()); - verify(seasonDAO, times(1)).getSeasonById("season-1", program.getId()); - } - - @Test - void exportObservationsFetchesYearsOnlyForRequestedEnvironments() throws Exception { - ExperimentExportQuery params = exportQuery(EXPORT_DATASET_ID); - setField(params, "environments", ENVIRONMENT_ID); - List observationUnits = List.of(createObservationUnit("ou-db-1", "plot-1")); - BrAPIStudy unrelatedStudy = new BrAPIStudy(); - unrelatedStudy.setStudyDbId("study-2"); - unrelatedStudy.setStudyName("Environment 2"); - unrelatedStudy.setLocationName("Location 2"); - unrelatedStudy.setSeasons(Collections.emptyList()); - unrelatedStudy.setExternalReferences(List.of(createExternalReference( - String.format("%s/%s", REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName()), - SECOND_ENVIRONMENT_ID - ))); - - when(trialDAO.getTrialsByExperimentIds(eq(List.of(UUID.fromString("11111111-1111-1111-1111-111111111111"))), eq(program))) - .thenReturn(List.of(experiment)); - when(studyDAO.getStudiesByExperimentID(eq(UUID.fromString("11111111-1111-1111-1111-111111111111")), eq(program))) - .thenReturn(List.of(study, unrelatedStudy)); - when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); - when(observationUnitDAO.getObservationUnitsForDatasetAndEnvs(EXPORT_DATASET_ID, List.of(ENVIRONMENT_ID), program)).thenReturn(observationUnits); - when(listDAO.getListsByTypeAndExternalRef(any(), eq(program.getId()), any(), any())).thenReturn(Collections.emptyList()); - when(observationDAO.getObservationsByObservationUnits(anyCollection(), eq(program))).thenReturn(Collections.emptyList()); - when(germplasmDAO.getGermplasmsByDBID(anyList(), eq(program.getId()))).thenReturn(List.of(germplasm)); - - DownloadFile downloadFile = service.exportObservations(program, UUID.fromString("11111111-1111-1111-1111-111111111111"), params); - - Table exportTable = FileUtil.parseTableFromCsv(new ByteArrayInputStream(downloadFile.getStreamedFile().getInputStream().readAllBytes())); - assertEquals(1, exportTable.rowCount()); - assertEquals(List.of(2023), exportTable.intColumn(Columns.ENV_YEAR).asList()); - verify(seasonDAO, times(1)).getSeasonById("season-1", program.getId()); - } - - @Test - void exportObservationsWritesDistinctYearsForMultipleEnvironments() throws Exception { + void exportObservationsFetchesSeasonOncePerDistinctSeasonAndWritesEnvYears() throws Exception { ExperimentExportQuery params = exportQuery(EXPORT_DATASET_ID); BrAPIStudy secondStudy = new BrAPIStudy(); secondStudy.setStudyDbId("study-2"); @@ -245,55 +184,7 @@ void exportObservationsWritesDistinctYearsForMultipleEnvironments() throws Excep } @Test - void exportObservationsThrowsWhenSeasonYearIsNull() throws Exception { - // This PR intentionally hardens null season years because experiment import - // requires Env Year when creating experiments/environments. - ExperimentExportQuery params = exportQuery(EXPORT_DATASET_ID); - List observationUnits = List.of(createObservationUnit("ou-db-1", "plot-1")); - season.setYear(null); - - when(trialDAO.getTrialsByExperimentIds(eq(List.of(UUID.fromString("11111111-1111-1111-1111-111111111111"))), eq(program))) - .thenReturn(List.of(experiment)); - when(studyDAO.getStudiesByExperimentID(eq(UUID.fromString("11111111-1111-1111-1111-111111111111")), eq(program))) - .thenReturn(List.of(study)); - when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); - when(observationUnitDAO.getObservationUnitsForDataset(EXPORT_DATASET_ID, program)).thenReturn(observationUnits); - when(listDAO.getListsByTypeAndExternalRef(any(), eq(program.getId()), any(), any())).thenReturn(Collections.emptyList()); - when(observationDAO.getObservationsByObservationUnits(anyCollection(), eq(program))).thenReturn(Collections.emptyList()); - when(germplasmDAO.getGermplasmsByDBID(anyList(), eq(program.getId()))).thenReturn(List.of(germplasm)); - - DoesNotExistException exception = assertThrows(DoesNotExistException.class, - () -> service.exportObservations(program, UUID.fromString("11111111-1111-1111-1111-111111111111"), params)); - - assertEquals("Env Year not found for Study DbId = 'study-1'.", exception.getMessage()); - } - - @Test - void getDatasetDataUsesStudyDbIdsAndCachesSeasonByStudy() throws Exception { - List observationUnits = new ArrayList<>(List.of( - createObservationUnit("ou-db-1", "plot-1"), - createObservationUnit("ou-db-2", "plot-2") - )); - - when(observationUnitDAO.getObservationUnitsForDataset(DATASET_ID, program)).thenReturn(observationUnits); - when(studyDAO.getStudiesByStudyDbId(eq(Set.of("study-1")), eq(program))).thenReturn(List.of(study)); - when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); - when(listDAO.getListsByTypeAndExternalRef(any(), eq(program.getId()), any(), any())).thenReturn(Collections.emptyList()); - when(observationDAO.getObservationsByObservationUnitsAndVariables(anyList(), eq(Collections.emptyList()), eq(program))) - .thenReturn(Collections.emptyList()); - - Dataset dataset = service.getDatasetData(program, UUID.randomUUID(), UUID.fromString(DATASET_ID), false); - - assertEquals(2, dataset.observationUnits.size()); - dataset.observationUnits.forEach(ou -> - assertEquals(2023, ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENV_YEAR).getAsInt())); - verify(studyDAO, times(1)).getStudiesByStudyDbId(eq(Set.of("study-1")), eq(program)); - verify(studyDAO, never()).getStudyByEnvironmentId(eq(UUID.fromString(ENVIRONMENT_ID)), eq(program)); - verify(seasonDAO, times(1)).getSeasonById("season-1", program.getId()); - } - - @Test - void getDatasetDataWritesDistinctYearsForMultipleStudies() throws Exception { + void getDatasetDataFetchesSeasonOncePerDistinctSeasonAndWritesEnvYears() throws Exception { BrAPIStudy secondStudy = new BrAPIStudy(); secondStudy.setStudyDbId("study-2"); secondStudy.setStudyName("Environment 2"); @@ -346,7 +237,7 @@ void getDatasetDataThrowsWhenSeasonYearIsNull() throws Exception { DoesNotExistException exception = assertThrows(DoesNotExistException.class, () -> service.getDatasetData(program, UUID.randomUUID(), UUID.fromString(DATASET_ID), false)); - assertEquals("Study DbId 'study-1' not found.", exception.getMessage()); + assertEquals("Env Year not found for Study DbId = 'study-1'.", exception.getMessage()); } private ExperimentExportQuery exportQuery(String datasetId) throws Exception { From 3950562c4d46af75d9d83116e8923985518105ee Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 28 Apr 2026 17:56:52 +0000 Subject: [PATCH 278/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 37bf4112d..1215d8b19 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1129 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/78441e199e08a0f02012108b0fa04548bad530f9 \ No newline at end of file +version=v1.3.0+1133 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/780602f094f3ce993622fbf637505b5e32a0dd78 \ No newline at end of file From 3371a8c532d6b564d94ce88a8c72cf0adb54ca9d Mon Sep 17 00:00:00 2001 From: Keerthi Humsika Kattamudi Date: Tue, 12 May 2026 15:40:12 -0700 Subject: [PATCH 279/289] BI-2860: Made changes to the end point to limit the access. --- .../v1/controller/ExperimentController.java | 2 +- .../ExperimentControllerIntegrationTest.java | 50 ++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index c846fb17d..988402889 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -122,7 +122,7 @@ public HttpResponse> getDatasetData( * @return An HttpResponse with a Response object containing the newly created Dataset. */ @Post("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/dataset") - @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) + @ProgramSecured(roles = {ProgramSecuredRole.PROGRAM_ADMIN}) @Produces(MediaType.APPLICATION_JSON) public HttpResponse> createSubEntityDataset( @PathVariable("programId") UUID programId, diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 7c5c2a872..506c89cef 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -10,7 +10,7 @@ import io.micronaut.http.client.annotation.Client; import io.micronaut.http.client.exceptions.HttpClientResponseException; import io.micronaut.http.netty.cookies.NettyCookie; -import io.micronaut.test.annotation.MicronautTest; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; import org.apache.commons.io.FileUtils; @@ -464,6 +464,54 @@ public void createSubEntityDatasetAllowsExpUnitNameUsedInOtherExperiment() throw assertEquals(HttpStatus.OK, response.getStatus()); } + @Test + public void createSubEntityDatasetForbiddenForExperimentalCollaborator() throws Exception { + // add otherTestUser to the program as an Experimental Collaborator + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + dsl.execute(securityFp.get("InsertProgramRolesExperimentalCollaborator"), otherTestUser.getId().toString(), program.getId()); + + // add that user to this experiment as a collaborator + JsonObject requestBody = new JsonObject(); + requestBody.addProperty("userId", otherTestUser.getId().toString()); + + Flowable> collaboratorCall = client.exchange( + POST(String.format("/programs/%s/experiments/%s/collaborators", program.getId().toString(), experimentId), requestBody.toString()) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + HttpResponse collaboratorResponse = collaboratorCall.blockingFirst(); + assertEquals(HttpStatus.OK, collaboratorResponse.getStatus()); + + JsonObject collaboratorResult = JsonParser.parseString(collaboratorResponse.body()).getAsJsonObject().getAsJsonObject("result"); + String collaboratorId = collaboratorResult.get("id").getAsString(); + + // collaborator should be blocked from creating sub-entity datasets directly + Flowable> call = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", program.getId(), experimentId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "other-registered-user")), + String.class + ); + + HttpClientResponseException e = assertThrows(HttpClientResponseException.class, call::blockingFirst); + assertEquals(HttpStatus.FORBIDDEN, e.getStatus()); + + // cleanup collaborator record + Flowable> deleteCall = client.exchange( + DELETE(String.format("/programs/%s/experiments/%s/collaborators/%s", program.getId().toString(), experimentId, collaboratorId)) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + HttpResponse deleteResponse = deleteCall.blockingFirst(); + assertEquals(HttpStatus.OK, deleteResponse.getStatus()); + + // cleanup program user role + dsl.execute(securityFp.get("DeleteProgramUser"), otherTestUser.getId().toString()); + } + @Test public void recommendedSubEntityDatasetNamesIncludeExpUnitNamesFromOtherExperiments() throws Exception { Program testProgram = createSeededProgram("Recommended Names"); From 055fc2b181cd30aeddaf8f353897426ddf327957 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 13 May 2026 18:38:44 +0000 Subject: [PATCH 280/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1215d8b19..fd16637df 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1133 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/780602f094f3ce993622fbf637505b5e32a0dd78 \ No newline at end of file +version=v1.3.0+1143 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/3aab236980401d087ddb6eb6f4c7e05783be5b84 \ No newline at end of file From 3613ad6e26fd38b58d59f8213cd3af3d64945fbc Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 20 May 2026 19:49:58 +0000 Subject: [PATCH 281/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 3c769284c..b440d42c4 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0 -versionInfo=https://github.com/Breeding-Insight/bi-api/releases/tag/v1.3.0 +version=v1.3.0+1149 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/503fda4b0d29813511c7123f69bdf32a4fba7bb3 From ae5302dd9fb6e60797a1ec32cdb5be2aebc34a66 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 20 May 2026 16:21:53 -0400 Subject: [PATCH 282/289] Update brapi-java-client version to 2.2.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 674e7ffea..1788b1604 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ 31.0.1-jre 4.9.3 4.3.1 - 2.2-SNAPSHOT + 2.2.0 2.11.0 2.2.1 From 2c928c0c4540ba6deeeb8e02f5b2130c2b4b3877 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 20 May 2026 20:22:04 +0000 Subject: [PATCH 283/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index b440d42c4..7bb1344b6 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1149 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/503fda4b0d29813511c7123f69bdf32a4fba7bb3 +version=v1.3.0+1151 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ae5302dd9fb6e60797a1ec32cdb5be2aebc34a66 From a0ced83fdce136189bf1a8482776e4cb700018f3 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 20 May 2026 16:49:11 -0400 Subject: [PATCH 284/289] Update brapi-java-client version to 2.2-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1788b1604..674e7ffea 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ 31.0.1-jre 4.9.3 4.3.1 - 2.2.0 + 2.2-SNAPSHOT 2.11.0 2.2.1 From 0e727756977be26589dc093e816e163d3dc8a489 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 20 May 2026 20:49:23 +0000 Subject: [PATCH 285/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 7bb1344b6..1d4bfda48 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1151 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ae5302dd9fb6e60797a1ec32cdb5be2aebc34a66 +version=v1.3.0+1153 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a0ced83fdce136189bf1a8482776e4cb700018f3 From 5844f9b01dca101d6eb4090ce03494b179d9e6c5 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 20 May 2026 17:10:40 -0400 Subject: [PATCH 286/289] Update brapi-java-client version to 2.2.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 674e7ffea..96e32029b 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ 31.0.1-jre 4.9.3 4.3.1 - 2.2-SNAPSHOT + 2.2.0-SNAPSHOT 2.11.0 2.2.1 From efd8b18eb5b49fed8759ef82d78144c51c200385 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 20 May 2026 21:10:51 +0000 Subject: [PATCH 287/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1d4bfda48..1857124f3 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1153 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a0ced83fdce136189bf1a8482776e4cb700018f3 +version=v1.3.0+1155 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/5844f9b01dca101d6eb4090ce03494b179d9e6c5 From 3a6361d30968c32ab97e175c53993991fcc429e9 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 20 May 2026 17:25:24 -0400 Subject: [PATCH 288/289] Update brapi-java-client version to 2.2.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 96e32029b..1788b1604 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ 31.0.1-jre 4.9.3 4.3.1 - 2.2.0-SNAPSHOT + 2.2.0 2.11.0 2.2.1 From 4346762d4ae168cb3301e2631d3d10ebd58b0a9a Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 20 May 2026 21:25:34 +0000 Subject: [PATCH 289/289] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1857124f3..d6ddc6d20 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.3.0+1155 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/5844f9b01dca101d6eb4090ce03494b179d9e6c5 +version=v1.3.0+1157 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/3a6361d30968c32ab97e175c53993991fcc429e9