diff --git a/.env.template b/.env.template index 1a481aae6..0e768adcd 100644 --- a/.env.template +++ b/.env.template @@ -3,6 +3,10 @@ USER_ID= GROUP_ID= +# GitHub OAuth variables. Only required if using GitHub as an OAuth provider. +GITHUB_OAUTH_CLIENT_ID= +GITHUB_OAUTH_CLIENT_SECRET= + ORCID_SANDBOX_AUTHENTICATION=use the Sandbox Orcid, false=>use the Production Orcid. Defaults to false.> # Authentication variables @@ -37,6 +41,7 @@ CACHE_BRAPI_FETCH_PAGE_SIZE=65000 # BrAPI Server Variables BRAPI_SERVER_PORT=8083 +BRAPI_DOCKER_IMAGE=breedinginsight/brapi-java-server:develop # Brapi Server Variables BRAPI_DEFAULT_URL=http://localhost:8083 @@ -73,4 +78,4 @@ STUDY_START_DELAY=10s TRIAL_START_DELAY=15s TRAIT_START_DELAY=20s OBSERVATION_START_DELAY=25s -OBSERVATION_UNIT_START_DELAY=30s \ No newline at end of file +OBSERVATION_UNIT_START_DELAY=30s diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9fcdc371a..2a083113d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -16,8 +16,6 @@ _Please include any details needed for reviewers to test this code_ - [ ] I have performed a self-review of my own code - [ ] I have tested my code and ensured it meets the acceptance criteria of the story -- [ ] I have tested that my code works with both the brapi-java-server and BreedBase -- [ ] I have create/modified unit tests to cover this change +- [ ] I have create/modified unit and/or integration tests to cover this change or tests are not applicable - [ ] I have commented my code, particularly in hard-to-understand areas -- [ ] I have made corresponding changes to documentation -- [ ] I have run TAF: _\_ +- [ ] I have either updated the source of truth or arranged for update with product owner if needed https://breedinginsight.atlassian.net/wiki/spaces/BI/pages/1559953409/Source+of+Truth diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0f77a41e0..3d848d159 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,12 +19,22 @@ name: maven build on: pull_request: - type: [opened, edited] + types: [opened, synchronize, edited] + workflow_dispatch: + inputs: + brapi_server_image: + description: 'BrAPI Server Docker Image' + required: false + default: 'breedinginsight/brapi-java-server:develop' + bi_api_branch: + description: 'BI API Branch' + required: false + default: 'develop' jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 services: postgres: @@ -38,11 +48,15 @@ jobs: options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v2 + - name: Checkout BI API + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.bi_api_branch || github.ref }} - name: Set up JDK 13 - uses: actions/setup-java@v1.4.3 + uses: actions/setup-java@v4 with: java-version: 13 + distribution: 'zulu' - name: Build with Maven run: mvn validate -B flyway:migrate clean install --file pom.xml --settings settings.xml @@ -59,4 +73,7 @@ jobs: JWT_SECRET: ${{ secrets.JWT_SECRET }} OAUTH_CLIENT_ID: 123abc OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf - BRAPI_REFERENCE_SOURCE: breedinginsight.org \ No newline at end of file + GITHUB_OAUTH_CLIENT_ID: 12345678901234567890 + GITHUB_OAUTH_CLIENT_SECRET: 1234567890123456789012345678901234567890 + BRAPI_REFERENCE_SOURCE: breedinginsight.org + BRAPI_DOCKER_IMAGE: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.brapi_server_image || 'breedinginsight/brapi-java-server:develop' }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 31d1bd0e6..a526b060e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,6 +66,8 @@ jobs: JWT_SECRET: ${{ secrets.JWT_SECRET }} OAUTH_CLIENT_ID: 123abc OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf + GITHUB_OAUTH_CLIENT_ID: 12345678901234567890 + GITHUB_OAUTH_CLIENT_SECRET: 1234567890123456789012345678901234567890 BRAPI_REFERENCE_SOURCE: breedinginsight.org - name: Login to Docker Hub uses: docker/login-action@v1 diff --git a/.github/workflows/versioner-develop.yml b/.github/workflows/versioner-develop.yml index 3d2dd8217..9822f045c 100644 --- a/.github/workflows/versioner-develop.yml +++ b/.github/workflows/versioner-develop.yml @@ -80,6 +80,8 @@ jobs: JWT_SECRET: ${{ secrets.JWT_SECRET }} OAUTH_CLIENT_ID: 123abc OAUTH_CLIENT_SECRET: asdfljkhalkbaldsfjasdfi238497098asdf + GITHUB_OAUTH_CLIENT_ID: 12345678901234567890 + GITHUB_OAUTH_CLIENT_SECRET: 1234567890123456789012345678901234567890 BRAPI_REFERENCE_SOURCE: breedinginsight.org - name: Login to Docker Hub uses: docker/login-action@v1 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..aefcee485 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,75 @@ +# AGENTS.md + +## Scope +- These instructions apply to the whole repository. +- If a subdirectory contains its own `AGENTS.md`, the more specific file wins for that subtree. + +## Repository Overview +- Java 13 Maven backend using Micronaut. +- Main application code: `src/main/java/org/breedinginsight` +- Tests: `src/test/java/org/breedinginsight` +- DB migrations: `src/main/resources/db/migration` +- API docs and related materials: `docs` +- Build config: `pom.xml` +- Docker and local setup: `README.md`, `docker-compose.yml` + +## Working Rules +- Keep changes narrow and task-focused. +- Prefer fixing root causes over adding one-off workarounds. +- Do not edit `.env`, `.env.test`, `.env.template`, or other secret-bearing config files unless explicitly asked. +- Do not modify `target/`, `.idea/`, or the large SQL dump files unless the task is specifically about them. +- Scope searches to relevant directories instead of scanning the entire repository when possible. +- Do not edit historical Flyway migrations unless explicitly requested. Add a new migration instead. + +## Files To Ignore By Default +- `target/` +- `.idea/` + +## Validation +- Use Maven for validation. +- Source envs from `.env` when running `mvn` commands by using `set -a`, `source .env`, and `set +a`. +- Preferred targeted test command: + ```sh + set -a + source .env + set +a + mvn -Dtest=ClassName test --settings settings.xml + ``` +- Full test command: + ```sh + set -a + source .env + set +a + mvn test --settings settings.xml + ``` +- Full build without tests: + ```sh + set -a + source .env + set +a + mvn clean validate install -D maven.test.skip=true --settings settings.xml + ``` +- Tests may require Docker, Testcontainers, and local services. If validation cannot run, say exactly what blocked it. + +## API Change Rules +- If endpoint behavior changes, do not update the relevant API docs or spec files in `docs`, they are no longer being maintained. +- If API code changes, add or update endpoint or integration tests. + +## Database Rules +- Schema changes belong in a new Flyway migration under `src/main/resources/db/migration`. +- If schema changes affect generated jOOQ code, run the documented generation flow. + +## Testing Guidance +- Prefer targeted tests for the changed area before suggesting a full test run. +- Prefer high-signal tests over broad or repetitive test coverage. +- Do not add unit tests by default for straightforward changes. +- Add unit tests selectively for hard-to-reproduce bugs, complex logic regressions, or behavior that is difficult or expensive to validate through integration tests. +- Prefer integration or endpoint tests for user-facing and API behavior changes. +- Avoid tests that mainly lock in implementation details or create disproportionate maintenance burden. +- When adding regression coverage, prefer the smallest number of tests that gives confidence in the fix. +- For controller or API changes, look first under `src/test/java/org/breedinginsight/api` and `src/test/java/org/breedinginsight/brapi`. +- For importer work, check `src/test/java/org/breedinginsight/brapps/importer`. + +## Response Expectations +- Summarize changed files, validation performed, and any remaining risks. +- If tests or docs updates were skipped, explain why. diff --git a/docker-compose.yml b/docker-compose.yml index efebf5fb6..f45227ea7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,8 @@ services: - API_INTERNAL_PORT=${API_INTERNAL_PORT} - API_INTERNAL_TEST_PORT=${API_INTERNAL_TEST_PORT} - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID} + - GITHUB_OAUTH_CLIENT_ID=${GITHUB_OAUTH_CLIENT_ID} + - GITHUB_OAUTH_CLIENT_SECRET=${GITHUB_OAUTH_CLIENT_SECRET} - JWT_DOMAIN=${JWT_DOMAIN} - DB_SERVER=${DB_SERVER} - DB_NAME=${DB_NAME} diff --git a/pom.xml b/pom.xml index fe89f2ca6..1788b1604 100644 --- a/pom.xml +++ b/pom.xml @@ -83,13 +83,13 @@ 1.14 1.9.0 3.12.0 - 1.16.3 + 1.21.4 7.7.3 31.0.1-jre 4.9.3 4.3.1 - 2.1-SNAPSHOT + 2.2.0 2.11.0 2.2.1 @@ -192,6 +192,11 @@ micronaut-inject compile + + io.micronaut + micronaut-http-client + compile + io.micronaut micronaut-validation @@ -536,6 +541,16 @@ ${maven.compiler.target} + + org.apache.maven.plugins + maven-surefire-plugin + + + + **/GigwaGenotypeServiceImplIntegrationTest.java + + + org.jooq jooq-codegen-maven @@ -619,6 +634,10 @@ jdbc:postgresql://${DB_SERVER}/${DB_NAME} ${DB_USER} ${DB_PASSWORD} + + filesystem:src/main/java/org/breedinginsight/db/migration + filesystem:src/main/resources/db/migration + diff --git a/settings.xml b/settings.xml index b5281bb65..9b5fb7aa5 100644 --- a/settings.xml +++ b/settings.xml @@ -52,7 +52,7 @@ github-fannypack FannyPack github repository - https://maven.pkg.github.com/Kowalski-IO/fannypack + https://maven.pkg.github.com/BrandonKowalski/fannypack true true diff --git a/src/main/java/org/breedinginsight/api/auth/AuthServiceLoginHandler.java b/src/main/java/org/breedinginsight/api/auth/AuthServiceLoginHandler.java index 3a25eae13..a03b21641 100644 --- a/src/main/java/org/breedinginsight/api/auth/AuthServiceLoginHandler.java +++ b/src/main/java/org/breedinginsight/api/auth/AuthServiceLoginHandler.java @@ -30,7 +30,6 @@ import io.micronaut.security.token.jwt.cookie.JwtCookieLoginHandler; import io.micronaut.security.token.jwt.generator.AccessRefreshTokenGenerator; import io.micronaut.security.token.jwt.generator.AccessTokenConfiguration; -import io.micronaut.security.token.jwt.generator.JwtGeneratorConfiguration; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.model.v1.auth.SignUpJWT; import org.breedinginsight.model.ProgramUser; @@ -83,7 +82,7 @@ public AuthServiceLoginHandler(JwtCookieConfiguration jwtCookieConfiguration, @Override public MutableHttpResponse loginSuccess(UserDetails userDetails, HttpRequest request) { - // Called when login to orcid is successful. + // Called when login to OAuth provider is successful. // Check if our login to our system is successful. if (request.getCookies().contains(accountTokenCookieName)) { Cookie accountTokenCookie = request.getCookies().get(accountTokenCookieName); @@ -124,7 +123,7 @@ public MutableHttpResponse loginSuccess(UserDetails userDetails, HttpRequest< private AuthenticatedUser getUserCredentials(UserDetails userDetails) throws AuthenticationException { - Optional user = userService.getByOrcid(userDetails.getUsername()); + Optional user = userService.getByOAuthId(userDetails.getUsername()); if (user.isPresent()) { if (user.get().getActive()) { @@ -159,9 +158,20 @@ public MutableHttpResponse loginFailed(AuthenticationResponse authenticationF } } + private String parseOAuthProvider(HttpRequest request) { + // The request path will be something like "/sso/success/github". + if (request.getPath().toLowerCase().contains("github")) { + return "github"; + } else { + // Default to ORCID. + return "orcid"; + } + } + private MutableHttpResponse newAccountCreationResponse(UserDetails userDetails, String accountToken, HttpRequest request) { - String orcid = userDetails.getUsername(); + String oAuthId = userDetails.getUsername(); + String oAuthProvider = parseOAuthProvider(request); SignUpJWT signUpJWT; try { signUpJWT = signUpJwtService.validateAndParseAccountSignUpJwt(accountToken); @@ -185,9 +195,9 @@ private MutableHttpResponse newAccountCreationResponse(UserDetails userDetails, } if (newUser.getAccountToken().equals(signUpJWT.getJwtId().toString())) { - // Assign orcid to that user + // Assign OAuth Id and provider to that user. try { - userService.updateOrcid(newUser.getId(), orcid); + userService.updateOAuthInfo(newUser.getId(), oAuthId, oAuthProvider); } catch (DoesNotExistException e) { MutableHttpResponse resp = HttpResponse.seeOther(URI.create(newAccountErrorUrl)); return resp; diff --git a/src/main/java/org/breedinginsight/api/auth/GithubApiClient.java b/src/main/java/org/breedinginsight/api/auth/GithubApiClient.java new file mode 100644 index 000000000..a25bb9f41 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/auth/GithubApiClient.java @@ -0,0 +1,32 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.auth; + +import io.micronaut.http.annotation.Get; +import io.micronaut.http.annotation.Header; +import io.micronaut.http.client.annotation.Client; +import io.reactivex.Flowable; + +@Header(name = "User-Agent", value = "Micronaut") +@Client("https://api.github.com") +public interface GithubApiClient { + + @Get("/user") + Flowable getUser(@Header("Authorization") String authorization); +} + diff --git a/src/main/java/org/breedinginsight/api/auth/GithubUser.java b/src/main/java/org/breedinginsight/api/auth/GithubUser.java new file mode 100644 index 000000000..a71dc64fc --- /dev/null +++ b/src/main/java/org/breedinginsight/api/auth/GithubUser.java @@ -0,0 +1,37 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.auth; + +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import io.micronaut.core.annotation.Introspected; +import lombok.Getter; + +@Introspected +@JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) +@Getter +public class GithubUser { + + private String id; + // The login will be the unique GitHub username. + private String login; + private String name; + private String email; + +} + diff --git a/src/main/java/org/breedinginsight/api/auth/GithubUserDetailsMapper.java b/src/main/java/org/breedinginsight/api/auth/GithubUserDetailsMapper.java new file mode 100644 index 000000000..02515f766 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/auth/GithubUserDetailsMapper.java @@ -0,0 +1,61 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.auth; + +import io.micronaut.core.annotation.Nullable; +import io.micronaut.core.async.publisher.Publishers; +import io.micronaut.security.authentication.AuthenticationResponse; +import io.micronaut.security.authentication.UserDetails; +import io.micronaut.security.oauth2.endpoint.authorization.state.State; +import io.micronaut.security.oauth2.endpoint.token.response.OauthUserDetailsMapper; +import io.micronaut.security.oauth2.endpoint.token.response.TokenResponse; +import lombok.extern.slf4j.Slf4j; +import org.reactivestreams.Publisher; + + +import javax.inject.Named; +import javax.inject.Singleton; +import java.util.Collections; +import java.util.List; + +@Slf4j +@Named("github") +@Singleton +class GithubUserDetailsMapper implements OauthUserDetailsMapper { + + private final GithubApiClient apiClient; + + GithubUserDetailsMapper(GithubApiClient apiClient) { + this.apiClient = apiClient; + } + + @Override + public Publisher createUserDetails(TokenResponse tokenResponse) { + return Publishers.just(new UnsupportedOperationException()); + } + + @Override + public Publisher createAuthenticationResponse(TokenResponse tokenResponse, @Nullable State state) { + return apiClient.getUser("token " + tokenResponse.getAccessToken()) + .map(user -> { + List roles = Collections.singletonList("ROLE_GITHUB"); + return new UserDetails(user.getLogin(), roles); + }); + } +} + diff --git a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java index 7fdd58347..d809d72fc 100644 --- a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java +++ b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java @@ -49,6 +49,18 @@ public void addError(Integer rowNumber, ValidationError validationError){ rowErrors.add(newRow); } + public boolean hasErrorAtCell(int rowIndex, String field) { + for (RowValidationErrors row: rowErrors) { + if (row.getRowIndex() == rowIndex){ + return row.getErrors() + .stream() + .anyMatch(error -> error.getField().equals(field)); + } + } + + return false; + } + public void merge(ValidationErrors validationErrors){ for (RowValidationErrors rowValidationErrors: validationErrors.getRowErrors()){ for (ValidationError validationError: rowValidationErrors.getErrors()) { diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index 8619a140a..988402889 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -28,7 +28,9 @@ import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.ProgramUserService; import org.breedinginsight.services.RoleService; +import org.breedinginsight.services.exceptions.AlreadyExistsException; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.CreationBusyException; import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; import javax.inject.Inject; @@ -120,7 +122,7 @@ public HttpResponse> getDatasetData( * @return An HttpResponse with a Response object containing the newly created Dataset. */ @Post("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/dataset") - @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) + @ProgramSecured(roles = {ProgramSecuredRole.PROGRAM_ADMIN}) @Produces(MediaType.APPLICATION_JSON) public HttpResponse> createSubEntityDataset( @PathVariable("programId") UUID programId, @@ -134,6 +136,12 @@ public HttpResponse> createSubEntityDataset( Response response = new Response(experimentService.createSubEntityDataset(programOptional.get(), experimentId, datasetRequest)); return HttpResponse.ok(response); + } catch (AlreadyExistsException e) { + log.info(e.getMessage()); + return HttpResponse.status(HttpStatus.CONFLICT, e.getMessage()); + } catch (CreationBusyException e) { + log.info(e.getMessage()); + return HttpResponse.status(HttpStatus.SERVICE_UNAVAILABLE, e.getMessage()); } catch (Exception e){ log.info(e.getMessage()); return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); @@ -167,6 +175,37 @@ public HttpResponse>> getDatasets( } + @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/recommended-sub-entity-dataset-names") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) + @Produces(MediaType.APPLICATION_JSON) + public HttpResponse>> getRecommendedSubEntityDatasetNames( + @PathVariable("programId") UUID programId, + @PathVariable("experimentId") UUID experimentId) { + try { + Optional programOptional = programService.getById(programId); + if (programOptional.isEmpty()) { + return HttpResponse.status(HttpStatus.NOT_FOUND, "Program does not exist"); + } + + List recommendedNames = experimentService.getRecommendedSubEntityDatasetNames(programOptional.get(), experimentId); + + List metadataStatus = new ArrayList<>(); + metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); + //TODO: paging if needed, unlikely to get very large + Pagination pagination = new Pagination(recommendedNames.size(), recommendedNames.size(), 1, 0); + Metadata metadata = new Metadata(pagination, metadataStatus); + + Response> response = new Response<>(metadata, new DataResponse<>(recommendedNames)); + return HttpResponse.ok(response); + } catch (DoesNotExistException e) { + log.info(e.getMessage()); + return HttpResponse.status(HttpStatus.NOT_FOUND, e.getMessage()); + } catch (Exception e) { + log.error("Error finding recommended sub-entity dataset names", e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error finding recommended sub-entity dataset names"); + } + } + /** * Adds a record to the experiment_program_user_role table * @param programId The UUID of the program diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 611a5e5fb..010bb15e9 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -36,7 +36,6 @@ public final class BrAPIAdditionalInfoFields { public static final String GERMPLASM_BREEDING_METHOD = "breedingMethod"; public static final String CREATED_DATE = "createdDate"; public static final String DEFAULT_OBSERVATION_LEVEL = "defaultObservationLevel"; - public static final String OBSERVATION_LEVEL = "observationLevel"; public static final String RTK = "rtk"; public static final String EXPERIMENT_TYPE = "experimentType"; public static final String EXPERIMENT_NUMBER = "experimentNumber"; @@ -47,7 +46,6 @@ public final class BrAPIAdditionalInfoFields { public static final String DATASETS = "datasets"; public static final String FEMALE_PARENT_UNKNOWN = "femaleParentUnknown"; public static final String MALE_PARENT_UNKNOWN = "maleParentUnknown"; - public static final String TREATMENTS = "treatments"; public static final String GID = "gid"; public static final String CHANGELOG = "changeLog"; public static final String ENV_YEAR = "envYear"; @@ -57,4 +55,5 @@ public final class BrAPIAdditionalInfoFields { public static final String OBS_UNIT_ID = "obsUnitID"; public static final String GERMPLASM_NAME = "germplasmName"; public static final String SUBMISSION_NAME = "submissionName"; + public static final String EXP_UNIT_ID = "expUnitID"; } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java index 5ed73df60..4f6cfdec7 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java @@ -43,6 +43,7 @@ import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; +import javax.inject.Singleton; import javax.validation.constraints.NotNull; import java.util.ArrayList; import java.util.Collections; @@ -50,6 +51,7 @@ import java.util.UUID; @Slf4j +@Singleton public class BrAPIListDAO { private ProgramDAO programDAO; diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java new file mode 100644 index 000000000..7cd38ea30 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java @@ -0,0 +1,139 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2.dao; + +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.phenotype.ObservationLevelNamesApi; +import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; +import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponse; +import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponseResult; +import org.brapi.v2.model.pheno.response.BrAPIObservationLevelSingleResponse; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.DatasetLevel; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.List; +import org.breedinginsight.utilities.Utilities; +import java.util.Optional; + +@Slf4j +@Singleton +public class BrAPIObservationLevelDAO { + + private final BrAPIEndpointProvider brAPIEndpointProvider; + private final ProgramDAO programDAO; + + @Inject + public BrAPIObservationLevelDAO(BrAPIEndpointProvider brAPIEndpointProvider, + ProgramDAO programDAO) { + this.brAPIEndpointProvider = brAPIEndpointProvider; + this.programDAO = programDAO; + } + + public BrAPIObservationUnitHierarchyLevel createLevelName(Program program, + String programDbId, + String levelName, + DatasetLevel levelOrder) throws ApiException { + ObservationLevelNamesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationLevelNamesApi.class); + + ApiResponse response; + + + BrAPIObservationUnitHierarchyLevel level = new BrAPIObservationUnitHierarchyLevel(); + + level.setLevelName(levelName.toLowerCase()); + level.setLevelOrder(levelOrder.getValue()); + level.setProgramDbId(programDbId); + + try { + response = api.observationLevelNamesPost(List.of(level)); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); + } + + return Optional.of(response) + .map(ApiResponse::getBody) + .map(BrAPIObservationLevelListResponse::getResult) + .map(BrAPIObservationLevelListResponseResult::getData) + .flatMap(data -> data.stream().findFirst()) + .orElseThrow(() -> new ApiException(String.format("BrAPI indicated level name [%s] was created but no levelNameDbId was returned upon its creation", levelName))); + } + + public List getObservationLevelNamesByProgramId(Program program, String programDbId) { + ObservationLevelNamesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationLevelNamesApi.class); + + ApiResponse response; + + int pageSize = 100; + + try { + response = api.observationLevelNamesGet(programDbId, + false, + 0, + pageSize); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); + } + + if (response.getBody().getMetadata().getPagination().getTotalCount() > 100) { + throw new InternalServerException(String.format("More level names exist than requested [%s]", pageSize)); + } + + return response.getBody().getResult().getData(); + } + + public List getGlobalObservationLevelNames(Program program) { + return getObservationLevelNamesByProgramId(program, null); + } + + public void deleteObservationLevelName(Program program, String levelDbId) { + ObservationLevelNamesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationLevelNamesApi.class); + + try { + api.observationLevelNameDbIdDelete(levelDbId); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); + } + } + + public BrAPIObservationUnitHierarchyLevel updateObservationLevelName(Program program, + String levelNameDbId, + BrAPIObservationUnitHierarchyLevel level) { + ObservationLevelNamesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationLevelNamesApi.class); + + ApiResponse response; + + try { + response = api.observationLevelNameDbIdPut(levelNameDbId, level); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); + } + + return response.getBody().getResult(); + } + +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index 9749bf093..76acb8076 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -176,7 +176,6 @@ public List createBrAPIObservationUnits(List> postFunction = () -> { - preprocessObservationUnits(brAPIObservationUnitList); List ous = brAPIDAOUtil.post(brAPIObservationUnitList, upload, api::observationunitsPost, importDAO::update); return processObservationUnitsForCache(ous, program, false); }; @@ -198,7 +197,6 @@ public List createBrAPIObservationUnits(List> postFunction = () -> { - preprocessObservationUnits(brAPIObservationUnitList); List ous = brAPIDAOUtil.post(brAPIObservationUnitList, api::observationunitsPost); return processObservationUnitsForCache(ous, program, false); }; @@ -387,15 +385,9 @@ private void processObservationUnits(Program program, List this.germplasmService.getGermplasm(program.getId()).forEach((germplasm -> germplasmByDbId.put(germplasm.getGermplasmDbId(), germplasm))); } - // if has treatments in additionalInfo, copy to treatments property for (BrAPIObservationUnit ou : brapiObservationUnits) { JsonObject additionalInfo = ou.getAdditionalInfo(); if (additionalInfo != null) { - JsonElement treatmentsElement = additionalInfo.get(BrAPIAdditionalInfoFields.TREATMENTS); - if (treatmentsElement != null) { - List treatments = gson.fromJson(treatmentsElement, treatmentlistType); - ou.setTreatments(treatments); - } if( withGID ){ BrAPIGermplasm germplasm = germplasmByDbId.get(ou.getGermplasmDbId()); ou.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, germplasm.getAccessionNumber()); @@ -432,14 +424,4 @@ private void processObservationUnits(Program program, List } } } - - private void preprocessObservationUnits(List brapiObservationUnits) { - // add treatments to additional info - for (BrAPIObservationUnit obsUnit : brapiObservationUnits) { - List treatments = obsUnit.getTreatments(); - if (treatments != null) { - obsUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.TREATMENTS, treatments); - } - } - } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationLevelService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationLevelService.java new file mode 100644 index 000000000..e1f3d9b98 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationLevelService.java @@ -0,0 +1,39 @@ +package org.breedinginsight.brapi.v2.services; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationLevelDAO; +import org.breedinginsight.model.DatasetLevel; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.List; + +@Slf4j +@Singleton +public class BrAPIObservationLevelService { + private final BrAPIObservationLevelDAO brAPIObservationLevelDAO; + + @Inject + public BrAPIObservationLevelService(BrAPIObservationLevelDAO brAPIObservationLevelDAO) { + this.brAPIObservationLevelDAO = brAPIObservationLevelDAO; + } + + public List getGlobalLevelNames(Program program) { + return brAPIObservationLevelDAO.getGlobalObservationLevelNames(program); + } + + public List getProgrammaticLevelNames(Program program, String brapiProgramDbId) { + return brAPIObservationLevelDAO.getObservationLevelNamesByProgramId(program, brapiProgramDbId); + } + + public BrAPIObservationUnitHierarchyLevel createObservationLevel(Program program, + String brapiProgramDbId, + String levelName, + DatasetLevel levelOrder) throws ApiException { + return brAPIObservationLevelDAO.createLevelName(program, brapiProgramDbId, levelName, levelOrder); + } + +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 6b8a3f233..bc1ce1ce0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -27,26 +27,30 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.FileMappingUtil; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.model.BrAPIConstants; import org.breedinginsight.model.Column; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; import org.breedinginsight.model.*; +import org.breedinginsight.model.delta.DeltaEntityFactory; +import org.breedinginsight.model.delta.Experiment; import org.breedinginsight.services.TraitService; +import org.breedinginsight.services.exceptions.AlreadyExistsException; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.CreationBusyException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; -import org.breedinginsight.utilities.DatasetUtil; -import org.breedinginsight.utilities.IntOrderComparator; -import org.breedinginsight.utilities.FileUtil; -import org.breedinginsight.utilities.Utilities; +import org.breedinginsight.utilities.*; import org.jetbrains.annotations.NotNull; +import org.breedinginsight.services.lock.DistributedLockService; import javax.inject.Inject; import javax.inject.Singleton; import java.io.IOException; import java.io.PipedInputStream; import java.io.PipedOutputStream; +import java.time.Duration; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.*; @@ -54,6 +58,9 @@ import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import java.util.concurrent.TimeoutException; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; @Slf4j @Singleton @@ -68,9 +75,14 @@ public class BrAPITrialService { private final BrAPIStudyDAO studyDAO; private final BrAPISeasonDAO seasonDAO; private final BrAPIObservationUnitDAO ouDAO; + private final BrAPIObservationLevelDAO observationLevelDAO; private final BrAPIGermplasmDAO germplasmDAO; private final FileMappingUtil fileMappingUtil; + private final DistributedLockService lockService; private static final String SHEET_NAME = "Data"; + private final DatasetService datasetService; + private final DeltaEntityFactory deltaEntityFactory; + private final BrAPIObservationLevelService observationLevelService; @Inject public BrAPITrialService(@Property(name = "brapi.server.reference-source") String referenceSource, @@ -82,8 +94,13 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin BrAPIStudyDAO studyDAO, BrAPISeasonDAO seasonDAO, BrAPIObservationUnitDAO ouDAO, + BrAPIObservationLevelDAO observationLevelDAO, BrAPIGermplasmDAO germplasmDAO, - FileMappingUtil fileMappingUtil) { + FileMappingUtil fileMappingUtil, + DistributedLockService lockService, + DatasetService datasetService, + DeltaEntityFactory deltaEntityFactory, + BrAPIObservationLevelService observationLevelService) { this.referenceSource = referenceSource; this.trialDAO = trialDAO; @@ -94,8 +111,13 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin this.studyDAO = studyDAO; this.seasonDAO = seasonDAO; this.ouDAO = ouDAO; + this.observationLevelDAO = observationLevelDAO; this.germplasmDAO = germplasmDAO; this.fileMappingUtil = fileMappingUtil; + this.lockService = lockService; + this.datasetService = datasetService; + this.deltaEntityFactory = deltaEntityFactory; + this.observationLevelService = observationLevelService; } public List getExperiments(UUID programId) throws ApiException, DoesNotExistException { @@ -136,6 +158,13 @@ private long countGermplasm(UUID programId, String trialDbId) throws ApiExceptio return obUnits.stream().map(BrAPIObservationUnit::getGermplasmDbId).distinct().count(); } + private BrAPIObservationUnitLevelRelationship getTopLevel(BrAPIObservationUnit ou) { + BrAPIObservationUnitLevelRelationship topLevel = ou.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(x -> (x.getLevelOrder() != null && x.getLevelOrder().equals(0))).findFirst().orElse(null); + return topLevel; + } + public DownloadFile exportObservations( Program program, UUID experimentId, @@ -147,14 +176,12 @@ public DownloadFile exportObservations( List obsVars = new ArrayList<>(); Map> rowByOUId = new HashMap<>(); Map studyByDbId = new HashMap<>(); + Map yearByStudyDbId = new HashMap<>(); Map studyDbIdByOUId = new HashMap<>(); List requestedEnvIds = StringUtils.isNotBlank(params.getEnvironments()) ? new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))) : new ArrayList<>(); FileType fileType = params.getFileExtension(); - // make columns present in all exports - List columns = ExperimentFileColumns.getOrderedColumns(); - // add columns for requested dataset obsvars and timestamps log.debug(logHash + ": fetching experiment for export"); BrAPITrial experiment = getExperiment(program, experimentId); @@ -185,6 +212,31 @@ public DownloadFile exportObservations( log.error(logHash + ": Error fetching observation units for a study by its DbId" + Utilities.generateApiExceptionLogMessage(err), err); } + yearByStudyDbId.putAll(getYearByStudyDbId(expStudies, program.getId())); + + boolean isSubObs = isSubEntityDataset(ous); + + List columns; + + //make columns present in all exports + if (isSubObs){ + columns = ExperimentFileColumns.getOrderedColumnsSubEntity(); + } else { + columns = ExperimentFileColumns.getOrderedColumns(); + } + + //add obsUnitID as dynamic column with observation level appended to header + if (isSubObs) { + //need to add top level obs unit ids as well + BrAPIObservationUnitLevelRelationship topLevel = getTopLevel(ous.get(0)); + if (topLevel != null) { + String topObservationLvl = StringUtils.capitalize(topLevel.getLevelName()); + columns = dynamicUpdateObsUnitIDLabel(columns, topObservationLvl); + } + } + String observationLvl = StringUtils.capitalize(ous.get(0).getObservationUnitPosition().getObservationLevel().getLevelName()); + columns = dynamicUpdateObsUnitIDLabel(columns, observationLvl); + if (params.getDatasetId() != null) { log.debug(logHash + ": fetching " + params.getDatasetId() + " dataset observation variables for export"); obsVars = getDatasetObsVars(params.getDatasetId(), program); @@ -213,7 +265,9 @@ public DownloadFile exportObservations( params.isIncludeTimestamps(), obsVars, studyDbIdByOUId, - programGermplasmByDbId + programGermplasmByDbId, + yearByStudyDbId, + isSubObs ); // make export rows for OUs without observations @@ -223,7 +277,7 @@ public DownloadFile exportObservations( // Map Observation Unit to the Study it belongs to. studyDbIdByOUId.put(ouId, ou.getStudyDbId()); if (!rowByOUId.containsKey(ouId)) { - rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId)); + rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId, yearByStudyDbId, isSubObs)); } } } @@ -233,9 +287,10 @@ public DownloadFile exportObservations( if (!requestedEnvIds.isEmpty()) { // This will hold a list of rows for each study, each list will become a separate file. Map>> rowsByStudyId = new HashMap<>(); + String obsUnitIDLabel = observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX; for (Map row: rowByOUId.values()) { - String studyId = studyDbIdByOUId.get((String)row.get(ExperimentObservation.Columns.OBS_UNIT_ID)); + String studyId = studyDbIdByOUId.get((String)row.get(obsUnitIDLabel)); // Initialize key with empty list if it is not present. if (!rowsByStudyId.containsKey(studyId)) { @@ -250,8 +305,7 @@ public DownloadFile exportObservations( List> rows = entry.getValue(); sortDefaultForExportRows(rows); StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, rows, fileType, SHEET_NAME); - // TODO: [BI-2183] remove hardcoded datasetName, use observation level. - String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName(), "Observation Dataset") + fileType.getExtension(); + String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName(), StringUtils.capitalize(observationLvl.toLowerCase())) + fileType.getExtension(); // Add to file list. files.add(new DownloadFile(name, streamedFile)); } @@ -263,7 +317,7 @@ public DownloadFile exportObservations( log.debug(logHash + ": zipping files for export"); // Zip, as there are multiple files. StreamedFile zipFile = zipFiles(files); - downloadFile = new DownloadFile(makeZipFileName(experiment, program), zipFile); + downloadFile = new DownloadFile(makeZipFileName(experiment, program, StringUtils.capitalize(observationLvl.toLowerCase())), zipFile); } } else { List> exportRows = new ArrayList<>(rowByOUId.values()); @@ -271,9 +325,8 @@ public DownloadFile exportObservations( // write export data to requested file format StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, exportRows, fileType, SHEET_NAME); // Set filename. - String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); - // TODO: [BI-2183] remove hardcoded datasetName, use observation level. - String fileName = makeFileName(experiment, program, envFilenameFragment, "Observation Dataset") + fileType.getExtension(); + String envFilenameFragment = params.getEnvironments() == null ? "All Env" : params.getEnvironments(); + String fileName = makeFileName(experiment, program, envFilenameFragment, StringUtils.capitalize(observationLvl.toLowerCase())) + fileType.getExtension(); downloadFile = new DownloadFile(fileName, streamedFile); } @@ -306,35 +359,25 @@ private StreamedFile zipFiles(List files) throws IOException { return new StreamedFile(in, new MediaType(MediaType.APPLICATION_OCTET_STREAM)); } + public List dynamicUpdateObsUnitIDLabel(List columns, String observationLvl){ + String dynamicLabel = observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX; + Column ObsUnitIDCol = new Column(dynamicLabel, Column.ColumnDataType.STRING); + columns.add(ObsUnitIDCol); + + return columns; + } + public Dataset getDatasetData(Program program, UUID experimentId, UUID datasetId, Boolean stats) throws ApiException, DoesNotExistException { log.debug("fetching dataset: " + datasetId + " for experiment: " + experimentId + ". including stats: " + stats); log.debug("fetching observationUnits for dataset: " + datasetId); List datasetOUs = ouDAO.getObservationUnitsForDataset(datasetId.toString(), program); - //Add years to the addition_info elements - //TODO yearByStudyDbId will no longer be needed, and should be removed, once the seasonDAO uses the redis cache (BI-2261). - Map yearByStudyDbId = new HashMap<>(); // used to prevent the same season from being fetched repeatedly. - for ( BrAPIObservationUnit ou: datasetOUs ) { - String environmentId = Utilities.getExternalReference(ou.getExternalReferences(), this.referenceSource, ExternalReferenceSource.STUDIES) - .orElseThrow( ()-> new DoesNotExistException("No BI external reference for STUDIES was found")) - .getReferenceId(); - if( !yearByStudyDbId.containsKey( environmentId )) { - // Get the Study and extract the year from its Season - BrAPIStudy study = studyDAO.getStudyByEnvironmentId(UUID.fromString(environmentId), program).orElseThrow( () -> new DoesNotExistException(String.format("Study Id '%s' not found.", environmentId)) ); - if(study.getSeasons().isEmpty()){ - throw new DoesNotExistException(String.format("No Seasons found in Study Id = '%s'.", environmentId)); - } - String seasonId = study.getSeasons().get(0); - BrAPISeason season = seasonDAO.getSeasonById(seasonId, program.getId()); - if(season==null){ - throw new DoesNotExistException(String.format("Seasons not found for Id = '%s'.", seasonId)); - } - Integer year = season.getYear(); - yearByStudyDbId.put(environmentId, year); - } - - ou.putAdditionalInfoItem(BrAPIAdditionalInfoFields.ENV_YEAR, yearByStudyDbId.get(environmentId)); - } + Map yearByStudyDbId = getYearByStudyDbIds( + datasetOUs.stream() + .map(BrAPIObservationUnit::getStudyDbId) + .collect(Collectors.toSet()), + program); + addEnvYearToObservationUnits(datasetOUs, yearByStudyDbId); log.debug("fetching dataset variables dataset: " + datasetId); List datasetObsVars = getDatasetObsVars(datasetId.toString(), program); @@ -377,69 +420,142 @@ public List getDatasetsMetadata(Program program, UUID experimen return datasets; } - public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEntityDatasetRequest request) throws ApiException, DoesNotExistException { - log.debug("creating sub-entity dataset: \"" + request.getName() + "\" for experiment: \"" + experimentId + "\" with: \"" + request.getRepeatedMeasures() + "\" repeated measures."); - UUID subEntityDatasetId = UUID.randomUUID(); - List subObsUnits = new ArrayList<>(); - BrAPITrial experiment = getExperiment(program, experimentId); - // Get top level dataset ObservationUnits. - DatasetMetadata topLevelDataset = DatasetUtil.getTopLevelDataset(experiment); - if (topLevelDataset == null) { - log.error("Experiment {} has no top level dataset.", experiment.getTrialDbId()); - throw new RuntimeException("Cannot create sub-entity dataset for experiment without top level dataset."); - } - - List expOUs = ouDAO.getObservationUnitsForDataset(topLevelDataset.getId().toString(), program); - for (BrAPIObservationUnit expUnit : expOUs) { - - // Get environment number from study. - String envSeqValue = studyDAO.getStudyByDbId(expUnit.getStudyDbId(), program).orElseThrow() - .getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER).getAsString(); - - for (int i=1; i<=request.getRepeatedMeasures(); i++) { - // Create subObsUnit and add to list. - subObsUnits.add( - createSubObservationUnit( - request.getName(), - Integer.toString(i), - program, - envSeqValue, - expUnit, - this.referenceSource, - subEntityDatasetId, - UUID.randomUUID() - ) + /** + * Returns list of recommended sub entity names based on observation levels for the program that exclude + * level names already used in the experiment and is deduplicated for same name at multiple levels + * @param program Program + * @param experimentId Experiment Id + * @return list of dataset name recommendations + * @throws DoesNotExistException If trial does not exist + * @throws ApiException If BrAPI trial retrieval fails + */ + public List getRecommendedSubEntityDatasetNames(Program program, UUID experimentId) throws DoesNotExistException, ApiException { + BrAPITrial experiment = trialDAO.getTrialById(program.getId(), experimentId).orElseThrow(() -> new DoesNotExistException("Trial does not exist")); + Experiment deltaExperiment = deltaEntityFactory.makeExperimentBean(experiment); + // set to eliminate possible duplicates like plant for exp unit and sub unit + Set currentExperimentDatasetNames = deltaExperiment.getDatasetsMetadata() + .stream() + .map(DatasetMetadata::getName) + .filter(Objects::nonNull) + .map(String::toLowerCase) + .collect(Collectors.toSet()); + + return getProgramObservationLevelNames(program).stream() + .filter(Objects::nonNull) + .map(String::toLowerCase) + .filter(name -> !currentExperimentDatasetNames.contains(name)) + .distinct() + .sorted() + .collect(Collectors.toList()); + } + + /** + * Creates sub-entity dataset + * TODO: Handle compensating transactions in event of failure. Currently brapi server does not support + * deleting observation units. Will need to add batch delete support for observation units before this + * can be done. + * + * @param program program object representing the program that the datasets belong to + * @param experimentId id of the experiment that the datasets are associated with + * @param request request body containing dataset name and repeated-measure count + * @return dataset metadata and data for the newly created sub-entity dataset + * @throws ApiException if the BrAPI server rejects a create request + * @throws DoesNotExistException if the experiment or required BrAPI entities are missing + * @throws AlreadyExistsException if a dataset with the requested name already exists in the experiment + * @throws CreationBusyException if the dataset creation lock cannot be acquired in time + */ + public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEntityDatasetRequest request) + throws ApiException, DoesNotExistException, AlreadyExistsException, CreationBusyException { + final String datasetName = request.getName().trim().toLowerCase(); + String lockKey = String.format("sub-entity-dataset:%s", experimentId); + try { + return lockService.withLock(lockKey, Duration.ofSeconds(30), Duration.ofMinutes(5), () -> { + log.debug("creating sub-entity dataset: \"{}\" for experiment: \"{}\" with: \"{}\" repeated measures.", datasetName, experimentId, request.getRepeatedMeasures()); + UUID subEntityDatasetId = UUID.randomUUID(); + List subObsUnits = new ArrayList<>(); + BrAPITrial experiment = getExperiment(program, experimentId); + DatasetMetadata topLevelDataset = DatasetUtil.getTopLevelDataset(experiment); + if (topLevelDataset == null) { + log.error("Experiment {} has no top level dataset.", experiment.getTrialDbId()); + throw new RuntimeException("Cannot create sub-entity dataset for experiment without top level dataset."); + } + + List existingDatasets = DatasetUtil.datasetsFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); + if (existingDatasets.stream().anyMatch(dataset -> dataset.getName().equalsIgnoreCase(datasetName))) { + throw new AlreadyExistsException("Dataset name already exists in this experiment"); + } + + String programBrapiDbId = program.getBrapiProgram() != null ? program.getBrapiProgram().getProgramDbId() : null; + + String subEntityLevelNameDbId = datasetService.getOrCreateLevelNameForDataset(program, + programBrapiDbId, + datasetName, + DatasetLevel.SUB_OBS_UNIT ); - } - } - List createdObservationUnits = observationUnitDAO.createBrAPIObservationUnits(subObsUnits, program.getId()); - // Add the new dataset metadata to the datasets array in the trial's additionalInfo. - DatasetMetadata subEntityDatasetMetadata = DatasetMetadata.builder() - .id(subEntityDatasetId) - .name(request.getName()) - .level(DatasetLevel.SUB_OBS_UNIT) - .build(); - List datasets = DatasetUtil.datasetsFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); - datasets.add(subEntityDatasetMetadata); - experiment.getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, DatasetUtil.jsonArrayFromDatasets(datasets)); - // Ask the DAO to persist the updated trial. - trialDAO.updateBrAPITrial(experiment.getTrialDbId(), experiment, program.getId()); + List expOUs = ouDAO.getObservationUnitsForDataset(topLevelDataset.getId().toString(), program); + for (BrAPIObservationUnit expUnit : expOUs) { + + String envSeqValue = studyDAO.getStudyByDbId(expUnit.getStudyDbId(), program).orElseThrow() + .getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER).getAsString(); + + for (int i=1; i<=request.getRepeatedMeasures(); i++) { + subObsUnits.add( + createSubObservationUnit( + Integer.toString(i), + program, + envSeqValue, + expUnit, + this.referenceSource, + subEntityDatasetId, + UUID.randomUUID(), + subEntityLevelNameDbId + ) + ); + } + } + + observationUnitDAO.createBrAPIObservationUnits(subObsUnits, program.getId()); + + DatasetMetadata subEntityDatasetMetadata = DatasetMetadata.builder() + .id(subEntityDatasetId) + .name(datasetName) + .level(DatasetLevel.SUB_OBS_UNIT) + .build(); - // Return the new dataset. - return getDatasetData(program, experimentId, subEntityDatasetId, false); + // Refresh experiment so we merge with the latest dataset metadata and avoid clobbering concurrent updates. + BrAPITrial latestExperiment = getExperiment(program, experimentId); + List datasets = DatasetUtil.datasetsFromJson(latestExperiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); + if (datasets.stream().anyMatch(dataset -> dataset.getName().equalsIgnoreCase(datasetName))) { + throw new AlreadyExistsException("Dataset name already exists in this experiment"); + } + datasets.add(subEntityDatasetMetadata); + latestExperiment.getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, DatasetUtil.jsonArrayFromDatasets(datasets)); + trialDAO.updateBrAPITrial(latestExperiment.getTrialDbId(), latestExperiment, program.getId()); + + datasetService.createBrAPIObsVarListForDataset(program, latestExperiment, subEntityDatasetMetadata); + + return getDatasetData(program, experimentId, subEntityDatasetId, false); + }); + } catch (TimeoutException e) { + throw new CreationBusyException("Dataset creation is busy, please retry"); + } catch (ApiException | DoesNotExistException | AlreadyExistsException | CreationBusyException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Unexpected error creating sub-entity dataset", e); + } } public BrAPIObservationUnit createSubObservationUnit( - String subEntityDatasetName, String subUnitId, Program program, String seqVal, BrAPIObservationUnit expUnit, String referenceSource, UUID datasetId, - UUID id + UUID id, + String subEntityLevelNameDbId ) { BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); @@ -485,17 +601,17 @@ public BrAPIObservationUnit createSubObservationUnit( } // Set treatment factors. - List treatmentFactors = new ArrayList<>(); - for (BrAPIObservationTreatment t : expUnit.getTreatments()) { - BrAPIObservationTreatment treatment = new BrAPIObservationTreatment(); - treatment.setFactor(t.getFactor()); - treatment.setModality(t.getModality()); - treatmentFactors.add(treatment); + if (!(expUnit.getTreatments() == null || expUnit.getTreatments().isEmpty())) { + List treatmentFactors = new ArrayList<>(); + for (BrAPIObservationTreatment t : expUnit.getTreatments()) { + BrAPIObservationTreatment treatment = new BrAPIObservationTreatment(); + treatment.setFactor(t.getFactor()); + treatment.setModality(t.getModality()); + treatmentFactors.add(treatment); + } + observationUnit.setTreatments(treatmentFactors); } - observationUnit.setTreatments(treatmentFactors); - // Put level in additional info: keep this in case we decide to rename levels in future. - observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, subEntityDatasetName); // Put RTK in additional info. JsonElement rtk = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.RTK); if (rtk != null) { @@ -515,46 +631,42 @@ public BrAPIObservationUnit createSubObservationUnit( // ObservationLevel entry for Sub-Obs Unit. BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); - // TODO: consider removing toLowerCase() after BI-2219 is implemented. - level.setLevelName(subEntityDatasetName.toLowerCase()); + level.setLevelNameDbId(subEntityLevelNameDbId); level.setLevelCode(Utilities.appendProgramKey(subUnitId, program.getKey(), seqVal)); - level.setLevelOrder(DatasetLevel.SUB_OBS_UNIT.getValue()); position.setObservationLevel(level); // ObservationLevelRelationships. List levelRelationships = new ArrayList<>(); - // ObservationLevelRelationships for block. - BrAPIObservationUnitLevelRelationship expBlockLevel = expUnit.getObservationUnitPosition() - .getObservationLevelRelationships().stream() - .filter(x -> x.getLevelName().equals(BrAPIConstants.REPLICATE.getValue())).findFirst().orElse(null); - if (expBlockLevel != null) { - BrAPIObservationUnitLevelRelationship blockLevel = new BrAPIObservationUnitLevelRelationship(); - blockLevel.setLevelName(expBlockLevel.getLevelName()); - blockLevel.setLevelCode(expBlockLevel.getLevelCode()); - blockLevel.setLevelOrder(expBlockLevel.getLevelOrder()); - levelRelationships.add(blockLevel); - } // ObservationLevelRelationships for rep. BrAPIObservationUnitLevelRelationship expRepLevel = expUnit.getObservationUnitPosition() .getObservationLevelRelationships().stream() - .filter(x -> x.getLevelName().equals(BrAPIConstants.BLOCK.getValue())).findFirst().orElse(null); + .filter(x -> x.getLevelName().equals(BrAPIConstants.REPLICATE.getValue())).findFirst().orElse(null); if (expRepLevel != null) { BrAPIObservationUnitLevelRelationship repLevel = new BrAPIObservationUnitLevelRelationship(); - repLevel.setLevelName(expRepLevel.getLevelName()); + repLevel.setLevelNameDbId(expRepLevel.getLevelNameDbId()); repLevel.setLevelCode(expRepLevel.getLevelCode()); - repLevel.setLevelOrder(expRepLevel.getLevelOrder()); levelRelationships.add(repLevel); } + // ObservationLevelRelationships for block. + BrAPIObservationUnitLevelRelationship expBlockLevel = expUnit.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(x -> x.getLevelName().equals(BrAPIConstants.BLOCK.getValue())).findFirst().orElse(null); + if (expBlockLevel != null) { + BrAPIObservationUnitLevelRelationship blockLevel = new BrAPIObservationUnitLevelRelationship(); + blockLevel.setLevelNameDbId(expBlockLevel.getLevelNameDbId()); + blockLevel.setLevelCode(expBlockLevel.getLevelCode()); + levelRelationships.add(blockLevel); + } // ObservationLevelRelationships for top-level Exp Unit linking. BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); - // TODO: consider removing toLowerCase() after BI-2219 is implemented. - expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString().toLowerCase()); + expUnitLevel.setLevelNameDbId(expUnit.getObservationUnitPosition().getObservationLevel().getLevelNameDbId()); String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId(); expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnitUUID, program.getKey(), seqVal)); - expUnitLevel.setLevelOrder(DatasetLevel.EXP_UNIT.getValue()); levelRelationships.add(expUnitLevel); position.setObservationLevelRelationships(levelRelationships); + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.EXP_UNIT_ID, expUnit.getObservationUnitName()); + // Set ObservationUnitPosition. observationUnit.setObservationUnitPosition(position); @@ -571,7 +683,9 @@ private void addBrAPIObsToRecords( boolean includeTimestamp, List obsVars, Map studyDbIdByOUId, - Map programGermplasmByDbId) throws ApiException, DoesNotExistException { + Map programGermplasmByDbId, + Map yearByStudyDbId, + boolean isSubObs) throws ApiException, DoesNotExistException { Map varByDbId = new HashMap<>(); obsVars.forEach(var -> varByDbId.put(var.getObservationVariableDbId(), var)); for (BrAPIObservation obs: dataset) { @@ -589,7 +703,7 @@ private void addBrAPIObsToRecords( } else { // otherwise make a new row - Map row = createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId); + Map row = createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId, yearByStudyDbId, isSubObs); addObsVarDataToRow(row, obs, includeTimestamp, var, program); rowByOUId.put(ouId, row); } @@ -702,12 +816,26 @@ public int deleteExperiment(Program program, UUID experimentId, boolean hard) th return existingObservations.size(); } + /** + * Builds the static export columns for a single observation unit. + * + * Env Year is resolved ahead of time and passed in so export generation and + * dataset retrieval share the same study/season lookup path + */ + private List getProgramObservationLevelNames(Program program) { + String programDbId = program.getBrapiProgram() != null ? program.getBrapiProgram().getProgramDbId() : null; + List levelNames = observationLevelService.getProgrammaticLevelNames(program, programDbId); + return levelNames.stream().map(BrAPIObservationUnitHierarchyLevel::getLevelName).collect(Collectors.toList()); + } + private Map createExportRow( BrAPITrial experiment, Program program, BrAPIObservationUnit ou, Map studyByDbId, - Map programGermplasmByDbId) throws ApiException, DoesNotExistException { + Map programGermplasmByDbId, + Map yearByStudyDbId, + boolean isSubEntity) throws ApiException, DoesNotExistException { HashMap row = new HashMap<>(); // get OU id, germplasm, and study @@ -718,7 +846,8 @@ private Map createExportRow( String ouId = ouXref.getReferenceID(); BrAPIGermplasm germplasm = Optional.ofNullable(programGermplasmByDbId.get(ou.getGermplasmDbId())) .orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service")); - BrAPIStudy study = studyByDbId.get(ou.getStudyDbId()); + BrAPIStudy study = Optional.ofNullable(studyByDbId.get(ou.getStudyDbId())) + .orElseThrow(() -> new DoesNotExistException(String.format("Study DbId '%s' not found.", ou.getStudyDbId()))); // make export row from BrAPI objects row.put(ExperimentObservation.Columns.GERMPLASM_NAME, Utilities.removeProgramKey(ou.getGermplasmName(), program.getKey(), germplasm.getAccessionNumber())); @@ -730,7 +859,7 @@ private Map createExportRow( row.put(ExperimentObservation.Columns.TEST_CHECK, testCheck); row.put(ExperimentObservation.Columns.EXP_TITLE, Utilities.removeProgramKey(experiment.getTrialName(), program.getKey())); row.put(ExperimentObservation.Columns.EXP_DESCRIPTION, experiment.getTrialDescription()); - row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); row.put(ExperimentObservation.Columns.ENV, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); @@ -754,9 +883,11 @@ private Map createExportRow( : additionalInfo.get(BrAPIAdditionalInfoFields.RTK).getAsString(); row.put(ExperimentObservation.Columns.RTK, rtk); - BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); - row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); - row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + // Treat a null season year as missing data. Experiment import requires Env Year, + // so a missing year here indicates unsupported upstream BrAPI data. + Integer year = Optional.ofNullable(yearByStudyDbId.get(study.getStudyDbId())) + .orElseThrow(() -> new DoesNotExistException(String.format("Env Year not found for Study DbId = '%s'.", study.getStudyDbId()))); + row.put(ExperimentObservation.Columns.ENV_YEAR, year); // get replicate number Optional repLevel = ou.getObservationUnitPosition() @@ -785,11 +916,114 @@ private Map createExportRow( } else { row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, null); } - row.put(ExperimentObservation.Columns.OBS_UNIT_ID, ouId); + + //Append observation level to obsUnitID + String observationLvl = StringUtils.capitalize(ou.getObservationUnitPosition().getObservationLevel().getLevelName()); + row.put(observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX, ouId); + + if (isSubEntity) { + BrAPIObservationUnitLevelRelationship topLevel = getTopLevel(ou); + + if (topLevel != null) { + String topLvlName = StringUtils.capitalize(topLevel.getLevelName()); + row.put(ExperimentObservation.Columns.EXP_UNIT, topLvlName); + + String topLvlOuId = Utilities.removeProgramKeyAndUnknownAdditionalData(topLevel.getLevelCode(), program.getKey()); + row.put(topLvlName + " " + OBSERVATION_UNIT_ID_SUFFIX, topLvlOuId); + } + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXP_UNIT_ID).getAsString()); + + row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, StringUtils.capitalize(ou.getObservationUnitPosition().getObservationLevel().getLevelName())); + row.put(ExperimentObservation.Columns.SUB_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + + } else { + row.put(ExperimentObservation.Columns.EXP_UNIT, StringUtils.capitalize(ou.getObservationUnitPosition().getObservationLevel().getLevelName())); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + } return row; } + /** + * Resolves Env Year once per study while caching repeated season lookups by season DbId. + * + * This keeps export and dataset retrieval on the same code path and avoids refetching + * a season for every observation unit in the same study. + */ + private Map getYearByStudyDbId(Collection studies, UUID programId) throws ApiException, DoesNotExistException { + Map yearByStudyDbId = new HashMap<>(); + Map yearBySeasonDbId = new HashMap<>(); + for (BrAPIStudy study : studies) { + yearByStudyDbId.put(study.getStudyDbId(), getYearForStudy(study, programId, yearBySeasonDbId)); + } + + return yearByStudyDbId; + } + + /** + * Bulk-loads studies by studyDbId so dataset retrieval can avoid per-observation-unit + * environment lookups and reuse the shared year resolution flow. + * + * Missing studies are rejected here so later Env Year writes can reserve their + * failures for actual year resolution problems. + */ + private Map getYearByStudyDbIds(Collection studyDbIds, Program program) throws ApiException, DoesNotExistException { + List studies = studyDAO.getStudiesByStudyDbId(studyDbIds, program); + Set resolvedStudyDbIds = studies.stream() + .map(BrAPIStudy::getStudyDbId) + .collect(Collectors.toSet()); + for (String studyDbId : studyDbIds) { + if (!resolvedStudyDbIds.contains(studyDbId)) { + throw new DoesNotExistException(String.format("Study DbId '%s' not found.", studyDbId)); + } + } + + return getYearByStudyDbId(studies, program.getId()); + } + + /** + * Adds the resolved Env Year to each observation unit for the dataset response. + */ + private void addEnvYearToObservationUnits(Collection observationUnits, Map yearByStudyDbId) throws DoesNotExistException { + // additionalinfo.envYear is for the frontend dataset view + for (BrAPIObservationUnit observationUnit : observationUnits) { + Integer year = yearByStudyDbId.get(observationUnit.getStudyDbId()); + if (year == null) { + throw new DoesNotExistException(String.format("Env Year not found for Study DbId = '%s'.", observationUnit.getStudyDbId())); + } + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.ENV_YEAR, year); + } + } + + /** + * Resolves the year for the study's first season reference. + * + * A season that exists but does not expose a numeric year is treated as invalid data. + * The experiment import workflow requires Env Year for created experiments/environments, + * so the stricter failure here is intentional + */ + private Integer getYearForStudy(BrAPIStudy study, UUID programId, Map yearBySeasonDbId) throws ApiException, DoesNotExistException { + if (study.getSeasons() == null || study.getSeasons().isEmpty()) { + throw new DoesNotExistException(String.format("No Seasons found in Study DbId = '%s'.", study.getStudyDbId())); + } + + String seasonId = study.getSeasons().get(0); + Integer year = yearBySeasonDbId.get(seasonId); + if (year == null) { + BrAPISeason season = seasonDAO.getSeasonById(seasonId, programId); + if (season == null) { + throw new DoesNotExistException(String.format("Seasons not found for Id = '%s'.", seasonId)); + } + year = season.getYear(); + if (year == null) { + throw new DoesNotExistException(String.format("Env Year not found for Study DbId = '%s'.", study.getStudyDbId())); + } + yearBySeasonDbId.put(seasonId, year); + } + + return year; + } + private String doubleToString(double val){ return Double.isNaN(val) ? null : String.valueOf( val ); } @@ -844,12 +1078,13 @@ private String makeFileName(BrAPITrial experiment, Program program, String envNa return Utilities.makePortableFilename(unsafeName); } - private String makeZipFileName(BrAPITrial experiment, Program program) { + private String makeZipFileName(BrAPITrial experiment, Program program, String datasetName) { // .zip DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_hh-mm-ssZ"); String timestamp = formatter.format(OffsetDateTime.now()); - String unsafeName = String.format("%s_%s.zip", + String unsafeName = String.format("%s_%s_%s.zip", Utilities.removeProgramKey(experiment.getTrialName(), program.getKey()), + datasetName, timestamp); // Make file name safe for all platforms. return Utilities.makePortableFilename(unsafeName); @@ -865,10 +1100,22 @@ private List filterDatasetByEnvironment( .collect(Collectors.toList()); } + private boolean isSubEntityDataset(List ous){ + return (ous.get(0).getObservationUnitPosition().getObservationLevelRelationships().size() > 2); + } + private void sortDefaultForObservationUnit(List ous) { Comparator studyNameComparator = Comparator.comparing(BrAPIObservationUnit::getStudyName, new IntOrderComparator()); - Comparator ouNameComparator = Comparator.comparing(BrAPIObservationUnit::getObservationUnitName, new IntOrderComparator()); - ous.sort( (studyNameComparator).thenComparing(ouNameComparator)); + + if (isSubEntityDataset(ous)) { + Comparator subUnitComparator = Comparator.comparing(BrAPIObservationUnit::getObservationUnitName, new IntOrderComparator()); + Comparator ouNameComparator = Comparator.comparing(row -> (row.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXP_UNIT_ID).toString()), new IntOrderComparator()); + ous.sort((studyNameComparator).thenComparing(ouNameComparator).thenComparing(subUnitComparator)); + } + else { + Comparator ouNameComparator = Comparator.comparing(BrAPIObservationUnit::getObservationUnitName, new IntOrderComparator()); + ous.sort((studyNameComparator).thenComparing(ouNameComparator)); + } } private void sortDefaultForExportRows(@NotNull List> exportRows) { @@ -876,7 +1123,9 @@ private void sortDefaultForExportRows(@NotNull List> exportR Comparator> expUnitIdComparator = Comparator.comparing(row -> (row.get(Columns.EXP_UNIT_ID).toString()), new IntOrderComparator()); - exportRows.sort(envComparator.thenComparing(expUnitIdComparator)); + Comparator> subUnitIdComparator = Comparator.comparing(row -> (row.get(Columns.SUB_UNIT_ID).toString()), new IntOrderComparator()); + + exportRows.sort(envComparator.thenComparing(expUnitIdComparator).thenComparing(subUnitIdComparator)); } public BrAPIStudy getEnvironment(Program program, UUID envId) throws ApiException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 8df8be09b..ddcb9d93b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -145,10 +145,6 @@ public class ExperimentObservation implements BrAPIImport { @ImportFieldMetadata(id = "treatmentFactors", name = Columns.TREATMENT_FACTORS, description = "Treatment factors in an experiment with applied variables, like fertilizer or water regimens.") private String treatmentFactors; - @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "ObsUnitID", name = Columns.OBS_UNIT_ID, description = "A database generated unique identifier for experimental observation units") - private String obsUnitID; - public BrAPITrial constructBrAPITrial(Program program, User user, boolean commit, String referenceSource, UUID id, String expSeqValue) { BrAPIProgram brapiProgram = program.getBrapiProgram(); BrAPITrial trial = new BrAPITrial(); @@ -232,23 +228,6 @@ public BrAPIStudy constructBrAPIStudy( return study; } - public BrAPIListDetails constructDatasetDetails( - String name, - UUID datasetId, - String referenceSourceBase, - Program program, String trialId) { - BrAPIListDetails dataSetDetails = new BrAPIListDetails(); - dataSetDetails.setListName(name); - dataSetDetails.setListType(BrAPIListTypes.OBSERVATIONVARIABLES); - dataSetDetails.setData(new ArrayList<>()); - dataSetDetails.putAdditionalInfoItem("datasetType", "observationDataset"); - List refs = new ArrayList<>(); - Utilities.addReference(refs, program.getId(), referenceSourceBase, ExternalReferenceSource.PROGRAMS); - Utilities.addReference(refs, UUID.fromString(trialId), referenceSourceBase, ExternalReferenceSource.TRIALS); - Utilities.addReference(refs, datasetId, referenceSourceBase, ExternalReferenceSource.DATASET); - dataSetDetails.setExternalReferences(refs); - return dataSetDetails; - } public BrAPIObservationUnit constructBrAPIObservationUnit( Program program, String seqVal, @@ -284,12 +263,10 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( // If expUnit is null, a validation error will be produced later on. if (getExpUnit() != null) { - // TODO: [BI-2219] BJTS only accepts hardcoded levels, need to handle dynamic levels. level.setLevelName(getExpUnit().toLowerCase()); // HACK: toLowerCase() is needed to match BJTS hardcoded levels. } level.setLevelCode(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); position.setObservationLevel(level); - observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getExpUnit()); // Exp Unit List levelRelationships = new ArrayList<>(); @@ -363,10 +340,6 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( observationUnit.setTreatments(List.of(treatment)); } - if (getObsUnitID() != null) { - observationUnit.setObservationUnitDbId(getObsUnitID()); - } - return observationUnit; } @@ -478,7 +451,6 @@ public static final class Columns { public static final String ELEVATION = "Elevation"; public static final String RTK = "RTK"; public static final String TREATMENT_FACTORS = "Treatment Factors"; - public static final String OBS_UNIT_ID = "ObsUnitID"; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 039c0b58c..2af599402 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -458,6 +458,13 @@ private void processFile(String workflowId, List finalBrAPIImportLi progress.setBody(JSONB.valueOf(json)); progress.setUpdatedBy(actingUser.getId()); importDAO.update(upload); + } catch (BadRequestException e) { + log.error(e.getMessage(), e); + ImportProgress progress = upload.getProgress(); + progress.setStatuscode((short) HttpStatus.BAD_REQUEST.getCode()); + progress.setMessage(e.getMessage()); + progress.setUpdatedBy(actingUser.getId()); + importDAO.update(upload); } catch (Exception e) { if(e instanceof ApiException) { log.error("Error making BrAPI call: " + Utilities.generateApiExceptionLogMessage((ApiException) e), e); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java deleted file mode 100644 index 0666870c6..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ /dev/null @@ -1,2567 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.breedinginsight.brapps.importer.services.processors; - - -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import io.micronaut.context.annotation.Property; -import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.HttpStatus; -import io.micronaut.http.exceptions.HttpStatusException; -import io.micronaut.http.server.exceptions.InternalServerException; -import io.reactivex.functions.Function; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.collections4.map.CaseInsensitiveMap; -import org.apache.commons.lang3.StringUtils; -import org.brapi.client.v2.JSON; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.*; -import org.brapi.v2.model.core.request.BrAPIListNewRequest; -import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; -import org.breedinginsight.api.auth.AuthenticatedUser; -import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; -import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapi.v2.dao.*; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; -import org.breedinginsight.brapps.importer.model.imports.PendingImport; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.FileMappingUtil; -import org.breedinginsight.dao.db.tables.pojos.TraitEntity; -import org.breedinginsight.model.*; -import org.breedinginsight.services.OntologyService; -import org.breedinginsight.services.ProgramLocationService; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; -import org.breedinginsight.utilities.DatasetUtil; -import org.breedinginsight.utilities.Utilities; -import org.jooq.DSLContext; -import tech.tablesaw.api.Table; -import tech.tablesaw.columns.Column; - -import javax.inject.Inject; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.*; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -@Slf4j -@Prototype -public class ExperimentProcessor implements Processor { - - private static final String NAME = "Experiment"; - private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs"; - private static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists"; - private static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; - private static final String MIDNIGHT = "T00:00:00-00:00"; - private static final String TIMESTAMP_PREFIX = "TS:"; - private static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; - private static final String COMMA_DELIMITER = ","; - private static final String BLANK_FIELD_EXPERIMENT = "Field is blank when creating a new experiment"; - private static final String BLANK_FIELD_ENV = "Field is blank when creating a new environment"; - private static final String BLANK_FIELD_OBS = "Field is blank when creating new observations"; - private static final String ENV_LOCATION_MISMATCH = "All locations must be the same for a given environment"; - private static final String ENV_YEAR_MISMATCH = "All years must be the same for a given environment"; - - @Property(name = "brapi.server.reference-source") - private String BRAPI_REFERENCE_SOURCE; - - private final DSLContext dsl; - private final BrAPITrialDAO brapiTrialDAO; - private final ProgramLocationService locationService; - private final BrAPIStudyDAO brAPIStudyDAO; - private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; - private final BrAPIObservationDAO brAPIObservationDAO; - private final BrAPISeasonDAO brAPISeasonDAO; - private final BrAPIGermplasmDAO brAPIGermplasmDAO; - private final BrAPIListDAO brAPIListDAO; - private final OntologyService ontologyService; - private final FileMappingUtil fileMappingUtil; - - - // used to make the yearsToSeasonDbId() function more efficient - private final Map yearToSeasonDbIdCache = new HashMap<>(); - // used to make the seasonDbIdtoYear() function more efficient - private final Map seasonDbIdToYearCache = new HashMap<>(); - - //These BrapiData-objects are initially populated by the getExistingBrapiData() method, - // then updated by the initNewBrapiData() method. - private Map> trialByNameNoScope = new HashMap<>(); - private Map> pendingTrialByOUId = new HashMap<>(); - private Map> locationByName = null; - private Map> pendingLocationByOUId = new HashMap<>(); - private Map> studyByNameNoScope = new HashMap<>(); - private Map> pendingStudyByOUId = new HashMap<>(); - private Map> obsVarDatasetByName = null; - private Map> pendingObsDatasetByOUId = new HashMap<>(); - private Map> observationUnitByNameNoScope = null; - private Map> pendingObsUnitByOUId = new HashMap<>(); - - private final Map> observationByHash = new HashMap<>(); - private Map existingObsByObsHash = new HashMap<>(); - // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the initNewBrapiData() method - private Map> existingGermplasmByGID = new HashMap<>(); - private Map> pendingGermplasmByOUId = new HashMap<>(); - - // Associates timestamp columns to associated phenotype column name for ease of storage - private final Map> timeStampColByPheno = new HashMap<>(); - private final Gson gson; - private boolean hasAllReferenceUnitIds = true; - private boolean hasNoReferenceUnitIds = true; - private Set referenceOUIds = new HashSet<>(); - - @Inject - public ExperimentProcessor(DSLContext dsl, - BrAPITrialDAO brapiTrialDAO, - ProgramLocationService locationService, - BrAPIStudyDAO brAPIStudyDAO, - BrAPIObservationUnitDAO brAPIObservationUnitDAO, - BrAPIObservationDAO brAPIObservationDAO, - BrAPISeasonDAO brAPISeasonDAO, - BrAPIGermplasmDAO brAPIGermplasmDAO, - BrAPIListDAO brAPIListDAO, OntologyService ontologyService, - FileMappingUtil fileMappingUtil) { - this.dsl = dsl; - this.brapiTrialDAO = brapiTrialDAO; - this.locationService = locationService; - this.brAPIStudyDAO = brAPIStudyDAO; - this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; - this.brAPIObservationDAO = brAPIObservationDAO; - this.brAPISeasonDAO = brAPISeasonDAO; - this.brAPIGermplasmDAO = brAPIGermplasmDAO; - this.brAPIListDAO = brAPIListDAO; - this.ontologyService = ontologyService; - this.fileMappingUtil = fileMappingUtil; - this.gson = new JSON().getGson(); - } - - @Override - public String getName() { - return NAME; - } - - /** - * Initialize the Map objects with existing BrAPI Data. - * - * @param importRows - * @param program - */ - @Override - public void getExistingBrapiData(List importRows, Program program) { - - List experimentImportRows = importRows.stream() - .map(trialImport -> (ExperimentObservation) trialImport) - .collect(Collectors.toList()); - - // check for references to Deltabreed-generated observation units - referenceOUIds = collateReferenceOUIds(importRows); - - if (hasAllReferenceUnitIds) { - try { - - // get all prior units referenced in import - pendingObsUnitByOUId = fetchReferenceObservationUnits(referenceOUIds, program); - observationUnitByNameNoScope = mapPendingObservationUnitByName(pendingObsUnitByOUId, program); - initializeTrialsForExistingObservationUnits(program, trialByNameNoScope); - initializeStudiesForExistingObservationUnits(program, studyByNameNoScope); - locationByName = initializeLocationByName(program, studyByNameNoScope); - obsVarDatasetByName = initializeObsVarDatasetForExistingObservationUnits(trialByNameNoScope, program); - existingGermplasmByGID = initializeGermplasmByGIDForExistingObservationUnits(observationUnitByNameNoScope, program); - for (Map.Entry> unitEntry : pendingObsUnitByOUId.entrySet()) { - String unitId = unitEntry.getKey(); - BrAPIObservationUnit unit = unitEntry.getValue().getBrAPIObject(); - mapPendingTrialByOUId(unitId, unit, trialByNameNoScope, studyByNameNoScope, pendingTrialByOUId, program); - mapPendingStudyByOUId(unitId, unit, studyByNameNoScope, pendingStudyByOUId, program); - mapPendingLocationByOUId(unitId, unit, pendingStudyByOUId, locationByName, pendingLocationByOUId); - mapPendingObsDatasetByOUId(unitId, pendingTrialByOUId, obsVarDatasetByName, pendingObsDatasetByOUId); - mapGermplasmByOUId(unitId, unit, existingGermplasmByGID, pendingGermplasmByOUId); - } - - } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (Exception e) { - log.error("Error processing experiment with ", e); - throw new InternalServerException(e.toString(), e); - } - } else if (hasNoReferenceUnitIds) { - observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); - trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); - studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); - locationByName = initializeUniqueLocationNames(program, experimentImportRows); - obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); - existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); - - } else { - - // can't proceed if the import has a mix of ObsUnitId for some but not all rows - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, MISSING_OBS_UNIT_ID_ERROR); - } - } - - /** - * @param importRows - one element of the list for every row of the import file. - * @param mappedBrAPIImport - passed in by reference and modified within this program (this will later be passed to the front end for the preview) - * @param program - * @param user - * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) - * false when used for preview only - * @return Map - used to display the summary statistics. - * @throws ValidatorException - */ - @Override - public Map process( - ImportUpload upload, - List importRows, - Map mappedBrAPIImport, - Table data, - Program program, - User user, - boolean commit) throws ApiException, ValidatorException, MissingRequiredInfoException, UnprocessableEntityException { - log.debug("processing experiment import"); - - ValidationErrors validationErrors = new ValidationErrors(); - - // Get dynamic phenotype columns for processing - List> dynamicCols = data.columns(upload.getDynamicColumnNames()); - List> phenotypeCols = new ArrayList<>(); - List> timestampCols = new ArrayList<>(); - for (Column dynamicCol : dynamicCols) { - //Distinguish between phenotype and timestamp columns - if (dynamicCol.name().startsWith(TIMESTAMP_PREFIX)) { - timestampCols.add(dynamicCol); - } else { - phenotypeCols.add(dynamicCol); - } - } - - List referencedTraits = verifyTraits(program.getId(), phenotypeCols, timestampCols); - - //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval - for (Column tsColumn : timestampCols) { - timeStampColByPheno.put(tsColumn.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), tsColumn); - } - - // add "New" pending data to the BrapiData objects - initNewBrapiData(importRows, phenotypeCols, program, user, referencedTraits, commit); - - prepareDataForValidation(importRows, phenotypeCols, mappedBrAPIImport); - - validateFields(importRows, validationErrors, mappedBrAPIImport, referencedTraits, program, phenotypeCols, commit, user); - - if (validationErrors.hasErrors()) { - throw new ValidatorException(validationErrors); - } - - log.debug("done processing experiment import"); - // Construct our response object - return generateStatisticsMap(importRows); - } - - @Override - public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { - // TODO - } - - @Override - public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { - log.debug("starting post of experiment data to BrAPI server"); - - List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); - Map mutatedTrialsById = ProcessorData - .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); - - Map mutatedObservationByDbId = ProcessorData - .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); - - List newLocations = ProcessorData.getNewObjects(this.locationByName) - .stream() - .map(location -> ProgramLocationRequest.builder() - .name(location.getName()) - .build()) - .collect(Collectors.toList()); - List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); - - List newDatasetRequests = ProcessorData.getNewObjects(obsVarDatasetByName).stream().map(details -> { - BrAPIListNewRequest request = new BrAPIListNewRequest(); - request.setListName(details.getListName()); - request.setListType(details.getListType()); - request.setExternalReferences(details.getExternalReferences()); - request.setAdditionalInfo(details.getAdditionalInfo()); - request.data(details.getData()); - return request; - }).collect(Collectors.toList()); - Map datasetNewDataById = ProcessorData - .getMutationsByObjectId(obsVarDatasetByName, BrAPIListSummary::getListDbId); - - List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); - - // filter out observations with no 'value' so they will not be saved - List newObservations = ProcessorData.getNewObjects(this.observationByHash) - .stream() - .filter(obs -> !obs.getValue().isBlank()) - .collect(Collectors.toList()); - - AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); - - try { - List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); - createdDatasets.forEach(summary -> obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId())); - - List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); - // set the DbId to the for each newly created trial - for (BrAPITrial createdTrial : createdTrials) { - String createdTrialName = Utilities.removeProgramKey(createdTrial.getTrialName(), program.getKey()); - this.trialByNameNoScope.get(createdTrialName) - .getBrAPIObject() - .setTrialDbId(createdTrial.getTrialDbId()); - } - - List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); - // set the DbId to the for each newly created location - for (ProgramLocation createdLocation : createdLocations) { - String createdLocationName = createdLocation.getName(); - this.locationByName.get(createdLocationName) - .getBrAPIObject() - .setLocationDbId(createdLocation.getLocationDbId()); - } - - updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); - List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); - - // set the DbId to the for each newly created study - for (BrAPIStudy createdStudy : createdStudies) { - String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); - this.studyByNameNoScope.get(createdStudy_name_no_key) - .getBrAPIObject() - .setStudyDbId(createdStudy.getStudyDbId()); - } - - updateObsUnitDependencyValues(program.getKey()); - List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); - - // set the DbId to the for each newly created Observation Unit - for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { - // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope - String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); - String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); - String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); - this.observationUnitByNameNoScope.get(createdObsUnit_key) - .getBrAPIObject() - .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); - } - - updateObservationDependencyValues(program); - brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); - } catch (ApiException e) { - log.error("Error saving experiment import: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error saving experiment import", e); - throw new InternalServerException(e.getMessage(), e); - } - - mutatedTrialsById.forEach((id, trial) -> { - try { - brapiTrialDAO.updateBrAPITrial(id, trial, program.getId()); - } catch (ApiException e) { - log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error updating dataset observation variables: ", e); - throw new InternalServerException(e.getMessage(), e); - } - }); - - datasetNewDataById.forEach((id, dataset) -> { - try { - List existingObsVarIds = brAPIListDAO.getListById(id, program.getId()).getResult().getData(); - List newObsVarIds = dataset - .getData() - .stream() - .filter(obsVarId -> !existingObsVarIds.contains(obsVarId)).collect(Collectors.toList()); - List obsVarIds = new ArrayList<>(existingObsVarIds); - obsVarIds.addAll(newObsVarIds); - dataset.setData(obsVarIds); - brAPIListDAO.updateBrAPIList(id, dataset, program.getId()); - } catch (ApiException e) { - log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error updating dataset observation variables: ", e); - throw new InternalServerException(e.getMessage(), e); - } - }); - - mutatedObservationByDbId.forEach((id, observation) -> { - try { - if (observation == null) { - throw new Exception("Null observation"); - } - - BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); - - if (updatedObs == null) { - throw new Exception("Null updated observation"); - } - - if (!Objects.equals(observation.getValue(), updatedObs.getValue()) - || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { - String message; - if(!Objects.equals(observation.getValue(), updatedObs.getValue())) { - message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); - } else { - message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); - } - throw new Exception(message); - } - } catch (ApiException e) { - log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException("Error saving experiment import", e); - } catch (Exception e) { - log.error("Error updating observation: ", e); - throw new InternalServerException(e.getMessage(), e); - } - }); - log.debug("experiment import complete"); - } - - private void prepareDataForValidation(List importRows, List> phenotypeCols, Map mappedBrAPIImport) { - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); - List> observations = mappedImportRow.getObservations(); - String observationHash; - if (hasAllReferenceUnitIds) { - String refOUId = importRow.getObsUnitID(); - mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); - mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); - mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); - mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); - mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); - - // loop over phenotype column observation data for current row - for (Column column : phenotypeCols) { - observationHash = getObservationHash( - pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + - pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), - getVariableNameFromColumn(column), - pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() - ); - - // if value was blank won't be entry in map for this observation - observations.add(observationByHash.get(observationHash)); - } - - } else { - mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); - mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); - mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); - mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); - mappedImportRow.setGermplasm(getGidPIO(importRow)); - - // loop over phenotype column observation data for current row - for (Column column : phenotypeCols) { - - // if value was blank won't be entry in map for this observation - observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); - } - } - - mappedBrAPIImport.put(rowNum, mappedImportRow); - } - } - - private List verifyTraits(UUID programId, List> phenotypeCols, List> timestampCols) { - Set varNames = phenotypeCols.stream() - .map(Column::name) - .collect(Collectors.toSet()); - Set tsNames = timestampCols.stream() - .map(Column::name) - .collect(Collectors.toSet()); - - // filter out just traits specified in file - List filteredTraits = fetchFileTraits(programId, varNames); - - // check that all specified ontology terms were found - if (filteredTraits.size() != varNames.size()) { - Set returnedVarNames = filteredTraits.stream() - .map(TraitEntity::getObservationVariableName) - .collect(Collectors.toSet()); - List differences = varNames.stream() - .filter(var -> !Utilities.containsCaseInsensitive(var, returnedVarNames)) - .collect(Collectors.toList()); - //TODO convert this to a ValidationError - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, - "Ontology term(s) not found: " + String.join(COMMA_DELIMITER, differences)); - } - - // Check that each ts column corresponds to a phenotype column - List unmatchedTimestamps = tsNames.stream() - .filter(e -> !(varNames.contains(e.replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY)))) - .collect(Collectors.toList()); - if (unmatchedTimestamps.size() > 0) { - //TODO convert this to a ValidationError - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, - "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(COMMA_DELIMITER, unmatchedTimestamps)); - } - - // sort the verified traits to match the order of the trait columns - List phenotypeColNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList()); - return fileMappingUtil.sortByField(phenotypeColNames, filteredTraits, TraitEntity::getObservationVariableName); - } - - private List fetchFileTraits(UUID programId, Collection varNames) { - try { - Collection upperCaseVarNames = varNames.stream().map(String::toUpperCase).collect(Collectors.toList()); - List traits = ontologyService.getTraitsByProgramId(programId, true); - // filter out just traits specified in file - return traits.stream() - .filter(e -> upperCaseVarNames.contains(e.getObservationVariableName().toUpperCase())) - .collect(Collectors.toList()); - } catch (DoesNotExistException e) { - log.error(e.getMessage(), e); - throw new InternalServerException(e.toString(), e); - } - } - - private String getVariableNameFromColumn(Column column) { - // TODO: timestamp stripping? - return column.name(); - } - - private void initNewBrapiData( - List importRows, - List> phenotypeCols, - Program program, - User user, - List referencedTraits, - boolean commit - ) throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { - - String expSequenceName = program.getExpSequence(); - if (expSequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); - } - Supplier expNextVal = () -> dsl.nextval(expSequenceName.toLowerCase()); - - String envSequenceName = program.getEnvSequence(); - if (envSequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the env sequence column.", program.getName())); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for environment import"); - } - Supplier envNextVal = () -> dsl.nextval(envSequenceName.toLowerCase()); - existingObsByObsHash = fetchExistingObservations(referencedTraits, program); - - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - - PendingImportObject trialPIO = null; - try { - trialPIO = fetchOrCreateTrialPIO(program, user, commit, importRow, expNextVal); - } catch (UnprocessableEntityException e) { - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); - } - - String expSeqValue = null; - if (commit) { - expSeqValue = trialPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) - .getAsString(); - } - - if (commit) { - fetchOrCreateDatasetPIO(importRow, program, referencedTraits); - } - - fetchOrCreateLocationPIO(importRow); - - PendingImportObject studyPIO = fetchOrCreateStudyPIO(program, commit, expSeqValue, importRow, envNextVal); - - String envSeqValue = null; - if (commit) { - envSeqValue = studyPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER) - .getAsString(); - } - - PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); - - for (Column column : phenotypeCols) { - //If associated timestamp column, add - String dateTimeValue = null; - if (timeStampColByPheno.containsKey(column.name())) { - dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); - //If no timestamp, set to midnight - if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { - dateTimeValue += MIDNIGHT; - } - } - - // get the study year either referenced from the observation unit or listed explicitly on the import row - String studyYear = hasAllReferenceUnitIds ? studyPIO.getBrAPIObject().getSeasons().get(0) : importRow.getEnvYear(); - String seasonDbId = yearToSeasonDbId(studyYear, program.getId()); - fetchOrCreateObservationPIO( - program, - user, - importRow, - column, //column.name() gets phenotype name - rowNum, - dateTimeValue, - commit, - seasonDbId, - obsUnitPIO, - studyPIO, - referencedTraits - ); - } - } - } - - private String createObservationUnitKey(ExperimentObservation importRow) { - return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); - } - - private String createObservationUnitKey(String studyName, String obsUnitName) { - return studyName + obsUnitName; - } - - /** - * This method is responsible for generating a hash based on the import observation unit information. - * It takes the observation unit name, variable name, and study name as input parameters. - * The observation unit key is created using the study name and observation unit name. - * The hash is generated based on the observation unit key, variable name, and study name. - * - * @param obsUnitName The name of the observation unit being imported. - * @param variableName The name of the variable associated with the observation unit. - * @param studyName The name of the study associated with the observation unit. - * @return A string representing the hash of the import observation unit information. - */ - private String getImportObservationHash(String obsUnitName, String variableName, String studyName) { - return getObservationHash(createObservationUnitKey(studyName, obsUnitName), variableName, studyName); - } - - private String getImportObservationHash(ExperimentObservation importRow, String variableName) { - return getObservationHash(createObservationUnitKey(importRow), variableName, importRow.getEnv()); - } - - private String getObservationHash(String observationUnitName, String variableName, String studyName) { - String concat = DigestUtils.sha256Hex(observationUnitName) + - DigestUtils.sha256Hex(variableName) + - DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); - return DigestUtils.sha256Hex(concat); - } - - private void validateFields(List importRows, ValidationErrors validationErrors, Map mappedBrAPIImport, List referencedTraits, Program program, - List> phenotypeCols, boolean commit, User user) { - //fetching any existing observations for any OUs in the import - CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); - for ( Trait trait: referencedTraits) { - colVarMap.put(trait.getObservationVariableName(),trait); - } - Set uniqueStudyAndObsUnit = new HashSet<>(); - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); - if (hasAllReferenceUnitIds) { - validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); - } else { - if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. - validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); - } - validateTestOrCheck(importRow, validationErrors, rowNum); - validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); - validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); - } - } - } - - private void validateObservationUnits( - ValidationErrors validationErrors, - Set uniqueStudyAndObsUnit, - int rowNum, - ExperimentObservation importRow - ) { - validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - - String key = createObservationUnitKey(importRow); - PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); - if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { - addRowError(Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); - } - - validateGeoCoordinates(validationErrors, rowNum, importRow); - } - - private void validateGeoCoordinates(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) { - - String lat = importRow.getLatitude(); - String lon = importRow.getLongitude(); - String elevation = importRow.getElevation(); - - // If any of Lat, Long, or Elevation are provided, Lat and Long must both be provided. - if (StringUtils.isNotBlank(lat) || StringUtils.isNotBlank(lon) || StringUtils.isNotBlank(elevation)) { - if (StringUtils.isBlank(lat)) { - addRowError(Columns.LAT, "Latitude must be provided for complete coordinate specification", validationErrors, rowNum); - } - if (StringUtils.isBlank(lon)) { - addRowError(Columns.LONG, "Longitude must be provided for complete coordinate specification", validationErrors, rowNum); - } - } - - // Validate coordinate values - boolean latBadValue = false; - boolean lonBadValue = false; - boolean elevationBadValue = false; - double latDouble; - double lonDouble; - double elevationDouble; - - // Only check latitude format if not blank since already had previous error - if (StringUtils.isNotBlank(lat)) { - try { - latDouble = Double.parseDouble(lat); - if (latDouble < -90 || latDouble > 90) { - latBadValue = true; - } - } catch (NumberFormatException e) { - latBadValue = true; - } - } - - // Only check longitude format if not blank since already had previous error - if (StringUtils.isNotBlank(lon)) { - try { - lonDouble = Double.parseDouble(lon); - if (lonDouble < -180 || lonDouble > 180) { - lonBadValue = true; - } - } catch (NumberFormatException e) { - lonBadValue = true; - } - } - - if (StringUtils.isNotBlank(elevation)) { - try { - elevationDouble = Double.parseDouble(elevation); - } catch (NumberFormatException e) { - elevationBadValue = true; - } - } - - if (latBadValue) { - addRowError(Columns.LAT, "Invalid Lat value (expected range -90 to 90)", validationErrors, rowNum); - } - - if (lonBadValue) { - addRowError(Columns.LONG, "Invalid Long value (expected range -180 to 180)", validationErrors, rowNum); - } - - if (elevationBadValue) { - addRowError(Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); - } - - } - - private Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { - Set ouDbIds = new HashSet<>(); - Set variableDbIds = new HashSet<>(); - Map variableNameByDbId = new HashMap<>(); - Map ouNameByDbId = new HashMap<>(); - Map studyNameByDbId = studyByNameNoScope.values() - .stream() - .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) - .map(PendingImportObject::getBrAPIObject) - .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - - studyNameByDbId.keySet().forEach(studyDbId -> { - try { - brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { - if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { - ouDbIds.add(ou.getObservationUnitDbId()); - } - ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); - }); - } catch (ApiException e) { - throw new RuntimeException(e); - } - }); - - for (Trait referencedTrait : referencedTraits) { - variableDbIds.add(referencedTrait.getObservationVariableDbId()); - variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); - } - - List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); - - return existingObservations.stream() - .map(obs -> { - String studyName = studyNameByDbId.get(obs.getStudyDbId()); - String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); - String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); - - String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); - - return Map.entry(key, obs); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - private void validateObservations(ValidationErrors validationErrors, - int rowNum, - ExperimentObservation importRow, - List> phenotypeCols, - CaseInsensitiveMap colVarMap, - boolean commit, - User user) { - phenotypeCols.forEach(phenoCol -> { - String importHash; - String importObsValue = phenoCol.getString(rowNum); - - if (hasAllReferenceUnitIds) { - importHash = getImportObservationHash( - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), - getVariableNameFromColumn(phenoCol), - pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() - ); - - } else { - importHash = getImportObservationHash(importRow, phenoCol.name()); - } - - // error if import observation data already exists and user has not selected to overwrite - if(commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && - this.existingObsByObsHash.containsKey(importHash) && - StringUtils.isNotBlank(phenoCol.getString(rowNum)) && - !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { - addRowError( - phenoCol.name(), - String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), - validationErrors, rowNum - ); - - // preview case where observation has already been committed and the import row ObsVar data differs from what - // had been saved prior to import - } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { - - // different data means validations still need to happen - // TODO consider moving these two calls into a separate method since called twice together - validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); - - //Timestamp validation - if(timeStampColByPheno.containsKey(phenoCol.name())) { - Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); - validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); - } - - // add a change log entry when updating the value of an observation - // only will update and thereby need change log entry if no error - if (commit && (!validationErrors.hasErrors())) { - BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); - String timestamp = formatter.format(OffsetDateTime.now()); - String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; - String prior = ""; - if (isValueMatched(importHash, importObsValue)) { - prior.concat(existingObsByObsHash.get(importHash).getValue()); - } - if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { - prior = prior.isEmpty() ? prior : prior.concat(" "); - prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); - } - ChangeLogEntry change = new ChangeLogEntry(prior, - reason, - user.getId(), - timestamp - ); - - // create the changelog field in additional info if it does not already exist - if (pendingObservation.getAdditionalInfo().isJsonNull()) { - pendingObservation.setAdditionalInfo(new JsonObject()); - pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } - - if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { - pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } - - // add a new entry to the changelog - pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); - } - - // preview case where observation has already been committed and import ObsVar data is the - // same as has been committed prior to import - } else if(isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { - BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); - existingObs.setObservationVariableName(phenoCol.name()); - observationByHash.get(importHash).setState(ImportObjectState.EXISTING); - observationByHash.get(importHash).setBrAPIObject(existingObs); - - // preview case where observation has already been committed and import ObsVar data is empty prior to import - } else if(!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { - observationByHash.get(importHash).setState(ImportObjectState.EXISTING); - } else { - validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); - - //Timestamp validation - if(timeStampColByPheno.containsKey(phenoCol.name())) { - Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); - validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); - } - } - }); - } - - /** - * Validate that the observation unit is unique within a study. - *
- * SIDE EFFECTS: validationErrors and uniqueStudyAndObsUnit can be modified. - * - * @param validationErrors can be modified as a side effect. - * @param uniqueStudyAndObsUnit can be modified as a side effect. - * @param rowNum counter that is always two less the file row being validated - * @param importRow the data row being validated - */ - private void validateUniqueObsUnits( - ValidationErrors validationErrors, - Set uniqueStudyAndObsUnit, - int rowNum, - ExperimentObservation importRow) { - String envIdPlusStudyId = createObservationUnitKey(importRow); - if (uniqueStudyAndObsUnit.contains(envIdPlusStudyId)) { - String errorMessage = String.format("The ID (%s) is not unique within the environment(%s)", importRow.getExpUnitId(), importRow.getEnv()); - this.addRowError(Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); - } else { - //Only want to add valid unique study-obs unit combos - //To avoid situations like system counting a null value as a unique combo - if (!envIdPlusStudyId.isBlank()) { - uniqueStudyAndObsUnit.add(envIdPlusStudyId); - } - } - } - - private void validateConditionallyRequired(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, Program program, boolean commit) { - ImportObjectState expState = this.trialByNameNoScope.get(importRow.getExpTitle()) - .getState(); - ImportObjectState envState = this.studyByNameNoScope.get(importRow.getEnv()).getState(); - - String errorMessage = BLANK_FIELD_EXPERIMENT; - if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { - errorMessage = BLANK_FIELD_ENV; - } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { - errorMessage = BLANK_FIELD_OBS; - } - - if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { - validateRequiredCell(importRow.getGid(), Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpTitle(),Columns.EXP_TITLE,errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpUnit(), Columns.EXP_UNIT, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpType(), Columns.EXP_TYPE, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getEnv(), Columns.ENV, errorMessage, validationErrors, rowNum); - if(validateRequiredCell(importRow.getEnvLocation(), Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum)) { - if(!Utilities.removeProgramKeyAndUnknownAdditionalData(this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getLocationName(), program.getKey()).equals(importRow.getEnvLocation())) { - addRowError(Columns.ENV_LOCATION, ENV_LOCATION_MISMATCH, validationErrors, rowNum); - } - } - if(validateRequiredCell(importRow.getEnvYear(), Columns.ENV_YEAR, errorMessage, validationErrors, rowNum)) { - String studyYear = StringUtils.defaultString( this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getSeasons().get(0) ); - String rowYear = importRow.getEnvYear(); - if(commit) { - rowYear = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); - } - if(StringUtils.isNotBlank(studyYear) && !studyYear.equals(rowYear)) { - addRowError(Columns.ENV_YEAR, ENV_YEAR_MISMATCH, validationErrors, rowNum); - } - } - validateRequiredCell(importRow.getExpUnitId(), Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpReplicateNo(), Columns.REP_NUM, errorMessage, validationErrors, rowNum); - validateRequiredCell(importRow.getExpBlockNo(), Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); - - if(StringUtils.isNotBlank(importRow.getObsUnitID())) { - addRowError(Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); - } - } else { - //Check if existing environment. If so, ObsUnitId must be assigned - validateRequiredCell( - importRow.getObsUnitID(), - Columns.OBS_UNIT_ID, - MISSING_OBS_UNIT_ID_ERROR, - validationErrors, - rowNum - ); - } - } - - private boolean validateRequiredCell(String value, String columnHeader, String errorMessage, ValidationErrors validationErrors, int rowNum) { - if (StringUtils.isBlank(value)) { - addRowError(columnHeader, errorMessage, validationErrors, rowNum); - return false; - } - return true; - } - - private void addRowError(String field, String errorMessage, ValidationErrors validationErrors, int rowNum) { - ValidationError ve = new ValidationError(field, errorMessage, HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(rowNum + 2, ve); // +2 instead of +1 to account for the column header row. - } - - private void addIfNotNull(HashSet set, String setValue) { - if (setValue != null) { - set.add(setValue); - } - } - - private Map generateStatisticsMap(List importRows) { - // Data for stats. - HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names - HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's - HashSet gidCounter = new HashSet<>(); // set of unique GID's - - for (BrAPIImport row : importRows) { - ExperimentObservation importRow = (ExperimentObservation) row; - // Collect date for stats. - addIfNotNull(environmentNameCounter, importRow.getEnv()); - addIfNotNull(obsUnitsIDCounter, createObservationUnitKey(importRow)); - addIfNotNull(gidCounter, importRow.getGid()); - } - - int numNewObservations = Math.toIntExact( - observationByHash.values() - .stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && - !StringUtils.isBlank(preview.getBrAPIObject() - .getValue())) - .count() - ); - - int numExistingObservations = Math.toIntExact( - this.observationByHash.values() - .stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING && - !StringUtils.isBlank(preview.getBrAPIObject() - .getValue())) - .count() - ); - - int numMutatedObservations = Math.toIntExact( - this.observationByHash.values() - .stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && - !StringUtils.isBlank(preview.getBrAPIObject() - .getValue())) - .count() - ); - - - ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() - .newObjectCount(environmentNameCounter.size()) - .build(); - ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() - .newObjectCount(obsUnitsIDCounter.size()) - .build(); - ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() - .newObjectCount(gidCounter.size()) - .build(); - ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() - .newObjectCount(numNewObservations) - .build(); - ImportPreviewStatistics existingObservationStats = ImportPreviewStatistics.builder() - .newObjectCount(numExistingObservations) - .build(); - ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() - .newObjectCount(numMutatedObservations) - .build(); - - return Map.of( - "Environments", environmentStats, - "Observation_Units", obdUnitStats, - "GIDs", gidStats, - "Observations", observationStats, - "Existing_Observations", existingObservationStats, - "Mutated_Observations", mutatedObservationStats - ); - } - - private void validateGermplasm(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum, PendingImportObject germplasmPIO) { - // error if GID is not blank but GID does not already exist - if (StringUtils.isNotBlank(importRow.getGid()) && germplasmPIO == null) { - addRowError(Columns.GERMPLASM_GID, "A non-existing GID", validationErrors, rowNum); - } - } - - private void validateTestOrCheck(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum) { - String testOrCheck = importRow.getTestOrCheck(); - if ( ! ( testOrCheck==null || testOrCheck.isBlank() - || "C".equalsIgnoreCase(testOrCheck) || "CHECK".equalsIgnoreCase(testOrCheck) - || "T".equalsIgnoreCase(testOrCheck) || "TEST".equalsIgnoreCase(testOrCheck) ) - ){ - addRowError(Columns.TEST_CHECK, String.format("Invalid value (%s)", testOrCheck), validationErrors, rowNum) ; - } - } - - private PendingImportObject getGidPIO(ExperimentObservation importRow) { - if (this.existingGermplasmByGID.containsKey(importRow.getGid())) { - return existingGermplasmByGID.get(importRow.getGid()); - } - - return null; - } - - private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { - PendingImportObject pio; - String key = createObservationUnitKey(importRow); - if (hasAllReferenceUnitIds) { - pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); - } else if (observationUnitByNameNoScope.containsKey(key)) { - pio = observationUnitByNameNoScope.get(key); - } else { - String germplasmName = ""; - if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { - germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) - .getBrAPIObject() - .getGermplasmName(); - } - PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; - UUID trialID = trialPIO.getId(); - UUID datasetId = null; - if (commit) { - JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); - DatasetMetadata dataset = DatasetUtil.getDatasetByNameFromJson(datasetsJson, importRow.getExpUnit()); - if (dataset != null) { - datasetId = dataset.getId(); - } - } - PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); - UUID studyID = studyPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); - - // check for existing units if this is an existing study - if (studyPIO.getBrAPIObject().getStudyDbId() != null) { - List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); - List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); - if (matchingOU.isEmpty()) { - throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); - } else { - pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); - } - } else { - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); - } - this.observationUnitByNameNoScope.put(key, pio); - } - return pio; - } - - boolean isTimestampMatched(String observationHash, String timeStamp) { - OffsetDateTime priorStamp = null; - if(existingObsByObsHash.get(observationHash)!=null){ - priorStamp = existingObsByObsHash.get(observationHash).getObservationTimeStamp(); - } - if (priorStamp == null) { - return timeStamp == null; - } - boolean isMatched = false; - try { - isMatched = priorStamp.isEqual(OffsetDateTime.parse(timeStamp)); - } catch(DateTimeParseException e){ - //if timestamp is invalid DateTime not equal to validated priorStamp - log.error(e.getMessage(), e); - } - return isMatched; - } - - boolean isValueMatched(String observationHash, String value) { - if (!existingObsByObsHash.containsKey(observationHash) || existingObsByObsHash.get(observationHash).getValue() == null) { - return value == null; - } - return existingObsByObsHash.get(observationHash).getValue().equals(value); - } - - boolean isObservationMatched(String observationHash, String value, Column phenoCol, Integer rowNum) { - if (timeStampColByPheno.isEmpty() || !timeStampColByPheno.containsKey(phenoCol.name())) { - return isValueMatched(observationHash, value); - } else { - String importObsTimestamp = timeStampColByPheno.get(phenoCol.name()).getString(rowNum); - return isTimestampMatched(observationHash, importObsTimestamp) && isValueMatched(observationHash, value); - } - } - - private void fetchOrCreateObservationPIO(Program program, - User user, - ExperimentObservation importRow, - Column column, - Integer rowNum, - String timeStampValue, - boolean commit, - String seasonDbId, - PendingImportObject obsUnitPIO, - PendingImportObject studyPIO, - List referencedTraits) throws ApiException, UnprocessableEntityException { - PendingImportObject pio; - BrAPIObservation newObservation; - String variableName = column.name(); - String value = column.getString(rowNum); - String key; - if (hasAllReferenceUnitIds) { - String unitName = obsUnitPIO.getBrAPIObject().getObservationUnitName(); - String studyName = studyPIO.getBrAPIObject().getStudyName(); - key = getObservationHash(studyName + unitName, variableName, studyName); - } else { - key = getImportObservationHash(importRow, variableName); - } - - if (existingObsByObsHash.containsKey(key)) { - // Update observation value only if it is changed and new value is not blank. - if (!isObservationMatched(key, value, column, rowNum) && StringUtils.isNotBlank(value)){ - - // prior observation with updated value - newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); - if (!isValueMatched(key, value)){ - newObservation.setValue(value); - } else if (!StringUtils.isBlank(timeStampValue) && !isTimestampMatched(key, timeStampValue)) { - DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; - String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); - newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); - } - pio = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); - } else { - - // prior observation - pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(existingObsByObsHash.get(key), BrAPIObservation.class, program)); - } - - observationByHash.put(key, pio); - } else if (!this.observationByHash.containsKey(key)){ - - // new observation - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - - UUID trialID = trialPIO.getId(); - UUID studyID = studyPIO.getId(); - UUID id = UUID.randomUUID(); - newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); - //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. - // Validation error should be thrown for offending value, but that doesn't happen until later downstream - if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) { - newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); - } - - newObservation.setStudyDbId(studyPIO.getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations - - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); - this.observationByHash.put(key, pio); - } - } - private void addObsVarsToDatasetDetails(PendingImportObject pio, List referencedTraits, Program program) { - BrAPIListDetails details = pio.getBrAPIObject(); - referencedTraits.forEach(trait -> { - String id = Utilities.appendProgramKey(trait.getObservationVariableName(), program.getKey()); - - // TODO - Don't append the key if connected to a brapi service operating with legacy data(no appended program key) - - if (!details.getData().contains(id) && ImportObjectState.EXISTING != pio.getState()) { - details.getData().add(id); - } - if (!details.getData().contains(id) && ImportObjectState.EXISTING == pio.getState()) { - details.getData().add(id); - pio.setState(ImportObjectState.MUTATED); - } - }); - } - - private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { - PendingImportObject pio; - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - String name = String.format("Observation Dataset [%s-%s]", - program.getKey(), - trialPIO.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) - .getAsString()); - if (obsVarDatasetByName.containsKey(name)) { - pio = obsVarDatasetByName.get(name); - } else { - UUID id = UUID.randomUUID(); - BrAPIListDetails newDataset = importRow.constructDatasetDetails( - name, - id, - BRAPI_REFERENCE_SOURCE, - program, - trialPIO.getId().toString()); - pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); - - JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); - // If datasets property does not yet exist, create it - String datasetName = StringUtils.isNotBlank(importRow.getSubObsUnit()) ? importRow.getSubObsUnit() : importRow.getExpUnit(); - List datasets = DatasetUtil.datasetsFromJson(datasetsJson); - DatasetMetadata dataset = DatasetMetadata.builder() - .name(datasetName) - .id(id) - .level(StringUtils.isNotBlank(importRow.getSubObsUnit()) ? DatasetLevel.SUB_OBS_UNIT : DatasetLevel.EXP_UNIT) - .build(); - - log.debug(dataset.getName()); - datasets.add(dataset); - datasetsJson = DatasetUtil.jsonArrayFromDatasets(datasets); - trialPIO.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, datasetsJson); - - if (ImportObjectState.EXISTING == trialPIO.getState()) { - trialPIO.setState(ImportObjectState.MUTATED); - } - obsVarDatasetByName.put(name, pio); - } - addObsVarsToDatasetDetails(pio, referencedTraits, program); - } - - private PendingImportObject fetchOrCreateStudyPIO( - Program program, - boolean commit, - String expSequenceValue, - ExperimentObservation importRow, - Supplier envNextVal - ) throws UnprocessableEntityException { - PendingImportObject pio; - if (hasAllReferenceUnitIds) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), - program.getKey() - ); - pio = studyByNameNoScope.get(studyName); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { - pio = studyByNameNoScope.get(importRow.getEnv()); - if (!commit){ - addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); - } - } else { - PendingImportObject trialPIO = hasAllReferenceUnitIds ? - getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - UUID trialID = trialPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); - newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies - - // It is assumed that the study has only one season, And that the Years and not - // the dbId's are stored in getSeason() list. - String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season - if (commit) { - if(StringUtils.isNotBlank(year)) { - String seasonID = this.yearToSeasonDbId(year, program.getId()); - newStudy.setSeasons(Collections.singletonList(seasonID)); - } - } else { - addYearToStudyAdditionalInfo(program, newStudy, year); - } - - pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); - this.studyByNameNoScope.put(importRow.getEnv(), pio); - } - return pio; - } - - - /* - * this finds the YEAR from the season list on the BrAPIStudy and then - * will add the year to the additionalInfo-field of the BrAPIStudy - * */ - private void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study) { - JsonObject additionalInfo = study.getAdditionalInfo(); - - //if it is already there, don't add it. - if(additionalInfo==null || additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { - String year = study.getSeasons().get(0); - addYearToStudyAdditionalInfo(program, study, year); - } - } - - - /* - * this will add the given year to the additionalInfo field of the BrAPIStudy (if it does not already exist) - * */ - private void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, String year) { - JsonObject additionalInfo = study.getAdditionalInfo(); - if (additionalInfo==null){ - additionalInfo = new JsonObject(); - study.setAdditionalInfo(additionalInfo); - } - if( additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { - additionalInfo.addProperty(BrAPIAdditionalInfoFields.ENV_YEAR, year); - } - } - - private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { - PendingImportObject pio; - String envLocationName = hasAllReferenceUnitIds ? - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName() : importRow.getEnvLocation(); - if (!locationByName.containsKey((importRow.getEnvLocation()))) { - ProgramLocation newLocation = new ProgramLocation(); - newLocation.setName(envLocationName); - pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); - this.locationByName.put(envLocationName, pio); - } - } - - private PendingImportObject fetchOrCreateTrialPIO( - Program program, - User user, - boolean commit, - ExperimentObservation importRow, - Supplier expNextVal - ) throws UnprocessableEntityException { - PendingImportObject trialPio; - - // use the prior trial if observation unit IDs are supplied - if (hasAllReferenceUnitIds) { - trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); - - // otherwise create a new trial, but there can be only one allowed - } else { - if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { - PendingImportObject envPio; - trialPio = trialByNameNoScope.get(importRow.getExpTitle()); - envPio = studyByNameNoScope.get(importRow.getEnv()); - - // creating new units for existing experiments and environments is not possible - if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && - (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ - throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); - } - } else if (!trialByNameNoScope.isEmpty()) { - throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); - } else { - UUID id = UUID.randomUUID(); - String expSeqValue = null; - if (commit) { - expSeqValue = expNextVal.get().toString(); - } - BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); - trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - trialByNameNoScope.put(importRow.getExpTitle(), trialPio); - } - } - return trialPio; - } - - private void updateObservationDependencyValues(Program program) { - String programKey = program.getKey(); - - // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds - this.observationUnitByNameNoScope.values().stream() - .map(PendingImportObject::getBrAPIObject) - .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); - - // Update ObservationVariable DbIds - List traits = getTraitList(program); - CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); - for ( Trait trait: traits) { - traitMap.put(trait.getObservationVariableName(),trait); - } - for (PendingImportObject observation : this.observationByHash.values()) { - String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); - if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { - String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); - observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); - } - } - } - - private List getTraitList(Program program) { - try { - return ontologyService.getTraitsByProgramId(program.getId(), true); - } catch (DoesNotExistException e) { - log.error(e.getMessage(), e); - throw new InternalServerException(e.toString(), e); - } - } - - // Update each ovservation's observationUnit DbId, study DbId, and germplasm DbId - private void updateObservationDbIds(BrAPIObservationUnit obsUnit, String programKey) { - // FILTER LOGIC: Match on Env and Exp Unit ID - this.observationByHash.values() - .stream() - .filter(obs -> obs.getBrAPIObject() - .getAdditionalInfo() != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) - .getAsString() - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getStudyName(), programKey)) - && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject().getObservationUnitName(), programKey) - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey)) - ) - .forEach(obs -> { - if (StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { - obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); - } - obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); - obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId()); - }); - } - - private void updateObsUnitDependencyValues(String programKey) { - - // update study DbIds - this.studyByNameNoScope.values() - .stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(study -> updateStudyDbId(study, programKey)); - - // update germplasm DbIds - this.existingGermplasmByGID.values() - .stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(this::updateGermplasmDbId); - } - - private void updateStudyDbId(BrAPIStudy study, String programKey) { - this.observationUnitByNameNoScope.values() - .stream() - .filter(obsUnit -> obsUnit.getBrAPIObject() - .getStudyName() - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey))) - .forEach(obsUnit -> { - obsUnit.getBrAPIObject() - .setStudyDbId(study.getStudyDbId()); - obsUnit.getBrAPIObject() - .setTrialDbId(study.getTrialDbId()); - }); - } - - private void updateGermplasmDbId(BrAPIGermplasm germplasm) { - this.observationUnitByNameNoScope.values() - .stream() - .filter(obsUnit -> germplasm.getAccessionNumber() != null && - germplasm.getAccessionNumber().equals(obsUnit - .getBrAPIObject() - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.GID).getAsString())) - .forEach(obsUnit -> obsUnit.getBrAPIObject() - .setGermplasmDbId(germplasm.getGermplasmDbId())); - } - - private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { - // update location DbIds in studies for all distinct locations - mappedBrAPIImport.values() - .stream() - .map(PendingImport::getLocation) - .forEach(this::updateStudyLocationDbId); - - // update trial DbIds in studies for all distinct trials - this.trialByNameNoScope.values() - .stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(trial -> this.updateTrialDbId(trial, programKey)); - } - - private void updateStudyLocationDbId(PendingImportObject location) { - this.studyByNameNoScope.values() - .stream() - .filter(study -> location.getId().toString() - .equals(study.getBrAPIObject() - .getLocationDbId())) - .forEach(study -> study.getBrAPIObject() - .setLocationDbId(location.getBrAPIObject().getLocationDbId())); - } - - private void updateTrialDbId(BrAPITrial trial, String programKey) { - this.studyByNameNoScope.values() - .stream() - .filter(study -> study.getBrAPIObject() - .getTrialName() - .equals(Utilities.removeProgramKey(trial.getTrialName(), programKey))) - .forEach(study -> study.getBrAPIObject() - .setTrialDbId(trial.getTrialDbId())); - } - - private ArrayList getGermplasmByAccessionNumber( - List germplasmAccessionNumbers, - UUID programId) throws ApiException { - List germplasmList = brAPIGermplasmDAO.getGermplasm(programId); - ArrayList resultGermplasm = new ArrayList<>(); - // Search for accession number matches - for (BrAPIGermplasm germplasm : germplasmList) { - for (String accessionNumber : germplasmAccessionNumbers) { - if (germplasm.getAccessionNumber() - .equals(accessionNumber)) { - resultGermplasm.add(germplasm); - break; - } - } - } - return resultGermplasm; - } - - private Map> initializeObservationUnits(Program program, List experimentImportRows) { - Map> ret = new HashMap<>(); - - Map rowByObsUnitId = new HashMap<>(); - experimentImportRows.forEach(row -> { - if (StringUtils.isNotBlank(row.getObsUnitID())) { - if(rowByObsUnitId.containsKey(row.getObsUnitID())) { - throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); - } - rowByObsUnitId.put(row.getObsUnitID(), row); - } - }); - - try { - List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); - - String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - if (existingObsUnits.size() == rowByObsUnitId.size()) { - existingObsUnits.forEach(brAPIObservationUnit -> { - processAndCacheObservationUnit(brAPIObservationUnit, refSource, program, ret, rowByObsUnitId); - - BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) - .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); - row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); - row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); - row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); - }); - } else { - List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); - missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(COMMA_DELIMITER, missingIds)); - } - - return ret; - } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - /** - * Maps pending locations by Observation Unit (OU) Id based on given parameters. - * - * This method takes in a unitId, BrAPIObservationUnit unit, Maps of studyByOUId, locationByName, - * and locationByOUId. It then associates the location of the observation unit with the respective OU Id. - * If the locationName is not null for the unit, it is directly added to locationByOUId. - * If the locationName is null, it checks the studyByOUId map for a location related to the unit. - * If a location related to the unit is found, it maps that location with the respective OU Id. - * If no location is found, it throws an IllegalStateException. - * - * @param unitId the Observation Unit Id - * @param unit the BrAPIObservationUnit object - * @param studyByOUId a Map of Study by Observation Unit Id - * @param locationByName a Map of Location by Name - * @param locationByOUId a Map of Location by Observation Unit Id - * @return the updated locationByOUId map after mapping the pending locations - * @throws IllegalStateException if the Observation Unit is missing a location - */ - private Map> mapPendingLocationByOUId( - String unitId, - BrAPIObservationUnit unit, - Map> studyByOUId, - Map> locationByName, - Map> locationByOUId - ) { - if (unit.getLocationName() != null) { - locationByOUId.put(unitId, locationByName.get(unit.getLocationName())); - } else if (studyByOUId.get(unitId) != null && studyByOUId.get(unitId).getBrAPIObject().getLocationName() != null) { - locationByOUId.put( - unitId, - locationByName.get(studyByOUId.get(unitId).getBrAPIObject().getLocationName()) - ); - } else { - throw new IllegalStateException("Observation unit missing location: " + unitId); - } - - return locationByOUId; - } - - private Map> mapPendingStudyByOUId( - String unitId, - BrAPIObservationUnit unit, - Map> studyByName, - Map> studyByOUId, - Program program - ) { - if (unit.getStudyName() != null) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); - studyByOUId.put(unitId, studyByName.get(studyName)); - } else { - throw new IllegalStateException("Observation unit missing study name: " + unitId); - } - - return studyByOUId; - } - private Map> mapPendingTrialByOUId( - String unitId, - BrAPIObservationUnit unit, - Map> trialByName, - Map> studyByName, - Map> trialByOUId, - Program program - ) { - String trialName; - if (unit.getTrialName() != null) { - trialName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getTrialName(), program.getKey()); - } else if (unit.getStudyName() != null) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); - trialName = Utilities.removeProgramKeyAndUnknownAdditionalData( - studyByName.get(studyName).getBrAPIObject().getTrialName(), - program.getKey() - ); - } else { - throw new IllegalStateException("Observation unit missing trial name and study name: " + unitId); - } - trialByOUId.put(unitId, trialByName.get(trialName)); - - return trialByOUId; - } - private Map> mapPendingObservationUnitByName( - Map> pendingUnitById, - Program program - ) { - Map> pendingUnitByName = new HashMap<>(); - for (Map.Entry> entry : pendingUnitById.entrySet()) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( - entry.getValue().getBrAPIObject().getStudyName(), - program.getKey() - ); - String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( - entry.getValue().getBrAPIObject().getObservationUnitName(), - program.getKey() - ); - pendingUnitByName.put(createObservationUnitKey(studyName, observationUnitName), entry.getValue()); - } - return pendingUnitByName; - } - - /** - * Retrieves reference Observation Units based on a set of reference Observation Unit IDs and a Program. - * Constructs DeltaBreed observation unit source for external references and sets up pending Observation Units. - * - * @param referenceOUIds A set of reference Observation Unit IDs to retrieve - * @param program The Program associated with the Observation Units - * @return A Map containing pending Observation Units by their ID - * @throws ApiException if an error occurs during the process - */ - private Map> fetchReferenceObservationUnits( - Set referenceOUIds, - Program program - ) throws ApiException { - Map> pendingUnitById = new HashMap<>(); - try { - // Retrieve reference Observation Units based on IDs - List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(referenceOUIds), - program - ); - - // Construct the DeltaBreed observation unit source for external references - String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - - if (referenceObsUnits.size() == referenceOUIds.size()) { - // Iterate through reference Observation Units - referenceObsUnits.forEach(unit -> { - // Get external reference for the Observation Unit - BrAPIExternalReference unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) - .orElseThrow(() -> new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID")); - - // Set pending Observation Unit by its ID - pendingUnitById.put( - unitXref.getReferenceId(), - new PendingImportObject( - ImportObjectState.EXISTING, unit, UUID.fromString(unitXref.getReferenceId())) - ); - }); - } else { - // Handle missing Observation Unit IDs - List missingIds = new ArrayList<>(referenceOUIds); - Set fetchedIds = referenceObsUnits.stream().map(unit -> - Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) - .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) - .getReferenceId()) - .collect(Collectors.toSet()); - missingIds.removeAll(fetchedIds); - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(COMMA_DELIMITER, missingIds)); - } - - return pendingUnitById; - } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new ApiException(e); - } - } - - private Map> initializeTrialByNameNoScope(Program program, List experimentImportRows) { - Map> trialByName = new HashMap<>(); - - initializeTrialsForExistingObservationUnits(program, trialByName); - - List uniqueTrialNames = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID())) - .map(ExperimentObservation::getExpTitle) - .distinct() - .collect(Collectors.toList()); - try { - brapiTrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> - processAndCacheTrial(existingTrial, program, trialByName) - ); - } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - return trialByName; - } - - private Map> initializeStudyByNameNoScope(Program program, List experimentImportRows) { - Map> studyByName = new HashMap<>(); - if (this.trialByNameNoScope.size() != 1) { - return studyByName; - } - - try { - initializeStudiesForExistingObservationUnits(program, studyByName); - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (Exception e) { - log.error("Error processing studies", e); - throw new InternalServerException(e.toString(), e); - } - - List existingStudies; - Optional> trial = getTrialPIO(experimentImportRows); - - try { - if (trial.isEmpty()) { - // TODO: throw ValidatorException and return 422 - } - UUID experimentId = trial.get().getId(); - existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); - for (BrAPIStudy existingStudy : existingStudies) { - processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); - } - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (Exception e) { - log.error("Error processing studies: ", e); - throw new InternalServerException(e.toString(), e); - } - - return studyByName; - } - - private void initializeStudiesForExistingObservationUnits( - Program program, - Map> studyByName - ) throws Exception { - Set studyDbIds = observationUnitByNameNoScope.values() - .stream() - .map(pio -> pio.getBrAPIObject() - .getStudyDbId()) - .collect(Collectors.toSet()); - - List studies = fetchStudiesByDbId(studyDbIds, program); - for (BrAPIStudy study : studies) { - processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); - } - } - - /** - * Fetches a list of BrAPI studies by their study database IDs for a given program. - * - * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. - * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. - * - * @param studyDbIds a Set of Strings representing the study database IDs to fetch - * @param program the Program object representing the program context in which to fetch studies - * @return a List of BrAPIStudy objects matching the provided study database IDs - * @throws ApiException if there is an issue fetching the studies - * @throws IllegalStateException if any requested study database IDs are not found in the result set - */ - private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { - List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); - if (studies.size() != studyDbIds.size()) { - List missingIds = new ArrayList<>(studyDbIds); - missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); - throw new IllegalStateException( - "Study not found for studyDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); - } - return studies; - } - - /** - * Initializes a map of ProgramLocation objects by their names using the given Program and a map of BrAPIStudy objects by their names. - * - * This method takes a Program object and a map of BrAPIStudy objects by their names, retrieves the location database IDs from the studies, - * and fetches existing ProgramLocation objects based on the database IDs. It then creates a map of ProgramLocation objects by their names - * with PendingImportObject wrappers that indicate the state of the object as existing. - * - * @param program the Program object to associate with the locations - * @param studyByName a map of BrAPIStudy objects by their names - * @return a map of ProgramLocation objects by their names with PendingImportObject wrappers - * @throws InternalServerException if an error occurs during the location retrieval process - */ - Map> initializeLocationByName( - Program program, - Map> studyByName) { - Map> locationByName = new HashMap<>(); - - List existingLocations = new ArrayList<>(); - if(studyByName.size() > 0) { - Set locationDbIds = studyByName.values() - .stream() - .map(study -> study.getBrAPIObject() - .getLocationDbId()) - .collect(Collectors.toSet()); - try { - existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - existingLocations.forEach(existingLocation -> locationByName.put( - existingLocation.getName(), - new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()) - ) - ); - return locationByName; - } - private Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { - Map> locationByName = new HashMap<>(); - - List existingLocations = new ArrayList<>(); - if(studyByNameNoScope.size() > 0) { - Set locationDbIds = studyByNameNoScope.values() - .stream() - .map(study -> study.getBrAPIObject() - .getLocationDbId()) - .collect(Collectors.toSet()); - try { - existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - List uniqueLocationNames = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) - .map(ExperimentObservation::getEnvLocation) - .distinct() - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - try { - existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); - } catch (ApiException e) { - log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); - return locationByName; - } - - /** - * Maps a given germplasm to an observation unit ID in a given map of germplasm by observation unit ID. - * - * This method retrieves the Global Identifier (GID) of the provided observation unit and uses it to lookup - * the corresponding PendingImportObject in the map of germplasm by name. The found germplasm - * object is then mapped to the observation unit ID in the provided map of germplasm by observation unit ID. - * The updated map is returned after the mapping operation has been performed. - * - * @param unitId The observation unit ID to which the germplasm should be mapped. - * @param unit The BrAPIObservationUnit object representing the observation unit. - * @param germplasmByName The map of germplasm objects by name used to lookup the desired germplasm. - * @param germplasmByOUId The map of germplasm objects by observation unit ID to update with the mapping result. - * @return The updated map of germplasm objects by observation unit ID after mapping the germplasm to the provided observation unit ID. - */ - private Map> mapGermplasmByOUId( - String unitId, - BrAPIObservationUnit unit, - Map> germplasmByName, - Map> germplasmByOUId) { - String gid = unit.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.GID).getAsString(); - germplasmByOUId.put(unitId, germplasmByName.get(gid)); - - return germplasmByOUId; - } - - /** - * Maps the pending observation dataset by OU Id based on the given inputs. - * This function checks if the trialByOUId map is not empty, the obsVarDatasetByName map is not empty, - * and if the first entry in the trialByOUId map contains observation dataset id in its additional info. - * If the conditions are met, it adds the pending import object from the obsVarDatasetByName map to the - * obsVarDatasetByOUId map using the unitId as the key. - * - * @param unitId the unit ID based on which the mapping is done - * @param trialByOUId a map containing pending import objects with BrAPITrial as the value, mapped by OU Id - * @param obsVarDatasetByName a map containing pending import objects with BrAPIListDetails as the value, mapped by dataset name - * @param obsVarDatasetByOUId a map containing pending import objects with BrAPIListDetails as the value, mapped by OU Id - * @return the updated obsVarDatasetByOUId map after potential addition of a pending import object - */ - private Map> mapPendingObsDatasetByOUId( - String unitId, - Map> trialByOUId, - Map> obsVarDatasetByName, - Map> obsVarDatasetByOUId) { - if (!trialByOUId.isEmpty() && - !obsVarDatasetByName.isEmpty() && - !trialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { - obsVarDatasetByOUId.put(unitId, obsVarDatasetByName.values().iterator().next()); - } - - return obsVarDatasetByOUId; - } - - /** - * Initializes observation variable dataset for existing observation units. This function retrieves existing datasets related to observation variables for the specified trial and program, processes the dataset details, and caches the data accordingly. - * - * @param trialByName A map containing trial information indexed by trial name. - * @param program The program to which the datasets are related. - * @return A map of observation variable dataset objects indexed by dataset name. - * @throws InternalServerException If the existing dataset summary is not retrieved from the BrAPI server, or an error occurs during API communication. - */ - // TODO: add dataset name or id parameter? This only supports top level for now. - private Map> initializeObsVarDatasetForExistingObservationUnits( - Map> trialByName, - Program program) { - Map> obsVarDatasetByName = new HashMap<>(); - - if (trialByName.size() > 0 && - !trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { - String datasetId = DatasetUtil.getDatasetIdByNameFromJson( - trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS), - trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString() - ); - - try { - List existingDatasets = brAPIListDAO - .getListsByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, - program.getId(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), - UUID.fromString(datasetId)); - if (existingDatasets == null || existingDatasets.isEmpty()) { - throw new InternalServerException("existing dataset summary not returned from brapi server"); - } - BrAPIListDetails dataSetDetails = brAPIListDAO - .getListById(existingDatasets.get(0).getListDbId(), program.getId()) - .getResult(); - processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - return obsVarDatasetByName; - } - - private Map> initializeObsVarDatasetByName(Program program, List experimentImportRows) { - Map> obsVarDatasetByName = new HashMap<>(); - - Optional> trialPIO = getTrialPIO(experimentImportRows); - - if (trialPIO.isPresent() && !trialPIO.get().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { - String datasetId = DatasetUtil.getTopLevelDatasetFromJson( - trialPIO.get().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS) - ).getId().toString(); - try { - List existingDatasets = brAPIListDAO - .getListsByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, - program.getId(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), - UUID.fromString(datasetId)); - if (existingDatasets == null || existingDatasets.isEmpty()) { - throw new InternalServerException("existing dataset summary not returned from brapi server"); - } - BrAPIListDetails dataSetDetails = brAPIListDAO - .getListById(existingDatasets.get(0).getListDbId(), program.getId()) - .getResult(); - processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - return obsVarDatasetByName; - } - - private Optional> getTrialPIO(List experimentImportRows) { - Optional expTitle = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID()) && StringUtils.isNotBlank(row.getExpTitle())) - .map(ExperimentObservation::getExpTitle) - .findFirst(); - - if (expTitle.isEmpty() && trialByNameNoScope.keySet().stream().findFirst().isEmpty()) { - return Optional.empty(); - } - if(expTitle.isEmpty()) { - expTitle = trialByNameNoScope.keySet().stream().findFirst(); - } - - return Optional.ofNullable(this.trialByNameNoScope.get(expTitle.get())); - } - - private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> obsVarDatasetByName) { - BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); - obsVarDatasetByName.put(existingList.getListName(), - new PendingImportObject(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); - } - - /** - * Initializes a mapping of BrAPI Germplasm objects by Germplasm ID for existing BrAPI Observation Units. - * This method retrieves existing Germplasms associated with the provided Observation Units and creates a mapping - * using their Accession Number as the key and a PendingImportObject containing the Germplasm object and a reference ID. - * If no existing Germplasms are found, an empty mapping is returned. - * - * @param unitByName A mapping of Observation Units by name. - * @param program The BrAPI Program object to which the Germplasms belong. - * @return A mapping of BrAPI Germplasm objects by Germplasm ID for existing Observation Units. - * @throws InternalServerException If an error occurs while fetching Germplasms from the database. - */ - private Map> initializeGermplasmByGIDForExistingObservationUnits( - Map> unitByName, - Program program) { - Map> existingGermplasmByGID = new HashMap<>(); - - List existingGermplasms = new ArrayList<>(); - if(unitByName.size() > 0) { - Set germplasmDbIds = unitByName.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); - try { - existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - existingGermplasms.forEach(existingGermplasm -> { - BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); - existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); - }); - return existingGermplasmByGID; - } - - - private Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { - Map> existingGermplasmByGID = new HashMap<>(); - - List existingGermplasms = new ArrayList<>(); - if(observationUnitByNameNoScope.size() > 0) { - Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); - try { - existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - List uniqueGermplasmGIDs = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) - .map(ExperimentObservation::getGid) - .distinct() - .collect(Collectors.toList()); - - try { - existingGermplasms.addAll(this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - existingGermplasms.forEach(existingGermplasm -> { - BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); - existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); - }); - return existingGermplasmByGID; - } - - private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservationUnit, String refSource, Program program, Map> observationUnitByName, - Map rowByObsUnitId) { - BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) - .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); - row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); - observationUnitByName.put(createObservationUnitKey(row), - new PendingImportObject<>(ImportObjectState.EXISTING, - brAPIObservationUnit, - UUID.fromString(idRef.getReferenceId()))); - } - private PendingImportObject processAndCacheStudy( - BrAPIStudy existingStudy, - Program program, - Function getterFunction, - Map> studyMap) throws Exception { - PendingImportObject pendingStudy; - BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); - // map season dbid to year - String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season - if(StringUtils.isNotBlank(seasonDbId)) { - String seasonYear = this.seasonDbIdToYear(seasonDbId, program.getId()); - existingStudy.setSeasons(Collections.singletonList(seasonYear)); - } - pendingStudy = new PendingImportObject<>( - ImportObjectState.EXISTING, - (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), - UUID.fromString(xref.getReferenceId()) - ); - studyMap.put( - Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), - pendingStudy - ); - return pendingStudy; - } - - private void initializeTrialsForExistingObservationUnits(Program program, Map> trialByName) { - if(observationUnitByNameNoScope.size() > 0) { - Set trialDbIds = new HashSet<>(); - Set studyDbIds = new HashSet<>(); - - observationUnitByNameNoScope.values() - .forEach(pio -> { - BrAPIObservationUnit existingOu = pio.getBrAPIObject(); - if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { - throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); - } - - if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { - trialDbIds.add(existingOu.getTrialDbId()); - } else { - studyDbIds.add(existingOu.getStudyDbId()); - } - }); - - //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId - if(!studyDbIds.isEmpty()) { - try { - trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - try { - List trials = brapiTrialDAO.getTrialsByDbIds(trialDbIds, program); - if (trials.size() != trialDbIds.size()) { - List missingIds = new ArrayList<>(trialDbIds); - missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); - throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); - } - - trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); - } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - } - - private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { - Set trialDbIds = new HashSet<>(); - List studies = fetchStudiesByDbId(studyDbIds, program); - studies.forEach(study -> { - if (StringUtils.isBlank(study.getTrialDbId())) { - throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); - } - trialDbIds.add(study.getTrialDbId()); - }); - - return trialDbIds; - } - - private void processAndCacheTrial( - BrAPITrial existingTrial, - Program program, - Map> trialByNameNoScope) { - - //get TrialId from existingTrial - BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) - .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); - UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); - - trialByNameNoScope.put( - Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), - new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); - } - - /** - * This function collates unique ObsUnitID values from a list of BrAPIImport objects. - * It iterates through the list and adds non-blank ObsUnitID values to a Set. - * It also checks for any repeated ObsUnitIDs. The instance variables hasAllReferenceUnitIds and - * hasNoReferenceUnitIds are updated. - * - * @param importRows a List of BrAPIImport objects containing ExperimentObservation data - * @return a Set of unique ObsUnitID strings - * @throws IllegalStateException if a repeated ObsUnitID is encountered - */ - private Set collateReferenceOUIds(List importRows) { - Set referenceOUIds = new HashSet<>(); - for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - - // Check if ObsUnitID is blank - if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { - hasAllReferenceUnitIds = false; - } else if (referenceOUIds.contains(importRow.getObsUnitID())) { - // Throw exception if ObsUnitID is repeated - throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); - } else { - // Add ObsUnitID to referenceOUIds - referenceOUIds.add(importRow.getObsUnitID()); - hasNoReferenceUnitIds = false; - } - } - return referenceOUIds; - } - - private void validateTimeStampValue(String value, - String columnHeader, ValidationErrors validationErrors, int row) { - if (StringUtils.isBlank(value)) { - log.debug(String.format("skipping validation of observation timestamp because there is no value.\n\tvariable: %s\n\trow: %d", columnHeader, row)); - return; - } - if (!validDateValue(value) && !validDateTimeValue(value)) { - addRowError(columnHeader, "Incorrect datetime format detected. Expected YYYY-MM-DD or YYYY-MM-DDThh:mm:ss+hh:mm", validationErrors, row); - } - - } - - private boolean isNAObservation(String value){ - return value.equalsIgnoreCase("NA"); - } - - private void validateObservationValue(Trait variable, String value, - String columnHeader, ValidationErrors validationErrors, int row) { - if (StringUtils.isBlank(value)) { - log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row)); - return; - } - - if (isNAObservation(value)) { - log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row)); - return; - } - - switch (variable.getScale().getDataType()) { - case NUMERICAL: - Optional number = validNumericValue(value); - if (number.isEmpty()) { - addRowError(columnHeader, "Non-numeric text detected detected", validationErrors, row); - } else if (!validNumericRange(number.get(), variable.getScale())) { - addRowError(columnHeader, "Value outside of min/max range detected", validationErrors, row); - } - break; - case DATE: - if (!validDateValue(value)) { - addRowError(columnHeader, "Incorrect date format detected. Expected YYYY-MM-DD", validationErrors, row); - } - break; - case ORDINAL: - if (!validCategory(variable.getScale().getCategories(), value)) { - addRowError(columnHeader, "Undefined ordinal category detected", validationErrors, row); - } - break; - case NOMINAL: - if (!validCategory(variable.getScale().getCategories(), value)) { - addRowError(columnHeader, "Undefined nominal category detected", validationErrors, row); - } - break; - default: - break; - } - - } - - private Optional validNumericValue(String value) { - BigDecimal number; - try { - number = new BigDecimal(value); - } catch (NumberFormatException e) { - return Optional.empty(); - } - return Optional.of(number); - } - - private boolean validNumericRange(BigDecimal value, Scale validValues) { - // account for empty min or max in valid determination - return (validValues.getValidValueMin() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMin())) >= 0) && - (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0); - } - - private boolean validDateValue(String value) { - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE; - try { - formatter.parse(value); - } catch (DateTimeParseException e) { - return false; - } - return true; - } - - private boolean validDateTimeValue(String value) { - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; - try { - formatter.parse(value); - } catch (DateTimeParseException e) { - return false; - } - return true; - } - - private boolean validCategory(List categories, String value) { - Set categoryValues = categories.stream() - .map(category -> category.getValue().toLowerCase()) - .collect(Collectors.toSet()); - return categoryValues.contains(value.toLowerCase()); - } - - /** - * Converts year String to SeasonDbId - *
- * NOTE: This assumes that the only Season records of interest are ones - * with a blank name or a name that is the same as the year. - * - * @param year The year as a string - * @param programId the program ID. - * @return the DbId of the season-record associated with the year - */ - private String yearToSeasonDbId(String year, UUID programId) { - String dbID = null; - if (this.yearToSeasonDbIdCache.containsKey(year)) { // get it from cache if possible - dbID = this.yearToSeasonDbIdCache.get(year); - } else { - dbID = this.yearToSeasonDbIdFromDatabase(year, programId); - this.yearToSeasonDbIdCache.put(year, dbID); - } - return dbID; - } - - private String seasonDbIdToYear(String seasonDbId, UUID programId) { - String year = null; - if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible - year = this.seasonDbIdToYearCache.get(seasonDbId); - } else { - year = this.seasonDbIdToYearFromDatabase(seasonDbId, programId); - this.seasonDbIdToYearCache.put(seasonDbId, year); - } - return year; - } - - private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { - BrAPISeason targetSeason = null; - List seasons; - try { - seasons = this.brAPISeasonDAO.getSeasonsByYear(year, programId); - for (BrAPISeason season : seasons) { - if (null == season.getSeasonName() || season.getSeasonName().isBlank() || season.getSeasonName().equals(year)) { - targetSeason = season; - break; - } - } - if (targetSeason == null) { - BrAPISeason newSeason = new BrAPISeason(); - Integer intYear = null; - if( StringUtils.isNotBlank(year) ){ - intYear = Integer.parseInt(year); - } - newSeason.setYear(intYear); - newSeason.setSeasonName(year); - targetSeason = this.brAPISeasonDAO.addOneSeason(newSeason, programId); - } - - } catch (ApiException e) { - log.warn(Utilities.generateApiExceptionLogMessage(e)); - log.error(e.getResponseBody(), e); - } - - return (targetSeason == null) ? null : targetSeason.getSeasonDbId(); - } - - private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { - BrAPISeason season = null; - try { - season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e), e); - } - Integer yearInt = (season == null) ? null : season.getYear(); - return (yearInt == null) ? "" : yearInt.toString(); - } - - /** - * Returns the single value from the given map, throwing an UnprocessableEntityException if the map does not contain exactly one entry. - * - * @param map The map from which to retrieve the single value. - * @param message The error message to include in the UnprocessableEntityException if the map does not contain exactly one entry. - * @return The single value from the map. - * @throws UnprocessableEntityException if the map does not contain exactly one entry. - */ - private V getSingleEntryValue(Map map, String message) throws UnprocessableEntityException { - if (map.size() != 1) { - throw new UnprocessableEntityException(message); - } - return map.values().iterator().next(); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 4c79478e8..44383d2ba 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -50,6 +50,7 @@ import org.breedinginsight.model.Scale; import org.breedinginsight.model.Trait; import tech.tablesaw.columns.Column; +import org.breedinginsight.services.exceptions.BadRequestException; import javax.inject.Singleton; import java.math.BigDecimal; @@ -59,6 +60,8 @@ import java.util.*; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.OZEX; + @Slf4j @Singleton public class ExperimentUtilities { @@ -68,6 +71,7 @@ public class ExperimentUtilities { public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; public static final String MIDNIGHT = "T00:00:00-00:00"; public static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; + public static final String MULTIPLE_EXP_UNITS = "File contains more than one Exp Unit"; public static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists"; public static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs"; public static final String UNMATCHED_COLUMN = "Ontology term(s) not found: "; @@ -89,7 +93,7 @@ public ExperimentUtilities() { */ public boolean isInvalidMemberListForClass(List list, Class clazz) { // Check if the input list is null, empty, or contains any member that is not an instance of the specified class - return list == null || list.isEmpty() || !list.stream().allMatch(clazz::isInstance); + return (list == null) || !list.stream().allMatch(clazz::isInstance); } /** @@ -153,32 +157,35 @@ public static List importRowsToExperimentObservations(Lis } /** - * This method generates a unique key for an observation unit based on the environment and experimental unit ID. + * This method generates a unique key for an observation unit based on the environment and experimental unit ID and germplasm GID. * - * @param importRow the ExperimentObservation object containing the environment and experimental unit ID + * @param importRow the ExperimentObservation object containing the environment and experimental unit ID and germplasm GID * @return a String representing the unique key for the observation unit */ public static String createObservationUnitKey(ExperimentObservation importRow) { - // Extract the environment and experimental unit ID from the ExperimentObservation object + // Extract the environment and experimental unit ID and germplasm GID from the ExperimentObservation object // and pass them to the createObservationUnitKey method - return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId(), importRow.getGid()); } /** * Create Observation Unit Key * - * This method takes in the name of a study and the name of an observation unit and concatenates them to create a unique key. + * This method takes in the name of a study and the name of an observation unit and the germplasm name and concatenates them to create a unique key. + * Sub-observation unit name needed when repeated measures due to how they are created and named * - * If one or both of the inputs is null, returns an empty string since not a valid combination + * If any of the inputs are null, returns an empty string since not a valid combination * * @param studyName The name of the study * @param obsUnitName The name of the observation unit - * @return A string representing the unique key formed by concatenating the study name and observation unit name + * @param germplasmGID The germplasm gid + * @return A string representing the unique key formed by concatenating the study name and observation unit name and germplasm gid */ - public static String createObservationUnitKey(String studyName, String obsUnitName) { - // Concatenate the study name and observation unit name to create the unique key - if (studyName != null && obsUnitName != null) { - return studyName + obsUnitName; + public static String createObservationUnitKey(String studyName, String obsUnitName, String germplasmGID) { + // Concatenate the study name and observation unit name and germplasm gid to create the unique key + if (studyName != null && obsUnitName != null && germplasmGID != null) { + String keyDelim = "@*"; + return studyName + keyDelim + obsUnitName + keyDelim + germplasmGID; } else { return ""; } @@ -299,7 +306,7 @@ public static void addYearToStudyAdditionalInfo(Program program, BrAPIStudy stud * This method iterates through all import rows in the given context and * extracts unique Observation Unit IDs (ObsUnit IDs) that are not null or blank. * - * @param context The AppendOverwriteMiddlewareContext containing the import data. + * @param ctx The AppendOverwriteMiddlewareContext containing the import data. * @return A Set of String containing all unique, non-null, non-blank Observation Unit IDs. * * @implNote The method performs the following steps: @@ -309,61 +316,38 @@ public static void addYearToStudyAdditionalInfo(Program program, BrAPIStudy stud * 4. If valid, adds the ObsUnit ID to the set. * 5. Returns the set of unique ObsUnit IDs. */ - public static Set collateUniqueOUIds(AppendOverwriteMiddlewareContext context) { - // Initialize variables to track the presence of ObsUnit IDs + public static Set collateUniqueOUIds(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() == null) { + throw new BadRequestException(OZEX.getValue()); + } + Set referenceOUIds = new HashSet<>(); + String idColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column idCol = ctx.getImportContext().getData().column(idColName); - // Iterate through the import rows to process ObsUnit IDs - for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); - if (importRow.getObsUnitID() != null && !importRow.getObsUnitID().isBlank()) { - referenceOUIds.add(importRow.getObsUnitID()); + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + String id = idCol.getString(rowNum); + if (id != null && !id.isBlank()) { + referenceOUIds.add(id); } } return referenceOUIds; } - /** - * Validates Observation Unit ID values in the import context. - * - * This method checks each import row for the validity of its Observation Unit ID (ObsUnitID). - * It performs the following validations: - * 1. Checks if the ObsUnitID is null or blank. - * 2. Checks if the ObsUnitID is a duplicate within the import data. - * - * @param context The AppendOverwriteMiddlewareContext containing import data and validation error storage. - * @throws HttpStatusException If there's an HTTP-related error during the validation process. - * @throws IllegalStateException If the system encounters an unexpected state during validation. - * - * @implNote The method performs the following steps: - * 1. Retrieves the ValidationErrors object from the context. - * 2. Initializes a HashSet to track unique ObsUnitIDs. - * 3. Iterates through each import row in the context. - * 4. For each row: - * - If ObsUnitID is null or blank, adds a "missing ObsUnitID" error. - * - If ObsUnitID is already in the set (duplicate), adds a "duplicate ObsUnitID" error. - * - Otherwise, adds the ObsUnitID to the set of unique IDs. - * 5. Errors are added using the addRowError method, specifying the OBS_UNIT_ID column and appropriate error messages. - */ - public static void validateReferenceOUIdValues(AppendOverwriteMiddlewareContext context) throws HttpStatusException, IllegalStateException { - ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + public static boolean hasUniqueIds(AppendOverwriteMiddlewareContext ctx, String colName) throws IllegalStateException { Set referenceOUIds = new HashSet<>(); + Column col = Optional.ofNullable(ctx.getImportContext().getData().column(colName)) + .orElseThrow(()->new IllegalStateException("Column "+colName+" not found")); - // Iterate through the import rows to process ObsUnit IDs - for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); - - if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { - // Check if ObsUnitID is blank - addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExpImportProcessConstants.ErrMessage.MISSING_OBS_UNIT_ID.getValue(), validationErrors, rowNum); - } else if (referenceOUIds.contains(importRow.getObsUnitID())) { - // Check if ObsUnitID is repeated - addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExpImportProcessConstants.ErrMessage.DUPLICATE_OBS_UNIT_ID.getValue(), validationErrors, rowNum); + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + if (referenceOUIds.contains(col.getString(rowNum))) { + return false; } else { - // Add ObsUnitID to referenceOUIds - referenceOUIds.add(importRow.getObsUnitID()); + referenceOUIds.add(col.getString(rowNum)); } } + + return true; } /** @@ -373,7 +357,7 @@ public static void validateReferenceOUIdValues(AppendOverwriteMiddlewareContext * to the context for each import row where the Observation Unit ID was not found. * * @param e The EntityNotFoundException containing information about missing Observation Unit IDs. - * @param context The AppendOverwriteMiddlewareContext containing import data and validation error storage. + * @param ctx The AppendOverwriteMiddlewareContext containing import data and validation error storage. * * @implNote The method performs the following steps: * 1. Retrieves the ValidationErrors object from the context. @@ -382,14 +366,15 @@ public static void validateReferenceOUIdValues(AppendOverwriteMiddlewareContext * 4. If a match is found, adds a validation error for that row, indicating an invalid Observation Unit ID. * 5. The error is added using the addRowError method, specifying the OBS_UNIT_ID column and using a predefined error message. */ - public static void addValidationErrorsForObsUnitsNotFound(EntityNotFoundException e, AppendOverwriteMiddlewareContext context) { - ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + public static void addValidationErrorsForObsUnitsNotFound(EntityNotFoundException e, AppendOverwriteMiddlewareContext ctx) { + ValidationErrors validationErrors = ctx.getAppendOverwriteWorkflowContext().getValidationErrors(); List errors = new ArrayList<>(); - for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { - String rowObsUnitId = ((ExperimentObservation)context.getImportContext().getImportRows().get(rowNum)).getObsUnitID(); - if (e.getMissingEntityIds().contains(rowObsUnitId)) { - addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExperimentUtilities.INVALID_OBS_UNIT_ID_ERROR, validationErrors, rowNum); + String obsUnitIDColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column idCol = ctx.getImportContext().getData().column(obsUnitIDColName); + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + if (e.getMissingEntityIds().contains(idCol.getString(rowNum))) { + addRowError(obsUnitIDColName, ExperimentUtilities.INVALID_OBS_UNIT_ID_ERROR, validationErrors, rowNum); } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index fbbc0afbd..1f815c80d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -19,6 +19,9 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; @@ -28,6 +31,7 @@ import org.breedinginsight.brapps.importer.services.ImportStatusService; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.AppendOverwriteIDValidation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.AppendOverwriteVariableValidation; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPICommit; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize.WorkflowInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.ImportTableProcess; @@ -36,6 +40,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.services.exceptions.ValidatorException; import javax.inject.Inject; import javax.inject.Singleton; @@ -46,25 +51,56 @@ @Singleton public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { private final ExperimentWorkflowNavigator.Workflow workflow; + private final AppendOverwriteMiddleware validationMiddleware; private final AppendOverwriteMiddleware importPreviewMiddleware; private final AppendOverwriteMiddleware brapiCommitMiddleware; private final ImportStatusService statusService; @Inject public AppendOverwritePhenotypesWorkflow(AppendOverwriteIDValidation expUnitIDValidation, + AppendOverwriteVariableValidation obsVariableValidation, WorkflowInitialization workflowInitialization, ImportTableProcess importTableProcess, BrAPICommit brAPICommit, ImportStatusService statusService){ this.statusService = statusService; this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; + this.validationMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link( + expUnitIDValidation, + obsVariableValidation); this.importPreviewMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link( - expUnitIDValidation, workflowInitialization, importTableProcess); this.brapiCommitMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link(brAPICommit); } + /** + * Determines if any validation or process errors are present in the context and handles result accordingly + * + * @param context The import service context containing upload, data, program, user, commit flag, and workflow information. + * @param result the ImportWorkflowResult to be modified in response to any errors found in context + * @return Pair containing a Boolean indicating whether any errors were found, and the potentially modified ImportWorkflowResult + */ + public Pair> checkForExistingErrors(AppendOverwriteMiddlewareContext context, Optional result){ + // Stop and return any validation errors + Optional validationErrorOptional = Optional + .ofNullable(context.getAppendOverwriteWorkflowContext().getValidationErrors()); + if (validationErrorOptional.isPresent() && validationErrorOptional.get().hasErrors()){ + result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.of(new ValidatorException(validationErrorOptional.get())))); + return new ImmutablePair<>(true, result); + } + + // Stop and return any errors that occurred while processing + Optional previewException = Optional + .ofNullable(context.getAppendOverwriteWorkflowContext().getProcessError()); + if (previewException.isPresent()) { + log.debug(String.format("%s in %s", previewException.get().getException().getClass().getName(), previewException.get().getLocalTransactionName())); + result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(previewException.get().getException()))); + return new ImmutablePair<>(true, result); + } + return new ImmutablePair<>(false, result); + } + /** * Processes the import workflow based on the provided import service context. * If the provided context is not valid or if the workflow is not equal to the context workflow, returns an empty Optional. @@ -101,6 +137,8 @@ public Optional process(ImportServiceContext context) { return result; } + Pair> errors; + // Build the workflow context for processing the import ImportContext importContext = ImportContext.builder() .upload(context.getUpload()) @@ -115,16 +153,19 @@ public Optional process(ImportServiceContext context) { .appendOverwriteWorkflowContext(new AppendOverwriteWorkflowContext()) .build(); + // Validate the import + AppendOverwriteMiddlewareContext validatedImportContext = this.validationMiddleware.process(workflowContext); + + //Stop and return any validation or process errors, needs to be done before processing import preview to avoid null pointer exceptions + errors = checkForExistingErrors(validatedImportContext, result); + if (errors.getLeft()) return errors.getRight(); + // Process the import preview - AppendOverwriteMiddlewareContext processedPreviewContext = this.importPreviewMiddleware.process(workflowContext); + AppendOverwriteMiddlewareContext processedPreviewContext = this.importPreviewMiddleware.process(validatedImportContext); - // Stop and return any errors that occurred while processing - Optional previewException = Optional.ofNullable(processedPreviewContext.getAppendOverwriteWorkflowContext().getProcessError()); - if (previewException.isPresent()) { - log.debug(String.format("%s in %s", previewException.get().getException().getClass().getName(), previewException.get().getLocalTransactionName())); - result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(previewException.get().getException()))); - return result; - } + // Stop and return any validation or process errors + errors = checkForExistingErrors(validatedImportContext, result); + if (errors.getLeft()) return errors.getRight(); // Build and return the preview response ImportPreviewResponse response = new ImportPreviewResponse(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java index 216006f23..0a831d56c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java @@ -17,9 +17,11 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity; +import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import com.google.gson.JsonArray; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; @@ -27,19 +29,16 @@ import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.DatasetUtil; +import org.breedinginsight.utilities.Utilities; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; @Prototype @@ -49,16 +48,19 @@ public class PendingDataset implements ExperimentImportEntity BrAPIListDAO brAPIListDAO; DatasetService datasetService; ExperimentUtilities experimentUtilities; + String referenceSourceBase; public PendingDataset(AppendOverwriteMiddlewareContext context, BrAPIListDAO brAPIListDAO, DatasetService datasetService, - ExperimentUtilities experimentUtilities) { + ExperimentUtilities experimentUtilities, + String referenceSourceBase) { this.cache = context.getAppendOverwriteWorkflowContext(); this.importContext = context.getImportContext(); this.brAPIListDAO = brAPIListDAO; this.datasetService = datasetService; this.experimentUtilities = experimentUtilities; + this.referenceSourceBase = referenceSourceBase; } /** * Create new objects generated by the workflow in the BrAPI service. @@ -103,14 +105,25 @@ public List brapiPost(List members) throws A */ @Override public List brapiRead() throws ApiException { - // Get the id of the dataset belonging to the required exp units - JsonArray datasetsJson = cache.getTrialByNameNoScope().values().iterator().next().getBrAPIObject() - .getAdditionalInfo() - .getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); - String datasetId = DatasetUtil.getTopLevelDatasetFromJson(datasetsJson).getId().toString(); - - // Get the dataset belonging to required exp units - return List.of(datasetService.fetchDatasetById(datasetId, importContext.getProgram()).orElseThrow(ApiException::new)); + // Get the dataset id from the xref of any observation unit in the import + List refs = cache + .getPendingObsUnitByOUId() + .values() + .stream() + .findFirst() + .get() + .getBrAPIObject() + .getExternalReferences(); + String datasetId = Utilities + .getExternalReference(refs, referenceSourceBase, ExternalReferenceSource.DATASET) + .map(BrAPIExternalReference::getReferenceId) + .get(); + + // Get the dataset + return datasetService + .fetchDatasetById(datasetId, importContext.getProgram()) + .map(List::of) + .orElseGet(List::of); } /** @@ -230,18 +243,22 @@ public void initializeWorkflow(List members) { .collect(Collectors.toMap(pio -> pio.getBrAPIObject().getListName(),pio -> pio)); // Construct a hashmap to look up the pending dataset by the observation unit ID of a unit stored in the BrAPI service - Map> pendingObsDatasetByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - e -> { - if (cache.getPendingTrialByOUId().isEmpty() || - pendingDatasetByName.isEmpty() || - cache.getPendingTrialByOUId().values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { - throw new IllegalStateException("There is not an observation data set for this unit: " + e.getKey()); + Map> pendingObsDatasetByOUId; + if (pendingDatasetByName.isEmpty()) { + pendingObsDatasetByOUId = Collections.emptyMap(); + } else { + pendingObsDatasetByOUId = cache.getPendingObsUnitByOUId().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> { + if (cache.getPendingTrialByOUId().isEmpty() || + cache.getPendingTrialByOUId().values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { + throw new IllegalStateException("There is not an observation data set for this unit: " + e.getKey()); + } + return pendingDatasetByName.values().iterator().next(); } - return pendingDatasetByName.values().iterator().next(); - } - )); + )); + } // Add the maps to the context for use in processing import cache.setObsVarDatasetByName(pendingDatasetByName); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java index 20e827f2f..440d663cc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingEntityFactory.java @@ -19,6 +19,7 @@ import io.micronaut.context.annotation.Bean; import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; @@ -45,6 +46,7 @@ public class PendingEntityFactory { private final ProgramLocationService programLocationService; private final LocationService locationService; private final ExperimentUtilities experimentUtilities; + private final String referenceSourceBase; @Inject public PendingEntityFactory(TrialService trialService, @@ -58,7 +60,8 @@ public PendingEntityFactory(TrialService trialService, DatasetService datasetService, BrAPIObservationDAO brAPIObservationDAO, OntologyService ontologyService, ProgramLocationService programLocationService, LocationService locationService, - ExperimentUtilities experimentUtilities) { + ExperimentUtilities experimentUtilities, + @Property(name = "brapi.server.reference-source") String referenceSourceBase) { this.trialService = trialService; this.brapiTrialDAO = brapiTrialDAO; this.observationUnitDAO = observationUnitDAO; @@ -73,6 +76,7 @@ public PendingEntityFactory(TrialService trialService, this.programLocationService = programLocationService; this.locationService = locationService; this.experimentUtilities = experimentUtilities; + this.referenceSourceBase = referenceSourceBase; } public static PendingTrial pendingTrial(AppendOverwriteMiddlewareContext context, @@ -105,8 +109,9 @@ public static PendingGermplasm pendingGermplasm(AppendOverwriteMiddlewareContext public static PendingDataset pendingDataset(AppendOverwriteMiddlewareContext context, BrAPIListDAO brAPIListDAO, DatasetService datasetService, - ExperimentUtilities experimentUtilities) { - return new PendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); + ExperimentUtilities experimentUtilities, + String referenceSourceBase) { + return new PendingDataset(context, brAPIListDAO, datasetService, experimentUtilities, referenceSourceBase); } public static PendingObservation pendingObservation(AppendOverwriteMiddlewareContext context, @@ -150,7 +155,7 @@ public PendingGermplasm pendingGermplasmBean(AppendOverwriteMiddlewareContext co @Bean @Prototype public PendingDataset pendingDatasetBean(AppendOverwriteMiddlewareContext context) { - return pendingDataset(context, brAPIListDAO, datasetService, experimentUtilities); + return pendingDataset(context, brAPIListDAO, datasetService, experimentUtilities, referenceSourceBase); } @Bean diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java index a2edbdf65..42fe01230 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java @@ -23,12 +23,14 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID.ObservationUnitIDValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIReadFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowReadInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.BadRequestException; import org.breedinginsight.services.exceptions.ValidatorException; import javax.inject.Inject; @@ -40,29 +42,41 @@ public class AppendOverwriteIDValidation extends AppendOverwriteMiddleware { WorkflowReadInitialization brAPIObservationUnitReadWorkflowInitialization; BrAPIReadFactory brAPIReadFactory; + ObservationUnitIDValidator ouIdValidator; @Inject - public AppendOverwriteIDValidation(BrAPIReadFactory brAPIReadFactory) { + public AppendOverwriteIDValidation(BrAPIReadFactory brAPIReadFactory, ObservationUnitIDValidator ouIdValidator) { this.brAPIReadFactory = brAPIReadFactory; + this.ouIdValidator = ouIdValidator; } + @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { brAPIObservationUnitReadWorkflowInitialization = brAPIReadFactory.observationUnitWorkflowReadInitializationBean(context); - // Initialize the validation error collection + // Initialize the tabular error collection Optional.ofNullable(context.getAppendOverwriteWorkflowContext().getValidationErrors()).orElseGet(() -> { context.getAppendOverwriteWorkflowContext().setValidationErrors(new ValidationErrors()); return new ValidationErrors(); }); ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); - ExperimentUtilities.validateReferenceOUIdValues(context); // Check for missing or duplicate OU ids - Set uniqueOUIds = ExperimentUtilities.collateUniqueOUIds(context); - context.getAppendOverwriteWorkflowContext().setReferenceOUIds(uniqueOUIds); + try { - brAPIObservationUnitReadWorkflowInitialization.execute(); // Fetch the obs units from the BrAPi service + ouIdValidator.validateDynamicColumns(context); + Set uniqueOUIds = ExperimentUtilities.collateUniqueOUIds(context); + context.getAppendOverwriteWorkflowContext().setReferenceOUIds(uniqueOUIds); + + // Check for tabular errors collected during validation if (validationErrors.hasErrors()) { throw new ValidatorException(validationErrors); } + + // Fetch the obs units from the BrAPi service + brAPIObservationUnitReadWorkflowInitialization.execute(); + + // Validate retrieved observation units + ouIdValidator.validateDynamicColumns(context); + return processNext(context); } catch (EntityNotFoundException e) { /** @@ -72,7 +86,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext ExperimentUtilities.addValidationErrorsForObsUnitsNotFound(e, context); context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(new ValidatorException(validationErrors))); return this.compensate(context); - } catch (ApiException | ValidatorException e) { + } catch (BadRequestException | ApiException | ValidatorException e) { /** * If OUs were fetched for all unique reference ids but some of the reference ids failed validation, * return an error response and a list of rows with duplicate or missing ids diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteVariableValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteVariableValidation.java new file mode 100644 index 000000000..381ba1113 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteVariableValidation.java @@ -0,0 +1,84 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIReadFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowReadInitialization; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationVariable.ObservationVariableValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; +import org.breedinginsight.services.exceptions.BadRequestException; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.api.model.v1.response.ValidationErrors; + +@Slf4j +@Prototype +public class AppendOverwriteVariableValidation extends AppendOverwriteMiddleware { + private final ObservationVariableValidator obsVarValidator; + private final BrAPIReadFactory brAPIReadFactory; + WorkflowReadInitialization brAPITrialReadWorkflowInitialization; + WorkflowReadInitialization brAPIDatasetReadWorkflowInitialization; + + public AppendOverwriteVariableValidation(ObservationVariableValidator obsVarValidator, + BrAPIReadFactory brAPIReadFactory) { + this.obsVarValidator = obsVarValidator; + this.brAPIReadFactory = brAPIReadFactory; + } + + @Override + public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + brAPITrialReadWorkflowInitialization = brAPIReadFactory.trialWorkflowReadInitializationBean(context); + brAPIDatasetReadWorkflowInitialization = brAPIReadFactory.datasetWorkflowReadInitializationBean(context); + ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + + try { + // Validate any dynamic columns that are phenotype variables + obsVarValidator.validateDynamicColumns(context); + + // Check for tabular errors collected during validation + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); + } + + // Fetch the observation variables owned by all datasets belonging to the experiment involved in the import + brAPITrialReadWorkflowInitialization.execute(); + brAPIDatasetReadWorkflowInitialization.execute(); + + // Validate again to check that none of the phenotypes in the import belong to other datasets + obsVarValidator.validateDynamicColumns(context); + + return processNext(context); + } catch ( EntityNotFoundException e) { + // TODO: change method to handle errors with other entities besides obs units + ExperimentUtilities.addValidationErrorsForObsUnitsNotFound(e, context); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(new ValidatorException(validationErrors))); + return this.compensate(context); + } catch (BadRequestException | ApiException | ValidatorException e) { + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); + return this.compensate(context); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index 2aa889fc2..decd495a2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -39,10 +39,8 @@ @Slf4j @Prototype public class WorkflowInitialization extends AppendOverwriteMiddleware { - WorkflowReadInitialization brAPITrialReadWorkflowInitialization; WorkflowReadInitialization brAPIStudyReadWorkflowInitialization; WorkflowReadInitialization locationReadWorkflowInitialization; - WorkflowReadInitialization brAPIDatasetReadWorkflowInitialization; WorkflowReadInitialization brAPIGermplasmReadWorkflowInitialization; BrAPIReadFactory brAPIReadFactory; @@ -52,18 +50,14 @@ public WorkflowInitialization(BrAPIReadFactory brAPIReadFactory) { } @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { - brAPITrialReadWorkflowInitialization = brAPIReadFactory.trialWorkflowReadInitializationBean(context); brAPIStudyReadWorkflowInitialization = brAPIReadFactory.studyWorkflowReadInitializationBean(context); locationReadWorkflowInitialization = brAPIReadFactory.locationWorkflowReadInitializationBean(context); - brAPIDatasetReadWorkflowInitialization = brAPIReadFactory.datasetWorkflowReadInitializationBean(context); brAPIGermplasmReadWorkflowInitialization = brAPIReadFactory.germplasmWorkflowReadInitializationBean(context); log.debug("reading required BrAPI data from BrAPI service"); try { - brAPITrialReadWorkflowInitialization.execute(); brAPIStudyReadWorkflowInitialization.execute(); locationReadWorkflowInitialization.execute(); - brAPIDatasetReadWorkflowInitialization.execute(); brAPIGermplasmReadWorkflowInitialization.execute(); } catch (ApiException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java index 0893df9f2..58381bcbe 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java @@ -18,6 +18,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; import io.micronaut.context.annotation.Prototype; +import lombok.Setter; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import java.util.HashSet; @@ -32,6 +33,8 @@ public class AppendStatistic { private int newCount; private int existingCount; private int mutatedCount; + @Setter + private int obsVarCount; public AppendStatistic() { this.clearData(); @@ -44,6 +47,7 @@ public void clearData() { this.newCount = 0; this.existingCount = 0; this.mutatedCount = 0; + this.obsVarCount = 0; } public int incrementNewCount(Integer value) { int increment = 0; @@ -94,6 +98,7 @@ public Map constructPreviewMap() { ImportPreviewStatistics newStats = ImportPreviewStatistics.builder().newObjectCount(newCount).build(); ImportPreviewStatistics existingStats = ImportPreviewStatistics.builder().newObjectCount(existingCount).build(); ImportPreviewStatistics mutatedStats = ImportPreviewStatistics.builder().newObjectCount(mutatedCount).build(); + ImportPreviewStatistics obsVarStats = ImportPreviewStatistics.builder().newObjectCount(obsVarCount).build(); return Map.of( "Environments", environmentStats, @@ -101,7 +106,8 @@ public Map constructPreviewMap() { "GIDs", gidStats, "Observations", newStats, "Existing_Observations", existingStats, - "Mutated_Observations", mutatedStats + "Mutated_Observations", mutatedStats, + "Observation_Variables", obsVarStats ); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index f2e74982e..9aa92fca7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -22,27 +22,23 @@ import com.google.gson.GsonBuilder; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.HttpStatus; -import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.ProcessedDataFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; @@ -60,7 +56,6 @@ import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; import org.breedinginsight.utilities.Utilities; -import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; import javax.inject.Inject; @@ -68,8 +63,6 @@ import java.util.stream.Collectors; import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_REGEX; @Slf4j @Prototype @@ -108,57 +101,14 @@ public ImportTableProcess(StudyService studyService, @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { - log.debug("verifying traits listed in import"); - - // Get all the dynamic columns of the import - ImportUpload upload = context.getImportContext().getUpload(); - Table data = context.getImportContext().getData(); - String[] dynamicColNames = upload.getDynamicColumnNames(); - - // don't allow periods (.) or square brackets in Dynamic Column Names - for (String dynamicColumnName: dynamicColNames) { - if(dynamicColumnName.contains(".") || dynamicColumnName.contains("[") || dynamicColumnName.contains("]")){ - String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", dynamicColumnName); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, errorMsg); - } - } - List> dynamicCols = data.columns(dynamicColNames); - - // Collect the columns for observation variable data - List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); - List varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList()); - - // Collect the columns for observation timestamps - List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); - Set tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toSet()); - - // Construct validation errors for any timestamp columns that don't have a matching variable column - List importRows = context.getImportContext().getImportRows(); - Optional.ofNullable(context.getAppendOverwriteWorkflowContext().getValidationErrors()).orElseGet(() -> { - context.getAppendOverwriteWorkflowContext().setValidationErrors(new ValidationErrors()); - return new ValidationErrors(); - }); - ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); - List tsValErrs = observationVariableService.validateMatchedTimestamps(Set.copyOf(varNames), timestampCols).orElse(new ArrayList<>()); - for (int i = 0; i < importRows.size(); i++) { - int rowNum = i; - tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); - } - try { - // Stop processing the import if there are unmatched timestamp columns - if (tsValErrs.size() > 0) { - throw new UnprocessableEntityException("One or more timestamp columns do not have a matching observation variable"); - } - - //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval - Map> tsColByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); - - // Add the map to the context for use in processing import - context.getAppendOverwriteWorkflowContext().setTimeStampColByPheno(tsColByPheno); + ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + Map> tsColByPheno = context.getAppendOverwriteWorkflowContext().getTimeStampColByPheno(); + List> phenotypeCols = context.getAppendOverwriteWorkflowContext().getPhenotypeCols(); + List varNames = context.getAppendOverwriteWorkflowContext().getVarNames(); + Program program = context.getImportContext().getProgram(); // Fetch the traits named in the observation variable columns - Program program = context.getImportContext().getProgram(); List traits = observationVariableService.fetchTraitsByName(Set.copyOf(varNames), program); // Map trait by phenotype column name @@ -174,17 +124,28 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext // Sort the traits to match the order of the headers in the import file List sortedTraits = experimentUtil.sortByField(varNames, new ArrayList<>(traits), TraitEntity::getObservationVariableName); - // Get the pending observation dataset - PendingImportObject pendingTrial = ExperimentUtilities.getSingleEntryValue(context.getAppendOverwriteWorkflowContext().getTrialByNameNoScope()).orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); - String datasetName = String.format("Observation Dataset [%s-%s]", program.getKey(), pendingTrial.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString()); - PendingImportObject pendingDataset = context.getAppendOverwriteWorkflowContext().getObsVarDatasetByName().get(datasetName); + // Get the pending observation dataset; there should only be a single dataset used for the import + PendingImportObject pendingTrial = ExperimentUtilities + .getSingleEntryValue(context.getAppendOverwriteWorkflowContext().getTrialByNameNoScope()) + .orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); + PendingImportObject pendingDataset = context + .getAppendOverwriteWorkflowContext() + .getPendingObsDatasetByOUId() + .values() + .stream() + .findAny() + .orElseGet(()-> new PendingImportObject(ImportObjectState.NEW, new BrAPIListDetails())); // Add new phenotypes to the pending observation dataset list (NOTE: "obsVarName [programKey]" is used instead of obsVarDbId) // TODO: Change to using actual dbIds as per the BrAPI spec, instead of namespaced obsVar names (was necessary for Breedbase) - List datasetObsVarDbIds = pendingDataset.getBrAPIObject().getData().stream().collect(Collectors.toList()); + List datasetObsVarDbIds = Optional.ofNullable(pendingDataset.getBrAPIObject().getData()) + .map(ArrayList::new) + .orElseGet(ArrayList::new); List phenoDbIds = sortedTraits.stream().map(t->Utilities.appendProgramKey(t.getObservationVariableName(), program.getKey())).collect(Collectors.toList()); phenoDbIds.removeAll(datasetObsVarDbIds); - pendingDataset.getBrAPIObject().getData().addAll(phenoDbIds); + for (String phenoDbId : phenoDbIds) { + pendingDataset.getBrAPIObject().addDataItem(phenoDbId); + } // Update pending status if (ImportObjectState.EXISTING == pendingDataset.getState()) { @@ -193,12 +154,19 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext // Read any observation data stored for these traits log.debug("fetching observation data stored for traits"); - Set ouDbIds = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); - Set varDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); - List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, varDbIds, program); + Set ouBrapiDbIds = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); + Set varBrapiDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); + List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouBrapiDbIds, varBrapiDbIds, program); + + // OU Exref Ids are what are displayed to users in the Obs Unit Id columns in experiments in DeltaBreed. + Map OuExRefIdByObsBrAPIDbId = new HashMap<>(); + + for (BrAPIObservation observation : observations) { + Optional ouExref = Utilities.getExternalReference(observation.getExternalReferences(), brapiReferenceSource, ExternalReferenceSource.OBSERVATION_UNITS); + ouExref.ifPresent(exRef -> OuExRefIdByObsBrAPIDbId.put(observation.getObservationDbId(), exRef.getReferenceId().toString())); + } // Construct helper lookup tables to use for hashing stored observation data - Map unitNameByDbId = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, BrAPIObservationUnit::getObservationUnitName)); Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); Map studyNameByDbId = context.getAppendOverwriteWorkflowContext().getStudyByNameNoScope().values().stream() .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) @@ -207,7 +175,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext // Hash stored observation data using a signature of unit, variable, and study names Map observationByObsHash = observations.stream().collect(Collectors.toMap(o->{ - return observationService.getObservationHash(unitNameByDbId.get(o.getObservationUnitDbId()), + return observationService.getObservationHash(OuExRefIdByObsBrAPIDbId.get(o.getObservationDbId()), variableNameByDbId.get(o.getObservationVariableDbId()), studyNameByDbId.get(o.getStudyDbId())); }, o->o)); @@ -233,7 +201,9 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext return new HashMap<>(); }); PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); - String unitId = row.getObsUnitID(); + String obsUnitIDColName = context.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column obsUnitIDCol = context.getImportContext().getData().column(obsUnitIDColName); + String unitId = obsUnitIDCol.getString(rowNum); String studyName = context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); mappedImportRow.setTrial(context.getAppendOverwriteWorkflowContext().getPendingTrialByOUId().get(unitId)); mappedImportRow.setLocation(context.getAppendOverwriteWorkflowContext().getPendingLocationByOUId().get(unitId)); @@ -248,7 +218,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext */ if (phenotypeCols.isEmpty()) { processedData = processedDataFactory.undefinedDatasetBean(); - updatePreviewStatistics(processedData, context, studyName, unitId); + updatePreviewStatistics(processedData, context, studyName, unitId, phenotypeCols.size()); } // Assemble the pending observation data for all phenotypes @@ -256,9 +226,8 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext String cellData = column.getString(rowNum); // Generate hash for looking up prior observation data - String unitName = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); String phenoColumnName = column.name(); - String observationHash = observationService.getObservationHash(unitName, phenoColumnName, studyName); + String observationHash = observationService.getObservationHash(unitId, phenoColumnName, studyName); // Get timestamp if associated column var cell = new Object() { // mutable reference object to make timestamp accessible in anonymous methods @@ -319,7 +288,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext } else if (!cellData.isBlank()) { // Clone the observation unit and trait - BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(row.getObsUnitID()).getBrAPIObject()), BrAPIObservationUnit.class); + BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(obsUnitIDCol.getString(rowNum)).getBrAPIObject()), BrAPIObservationUnit.class); Trait initialTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); // create new instance of InitialData @@ -342,7 +311,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context.getImportContext().getProgram()); } else { // Clone the observation unit - BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(row.getObsUnitID()).getBrAPIObject()), BrAPIObservationUnit.class); + BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(obsUnitIDCol.getString(rowNum)).getBrAPIObject()), BrAPIObservationUnit.class); processedData = processedDataFactory.emptyDataBean(brapiReferenceSource, context.getImportContext().isCommit(), @@ -363,7 +332,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e -> validationErrors.addError(rowNum + 2, e))); // +2 to account for header row and excel file 1-based row index // Update import preview statistics and set in the context - updatePreviewStatistics(processedData, context, studyName, unitId); + updatePreviewStatistics(processedData, context, studyName, unitId, varNames.size()); // Construct a pending observation Optional> pendingProcessedData = Optional.ofNullable(processedData.constructPendingObservation()); @@ -426,12 +395,14 @@ private boolean isChanged(String cellData, BrAPIObservation observation, String private void updatePreviewStatistics(VisitedObservationData processedData, AppendOverwriteMiddlewareContext context, String studyName, - String unitId) { + String unitId, + int obsVarCount) { // Update import preview statistics and set in the context processedData.updateTally(statistic); statistic.addEnvironmentName(studyName); statistic.addObservationUnitId(unitId); statistic.addGid(context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getAccessionNumber()); + statistic.setObsVarCount(obsVarCount); context.getAppendOverwriteWorkflowContext().setStatistic(statistic); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java index 13d892722..e42b6185a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/AppendOverwriteWorkflowContext.java @@ -31,14 +31,14 @@ import org.breedinginsight.model.ProgramLocation; import tech.tablesaw.columns.Column; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; @Getter @Setter public class AppendOverwriteWorkflowContext { + // Dynamic Columns + private String obsUnitColName; + // Cache maps keyed by existing observation unit ids private Set referenceOUIds = new HashSet<>(); private Map> pendingTrialByOUId = new HashMap<>(); @@ -55,6 +55,10 @@ public class AppendOverwriteWorkflowContext { private MiddlewareException processError; private ValidationErrors validationErrors; + // Dynamic Columns + private List> phenotypeCols; + private List varNames; + // Cache maps keyed by name without program scope private Map> observationUnitByNameNoScope; private Map> trialByNameNoScope; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/DynamicObsUnitValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/DynamicObsUnitValidator.java new file mode 100644 index 000000000..166a6fd0e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/DynamicObsUnitValidator.java @@ -0,0 +1,28 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; + +import io.micronaut.core.order.Ordered; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.services.exceptions.BadRequestException; + +@FunctionalInterface +public interface DynamicObsUnitValidator extends Ordered { + void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException, ApiException; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java new file mode 100644 index 000000000..54a70cf57 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitDuplicateIDValidator.java @@ -0,0 +1,66 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.services.exceptions.BadRequestException; +import tech.tablesaw.columns.Column; + +import javax.inject.Singleton; +import java.util.HashSet; +import java.util.Set; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.OZEX; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.VVCN; + +@Slf4j +@Singleton +public class ObservationUnitDuplicateIDValidator implements DynamicObsUnitValidator { + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + // Skip this validation if the observation units have already been fetched from the BrAPI service + if (!ctx.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().isEmpty()) return; + + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() == null) { + throw new BadRequestException(OZEX.getValue()); + } + + ValidationErrors rowErrors = ctx.getAppendOverwriteWorkflowContext().getValidationErrors(); + Set referenceOUIds = new HashSet<>(); + String idColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column idCol = ctx.getImportContext().getData().columns(idColName).get(0); + + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + if (referenceOUIds.contains(idCol.get(rowNum).toString())) { + // Check if ObsUnitID is duplicated + ExperimentUtilities.addRowError(idColName, VVCN.getValue(), rowErrors, rowNum); + } else { + // Add ObsUnitID to referenceOUIds + referenceOUIds.add(idCol.get(rowNum).toString()); + } + } + } + + @Override + public int getOrder() { + return 4; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java new file mode 100644 index 000000000..b69234a1c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDBlankValidator.java @@ -0,0 +1,62 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.services.exceptions.BadRequestException; +import tech.tablesaw.columns.Column; + +import javax.inject.Singleton; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.BITB; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.OZEX; + +@Slf4j +@Singleton +public class ObservationUnitIDBlankValidator implements DynamicObsUnitValidator { + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + // Skip this validation if the observation units have already been fetched from the BrAPI service + if (!ctx.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().isEmpty()) return; + + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() == null) { + throw new BadRequestException(OZEX.getValue()); + } + + ValidationErrors rowErrors = ctx.getAppendOverwriteWorkflowContext().getValidationErrors(); + String idColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column idCol = ctx.getImportContext().getData().columns(idColName).get(0); + + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + Object cellValue = idCol.get(rowNum); + String id = (cellValue != null) ? cellValue.toString() : null; + if ( id == null || id.isBlank()) { + // Check if ObsUnitID is blank + ExperimentUtilities.addRowError(idColName, BITB.getValue(), rowErrors, rowNum); + } + } + } + + @Override + public int getOrder() { + return 2; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java new file mode 100644 index 000000000..98a9cb8f1 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDColumnNameValidator.java @@ -0,0 +1,82 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.services.exceptions.BadRequestException; + +import javax.inject.Singleton; +import java.util.Arrays; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.OZEX; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.SUB_UNIT_ID; + +@Slf4j +@Singleton +public class ObservationUnitIDColumnNameValidator implements DynamicObsUnitValidator { + + public ObservationUnitIDColumnNameValidator() {} + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + // Skip this validation if it has already been successfully completed + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() != null) return; + + // Skip this validation if the observation units have already been fetched from the BrAPI service + if (!ctx.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().isEmpty()) return; + + // Get the names of all the dynamic columns with observation unit ids + String[] idColNames = Arrays.stream(ctx.getImportContext().getUpload().getDynamicColumnNames()) + .filter(name->name.endsWith(OBSERVATION_UNIT_ID_SUFFIX)).toArray(String[]::new); + + // throw an error if there are no obs unit id columns in the import + int idColCount = idColNames.length; + if (idColCount == 0) throw new BadRequestException(OZEX.getValue()); + + // count the number of columns and throw an error if the count is neither 1 and unique nor 2 with 1 unique and 1 non-unique + if (!(idColCount == 1 || idColCount == 2)) throw new BadRequestException(OZEX.getValue()); + + if (idColCount == 2) { + // if sub-entity ids in import then check for presence of sub-unit # column + Arrays.stream(ctx.getImportContext().getUpload().getDynamicColumnNames()) + .filter(name-> name.equals(SUB_UNIT_ID)) + .findAny() + .orElseThrow(()->new BadRequestException(OZEX.getValue())); + + // check that if there is a column with unique ids, it is the right-most column + boolean leftIsUnique = ExperimentUtilities.hasUniqueIds(ctx, idColNames[0]); + boolean rightIsUnique = ExperimentUtilities.hasUniqueIds(ctx, idColNames[1]); + if (leftIsUnique && !rightIsUnique) throw new BadRequestException(OZEX.getValue()); + + // the right column should be the most nested level and this column is to be used for processing the import + ctx.getAppendOverwriteWorkflowContext().setObsUnitColName(idColNames[1]); + } else if (idColCount == 1) { + + // there is only one top level whose column is used for processing the import + ctx.getAppendOverwriteWorkflowContext().setObsUnitColName(idColNames[0]); + } + } + + @Override + public int getOrder() { + return 1; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java new file mode 100644 index 000000000..e35ffd884 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDFormatValidator.java @@ -0,0 +1,71 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.services.exceptions.BadRequestException; +import tech.tablesaw.columns.Column; +import java.util.regex.Pattern; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.BITB; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.OZEX; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; + +import javax.inject.Singleton; + +@Slf4j +@Singleton +public class ObservationUnitIDFormatValidator implements DynamicObsUnitValidator { + private static final Pattern UUID_PATTERN = Pattern.compile( + "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" + ); + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + // Skip this validation if the observation units have already been fetched from the BrAPI service + if (!ctx.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().isEmpty()) return; + + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() == null) { + throw new BadRequestException(OZEX.getValue()); + } + + ValidationErrors rowErrors = ctx.getAppendOverwriteWorkflowContext().getValidationErrors(); + String idColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column idCol = ctx.getImportContext().getData().column(idColName); + + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + Object value = idCol.get(rowNum); + String id = value != null ? value.toString() : null; + + // Validate UUID format + if (id == null || !UUID_PATTERN.matcher(id).matches()) { + if (!rowErrors.hasErrorAtCell(rowNum + 2, idColName)) { // take header row into account + // don't add another error for format if it already has an error for being blank + ExperimentUtilities.addRowError(idColName, BITB.getValue(), rowErrors, rowNum); + } + } + } + } + + @Override + public int getOrder() { + return 3; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDValidator.java new file mode 100644 index 000000000..dc37d7e16 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitIDValidator.java @@ -0,0 +1,44 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; + +import io.micronaut.context.annotation.Primary; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.services.exceptions.BadRequestException; + +import javax.inject.Singleton; +import java.util.List; + +@Primary +@Singleton +public class ObservationUnitIDValidator implements DynamicObsUnitValidator { + private final List validators; + + public ObservationUnitIDValidator(List validators) { + this.validators = validators; + } + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) + throws BadRequestException, ApiException { + for (DynamicObsUnitValidator validator : validators) { + validator.validateDynamicColumns(ctx); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitSingleDatasetValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitSingleDatasetValidator.java new file mode 100644 index 000000000..ef01603ed --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationUnitID/ObservationUnitSingleDatasetValidator.java @@ -0,0 +1,89 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationUnitID; + +import io.micronaut.context.annotation.Property; +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.services.exceptions.BadRequestException; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.columns.Column; + +import javax.inject.Singleton; + +import java.util.List; +import java.util.Map; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.OZEX; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.PJZH; + +@Slf4j +@Singleton +public class ObservationUnitSingleDatasetValidator implements DynamicObsUnitValidator { + private final String referenceSourceBase; + + public ObservationUnitSingleDatasetValidator(@Property(name = "brapi.server.reference-source") String referenceSourceBase) { + this.referenceSourceBase = referenceSourceBase; + } + + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + // Skip this validation if the observation units have not been fetched from the BrAPI service + if (ctx.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().isEmpty()) return; + + if (ctx.getAppendOverwriteWorkflowContext().getObsUnitColName() == null) { + throw new BadRequestException(OZEX.getValue()); + } + + String idColName = ctx.getAppendOverwriteWorkflowContext().getObsUnitColName(); + Column idCol = ctx.getImportContext().getData().columns(idColName).get(0); + Map> unitPioCache = ctx.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId(); + String singleDatasetId = null; + for (int rowNum = 0; rowNum < ctx.getImportContext().getImportRows().size(); rowNum++) { + // Get the external references for the BrAPI Observation Unit stored with the given id + List refs = unitPioCache + .get(idCol.get(rowNum).toString()) + .getBrAPIObject() + .getExternalReferences(); + + // Find the id of the dataset that owns the given observation unit + String datasetId = Utilities + .getExternalReference(refs, referenceSourceBase, ExternalReferenceSource.DATASET) + .map(BrAPIExternalReference::getReferenceId) + .orElseThrow(() -> new BadRequestException(PJZH.getValue())); + + // Make sure there is only a single unique dataset used in the import + if (singleDatasetId == null) { + singleDatasetId = datasetId; + } else if (!singleDatasetId.equals(datasetId)) { + throw new BadRequestException(PJZH.getValue()); + } + } + + } + + @Override + public int getOrder() { + return 5; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/DynamicObsVarValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/DynamicObsVarValidator.java new file mode 100644 index 000000000..b42cbee7c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/DynamicObsVarValidator.java @@ -0,0 +1,28 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationVariable; + +import io.micronaut.core.order.Ordered; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.services.exceptions.BadRequestException; + +@FunctionalInterface +public interface DynamicObsVarValidator extends Ordered { + void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException, ApiException; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java new file mode 100644 index 000000000..70ba472d9 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariablePriorDatasetValidator.java @@ -0,0 +1,154 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationVariable; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import io.micronaut.context.annotation.Property; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; +import org.breedinginsight.services.exceptions.BadRequestException; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.JABH; + +@Slf4j +@Singleton +public class ObservationVariablePriorDatasetValidator implements DynamicObsVarValidator { + private final String referenceSourceBase; + private final DatasetService datasetService; + + public ObservationVariablePriorDatasetValidator( + @Property(name = "brapi.server.reference-source") String referenceSourceBase, + DatasetService datasetService) { + this.referenceSourceBase = referenceSourceBase; + this.datasetService = datasetService; + } + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException, ApiException { + // Skip the validation if the dependencies have not been fetched from the BrAPI service + if(noMappings(ctx)) return; + + // Get the dataset id used for this import + String datasetId = getImportDatasetId(ctx); + + // Get the ids for the other datasets in the same experiment + Set otherIds = getExperimentDatasetIds(ctx); + otherIds.remove(datasetId); + + // Get the names of any observation variables owned by the other datasets + Set forbiddenVariables = getDatasetVariables(ctx, otherIds); + + // Check that no phenotype name used in the import already belongs to another dataset + if (forbidden(ctx, forbiddenVariables)) { + throw new BadRequestException(JABH.getValue()); + } + } + + @Override + public int getOrder() { + return 2; + } + + private Set getDatasetVariables(AppendOverwriteMiddlewareContext ctx, Set ids) throws ApiException { + Set variables = new HashSet<>(); + String progKey = ctx.getImportContext().getProgram().getKey(); + List varListDetails = datasetService + .fetchDatasetsByIds(ids, ctx.getImportContext().getProgram()).orElse(new ArrayList<>()); + for (BrAPIListDetails brAPIListDetails : varListDetails) { + List priorVariablesNoScope = brAPIListDetails + .getData() + .stream() + .map((scopedVariable) -> Utilities.removeProgramKey(scopedVariable, progKey)) + .collect(Collectors.toList()); + variables.addAll(priorVariablesNoScope); + } + + return variables; + } + + private boolean forbidden(AppendOverwriteMiddlewareContext ctx, Set forbiddenVariables) { + List importVarNames = ctx.getAppendOverwriteWorkflowContext().getVarNames(); + for (String importVarName : importVarNames) { + if (forbiddenVariables.contains(importVarName)) return true; + } + + return false; + } + + private Set getExperimentDatasetIds(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + JsonArray expDatasets = ctx.getAppendOverwriteWorkflowContext() + .getPendingTrialByOUId() + .values() + .stream() + .findFirst() + .orElseThrow(() -> new BadRequestException("No pending trial found")) + .getBrAPIObject() + .getAdditionalInfo() + .getAsJsonArray("datasets"); + Set datasetIds = new HashSet<>(); + for (JsonElement expDataset : expDatasets) { + String datasetId = expDataset.getAsJsonObject().get("id").getAsString(); + datasetIds.add(datasetId); + } + + return datasetIds; + } + + private String getImportDatasetId(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + List refs = ctx.getAppendOverwriteWorkflowContext() + .getPendingObsUnitByOUId() + .values() + .stream() + .findFirst() + .orElseThrow(()->new BadRequestException("No pending obs unit found")) + .getBrAPIObject() + .getExternalReferences(); + + return Utilities + .getExternalReference(refs, referenceSourceBase, ExternalReferenceSource.DATASET) + .map(BrAPIExternalReference::getReferenceId) + .orElseThrow(() -> new BadRequestException("No dataset associated with observation unit")); + } + + private boolean noMappings(AppendOverwriteMiddlewareContext context) { + Map> trialByOUId = context + .getAppendOverwriteWorkflowContext() + .getPendingTrialByOUId(); + Map> datasetByOUId = context + .getAppendOverwriteWorkflowContext() + .getPendingObsDatasetByOUId(); + + if (trialByOUId == null) return true; + if (trialByOUId.isEmpty()) return true; + if (datasetByOUId == null) return true; + return false; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java new file mode 100644 index 000000000..cb9b4ac21 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableTimestampValidator.java @@ -0,0 +1,110 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationVariable; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; +import org.breedinginsight.services.exceptions.BadRequestException; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; + +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.*; + +@Slf4j +@Singleton +public class ObservationVariableTimestampValidator implements DynamicObsVarValidator { + private final ObservationVariableService observationVariableService; + + public ObservationVariableTimestampValidator(ObservationVariableService observationVariableService) { + this.observationVariableService = observationVariableService; + } + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) throws BadRequestException { + log.debug("verifying traits listed in import"); + + // Get all the phenotypic columns of the import + ImportUpload upload = ctx.getImportContext().getUpload(); + Table data = ctx.getImportContext().getData(); + List phenotypeColNames = upload + .getDynamicColumnNamesList() + .stream() + .filter(name -> !name.endsWith(OBSERVATION_UNIT_ID_SUFFIX)) + .filter(name -> !name.contains(SUB_UNIT_ID)) + .filter(name -> !name.contains(SUB_OBS_UNIT)) + .collect(Collectors.toList()); + + // don't allow periods (.) or square brackets in Phenotype Column Names + for (String phenotypeColumnName: phenotypeColNames) { + if(phenotypeColumnName.contains(".") || phenotypeColumnName.contains("[") || phenotypeColumnName.contains("]")){ + String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", phenotypeColumnName); + throw new BadRequestException(errorMsg); + } + } + List> dynamicCols = data.columns(phenotypeColNames.toArray(new String[0])); + + // Collect the columns for observation variable data + List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); + List varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList()); + + // Add the phenotypes to the context for use in processing import + ctx.getAppendOverwriteWorkflowContext().setPhenotypeCols(phenotypeCols); + ctx.getAppendOverwriteWorkflowContext().setVarNames(varNames); + + // Collect the columns for observation timestamps + List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); + + // Construct validation errors for any timestamp columns that don't have a matching variable column + List importRows = ctx.getImportContext().getImportRows(); + ValidationErrors validationErrors = ctx.getAppendOverwriteWorkflowContext().getValidationErrors(); + List tsValErrs = observationVariableService + .validateMatchedTimestamps(Set.copyOf(varNames), timestampCols) + .orElse(new ArrayList<>()); + for (int i = 0; i < importRows.size(); i++) { + int rowNum = i; + tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); + } + + if (tsValErrs.isEmpty()) { + //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval + Map> tsColByPheno = timestampCols + .stream() + .collect(Collectors + .toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), + col -> col)); + + // Add the map to the context for use in processing import + ctx.getAppendOverwriteWorkflowContext().setTimeStampColByPheno(tsColByPheno); + } + } + + @Override + public int getOrder() { + return 1; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableValidator.java new file mode 100644 index 000000000..83cd9cd35 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/validator/dynamicColumns/observationVariable/ObservationVariableValidator.java @@ -0,0 +1,44 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.validator.dynamicColumns.observationVariable; + +import io.micronaut.context.annotation.Primary; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; +import org.breedinginsight.services.exceptions.BadRequestException; + +import javax.inject.Singleton; +import java.util.List; + +@Primary +@Singleton +public class ObservationVariableValidator implements DynamicObsVarValidator { + private final List validators; + + public ObservationVariableValidator(List validators) { + this.validators = validators; + } + + @Override + public void validateDynamicColumns(AppendOverwriteMiddlewareContext ctx) + throws BadRequestException, ApiException { + for (DynamicObsVarValidator validator : validators) { + validator.validateDynamicColumns(ctx); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index de3f60844..fe71cd39f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -45,7 +45,7 @@ public class PendingData { private Map> locationByName; private Map> obsVarDatasetByName; private Map> existingGermplasmByGID; - + private Map expUnitByTrialName; private Map> pendingObservationByHash; private Map> timeStampColByPheno; private ValidationErrors validationErrors; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index d0a4ca975..a22cb98bf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -17,11 +17,9 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; -import io.micronaut.context.annotation.Prototype; import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; -import lombok.val; import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.api.model.v1.response.ValidationErrors; @@ -35,7 +33,6 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.model.workflow.ImportContext; import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.ImportStatusService; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; @@ -46,14 +43,11 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateNewPendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.ValidatePendingImportObjectsStep; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentPhenotypeService; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; import javax.inject.Inject; -import javax.inject.Named; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; +import java.util.*; import lombok.Getter; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; @@ -63,7 +57,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Singleton; -import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; @Slf4j @Getter @@ -105,10 +101,16 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "ObsUnitIDs are detected"); } + //Make sure file only contains one exp title, check early cause avoids issues with titles corresponding to existing and new trials + List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(importRows); + if (experimentImportRows.stream().map(ExperimentObservation::getExpTitle).distinct().count() > 1) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue()); + } + statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); - ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); + ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context); populateNewPendingImportObjectsStep.process(processContext, phenotypeData); ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context, processContext.getPendingData(), phenotypeData, processedData); @@ -216,12 +218,8 @@ private long getNewObjectCount(ImportPreviewResponse response) { } private boolean containsObsUnitIDs(ImportContext importContext) { - List importRows = importContext.getImportRows(); - return importRows.stream() - .anyMatch(row -> { - ExperimentObservation expRow = (ExperimentObservation) row; - return StringUtils.isNotBlank(expRow.getObsUnitID()); - }); + return Arrays.stream(importContext.getUpload().getDynamicColumnNames()) + .anyMatch(name->name.endsWith(OBSERVATION_UNIT_ID_SUFFIX)); } // TODO: move to shared area: experiment import service @@ -268,6 +266,15 @@ private Map generateStatisticsMap(PendingData p .count() ); + // Assume observationVariableName is set when pending observations are created. + int obsVarCount = Math.toIntExact( + observationByHash.values() + .stream() + .map(o -> o.getBrAPIObject().getObservationVariableName()) + .distinct() + .count() + ); + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() .newObjectCount(environmentNameCounter.size()) .build(); @@ -286,6 +293,9 @@ private Map generateStatisticsMap(PendingData p ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() .newObjectCount(numMutatedObservations) .build(); + ImportPreviewStatistics obsVarStats = ImportPreviewStatistics.builder() + .newObjectCount(obsVarCount) + .build(); return Map.of( "Environments", environmentStats, @@ -293,7 +303,8 @@ private Map generateStatisticsMap(PendingData p "GIDs", gidStats, "Observations", observationStats, "Existing_Observations", existingObservationStats, - "Mutated_Observations", mutatedObservationStats + "Mutated_Observations", mutatedObservationStats, + "Observation_Variables", obsVarStats ); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index 12e6bcc99..bdcc8956f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -16,12 +16,14 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps; +import io.micronaut.http.HttpResponse; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPIProgram; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPIListNewRequest; @@ -42,22 +44,24 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; +import org.breedinginsight.model.DatasetLevel; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.model.Trait; import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities.PREEXISTING_EXPERIMENT_TITLE; + @Singleton @Slf4j public class CommitPendingImportObjectsStep { @@ -69,6 +73,7 @@ public class CommitPendingImportObjectsStep { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final ProgramLocationService locationService; private final OntologyService ontologyService; + private final DatasetService datasetService; @Inject public CommitPendingImportObjectsStep(BrAPIListDAO brAPIListDAO, @@ -77,7 +82,8 @@ public CommitPendingImportObjectsStep(BrAPIListDAO brAPIListDAO, BrAPIObservationDAO brAPIObservationDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO, ProgramLocationService locationService, - OntologyService ontologyService) { + OntologyService ontologyService, + DatasetService datasetService) { this.brAPIListDAO = brAPIListDAO; this.brapiTrialDAO = brapiTrialDAO; this.brAPIStudyDAO = brAPIStudyDAO; @@ -85,10 +91,13 @@ public CommitPendingImportObjectsStep(BrAPIListDAO brAPIListDAO, this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.locationService = locationService; this.ontologyService = ontologyService; + this.datasetService = datasetService; } // TODO: some common code between workflows here that could be broken out, removed append/update specific code - public void process(ProcessContext processContext, ProcessedData processedData) { + // TODO: For instance, multiple trials don't really exist in the use case which this code is used for: creating an experiment. + // TODO: This means having a Map of trials doesn't really make sense anymore, and should be replaced with a singular PendingImportObject + public void process(ProcessContext processContext, ProcessedData processedData) throws UnprocessableEntityException, ApiException { PendingData pendingData = processContext.getPendingData(); ImportContext importContext = processContext.getImportContext(); @@ -103,6 +112,33 @@ public void process(ProcessContext processContext, ProcessedData processedData) Map> locationByName = pendingData.getLocationByName(); Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); Map> observationByHash = pendingData.getObservationByHash(); + Map expUnitbyTrialName = pendingData.getExpUnitByTrialName(); + + // Assume that there's only one expUnit per trial in this workflow. + // This should be a safe assumption to make considering this workflow is for the creation of a single experiment. + // If that changes in the future, this code will break and need to be dealt with. + + if (expUnitbyTrialName.size() > 1) { + throw new InternalServerException("Multiple Experiment names exist during commit stage"); + } + + String expUnitName = expUnitbyTrialName.values() + .stream() + .findFirst() + .orElse(null); + + if (expUnitName == null) { + throw new InternalServerException("Experiment unit name not found during commit stage"); + } + + String brapiProgramId = Optional.of(program) + .map(Program::getBrapiProgram) + .map(BrAPIProgram::getProgramDbId) + .orElse(null); + + if (brapiProgramId == null) { + throw new InternalServerException("BrAPI program not found during commit stage"); + } List newTrials = ProcessorData.getNewObjects(pendingData.getTrialByNameNoScope()); @@ -130,6 +166,14 @@ public void process(ProcessContext processContext, ProcessedData processedData) List newObservationUnits = ProcessorData.getNewObjects(pendingData.getObservationUnitByNameNoScope()); + // Inject level names here + datasetService.updateObservationUnitsWithLevelNameDbIds(newObservationUnits, + program, + brapiProgramId, + expUnitName, + DatasetLevel.EXP_UNIT + ); + // filter out observations with no 'value' so they will not be saved List newObservations = ProcessorData.getNewObjects(observationByHash) .stream() @@ -138,6 +182,24 @@ public void process(ProcessContext processContext, ProcessedData processedData) AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); + // TODO: Implement more robust solution either brapi server side or possibly redis SETNX client side + // Do this check here, directly before creating new trials instead of in earlier step to minimize time window of race condition + if (!newTrials.isEmpty()) { + try { + List cachedTrials = brapiTrialDAO.getTrials(program.getId()); + List existingTrialNames = cachedTrials.stream().map(BrAPITrial::getTrialName).collect(Collectors.toList()); + List newTrialNames = newTrials.stream().map(t -> Utilities.removeProgramKey(t.getTrialName(), program.getKey())).collect(Collectors.toList()); + log.debug("** Trials Duplicate Check: {} -> {}", existingTrialNames, newTrialNames); + if (newTrialNames.stream().anyMatch(existingTrialNames::contains)) { + log.debug("** New matches existing"); + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } catch (ApiException e) { + log.error("Error getting trials for duplicate name check", e); + throw new InternalServerException(e.getMessage(), e); + } + } + try { List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); createdDatasets.forEach(summary -> obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId())); @@ -179,7 +241,8 @@ public void process(ProcessContext processContext, ProcessedData processedData) // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); - String createdObsUnit_key = ExperimentUtilities.createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); + String createdObservationUnit_GID = createdObservationUnit.getAdditionalInfo().get("gid").getAsString(); + String createdObsUnit_key = ExperimentUtilities.createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName, createdObservationUnit_GID); observationUnitByNameNoScope.get(createdObsUnit_key) .getBrAPIObject() .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); @@ -320,10 +383,24 @@ private void updateObservationDependencyValues(PendingData pendingData, Program Map> observationUnitByNameNoScope = pendingData.getObservationUnitByNameNoScope(); Map> observationByHash = pendingData.getObservationByHash(); + // Create a lookup map for observations + // Key: studyName_observationUnitName (composite key) + // Value: List of observations matching the key + Map>> observationsByStudyAndUnit = new java.util.HashMap<>(); + for (PendingImportObject obsPio : observationByHash.values()) { + BrAPIObservation obs = obsPio.getBrAPIObject(); + if (obs.getAdditionalInfo() != null && obs.getAdditionalInfo().get(BrAPIAdditionalInfoFields.STUDY_NAME) != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getAdditionalInfo().get(BrAPIAdditionalInfoFields.STUDY_NAME).getAsString(), programKey); + String obsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getObservationUnitName(), programKey); + String key = studyName + "_" + obsUnitName; + observationsByStudyAndUnit.computeIfAbsent(key, k -> new ArrayList<>()).add(obsPio); + } + } + // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds observationUnitByNameNoScope.values().stream() .map(PendingImportObject::getBrAPIObject) - .forEach(obsUnit -> updateObservationDbIds(pendingData, obsUnit, programKey)); + .forEach(obsUnit -> updateObservationDbIds(observationsByStudyAndUnit, obsUnit, programKey)); // Pass the new map // Update ObservationVariable DbIds List traits = getTraitList(program); @@ -340,33 +417,26 @@ private void updateObservationDependencyValues(PendingData pendingData, Program } } - // Update each ovservation's observationUnit DbId, study DbId, and germplasm DbId - private void updateObservationDbIds(PendingData pendingData, BrAPIObservationUnit obsUnit, String programKey) { - Map> observationByHash = pendingData.getObservationByHash(); + // Update each observation's observationUnit DbId, study DbId, and germplasm DbId + private void updateObservationDbIds(Map>> observationsByStudyAndUnit, BrAPIObservationUnit obsUnit, String programKey) { // Modified signature - // FILTER LOGIC: Match on Env and Exp Unit ID - observationByHash.values() - .stream() - .filter(obs -> obs.getBrAPIObject() - .getAdditionalInfo() != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) - .getAsString() - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getStudyName(), programKey)) - && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject().getObservationUnitName(), programKey) - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey)) - ) - .forEach(obs -> { - if (StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { - obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); - } - obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); - obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId()); - }); + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getStudyName(), programKey); + String obsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey); + String key = studyName + "_" + obsUnitName; + + List> matchingObservations = observationsByStudyAndUnit.get(key); + + if (matchingObservations != null) { + for (PendingImportObject obsPio : matchingObservations) { + BrAPIObservation obs = obsPio.getBrAPIObject(); + + if (StringUtils.isBlank(obs.getObservationUnitDbId())) { + obs.setObservationUnitDbId(obsUnit.getObservationUnitDbId()); + } + obs.setStudyDbId(obsUnit.getStudyDbId()); + obs.setGermplasmDbId(obsUnit.getGermplasmDbId()); + } + } } private List getTraitList(Program program) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index b6bca19d0..f30512d2b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -41,12 +41,10 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentStudyService; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentTrialService; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.model.Trait; import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.Utilities; @@ -65,13 +63,10 @@ @Slf4j public class PopulateExistingPendingImportObjectsStep { - private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; - private final BrAPITrialDAO brAPITrialDAO; private final BrAPIStudyDAO brAPIStudyDAO; private final ProgramLocationService locationService; private final BrAPIListDAO brAPIListDAO; private final BrAPIGermplasmDAO brAPIGermplasmDAO; - private final BrAPIObservationDAO brAPIObservationDAO; private final ExperimentStudyService experimentStudyService; private final ExperimentTrialService experimentTrialService; @@ -80,39 +75,35 @@ public class PopulateExistingPendingImportObjectsStep { @Inject public PopulateExistingPendingImportObjectsStep(BrAPIObservationUnitDAO brAPIObservationUnitDAO, - BrAPITrialDAO brAPITrialDAO, BrAPIStudyDAO brAPIStudyDAO, ProgramLocationService locationService, BrAPIListDAO brAPIListDAO, BrAPIGermplasmDAO brAPIGermplasmDAO, - BrAPIObservationDAO brAPIObservationDAO, ExperimentStudyService experimentStudyService, ExperimentTrialService experimentTrialService) { - this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; - this.brAPITrialDAO = brAPITrialDAO; this.brAPIStudyDAO = brAPIStudyDAO; this.locationService = locationService; this.brAPIListDAO = brAPIListDAO; this.brAPIGermplasmDAO = brAPIGermplasmDAO; - this.brAPIObservationDAO = brAPIObservationDAO; this.experimentStudyService = experimentStudyService; this.experimentTrialService = experimentTrialService; } - public ProcessContext process(ImportContext input, ProcessedPhenotypeData phenotypeData) { + public ProcessContext process(ImportContext input) { List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); Program program = input.getProgram(); // Populate pending objects with existing status - Map> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); - Map> trialByNameNoScope = experimentTrialService.initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); - Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); + Map> observationUnitByNameNoScope = new HashMap<>(); + Map> trialByNameNoScope = experimentTrialService.initializeTrialByNameNoScope(program, experimentImportRows); + Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, experimentImportRows); // interesting we're using our data model instead of brapi for locations - Map> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); - Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, trialByNameNoScope, experimentImportRows); - Map> existingGermplasmByGID = initializeExistingGermplasmByGID(program, observationUnitByNameNoScope, experimentImportRows); - Map existingObsByObsHash = fetchExistingObservations(phenotypeData.getReferencedTraits(), studyByNameNoScope, program); + Map> locationByName = initializeUniqueLocationNames(program, experimentImportRows); + Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); + Map> existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); + Map existingObsByObsHash = new HashMap<>(); + Map expUnitByTrialName = new HashMap<>(); PendingData existing = PendingData.builder() .observationUnitByNameNoScope(observationUnitByNameNoScope) @@ -123,6 +114,7 @@ public ProcessContext process(ImportContext input, ProcessedPhenotypeData phenot .existingGermplasmByGID(existingGermplasmByGID) .existingObsByObsHash(existingObsByObsHash) .observationByHash(new HashMap<>()) + .expUnitByTrialName(expUnitByTrialName) .build(); return ProcessContext.builder() @@ -131,144 +123,6 @@ public ProcessContext process(ImportContext input, ProcessedPhenotypeData phenot .build(); } - /** - * Initializes the observation units for the given program and experimentImportRows. - * - * @param program The program object - * @param experimentImportRows A list of ExperimentObservation objects - * @return A map of Observation Unit IDs to PendingImportObject objects - * - * @throws InternalServerException - * @throws IllegalStateException - */ - private Map> initializeObservationUnits(Program program, List experimentImportRows) { - Map> observationUnitByName = new HashMap<>(); - - Map rowByObsUnitId = new HashMap<>(); - experimentImportRows.forEach(row -> { - if (StringUtils.isNotBlank(row.getObsUnitID())) { - if(rowByObsUnitId.containsKey(row.getObsUnitID())) { - throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); - } - rowByObsUnitId.put(row.getObsUnitID(), row); - } - }); - - try { - List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); - - // TODO: grab from externalReferences - /* - observationUnitByObsUnitId = existingObsUnits.stream() - .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, - (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false))); - */ - - String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - if (existingObsUnits.size() == rowByObsUnitId.size()) { - existingObsUnits.forEach(brAPIObservationUnit -> { - processAndCacheObservationUnit(brAPIObservationUnit, refSource, program, observationUnitByName, rowByObsUnitId); - - BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) - .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); - row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); - row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); - row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); - }); - } else { - List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); - missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); - } - - return observationUnitByName; - } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - /** - * Adds a new map entry to observationUnitByName based on the brAPIObservationUnit passed in and sets the - * expUnitId in the rowsByObsUnitId map. - * - * @param brAPIObservationUnit the BrAPI observation unit object - * @param refSource the reference source - * @param program the program object - * @param observationUnitByName the map of observation units by name (will be modified in place) - * @param rowByObsUnitId the map of rows by observation unit ID (will be modified in place) - * - * @throws InternalServerException - */ - private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservationUnit, String refSource, Program program, - Map> observationUnitByName, - Map rowByObsUnitId) { - BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) - .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); - row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); - observationUnitByName.put(ExperimentUtilities.createObservationUnitKey(row), - new PendingImportObject<>(ImportObjectState.EXISTING, - brAPIObservationUnit, - UUID.fromString(idRef.getReferenceId()))); - } - - - - /** - * Initializes studies by name without scope. - * - * @param program The program object. - * @param trialByNameNoScope A map of trial names with their corresponding pending import objects. - * @param experimentImportRows A list of experiment observation objects. - * @return A map of study names with their corresponding pending import objects. - * @throws InternalServerException If there is an error while processing the method. - */ - private Map> initializeStudyByNameNoScope(Program program, - Map> trialByNameNoScope, - Map> observationUnitByNameNoScope, - List experimentImportRows) { - Map> studyByName = new HashMap<>(); - if (trialByNameNoScope.size() != 1) { - return studyByName; - } - - try { - initializeStudiesForExistingObservationUnits(program, studyByName, observationUnitByNameNoScope); - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (Exception e) { - log.error("Error processing studies", e); - throw new InternalServerException(e.toString(), e); - } - - List existingStudies; - Optional> trial = getTrialPIO(experimentImportRows, trialByNameNoScope); - - try { - if (trial.isEmpty()) { - // TODO: throw ValidatorException and return 422 - } - UUID experimentId = trial.get().getId(); - existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); - for (BrAPIStudy existingStudy : existingStudies) { - experimentStudyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); - } - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } catch (Exception e) { - log.error("Error processing studies: ", e); - throw new InternalServerException(e.toString(), e); - } - - return studyByName; - } - /** * Retrieves the PendingImportObject of a BrAPITrial based on the given list of ExperimentObservation and trialByNameNoScope map. * @@ -279,7 +133,7 @@ private Map> initializeStudyByNameNoScop private Optional> getTrialPIO(List experimentImportRows, Map> trialByNameNoScope) { Optional expTitle = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID()) && StringUtils.isNotBlank(row.getExpTitle())) + .filter(row -> StringUtils.isNotBlank(row.getExpTitle())) .map(ExperimentObservation::getExpTitle) .findFirst(); @@ -293,62 +147,27 @@ private Optional> getTrialPIO(List> studyByName, - Map> observationUnitByNameNoScope - ) throws Exception { - Set studyDbIds = observationUnitByNameNoScope.values() - .stream() - .map(pio -> pio.getBrAPIObject() - .getStudyDbId()) - .collect(Collectors.toSet()); - - List studies = experimentStudyService.fetchStudiesByDbId(studyDbIds, program); - for (BrAPIStudy study : studies) { - experimentStudyService.processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); - } - } - /** * Initializes unique location names for a program. * * @param program The program object. - * @param studyByNameNoScope A map of study names and corresponding BrAPI study objects. * @param experimentImportRows A list of experiment observation objects for import. * @return A map of location names and their corresponding pending import objects. * @throws InternalServerException If there is an error fetching locations. */ private Map> initializeUniqueLocationNames(Program program, - Map> studyByNameNoScope, List experimentImportRows) { Map> locationByName = new HashMap<>(); - List existingLocations = new ArrayList<>(); - if(studyByNameNoScope.size() > 0) { - Set locationDbIds = studyByNameNoScope.values() - .stream() - .map(study -> study.getBrAPIObject() - .getLocationDbId()) - .collect(Collectors.toSet()); - try { - existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - + List existingLocations; List uniqueLocationNames = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) .map(ExperimentObservation::getEnvLocation) .distinct() .filter(Objects::nonNull) .collect(Collectors.toList()); try { - existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); + existingLocations = new ArrayList<>(locationService.getLocationsByName(uniqueLocationNames, program.getId())); } catch (ApiException e) { log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -359,19 +178,57 @@ private Map> initializeUniqueLocati } /** - * Initializes observation variable dataset by name. - * - * @param program The program associated with the dataset. - * @param trialByNameNoScope The map of trials identified by name without scope. - * @param experimentImportRows The list of experiment observation rows. - * @return The map of observation variable dataset indexed by name. + * Initializes studies by name without scope. * - * @throws InternalServerException + * @param program The program object. + * @param trialByNameNoScope A map of trial names with their corresponding pending import objects. + * @param experimentImportRows A list of experiment observation objects. + * @return A map of study names with their corresponding pending import objects. + * @throws InternalServerException If there is an error while processing the method. */ + private Map> initializeStudyByNameNoScope(Program program, + Map> trialByNameNoScope, + List experimentImportRows) { + Map> studyByName = new HashMap<>(); + if (trialByNameNoScope.size() != 1) { + return studyByName; + } + + + List existingStudies; + Optional> trial = getTrialPIO(experimentImportRows, trialByNameNoScope); + + try { + // the 'trial' variable will never be "null". + UUID experimentId = trial.get().getId(); + existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); + for (BrAPIStudy existingStudy : existingStudies) { + experimentStudyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); + } + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies: ", e); + throw new InternalServerException(e.toString(), e); + } + + return studyByName; + } + + /** + * Initializes observation variable dataset by name. + * + * @param program The program associated with the dataset. + * @param experimentImportRows The list of experiment observation rows. + * @return The map of observation variable dataset indexed by name. + * + * @throws InternalServerException + */ private Map> initializeObsVarDatasetByName(Program program, - Map> trialByNameNoScope, List experimentImportRows) { Map> obsVarDatasetByName = new HashMap<>(); + Map> trialByNameNoScope = new HashMap<>(); Optional> trialPIO = getTrialPIO(experimentImportRows, trialByNameNoScope); @@ -422,36 +279,24 @@ private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> initializeExistingGermplasmByGID(Program program, - Map> observationUnitByNameNoScope, List experimentImportRows) { Map> existingGermplasmByGID = new HashMap<>(); - List existingGermplasms = new ArrayList<>(); - if(observationUnitByNameNoScope.size() > 0) { - Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); - try { - existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } + List existingGermplasms; List uniqueGermplasmGIDs = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) .map(ExperimentObservation::getGid) .distinct() .collect(Collectors.toList()); try { - existingGermplasms.addAll(getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); + existingGermplasms = new ArrayList<>(getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); } catch (ApiException e) { log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -490,64 +335,4 @@ private ArrayList getGermplasmByAccessionNumber( } return resultGermplasm; } - - /** - * Fetches existing observations based on the given referenced traits, studyByNameNoScope map, and program. - * - * @param referencedTraits The list of referenced traits. - * @param studyByNameNoScope The map of studies by name without scope. - * @param program The program. - * @return A map of existing observations with their unique keys. - */ - private Map fetchExistingObservations(List referencedTraits, - Map> studyByNameNoScope, - Program program) { - Set ouDbIds = new HashSet<>(); - Set variableDbIds = new HashSet<>(); - Map variableNameByDbId = new HashMap<>(); - Map ouNameByDbId = new HashMap<>(); - Map studyNameByDbId = studyByNameNoScope.values() - .stream() - .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) - .map(PendingImportObject::getBrAPIObject) - .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - - studyNameByDbId.keySet().forEach(studyDbId -> { - try { - brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { - if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { - ouDbIds.add(ou.getObservationUnitDbId()); - } - ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); - }); - } catch (ApiException e) { - throw new RuntimeException(e); - } - }); - - for (Trait referencedTrait : referencedTraits) { - variableDbIds.add(referencedTrait.getObservationVariableDbId()); - variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); - } - - List existingObservations = new ArrayList<>(); - try { - existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); - } catch (ApiException e) { - throw new RuntimeException(e); - } - - return existingObservations.stream() - .map(obs -> { - String studyName = studyNameByDbId.get(obs.getStudyDbId()); - String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); - String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); - - String key = ExperimentUtilities.getObservationHash(ExperimentUtilities.createObservationUnitKey(studyName, ouName), variableName, studyName); - - return Map.entry(key, obs); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index 0e2e465cf..f2b6ba133 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -1,4 +1,4 @@ -/* + /* * See the NOTICE file distributed with this work for additional information * regarding copyright ownership. * @@ -43,6 +43,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingImportObjectData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; @@ -76,6 +77,7 @@ public class PopulateNewPendingImportObjectsStep { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final DSLContext dsl; private final Gson gson; + private final DatasetService datasetService; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -83,11 +85,12 @@ public class PopulateNewPendingImportObjectsStep { @Inject public PopulateNewPendingImportObjectsStep(ExperimentSeasonService experimentSeasonService, BrAPIObservationUnitDAO brAPIObservationUnitDAO, - DSLContext dsl) { + DSLContext dsl, DatasetService datasetService) { this.experimentSeasonService = experimentSeasonService; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.dsl = dsl; this.gson = new JSON().getGson(); + this.datasetService = datasetService; } /** @@ -302,7 +305,9 @@ public PendingImportObject populateTrial(ImportContext importContext boolean commit = importContext.isCommit(); Map> trialByNameNoScope = pendingData.getTrialByNameNoScope(); Map> studyByNameNoScope = pendingData.getStudyByNameNoScope(); + Map expUnitByTrialName = pendingData.getExpUnitByTrialName(); + //Experiment titles and environment checks if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { PendingImportObject envPio; trialPio = trialByNameNoScope.get(importRow.getExpTitle()); @@ -310,7 +315,7 @@ public PendingImportObject populateTrial(ImportContext importContext // creating new units for existing experiments and environments is not possible if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && - (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); } } else if (!trialByNameNoScope.isEmpty()) { @@ -327,6 +332,13 @@ public PendingImportObject populateTrial(ImportContext importContext //trialByNameNoScope.put(importRow.getExpTitle(), trialPio); } + //Experiment unit duplicate check + if (expUnitByTrialName.get(importRow.getExpTitle()) == null) { + expUnitByTrialName.put(importRow.getExpTitle(), importRow.getExpUnit()); + } else if (!expUnitByTrialName.get(importRow.getExpTitle()).equalsIgnoreCase(importRow.getExpUnit())) { + throw new UnprocessableEntityException(MULTIPLE_EXP_UNITS); + } + return trialPio; } @@ -361,7 +373,7 @@ public void fetchOrCreateDatasetPIO(ImportContext importContext, pio = obsVarDatasetByName.get(name); } else { UUID id = UUID.randomUUID(); - BrAPIListDetails newDataset = importRow.constructDatasetDetails( + BrAPIListDetails newDataset = datasetService.constructDatasetDetails( name, id, BRAPI_REFERENCE_SOURCE, @@ -517,7 +529,7 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Import String key = ExperimentUtilities.createObservationUnitKey(importRow); // NOTE: removed other workflow - if (observationUnitByNameNoScope.containsKey(key)) { + if (observationUnitByNameNoScope!=null && observationUnitByNameNoScope.containsKey(key)) { pio = observationUnitByNameNoScope.get(key); } else { String germplasmName = ""; @@ -552,6 +564,9 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Import } else { pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); } + if(observationUnitByNameNoScope==null){ + observationUnitByNameNoScope = new HashMap<>(); + } observationUnitByNameNoScope.put(key, pio); } return pio; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java index ee570c238..1ed8f9f97 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/ValidatePendingImportObjectsStep.java @@ -228,18 +228,10 @@ private void validateConditionallyRequired(PendingData pendingData, ValidationEr validateRequiredCell(importRow.getExpReplicateNo(), ExperimentObservation.Columns.REP_NUM, errorMessage, validationErrors, rowNum); validateRequiredCell(importRow.getExpBlockNo(), ExperimentObservation.Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); - if(StringUtils.isNotBlank(importRow.getObsUnitID())) { - ExperimentUtilities.addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); - } - } else { - //Check if existing environment. If so, ObsUnitId must be assigned - validateRequiredCell( - importRow.getObsUnitID(), - ExperimentObservation.Columns.OBS_UNIT_ID, - ExperimentUtilities.MISSING_OBS_UNIT_ID_ERROR, - validationErrors, - rowNum - ); + // TODO: replace validating each row for ObsUnitID with a single validation for the absence of the entire column " ObsUnitID" +// if(StringUtils.isNotBlank(importRow.getObsUnitID())) { +// ExperimentUtilities.addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); +// } } } @@ -263,9 +255,10 @@ private void validateObservationUnits( String key = ExperimentUtilities.createObservationUnitKey(importRow); PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); - if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { - ExperimentUtilities.addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); - } + // TODO: Is this check still needed for new observation units? +// if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { +// ExperimentUtilities.addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); +// } validateGeoCoordinates(validationErrors, rowNum, importRow); } @@ -397,7 +390,7 @@ private void validateObservations(PendingData pendingData, !existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { ExperimentUtilities.addRowError( phenoCol.name(), - String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), + String.format("Value already exists for Phenotype: %s", phenoCol.name()), validationErrors, rowNum ); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index 2c0ae2e53..b8ea2b6af 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -18,17 +18,19 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.model; import com.fasterxml.jackson.annotation.JsonValue; -import io.micronaut.context.annotation.Property; import lombok.extern.slf4j.Slf4j; @Slf4j public class ExpImportProcessConstants { public static final CharSequence COMMA_DELIMITER = ","; + public static final String OBSERVATION_UNIT_ID_SUFFIX = "ObsUnitID"; public static final String TIMESTAMP_PREFIX = "TS:"; public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; public static String BRAPI_REFERENCE_SOURCE; public static final String MIDNIGHT = "T00:00:00-00:00"; + public static final String SUB_UNIT_ID = "Sub Unit ID"; + public static final String SUB_OBS_UNIT = "Sub-Obs Unit"; public enum ErrMessage { MULTIPLE_EXP_TITLES("File contains more than one Experiment Title"), @@ -36,7 +38,13 @@ public enum ErrMessage { PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"), UNMATCHED_COLUMN("Ontology term(s) not found: "), OBS_UNIT_NOT_FOUND("Invalid ObsUnitID"), - DUPLICATE_OBS_UNIT_ID("ObsUnitId is repeated"); + DUPLICATE_OBS_UNIT_ID("ObsUnitId is repeated"), + DATASET_NOT_FOUND("Dataset not found"), + OZEX("Missing ObsUnitID column"), + VVCN("ObsUnitID is duplicated"), + BITB("Invalid or missing ObsUnitID"), + PJZH("Required field is blank"), + JABH("Observation variable(s) are already associated with another dataset(s) in this experiment"); private String value; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index 68aca9e2b..0bba7f48a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -19,33 +19,46 @@ import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; +import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIListTypes; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; +import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapi.v2.services.BrAPIObservationLevelService; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.BrAPIConstants; +import org.breedinginsight.model.DatasetLevel; +import org.breedinginsight.model.DatasetMetadata; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; -import java.util.List; -import java.util.Optional; -import java.util.UUID; +import java.util.*; +import java.util.stream.Collectors; @Singleton public class DatasetService { private final BrAPIListDAO brAPIListDAO; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; + private final BrAPIObservationLevelService observationLevelService; @Inject - public DatasetService(BrAPIListDAO brapiListDAO) { + public DatasetService(BrAPIListDAO brapiListDAO, + BrAPIObservationLevelService brAPIObservationLevelService) { this.brAPIListDAO = brapiListDAO; + this.observationLevelService = brAPIObservationLevelService; } /** * Module: Dataset Utility @@ -74,10 +87,8 @@ public Optional fetchDatasetById(String id, Program program) t program.getId(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), UUID.fromString(id)); - - // Check if the existing dataset summaries are returned, throw exception if not if (existingDatasets == null || existingDatasets.isEmpty()) { - throw new InternalServerException("Existing dataset summary not returned from BrAPI server"); + return Optional.empty(); } // Retrieve dataset details using the list DB ID from the existing dataset summary @@ -88,6 +99,16 @@ public Optional fetchDatasetById(String id, Program program) t return dataSetDetails; } + public Optional> fetchDatasetsByIds(Set datasetIds, Program program) throws ApiException { + List datasets = new ArrayList<>(); + for (String datasetId : datasetIds) { + Optional dataSetDetailsOptional = fetchDatasetById(datasetId, program); + dataSetDetailsOptional.ifPresent(datasets::add); + } + + return datasets.isEmpty() ? Optional.empty() : Optional.of(datasets); + } + /** * Constructs a PendingImportObject for a BrAPIListDetails dataset. * This method retrieves the external reference for the dataset from the existing list @@ -108,4 +129,132 @@ public PendingImportObject constructPIOFromDataset(BrAPIListDe // Create a PendingImportObject for the dataset with the existing list and reference ID return new PendingImportObject(ImportObjectState.EXISTING, dataset, UUID.fromString(xref.getReferenceId())); } + + public void createBrAPIObsVarListForDataset(Program program, + BrAPITrial trial, + DatasetMetadata subEntityDatasetMetadata) throws ApiException { + + String name = String.format("Observation Dataset [%s-%s-%s]", + program.getKey(), + trial.getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString(), + subEntityDatasetMetadata.getName()); + + BrAPIListDetails subEntityObsVarsList = constructDatasetDetails(name, + subEntityDatasetMetadata.getId(), + BRAPI_REFERENCE_SOURCE, + program, + trial.getTrialDbId()); + + BrAPIListNewRequest listRq = new BrAPIListNewRequest(); + listRq.setListName(subEntityObsVarsList.getListName()); + listRq.setListType(subEntityObsVarsList.getListType()); + listRq.setExternalReferences(subEntityObsVarsList.getExternalReferences()); + listRq.setAdditionalInfo(subEntityObsVarsList.getAdditionalInfo()); + listRq.data(subEntityObsVarsList.getData()); + + brAPIListDAO.createBrAPILists(List.of(listRq), program.getId(), null); + } + + public BrAPIListDetails constructDatasetDetails( + String name, + UUID datasetId, + String referenceSourceBase, + Program program, String trialId) { + BrAPIListDetails dataSetDetails = new BrAPIListDetails(); + dataSetDetails.setListName(name); + dataSetDetails.setListType(BrAPIListTypes.OBSERVATIONVARIABLES); + dataSetDetails.setData(new ArrayList<>()); + dataSetDetails.putAdditionalInfoItem("datasetType", "observationDataset"); + List refs = new ArrayList<>(); + Utilities.addReference(refs, program.getId(), referenceSourceBase, ExternalReferenceSource.PROGRAMS); + Utilities.addReference(refs, UUID.fromString(trialId), referenceSourceBase, ExternalReferenceSource.TRIALS); + Utilities.addReference(refs, datasetId, referenceSourceBase, ExternalReferenceSource.DATASET); + dataSetDetails.setExternalReferences(refs); + return dataSetDetails; + } + + /** + * @return brapiLevelNameDbId of found or created record + */ + public String getOrCreateLevelNameForDataset(Program program, + String brapiProgramDbId, + String levelName, + DatasetLevel levelOrder) throws ApiException { + + String existingLevelNameDbId = findLevelNameByNameAndOrder(program, brapiProgramDbId, levelName, levelOrder); + + if (StringUtils.isNotBlank(existingLevelNameDbId)) { + return existingLevelNameDbId; + } + + // Level name does not exist and needs to be created. + BrAPIObservationUnitHierarchyLevel createdLevelName = observationLevelService.createObservationLevel(program, brapiProgramDbId, levelName, levelOrder); + + return createdLevelName.getLevelNameDbId(); + } + + /** + * This method retrieves the programmatic level names and then matches the level names on the submitted + * level name and order. + * + * @return levelNameDbId of the matched level name + */ + private String findLevelNameByNameAndOrder(Program program, + String brapiProgramDbId, + String levelName, + DatasetLevel levelOrder) { + var programmaticLevelNames = observationLevelService.getProgrammaticLevelNames(program, brapiProgramDbId); + + List levelNameStreamResult + = programmaticLevelNames.stream() + .filter(ouln -> ouln.getLevelName().equals(levelName.toLowerCase()) && ouln.getLevelOrder() == levelOrder.getValue()) + .limit(1) + .collect(Collectors.toList()); + + if (levelNameStreamResult.isEmpty()) { + return null; + } + + return levelNameStreamResult.get(0).getLevelNameDbId(); + } + + public void updateObservationUnitsWithLevelNameDbIds(List observationUnits, + Program program, + String brapiProgramDbId, + String expUnitName, + DatasetLevel levelOrder) throws ApiException { + Map levelNameDbIdByName = new HashMap<>(); + + String expLevelName = expUnitName.toLowerCase(); + + String existingLevelNameDbId = getOrCreateLevelNameForDataset(program, brapiProgramDbId, expLevelName, levelOrder); + levelNameDbIdByName.put(expLevelName, existingLevelNameDbId); + + List globalLevelNames = observationLevelService.getGlobalLevelNames(program); + + globalLevelNames.forEach(ouln -> levelNameDbIdByName.put(ouln.getLevelName(), ouln.getLevelNameDbId())); + + for (BrAPIObservationUnit observationUnit : observationUnits) { + + String positionLevelName = observationUnit.getObservationUnitPosition().getObservationLevel().getLevelName().toLowerCase(); + + observationUnit.getObservationUnitPosition().getObservationLevel().setLevelNameDbId(levelNameDbIdByName.get(positionLevelName)); + + for (BrAPIObservationUnitLevelRelationship lvlRelationship : observationUnit.getObservationUnitPosition().getObservationLevelRelationships()) { + if (lvlRelationship.getLevelName().equals(BrAPIConstants.BLOCK.getValue())) { + lvlRelationship.setLevelNameDbId(levelNameDbIdByName.get(lvlRelationship.getLevelName())); + } else if (lvlRelationship.getLevelName().equals(BrAPIConstants.REPLICATE.getValue())) { + lvlRelationship.setLevelNameDbId(levelNameDbIdByName.get(lvlRelationship.getLevelName())); + } else { + throw new InternalServerException(String.format("Level name [%s] detected in OU Level Relationship " + + "for experiment with Exp Unit name [%s]. This is unexpected and the new level " + + "name must be retrieved properly from BrAPI to insert its DbId into BrAPI request for proper creation and assignment.", + lvlRelationship.getLevelName(), expUnitName)); + } + } + } + + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index de98202fa..ed432fb7e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -99,8 +99,8 @@ public boolean validDateValue(String value) { } return true; } - public String getObservationHash(String observationUnitName, String variableName, String studyName) { - String concat = DigestUtils.sha256Hex(observationUnitName) + + public String getObservationHash(String observationUnitId, String variableName, String studyName) { + String concat = DigestUtils.sha256Hex(observationUnitId) + DigestUtils.sha256Hex(variableName) + DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); return DigestUtils.sha256Hex(concat); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index d66507558..3c387f061 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -160,7 +160,9 @@ public Map> mapPendingUnitByNa pio.getBrAPIObject().getObservationUnitName(), program.getKey() ); - pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), pio); + String germplasmGID = pio.getBrAPIObject().getAdditionalInfo().get("gid").getAsString(); + + pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName, germplasmGID), pio); } return pendingUnitByNameNoScope; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index dcbc79159..1a9a7f02f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -306,42 +306,6 @@ public PendingImportObject constructPIOFromBrapiTrial(BrAPITrial tri return pio; } - /** - * Initializes trials by name without scope for the given program. - * - * @param program the program to initialize trials for - * @param observationUnitByNameNoScope a map of observation units by name without scope - * @param experimentImportRows a list of experiment observation rows - * @return a map of trials by name with pending import objects - * - * @throws InternalServerException - */ - private Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, - List experimentImportRows) { - Map> trialByName = new HashMap<>(); - - initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); - - List uniqueTrialNames = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID())) - .map(ExperimentObservation::getExpTitle) - .distinct() - .collect(Collectors.toList()); - try { - brAPITrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> - processAndCacheTrial(existingTrial, program, trialByName) - ); - } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - return trialByName; - } - - private void initializeTrialsForExistingObservationUnits(Program program, Map> observationUnitByNameNoScope, Map> trialByName) { - } - // TODO: used by expunit workflow public Map> mapPendingTrialByOUId( String unitId, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java index 9db927345..2e3a2ec33 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java @@ -164,20 +164,15 @@ private void processAndCacheTrial( * Initializes trials by name without scope for the given program. * * @param program the program to initialize trials for - * @param observationUnitByNameNoScope a map of observation units by name without scope * @param experimentImportRows a list of experiment observation rows * @return a map of trials by name with pending import objects * - * @throws InternalServerException */ - public Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, - List experimentImportRows) { + public Map> initializeTrialByNameNoScope(Program program, + List experimentImportRows) { Map> trialByName = new HashMap<>(); - initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); - List uniqueTrialNames = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID())) .map(ExperimentObservation::getExpTitle) .distinct() .collect(Collectors.toList()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index 491706493..5defe7d10 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -91,8 +91,9 @@ public class GermplasmProcessor implements Processor { public static String missingGIDsMsg = "The following GIDs were not found in the database: %s"; public static String missingParentalGIDsMsg = "The following parental GIDs were not found in the database: %s"; - public static String missingParentalEntryNoMsg = "The following parental entry numbers were not found in the database: %s"; + public static String missingParentalEntryNoMsg = "The following parental entry numbers were not found in the file: %s"; public static String badBreedMethodsMsg = "Invalid breeding method"; + public static String badGermplasmNameMsg = "Germplasm name cannot contain /"; public static String missingEntryNumbersMsg = "Either all or none of the germplasm must have entry numbers"; public static String duplicateEntryNoMsg = "Entry numbers must be unique. Duplicated entry numbers found: %s"; public static String circularDependency = "Circular dependency in the pedigree tree"; @@ -136,14 +137,6 @@ public void getExistingBrapiData(List importRows, Program program) BrAPIImport germplasmImport = importRows.get(i); Germplasm germplasm = germplasmImport.getGermplasm(); if (germplasm != null) { - - // Retrieve parent accession numbers to assess if already in db - if (germplasm.getFemaleParentAccessionNumber() != null) { - germplasmAccessionNumbers.put(germplasm.getFemaleParentAccessionNumber(), true); - } - if (germplasm.getMaleParentAccessionNumber() != null) { - germplasmAccessionNumbers.put(germplasm.getMaleParentAccessionNumber(), true); - } if (germplasm.getAccessionNumber() != null) { germplasmAccessionNumbers.put(germplasm.getAccessionNumber(), false); } @@ -158,6 +151,29 @@ public void getExistingBrapiData(List importRows, Program program) } } + // Get existing germplasm names. This should be used ONLY to identify duplicate germplasm names. + // In general you should look up germplasm by GID. + List dbGermplasm = brAPIGermplasmService.getGermplasmByDisplayName(new ArrayList<>(fileGermplasmByName.keySet()), program.getId()); + dbGermplasm.forEach(germplasm -> { + dbGermplasmByName.put(germplasm.getDefaultDisplayName(), germplasm); + dbGermplasmByAccessionNo.put(germplasm.getAccessionNumber(), germplasm); + }); + + // Get parental accession nos in file + for (int i = 0; i < importRows.size(); i++) { + BrAPIImport germplasmImport = importRows.get(i); + Germplasm germplasm = germplasmImport.getGermplasm(); + if (germplasm != null) { + // Retrieve parent accession numbers to assess if already in db + if (germplasm.getFemaleParentAccessionNumber() != null) { + germplasmAccessionNumbers.put(germplasm.getFemaleParentAccessionNumber(), true); + } + if (germplasm.getMaleParentAccessionNumber() != null) { + germplasmAccessionNumbers.put(germplasm.getMaleParentAccessionNumber(), true); + } + } + } + // If a parental accession number is present, it should exist in the database. existingGermplasm = new ArrayList<>(); List missingParentalAccessionNumbers = germplasmAccessionNumbers.entrySet().stream().filter(Map.Entry::getValue).map(Map.Entry::getKey).collect(Collectors.toList()); @@ -180,13 +196,6 @@ public void getExistingBrapiData(List importRows, Program program) } } - // Get existing germplasm names - List dbGermplasm = brAPIGermplasmService.getGermplasmByDisplayName(new ArrayList<>(fileGermplasmByName.keySet()), program.getId()); - dbGermplasm.forEach(germplasm -> { - dbGermplasmByName.put(germplasm.getDefaultDisplayName(), germplasm); - dbGermplasmByAccessionNo.put(germplasm.getAccessionNumber(), germplasm); - }); - // Check for existing germplasm lists Boolean listNameDup = false; if (importRows.size() > 0 && importRows.get(0).getGermplasm().getListName() != null) { @@ -222,27 +231,28 @@ public void getExistingBrapiData(List importRows, Program program) arrayOfStringFormatter.apply(missingAccessionNumbers))); } - List missingEntryNumbers = new ArrayList<>(); - for (BrAPIImport importRow: importRows) { - Germplasm germplasm = importRow.getGermplasm(); - // Check Female Parent - if (germplasm.getFemaleParentEntryNo() != null) { - if ((!germplasmIndexByEntryNo.containsKey(germplasm.getFemaleParentEntryNo())) && !(germplasm.getFemaleParentEntryNo().equals("0"))) { - missingEntryNumbers.add(germplasm.getFemaleParentEntryNo()); + List missingEntryNumbers = new ArrayList<>(); + for (BrAPIImport importRow : importRows) { + Germplasm germplasm = importRow.getGermplasm(); + + // Check Female Parent + if (germplasm.getFemaleParentEntryNo() != null) { + if ((!germplasmIndexByEntryNo.containsKey(germplasm.getFemaleParentEntryNo())) && !(germplasm.getFemaleParentEntryNo().equals("0"))) { + missingEntryNumbers.add(germplasm.getFemaleParentEntryNo()); + } } - } - // Check Male Parent - if (germplasm.getMaleParentEntryNo() != null) { - if ((!germplasmIndexByEntryNo.containsKey(germplasm.getMaleParentEntryNo())) && !(germplasm.getMaleParentEntryNo().equals("0"))) { - missingEntryNumbers.add(germplasm.getMaleParentEntryNo()); + // Check Male Parent + if (germplasm.getMaleParentEntryNo() != null) { + if ((!germplasmIndexByEntryNo.containsKey(germplasm.getMaleParentEntryNo())) && !(germplasm.getMaleParentEntryNo().equals("0"))) { + missingEntryNumbers.add(germplasm.getMaleParentEntryNo()); + } } } - } - if (missingEntryNumbers.size() > 0) { - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, - String.format(missingParentalEntryNoMsg, - arrayOfStringFormatter.apply(missingEntryNumbers))); - } + if (missingEntryNumbers.size() > 0) { + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, + String.format(missingParentalEntryNoMsg, + arrayOfStringFormatter.apply(missingEntryNumbers))); + } if (listNameDup) { throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, listNameAlreadyExists); @@ -359,6 +369,7 @@ private void processNewGermplasm(Germplasm germplasm, ValidationErrors validatio } } + validateGermplasmName(germplasm, i+2, validationErrors); validatePedigree(germplasm, i + 2, validationErrors); if (germplasm.pedigreeExists()) { @@ -458,6 +469,16 @@ private boolean hasPedigree(BrAPIGermplasm germplasm) { germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.MALE_PARENT_UNKNOWN).getAsBoolean()); } + //Used to check if germplasm already has a pedigree in the database + private boolean databaseGermplasmHasPedigree(Germplasm germplasm) { + if (germplasm.getAccessionNumber() == null || dbGermplasmByAccessionNo.get(germplasm.getAccessionNumber()) == null) { + return false; + } else { + BrAPIGermplasm dbGermplasm = dbGermplasmByAccessionNo.get(germplasm.getAccessionNumber()); + return hasPedigree(dbGermplasm); + } + } + /** * Compare an existing germplasm's pedigree to the incoming germplasm's pedigree to ensure they are the same.

* Assumes that an empty value for a given parent in the incoming germplasm is equal to the existing germplasm.

@@ -558,6 +579,31 @@ private Map getStatisticsMap() { } + /** + * Validates the name of the given Germplasm, ensuring it does not contain any slash ("/") characters. + *

+ * If the germplasm name contains a "/", a new {@link ValidationError} with status + * {@code 422 Unprocessable Entity} is created and added to the provided {@code ValidationErrors} object. + * This method does not throw an exception; instead, it records validation failures by mutating + * the {@code validationErrors} parameter. + *

+ * + * @param germplasm + * the {@link Germplasm} instance whose name is to be validated; must not be {@code null} + * @param rowNumber + * the row index (for example, in a spreadsheet or CSV file) corresponding to this + * germplasm entry; used when reporting errors + * @param validationErrors + * the {@link ValidationErrors} collector into which any detected errors will be added; + * this object is modified by this method to record validation issues; must not be {@code null} + */ + private void validateGermplasmName(Germplasm germplasm, Integer rowNumber, ValidationErrors validationErrors) { + if (germplasm.getGermplasmName().contains("/")) { + ValidationError error = new ValidationError("Germplasm Name", badGermplasmNameMsg, HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowNumber, error); + } + } + private void validatePedigree(Germplasm germplasm, Integer rowNumber, ValidationErrors validationErrors) { String femaleParentEntryNo = germplasm.getFemaleParentEntryNo(); String maleParentEntryNo = germplasm.getMaleParentEntryNo(); @@ -766,6 +812,8 @@ else if (germplasmIndexByEntryNo.containsKey(germplasm.getFemaleParentEntryNo()) if (commit) { if (femaleParentFound) { brAPIGermplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_GID, femaleParent.getAccessionNumber()); + //entry number no longer needed for figuring out parentage, can remove (since same germplasm can have different entry numbers across multiple lists) + brAPIGermplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_ENTRY_NO, null); // Add femaleParentUUID to additionalInfo. Optional femaleParentUUID = Utilities.getExternalReference(femaleParent.getExternalReferences(), BRAPI_REFERENCE_SOURCE); if (femaleParentUUID.isPresent()) { @@ -775,6 +823,8 @@ else if (germplasmIndexByEntryNo.containsKey(germplasm.getFemaleParentEntryNo()) if (maleParent != null) { brAPIGermplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_GID, maleParent.getAccessionNumber()); + //entry number no longer needed for figuring out parentage, can remove (since same germplasm can have different entry numbers across multiple lists) + brAPIGermplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO, null); // Add maleParentUUID to additionalInfo. Optional maleParentUUID = Utilities.getExternalReference(maleParent.getExternalReferences(), BRAPI_REFERENCE_SOURCE); if (maleParentUUID.isPresent()) { diff --git a/src/main/java/org/breedinginsight/daos/ProgramUserDAO.java b/src/main/java/org/breedinginsight/daos/ProgramUserDAO.java index 106a34688..94bb9b3a6 100644 --- a/src/main/java/org/breedinginsight/daos/ProgramUserDAO.java +++ b/src/main/java/org/breedinginsight/daos/ProgramUserDAO.java @@ -120,14 +120,14 @@ public List getProgramUsersByUserId(UUID userId) { return parseRecords(records, createdByUser, updatedByUser); } - public List getProgramUsersByOrcid(String orcid) { + public List getProgramUsersByOAuthId(String oAuthId) { BiUserTable createdByUser = BI_USER.as("createdByUser"); BiUserTable updatedByUser = BI_USER.as("updatedByUser"); // TODO: When we allow for pulling archived users, active condition won't be hardcoded. Result records = getProgramUsersQuery(createdByUser, updatedByUser) - .where(BI_USER.ORCID.eq(orcid)) + .where(BI_USER.OAUTH_ID.eq(oAuthId)) .and(PROGRAM.ACTIVE.eq(true)) .fetch(); diff --git a/src/main/java/org/breedinginsight/daos/UserDAO.java b/src/main/java/org/breedinginsight/daos/UserDAO.java index 562801735..a89376982 100644 --- a/src/main/java/org/breedinginsight/daos/UserDAO.java +++ b/src/main/java/org/breedinginsight/daos/UserDAO.java @@ -32,11 +32,11 @@ public interface UserDAO extends DAO { Optional getUser(UUID id); - Optional getUserByOrcId(String orcid); + Optional getUserByOAuthId(String oAuthId); BiUserEntity fetchOneById(UUID value); List fetchByEmail(String... values); - List fetchByOrcid(String... values); + List fetchByOauthId(String... values); } diff --git a/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java index e179bc42d..1e6e391b2 100644 --- a/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java @@ -67,11 +67,11 @@ public Optional getUser(UUID id) { return Utilities.getSingleOptional(users); } - public Optional getUserByOrcId(String orcid) { + public Optional getUserByOAuthId(String oAuthId) { List records = getUsersQuery() - .where(BI_USER.ORCID.eq(orcid)) + .where(BI_USER.OAUTH_ID.eq(oAuthId)) .fetch(); - List programUsers = programUserDAO.getProgramUsersByOrcid(orcid); + List programUsers = programUserDAO.getProgramUsersByOAuthId(oAuthId); List users = parseRecords(records, programUsers); return Utilities.getSingleOptional(users); diff --git a/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java b/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java index 8ffcc02e7..36e3a6700 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java @@ -69,19 +69,56 @@ private void deleteUserChris(Context context) throws SQLException { private void addConstraint(Context context) throws SQLException { final String CONSTRAINT_NAME = "email_orcid"; - // First, drop the constraint if it already exist. + // First, drop the constraint if it already exists. try (Statement altTable = context.getConnection().createStatement()) { - String sql = "ALTER TABLE bi_user DROP CONSTRAINT IF EXISTS "+ CONSTRAINT_NAME; + String sql = "ALTER TABLE bi_user DROP CONSTRAINT IF EXISTS " + CONSTRAINT_NAME; log.debug(sql); altTable.executeUpdate(sql); } - // Add new constraint - try (Statement altTable = context.getConnection().createStatement()) { - String sql = "ALTER TABLE bi_user\n" + - "ADD CONSTRAINT " +CONSTRAINT_NAME+ " CHECK ( (email IS NOT NULL ) OR (orcid IS NULL) ) ;"; - log.debug(sql); - altTable.executeUpdate(sql); + /* NOTE: + * (1) Because Java-based migrations don't run at build time, they end up running last (out of order) on + * application startup. If they haven't been applied to the database already. For that reason, the reference + * to column `orcid` was changed to `oauth_id` 2025-08-25 to allow this migration to run after + * V1.34.0__rename-orcid.sql. + * + * (2) At test time, however, the build has already run, but a fresh database is set up at runtime. + * Migrations run in order in this context, meaning this Java-based migration runs before + * V1.34.0__rename-orcid.sql, hence the logic below to check for the existence of the old column. + */ + + // Determine if the old column, orcid, exists (true when migrations run in order). + String orcidExistsSQL = "SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='bi_user' AND column_name='orcid') AS exists;"; + Boolean orcidExists = false; + try (Statement existsQuery = context.getConnection().createStatement()) { + var resultSet = existsQuery.executeQuery(orcidExistsSQL); + if (resultSet.next()) { + orcidExists = resultSet.getBoolean("exists"); + } + } + + if (orcidExists) { + try (Statement altTable = context.getConnection().createStatement()) { + + String oldSql = "ALTER TABLE bi_user\n" + + "ADD CONSTRAINT " + CONSTRAINT_NAME + " CHECK ( (email IS NOT NULL ) OR (orcid IS NULL) ) ;"; + + log.debug("Attempting to reference `orcid` column."); + altTable.executeUpdate(oldSql); + log.debug(oldSql); + log.debug("Successfully referenced `orcid` column."); + } + } else { + try (Statement newAltTable = context.getConnection().createStatement()) { + String newSql = "ALTER TABLE bi_user\n" + + "ADD CONSTRAINT " +CONSTRAINT_NAME+ " CHECK ( (email IS NOT NULL ) OR (oauth_id IS NULL) ) ;"; + + log.debug("Referencing `orcid` column failed."); + log.debug("Attempting to reference `oauth_id` column."); + newAltTable.executeUpdate(newSql); + log.debug(newSql); + log.debug("Successfully referenced `oauth_id` column."); + } } } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/model/Column.java b/src/main/java/org/breedinginsight/model/Column.java index 054dcb435..52b4a9de3 100644 --- a/src/main/java/org/breedinginsight/model/Column.java +++ b/src/main/java/org/breedinginsight/model/Column.java @@ -21,6 +21,8 @@ import lombok.experimental.Accessors; import lombok.experimental.SuperBuilder; +import java.util.Objects; + @Getter @Setter @Accessors(chain=true) @@ -46,4 +48,17 @@ public enum ColumnDataType { INTEGER, DOUBLE } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Column that = (Column) o; + return Objects.equals(getValue(), that.getValue()) && Objects.equals(getDataType(), that.getDataType()); + } + + @Override + public int hashCode() { + return Objects.hash(getValue(), getDataType()); + } } diff --git a/src/main/java/org/breedinginsight/model/Country.java b/src/main/java/org/breedinginsight/model/Country.java index e2d93dbfe..4cbe21997 100644 --- a/src/main/java/org/breedinginsight/model/Country.java +++ b/src/main/java/org/breedinginsight/model/Country.java @@ -81,4 +81,9 @@ public boolean equals(Object o) { Country that = (Country) o; return Objects.equals(getId(), that.getId()) && Objects.equals(getName(), that.getName()); } + + @Override + public int hashCode() { + return Objects.hash(getId(), getName()); + } } diff --git a/src/main/java/org/breedinginsight/model/User.java b/src/main/java/org/breedinginsight/model/User.java index 30386b79d..9d72b1841 100644 --- a/src/main/java/org/breedinginsight/model/User.java +++ b/src/main/java/org/breedinginsight/model/User.java @@ -56,13 +56,14 @@ public class User extends BiUserEntity { public User(BiUserEntity biUser) { this.setId(biUser.getId()); - this.setOrcid(biUser.getOrcid()); + this.setOauthId(biUser.getOauthId()); this.setName(biUser.getName()); this.setEmail(biUser.getEmail()); this.setSystemRoles(new ArrayList<>()); this.setProgramRoles(new ArrayList<>()); this.setActive(biUser.getActive()); this.setAccountToken(biUser.getAccountToken()); + this.setOauthProvider(biUser.getOauthProvider()); } public User() { @@ -72,13 +73,14 @@ public User() { public static User parseSQLRecord(Record record, @NotNull BiUserTable tableName){ return User.builder() .id(record.getValue(tableName.ID)) - .orcid(record.getValue(tableName.ORCID)) + .oauthId(record.getValue(tableName.OAUTH_ID)) .name(record.getValue(tableName.NAME)) .email(record.getValue(tableName.EMAIL)) .systemRoles(new ArrayList<>()) .programRoles(new ArrayList<>()) .active(record.getValue(tableName.ACTIVE)) .accountToken(record.getValue(tableName.ACCOUNT_TOKEN)) + .oauthProvider(record.getValue(tableName.OAUTH_PROVIDER)) .build(); } @@ -98,7 +100,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; User user = (User) o; return Objects.equals(getId(), user.getId()) && - Objects.equals(getOrcid(), user.getOrcid()) && + Objects.equals(getOauthId(), user.getOauthId()) && Objects.equals(getName(), user.getName()) && Objects.equals(getEmail(), user.getEmail()) && Objects.equals(getCreatedAt(), user.getCreatedAt()) && @@ -106,11 +108,12 @@ public boolean equals(Object o) { Objects.equals(getCreatedBy(), user.getCreatedBy()) && Objects.equals(getUpdatedBy(), user.getUpdatedBy()) && Objects.equals(getActive(), user.getActive()) && - Objects.equals(getAccountToken(), user.getAccountToken()); + Objects.equals(getAccountToken(), user.getAccountToken()) && + Objects.equals(getOauthProvider(), user.getOauthProvider()); } @Override public int hashCode() { - return Objects.hash(getId(), getOrcid(), getName(), getEmail(), getCreatedAt(), getUpdatedAt(), getCreatedBy(), getUpdatedBy(), getActive(), getAccountToken()); + return Objects.hash(getId(), getOauthId(), getName(), getEmail(), getCreatedAt(), getUpdatedAt(), getCreatedBy(), getUpdatedBy(), getActive(), getAccountToken(), getOauthProvider()); } } diff --git a/src/main/java/org/breedinginsight/services/UserService.java b/src/main/java/org/breedinginsight/services/UserService.java index b78fdf8a6..33c459254 100644 --- a/src/main/java/org/breedinginsight/services/UserService.java +++ b/src/main/java/org/breedinginsight/services/UserService.java @@ -91,10 +91,10 @@ public UserService(UserDAO dao, SystemUserRoleDao systemUserRoleDao, SystemRoleD } - public Optional getByOrcid(String orcid) { + public Optional getByOAuthId(String oAuthId) { - // User has been authenticated against orcid, check they have a bi account. - Optional users = dao.getUserByOrcId(orcid); + // User has been authenticated against OAuth provider, check they have a bi account. + Optional users = dao.getUserByOAuthId(oAuthId); if (users.isEmpty()) { return Optional.empty(); @@ -113,7 +113,7 @@ public List getAll() { public Optional getById(UUID userId) { - // User has been authenticated against orcid, check they have a bi account. + // User has been authenticated against OAuth provider, check they have a bi account. Optional user = dao.getUser(userId); if (!user.isPresent()) { @@ -146,7 +146,7 @@ public User create(AuthenticatedUser actingUser, UserRequest userRequest, Config insertSystemRoles(actingUser, newUser.getId(), systemRoles); } - // Start OrcID association flow + // Start OAuth account association flow createAndSendAccountToken(newUser.getId()); return getById(newUser.getId()).get(); @@ -352,7 +352,7 @@ public void createAndSendAccountToken(UUID userId) throws DoesNotExistException sendAccountSignUpEmail(biUser, jwt.getSignedJWT()); } - public void updateOrcid(UUID userId, String orcid) throws DoesNotExistException, AlreadyExistsException { + public void updateOAuthInfo(UUID userId, String oAuthId, String oAuthProvider) throws DoesNotExistException, AlreadyExistsException { BiUserEntity biUser = dao.fetchOneById(userId); @@ -360,14 +360,15 @@ public void updateOrcid(UUID userId, String orcid) throws DoesNotExistException, throw new DoesNotExistException("UUID for user does not exist"); } - List biUserWithOrcidList = dao.fetchByOrcid(orcid); - for (BiUserEntity biUserWithOrcid: biUserWithOrcidList){ - if (!biUserWithOrcid.getId().equals(userId)){ - throw new AlreadyExistsException("Orcid already in use"); + List biUserWithOAuthIdList = dao.fetchByOauthId(oAuthId); + for (BiUserEntity biUserWithOAuthId: biUserWithOAuthIdList){ + if (!biUserWithOAuthId.getId().equals(userId)){ + throw new AlreadyExistsException("OAuth Id already in use"); } } - biUser.setOrcid(orcid); + biUser.setOauthId(oAuthId); + biUser.setOauthProvider(oAuthProvider); biUser.setAccountToken(null); dao.update(biUser); } diff --git a/src/main/java/org/breedinginsight/services/exceptions/CreationBusyException.java b/src/main/java/org/breedinginsight/services/exceptions/CreationBusyException.java new file mode 100644 index 000000000..d32536159 --- /dev/null +++ b/src/main/java/org/breedinginsight/services/exceptions/CreationBusyException.java @@ -0,0 +1,7 @@ +package org.breedinginsight.services.exceptions; + +public class CreationBusyException extends Exception { + public CreationBusyException(String message) { + super(message); + } +} diff --git a/src/main/java/org/breedinginsight/services/lock/DistributedLockService.java b/src/main/java/org/breedinginsight/services/lock/DistributedLockService.java new file mode 100644 index 000000000..06f141f33 --- /dev/null +++ b/src/main/java/org/breedinginsight/services/lock/DistributedLockService.java @@ -0,0 +1,61 @@ +package org.breedinginsight.services.lock; + +import lombok.extern.slf4j.Slf4j; +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.time.Duration; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Small helper to provide a consistent pattern for distributed locks across the service layer. + */ +@Slf4j +@Singleton +public class DistributedLockService { + + private final RedissonClient redissonClient; + + @Inject + public DistributedLockService(RedissonClient redissonClient) { + this.redissonClient = redissonClient; + } + + /** + * Execute the given callback guarded by a distributed lock. + * + * @param lockKey the key for the distributed lock + * @param waitTime how long to wait to acquire the lock + * @param leaseTime how long before the lock automatically releases + * @param action the work to run while holding the lock + * @return result of the callback + * @throws TimeoutException if the lock cannot be acquired within the wait time + * @throws Exception bubbled up from the callback + */ + public T withLock(String lockKey, Duration waitTime, Duration leaseTime, Callable action) throws Exception { + RLock lock = redissonClient.getLock(lockKey); + boolean acquired = false; + try { + acquired = lock.tryLock(waitTime.toMillis(), leaseTime.toMillis(), TimeUnit.MILLISECONDS); + if (!acquired) { + throw new TimeoutException("Unable to acquire lock " + lockKey); + } + return action.call(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TimeoutException("Interrupted while acquiring lock " + lockKey); + } finally { + if (acquired && lock.isHeldByCurrentThread()) { + try { + lock.unlock(); + } catch (Exception e) { + log.warn("Failed to release lock {}", lockKey, e); + } + } + } + } +} diff --git a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java index 3eedbaf11..dd2e4789f 100644 --- a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java +++ b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.*; + public enum ExperimentFileColumns { GERMPLASM_NAME(ExperimentObservation.Columns.GERMPLASM_NAME, Column.ColumnDataType.STRING), @@ -32,13 +34,13 @@ public enum ExperimentFileColumns { EXP_TITLE(ExperimentObservation.Columns.EXP_TITLE, Column.ColumnDataType.STRING), EXP_DESCRIPTION(ExperimentObservation.Columns.EXP_DESCRIPTION, Column.ColumnDataType.STRING), EXP_UNIT(ExperimentObservation.Columns.EXP_UNIT, Column.ColumnDataType.STRING), - //SUB_OBS_UNIT(ExperimentObservation.Columns.SUB_OBS_UNIT, Column.ColumnDataType.STRING), + SUB_OBS_UNIT(ExperimentObservation.Columns.SUB_OBS_UNIT, Column.ColumnDataType.STRING), EXP_TYPE(ExperimentObservation.Columns.EXP_TYPE, Column.ColumnDataType.STRING), ENV(ExperimentObservation.Columns.ENV, Column.ColumnDataType.STRING), ENV_LOCATION(ExperimentObservation.Columns.ENV_LOCATION, Column.ColumnDataType.STRING), ENV_YEAR(ExperimentObservation.Columns.ENV_YEAR, Column.ColumnDataType.INTEGER), EXP_UNIT_ID(ExperimentObservation.Columns.EXP_UNIT_ID, Column.ColumnDataType.STRING), - //SUB_UNIT_ID(ExperimentObservation.Columns.SUB_UNIT_ID, Column.ColumnDataType.STRING), + SUB_UNIT_ID(ExperimentObservation.Columns.SUB_UNIT_ID, Column.ColumnDataType.STRING), REP_NUM(ExperimentObservation.Columns.REP_NUM, Column.ColumnDataType.INTEGER), BLOCK_NUM(ExperimentObservation.Columns.BLOCK_NUM, Column.ColumnDataType.INTEGER), ROW(ExperimentObservation.Columns.ROW, Column.ColumnDataType.STRING), @@ -47,10 +49,10 @@ public enum ExperimentFileColumns { LONG(ExperimentObservation.Columns.LONG, Column.ColumnDataType.STRING), ELEVATION(ExperimentObservation.Columns.ELEVATION, Column.ColumnDataType.STRING), RTK(ExperimentObservation.Columns.RTK, Column.ColumnDataType.STRING), - TREATMENT_FACTORS(ExperimentObservation.Columns.TREATMENT_FACTORS, Column.ColumnDataType.STRING), - OBS_UNIT_ID(ExperimentObservation.Columns.OBS_UNIT_ID, Column.ColumnDataType.STRING); + TREATMENT_FACTORS(ExperimentObservation.Columns.TREATMENT_FACTORS, Column.ColumnDataType.STRING); private final Column column; + private static List subEntityOnlyColumns = Arrays.asList(SUB_OBS_UNIT, SUB_UNIT_ID); ExperimentFileColumns(String value, Column.ColumnDataType dataType) { this.column = new Column(value, dataType); @@ -62,6 +64,14 @@ public String toString() { } public static List getOrderedColumns() { + //Don't include subentity columns + return Arrays.stream(ExperimentFileColumns.values()) + .filter(val -> !(subEntityOnlyColumns.contains(val))) + .map(value -> value.column) + .collect(Collectors.toList()); + } + + public static List getOrderedColumnsSubEntity() { return Arrays.stream(ExperimentFileColumns.values()) .map(value -> value.column) .collect(Collectors.toList()); diff --git a/src/main/java/org/breedinginsight/utilities/response/mappers/UserQueryMapper.java b/src/main/java/org/breedinginsight/utilities/response/mappers/UserQueryMapper.java index 2eab94b77..c1ba7f0c6 100644 --- a/src/main/java/org/breedinginsight/utilities/response/mappers/UserQueryMapper.java +++ b/src/main/java/org/breedinginsight/utilities/response/mappers/UserQueryMapper.java @@ -35,7 +35,7 @@ public UserQueryMapper() { fields = Map.ofEntries( Map.entry("name", User::getName), Map.entry("email", User::getEmail), - Map.entry("orcid", User::getOrcid), + Map.entry("oauthId", User::getOauthId), Map.entry("systemRoles", user -> user.getSystemRoles() != null ? user.getSystemRoles().stream() .map(role -> role.getDomain()).collect(Collectors.toList()) : null), diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 313540dbd..d58a8d8d2 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -59,6 +59,17 @@ micronaut: jwks-uri: ${OAUTH_OPENID_JWKSURI:`https://sandbox.orcid.org/oauth/jwks`} user-info: url: ${OAUTH_OPENID_USERINFOURL:`https://sandbox.orcid.org/oauth/userinfo`} + github: + client-id: ${GITHUB_OAUTH_CLIENT_ID:`placeholder`} + client-secret: ${GITHUB_OAUTH_CLIENT_SECRET:`placeholder`} + scopes: + - user:email + - read:user + authorization: + url: https://github.com/login/oauth/authorize + token: + url: https://github.com/login/oauth/access_token + auth-method: client-secret-basic state: cookie: cookie-max-age: 10m diff --git a/src/main/resources/brapi/sql/R__species.sql b/src/main/resources/brapi/sql/R__species.sql index ec6ae4c6c..150c74f7e 100644 --- a/src/main/resources/brapi/sql/R__species.sql +++ b/src/main/resources/brapi/sql/R__species.sql @@ -17,34 +17,35 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; DO $$ DECLARE -v_auth_id CONSTANT uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; + v_auth_id CONSTANT uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; BEGIN /* ------------------------------------------------------------------------------------------ • uuid_generate_v5(namespace, crop_name) → deterministic UUID can be used for idempotency • Do it this way so no schema changes are required • Removed the Honey Bee special case because all systems will be starting fresh ------------------------------------------------------------------------------------------ */ -INSERT INTO crop (id, auth_user_id, crop_name) -SELECT - uuid_generate_v5('9a4deca9-4068-46a3-9efe-db0c181f491a'::uuid, + + INSERT INTO crop (id, auth_user_id, crop_name) + SELECT + uuid_generate_v5('9a4deca9-4068-46a3-9efe-db0c181f491a'::uuid, -- 1) lower‑case -- 2) trim leading/trailing space -- 3) REMOVE every space or tab regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), v_auth_id, crop_name -FROM (VALUES + FROM (VALUES ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), ('Strawberry'), ('Honey Bee'), ('Pecan'), ('Lettuce'), ('Cotton'), ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), ('Red Clover'), ('Potato'), ('Blackberry'), - ('Raspberry'), ('Sugar Beet'), ('Coffee') - ) AS src(crop_name) + ('Raspberry'), ('Sugar Beet'), ('Coffee'), ('Sunflower') + ) AS src(crop_name) ON CONFLICT (id) DO --- want case changes or space changes to overwrite existing --- Only rewrite the row if name changed -UPDATE SET crop_name = EXCLUDED.crop_name -WHERE crop.crop_name IS DISTINCT FROM EXCLUDED.crop_name; -END $$; \ No newline at end of file + -- want case changes or space changes to overwrite existing + -- Only rewrite the row if name changed + UPDATE SET crop_name = EXCLUDED.crop_name + WHERE crop.crop_name IS DISTINCT FROM EXCLUDED.crop_name; +END $$; diff --git a/src/main/resources/db/migration/V1.33.0__experiment_template_delete_obsUnitID_col.sql b/src/main/resources/db/migration/V1.33.0__experiment_template_delete_obsUnitID_col.sql new file mode 100644 index 000000000..01bcbb1c5 --- /dev/null +++ b/src/main/resources/db/migration/V1.33.0__experiment_template_delete_obsUnitID_col.sql @@ -0,0 +1,27 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DO $$ + +BEGIN +-- fix file mapping for ObsUnitID +update importer_mapping +set + mapping = '[{"id": "726a9f10-4892-4204-9e52-bd2b1d735f65", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "98774e20-6f89-4d6a-a7c9-f88887228ed6", "value": {"fileFieldName": "Germplasm GID"}, "objectId": "gid"}, {"id": "880ef0c9-4e3e-42d4-9edc-667684a91889", "value": {"fileFieldName": "Test (T) or Check (C)"}, "objectId": "test_or_check"}, {"id": "b693eca7-efcd-4518-a9d3-db0b037a76ee", "value": {"fileFieldName": "Exp Title"}, "objectId": "exp_title"}, {"id": "df340215-db6e-4219-a3b7-119f297b81c3", "value": {"fileFieldName": "Exp Description"}, "objectId": "expDescription"}, {"id": "9ca7cc81-562c-43a7-989a-41da309f603d", "value": {"fileFieldName": "Exp Unit"}, "objectId": "expUnit"}, {"id": "27215777-c8f9-4fe7-a7ac-918d6168b0dd", "value": {"fileFieldName": "Exp Type"}, "objectId": "expType"}, {"id": "19d220e2-dff0-4a3a-ad6e-32f4d8602b5c", "value": {"fileFieldName": "Env"}, "objectId": "env"}, {"id": "861518b9-5c9e-4fe5-b31e-baf16e27155d", "value": {"fileFieldName": "Env Location"}, "objectId": "envLocation"}, {"id": "667355c3-dae1-4a64-94c8-ac2d543bd474", "value": {"fileFieldName": "Env Year"}, "objectId": "envYear"}, {"id": "ad11f2df-c5b4-4a05-8e52-c57625140061", "value": {"fileFieldName": "Exp Unit ID"}, "objectId": "expUnitId"}, {"id": "639b40ec-20f8-4659-8464-6a4be997ac7a", "value": {"fileFieldName": "Exp Replicate #"}, "objectId": "expReplicateNo"}, {"id": "2a62a80f-d8ba-42c4-9997-3b4ac8a965aa", "value": {"fileFieldName": "Exp Block #"}, "objectId": "expBlockNo"}, {"id": "f3e7de69-21ad-4cda-b1cc-a5e1987fb931", "value": {"fileFieldName": "Row"}, "objectId": "row"}, {"id": "251c5bbd-fc4d-4371-a4ce-4e2686f6837e", "value": {"fileFieldName": "Column"}, "objectId": "column"}, {"id": "a0edcc27-d423-478f-8eed-05b554805ec9", "value": {"fileFieldName": "Lat"}, "objectId": "lat"}, {"id": "a8fd2190-d277-4369-ae72-af32416f14ac", "value": {"fileFieldName": "Long"}, "objectId": "long"}, {"id": "95caefec-2b12-4728-9f2e-3bc17478b662", "value": {"fileFieldName": "Elevation"}, "objectId": "elevation"}, {"id": "e8f80336-0982-4a48-85ac-4b0278e28b70", "value": {"fileFieldName": "RTK"}, "objectId": "rtk"}, {"id": "ce5f61f2-f1de-45a4-8baf-e2471a5d863d", "value": {"fileFieldName": "Treatment Factors"}, "objectId": "treatmentFactors"}]', + file = '[{"Env": "New Study", "Lat": 42.4440, "RTK": "RTK description", "Row": 4, "Long": 76.5019, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "plot", "Elevation": 123, "Exp Title": "New Trial", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Treatment Factors": "Jam application", "Test (T) or Check (C)": true}]' +where import_type_id = 'ExperimentImport'; +END $$; \ No newline at end of file diff --git a/src/main/resources/db/migration/V1.34.0__rename-orcid.sql b/src/main/resources/db/migration/V1.34.0__rename-orcid.sql new file mode 100644 index 000000000..5c559237c --- /dev/null +++ b/src/main/resources/db/migration/V1.34.0__rename-orcid.sql @@ -0,0 +1,28 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Rename orcid column to more generic oauth_id. +ALTER TABLE bi_user +RENAME COLUMN orcid TO oauth_id; + +-- Rename unique constraint. +ALTER TABLE bi_user +RENAME CONSTRAINT orcid_unique TO oauth_id_unique; + +-- Add a column to store the OAuth provider with 'orcid' as the default value. +ALTER TABLE bi_user +ADD COLUMN oauth_provider text DEFAULT 'orcid'; diff --git a/src/main/resources/db/migration/V1.35.0__add_sunflower_to_species.sql b/src/main/resources/db/migration/V1.35.0__add_sunflower_to_species.sql new file mode 100644 index 000000000..7f456bf03 --- /dev/null +++ b/src/main/resources/db/migration/V1.35.0__add_sunflower_to_species.sql @@ -0,0 +1,12 @@ +DO $$ +DECLARE +user_id UUID; +BEGIN + +user_id := (SELECT id FROM bi_user WHERE name = 'system'); + +-- just putting blank strings in for descriptions until later date, can update if needed +INSERT INTO species (common_name, description, created_by, updated_by) +VALUES + ('Sunflower', '', user_id, user_id) ON CONFLICT DO NOTHING; +END $$; diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 05d3816fe..d6ddc6d20 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,6 +14,5 @@ # limitations under the License. # - -version=v1.2.0 -versionInfo=https://github.com/Breeding-Insight/bi-api/releases/tag/v1.2.0 +version=v1.3.0+1157 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/3a6361d30968c32ab97e175c53993991fcc429e9 diff --git a/src/test/java/org/breedinginsight/BrAPITest.java b/src/test/java/org/breedinginsight/BrAPITest.java index bdf362979..37c94a61a 100644 --- a/src/test/java/org/breedinginsight/BrAPITest.java +++ b/src/test/java/org/breedinginsight/BrAPITest.java @@ -49,7 +49,12 @@ public class BrAPITest extends DatabaseTest { public BrAPITest() { super(); - brapiContainer = new GenericContainer<>("breedinginsight/brapi-java-server:develop") + String dockerImage = System.getenv("BRAPI_DOCKER_IMAGE"); + if (dockerImage == null || dockerImage.isEmpty()) { + dockerImage = "breedinginsight/brapi-java-server:develop"; + } + + brapiContainer = new GenericContainer<>(dockerImage) .withNetwork(super.getNetwork()) .withImagePullPolicy(PullPolicy.ageBased(Duration.ofMinutes(60))) .withExposedPorts(8080) diff --git a/src/test/java/org/breedinginsight/api/auth/rules/ProgramSecuredAnnotationRuleIntegrationTest.java b/src/test/java/org/breedinginsight/api/auth/rules/ProgramSecuredAnnotationRuleIntegrationTest.java index 26a53b9ca..58a35c919 100644 --- a/src/test/java/org/breedinginsight/api/auth/rules/ProgramSecuredAnnotationRuleIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/auth/rules/ProgramSecuredAnnotationRuleIntegrationTest.java @@ -45,11 +45,9 @@ import java.time.OffsetDateTime; import java.util.List; -import java.util.UUID; import static io.micronaut.http.HttpRequest.GET; import static io.micronaut.http.HttpRequest.POST; -import static org.breedinginsight.TestUtils.insertAndFetchTestProgram; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -106,8 +104,8 @@ void setup() { dsl.execute(fp.get("InsertPrograms")); programs = programDAO.getAll(); - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); - otherTestUser = userDAO.getUserByOrcId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); + otherTestUser = userDAO.getUserByOAuthId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(); // Insert system roles dsl.execute(fp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java index 0cf098c04..c3ba499c4 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java @@ -90,7 +90,7 @@ void setup() throws Exception { securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index a3c47e573..506c89cef 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -10,10 +10,11 @@ import io.micronaut.http.client.annotation.Client; import io.micronaut.http.client.exceptions.HttpClientResponseException; import io.micronaut.http.netty.cookies.NettyCookie; -import io.micronaut.test.annotation.MicronautTest; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; @@ -53,6 +54,7 @@ import tech.tablesaw.api.Table; import javax.inject.Inject; import java.io.*; +import java.math.BigDecimal; import java.time.OffsetDateTime; import java.util.*; import java.util.stream.Collectors; @@ -62,7 +64,6 @@ @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ExperimentControllerIntegrationTest extends BrAPITest { private Program program; @@ -113,8 +114,8 @@ void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); - otherTestUser = userDAO.getUserByOrcId(TestTokenValidator.OTHER_TEST_USER_ORCID).orElseThrow(Exception::new); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + otherTestUser = userDAO.getUserByOAuthId(TestTokenValidator.OTHER_TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); @@ -213,12 +214,21 @@ void setup() throws Exception { // Create an experiment with no observations. private String uploadExperimentWithoutObs() throws Exception { + return uploadExperimentWithoutObs("Without Obs", "Plot"); + } + + private String uploadExperimentWithoutObs(String title, String expUnit) throws Exception { + return uploadExperimentWithoutObs(program, title, expUnit); + } + + private String uploadExperimentWithoutObs(Program targetProgram, String title, String expUnit) throws Exception { ImportTestUtils importTestUtils = new ImportTestUtils(); List> expRows = new ArrayList<>(); // Make test experiment import. - Map row1 = makeExpImportRow("Without Obs", "NewEnv1"); - Map row2 = makeExpImportRow("Without Obs", "NewEnv2"); + String envBase = title.replaceAll("\\s+", ""); + Map row1 = makeExpImportRow(title, envBase + "1", expUnit); + Map row2 = makeExpImportRow(title, envBase + "2", expUnit); expRows.add(row1); expRows.add(row2); @@ -229,7 +239,7 @@ private String uploadExperimentWithoutObs() throws Exception { null, true, client, - program, + targetProgram, mappingId, newExperimentWorkflowId); String expId = importResult @@ -265,7 +275,7 @@ private String uploadExperimentWithoutObs() throws Exception { @SneakyThrows void downloadDatasets(boolean includeTimestamps, String extension, int numberOfEnvsRequested) { // How many columns are expected in the output? - int expectedColNumber = columns.size(); + int expectedColNumber = columns.size() + 1; //Need to account for ObsUnitID column which is present in export but not import if (includeTimestamps) { expectedColNumber += traits.size(); } @@ -313,14 +323,14 @@ void downloadDatasets(boolean includeTimestamps, String extension, int numberOfE List> filteredRows = rows.stream() .filter(row -> file.getName().contains(row.get(ExperimentObservation.Columns.ENV).toString())) .collect(Collectors.toList()); - parseAndCheck(fileStream, extension, true, filteredRows, includeTimestamps, expectedColNumber); + parseAndCheck(fileStream, extension, true, filteredRows, includeTimestamps, expectedColNumber, "Plot ObsUnitID", filteredRows.size()); } } else { assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); // All (both) rows when 0 or 2 envs sent, first row when 1 env sent as query param. List> filteredRows = numberOfEnvsRequested == 1 ? List.of(rows.get(0)) : rows; - parseAndCheck(bodyStream, extension, numberOfEnvsRequested > 0, filteredRows, includeTimestamps, expectedColNumber); + parseAndCheck(bodyStream, extension, numberOfEnvsRequested > 0, filteredRows, includeTimestamps, expectedColNumber, "Plot ObsUnitID", filteredRows.size()); } // Remove temp directory after each test run. FileUtils.deleteDirectory(new File(tempDir)); @@ -333,14 +343,19 @@ void downloadDatasets(boolean includeTimestamps, String extension, int numberOfE @ParameterizedTest @CsvSource(value = {"CSV", "XLSX", "XLS"}) @SneakyThrows - @Disabled // disabled for now until we re-enable subentity support void downloadSubEntityDataset(String extension) { + Program subEntityProgram = createSeededProgram("SubEntity Download"); + String subEntityExperimentTitle = "SubEntity Download " + extension; + List> topLevelRows = buildObservedRows(subEntityExperimentTitle); + String subEntityExperimentId = uploadExperimentWithObs(subEntityProgram, subEntityExperimentTitle, topLevelRows); + String plantDatasetName = "plant" + extension.toLowerCase(Locale.ROOT); + String plantObservationLevel = StringUtils.capitalize(plantDatasetName); // Create sub-entity dataset. Flowable> postCall = client.exchange( POST(String.format("/programs/%s/experiments/%s/dataset", - program.getId().toString(), experimentId), - "{\"name\":\"Plant\",\"repeatedMeasures\":3}") + subEntityProgram.getId().toString(), subEntityExperimentId), + String.format("{\"name\":\"%s\",\"repeatedMeasures\":3}", plantDatasetName)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class); HttpResponse postResponse = postCall.blockingFirst(); @@ -349,11 +364,11 @@ void downloadSubEntityDataset(String extension) { assertEquals(HttpStatus.OK, postResponse.getStatus()); // Get top-level datasetId to include in export request. - BrAPITrial experiment = experimentService.getTrialDataByUUID(program.getId(), UUID.fromString(experimentId), false); + BrAPITrial experiment = experimentService.getTrialDataByUUID(subEntityProgram.getId(), UUID.fromString(subEntityExperimentId), false); String topLevelDatasetId = DatasetUtil.getTopLevelDataset(experiment).getId().toString(); Flowable> topLevelExportCall = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", - program.getId().toString(), experimentId, extension, topLevelDatasetId)) + subEntityProgram.getId().toString(), subEntityExperimentId, extension, topLevelDatasetId)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class ); HttpResponse topLevelResponse = topLevelExportCall.blockingFirst(); @@ -371,13 +386,22 @@ void downloadSubEntityDataset(String extension) { // Check file contents. ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(topLevelResponse.body())); - parseAndCheck(bodyStream, extension, false, rows, false, 25); + parseAndCheck( + bodyStream, + extension, + false, + topLevelRows, + false, + getExpectedExportColumnCount(false, traits.size(), false), + "Plot ObsUnitID", + topLevelRows.size() + ); // Make sub-entity dataset export request. - String plantDatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), "Plant"); + String plantDatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), plantDatasetName); Flowable> plantExportCall = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", - program.getId().toString(), experimentId, extension, plantDatasetId)) + subEntityProgram.getId().toString(), subEntityExperimentId, extension, plantDatasetId)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class ); HttpResponse plantResponse = plantExportCall.blockingFirst(); @@ -389,11 +413,137 @@ void downloadSubEntityDataset(String extension) { assertEquals(mediaTypeByExtension.get(extension), plantResponse.getHeaders().getContentType().orElseThrow(Exception::new)); // The expected contents of the exported Plant dataset (3 sub-obs units for each top-level unit were requested). - List> plantRows = buildSubEntityRows(rows, "Plant", 3); + List> plantRows = buildSubEntityRows(topLevelRows, plantObservationLevel, 3); // Check file contents. ByteArrayInputStream plantBodyStream = new ByteArrayInputStream(Objects.requireNonNull(plantResponse.body())); - parseAndCheck(plantBodyStream, extension, false, plantRows, false, 23); + parseAndCheck( + plantBodyStream, + extension, + false, + plantRows, + false, + getExpectedExportColumnCount(true, experimentService.getDatasetObsVars(plantDatasetId, subEntityProgram).size(), false), + plantObservationLevel + " ObsUnitID", + plantRows.size() + ); + } + + @Test + public void createSubEntityDatasetRejectsExpUnitNameAlreadyUsedInSameExperiment() throws Exception { + Program testProgram = createSeededProgram("Reject Already Used"); + String plantExperimentId = uploadExperimentWithoutObs(testProgram, "Plant Same Experiment", "Plant"); + + Flowable> call = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", testProgram.getId(), plantExperimentId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + + HttpClientResponseException e = assertThrows(HttpClientResponseException.class, call::blockingFirst); + assertEquals(HttpStatus.CONFLICT, e.getStatus()); + } + + @Test + public void createSubEntityDatasetAllowsExpUnitNameUsedInOtherExperiment() throws Exception { + Program testProgram = createSeededProgram("Allow Other Experiment"); + uploadExperimentWithoutObs(testProgram, "Plant Source Experiment", "Plant"); + String recipientExperimentId = uploadExperimentWithoutObs(testProgram, "Plot Recipient Experiment", "Plot"); + + Flowable> call = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", testProgram.getId(), recipientExperimentId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + } + + @Test + public void createSubEntityDatasetForbiddenForExperimentalCollaborator() throws Exception { + // add otherTestUser to the program as an Experimental Collaborator + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + dsl.execute(securityFp.get("InsertProgramRolesExperimentalCollaborator"), otherTestUser.getId().toString(), program.getId()); + + // add that user to this experiment as a collaborator + JsonObject requestBody = new JsonObject(); + requestBody.addProperty("userId", otherTestUser.getId().toString()); + + Flowable> collaboratorCall = client.exchange( + POST(String.format("/programs/%s/experiments/%s/collaborators", program.getId().toString(), experimentId), requestBody.toString()) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + HttpResponse collaboratorResponse = collaboratorCall.blockingFirst(); + assertEquals(HttpStatus.OK, collaboratorResponse.getStatus()); + + JsonObject collaboratorResult = JsonParser.parseString(collaboratorResponse.body()).getAsJsonObject().getAsJsonObject("result"); + String collaboratorId = collaboratorResult.get("id").getAsString(); + + // collaborator should be blocked from creating sub-entity datasets directly + Flowable> call = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", program.getId(), experimentId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "other-registered-user")), + String.class + ); + + HttpClientResponseException e = assertThrows(HttpClientResponseException.class, call::blockingFirst); + assertEquals(HttpStatus.FORBIDDEN, e.getStatus()); + + // cleanup collaborator record + Flowable> deleteCall = client.exchange( + DELETE(String.format("/programs/%s/experiments/%s/collaborators/%s", program.getId().toString(), experimentId, collaboratorId)) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + HttpResponse deleteResponse = deleteCall.blockingFirst(); + assertEquals(HttpStatus.OK, deleteResponse.getStatus()); + + // cleanup program user role + dsl.execute(securityFp.get("DeleteProgramUser"), otherTestUser.getId().toString()); + } + + @Test + public void recommendedSubEntityDatasetNamesIncludeExpUnitNamesFromOtherExperiments() throws Exception { + Program testProgram = createSeededProgram("Recommended Names"); + uploadExperimentWithoutObs(testProgram, "Plant Autocomplete Source", "Plant"); + String recipientExperimentId = uploadExperimentWithoutObs(testProgram, "Autocomplete Recipient", "Plot"); + + List recommendedNames = getRecommendedSubEntityDatasetNames(testProgram, recipientExperimentId); + + assertTrue(recommendedNames.stream().anyMatch(name -> name.equalsIgnoreCase("plant"))); + assertFalse(recommendedNames.stream().anyMatch(name -> name.equalsIgnoreCase("plot"))); + } + + @Test + public void recommendedSubEntityDatasetNamesDeDuplicateExpUnitAndSubUnitNamesAcrossExperiments() throws Exception { + Program testProgram = createSeededProgram("Dedup Across Experiments"); + uploadExperimentWithoutObs(testProgram, "Plant Exp Unit Source", "Plant"); + String subEntitySourceExperimentId = uploadExperimentWithoutObs(testProgram, "Plant Sub Unit Source", "Plot"); + String recipientExperimentId = uploadExperimentWithoutObs(testProgram, "Plant Unique Recipient", "Plot"); + + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", testProgram.getId(), subEntitySourceExperimentId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + HttpResponse postResponse = postCall.blockingFirst(); + assertEquals(HttpStatus.OK, postResponse.getStatus()); + + List recommendedNames = getRecommendedSubEntityDatasetNames(testProgram, recipientExperimentId); + + assertEquals(1L, recommendedNames.stream().filter(name -> name.equalsIgnoreCase("plant")).count()); } /** @@ -752,21 +902,20 @@ public void deleteExperimentInvalid() { @CsvSource(value = {"true,true", "false,true", "true,false", "false,false"}) @SneakyThrows public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations) { + Program deleteProgram = createSeededProgram("Delete Experiment"); + String deleteExperimentTitle = "Delete Experiment " + UUID.randomUUID(); // Set up a test trial and get the trialDbId. String trialDbId; if (withObservations) { - JsonArray beforeData = getProgramTrials(program.getId().toString()); - - // The trial created by setup has observations. - trialDbId = beforeData.get(0).getAsJsonObject().get("trialDbId").getAsString(); + trialDbId = uploadExperimentWithObs(deleteProgram, deleteExperimentTitle, buildObservedRows(deleteExperimentTitle)); } else { // Create a trial without observations. - trialDbId = uploadExperimentWithoutObs(); + trialDbId = uploadExperimentWithoutObs(deleteProgram, deleteExperimentTitle, "Plot"); } // A DELETE request should delete an experiment with observations unless there are observations and hardDelete = true. Flowable> deleteCall = client.exchange( - DELETE(String.format("/programs/%s/experiments/%s?hard=%s", program.getId().toString(), trialDbId, hardDelete)) + DELETE(String.format("/programs/%s/experiments/%s?hard=%s", deleteProgram.getId().toString(), trialDbId, hardDelete)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class ); @@ -780,15 +929,15 @@ public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations assertEquals(HttpStatus.CONFLICT, e.getStatus()); // Check that the trial was not deleted. - JsonArray trials = getProgramTrials(program.getId().toString()); + JsonArray trials = getProgramTrials(deleteProgram.getId().toString()); assertEquals(1, trials.size()); // Check that the studies were not deleted. - JsonArray studies = getProgramStudies(program.getId().toString()); + JsonArray studies = getProgramStudies(deleteProgram.getId().toString()); assertEquals(2, studies.size()); // Check that lists were not deleted. - JsonArray lists = getProgramObsVarLists(program.getId().toString()); + JsonArray lists = getProgramObsVarLists(deleteProgram.getId().toString()); assertEquals(1, lists.size()); } else { HttpResponse deleteResponse = deleteCall.blockingFirst(); @@ -796,20 +945,55 @@ public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations assertEquals(HttpStatus.NO_CONTENT, deleteResponse.getStatus()); // Check that the trial was deleted. - JsonArray trials = getProgramTrials(program.getId().toString()); + JsonArray trials = getProgramTrials(deleteProgram.getId().toString()); assertEquals(0, trials.size()); // Check that the studies were deleted. - JsonArray studies = getProgramStudies(program.getId().toString()); + JsonArray studies = getProgramStudies(deleteProgram.getId().toString()); assertEquals(0, studies.size()); // Check that the BrAPI lists were deleted. - JsonArray lists = getProgramObsVarLists(program.getId().toString()); + JsonArray lists = getProgramObsVarLists(deleteProgram.getId().toString()); assertEquals(0, lists.size()); } } + private List> buildObservedRows(String title) { + List> observedRows = new ArrayList<>(); + String envBase = title.replaceAll("\\s+", ""); + Map row1 = makeExpImportRow(title, envBase + "1"); + Map row2 = makeExpImportRow(title, envBase + "2"); + + for (int i = 0; i < traits.size(); i++) { + row1.put(traits.get(i).getObservationVariableName(), (float) (i + 1)); + } + + observedRows.add(row1); + observedRows.add(row2); + return observedRows; + } + + private String uploadExperimentWithObs(Program targetProgram, String title, List> expRows) throws Exception { + ImportTestUtils importTestUtils = new ImportTestUtils(); + + JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( + importTestUtils.writeExperimentDataToFile(expRows, traits, false, false, null), + null, + true, + client, + targetProgram, + mappingId, + newExperimentWorkflowId); + + return importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + } + private List> buildSubEntityRows(List> topLevelRows, String entityName, int repeatedMeasures) { List> plantRows = new ArrayList<>(); for (Map row : topLevelRows) { @@ -817,8 +1001,8 @@ private List> buildSubEntityRows(List> t // Deep copy map entries. Map plantRow = new HashMap<>(row); - plantRow.put("Exp Unit", entityName); - plantRow.put("Exp Unit ID", i.toString()); + plantRow.put(ExperimentObservation.Columns.SUB_OBS_UNIT, entityName); + plantRow.put(ExperimentObservation.Columns.SUB_UNIT_ID, i.toString()); plantRow.remove("tt_test_1"); plantRow.remove("tt_test_2"); plantRows.add(plantRow); @@ -827,6 +1011,14 @@ private List> buildSubEntityRows(List> t return plantRows; } + private int getExpectedExportColumnCount(boolean isSubEntity, int obsVarCount, boolean includeTimestamps) { + int baseColumnCount = isSubEntity + ? ExperimentFileColumns.getOrderedColumnsSubEntity().size() + 2 + : ExperimentFileColumns.getOrderedColumns().size() + 1; + int timestampColumnCount = includeTimestamps ? obsVarCount : 0; + return baseColumnCount + obsVarCount + timestampColumnCount; + } + private File writeDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); @@ -845,12 +1037,77 @@ private File writeDataToFile(List> data, List traits) return file; } + private List getRecommendedSubEntityDatasetNames(String targetExperimentId) { + return getRecommendedSubEntityDatasetNames(program, targetExperimentId); + } + + private List getRecommendedSubEntityDatasetNames(Program targetProgram, String targetExperimentId) { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/experiments/%s/recommended-sub-entity-dataset-names", + targetProgram.getId(), targetExperimentId)) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonArray result = JsonParser.parseString(Objects.requireNonNull(response.body())) + .getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data"); + + List recommendedNames = new ArrayList<>(); + result.forEach(name -> recommendedNames.add(name.getAsString())); + return recommendedNames; + } + + private Program createSeededProgram(String prefix) throws Exception { + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + String suffix = UUID.randomUUID().toString().replaceAll("[^a-fA-F]", "").toUpperCase(Locale.ROOT); + while (suffix.length() < 4) { + suffix += UUID.randomUUID().toString().replaceAll("[^a-fA-F]", "").toUpperCase(Locale.ROOT); + } + suffix = suffix.substring(0, 4); + ProgramRequest programRequest = ProgramRequest.builder() + .name(prefix + " " + suffix) + .abbreviation("OT" + suffix) + .documentationUrl("localhost:8080") + .objective("To test ordered experiment scenarios") + .species(SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build()) + .key("OT" + suffix) + .build(); + Program seededProgram = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), seededProgram.getId()); + + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + ontologyService.createTraits(seededProgram.getId(), createTraits(2), user, false); + + List germplasm = createGermplasm(1); + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + newReference.setReferenceID(seededProgram.getId().toString()); + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + germplasmDAO.createBrAPIGermplasm(germplasm, seededProgram.getId(), null); + + return seededProgram; + } + private Map makeExpImportRow(String title, String environment) { + return makeExpImportRow(title, environment, "Plot"); + } + + private Map makeExpImportRow(String title, String environment, String expUnit) { Map row = new HashMap<>(); row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); row.put(ExperimentObservation.Columns.EXP_TITLE, title); - row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_UNIT, expUnit); //row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, ""); row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); row.put(ExperimentObservation.Columns.ENV, environment); @@ -927,7 +1184,9 @@ private void parseAndCheck(InputStream stream, boolean requestEnv, List> rows, boolean includeTimestamps, - Integer expectedColNumber) throws ParsingException { + Integer expectedColNumber, + String expectedObsUnitIdColumn, + Integer expectedObsUnitIdUniqueCount) throws ParsingException { Table download = Table.create(); if (extension.equals("CSV")) { download = FileUtil.parseTableFromCsv(stream); @@ -937,7 +1196,16 @@ private void parseAndCheck(InputStream stream, } // Assert import/export fidelity and presence of observation units in export - checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension, expectedColNumber); + checkDownloadTable( + requestEnv, + rows, + download, + includeTimestamps, + extension, + expectedColNumber, + expectedObsUnitIdColumn, + expectedObsUnitIdUniqueCount + ); } private void checkDownloadTable( @@ -946,12 +1214,15 @@ private void checkDownloadTable( Table table, boolean includeTimestamps, String extension, - Integer expectedColNumber) { + Integer expectedColNumber, + String expectedObsUnitIdColumn, + Integer expectedObsUnitIdUniqueCount) { // Filename is correct: _Observation Dataset [-]__ List expectedEnvNames = requestedImportRows.stream() .map(row -> row.get(ExperimentObservation.Columns.ENV).toString()).collect(Collectors.toList()); assertEquals(expectedColNumber, table.columnCount()); + assertEquals(requestedImportRows.size(), table.rowCount()); // Check that requested envs are present. expectedEnvNames.forEach(envName -> assertTrue(table.stringColumn("Env").contains(envName))); @@ -968,14 +1239,24 @@ private void checkDownloadTable( String gid = ExperimentObservation.Columns.GERMPLASM_GID; String env = ExperimentObservation.Columns.ENV; String expUnitId = ExperimentObservation.Columns.EXP_UNIT_ID; + String subObsUnit = ExperimentObservation.Columns.SUB_OBS_UNIT; + String subUnitId = ExperimentObservation.Columns.SUB_UNIT_ID; + boolean subObsUnitMatches = !row.containsKey(subObsUnit) || + row.get(subObsUnit).equals(downloadRow.getString(subObsUnit)); + boolean subUnitIdMatches = !row.containsKey(subUnitId) || + row.get(subUnitId).equals(downloadRow.getObject(subUnitId).toString()); if (extension.equalsIgnoreCase(FileType.CSV.getName())) { return Integer.parseInt(row.get(gid).toString()) == downloadRow.getInt(gid) && row.get(env).equals(downloadRow.getString(env)) && - row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); + row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()) && + subObsUnitMatches && + subUnitIdMatches; } else { return row.get(gid).equals(downloadRow.getString(gid)) && row.get(env).equals(downloadRow.getString(env)) && - row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); + row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()) && + subObsUnitMatches && + subUnitIdMatches; } }).findAny(); assertTrue(matchingImportRow.isPresent() && !matchingImportRow.get().isEmpty()); @@ -988,56 +1269,73 @@ private void checkDownloadTable( assertEquals(requestedImportRows.size(),matchingImportRows.size()); // Observation units populated. - assertEquals(0, table.column("ObsUnitID").countMissing()); + assertTrue(table.columnNames().contains(expectedObsUnitIdColumn)); + assertEquals(0, table.column(expectedObsUnitIdColumn).countMissing()); // Observation Unit IDs are assigned. - assertEquals(requestedImportRows.size(), table.column("ObsUnitID").countUnique()); + assertEquals(expectedObsUnitIdUniqueCount, table.column(expectedObsUnitIdColumn).countUnique()); } private boolean isMatchedRow(Map importRow, Row downloadRow) { - System.out.println("Validating row: " + downloadRow.getRowNumber()); - System.out.println("import columns: " + importRow.size()); return importRow.entrySet().stream().filter(e -> { String header = e.getKey(); - List importColumns = columns - .stream() - .filter(col -> header.equals(col.getValue())).collect(Collectors.toList()); - if (importColumns.size() != 1) { - return false; - } - Object expectedVal = null; - Object downloadedVal = null; - boolean doCompare = false; - - if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.STRING)) { - expectedVal = e.getValue().toString(); - downloadedVal = downloadRow.getString(e.getKey()); - doCompare = true; - } - if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.INTEGER)) { - expectedVal = Integer.parseInt(e.getValue().toString()); - downloadedVal = downloadRow.getInt(e.getKey()); - doCompare = true; - } - if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.DOUBLE)) { - expectedVal = Double.parseDouble(e.getValue().toString()); - downloadedVal = downloadRow.getDouble(e.getKey()); - doCompare = true; - } - if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.FLOAT)) { - expectedVal = e.getValue(); - downloadedVal = downloadRow.getFloat(e.getKey()); - doCompare = true; - } - System.out.println("Column: "+e.getKey()+", Expected: '"+ expectedVal +"', Received: '" + downloadedVal+"'"); - if(doCompare) { - assertEquals(expectedVal, downloadedVal); - return expectedVal.equals(downloadedVal); - } else { + if (!downloadRow.columnNames().contains(header)) { return false; } + + Object expectedVal = e.getValue(); + Object downloadedVal = getDownloadedValue(downloadRow, header); + assertValuesMatch(header, expectedVal, downloadedVal); + return true; }).count() == importRow.size(); } + private Object getDownloadedValue(Row downloadRow, String header) { + ColumnType columnType = downloadRow.getColumnType(header); + if (columnType.equals(ColumnType.STRING)) { + return downloadRow.getString(header); + } + if (columnType.equals(ColumnType.INTEGER)) { + return downloadRow.getInt(header); + } + if (columnType.equals(ColumnType.DOUBLE)) { + return downloadRow.getDouble(header); + } + if (columnType.equals(ColumnType.FLOAT)) { + return downloadRow.getFloat(header); + } + return downloadRow.getObject(header); + } + + private void assertValuesMatch(String header, Object expectedVal, Object downloadedVal) { + if (isNumeric(expectedVal) && isNumeric(downloadedVal)) { + assertEquals( + 0, + toBigDecimal(expectedVal).compareTo(toBigDecimal(downloadedVal)), + "Column " + header + " mismatch"); + return; + } + assertEquals( + Objects.toString(expectedVal, null), + Objects.toString(downloadedVal, null), + "Column " + header + " mismatch"); + } + + private boolean isNumeric(Object value) { + if (value == null) { + return false; + } + try { + new BigDecimal(value.toString()); + return true; + } catch (NumberFormatException e) { + return false; + } + } + + private BigDecimal toBigDecimal(Object value) { + return new BigDecimal(value.toString()); + } + private String getEnvId(JsonObject result, int index) { return result .get("preview").getAsJsonObject() diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ImportControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ImportControllerIntegrationTest.java index 378e99085..ebd602d3d 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ImportControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ImportControllerIntegrationTest.java @@ -109,7 +109,7 @@ public void setup() { validProgram = insertAndFetchTestProgram(program); // Set program user - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/InternalServerErrorHandlerUnitTest.java b/src/test/java/org/breedinginsight/api/v1/controller/InternalServerErrorHandlerUnitTest.java index 4fc66e151..42d5d976c 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/InternalServerErrorHandlerUnitTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/InternalServerErrorHandlerUnitTest.java @@ -112,7 +112,7 @@ void setup() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); } @BeforeEach diff --git a/src/test/java/org/breedinginsight/api/v1/controller/JobControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/JobControllerIntegrationTest.java index 6949499e9..e964cbb92 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/JobControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/JobControllerIntegrationTest.java @@ -39,7 +39,6 @@ import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.jooq.DSLContext; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -95,7 +94,7 @@ public void setup() { program = programs.get(0); // Insert system roles - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID) + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID) .get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId() diff --git a/src/test/java/org/breedinginsight/api/v1/controller/OntologyControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/OntologyControllerIntegrationTest.java index bf8dde632..a8293b5d6 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/OntologyControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/OntologyControllerIntegrationTest.java @@ -72,7 +72,7 @@ void setup() throws Exception { brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); brapiObservationFp = FannyPack.fill("src/test/resources/sql/brapi/BrAPIOntologyControllerIntegrationTest.sql"); - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ProgramControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ProgramControllerIntegrationTest.java index b3ef32a2b..32a044fae 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ProgramControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ProgramControllerIntegrationTest.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.*; -import com.google.gson.reflect.TypeToken; import io.kowalski.fannypack.FannyPack; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; @@ -42,7 +41,6 @@ import org.breedinginsight.api.model.v1.request.*; import org.breedinginsight.api.model.v1.request.query.FilterRequest; import org.breedinginsight.api.model.v1.request.query.SearchRequest; -import org.breedinginsight.api.model.v1.response.Response; import org.breedinginsight.api.v1.controller.metadata.SortOrder; import org.breedinginsight.dao.db.tables.daos.ProgramDao; import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; @@ -58,7 +56,6 @@ import javax.inject.Inject; import javax.inject.Named; -import java.lang.reflect.Type; import java.math.BigDecimal; import java.time.OffsetDateTime; import java.util.*; @@ -165,13 +162,13 @@ void setup() throws Exception { securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); - otherUser = userDAO.getUserByOrcId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); + otherUser = userDAO.getUserByOAuthId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); - Optional optionalUser = userService.getByOrcid(TestTokenValidator.TEST_USER_ORCID); + Optional optionalUser = userService.getByOAuthId(TestTokenValidator.TEST_USER_ORCID); testUser = optionalUser.get(); // Get species for tests @@ -275,7 +272,7 @@ public ProgramLocation insertAndFetchTestLocation() throws Exception { public User fetchTestUser() throws Exception{ - Optional user = userService.getByOrcid(TestTokenValidator.TEST_USER_ORCID); + Optional user = userService.getByOAuthId(TestTokenValidator.TEST_USER_ORCID); if (!user.isPresent()){ throw new Exception("Failed to insert test user"); } diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java index ec28c689b..79c8095cb 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -48,7 +48,6 @@ import org.breedinginsight.model.*; import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.parsers.ParsingException; -import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.Utilities; @@ -102,7 +101,7 @@ void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species @@ -407,7 +406,7 @@ private String createExperiment(Program program) throws IOException, Interrupted .get(0).getAsJsonObject().get("id").getAsString(); JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( - importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null), + importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null, false, false, null), null, true, client, diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TestTokenValidator.java b/src/test/java/org/breedinginsight/api/v1/controller/TestTokenValidator.java index 644ddc046..0e40555ab 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TestTokenValidator.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TestTokenValidator.java @@ -53,7 +53,7 @@ public TestTokenValidator(Collection signatureConfigurat public Publisher validateToken(String token) { if (token.equals("test-registered-user")) { - Optional testUser = userService.getByOrcid(TEST_USER_ORCID); + Optional testUser = userService.getByOAuthId(TEST_USER_ORCID); Map adminClaims = new HashMap<>(); List roles = new ArrayList<>(); roles.add("SYSTEM ADMINISTRATOR"); @@ -61,14 +61,14 @@ public Publisher validateToken(String token) { adminClaims.put("id", testUser.get().getId().toString()); return Flowable.just(new DefaultAuthentication(TEST_USER_ORCID, adminClaims)); } else if (token.equals("other-registered-user")) { - Optional otherTestUser = userService.getByOrcid(OTHER_TEST_USER_ORCID); + Optional otherTestUser = userService.getByOAuthId(OTHER_TEST_USER_ORCID); Map userClaims = new HashMap<>(); List roles = new ArrayList<>(); userClaims.put("roles", roles); userClaims.put("id", otherTestUser.get().getId().toString()); return Flowable.just(new DefaultAuthentication(OTHER_TEST_USER_ORCID, userClaims)); } else if (token.equals("another-registered-user")) { - Optional anotherTestUser = userService.getByOrcid(ANOTHER_TEST_USER_ORCID); + Optional anotherTestUser = userService.getByOAuthId(ANOTHER_TEST_USER_ORCID); Map userClaims = new HashMap<>(); List roles = new ArrayList<>(); userClaims.put("roles", roles); @@ -82,7 +82,7 @@ public Publisher validateToken(String token) { adminClaims.put("id", NON_EXISTENT_USER_ID); return Flowable.just(new DefaultAuthentication(NON_EXISTENT_USER_ID, adminClaims)); } else if (token.equals("inactive-user")) { - Optional inactiveUser = userService.getByOrcid(INACTIVE_USER_ORCID); + Optional inactiveUser = userService.getByOAuthId(INACTIVE_USER_ORCID); Map adminClaims = new HashMap<>(); List roles = new ArrayList<>(); roles.add("SYSTEM ADMINISTRATOR"); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java index ebfeec7ed..08c7edd37 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java @@ -160,7 +160,7 @@ public void setup() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Insert program diff --git a/src/test/java/org/breedinginsight/api/v1/controller/UploadControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/UploadControllerIntegrationTest.java index 6549586e0..b008ccf51 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/UploadControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/UploadControllerIntegrationTest.java @@ -107,7 +107,7 @@ public void setup() { validProgram = insertAndFetchTestProgram(program); // Set program user - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java index 23fcfa32c..4914824f3 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java @@ -110,8 +110,8 @@ void setup() { dsl.execute(fp.get("InsertProgram")); dsl.execute(fp.get("InsertUserProgramAssociations")); - testUser = biUserDao.fetchByOrcid(TestTokenValidator.TEST_USER_ORCID).get(0); - otherTestUser = biUserDao.fetchByOrcid(TestTokenValidator.OTHER_TEST_USER_ORCID).get(0); + testUser = biUserDao.fetchByOauthId(TestTokenValidator.TEST_USER_ORCID).get(0); + otherTestUser = biUserDao.fetchByOauthId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(0); validSystemRole = systemRoleDao.findAll().get(0); validRole = roleDao.findAll().get(0); validPrograms = programDao.findAll(); @@ -144,7 +144,7 @@ public void getUsersExistingId() { JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(testUser.getId().toString(), result.get("id").getAsString(), "Wrong id"); assertEquals("Test User", result.get("name").getAsString(), "Wrong name"); - assertEquals(testUser.getOrcid(), result.get("orcid").getAsString(), "Wrong orcid"); + assertEquals(testUser.getOauthId(), result.get("oauthId").getAsString(), "Wrong OAuthId"); assertEquals("test@test.com", result.get("email").getAsString(), "Wrong email"); JsonArray resultRoles = (JsonArray) result.get("systemRoles"); @@ -203,12 +203,12 @@ void postUsersRolesNotFound() { String name = "Test User4"; String email = "test4@test.com"; - String orcid = "0000-0000-0000-0000"; + String oAuthId = "0000-0000-0000-0000"; JsonObject requestBody = new JsonObject(); requestBody.addProperty("name", name); requestBody.addProperty("email", email); - requestBody.addProperty("orcid", orcid); + requestBody.addProperty("oauthId", oAuthId); JsonObject role = new JsonObject(); role.addProperty("id", invalidUUID); JsonArray roles = new JsonArray(); @@ -584,7 +584,7 @@ public void getUserInfoRegisteredUser() { JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(otherTestUser.getName(), result.get("name").getAsString(), "Wrong name"); - assertEquals(otherTestUser.getOrcid(), result.get("orcid").getAsString(), "Wrong orcid"); + assertEquals(otherTestUser.getOauthId(), result.get("oauthId").getAsString(), "Wrong orcid"); assertEquals(otherTestUser.getEmail(), result.get("email").getAsString(), "Wrong email"); assertEquals(otherTestUser.getId().toString(), result.get("id").getAsString(), "Wrong id"); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java index bd565374e..0fbfe4035 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java @@ -181,7 +181,7 @@ public void insertTestData() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Insert program diff --git a/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java index 4257f90d6..4f04c3578 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java @@ -84,8 +84,8 @@ public void setup() { dsl.execute(fp.get("InsertUserProgramAssociations")); dsl.execute(fp.get("InsertManyUsers")); - testUser = biUserDao.fetchByOrcid(TestTokenValidator.TEST_USER_ORCID).get(0); - otherTestUser = biUserDao.fetchByOrcid(TestTokenValidator.OTHER_TEST_USER_ORCID).get(0); + testUser = biUserDao.fetchByOauthId(TestTokenValidator.TEST_USER_ORCID).get(0); + otherTestUser = biUserDao.fetchByOauthId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(0); dsl.execute(fp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); dsl.execute(fp.get("InsertSystemRoleAdmin"), otherTestUser.getId().toString()); diff --git a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiAuthorizeControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiAuthorizeControllerIntegrationTest.java index 861bf27db..5567f7db8 100644 --- a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiAuthorizeControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiAuthorizeControllerIntegrationTest.java @@ -67,7 +67,7 @@ public void setup() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Insert program diff --git a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java index 038a691ec..9173d3cdf 100644 --- a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java @@ -105,8 +105,8 @@ public void setup() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); - User otherTestUser = userDAO.getUserByOrcId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); + User otherTestUser = userDAO.getUserByOAuthId(TestTokenValidator.OTHER_TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Insert program diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java index 61ce61a57..4f15c553d 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java @@ -106,7 +106,7 @@ void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java index 6bea444f9..44792ea64 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java @@ -110,7 +110,7 @@ void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java index 27cf6defd..59cd33630 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java @@ -111,7 +111,7 @@ void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java index d0b440f79..936bd401c 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java @@ -96,7 +96,7 @@ public Tuple2> setupTestProgram(DSLContext brAPIDslContext FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species @@ -121,7 +121,7 @@ public Tuple2> setupTestProgram(DSLContext brAPIDslContext dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); // Add Experimental Collaborator user. - User collaborator = userDAO.getUserByOrcId(TestTokenValidator.OTHER_TEST_USER_ORCID).orElseThrow(Exception::new); + User collaborator = userDAO.getUserByOAuthId(TestTokenValidator.OTHER_TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertProgramRolesExperimentalCollaborator"), collaborator.getId().toString(), program.getId().toString()); // Get experiment import map @@ -208,7 +208,7 @@ public Tuple2> setupTestProgram(DSLContext brAPIDslContext .get("id").getAsString(); // Refetch collaborator, since we updated program_user_roles. - collaborator = userDAO.getUserByOrcId(TestTokenValidator.OTHER_TEST_USER_ORCID).orElseThrow(Exception::new); + collaborator = userDAO.getUserByOAuthId(TestTokenValidator.OTHER_TEST_USER_ORCID).orElseThrow(Exception::new); // Add collaborator to experiment. dsl.execute(securityFp.get("AddCollaborator"), experiment2Id, collaborator.getProgramRoles().get(0).getId()); diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java index 1e1cb922b..24868f3fe 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java @@ -89,7 +89,7 @@ public void setup() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID) + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID) .get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId() diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java index 42dfa47c2..9fc0aac90 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java @@ -108,7 +108,7 @@ void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species diff --git a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java index 760974150..2a0ad750f 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java @@ -31,6 +31,7 @@ import org.junit.jupiter.api.*; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import tech.tablesaw.api.Table; import javax.inject.Inject; import java.io.ByteArrayInputStream; @@ -40,7 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import tech.tablesaw.api.Table; + import static io.micronaut.http.HttpRequest.GET; import static io.micronaut.http.HttpRequest.POST; @@ -87,7 +88,7 @@ public void setup() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); var brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Set up BrAPI @@ -243,7 +244,7 @@ public void getAllGermplasmListsSuccess() { } } } - + @ParameterizedTest @CsvSource(value = {"CSV", "XLSX", "XLS"}) @SneakyThrows @@ -276,6 +277,7 @@ public void germplasmListExport(String extension) { int dataSize = download.rowCount(); assertEquals(3, dataSize, "Wrong number of germplasm were returned"); } + @Test @SneakyThrows public void getAllGermplasmByListSuccess() { diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index 108eb2fea..b5ee8ce0d 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -148,7 +148,7 @@ public void setup() { newExp.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetchWorkflow( - importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); } @Test @@ -198,7 +198,6 @@ public void getAllListsSuccess() { } } - @Test @SneakyThrows @Order(2) diff --git a/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java new file mode 100644 index 000000000..d958d0b44 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java @@ -0,0 +1,112 @@ +package org.breedinginsight.brapi.v2; + +import com.google.gson.*; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpRequest; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.model.Program; +import org.junit.jupiter.api.*; + +import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.*; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class SubEntityDatasetLockIntegrationTest extends BrAPITest { + + private Program program; + private String experimentId; + + @Inject + private BrAPITestUtils brAPITestUtils; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private final Gson gson = new GsonBuilder() + .registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .create(); + + @BeforeAll + void setup() throws Exception { + var setup = brAPITestUtils.setupTestProgram(super.getBrapiDsl(), gson); + program = setup.getV1(); + experimentId = setup.getV2().get(0); + } + + @Test + void concurrentDatasetCreateReturnsSingleSuccessAndConflict() throws Exception { + // Use a fresh name to avoid interference with other runs + String datasetName = "LockTest-" + UUID.randomUUID(); + JsonObject request = new JsonObject(); + request.addProperty("name", datasetName); + request.addProperty("repeatedMeasures", 1); + + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch start = new CountDownLatch(1); + + Callable call = () -> { + start.await(1, TimeUnit.SECONDS); + try { + Flowable> response = client.exchange( + HttpRequest.POST(String.format("/programs/%s/experiments/%s/dataset", program.getId(), experimentId), request.toString()) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), + String.class + ); + return response.blockingFirst().getStatus(); + } catch (io.micronaut.http.client.exceptions.HttpClientResponseException e) { + return e.getStatus(); + } + }; + + Future first = executor.submit(call); + Future second = executor.submit(call); + start.countDown(); + + HttpStatus status1 = first.get(10, TimeUnit.SECONDS); + HttpStatus status2 = second.get(10, TimeUnit.SECONDS); + executor.shutdownNow(); + + List statuses = List.of(status1, status2); + assertTrue(statuses.contains(HttpStatus.OK)); + assertTrue(statuses.contains(HttpStatus.CONFLICT)); + + // Confirm only one dataset with that name exists + Flowable> datasetsCall = client.exchange( + HttpRequest.GET(String.format("/programs/%s/experiments/%s/datasets", program.getId(), experimentId)) + .bearerAuth("test-registered-user"), + String.class + ); + HttpResponse datasetsResponse = datasetsCall.blockingFirst(); + assertEquals(HttpStatus.OK, datasetsResponse.getStatus()); + JsonObject parsed = JsonParser.parseString(Objects.requireNonNull(datasetsResponse.body())).getAsJsonObject(); + JsonArray resultArray = parsed.has("result") && parsed.get("result").isJsonArray() + ? parsed.getAsJsonArray("result") + : null; + long matching = 0; + assertNotEquals(null, resultArray); + for (int i = 0; i < resultArray.size(); i++) { + String name = resultArray.get(i).getAsJsonObject().get("name").getAsString(); + if (name.equalsIgnoreCase(datasetName)) { + matching++; + } + } + assertEquals(1, matching); + } +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java b/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java new file mode 100644 index 000000000..5a055bdd0 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java @@ -0,0 +1,316 @@ +package org.breedinginsight.brapi.v2.services; + +import com.google.gson.JsonObject; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPISeason; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; +import org.brapi.v2.model.pheno.BrAPIObservationUnitPosition; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationLevelDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; +import org.breedinginsight.brapps.importer.model.exports.FileType; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.FileMappingUtil; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; +import org.breedinginsight.model.BrAPIConstants; +import org.breedinginsight.model.Dataset; +import org.breedinginsight.model.DownloadFile; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.delta.DeltaEntityFactory; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.TraitService; +import org.breedinginsight.services.lock.DistributedLockService; +import org.breedinginsight.utilities.FileUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.tablesaw.api.Table; + +import java.io.ByteArrayInputStream; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class BrAPITrialServiceUnitTest { + private static final String REFERENCE_SOURCE = "breedinginsight.org"; + private static final String EXPORT_DATASET_ID = "33333333-3333-3333-3333-333333333333"; + private static final String DATASET_ID = "22222222-2222-2222-2222-222222222222"; + private static final String ENVIRONMENT_ID = "44444444-4444-4444-4444-444444444444"; + private static final String SECOND_ENVIRONMENT_ID = "55555555-5555-5555-5555-555555555555"; + + private final BrAPITrialDAO trialDAO = mock(BrAPITrialDAO.class); + private final BrAPIObservationDAO observationDAO = mock(BrAPIObservationDAO.class); + private final BrAPIObservationUnitDAO observationUnitDAO = mock(BrAPIObservationUnitDAO.class); + private final BrAPIListDAO listDAO = mock(BrAPIListDAO.class); + private final TraitService traitService = mock(TraitService.class); + private final BrAPIStudyDAO studyDAO = mock(BrAPIStudyDAO.class); + private final BrAPISeasonDAO seasonDAO = mock(BrAPISeasonDAO.class); + private final BrAPIObservationLevelDAO observationLevelDAO = mock(BrAPIObservationLevelDAO.class); + private final BrAPIGermplasmDAO germplasmDAO = mock(BrAPIGermplasmDAO.class); + private final FileMappingUtil fileMappingUtil = mock(FileMappingUtil.class); + private final DistributedLockService lockService = mock(DistributedLockService.class); + private final DatasetService datasetService = mock(DatasetService.class); + private final DeltaEntityFactory deltaEntityFactory = mock(DeltaEntityFactory.class); + private final BrAPIObservationLevelService observationLevelService = mock(BrAPIObservationLevelService.class); + + private BrAPITrialService service; + private Program program; + private BrAPITrial experiment; + private BrAPIStudy study; + private BrAPIGermplasm germplasm; + private BrAPISeason season; + + @BeforeEach + void setup() { + service = new BrAPITrialService( + REFERENCE_SOURCE, + trialDAO, + observationDAO, + observationUnitDAO, + listDAO, + traitService, + studyDAO, + seasonDAO, + observationUnitDAO, + observationLevelDAO, + germplasmDAO, + fileMappingUtil, + lockService, + datasetService, + deltaEntityFactory, + observationLevelService + ); + + program = new Program(); + program.setId(UUID.randomUUID()); + program.setKey("TEST"); + + experiment = new BrAPITrial(); + experiment.setTrialDbId("trial-db-id"); + experiment.setTrialName("Unit Test Experiment"); + experiment.setTrialDescription("Trial description"); + JsonObject experimentInfo = new JsonObject(); + experimentInfo.addProperty(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE, "Phenotyping"); + experiment.setAdditionalInfo(experimentInfo); + + study = new BrAPIStudy(); + study.setStudyDbId("study-1"); + study.setStudyName("Environment 1"); + study.setLocationName("Location 1"); + study.setSeasons(List.of("season-1")); + study.setExternalReferences(List.of(createExternalReference( + String.format("%s/%s", REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName()), + ENVIRONMENT_ID + ))); + + germplasm = new BrAPIGermplasm(); + germplasm.setGermplasmDbId("germ-1"); + germplasm.setAccessionNumber("G-1"); + + season = new BrAPISeason(); + season.setSeasonDbId("season-1"); + season.setYear(2023); + } + + @Test + void exportObservationsFetchesSeasonOncePerDistinctSeasonAndWritesEnvYears() throws Exception { + ExperimentExportQuery params = exportQuery(EXPORT_DATASET_ID); + BrAPIStudy secondStudy = new BrAPIStudy(); + secondStudy.setStudyDbId("study-2"); + secondStudy.setStudyName("Environment 2"); + secondStudy.setLocationName("Location 2"); + secondStudy.setSeasons(List.of("season-2")); + secondStudy.setExternalReferences(List.of(createExternalReference( + String.format("%s/%s", REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName()), + SECOND_ENVIRONMENT_ID + ))); + + BrAPISeason secondSeason = new BrAPISeason(); + secondSeason.setSeasonDbId("season-2"); + secondSeason.setYear(2024); + + List observationUnits = new ArrayList<>(List.of( + createObservationUnit("ou-db-1", "plot-1"), + createObservationUnit("ou-db-2", "plot-2", "study-2", "Environment 2", SECOND_ENVIRONMENT_ID) + )); + + when(trialDAO.getTrialsByExperimentIds(eq(List.of(UUID.fromString("11111111-1111-1111-1111-111111111111"))), eq(program))) + .thenReturn(List.of(experiment)); + when(studyDAO.getStudiesByExperimentID(eq(UUID.fromString("11111111-1111-1111-1111-111111111111")), eq(program))) + .thenReturn(List.of(study, secondStudy)); + when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); + when(seasonDAO.getSeasonById("season-2", program.getId())).thenReturn(secondSeason); + when(observationUnitDAO.getObservationUnitsForDataset(EXPORT_DATASET_ID, program)).thenReturn(observationUnits); + when(listDAO.getListsByTypeAndExternalRef(any(), eq(program.getId()), any(), any())).thenReturn(Collections.emptyList()); + when(observationDAO.getObservationsByObservationUnits(anyCollection(), eq(program))).thenReturn(Collections.emptyList()); + when(germplasmDAO.getGermplasmsByDBID(anyList(), eq(program.getId()))).thenReturn(List.of(germplasm)); + + DownloadFile downloadFile = service.exportObservations(program, UUID.fromString("11111111-1111-1111-1111-111111111111"), params); + + Table exportTable = FileUtil.parseTableFromCsv(new ByteArrayInputStream(downloadFile.getStreamedFile().getInputStream().readAllBytes())); + assertEquals(2, exportTable.rowCount()); + assertEquals(List.of("Environment 1", "Environment 2"), exportTable.stringColumn(Columns.ENV).asList()); + assertEquals(List.of(2023, 2024), exportTable.intColumn(Columns.ENV_YEAR).asList()); + verify(seasonDAO, times(1)).getSeasonById("season-1", program.getId()); + verify(seasonDAO, times(1)).getSeasonById("season-2", program.getId()); + } + + @Test + void getDatasetDataFetchesSeasonOncePerDistinctSeasonAndWritesEnvYears() throws Exception { + BrAPIStudy secondStudy = new BrAPIStudy(); + secondStudy.setStudyDbId("study-2"); + secondStudy.setStudyName("Environment 2"); + secondStudy.setLocationName("Location 2"); + secondStudy.setSeasons(List.of("season-2")); + secondStudy.setExternalReferences(List.of(createExternalReference( + String.format("%s/%s", REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName()), + SECOND_ENVIRONMENT_ID + ))); + + BrAPISeason secondSeason = new BrAPISeason(); + secondSeason.setSeasonDbId("season-2"); + secondSeason.setYear(2024); + + List observationUnits = new ArrayList<>(List.of( + createObservationUnit("ou-db-1", "plot-1"), + createObservationUnit("ou-db-2", "plot-2", "study-2", "Environment 2", SECOND_ENVIRONMENT_ID) + )); + + when(observationUnitDAO.getObservationUnitsForDataset(DATASET_ID, program)).thenReturn(observationUnits); + when(studyDAO.getStudiesByStudyDbId(eq(Set.of("study-1", "study-2")), eq(program))).thenReturn(List.of(study, secondStudy)); + when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); + when(seasonDAO.getSeasonById("season-2", program.getId())).thenReturn(secondSeason); + when(listDAO.getListsByTypeAndExternalRef(any(), eq(program.getId()), any(), any())).thenReturn(Collections.emptyList()); + when(observationDAO.getObservationsByObservationUnitsAndVariables(anyList(), eq(Collections.emptyList()), eq(program))) + .thenReturn(Collections.emptyList()); + + Dataset dataset = service.getDatasetData(program, UUID.randomUUID(), UUID.fromString(DATASET_ID), false); + + assertEquals(2, dataset.observationUnits.size()); + assertEquals(List.of(2023, 2024), dataset.observationUnits.stream() + .map(ou -> ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENV_YEAR).getAsInt()) + .collect(Collectors.toList())); + verify(studyDAO, times(1)).getStudiesByStudyDbId(eq(Set.of("study-1", "study-2")), eq(program)); + verify(seasonDAO, times(1)).getSeasonById("season-1", program.getId()); + verify(seasonDAO, times(1)).getSeasonById("season-2", program.getId()); + } + + @Test + void getDatasetDataThrowsWhenSeasonYearIsNull() throws Exception { + // Dataset retrieval now shares the same year resolution path as export, so + // unsupported null season years fail consistently across both entry points. + List observationUnits = List.of(createObservationUnit("ou-db-1", "plot-1")); + season.setYear(null); + + when(observationUnitDAO.getObservationUnitsForDataset(DATASET_ID, program)).thenReturn(observationUnits); + when(studyDAO.getStudiesByStudyDbId(eq(Set.of("study-1")), eq(program))).thenReturn(List.of(study)); + when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); + + DoesNotExistException exception = assertThrows(DoesNotExistException.class, + () -> service.getDatasetData(program, UUID.randomUUID(), UUID.fromString(DATASET_ID), false)); + + assertEquals("Env Year not found for Study DbId = 'study-1'.", exception.getMessage()); + } + + private ExperimentExportQuery exportQuery(String datasetId) throws Exception { + ExperimentExportQuery params = new ExperimentExportQuery(); + setField(params, "datasetId", datasetId); + setField(params, "fileExtension", FileType.CSV); + setField(params, "includeTimestamps", false); + return params; + } + + private BrAPIObservationUnit createObservationUnit(String observationUnitDbId, String observationUnitName) { + return createObservationUnit(observationUnitDbId, observationUnitName, "study-1", "Environment 1", ENVIRONMENT_ID); + } + + private BrAPIObservationUnit createObservationUnit( + String observationUnitDbId, + String observationUnitName, + String studyDbId, + String studyName, + String environmentId) { + BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); + observationUnit.setObservationUnitDbId(observationUnitDbId); + observationUnit.setObservationUnitName(observationUnitName); + observationUnit.setStudyDbId(studyDbId); + observationUnit.setStudyName(studyName); + observationUnit.setGermplasmDbId("germ-1"); + observationUnit.setGermplasmName("Germplasm 1"); + observationUnit.setExternalReferences(List.of( + createExternalReference( + String.format("%s/%s", REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()), + observationUnitDbId + ), + createExternalReference( + String.format("%s/%s", REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName()), + environmentId + ))); + observationUnit.setObservationUnitPosition(createObservationUnitPosition()); + return observationUnit; + } + + private BrAPIObservationUnitPosition createObservationUnitPosition() { + BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); + BrAPIObservationUnitLevelRelationship observationLevel = new BrAPIObservationUnitLevelRelationship(); + observationLevel.setLevelName("plot"); + observationLevel.setLevelCode("plot"); + observationLevel.setLevelOrder(1); + position.setObservationLevel(observationLevel); + + BrAPIObservationUnitLevelRelationship repLevel = new BrAPIObservationUnitLevelRelationship(); + repLevel.setLevelName(BrAPIConstants.REPLICATE.getValue()); + repLevel.setLevelCode("1"); + repLevel.setLevelOrder(2); + + BrAPIObservationUnitLevelRelationship blockLevel = new BrAPIObservationUnitLevelRelationship(); + blockLevel.setLevelName(BrAPIConstants.BLOCK.getValue()); + blockLevel.setLevelCode("1"); + blockLevel.setLevelOrder(3); + + position.setObservationLevelRelationships(new ArrayList<>(List.of(repLevel, blockLevel))); + return position; + } + + private BrAPIExternalReference createExternalReference(String source, String id) { + BrAPIExternalReference externalReference = new BrAPIExternalReference(); + externalReference.setReferenceSource(source); + externalReference.setReferenceId(id); + externalReference.setReferenceID(id); + return externalReference; + } + + private void setField(Object target, String fieldName, Object value) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 7a984072c..802bf6889 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -48,6 +48,7 @@ import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; +import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; @@ -63,6 +64,8 @@ import org.breedinginsight.services.exceptions.BadRequestException; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.utilities.DatasetUtil; +import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import org.junit.jupiter.api.*; @@ -70,8 +73,10 @@ import org.junit.jupiter.params.provider.ValueSource; import org.junit.platform.commons.util.StringUtils; import org.opentest4j.AssertionFailedError; +import tech.tablesaw.api.Table; import javax.inject.Inject; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.time.OffsetDateTime; @@ -80,6 +85,8 @@ import java.util.stream.StreamSupport; import static io.micronaut.http.HttpRequest.GET; +import static io.micronaut.http.HttpRequest.POST; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.*; @@ -92,6 +99,7 @@ public class ExperimentFileImportTest extends BrAPITest { private FannyPack securityFp; private String mappingId; private BiUserEntity testUser; + private Program program; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -135,6 +143,9 @@ public class ExperimentFileImportTest extends BrAPITest { @Inject private ProgramLocationService locationService; + @Inject + private BrAPITrialService experimentService; + @Inject private BrAPIGermplasmDAO germplasmDAO; @@ -159,6 +170,7 @@ public void setup() { mappingId = (String) setupObjects.get("mappingId"); testUser = (BiUserEntity) setupObjects.get("testUser"); securityFp = (FannyPack) setupObjects.get("securityFp"); + program = (Program) setupObjects.get("program"); newExperimentWorkflowId = importTestUtils.getExperimentWorkflowId(client, 0); appendOverwriteWorkflowId = importTestUtils.getExperimentWorkflowId(client, 1); } @@ -179,6 +191,257 @@ public void setup() { - existing env that already has observation variables (existing dataset) */ + @Test + @SneakyThrows + @Disabled + /** + * TODO: Re-enable this test after dynamic observation level name migration + * Currently TrialService::isSubEntityDataset has an index out of bounds exception because + * plot has a level order of 6 in the new brapi server but the code is expecting level order 0 + */ + public void appendExperimentWithObsVarFromPriorDataset() { + log.debug("appendExperimentWithObsVarFromPriorDataset"); + + // Create a plot-level dataset that includes observation variable tt_test_1 + List traits = importTestUtils.createTraits(1); + Program program = createProgram("Append Exp with Prior Observations Vars", "EXPPRI", "EXPPRI", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), "1"); + + JsonObject importResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); + String expId = importResponse + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + + // Create sub-entity dataset that has two plant-level units but no obsvars and, therefore, no observations. + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", + program.getId().toString(), expId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class); + HttpResponse postResponse = postCall.blockingFirst(); + + // Assert 200 response + assertEquals(HttpStatus.OK, postResponse.getStatus()); + JsonObject result = JsonParser.parseString(postResponse.body()).getAsJsonObject().getAsJsonObject("result"); + + // Export the plant-level sub-entity dataset + String extension = "CSV"; + BrAPITrial experiment = experimentService.getTrialDataByUUID(program.getId(), UUID.fromString(expId), false); + String plantDatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), "Plant"); + Flowable> plantExportCall = client.exchange( + GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", + program.getId().toString(), expId, extension, plantDatasetId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse plantResponse = plantExportCall.blockingFirst(); + + // Assert 200 response + assertEquals(HttpStatus.OK, plantResponse.getStatus()); + + // Parse the export table + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(plantResponse.body())); + Table exportTable = FileUtil.parseTableFromCsv(bodyStream); + + // Build a request to append tt_test_1 observation data on the sub-entity dataset + String sub1ObsUnitId = exportTable.row(0).getString("Plant ObsUnitID"); + String sub2ObsUnitId = exportTable.row(1).getString("Plant ObsUnitID"); + + Map sub1 = new HashMap<>(); + sub1.put(Columns.GERMPLASM_GID, "1"); + sub1.put(Columns.TEST_CHECK, "T"); + sub1.put(Columns.EXP_TITLE, "Test Exp"); + sub1.put(Columns.EXP_UNIT, "Plot"); + sub1.put(Columns.EXP_TYPE, "Phenotyping"); + sub1.put(Columns.ENV, "New Env"); + sub1.put(Columns.ENV_LOCATION, "Location A"); + sub1.put(Columns.ENV_YEAR, "2023"); + sub1.put(Columns.EXP_UNIT_ID, "a-1"); + sub1.put(Columns.REP_NUM, "1"); + sub1.put(Columns.BLOCK_NUM, "1"); + sub1.put(Columns.ROW, "1"); + sub1.put(Columns.COLUMN, "1"); + sub1.put("Plant " + OBSERVATION_UNIT_ID_SUFFIX, sub1ObsUnitId); + sub1.put(traits.get(0).getObservationVariableName(), "1"); + + Map sub2 = new HashMap<>(); + sub2.put(Columns.GERMPLASM_GID, "1"); + sub2.put(Columns.TEST_CHECK, "T"); + sub2.put(Columns.EXP_TITLE, "Test Exp"); + sub2.put(Columns.EXP_UNIT, "Plot"); + sub2.put(Columns.EXP_TYPE, "Phenotyping"); + sub2.put(Columns.ENV, "New Env"); + sub2.put(Columns.ENV_LOCATION, "Location A"); + sub2.put(Columns.ENV_YEAR, "2023"); + sub2.put(Columns.EXP_UNIT_ID, "a-1"); + sub2.put(Columns.REP_NUM, "1"); + sub2.put(Columns.BLOCK_NUM, "1"); + sub2.put(Columns.ROW, "1"); + sub2.put(Columns.COLUMN, "1"); + sub2.put("Plant " + OBSERVATION_UNIT_ID_SUFFIX, sub2ObsUnitId); + sub2.put(traits.get(0).getObservationVariableName(), "2"); + + // Verify that the validation check returns a 400-level response since tt_test_1 is already used in the plot-level dataset + JsonObject previewResponse = importTestUtils.uploadAndFetchWorkflowPreview(importTestUtils.writeExperimentDataToFile(List.of(sub1, sub2), traits, true, true, "Plant"), null, true, client, program, mappingId, appendOverwriteWorkflowId); + assertEquals(400, previewResponse.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + } + + @Test + @SneakyThrows + public void appendExperimentMultipleDatasets() { + log.debug("appendExperimentMultipleDatasets"); + + // Create a plot-level dataset + List traits = importTestUtils.createTraits(1); + Program program = createProgram("Append Exp with Multiple Datasets", "MULSET", "MULSET", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + + JsonObject importResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); + String expId = importResponse + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + + // Create two sub-entity datasets that have two different sub entity units + Flowable> sub1PostCall = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", + program.getId().toString(), expId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class); + HttpResponse sub1PostResponse = sub1PostCall.blockingFirst(); + Flowable> sub2PostCall = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", + program.getId().toString(), expId), + "{\"name\":\"Tree\",\"repeatedMeasures\":2}") + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class); + HttpResponse sub2PostResponse = sub2PostCall.blockingFirst(); + + // Assert 200 response + assertEquals(HttpStatus.OK, sub1PostResponse.getStatus()); + assertEquals(HttpStatus.OK, sub2PostResponse.getStatus()); + JsonObject sub1Result = JsonParser.parseString(sub1PostResponse.body()) + .getAsJsonObject() + .getAsJsonObject("result"); + JsonObject sub2Result = JsonParser.parseString(sub2PostResponse.body()) + .getAsJsonObject() + .getAsJsonObject("result"); + + // Export the plant-level sub-entity datasets + String extension = "CSV"; + + // Get the dataset ids from the experiment additional info + BrAPITrial experiment = experimentService.getTrialDataByUUID(program.getId(), UUID.fromString(expId), false); + JsonObject additionalInfo = experiment.getAdditionalInfo(); + JsonArray datasetsJsonArray = additionalInfo.getAsJsonArray("datasets"); + List subEntityDatasetIds = new ArrayList<>(); + for (JsonElement elem : datasetsJsonArray) { + JsonObject datasetObj = elem.getAsJsonObject(); + int level = datasetObj.get("level").getAsInt(); + if (level == 1) { + String id = datasetObj.get("id").getAsString(); + subEntityDatasetIds.add(id); + } + } + + String plant1DatasetId = subEntityDatasetIds.get(0); + Flowable> plantExportCall = client.exchange( + GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", + program.getId().toString(), expId, extension, plant1DatasetId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse plantResponse = plantExportCall.blockingFirst(); + + String plant2DatasetId = subEntityDatasetIds.get(1); + Flowable> treeExportCall = client.exchange( + GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", + program.getId().toString(), expId, extension, plant2DatasetId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse treeResponse = treeExportCall.blockingFirst(); + + // Parse the export tables + ByteArrayInputStream bodyStream1 = new ByteArrayInputStream(Objects.requireNonNull(plantResponse.body())); + Table exportTable1 = FileUtil.parseTableFromCsv(bodyStream1); + ByteArrayInputStream bodyStream2 = new ByteArrayInputStream(Objects.requireNonNull(treeResponse.body())); + Table exportTable2 = FileUtil.parseTableFromCsv(bodyStream2); + + // Build a request to append tt_test_1 observation data on observation units from two separate datasets + String sub1ObsUnitId = exportTable1.row(0).getString("Plant ObsUnitID"); + String sub2ObsUnitId = exportTable2.row(0).getString("Tree ObsUnitID"); + + Map sub1 = new HashMap<>(); + sub1.put(Columns.GERMPLASM_GID, "1"); + sub1.put(Columns.TEST_CHECK, "T"); + sub1.put(Columns.EXP_TITLE, "Test Exp"); + sub1.put(Columns.EXP_UNIT, "Plot"); + sub1.put(Columns.EXP_TYPE, "Phenotyping"); + sub1.put(Columns.ENV, "New Env"); + sub1.put(Columns.ENV_LOCATION, "Location A"); + sub1.put(Columns.ENV_YEAR, "2023"); + sub1.put(Columns.EXP_UNIT_ID, "a-1"); + sub1.put(Columns.REP_NUM, "1"); + sub1.put(Columns.BLOCK_NUM, "1"); + sub1.put(Columns.ROW, "1"); + sub1.put(Columns.COLUMN, "1"); + sub1.put("Plant " + OBSERVATION_UNIT_ID_SUFFIX, sub1ObsUnitId); + sub1.put(traits.get(0).getObservationVariableName(), "1"); + + Map sub2 = new HashMap<>(); + sub2.put(Columns.GERMPLASM_GID, "1"); + sub2.put(Columns.TEST_CHECK, "T"); + sub2.put(Columns.EXP_TITLE, "Test Exp"); + sub2.put(Columns.EXP_UNIT, "Plot"); + sub2.put(Columns.EXP_TYPE, "Phenotyping"); + sub2.put(Columns.ENV, "New Env"); + sub2.put(Columns.ENV_LOCATION, "Location A"); + sub2.put(Columns.ENV_YEAR, "2023"); + sub2.put(Columns.EXP_UNIT_ID, "a-1"); + sub2.put(Columns.REP_NUM, "1"); + sub2.put(Columns.BLOCK_NUM, "1"); + sub2.put(Columns.ROW, "1"); + sub2.put(Columns.COLUMN, "1"); + sub2.put("Plant " + OBSERVATION_UNIT_ID_SUFFIX, sub2ObsUnitId); + sub2.put(traits.get(0).getObservationVariableName(), "2"); + + // Verify that the validation check returns a 400-level response since obervation units belonging to different datasets are included + JsonObject previewResponse = importTestUtils.uploadAndFetchWorkflowPreview(importTestUtils.writeExperimentDataToFile(List.of(sub1, sub2), null, true, false, "Plant"), null, true, client, program, mappingId, appendOverwriteWorkflowId); + assertEquals(400, previewResponse.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + sub1Result + sub2Result); + } + @Test @SneakyThrows public void importNewExpNewLocNoObsSuccess() { @@ -201,7 +464,7 @@ public void importNewExpNewLocNoObsSuccess() { validRow.put(Columns.COLUMN, "1"); validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(validRow), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -253,7 +516,7 @@ public void importNewExpMultiNewEnvSuccess() { secondEnv.put(Columns.COLUMN, "1"); secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(2, previewRows.size()); @@ -294,7 +557,7 @@ public void importExistingExpAndEnvErrorMessage() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); Map dupExp = new HashMap<>(); dupExp.put(Columns.GERMPLASM_GID, "1"); @@ -311,7 +574,7 @@ public void importExistingExpAndEnvErrorMessage() { dupExp.put(Columns.ROW, "1"); dupExp.put(Columns.COLUMN, "1"); - JsonObject expResult = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject expResult = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); assertEquals(422, expResult.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + expResult); assertTrue(expResult.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); @@ -338,7 +601,7 @@ public void importNewEnvNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); - JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -374,52 +637,52 @@ public void verifyMissingDataThrowsError(boolean commit) { Map noGID = new HashMap<>(base); noGID.remove(Columns.GERMPLASM_GID); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null, false, false, null), Columns.GERMPLASM_GID, commit, newExperimentWorkflowId); Map noExpTitle = new HashMap<>(base); noExpTitle.remove(Columns.EXP_TITLE); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null, false, false, null), Columns.EXP_TITLE, commit, newExperimentWorkflowId); Map noExpUnit = new HashMap<>(base); noExpUnit.remove(Columns.EXP_UNIT); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null, false, false, null), Columns.EXP_UNIT, commit, newExperimentWorkflowId); Map noExpType = new HashMap<>(base); noExpType.remove(Columns.EXP_TYPE); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null, false, false, null), Columns.EXP_TYPE, commit, newExperimentWorkflowId); Map noEnv = new HashMap<>(base); noEnv.remove(Columns.ENV); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null, false, false, null), Columns.ENV, commit, newExperimentWorkflowId); Map noEnvLoc = new HashMap<>(base); noEnvLoc.remove(Columns.ENV_LOCATION); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null, false, false, null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); Map noExpUnitId = new HashMap<>(base); noExpUnitId.remove(Columns.EXP_UNIT_ID); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null, false, false, null), Columns.EXP_UNIT_ID, commit, newExperimentWorkflowId); Map noExpRep = new HashMap<>(base); noExpRep.remove(Columns.REP_NUM); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null, false, false, null), Columns.REP_NUM, commit, newExperimentWorkflowId); Map noExpBlock = new HashMap<>(base); noExpBlock.remove(Columns.BLOCK_NUM); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null, false, false, null), Columns.BLOCK_NUM, commit, newExperimentWorkflowId); Map noEnvYear = new HashMap<>(base); noEnvYear.remove(Columns.ENV_YEAR); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null, false, false, null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); } @Test @@ -444,7 +707,7 @@ public void importNewExpWithObsVar() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -496,7 +759,7 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null, false, false, null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); } @@ -536,7 +799,7 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null, false, false, null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); } @ParameterizedTest @@ -562,7 +825,7 @@ public void importNewExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -603,7 +866,7 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "Red"); - uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); } @@ -628,14 +891,14 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); Map newOU = new HashMap<>(newExp); newOU.put(Columns.EXP_UNIT_ID, "a-2"); newOU.put(Columns.ROW, "1"); newOU.put(Columns.COLUMN, "2"); - JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(newOU), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); @@ -663,7 +926,7 @@ public void importNewObsVarExistingOu() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -688,10 +951,10 @@ public void importNewObsVarExistingOu() { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), null, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -729,7 +992,7 @@ public void importNewObsVarByObsUnitId() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -741,10 +1004,10 @@ public void importNewObsVarByObsUnitId() { assertTrue(ouIdXref.isPresent()); Map newObsVar = new HashMap<>(); - newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), null, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -783,7 +1046,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); // empty dataset - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -808,10 +1071,10 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -859,7 +1122,7 @@ public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); // Valid observation value. - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); // Fetch the ObsUnitId to use in the overwrite upload. BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); @@ -886,14 +1149,14 @@ public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); // Indicates this is an overwrite. + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); // Indicates this is an overwrite. newObsVar.put(traits.get(0).getObservationVariableName(), ""); // Empty string should be no op. Map requestBody = new HashMap<>(); requestBody.put("overwrite", "true"); requestBody.put("overwriteReason", "testing"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), requestBody, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), requestBody, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -929,7 +1192,7 @@ public void importNewObsExistingOu(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -954,10 +1217,10 @@ public void importNewObsExistingOu(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -997,7 +1260,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1022,10 +1285,10 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); - uploadAndVerifyWorkflowFailureNonTabular(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + uploadAndVerifyWorkflowFailureNonTabular(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); } /* @@ -1056,7 +1319,7 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsA.size()); @@ -1084,7 +1347,7 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsB.size()); @@ -1126,7 +1389,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1151,11 +1414,11 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), null, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1203,7 +1466,7 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { // In the first upload, only 1 trait should be present. List initialTraits = List.of(traits.get(0)); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), initialTraits), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), initialTraits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1228,13 +1491,13 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "1"); // Send overwrite parameters in request body to allow the append workflow to work normally. Map userData = Map.of("overwrite", "true", "overwriteReason", "testing"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), userData, true, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), userData, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1279,7 +1542,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newExp.put(traits.get(0).getObservationVariableName(), originalValue); newExp.put(traits.get(1).getObservationVariableName(), "2"); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1306,7 +1569,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), ""); // This blank value should not overwrite. newObservation.put(traits.get(1).getObservationVariableName(), "3"); // This valid value should overwrite. newObservation.put(traits.get(2).getObservationVariableName(), "4"); // This valid new observation should be appended. @@ -1315,7 +1578,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { Map requestBody = new HashMap<>(); requestBody.put("overwrite", "true"); requestBody.put("overwriteReason", "testing"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), requestBody, commit, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), requestBody, commit, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1340,7 +1603,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { private Map assertRowReferencedByOUIdSaved(Map expected, Program program, List traits) throws ApiException { Map ret = new HashMap<>(); - List units = ouDAO.getObservationUnitsById(List.of((String)expected.get(Columns.OBS_UNIT_ID)), program); + List units = ouDAO.getObservationUnitsById(List.of((String)expected.get("Plot "+OBSERVATION_UNIT_ID_SUFFIX)), program); assertFalse(units.isEmpty()); List observations = null; @@ -1438,7 +1701,7 @@ private Map assertRowSaved(Map expected, Program assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_DESCRIPTION), trial.getTrialDescription()); assertEquals(expected.get(Columns.EXP_UNIT), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString()); - assertEquals(expected.get(Columns.EXP_UNIT), ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + assertEquals(expected.get(Columns.EXP_UNIT).toString().toLowerCase(), ou.getObservationUnitPosition().getObservationLevel().getLevelName().toLowerCase()); assertEquals(expected.get(Columns.EXP_TYPE), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); assertEquals(expected.get(Columns.EXP_TYPE), study.getStudyType()); assertEquals(expected.get(Columns.ENV), Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); @@ -1545,7 +1808,7 @@ private Map assertValidPreviewRow(Map expected, assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_DESCRIPTION), trial.getTrialDescription()); assertEquals(expected.get(Columns.EXP_UNIT), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString()); - assertEquals(expected.get(Columns.EXP_UNIT), ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + assertEquals(expected.get(Columns.EXP_UNIT).toString().toLowerCase(), ou.getObservationUnitPosition().getObservationLevel().getLevelName().toLowerCase()); assertEquals(expected.get(Columns.EXP_TYPE), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); assertEquals(expected.get(Columns.EXP_TYPE), study.getStudyType()); assertEquals(expected.get(Columns.ENV), Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index 13d60a714..ac2860441 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -542,6 +542,35 @@ public void badBreedingMethods() { assertEquals("Breeding Method", firstError.get("field").getAsString()); } + /** + * Test for BI-2185 adding validation for germplasm names to not allow / characters + */ + @Test + @SneakyThrows + public void badGermplasmNames() { + File file = new File("src/test/resources/files/germplasm_import/bad_germplasm_names.csv"); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad Germplasm Names"), true, client, validProgram, germplasmMappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); + + JsonArray errorList = result + .getAsJsonObject("progress") + .getAsJsonArray("rowErrors"); + + assertEquals(3, errorList.size()); + + JsonObject firstError = errorList + .get(0).getAsJsonObject() + .getAsJsonArray("errors").get(0).getAsJsonObject(); + assertEquals("Germplasm name cannot contain /", firstError.get("errorMessage").getAsString()); + assertEquals("Germplasm Name", firstError.get("field").getAsString()); + } + @Test @SneakyThrows public void someEntryNumbersError() { diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 53abd8483..531875902 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -52,6 +52,7 @@ import static io.micronaut.http.HttpRequest.*; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; /** * Intended to be a utility class, but methods being static was causing issues. @@ -180,7 +181,7 @@ public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, .getAsJsonArray("data") .get(0).getAsJsonObject().get("id").getAsString(); - BiUserEntity testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + BiUserEntity testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); @@ -239,16 +240,34 @@ public JsonObject uploadAndFetchWorkflow(File file, Program program, String mappingId, String workflowId) throws InterruptedException { + JsonObject result = generatePreview(file,userData, commit, client, program, mappingId, workflowId); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + return result; + } + + public JsonObject uploadAndFetchWorkflowPreview(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + return generatePreview(file, userData, commit, client, program, mappingId, workflowId); + } + + private JsonObject generatePreview(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - return result; + + return JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); } public JsonObject uploadAndFetchWorkflowNoStatusCheck(File file, @@ -296,7 +315,8 @@ public List createTraits(int numToCreate) { return traits; } - public File writeExperimentDataToFile(List> data, List traits) throws IOException { + public File writeExperimentDataToFile(List> data, List traits, boolean ObsUnitIDCol, boolean isSubEntity, String level) throws IOException { + String obsLevel = level == null ? "Plot" : level; File file = File.createTempFile("test", ".csv"); List columns = new ArrayList<>(); @@ -306,13 +326,17 @@ public File writeExperimentDataToFile(List> data, List> data, List { diff --git a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java index 71bf6197c..f19803127 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java @@ -568,7 +568,7 @@ private String createExperiment(Program program) throws IOException, Interrupted .get(0).getAsJsonObject().get("id").getAsString(); JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( - importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null), + importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null, false, false, null), null, true, client, diff --git a/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java b/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java index c4c214682..bd2048b61 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java @@ -36,6 +36,7 @@ import static org.breedinginsight.TestUtils.insertAndFetchTestProgram; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -92,7 +93,7 @@ public void setup() { // Insert system admin role so can create program FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); SpeciesRequest speciesRequest = SpeciesRequest.builder() @@ -119,12 +120,14 @@ public void setup() { @SneakyThrows @Order(1) public void testCreateObservationUnitAdditionalInfoSingleTreatmentFactor() { - // create observation unit with treatments only in additional info to simulate breedbase not populating // treatments field BrAPIObservationUnit ou1 = new BrAPIObservationUnit(); ou1.setObservationUnitName("test1"); - ou1.putAdditionalInfoItem(BrAPIAdditionalInfoFields.TREATMENTS, List.of(testTreatment)); ou1.setProgramDbId(validProgram.getBrapiProgram().getProgramDbId()); + + var treatment = new BrAPIObservationTreatment(); + treatment.setFactor("ou1 treatment"); + ou1.setTreatments(List.of(treatment)); // Set xref. BrAPIExternalReference xref = new BrAPIExternalReference(); xref.setReferenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)); @@ -153,6 +156,9 @@ private void singleTreatmentAsserts(List obsUnits, BrAPIOb BrAPIObservationTreatment treatment = treatments.get(0); assertEquals(expectedTreatment, treatment, "Expected treatments to be same"); + + // Storing treatments in additionalInfo is no longer necessary since BrAPI server stores treatments in observation_unit_treatment + assertNull(ou.getAdditionalInfo(), "Expected OU additionalInfo to be null"); } } diff --git a/src/test/java/org/breedinginsight/daos/DSLTransactionResultIntegrationTest.java b/src/test/java/org/breedinginsight/daos/DSLTransactionResultIntegrationTest.java index a298361fa..e0cd9bb31 100644 --- a/src/test/java/org/breedinginsight/daos/DSLTransactionResultIntegrationTest.java +++ b/src/test/java/org/breedinginsight/daos/DSLTransactionResultIntegrationTest.java @@ -28,7 +28,6 @@ import org.jooq.DSLContext; import org.jooq.exception.DataAccessException; import org.jooq.impl.DSL; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -60,7 +59,7 @@ public class DSLTransactionResultIntegrationTest extends DatabaseTest { @BeforeAll void setup() throws Exception { - Optional userOptional = userService.getByOrcid(TestTokenValidator.TEST_USER_ORCID); + Optional userOptional = userService.getByOAuthId(TestTokenValidator.TEST_USER_ORCID); actingUser = userOptional.get(); } diff --git a/src/test/java/org/breedinginsight/services/lock/DistributedLockServiceTest.java b/src/test/java/org/breedinginsight/services/lock/DistributedLockServiceTest.java new file mode 100644 index 000000000..bb3275beb --- /dev/null +++ b/src/test/java/org/breedinginsight/services/lock/DistributedLockServiceTest.java @@ -0,0 +1,59 @@ +package org.breedinginsight.services.lock; + +import org.breedinginsight.DatabaseTest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class DistributedLockServiceTest extends DatabaseTest { + + private final ExecutorService executor = Executors.newFixedThreadPool(2); + + @AfterEach + void cleanup() { + executor.shutdownNow(); + } + + @Test + void secondLockAttemptTimesOutWhileFirstHolds() throws Exception { + DistributedLockService lockService = new DistributedLockService(super.getRedisConnection()); + String lockKey = "test-lock-key"; + + CountDownLatch firstAcquired = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + + Future firstCall = executor.submit(() -> + lockService.withLock(lockKey, Duration.ofMillis(500), Duration.ofSeconds(5), () -> { + firstAcquired.countDown(); + // keep the lock held until signaled + releaseFirst.await(2, TimeUnit.SECONDS); + return "first"; + }) + ); + + assertTrue(firstAcquired.await(1, TimeUnit.SECONDS), "First lock holder did not start in time"); + + assertThrows(TimeoutException.class, () -> + lockService.withLock(lockKey, Duration.ofMillis(100), Duration.ofSeconds(2), () -> "second") + ); + + releaseFirst.countDown(); + assertEquals("first", firstCall.get(2, TimeUnit.SECONDS)); + + String afterRelease = lockService.withLock(lockKey, Duration.ofMillis(500), Duration.ofSeconds(2), () -> "after-release"); + assertEquals("after-release", afterRelease); + } +} diff --git a/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java b/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java index 3006bc7b5..b31839ff2 100644 --- a/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java +++ b/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java @@ -86,7 +86,7 @@ public void setup() throws Exception { FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); // Test User - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Species diff --git a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java index 7cd16622a..56f977a47 100644 --- a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java +++ b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java @@ -93,7 +93,7 @@ public void setup() throws MissingRequiredInfoException, UnprocessableEntityExce var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); // Insert system roles - User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + User testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); // Insert program diff --git a/src/test/java/org/breedinginsight/utilities/response/mappers/UserQueryMapperUnitTest.java b/src/test/java/org/breedinginsight/utilities/response/mappers/UserQueryMapperUnitTest.java index 8e5be117d..50545644b 100644 --- a/src/test/java/org/breedinginsight/utilities/response/mappers/UserQueryMapperUnitTest.java +++ b/src/test/java/org/breedinginsight/utilities/response/mappers/UserQueryMapperUnitTest.java @@ -48,7 +48,7 @@ public void testMappings() { User user = User.builder() .name("Test User") .email("test@user.com") - .orcid("000000-000000-000000-00000") + .oauthId("000000-000000-000000-00000") .systemRoles(List.of(SystemRole.builder().domain("System Administrator").build())) .programRoles(List.of(ProgramUser.builder().program(Program.builder().name("Test program").build()).build())) .active(false) @@ -60,7 +60,7 @@ public void testMappings() { assertEquals(user.getName(), userQueryMapper.getField("name").apply(user), "Wrong getter"); assertEquals(user.getEmail(), userQueryMapper.getField("email").apply(user), "Wrong getter"); - assertEquals(user.getOrcid(), userQueryMapper.getField("orcid").apply(user), "Wrong getter"); + assertEquals(user.getOauthId(), userQueryMapper.getField("oauthId").apply(user), "Wrong getter"); assertEquals(user.getSystemRoles().stream().map(role -> role.getDomain()).collect(Collectors.toList()), userQueryMapper.getField("systemRoles").apply(user), "Wrong getter"); assertEquals(user.getProgramRoles().stream().map(role -> role.getProgram().getName()).collect(Collectors.toList()), diff --git a/src/test/resources/files/germplasm_import/bad_germplasm_names.csv b/src/test/resources/files/germplasm_import/bad_germplasm_names.csv new file mode 100644 index 000000000..22f177757 --- /dev/null +++ b/src/test/resources/files/germplasm_import/bad_germplasm_names.csv @@ -0,0 +1,4 @@ +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +,Germplasm 1/Germplasm 2,BCR,Test,,,1,,,, +,Germplasm3 / Germaplasm4,BCR,Test,,,2,,,, +,Germplasm5/Germplasm6/Germplasm7,BCR,Test,,,3,1,2,, \ No newline at end of file diff --git a/src/test/resources/sql/ExperimentalCollaboratorServiceTest.sql b/src/test/resources/sql/ExperimentalCollaboratorServiceTest.sql index b1b1a26c9..10f361cf4 100644 --- a/src/test/resources/sql/ExperimentalCollaboratorServiceTest.sql +++ b/src/test/resources/sql/ExperimentalCollaboratorServiceTest.sql @@ -17,8 +17,8 @@ */ -- name: CreateUser -INSERT INTO bi_user (id, orcid, name, email, created_by, updated_by, active) -VALUES ('594ec70e-0476-4c40-baf5-581ab0cfcd75', '0000-0001-2345-6789', 'Tester', 'tester@mailinator.com', '00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000000', true); +INSERT INTO bi_user (id, oauth_id, name, email, created_by, updated_by, active, oauth_provider) +VALUES ('594ec70e-0476-4c40-baf5-581ab0cfcd75', '0000-0001-2345-6789', 'Tester', 'tester@mailinator.com', '00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000000', true, 'orcid'); -- name: CreateProgram INSERT INTO program (id, species_id, name, created_by, updated_by, active, key, germplasm_sequence, exp_sequence, env_sequence) diff --git a/src/test/resources/sql/ImportControllerIntegrationTest.sql b/src/test/resources/sql/ImportControllerIntegrationTest.sql index 7bab169b8..34afd6e2b 100644 --- a/src/test/resources/sql/ImportControllerIntegrationTest.sql +++ b/src/test/resources/sql/ImportControllerIntegrationTest.sql @@ -33,7 +33,7 @@ INSERT INTO public.importer_mapping (name,import_type_id,mapping,file,draft,crea INSERT INTO public.importer_mapping (name,import_type_id,mapping,file,draft,created_by,updated_by) values ('ExperimentsTemplateMap','ExperimentImport', '[{"id": "726a9f10-4892-4204-9e52-bd2b1d735f65", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "98774e20-6f89-4d6a-a7c9-f88887228ed6", "value": {"fileFieldName": "Germplasm GID"}, "objectId": "gid"}, {"id": "880ef0c9-4e3e-42d4-9edc-667684a91889", "value": {"fileFieldName": "Test (T) or Check (C)"}, "objectId": "test_or_check"}, {"id": "b693eca7-efcd-4518-a9d3-db0b037a76ee", "value": {"fileFieldName": "Exp Title"}, "objectId": "exp_title"}, {"id": "df340215-db6e-4219-a3b7-119f297b81c3", "value": {"fileFieldName": "Exp Description"}, "objectId": "expDescription"}, {"id": "9ca7cc81-562c-43a7-989a-41da309f603d", "value": {"fileFieldName": "Exp Unit"}, "objectId": "expUnit"}, {"id": "27215777-c8f9-4fe7-a7ac-918d6168b0dd", "value": {"fileFieldName": "Exp Type"}, "objectId": "expType"}, {"id": "19d220e2-dff0-4a3a-ad6e-32f4d8602b5c", "value": {"fileFieldName": "Env"}, "objectId": "env"}, {"id": "861518b9-5c9e-4fe5-b31e-baf16e27155d", "value": {"fileFieldName": "Env Location"}, "objectId": "envLocation"}, {"id": "667355c3-dae1-4a64-94c8-ac2d543bd474", "value": {"fileFieldName": "Env Year"}, "objectId": "envYear"}, {"id": "ad11f2df-c5b4-4a05-8e52-c57625140061", "value": {"fileFieldName": "Exp Unit ID"}, "objectId": "expUnitId"}, {"id": "639b40ec-20f8-4659-8464-6a4be997ac7a", "value": {"fileFieldName": "Exp Replicate #"}, "objectId": "expReplicateNo"}, {"id": "2a62a80f-d8ba-42c4-9997-3b4ac8a965aa", "value": {"fileFieldName": "Exp Block #"}, "objectId": "expBlockNo"}, {"id": "f3e7de69-21ad-4cda-b1cc-a5e1987fb931", "value": {"fileFieldName": "Row"}, "objectId": "row"}, {"id": "251c5bbd-fc4d-4371-a4ce-4e2686f6837e", "value": {"fileFieldName": "Column"}, "objectId": "column"}, {"id": "ce5f61f2-f1de-45a4-8baf-e2471a5d863d", "value": {"fileFieldName": "Treatment Factors"}, "objectId": "treatmentFactors"}, {"id": "c5b8276f-e777-4385-a80f-5199abe63aac", "value": {"fileFieldName": "ObsUnitID"}, "objectId": "ObsUnitID"}]', - '[{"Env": "New Study", "Row": 4, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "plot", "Exp Title": "New Trial", "ObsUnitID": "", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Treatment Factors": "Jam application", "Test (T) or Check (C)": true}]', + '[{"Env": "New Study", "Row": 4, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "Plot", "Exp Title": "New Trial", "ObsUnitID": "", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Treatment Factors": "Jam application", "Test (T) or Check (C)": true}]', false,user_id,user_id); END $$; diff --git a/src/test/resources/sql/JobControllerIntegrationTest.sql b/src/test/resources/sql/JobControllerIntegrationTest.sql index 596c44bda..d238741eb 100644 --- a/src/test/resources/sql/JobControllerIntegrationTest.sql +++ b/src/test/resources/sql/JobControllerIntegrationTest.sql @@ -18,12 +18,12 @@ -- name: InsertJobs -INSERT INTO public.importer_progress (id, statuscode, message, body, total, finished, in_progress, created_at, updated_at, created_by, updated_by) VALUES ('d02090c3-e40d-45f4-9a63-b8c701c9531b', 202, 'Uploading to brapi service', null, 500, 100, 400, '2022-05-05 21:42:42 +00:00', '2022-05-05 21:42:42 +00:00', (select id from bi_user where orcid = '1111-2222-3333-4444'), (select id from bi_user where orcid = '1111-2222-3333-4444')); -INSERT INTO public.importer_progress (id, statuscode, message, body, total, finished, in_progress, created_at, updated_at, created_by, updated_by) VALUES ('066a37e4-c2e7-4d04-9175-7164c7dce70f', 200, 'Completed upload to brapi service', null, 17, 1, 0, '2022-05-17 14:19:33 +00:00', '2022-05-17 14:19:33 +00:00', (select id from bi_user where orcid = '1111-2222-3333-4444'), (select id from bi_user where orcid = '1111-2222-3333-4444')); +INSERT INTO public.importer_progress (id, statuscode, message, body, total, finished, in_progress, created_at, updated_at, created_by, updated_by) VALUES ('d02090c3-e40d-45f4-9a63-b8c701c9531b', 202, 'Uploading to brapi service', null, 500, 100, 400, '2022-05-05 21:42:42 +00:00', '2022-05-05 21:42:42 +00:00', (select id from bi_user where oauth_id = '1111-2222-3333-4444'), (select id from bi_user where oauth_id = '1111-2222-3333-4444')); +INSERT INTO public.importer_progress (id, statuscode, message, body, total, finished, in_progress, created_at, updated_at, created_by, updated_by) VALUES ('066a37e4-c2e7-4d04-9175-7164c7dce70f', 200, 'Completed upload to brapi service', null, 17, 1, 0, '2022-05-17 14:19:33 +00:00', '2022-05-17 14:19:33 +00:00', (select id from bi_user where oauth_id = '1111-2222-3333-4444'), (select id from bi_user where oauth_id = '1111-2222-3333-4444')); -INSERT INTO public.importer_import (id, program_id, user_id, importer_mapping_id, importer_progress_id, upload_file_name, file_data, modified_data, mapped_data, created_at, updated_at, created_by, updated_by) VALUES ('9aeeace2-7370-4b7d-a574-0ea799a342ef', ?::uuid, (select id from bi_user where orcid = '1111-2222-3333-4444'), (select id from importer_mapping where name = 'GermplasmTemplateMap'), '066a37e4-c2e7-4d04-9175-7164c7dce70f', 'germ import 2.xlsx', '[{"Name": "Gnome-1", "Source": "USDA", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-2", "Source": "USDA", "Entry No": "2", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3", "Source": "Breeding Insight", "Entry No": "3", "External UID": "", "Breeding Method": "BPC", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "2", "Female Parent Entry No": "1"}, {"Name": "Gnome-3-1", "Source": "Breeding Insight", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-2", "Source": "Breeding Insight", "Entry No": "5", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-3", "Source": "Breeding Insight", "Entry No": "6", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-4", "Source": "Breeding Insight", "Entry No": "7", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-5", "Source": "Breeding Insight", "Entry No": "8", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-6", "Source": "Breeding Insight", "Entry No": "9", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-7", "Source": "Breeding Insight", "Entry No": "10", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "11", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "9"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "12", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "3"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "13", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "7"}]', null, '{"rows": [{"germplasm": {"id": "1580b261-f504-4a7e-bbc7-e5599f0b2f4b", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-1 [MNTTIM-1]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "1"}, "commonCropName": "test", "accessionNumber": "1", "defaultDisplayName": "Gnome-1", "externalReferences": [{"referenceID": "6a1269cb-6e7a-4b87-a353-c681c214bd69", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "7e665eb0-2f7b-4b14-85a4-50f5508de2a8", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-2 [MNTTIM-2]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "2"}, "commonCropName": "test", "accessionNumber": "2", "defaultDisplayName": "Gnome-2", "externalReferences": [{"referenceID": "bcc3fab6-04ed-4e2f-b0b0-9e9b247462ef", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "eddb03ff-09c5-481d-9c1c-8fc609d432e5", "state": "NEW", "brAPIObject": {"pedigree": "Gnome-1 [MNTTIM-1]/Gnome-2 [MNTTIM-2]", "seedSource": "Breeding Insight", "germplasmName": "Gnome-3 [MNTTIM-3]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Biparental cross", "breedingMethodId": "aee4d3a3-0df8-422f-b052-d9f0b81e5b4d", "importEntryNumber": "3", "maleParentEntryNo": "2", "femaleParentEntryNo": "1"}, "commonCropName": "test", "accessionNumber": "3", "defaultDisplayName": "Gnome-3", "externalReferences": [{"referenceID": "24c3164e-0b37-44b4-8b18-9118ca9b2949", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "42ff9b3e-3f04-42f3-909b-2aaa158457e5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-1 [MNTTIM-4]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "4"}, "commonCropName": "test", "accessionNumber": "4", "defaultDisplayName": "Gnome-3-1", "externalReferences": [{"referenceID": "c271722b-488b-41c8-9671-73ac008be522", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "883cef8e-34c9-43f7-8062-d7d80c8a65fd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-2 [MNTTIM-5]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "5"}, "commonCropName": "test", "accessionNumber": "5", "defaultDisplayName": "Gnome-3-2", "externalReferences": [{"referenceID": "6bb014ce-0c5f-480c-a1e2-9b87a914a733", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "6215c013-2ed9-43c1-a964-15f87024da36", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-3 [MNTTIM-6]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "6"}, "commonCropName": "test", "accessionNumber": "6", "defaultDisplayName": "Gnome-3-3", "externalReferences": [{"referenceID": "988fe16a-6308-4146-9cae-be9d0cd9c295", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "bd62dec0-25d3-4bef-b9f7-5af634f78366", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-4 [MNTTIM-7]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "7"}, "commonCropName": "test", "accessionNumber": "7", "defaultDisplayName": "Gnome-3-4", "externalReferences": [{"referenceID": "28e6482a-2804-4aa8-b98d-6c4a56f8d8d2", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "c0067078-7a6d-4c99-b6ee-ab220feccddd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-5 [MNTTIM-8]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "8"}, "commonCropName": "test", "accessionNumber": "8", "defaultDisplayName": "Gnome-3-5", "externalReferences": [{"referenceID": "e7c25b55-61f5-4845-9080-d2243171f70b", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "cdb38aac-51fc-4ea7-9b38-8072258ceac5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-6 [MNTTIM-9]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "9"}, "commonCropName": "test", "accessionNumber": "9", "defaultDisplayName": "Gnome-3-6", "externalReferences": [{"referenceID": "ade6b8ca-b2e0-49cc-ac09-09476b1416e7", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "3f6c7d4b-fca2-4b22-aa6a-aadb347cd625", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-7 [MNTTIM-10]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "10"}, "commonCropName": "test", "accessionNumber": "10", "defaultDisplayName": "Gnome-3-7", "externalReferences": [{"referenceID": "6cd34ea3-86b0-41a2-aef0-57c101bd68fb", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "68a72f81-42b5-4425-8034-eef7d6921928", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-6 [MNTTIM-9]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-11]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "11", "maleParentEntryNo": "1", "femaleParentEntryNo": "9"}, "commonCropName": "test", "accessionNumber": "11", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "354faf95-682c-4a62-90f4-4f6c9c2d4191", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "25d84084-45a6-4d93-a683-004f95d3dc3e", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3 [MNTTIM-3]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-12]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "12", "maleParentEntryNo": "1", "femaleParentEntryNo": "3"}, "commonCropName": "test", "accessionNumber": "12", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "936a608f-0e2b-4cee-b348-dbbd57cbdf65", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "569fd3a0-1960-45e0-86d4-5157e21e8130", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-4 [MNTTIM-7]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-13]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "13", "maleParentEntryNo": "1", "femaleParentEntryNo": "7"}, "commonCropName": "test", "accessionNumber": "13", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "4e708554-e088-48ce-a16d-43f1778549fa", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}], "statistics": {"Germplasm": {"newObjectCount": 13}, "Pedigree Connections": {"newObjectCount": 4}}}', '2022-05-17 14:19:33 +00:00', '2022-05-17 14:19:33 +00:00', (select id from bi_user where orcid = '1111-2222-3333-4444'), (select id from bi_user where orcid = '1111-2222-3333-4444')); +INSERT INTO public.importer_import (id, program_id, user_id, importer_mapping_id, importer_progress_id, upload_file_name, file_data, modified_data, mapped_data, created_at, updated_at, created_by, updated_by) VALUES ('9aeeace2-7370-4b7d-a574-0ea799a342ef', ?::uuid, (select id from bi_user where oauth_id = '1111-2222-3333-4444'), (select id from importer_mapping where name = 'GermplasmTemplateMap'), '066a37e4-c2e7-4d04-9175-7164c7dce70f', 'germ import 2.xlsx', '[{"Name": "Gnome-1", "Source": "USDA", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-2", "Source": "USDA", "Entry No": "2", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3", "Source": "Breeding Insight", "Entry No": "3", "External UID": "", "Breeding Method": "BPC", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "2", "Female Parent Entry No": "1"}, {"Name": "Gnome-3-1", "Source": "Breeding Insight", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-2", "Source": "Breeding Insight", "Entry No": "5", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-3", "Source": "Breeding Insight", "Entry No": "6", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-4", "Source": "Breeding Insight", "Entry No": "7", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-5", "Source": "Breeding Insight", "Entry No": "8", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-6", "Source": "Breeding Insight", "Entry No": "9", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-7", "Source": "Breeding Insight", "Entry No": "10", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "11", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "9"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "12", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "3"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "13", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "7"}]', null, '{"rows": [{"germplasm": {"id": "1580b261-f504-4a7e-bbc7-e5599f0b2f4b", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-1 [MNTTIM-1]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "1"}, "commonCropName": "test", "accessionNumber": "1", "defaultDisplayName": "Gnome-1", "externalReferences": [{"referenceID": "6a1269cb-6e7a-4b87-a353-c681c214bd69", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "7e665eb0-2f7b-4b14-85a4-50f5508de2a8", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-2 [MNTTIM-2]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "2"}, "commonCropName": "test", "accessionNumber": "2", "defaultDisplayName": "Gnome-2", "externalReferences": [{"referenceID": "bcc3fab6-04ed-4e2f-b0b0-9e9b247462ef", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "eddb03ff-09c5-481d-9c1c-8fc609d432e5", "state": "NEW", "brAPIObject": {"pedigree": "Gnome-1 [MNTTIM-1]/Gnome-2 [MNTTIM-2]", "seedSource": "Breeding Insight", "germplasmName": "Gnome-3 [MNTTIM-3]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Biparental cross", "breedingMethodId": "aee4d3a3-0df8-422f-b052-d9f0b81e5b4d", "importEntryNumber": "3", "maleParentEntryNo": "2", "femaleParentEntryNo": "1"}, "commonCropName": "test", "accessionNumber": "3", "defaultDisplayName": "Gnome-3", "externalReferences": [{"referenceID": "24c3164e-0b37-44b4-8b18-9118ca9b2949", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "42ff9b3e-3f04-42f3-909b-2aaa158457e5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-1 [MNTTIM-4]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "4"}, "commonCropName": "test", "accessionNumber": "4", "defaultDisplayName": "Gnome-3-1", "externalReferences": [{"referenceID": "c271722b-488b-41c8-9671-73ac008be522", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "883cef8e-34c9-43f7-8062-d7d80c8a65fd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-2 [MNTTIM-5]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "5"}, "commonCropName": "test", "accessionNumber": "5", "defaultDisplayName": "Gnome-3-2", "externalReferences": [{"referenceID": "6bb014ce-0c5f-480c-a1e2-9b87a914a733", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "6215c013-2ed9-43c1-a964-15f87024da36", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-3 [MNTTIM-6]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "6"}, "commonCropName": "test", "accessionNumber": "6", "defaultDisplayName": "Gnome-3-3", "externalReferences": [{"referenceID": "988fe16a-6308-4146-9cae-be9d0cd9c295", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "bd62dec0-25d3-4bef-b9f7-5af634f78366", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-4 [MNTTIM-7]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "7"}, "commonCropName": "test", "accessionNumber": "7", "defaultDisplayName": "Gnome-3-4", "externalReferences": [{"referenceID": "28e6482a-2804-4aa8-b98d-6c4a56f8d8d2", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "c0067078-7a6d-4c99-b6ee-ab220feccddd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-5 [MNTTIM-8]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "8"}, "commonCropName": "test", "accessionNumber": "8", "defaultDisplayName": "Gnome-3-5", "externalReferences": [{"referenceID": "e7c25b55-61f5-4845-9080-d2243171f70b", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "cdb38aac-51fc-4ea7-9b38-8072258ceac5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-6 [MNTTIM-9]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "9"}, "commonCropName": "test", "accessionNumber": "9", "defaultDisplayName": "Gnome-3-6", "externalReferences": [{"referenceID": "ade6b8ca-b2e0-49cc-ac09-09476b1416e7", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "3f6c7d4b-fca2-4b22-aa6a-aadb347cd625", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-7 [MNTTIM-10]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "10"}, "commonCropName": "test", "accessionNumber": "10", "defaultDisplayName": "Gnome-3-7", "externalReferences": [{"referenceID": "6cd34ea3-86b0-41a2-aef0-57c101bd68fb", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "68a72f81-42b5-4425-8034-eef7d6921928", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-6 [MNTTIM-9]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-11]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "11", "maleParentEntryNo": "1", "femaleParentEntryNo": "9"}, "commonCropName": "test", "accessionNumber": "11", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "354faf95-682c-4a62-90f4-4f6c9c2d4191", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "25d84084-45a6-4d93-a683-004f95d3dc3e", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3 [MNTTIM-3]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-12]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "12", "maleParentEntryNo": "1", "femaleParentEntryNo": "3"}, "commonCropName": "test", "accessionNumber": "12", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "936a608f-0e2b-4cee-b348-dbbd57cbdf65", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "569fd3a0-1960-45e0-86d4-5157e21e8130", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-4 [MNTTIM-7]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-13]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "13", "maleParentEntryNo": "1", "femaleParentEntryNo": "7"}, "commonCropName": "test", "accessionNumber": "13", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "4e708554-e088-48ce-a16d-43f1778549fa", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}], "statistics": {"Germplasm": {"newObjectCount": 13}, "Pedigree Connections": {"newObjectCount": 4}}}', '2022-05-17 14:19:33 +00:00', '2022-05-17 14:19:33 +00:00', (select id from bi_user where oauth_id = '1111-2222-3333-4444'), (select id from bi_user where oauth_id = '1111-2222-3333-4444')); -INSERT INTO public.importer_import (id, program_id, user_id, importer_mapping_id, importer_progress_id, upload_file_name, file_data, modified_data, mapped_data, created_at, updated_at, created_by, updated_by) VALUES ('210c7175-9d7a-454c-b35f-c8aa0ee0a31e', ?::uuid, (select id from bi_user where orcid = '1111-2222-3333-4444'), (select id from importer_mapping where name = 'GermplasmTemplateMap'), 'd02090c3-e40d-45f4-9a63-b8c701c9531b', 'germ import 2.xlsx', '[{"Name": "Gnome-1", "Source": "USDA", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-2", "Source": "USDA", "Entry No": "2", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3", "Source": "Breeding Insight", "Entry No": "3", "External UID": "", "Breeding Method": "BPC", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "2", "Female Parent Entry No": "1"}, {"Name": "Gnome-3-1", "Source": "Breeding Insight", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-2", "Source": "Breeding Insight", "Entry No": "5", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-3", "Source": "Breeding Insight", "Entry No": "6", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-4", "Source": "Breeding Insight", "Entry No": "7", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-5", "Source": "Breeding Insight", "Entry No": "8", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-6", "Source": "Breeding Insight", "Entry No": "9", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-7", "Source": "Breeding Insight", "Entry No": "10", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "11", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "9"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "12", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "3"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "13", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "7"}]', null, '{"rows": [{"germplasm": {"id": "1580b261-f504-4a7e-bbc7-e5599f0b2f4b", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-1 [MNTTIM-1]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "1"}, "commonCropName": "test", "accessionNumber": "1", "defaultDisplayName": "Gnome-1", "externalReferences": [{"referenceID": "6a1269cb-6e7a-4b87-a353-c681c214bd69", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "7e665eb0-2f7b-4b14-85a4-50f5508de2a8", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-2 [MNTTIM-2]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "2"}, "commonCropName": "test", "accessionNumber": "2", "defaultDisplayName": "Gnome-2", "externalReferences": [{"referenceID": "bcc3fab6-04ed-4e2f-b0b0-9e9b247462ef", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "eddb03ff-09c5-481d-9c1c-8fc609d432e5", "state": "NEW", "brAPIObject": {"pedigree": "Gnome-1 [MNTTIM-1]/Gnome-2 [MNTTIM-2]", "seedSource": "Breeding Insight", "germplasmName": "Gnome-3 [MNTTIM-3]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Biparental cross", "breedingMethodId": "aee4d3a3-0df8-422f-b052-d9f0b81e5b4d", "importEntryNumber": "3", "maleParentEntryNo": "2", "femaleParentEntryNo": "1"}, "commonCropName": "test", "accessionNumber": "3", "defaultDisplayName": "Gnome-3", "externalReferences": [{"referenceID": "24c3164e-0b37-44b4-8b18-9118ca9b2949", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "42ff9b3e-3f04-42f3-909b-2aaa158457e5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-1 [MNTTIM-4]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "4"}, "commonCropName": "test", "accessionNumber": "4", "defaultDisplayName": "Gnome-3-1", "externalReferences": [{"referenceID": "c271722b-488b-41c8-9671-73ac008be522", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "883cef8e-34c9-43f7-8062-d7d80c8a65fd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-2 [MNTTIM-5]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "5"}, "commonCropName": "test", "accessionNumber": "5", "defaultDisplayName": "Gnome-3-2", "externalReferences": [{"referenceID": "6bb014ce-0c5f-480c-a1e2-9b87a914a733", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "6215c013-2ed9-43c1-a964-15f87024da36", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-3 [MNTTIM-6]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "6"}, "commonCropName": "test", "accessionNumber": "6", "defaultDisplayName": "Gnome-3-3", "externalReferences": [{"referenceID": "988fe16a-6308-4146-9cae-be9d0cd9c295", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "bd62dec0-25d3-4bef-b9f7-5af634f78366", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-4 [MNTTIM-7]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "7"}, "commonCropName": "test", "accessionNumber": "7", "defaultDisplayName": "Gnome-3-4", "externalReferences": [{"referenceID": "28e6482a-2804-4aa8-b98d-6c4a56f8d8d2", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "c0067078-7a6d-4c99-b6ee-ab220feccddd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-5 [MNTTIM-8]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "8"}, "commonCropName": "test", "accessionNumber": "8", "defaultDisplayName": "Gnome-3-5", "externalReferences": [{"referenceID": "e7c25b55-61f5-4845-9080-d2243171f70b", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "cdb38aac-51fc-4ea7-9b38-8072258ceac5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-6 [MNTTIM-9]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "9"}, "commonCropName": "test", "accessionNumber": "9", "defaultDisplayName": "Gnome-3-6", "externalReferences": [{"referenceID": "ade6b8ca-b2e0-49cc-ac09-09476b1416e7", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "3f6c7d4b-fca2-4b22-aa6a-aadb347cd625", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-7 [MNTTIM-10]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "10"}, "commonCropName": "test", "accessionNumber": "10", "defaultDisplayName": "Gnome-3-7", "externalReferences": [{"referenceID": "6cd34ea3-86b0-41a2-aef0-57c101bd68fb", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "68a72f81-42b5-4425-8034-eef7d6921928", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-6 [MNTTIM-9]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-11]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "11", "maleParentEntryNo": "1", "femaleParentEntryNo": "9"}, "commonCropName": "test", "accessionNumber": "11", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "354faf95-682c-4a62-90f4-4f6c9c2d4191", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "25d84084-45a6-4d93-a683-004f95d3dc3e", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3 [MNTTIM-3]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-12]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "12", "maleParentEntryNo": "1", "femaleParentEntryNo": "3"}, "commonCropName": "test", "accessionNumber": "12", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "936a608f-0e2b-4cee-b348-dbbd57cbdf65", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "569fd3a0-1960-45e0-86d4-5157e21e8130", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-4 [MNTTIM-7]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-13]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "13", "maleParentEntryNo": "1", "femaleParentEntryNo": "7"}, "commonCropName": "test", "accessionNumber": "13", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "4e708554-e088-48ce-a16d-43f1778549fa", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}], "statistics": {"Germplasm": {"newObjectCount": 13}, "Pedigree Connections": {"newObjectCount": 4}}}', '2022-05-18 14:19:33 +00:00', '2022-05-18 14:19:33 +00:00', (select id from bi_user where orcid = '1111-2222-3333-4444'), (select id from bi_user where orcid = '1111-2222-3333-4444')); +INSERT INTO public.importer_import (id, program_id, user_id, importer_mapping_id, importer_progress_id, upload_file_name, file_data, modified_data, mapped_data, created_at, updated_at, created_by, updated_by) VALUES ('210c7175-9d7a-454c-b35f-c8aa0ee0a31e', ?::uuid, (select id from bi_user where oauth_id = '1111-2222-3333-4444'), (select id from importer_mapping where name = 'GermplasmTemplateMap'), 'd02090c3-e40d-45f4-9a63-b8c701c9531b', 'germ import 2.xlsx', '[{"Name": "Gnome-1", "Source": "USDA", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-2", "Source": "USDA", "Entry No": "2", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3", "Source": "Breeding Insight", "Entry No": "3", "External UID": "", "Breeding Method": "BPC", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "2", "Female Parent Entry No": "1"}, {"Name": "Gnome-3-1", "Source": "Breeding Insight", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-2", "Source": "Breeding Insight", "Entry No": "5", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-3", "Source": "Breeding Insight", "Entry No": "6", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-4", "Source": "Breeding Insight", "Entry No": "7", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-5", "Source": "Breeding Insight", "Entry No": "8", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-6", "Source": "Breeding Insight", "Entry No": "9", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-3-7", "Source": "Breeding Insight", "Entry No": "10", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "11", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "9"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "12", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "3"}, {"Name": "Gnome-11", "Source": "Theoretical Breeding Institute", "Entry No": "13", "External UID": "", "Breeding Method": "BCR", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "1", "Female Parent Entry No": "7"}]', null, '{"rows": [{"germplasm": {"id": "1580b261-f504-4a7e-bbc7-e5599f0b2f4b", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-1 [MNTTIM-1]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "1"}, "commonCropName": "test", "accessionNumber": "1", "defaultDisplayName": "Gnome-1", "externalReferences": [{"referenceID": "6a1269cb-6e7a-4b87-a353-c681c214bd69", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "7e665eb0-2f7b-4b14-85a4-50f5508de2a8", "state": "NEW", "brAPIObject": {"seedSource": "USDA", "germplasmName": "Gnome-2 [MNTTIM-2]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Unknown maintenance method", "breedingMethodId": "af40db90-64ab-444d-a749-2f857bc90f64", "importEntryNumber": "2"}, "commonCropName": "test", "accessionNumber": "2", "defaultDisplayName": "Gnome-2", "externalReferences": [{"referenceID": "bcc3fab6-04ed-4e2f-b0b0-9e9b247462ef", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "eddb03ff-09c5-481d-9c1c-8fc609d432e5", "state": "NEW", "brAPIObject": {"pedigree": "Gnome-1 [MNTTIM-1]/Gnome-2 [MNTTIM-2]", "seedSource": "Breeding Insight", "germplasmName": "Gnome-3 [MNTTIM-3]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Biparental cross", "breedingMethodId": "aee4d3a3-0df8-422f-b052-d9f0b81e5b4d", "importEntryNumber": "3", "maleParentEntryNo": "2", "femaleParentEntryNo": "1"}, "commonCropName": "test", "accessionNumber": "3", "defaultDisplayName": "Gnome-3", "externalReferences": [{"referenceID": "24c3164e-0b37-44b4-8b18-9118ca9b2949", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "42ff9b3e-3f04-42f3-909b-2aaa158457e5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-1 [MNTTIM-4]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "4"}, "commonCropName": "test", "accessionNumber": "4", "defaultDisplayName": "Gnome-3-1", "externalReferences": [{"referenceID": "c271722b-488b-41c8-9671-73ac008be522", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "883cef8e-34c9-43f7-8062-d7d80c8a65fd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-2 [MNTTIM-5]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "5"}, "commonCropName": "test", "accessionNumber": "5", "defaultDisplayName": "Gnome-3-2", "externalReferences": [{"referenceID": "6bb014ce-0c5f-480c-a1e2-9b87a914a733", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "6215c013-2ed9-43c1-a964-15f87024da36", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-3 [MNTTIM-6]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "6"}, "commonCropName": "test", "accessionNumber": "6", "defaultDisplayName": "Gnome-3-3", "externalReferences": [{"referenceID": "988fe16a-6308-4146-9cae-be9d0cd9c295", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "bd62dec0-25d3-4bef-b9f7-5af634f78366", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-4 [MNTTIM-7]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "7"}, "commonCropName": "test", "accessionNumber": "7", "defaultDisplayName": "Gnome-3-4", "externalReferences": [{"referenceID": "28e6482a-2804-4aa8-b98d-6c4a56f8d8d2", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "c0067078-7a6d-4c99-b6ee-ab220feccddd", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-5 [MNTTIM-8]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "8"}, "commonCropName": "test", "accessionNumber": "8", "defaultDisplayName": "Gnome-3-5", "externalReferences": [{"referenceID": "e7c25b55-61f5-4845-9080-d2243171f70b", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "cdb38aac-51fc-4ea7-9b38-8072258ceac5", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-6 [MNTTIM-9]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "9"}, "commonCropName": "test", "accessionNumber": "9", "defaultDisplayName": "Gnome-3-6", "externalReferences": [{"referenceID": "ade6b8ca-b2e0-49cc-ac09-09476b1416e7", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "3f6c7d4b-fca2-4b22-aa6a-aadb347cd625", "state": "NEW", "brAPIObject": {"seedSource": "Breeding Insight", "germplasmName": "Gnome-3-7 [MNTTIM-10]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Clone formation (Veg)", "breedingMethodId": "48409513-d05d-458a-acf6-a5c501734145", "importEntryNumber": "10"}, "commonCropName": "test", "accessionNumber": "10", "defaultDisplayName": "Gnome-3-7", "externalReferences": [{"referenceID": "6cd34ea3-86b0-41a2-aef0-57c101bd68fb", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "68a72f81-42b5-4425-8034-eef7d6921928", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-6 [MNTTIM-9]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-11]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "11", "maleParentEntryNo": "1", "femaleParentEntryNo": "9"}, "commonCropName": "test", "accessionNumber": "11", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "354faf95-682c-4a62-90f4-4f6c9c2d4191", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "25d84084-45a6-4d93-a683-004f95d3dc3e", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3 [MNTTIM-3]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-12]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "12", "maleParentEntryNo": "1", "femaleParentEntryNo": "3"}, "commonCropName": "test", "accessionNumber": "12", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "936a608f-0e2b-4cee-b348-dbbd57cbdf65", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}, {"germplasm": {"id": "569fd3a0-1960-45e0-86d4-5157e21e8130", "state": "EXISTING", "brAPIObject": {"pedigree": "Gnome-3-4 [MNTTIM-7]/Gnome-1 [MNTTIM-1]", "seedSource": "Theoretical Breeding Institute", "germplasmName": "Gnome-11 [MNTTIM-13]", "additionalInfo": {"createdBy": {"userId": "55e268e9-cc02-44e6-ad1e-d99ea9770830", "userName": "BI-DEV Admin"}, "createdDate": "17/05/2022 10:19:54", "breedingMethod": "Backcross", "breedingMethodId": "2ac64ec4-19af-4b4a-946f-9b1be0cc1730", "importEntryNumber": "13", "maleParentEntryNo": "1", "femaleParentEntryNo": "7"}, "commonCropName": "test", "accessionNumber": "13", "defaultDisplayName": "Gnome-11", "externalReferences": [{"referenceID": "4e708554-e088-48ce-a16d-43f1778549fa", "referenceSource": "breedinginsight.org"}, {"referenceID": "ee801336-c885-44f1-873f-666ec6e0740a", "referenceSource": "breedinginsight.org/programs"}]}}}], "statistics": {"Germplasm": {"newObjectCount": 13}, "Pedigree Connections": {"newObjectCount": 4}}}', '2022-05-18 14:19:33 +00:00', '2022-05-18 14:19:33 +00:00', (select id from bi_user where oauth_id = '1111-2222-3333-4444'), (select id from bi_user where oauth_id = '1111-2222-3333-4444')); diff --git a/src/test/resources/sql/brapi/species.sql b/src/test/resources/sql/brapi/species.sql index 2b8fcf26d..acb173fb2 100644 --- a/src/test/resources/sql/brapi/species.sql +++ b/src/test/resources/sql/brapi/species.sql @@ -36,7 +36,7 @@ FROM (VALUES ('Strawberry'), ('Honey Bee'), ('Pecan'), ('Lettuce'), ('Cotton'), ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), ('Red Clover'), ('Potato'), ('Blackberry'), - ('Raspberry'), ('Sugar Beet'), ('Coffee') + ('Raspberry'), ('Sugar Beet'), ('Coffee'), ('Sunflower') ) AS src(crop_name) ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name