diff --git a/src/main/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadController.java b/src/main/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadController.java index df6615458..5f8540cae 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadController.java @@ -32,15 +32,15 @@ public GenotypeDataUploadController(GenotypeService genoService, SecurityService this.securityService = securityService; } - @Post("programs/{programId}/experiments/{experimentId}/geno/import") + @Post("programs/{programId}/submissions/{submissionId}/geno/import") @Consumes(MediaType.MULTIPART_FORM_DATA) @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.PROGRAM_ADMIN}) - public HttpResponse> uploadData(@PathVariable UUID programId, @PathVariable UUID experimentId, @Part("file") CompletedFileUpload upload) { + public HttpResponse> uploadData(@PathVariable UUID programId, @PathVariable UUID submissionId, @Part("file") CompletedFileUpload upload) { AuthenticatedUser actingUser = securityService.getUser(); try { - ImportResponse result = genoService.submitGenotypeData(actingUser.getId(), programId, experimentId, upload); + ImportResponse result = genoService.submitGenotypeData(actingUser.getId(), programId, submissionId, upload); Response response = new Response<>(result); return HttpResponse.ok(response); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/services/geno/GenotypeService.java b/src/main/java/org/breedinginsight/services/geno/GenotypeService.java index bceb33503..7da2f1fbd 100644 --- a/src/main/java/org/breedinginsight/services/geno/GenotypeService.java +++ b/src/main/java/org/breedinginsight/services/geno/GenotypeService.java @@ -11,7 +11,7 @@ import java.util.UUID; public interface GenotypeService { - ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID experimentId, CompletedFileUpload uploadedFile) throws DoesNotExistException, AuthorizationException, ApiException; + ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID submissionId, CompletedFileUpload uploadedFile) throws DoesNotExistException, AuthorizationException, ApiException; GermplasmGenotype retrieveGenotypeData(UUID programId, BrAPIGermplasm germplasm) throws DoesNotExistException, AuthorizationException, ApiException; } diff --git a/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java b/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java index 97c78cf05..dbc7e89ad 100644 --- a/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java +++ b/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java @@ -18,18 +18,13 @@ import org.brapi.client.v2.auth.OAuth; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.model.queryParams.core.ProgramQueryParams; -import org.brapi.client.v2.model.queryParams.core.TrialQueryParams; import org.brapi.client.v2.modules.core.ProgramsApi; -import org.brapi.client.v2.modules.core.TrialsApi; import org.brapi.client.v2.modules.genotype.CallSetsApi; import org.brapi.client.v2.modules.genotype.CallsApi; import org.brapi.client.v2.modules.genotype.SamplesApi; import org.brapi.client.v2.modules.genotype.VariantsApi; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIProgramListResponse; -import org.brapi.v2.model.core.response.BrAPITrialListResponse; import org.brapi.v2.model.geno.BrAPICall; import org.brapi.v2.model.geno.BrAPICallSet; import org.brapi.v2.model.geno.BrAPISample; @@ -42,14 +37,15 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapps.importer.daos.BrAPISampleDAO; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; import org.breedinginsight.brapps.importer.model.response.ImportResponse; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.SampleSubmissionDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.GermplasmGenotype; import org.breedinginsight.model.Program; @@ -100,6 +96,8 @@ public class GigwaGenotypeServiceImpl implements GenotypeService { private final ProgramDAO programDAO; private final UserDAO userDAO; private final ImportDAO importDAO; + private final SampleSubmissionDAO sampleSubmissionDAO; + private final BrAPISampleDAO sampleDAO; private final ImportMappingDAO importMappingDAO; private final SimpleStorageService storageService; @@ -122,6 +120,8 @@ public GigwaGenotypeServiceImpl(@Property(name = "gigwa.host") String gigwaHost, ProgramDAO programDAO, UserDAO userDAO, ImportDAO importDAO, + SampleSubmissionDAO sampleSubmissionDAO, + BrAPISampleDAO sampleDAO, ImportMappingDAO importMappingDAO, @Named("genotype") SimpleStorageService storageService, S3Client s3Client, @@ -137,6 +137,8 @@ public GigwaGenotypeServiceImpl(@Property(name = "gigwa.host") String gigwaHost, this.programDAO = programDAO; this.userDAO = userDAO; this.importDAO = importDAO; + this.sampleSubmissionDAO = sampleSubmissionDAO; + this.sampleDAO = sampleDAO; this.importMappingDAO = importMappingDAO; this.storageService = storageService; this.s3Client = s3Client; @@ -147,7 +149,7 @@ public GigwaGenotypeServiceImpl(@Property(name = "gigwa.host") String gigwaHost, } @Override - public ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID experimentId, CompletedFileUpload uploadedFile) throws DoesNotExistException, AuthorizationException, ApiException { + public ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID submissionId, CompletedFileUpload uploadedFile) throws DoesNotExistException, AuthorizationException, ApiException { Program program = getProgram(programId); User user = userDAO.getUser(userId) @@ -204,10 +206,10 @@ public ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID exper try { byte[] fileContents = uploadedFile.getBytes(); - if(validateSamples(program, experimentId, fileContents, upload, gigwaAuthToken)) { + if(validateSamples(program, submissionId, fileContents, upload)) { executor.execute(() -> { try { - processSubmission(gigwaAuthToken, program, experimentId, fileContents, uploadedFile.getFilename(), upload, progress); + processSubmission(gigwaAuthToken, program, submissionId, fileContents, uploadedFile.getFilename(), upload, progress); } catch (Exception e) { log.error(e.getMessage(), e); } @@ -261,18 +263,13 @@ public GermplasmGenotype retrieveGenotypeData(UUID programId, BrAPIGermplasm ger } } - private boolean validateSamples(Program program, UUID experimentId, byte[] fileContents, ImportUpload upload, String gigwaAuthToken) throws DoesNotExistException, ApiException { - log.debug("Validating samples in submitted VCF file for experiment: " + experimentId); + private boolean validateSamples(Program program, UUID submissionId, byte[] fileContents, ImportUpload upload) throws DoesNotExistException, ApiException { + log.debug("Validating samples in submitted VCF file for submission: " + submissionId); - BrAPIClient brAPIClient = programDAO.getCoreClient(program.getId()); - brAPIClient.setBasePath(gigwaHost + GIGWA_BRAPI_BASE_PATH); - Authentication authorizationToken = brAPIClient.getAuthentication("AuthorizationToken"); - if(authorizationToken instanceof OAuth) { - ((OAuth)authorizationToken).setAccessToken(gigwaAuthToken); - } - BrAPIClient brapiPhenoClient = programDAO.getPhenoClient(program.getId()); - - Set obsUnitNames = fetchObservationUnits(brapiPhenoClient, experimentId).stream().map(ou -> Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())).collect(Collectors.toSet()); + Set submissionSampleNames = fetchSubmissionSamples(program, submissionId).stream() + .map(BrAPISample::getSampleName) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); log.debug("searching for the VCF header row"); String[] headerParts = null; @@ -317,17 +314,17 @@ private boolean validateSamples(Program program, UUID experimentId, byte[] fileC return false; } - log.debug("pulled all the samples from the VCF, now checking each one has an OU record"); - List samplesMissingOu = new ArrayList<>(); + log.debug("pulled all the samples from the VCF, now checking each one belongs to the submission"); + List samplesMissingSubmission = new ArrayList<>(); samples.forEach(s -> { - if(!obsUnitNames.contains(s)) { - samplesMissingOu.add(s); + if(!submissionSampleNames.contains(s)) { + samplesMissingSubmission.add(s); } }); - if(!samplesMissingOu.isEmpty()) { + if(!samplesMissingSubmission.isEmpty()) { upload.getProgress().setStatuscode((short)HttpStatus.BAD_REQUEST.getCode()); - upload.getProgress().setMessage("There are samples that do not have an existing observation unit"); + upload.getProgress().setMessage("There are samples that are not linked to the selected submission"); importDAO.updateProgress(upload.getProgress()); return false; } @@ -408,43 +405,12 @@ private List fetchObservationUnits(BrAPIClient phenoBrAPIC return brAPIDAOUtil.search(observationUnitsApi::searchObservationunitsPost, observationUnitsApi::searchObservationunitsSearchResultsDbIdGet, searchRequest); } - private List fetchObservationUnits(BrAPIClient phenoBrAPIClient, UUID experimentId) throws ApiException, DoesNotExistException { - log.debug("fetching observationUnits for experiment: " + experimentId); - TrialsApi trialsApi = brAPIEndpointProvider.get(phenoBrAPIClient, TrialsApi.class); - ApiResponse brAPITrialListResponseApiResponse = trialsApi.trialsGet(new TrialQueryParams().externalReferenceID(experimentId.toString()) - .externalReferenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS))); - - BrAPITrial brAPITrial = null; - if(brAPITrialListResponseApiResponse.getBody().getResult().getData() != null) { - if (brAPITrialListResponseApiResponse.getBody().getResult().getData().size() == 1) { - brAPITrial = brAPITrialListResponseApiResponse.getBody().getResult().getData().get(0); - } else { - String trialReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS); - for (BrAPITrial trial : brAPITrialListResponseApiResponse.getBody().getResult().getData()) { - if (trial.getExternalReferences() != null) { - Optional xref = trial.getExternalReferences() - .stream() - .filter(externalReference -> externalReference.getReferenceSource().equals(trialReferenceSource)) - .findFirst(); - if (xref.isPresent() && xref.get().getReferenceID().equals(experimentId.toString())) { - brAPITrial = trial; - break; - } - } - } - } + private List fetchSubmissionSamples(Program program, UUID submissionId) throws ApiException, DoesNotExistException { + if(sampleSubmissionDAO.getBySubmissionId(program, submissionId).isEmpty()) { + throw new DoesNotExistException("Could not find sample submission in database"); } - if(brAPITrial != null) { - ObservationUnitsApi observationUnitsApi = brAPIEndpointProvider.get(phenoBrAPIClient, ObservationUnitsApi.class); - - BrAPIObservationUnitSearchRequest searchRequest = new BrAPIObservationUnitSearchRequest(); - searchRequest.addTrialDbIdsItem(brAPITrial.getTrialDbId()); - - return brAPIDAOUtil.search(observationUnitsApi::searchObservationunitsPost, observationUnitsApi::searchObservationunitsSearchResultsDbIdGet, searchRequest); - } else { - throw new DoesNotExistException("Could not find experiment in database"); - } + return sampleDAO.readSamplesBySubmissionIds(program, List.of(submissionId.toString())); } private List fetchCallsets(BrAPIClient genoBrAPIClient, List germplasmSamples) throws ApiException { @@ -500,12 +466,12 @@ private List fetchVariants(BrAPIClient genoBrAPIClient, List uploadedFileResult; try { progress.setMessage("Uploading file"); importDAO.updateProgress(progress); - uploadedFileResult = uploadGenotypeData(program.getId(), experimentId, upload.getId(), fileContents, filename); + uploadedFileResult = uploadGenotypeData(program.getId(), submissionId, upload.getId(), fileContents, filename); log.debug("file saved to: " + uploadedFileResult.getLeft()); } catch (Exception e) { progress.setStatuscode((short) HttpStatus.INTERNAL_SERVER_ERROR.getCode()); @@ -518,7 +484,7 @@ protected void processSubmission(String gigwaAuthToken, Program program, UUID ex importDAO.updateProgress(progress); OkHttpClient client = new OkHttpClient(); - String gigwaProgressToken = submitRequestToGigwa(client, program, experimentId, uploadedFileResult.getLeft(), gigwaAuthToken, progress); + String gigwaProgressToken = submitRequestToGigwa(client, program, submissionId, uploadedFileResult.getLeft(), gigwaAuthToken, progress); if(checkGigwaProgress(client, gigwaAuthToken, gigwaProgressToken, progress)) { log.debug("Gigwa import was successful!"); @@ -593,19 +559,19 @@ private boolean checkGigwaProgress(OkHttpClient client, String gigwaAuthToken, S * Submits the upload request to Gigwa, and returns the progress token * @param client * @param program - * @param experimentId + * @param submissionId * @param fileUrl * @param gigwaAuthToken * @param progress * @return the progress token to check on the import's progress * @throws IOException */ - private String submitRequestToGigwa(OkHttpClient client, Program program, UUID experimentId, String fileUrl, String gigwaAuthToken, ImportProgress progress) throws IOException { + private String submitRequestToGigwa(OkHttpClient client, Program program, UUID submissionId, String fileUrl, String gigwaAuthToken, ImportProgress progress) throws IOException { Request request = new Request.Builder() .url(HttpUrl.parse(buildPath("gigwa/genotypeImport")) .newBuilder() .addQueryParameter("module", program.getKey()) - .addQueryParameter("project", experimentId.toString()) + .addQueryParameter("project", submissionId.toString()) .addQueryParameter("run", LocalDateTime.now().toString()) .addQueryParameter("dataFile1", fileUrl) @@ -631,7 +597,7 @@ private String submitRequestToGigwa(OkHttpClient client, Program program, UUID e } } - private Pair uploadGenotypeData(UUID programId, UUID experimentId, UUID uploadId, byte[] fileContents, String filename) throws IOException, MimeTypeException { + private Pair uploadGenotypeData(UUID programId, UUID submissionId, UUID uploadId, byte[] fileContents, String filename) throws IOException, MimeTypeException { log.debug("saving genotype data to S3"); if(!storageService.listBucketNames().contains(storageService.getDefaultBucketName())) { @@ -641,7 +607,7 @@ private Pair uploadGenotypeData(UUID programId, UUID experimentId, var mimeType = mimeTypeParser.getMimeType(fileContents, filename); - var key = programId.toString() + "/" + experimentId.toString() + "/" + uploadId + mimeType.getExtension(); + var key = programId.toString() + "/" + submissionId.toString() + "/" + uploadId + mimeType.getExtension(); var path = storeMultipartFile(key, fileContents, Map.of("originalFileName", filename)); Long fileSize = Long.valueOf(fileContents.length); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadControllerIntegrationTest.java new file mode 100644 index 000000000..7398b6fa7 --- /dev/null +++ b/src/test/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadControllerIntegrationTest.java @@ -0,0 +1,145 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.v1.controller.geno; + +import io.kowalski.fannypack.FannyPack; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.client.exceptions.HttpClientResponseException; +import io.micronaut.http.client.multipart.MultipartBody; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.http.multipart.CompletedFileUpload; +import io.micronaut.test.annotation.MockBean; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import org.breedinginsight.DatabaseTest; +import org.breedinginsight.api.v1.controller.TestTokenValidator; +import org.breedinginsight.brapps.importer.model.ImportProgress; +import org.breedinginsight.brapps.importer.model.response.ImportResponse; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.services.geno.GenotypeService; +import org.jooq.DSLContext; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import javax.inject.Inject; +import java.io.File; +import java.util.UUID; + +import static io.micronaut.http.HttpRequest.POST; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class GenotypeDataUploadControllerIntegrationTest extends DatabaseTest { + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + @Inject + private GenotypeService genotypeService; + + @Inject + private DSLContext dsl; + + @Inject + private ProgramDAO programDAO; + + @Inject + private UserDAO userDAO; + + private Program program; + private User testUser; + + @MockBean(GenotypeService.class) + GenotypeService genotypeService() { + return mock(GenotypeService.class); + } + + @BeforeAll + void setup() { + FannyPack fp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + dsl.execute(fp.get("InsertPrograms")); + program = programDAO.getAll().get(0); + testUser = userDAO.getUserByOAuthId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(); + dsl.execute(fp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); + } + + @BeforeEach + void resetMocks() { + reset(genotypeService); + } + + @Test + void uploadDataUsesSubmissionScopedRouteAndServiceContract() throws Exception { + UUID submissionId = UUID.randomUUID(); + UUID importId = UUID.randomUUID(); + + ImportResponse importResponse = new ImportResponse(); + importResponse.setImportId(importId); + importResponse.setProgress(ImportProgress.builder() + .statuscode((short) HttpStatus.ACCEPTED.getCode()) + .build()); + + doReturn(importResponse).when(genotypeService) + .submitGenotypeData(eq(testUser.getId()), eq(program.getId()), eq(submissionId), any(CompletedFileUpload.class)); + + HttpResponse response = client.exchange( + POST(String.format("/programs/%s/submissions/%s/geno/import", program.getId(), submissionId), multipartBody()) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ).blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + verify(genotypeService).submitGenotypeData(eq(testUser.getId()), eq(program.getId()), eq(submissionId), any(CompletedFileUpload.class)); + } + + @Test + void experimentScopedUploadRouteIsRemoved() { + UUID experimentId = UUID.randomUUID(); + + HttpClientResponseException exception = assertThrows(HttpClientResponseException.class, () -> client.exchange( + POST(String.format("/programs/%s/experiments/%s/geno/import", program.getId(), experimentId), multipartBody()) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ).blockingFirst()); + + assertEquals(HttpStatus.FORBIDDEN, exception.getStatus()); + verifyNoInteractions(genotypeService); + } + + private MultipartBody multipartBody() { + return MultipartBody.builder() + .addPart("file", new File("src/test/resources/files/geno/sample.vcf")) + .build(); + } +} diff --git a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java index 94770d912..c1009977b 100644 --- a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java +++ b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java @@ -5,7 +5,10 @@ import com.agorapulse.micronaut.amazon.awssdk.s3.SimpleStorageServiceConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; import io.micronaut.context.ApplicationContext; +import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Replaces; +import io.micronaut.context.annotation.Requires; import io.micronaut.context.event.BeanCreatedEventListener; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; @@ -13,8 +16,6 @@ import io.micronaut.inject.qualifiers.Qualifiers; import io.micronaut.test.annotation.MockBean; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; -import io.reactivex.functions.Function; -import io.reactivex.functions.Function3; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.tika.mime.MimeTypeException; @@ -25,50 +26,42 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.model.queryParams.core.ProgramQueryParams; import org.brapi.client.v2.model.queryParams.core.StudyQueryParams; -import org.brapi.client.v2.model.queryParams.core.TrialQueryParams; import org.brapi.client.v2.modules.core.ProgramsApi; import org.brapi.client.v2.modules.core.StudiesApi; -import org.brapi.client.v2.modules.core.TrialsApi; +import org.brapi.client.v2.modules.genotype.SamplesApi; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIProgramListResponse; import org.brapi.v2.model.core.response.BrAPIStudyListResponse; -import org.brapi.v2.model.core.response.BrAPITrialListResponse; -import org.brapi.v2.model.core.response.BrAPITrialListResponseResult; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.geno.request.BrAPISampleSearchRequest; +import org.brapi.v2.model.geno.response.BrAPISampleListResponse; +import org.brapi.v2.model.geno.response.BrAPISampleListResponseResult; import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; -import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse; -import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponseResult; import org.breedinginsight.DatabaseTest; -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.daos.BrAPISampleDAO; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; -import org.breedinginsight.brapi.v2.dao.impl.BrAPITrialDAOImpl; import org.breedinginsight.brapi.v2.dao.impl.ImportMappingDAOImpl; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; import org.breedinginsight.brapps.importer.model.response.ImportResponse; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.tables.pojos.ImporterImportEntity; import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.SampleSubmissionDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.daos.impl.ProgramDAOImpl; import org.breedinginsight.daos.impl.UserDAOImpl; import org.breedinginsight.model.BrAPIConstants; import org.breedinginsight.model.GermplasmGenotype; import org.breedinginsight.model.Program; +import org.breedinginsight.model.SampleSubmission; import org.breedinginsight.model.User; -import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.brapi.BrAPIClientProvider; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.services.brapi.BrAPIProvider; import org.breedinginsight.services.exceptions.AuthorizationException; import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.utilities.BrAPIDAOUtil; -import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; import org.jooq.Configuration; import org.jooq.DSLContext; @@ -83,6 +76,7 @@ import javax.inject.Inject; import javax.inject.Named; +import javax.inject.Singleton; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -129,7 +123,10 @@ public class GigwaGenotypeServiceImplIntegrationTest extends DatabaseTest { private ImportDAO importDAO; @Inject - private BrAPITrialDAO trialDAO; + private SampleSubmissionDAO sampleSubmissionDAO; + + @Inject + private BrAPISampleDAO sampleDAO; @Inject private ObjectMapper objectMapper; @@ -147,9 +144,6 @@ public class GigwaGenotypeServiceImplIntegrationTest extends DatabaseTest { @Inject private S3Presigner presigner; - @Inject - private BrAPIDAOUtil brAPIDAOUtil; - @Inject private BrAPIEndpointProvider brAPIEndpointProvider; @@ -188,20 +182,14 @@ ImportDAO importDAO() { return mock(ImportDAO.class); } - - @MockBean(BrAPITrialDAOImpl.class) - BrAPITrialDAO trialDAO() { - return mock(BrAPITrialDAOImpl.class); - } - - @MockBean(ProgramService.class) - ProgramService programService() { - return mock(ProgramService.class); + @MockBean(BrAPIEndpointProvider.class) + BrAPIEndpointProvider brAPIEndpointProvider() { + return spy(new BrAPIEndpointProvider()); } - @MockBean(BrAPIDAOUtil.class) - BrAPIDAOUtil brAPIDAOUtil() { - return spy(new BrAPIDAOUtil(1000, Duration.of(10, ChronoUnit.MINUTES), 1000, 100, 65000, programService())); + @MockBean(BrAPISampleDAO.class) + BrAPISampleDAO sampleDAO() { + return mock(BrAPISampleDAO.class); } @MockBean(SimpleStorageService.class) @@ -210,9 +198,17 @@ SimpleStorageService simpleStorageService() { return spy(new DefaultSimpleStorageService(bucketName, s3Client, presigner)); } - @MockBean(BrAPIEndpointProvider.class) - BrAPIEndpointProvider brAPIEndpointProvider() { - return spy(new BrAPIEndpointProvider()); + // @MockBean cannot replace SampleSubmissionDAO here because it extends generated jOOQ DAO code; + // Scope this replacement to this spec so it does not affect other test contexts. + @Factory + @Requires(property = "micronaut.test.active.spec", value = "org.breedinginsight.services.geno.impl.GigwaGenotypeServiceImplIntegrationTest") + static class SampleSubmissionTestFactory { + + @Singleton + @Replaces(SampleSubmissionDAO.class) + SampleSubmissionDAO sampleSubmissionDAO() { + return mock(SampleSubmissionDAO.class); + } } private GenericContainer gigwa; @@ -306,11 +302,11 @@ public void teardown() { public void testUpload() throws ApiException, AuthorizationException { UUID programId = UUID.fromString("360766b8-480b-4b0a-862c-7eaa651dda28"); String programKey = "TEST"; - UUID expId = UUID.randomUUID(); + UUID submissionId = UUID.randomUUID(); UUID importId = UUID.randomUUID(); - assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> uploadGenoData(programId, programKey, expId, importId), "Upload did not complete within the time period"); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> uploadGenoData(programId, programKey, submissionId, importId), "Upload did not complete within the time period"); - assertTrue(storageService.exists(storageService.getDefaultBucketName(), programId + "/" + expId + "/" + importId + ".vcf"), "File was not uploaded to s3"); + assertTrue(storageService.exists(storageService.getDefaultBucketName(), programId + "/" + submissionId + "/" + importId + ".vcf"), "File was not uploaded to s3"); BrAPIClient brAPIClient = new BrAPIClient(gigwaHost + "gigwa/rest/brapi/v2"); Authentication authorizationToken = brAPIClient.getAuthentication("AuthorizationToken"); @@ -339,7 +335,7 @@ public void testUpload() throws ApiException, AuthorizationException { .getData() .stream() .filter(brAPIStudy -> brAPIStudy.getStudyName() - .equals(expId.toString())) + .equals(submissionId.toString())) .count()); } catch (ApiException e) { System.err.println(e.getMessage()); @@ -348,38 +344,38 @@ public void testUpload() throws ApiException, AuthorizationException { } } + //TODO: Enable in BI-2841 @Test + @Disabled("BI-2841: retrieveGenotypeData should target BrAPI samples directly instead of resolving samples through observation units") public void testFetchGermplasmGenotype() throws AuthorizationException, ApiException, DoesNotExistException { UUID programId = UUID.fromString("8b667063-480b-4b0a-862c-7eaa651dda28"); String programKey = "TESTFETCH"; - UUID expId = UUID.randomUUID(); + UUID submissionId = UUID.randomUUID(); UUID importId = UUID.randomUUID(); - assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> uploadGenoData(programId, programKey, expId, importId), "Upload did not complete within the time period"); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> uploadGenoData(programId, programKey, submissionId, importId), "Upload did not complete within the time period"); BrAPIGermplasm germplasm = new BrAPIGermplasm().germplasmDbId(UUID.randomUUID().toString()).germplasmName("Test Germ"); - BrAPITrial trial = new BrAPITrial().externalReferences(List.of(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) - .referenceID(UUID.randomUUID() - .toString()))); - - - doReturn(List.of(trial)).when(trialDAO).getTrials(any(UUID.class)); + SamplesApi mockSamplesApi = spy(new SamplesApi()); + BrAPISample sample = new BrAPISample().sampleName("USDAMSP1_A01") + .germplasmDbId(germplasm.getGermplasmDbId()); + doReturn(new ApiResponse<>(200, + new HashMap<>(), + Pair.of(Optional.of(new BrAPISampleListResponse().result(new BrAPISampleListResponseResult().data(List.of(sample)))), + Optional.empty()))) + .when(mockSamplesApi).searchSamplesPost(any(BrAPISampleSearchRequest.class)); - doAnswer(invocation -> { - Object searchObject = invocation.getArgument(2); - if(searchObject instanceof BrAPIObservationUnitSearchRequest) { - return List.of(new BrAPIObservationUnit().observationUnitName("USDAMSP1_A01")); - } else { - return invocation.callRealMethod(); - } - }).when(brAPIDAOUtil) - .search(any(Function.class), any(Function3.class), any()); + doReturn(mockSamplesApi).when(brAPIEndpointProvider).get(any(BrAPIClient.class), eq(SamplesApi.class)); doReturn(new BrAPIClient("", 300000)).when(programDAO).getCoreClient(any(UUID.class)); doReturn(new BrAPIClient("", 300000)).when(programDAO).getPhenoClient(any(UUID.class)); GermplasmGenotype germplasmGenotype = gigwaGenoStorageService.retrieveGenotypeData(programId, germplasm); + verify(brAPIEndpointProvider, never()).get(any(BrAPIClient.class), eq(ObservationUnitsApi.class)); + verify(mockSamplesApi).searchSamplesPost(argThat(searchRequest -> searchRequest.getGermplasmDbIds() != null && + searchRequest.getGermplasmDbIds().contains(germplasm.getGermplasmDbId()) && + (searchRequest.getObservationUnitDbIds() == null || searchRequest.getObservationUnitDbIds().isEmpty()))); assertNotNull(germplasmGenotype); assertFalse(germplasmGenotype.getCalls().isEmpty()); assertFalse(germplasmGenotype.getCallSets().isEmpty()); @@ -390,7 +386,7 @@ public void testFetchGermplasmGenotype() throws AuthorizationException, ApiExcep public void testSubmitValidFile() throws IOException, ApiException { UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956"); String programKey = "TESTSUBMITVALID"; - UUID expId = UUID.randomUUID(); + UUID submissionId = UUID.randomUUID(); Scanner sc = new Scanner(new FileInputStream("src/test/resources/files/geno/sample.vcf"), "UTF-8"); String[] headerParts = null; @@ -404,21 +400,14 @@ public void testSubmitValidFile() throws IOException, ApiException { } assertTrue(foundHeader, "Could not find sample.vcf header file"); - List ous = new ArrayList<>(); + List samples = new ArrayList<>(); for(int i = 9; i < headerParts.length; i++) { - ous.add(new BrAPIObservationUnit().observationUnitName(headerParts[i] + " ["+programKey+"-"+(i-7)+"]")); + samples.add(new BrAPISample().sampleName(headerParts[i])); } - setupMocksForSubmitGenoData(expId, ous); - - doAnswer(invocation -> { - if(invocation.getArgument(2) instanceof BrAPIObservationUnitSearchRequest) { - return invocation.callRealMethod(); - } - return invocation.getMock(); - }).when(brAPIDAOUtil).search(any(Function.class), any(Function3.class), any()); + setupMocksForSubmitGenoData(programId, submissionId, samples); AtomicReference importResponse = new AtomicReference<>(); - assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, expId, "sample.vcf")), "Upload did not complete within the time period"); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample.vcf")), "Upload did not complete within the time period"); ImportResponse response = importResponse.get(); assertNotNull(response); @@ -430,10 +419,10 @@ public void testSubmitValidFile() throws IOException, ApiException { public void testSubmitInvalidHeader() throws ApiException { UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956"); String programKey = "TESTSUBMITINVALID"; - UUID expId = UUID.randomUUID(); - setupMocksForSubmitGenoData(expId, Collections.emptyList()); + UUID submissionId = UUID.randomUUID(); + setupMocksForSubmitGenoData(programId, submissionId, Collections.emptyList()); AtomicReference importResponse = new AtomicReference<>(); - assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, expId, "sample_invalid.vcf")), "Upload did not complete within the time period"); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_invalid.vcf")), "Upload did not complete within the time period"); ImportResponse response = importResponse.get(); assertNotNull(response); @@ -443,43 +432,35 @@ public void testSubmitInvalidHeader() throws ApiException { } @Test - public void testSubmitMissingOUs() throws ApiException { + public void testSubmitMissingSubmissionSamples() throws ApiException { UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956"); String programKey = "TESTSUBMITMISSINGOU"; - UUID expId = UUID.randomUUID(); + UUID submissionId = UUID.randomUUID(); - setupMocksForSubmitGenoData(expId, Collections.emptyList()); + setupMocksForSubmitGenoData(programId, submissionId, Collections.emptyList()); AtomicReference importResponse = new AtomicReference<>(); - assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, expId, "sample.vcf")), "Upload did not complete within the time period"); + assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample.vcf")), "Upload did not complete within the time period"); ImportResponse response = importResponse.get(); assertNotNull(response); assertNotNull(response.getProgress()); assertEquals((short)HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode()); - assertEquals("There are samples that do not have an existing observation unit", response.getProgress().getMessage()); + assertEquals("There are samples that are not linked to the selected submission", response.getProgress().getMessage()); } - private void setupMocksForSubmitGenoData(UUID expId, List ous) throws ApiException { - TrialsApi mockTrialsApi = spy(new TrialsApi()); - doReturn(new ApiResponse(200, - new HashMap<>(), - new BrAPITrialListResponse().result(new BrAPITrialListResponseResult().addDataItem(new BrAPITrial().trialDbId(expId.toString()))))) - .when(mockTrialsApi).trialsGet(any(TrialQueryParams.class)); - - doReturn(mockTrialsApi).when(brAPIEndpointProvider).get(any(BrAPIClient.class), eq(TrialsApi.class)); - - ObservationUnitsApi mockOUsApi = spy(new ObservationUnitsApi()); - doReturn(new ApiResponse<>(200, - new HashMap<>(), - Pair.of(Optional.of(new BrAPIObservationUnitListResponse().result(new BrAPIObservationUnitListResponseResult().data(ous))), - Optional.empty()))) - .when(mockOUsApi).searchObservationunitsPost(any(BrAPIObservationUnitSearchRequest.class)); + private void setupMocksForSubmitGenoData(UUID programId, UUID submissionId, List samples) throws ApiException { + SampleSubmission submission = new SampleSubmission(); + submission.setId(submissionId); + submission.setProgramId(programId); - doReturn(mockOUsApi).when(brAPIEndpointProvider).get(any(BrAPIClient.class), eq(ObservationUnitsApi.class)); + doReturn(List.of(submission)).when(sampleSubmissionDAO) + .getBySubmissionId(any(Program.class), eq(submissionId)); + doReturn(samples).when(sampleDAO) + .readSamplesBySubmissionIds(any(Program.class), eq(List.of(submissionId.toString()))); } - private void uploadGenoData(UUID programId, String programKey, UUID expId, UUID importId) throws AuthorizationException, MimeTypeException, IOException, ApiException { + private void uploadGenoData(UUID programId, String programKey, UUID submissionId, UUID importId) throws AuthorizationException, MimeTypeException, IOException, ApiException { Program program = Program.builder() .id(programId) .key(programKey) @@ -526,11 +507,11 @@ private void uploadGenoData(UUID programId, String programKey, UUID expId, UUID .build(); System.out.println("====================== program ID: " + program.getId() + " ==============="); - System.out.println("=================== experiment ID: " + expId + " ==============="); - gigwaGenoStorageService.processSubmission(gigwaGenoStorageService.getAuthToken(), program, expId, new TestFileUpload("src/test/resources/files/geno/sample.vcf", MediaType.of("application/vcard")).getBytes(), "sample.vcf", importUpload, progress); + System.out.println("=================== submission ID: " + submissionId + " ==============="); + gigwaGenoStorageService.processSubmission(gigwaGenoStorageService.getAuthToken(), program, submissionId, new TestFileUpload("src/test/resources/files/geno/sample.vcf", MediaType.of("application/vcard")).getBytes(), "sample.vcf", importUpload, progress); } - private ImportResponse submitGenoData(UUID programId, String programKey, UUID expId, String file) throws AuthorizationException, IOException, ApiException, DoesNotExistException { + private ImportResponse submitGenoData(UUID programId, String programKey, UUID submissionId, String file) throws AuthorizationException, IOException, ApiException, DoesNotExistException { Program program = Program.builder() .id(programId) .key(programKey) @@ -562,8 +543,8 @@ private ImportResponse submitGenoData(UUID programId, String programKey, UUID ex doReturn(new BrAPIClient("", 300000)).when(programDAO).getPhenoClient(any(UUID.class)); System.out.println("====================== program ID: " + program.getId() + " ==============="); - System.out.println("=================== experiment ID: " + expId + " ==============="); - return gigwaGenoStorageService.submitGenotypeData(user.getId(), programId, expId, new TestFileUpload("src/test/resources/files/geno/"+file, MediaType.of("application/vcard"))); + System.out.println("=================== submission ID: " + submissionId + " ==============="); + return gigwaGenoStorageService.submitGenotypeData(user.getId(), programId, submissionId, new TestFileUpload("src/test/resources/files/geno/"+file, MediaType.of("application/vcard"))); } private class TestFileUpload implements CompletedFileUpload {