Skip to content

BI-2839 - Submission-Scoped Genotype Upload API #515

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
HMS17 merged 4 commits into from
May 12, 2026
Merged

BI-2839 - Submission-Scoped Genotype Upload API #515

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ae77ecc
codex first pass
nickpalladino Apr 29, 2026
0c6a047
Merge branch 'develop' into feature/BI-2839
nickpalladino May 1, 2026
88ffab8
Clean up tests
nickpalladino May 5, 2026
2d10f92
Merge branch 'develop' into feature/BI-2839
nickpalladino May 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/main/java/org/breedinginsight/api/v1/controller/geno/GenotypeDataUploadController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,15 @@ public GenotypeDataUploadController(GenotypeService genoService, SecurityService
this.securityService = securityService;
}

@Post("programs/{programId}/experiments/{experimentId}/geno/import")
@Post("programs/{programId}/submissions/{submissionId}/geno/import")
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@AddMetadata
@ProgramSecured(roles = {ProgramSecuredRole.PROGRAM_ADMIN})
public HttpResponse<Response<ImportResponse>> uploadData(@PathVariable UUID programId, @PathVariable UUID experimentId, @Part("file") CompletedFileUpload upload) {
public HttpResponse<Response<ImportResponse>> uploadData(@PathVariable UUID programId, @PathVariable UUID submissionId, @Part("file") CompletedFileUpload upload) {
AuthenticatedUser actingUser = securityService.getUser();
try {
ImportResponse result = genoService.submitGenotypeData(actingUser.getId(), programId, experimentId, upload);
ImportResponse result = genoService.submitGenotypeData(actingUser.getId(), programId, submissionId, upload);
Response<ImportResponse> response = new Response<>(result);
return HttpResponse.ok(response);
} catch (DoesNotExistException e) {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/breedinginsight/services/geno/GenotypeService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import java.util.UUID;

public interface GenotypeService {
ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID experimentId, CompletedFileUpload uploadedFile) throws DoesNotExistException, AuthorizationException, ApiException;
ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID submissionId, CompletedFileUpload uploadedFile) throws DoesNotExistException, AuthorizationException, ApiException;

GermplasmGenotype retrieveGenotypeData(UUID programId, BrAPIGermplasm germplasm) throws DoesNotExistException, AuthorizationException, ApiException;
}
104 changes: 35 additions & 69 deletions src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,13 @@
import org.brapi.client.v2.auth.OAuth;
import org.brapi.client.v2.model.exceptions.ApiException;
import org.brapi.client.v2.model.queryParams.core.ProgramQueryParams;
import org.brapi.client.v2.model.queryParams.core.TrialQueryParams;
import org.brapi.client.v2.modules.core.ProgramsApi;
import org.brapi.client.v2.modules.core.TrialsApi;
import org.brapi.client.v2.modules.genotype.CallSetsApi;
import org.brapi.client.v2.modules.genotype.CallsApi;
import org.brapi.client.v2.modules.genotype.SamplesApi;
import org.brapi.client.v2.modules.genotype.VariantsApi;
import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi;
import org.brapi.v2.model.BrAPIExternalReference;
import org.brapi.v2.model.core.BrAPITrial;
import org.brapi.v2.model.core.response.BrAPIProgramListResponse;
import org.brapi.v2.model.core.response.BrAPITrialListResponse;
import org.brapi.v2.model.geno.BrAPICall;
import org.brapi.v2.model.geno.BrAPICallSet;
import org.brapi.v2.model.geno.BrAPISample;
Expand All @@ -42,14 +37,15 @@
import org.brapi.v2.model.pheno.BrAPIObservationUnit;
import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest;
import org.breedinginsight.brapi.v1.controller.BrapiVersion;
import org.breedinginsight.brapps.importer.daos.BrAPISampleDAO;
import org.breedinginsight.brapps.importer.daos.ImportDAO;
import org.breedinginsight.brapps.importer.daos.ImportMappingDAO;
import org.breedinginsight.brapps.importer.model.ImportProgress;
import org.breedinginsight.brapps.importer.model.ImportUpload;
import org.breedinginsight.brapps.importer.model.mapping.ImportMapping;
import org.breedinginsight.brapps.importer.model.response.ImportResponse;
import org.breedinginsight.brapps.importer.services.ExternalReferenceSource;
import org.breedinginsight.daos.ProgramDAO;
import org.breedinginsight.daos.SampleSubmissionDAO;
import org.breedinginsight.daos.UserDAO;
import org.breedinginsight.model.GermplasmGenotype;
import org.breedinginsight.model.Program;
Expand Down Expand Up @@ -100,6 +96,8 @@ public class GigwaGenotypeServiceImpl implements GenotypeService {
private final ProgramDAO programDAO;
private final UserDAO userDAO;
private final ImportDAO importDAO;
private final SampleSubmissionDAO sampleSubmissionDAO;
private final BrAPISampleDAO sampleDAO;

private final ImportMappingDAO importMappingDAO;
private final SimpleStorageService storageService;
Expand All @@ -122,6 +120,8 @@ public GigwaGenotypeServiceImpl(@Property(name = "gigwa.host") String gigwaHost,
ProgramDAO programDAO,
UserDAO userDAO,
ImportDAO importDAO,
SampleSubmissionDAO sampleSubmissionDAO,
BrAPISampleDAO sampleDAO,
ImportMappingDAO importMappingDAO,
@Named("genotype") SimpleStorageService storageService,
S3Client s3Client,
Expand All @@ -137,6 +137,8 @@ public GigwaGenotypeServiceImpl(@Property(name = "gigwa.host") String gigwaHost,
this.programDAO = programDAO;
this.userDAO = userDAO;
this.importDAO = importDAO;
this.sampleSubmissionDAO = sampleSubmissionDAO;
this.sampleDAO = sampleDAO;
this.importMappingDAO = importMappingDAO;
this.storageService = storageService;
this.s3Client = s3Client;
Expand All @@ -147,7 +149,7 @@ public GigwaGenotypeServiceImpl(@Property(name = "gigwa.host") String gigwaHost,
}

@Override
public ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID experimentId, CompletedFileUpload uploadedFile) throws DoesNotExistException, AuthorizationException, ApiException {
public ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID submissionId, CompletedFileUpload uploadedFile) throws DoesNotExistException, AuthorizationException, ApiException {
Program program = getProgram(programId);

User user = userDAO.getUser(userId)
Expand Down Expand Up @@ -204,10 +206,10 @@ public ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID exper

try {
byte[] fileContents = uploadedFile.getBytes();
if(validateSamples(program, experimentId, fileContents, upload, gigwaAuthToken)) {
if(validateSamples(program, submissionId, fileContents, upload)) {
executor.execute(() -> {
try {
processSubmission(gigwaAuthToken, program, experimentId, fileContents, uploadedFile.getFilename(), upload, progress);
processSubmission(gigwaAuthToken, program, submissionId, fileContents, uploadedFile.getFilename(), upload, progress);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
Expand Down Expand Up @@ -261,18 +263,13 @@ public GermplasmGenotype retrieveGenotypeData(UUID programId, BrAPIGermplasm ger
}
}

private boolean validateSamples(Program program, UUID experimentId, byte[] fileContents, ImportUpload upload, String gigwaAuthToken) throws DoesNotExistException, ApiException {
log.debug("Validating samples in submitted VCF file for experiment: " + experimentId);
private boolean validateSamples(Program program, UUID submissionId, byte[] fileContents, ImportUpload upload) throws DoesNotExistException, ApiException {
log.debug("Validating samples in submitted VCF file for submission: " + submissionId);

BrAPIClient brAPIClient = programDAO.getCoreClient(program.getId());
brAPIClient.setBasePath(gigwaHost + GIGWA_BRAPI_BASE_PATH);
Authentication authorizationToken = brAPIClient.getAuthentication("AuthorizationToken");
if(authorizationToken instanceof OAuth) {
((OAuth)authorizationToken).setAccessToken(gigwaAuthToken);
}
BrAPIClient brapiPhenoClient = programDAO.getPhenoClient(program.getId());

Set<String> obsUnitNames = fetchObservationUnits(brapiPhenoClient, experimentId).stream().map(ou -> Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())).collect(Collectors.toSet());
Set<String> submissionSampleNames = fetchSubmissionSamples(program, submissionId).stream()
.map(BrAPISample::getSampleName)
.filter(Objects::nonNull)
.collect(Collectors.toSet());

log.debug("searching for the VCF header row");
String[] headerParts = null;
Expand Down Expand Up @@ -317,17 +314,17 @@ private boolean validateSamples(Program program, UUID experimentId, byte[] fileC
return false;
}

log.debug("pulled all the samples from the VCF, now checking each one has an OU record");
List<String> samplesMissingOu = new ArrayList<>();
log.debug("pulled all the samples from the VCF, now checking each one belongs to the submission");
List<String> samplesMissingSubmission = new ArrayList<>();
samples.forEach(s -> {
if(!obsUnitNames.contains(s)) {
samplesMissingOu.add(s);
if(!submissionSampleNames.contains(s)) {
samplesMissingSubmission.add(s);
}
});

if(!samplesMissingOu.isEmpty()) {
if(!samplesMissingSubmission.isEmpty()) {
upload.getProgress().setStatuscode((short)HttpStatus.BAD_REQUEST.getCode());
upload.getProgress().setMessage("There are samples that do not have an existing observation unit");
upload.getProgress().setMessage("There are samples that are not linked to the selected submission");
importDAO.updateProgress(upload.getProgress());
return false;
}
Expand Down Expand Up @@ -408,43 +405,12 @@ private List<BrAPIObservationUnit> fetchObservationUnits(BrAPIClient phenoBrAPIC
return brAPIDAOUtil.search(observationUnitsApi::searchObservationunitsPost, observationUnitsApi::searchObservationunitsSearchResultsDbIdGet, searchRequest);
}

private List<BrAPIObservationUnit> fetchObservationUnits(BrAPIClient phenoBrAPIClient, UUID experimentId) throws ApiException, DoesNotExistException {
log.debug("fetching observationUnits for experiment: " + experimentId);
TrialsApi trialsApi = brAPIEndpointProvider.get(phenoBrAPIClient, TrialsApi.class);
ApiResponse<BrAPITrialListResponse> brAPITrialListResponseApiResponse = trialsApi.trialsGet(new TrialQueryParams().externalReferenceID(experimentId.toString())
.externalReferenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)));

BrAPITrial brAPITrial = null;
if(brAPITrialListResponseApiResponse.getBody().getResult().getData() != null) {
if (brAPITrialListResponseApiResponse.getBody().getResult().getData().size() == 1) {
brAPITrial = brAPITrialListResponseApiResponse.getBody().getResult().getData().get(0);
} else {
String trialReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS);
for (BrAPITrial trial : brAPITrialListResponseApiResponse.getBody().getResult().getData()) {
if (trial.getExternalReferences() != null) {
Optional<BrAPIExternalReference> xref = trial.getExternalReferences()
.stream()
.filter(externalReference -> externalReference.getReferenceSource().equals(trialReferenceSource))
.findFirst();
if (xref.isPresent() && xref.get().getReferenceID().equals(experimentId.toString())) {
brAPITrial = trial;
break;
}
}
}
}
private List<BrAPISample> fetchSubmissionSamples(Program program, UUID submissionId) throws ApiException, DoesNotExistException {
if(sampleSubmissionDAO.getBySubmissionId(program, submissionId).isEmpty()) {
throw new DoesNotExistException("Could not find sample submission in database");
}

if(brAPITrial != null) {
ObservationUnitsApi observationUnitsApi = brAPIEndpointProvider.get(phenoBrAPIClient, ObservationUnitsApi.class);

BrAPIObservationUnitSearchRequest searchRequest = new BrAPIObservationUnitSearchRequest();
searchRequest.addTrialDbIdsItem(brAPITrial.getTrialDbId());

return brAPIDAOUtil.search(observationUnitsApi::searchObservationunitsPost, observationUnitsApi::searchObservationunitsSearchResultsDbIdGet, searchRequest);
} else {
throw new DoesNotExistException("Could not find experiment in database");
}
return sampleDAO.readSamplesBySubmissionIds(program, List.of(submissionId.toString()));
}

private List<BrAPICallSet> fetchCallsets(BrAPIClient genoBrAPIClient, List<BrAPISample> germplasmSamples) throws ApiException {
Expand Down Expand Up @@ -500,12 +466,12 @@ private List<BrAPIVariant> fetchVariants(BrAPIClient genoBrAPIClient, List<BrAPI
return brAPIDAOUtil.searchWithToken(variantsApi::searchVariantsPost, variantsApi::searchVariantsSearchResultsDbIdGet, searchRequest);
}

protected void processSubmission(String gigwaAuthToken, Program program, UUID experimentId, byte[] fileContents, String filename, ImportUpload upload, ImportProgress progress) throws MimeTypeException, IOException, ApiException {
protected void processSubmission(String gigwaAuthToken, Program program, UUID submissionId, byte[] fileContents, String filename, ImportUpload upload, ImportProgress progress) throws MimeTypeException, IOException, ApiException {
Pair<String, Long> uploadedFileResult;
try {
progress.setMessage("Uploading file");
importDAO.updateProgress(progress);
uploadedFileResult = uploadGenotypeData(program.getId(), experimentId, upload.getId(), fileContents, filename);
uploadedFileResult = uploadGenotypeData(program.getId(), submissionId, upload.getId(), fileContents, filename);
log.debug("file saved to: " + uploadedFileResult.getLeft());
} catch (Exception e) {
progress.setStatuscode((short) HttpStatus.INTERNAL_SERVER_ERROR.getCode());
Expand All @@ -518,7 +484,7 @@ protected void processSubmission(String gigwaAuthToken, Program program, UUID ex
importDAO.updateProgress(progress);

OkHttpClient client = new OkHttpClient();
String gigwaProgressToken = submitRequestToGigwa(client, program, experimentId, uploadedFileResult.getLeft(), gigwaAuthToken, progress);
String gigwaProgressToken = submitRequestToGigwa(client, program, submissionId, uploadedFileResult.getLeft(), gigwaAuthToken, progress);

if(checkGigwaProgress(client, gigwaAuthToken, gigwaProgressToken, progress)) {
log.debug("Gigwa import was successful!");
Expand Down Expand Up @@ -593,19 +559,19 @@ private boolean checkGigwaProgress(OkHttpClient client, String gigwaAuthToken, S
* Submits the upload request to Gigwa, and returns the progress token
* @param client
* @param program
* @param experimentId
* @param submissionId
* @param fileUrl
* @param gigwaAuthToken
* @param progress
* @return the progress token to check on the import's progress
* @throws IOException
*/
private String submitRequestToGigwa(OkHttpClient client, Program program, UUID experimentId, String fileUrl, String gigwaAuthToken, ImportProgress progress) throws IOException {
private String submitRequestToGigwa(OkHttpClient client, Program program, UUID submissionId, String fileUrl, String gigwaAuthToken, ImportProgress progress) throws IOException {
Request request = new Request.Builder()
.url(HttpUrl.parse(buildPath("gigwa/genotypeImport"))
.newBuilder()
.addQueryParameter("module", program.getKey())
.addQueryParameter("project", experimentId.toString())
.addQueryParameter("project", submissionId.toString())
.addQueryParameter("run", LocalDateTime.now().toString())
.addQueryParameter("dataFile1", fileUrl)

Expand All @@ -631,7 +597,7 @@ private String submitRequestToGigwa(OkHttpClient client, Program program, UUID e
}
}

private Pair<String, Long> uploadGenotypeData(UUID programId, UUID experimentId, UUID uploadId, byte[] fileContents, String filename) throws IOException, MimeTypeException {
private Pair<String, Long> uploadGenotypeData(UUID programId, UUID submissionId, UUID uploadId, byte[] fileContents, String filename) throws IOException, MimeTypeException {
log.debug("saving genotype data to S3");

if(!storageService.listBucketNames().contains(storageService.getDefaultBucketName())) {
Expand All @@ -641,7 +607,7 @@ private Pair<String, Long> uploadGenotypeData(UUID programId, UUID experimentId,

var mimeType = mimeTypeParser.getMimeType(fileContents, filename);

var key = programId.toString() + "/" + experimentId.toString() + "/" + uploadId + mimeType.getExtension();
var key = programId.toString() + "/" + submissionId.toString() + "/" + uploadId + mimeType.getExtension();
var path = storeMultipartFile(key, fileContents, Map.of("originalFileName", filename));

Long fileSize = Long.valueOf(fileContents.length);
Expand Down
Loading
Loading