diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index 5741e1313..5ee72f3a2 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -175,6 +175,29 @@ public HttpResponse>> getDatasets( } + @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/recommended-sub-entity-dataset-names") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) + @Produces(MediaType.APPLICATION_JSON) + public HttpResponse>> getRecommendedSubEntityDatasetNames( + @PathVariable("programId") UUID programId, + @PathVariable("experimentId") UUID experimentId) { + try { + Optional programOptional = programService.getById(programId); + if (programOptional.isEmpty()) { + return HttpResponse.status(HttpStatus.NOT_FOUND, "Program does not exist"); + } + + Response> response = new Response<>(experimentService.getRecommendedSubEntityDatasetNames(programOptional.get(), experimentId)); + return HttpResponse.ok(response); + } catch (DoesNotExistException e) { + log.info(e.getMessage()); + return HttpResponse.status(HttpStatus.NOT_FOUND, e.getMessage()); + } catch (Exception e) { + log.error("Error finding recommended sub-entity dataset names", e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error finding recommended sub-entity dataset names"); + } + } + /** * Adds a record to the experiment_program_user_role table * @param programId The UUID of the program diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java index 258ffbaf6..7cd38ea30 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationLevelDAO.java @@ -17,22 +17,12 @@ package org.breedinginsight.brapi.v2.dao; -import com.google.gson.Gson; -import io.micronaut.http.HttpResponse; -import io.micronaut.http.HttpStatus; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; -import okhttp3.HttpUrl; -import okhttp3.MediaType; -import okhttp3.Request; -import okhttp3.RequestBody; import org.brapi.client.v2.ApiResponse; -import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationLevelNamesApi; -import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; -import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponse; import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponseResult; import org.brapi.v2.model.pheno.response.BrAPIObservationLevelSingleResponse; @@ -40,63 +30,26 @@ import org.breedinginsight.model.DatasetLevel; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.utilities.BrAPIDAOUtil; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import org.breedinginsight.utilities.Utilities; - import javax.inject.Inject; import javax.inject.Singleton; import java.util.List; +import org.breedinginsight.utilities.Utilities; import java.util.Optional; @Slf4j @Singleton public class BrAPIObservationLevelDAO { - private static final MediaType JSON_MEDIA_TYPE = MediaType.get("application/json"); - private final BrAPIDAOUtil brAPIDAOUtil; - private final Gson gson = new JSON().getGson(); - private final BrAPIEndpointProvider brAPIEndpointProvider; private final ProgramDAO programDAO; @Inject - public BrAPIObservationLevelDAO(BrAPIDAOUtil brAPIDAOUtil, - BrAPIEndpointProvider brAPIEndpointProvider, + public BrAPIObservationLevelDAO(BrAPIEndpointProvider brAPIEndpointProvider, ProgramDAO programDAO) { - this.brAPIDAOUtil = brAPIDAOUtil; this.brAPIEndpointProvider = brAPIEndpointProvider; this.programDAO = programDAO; } - public HttpResponse createObservationLevelName(Program program, String levelName, DatasetLevel levelOrder, String programDbId) { - HttpUrl url = HttpUrl.parse(brAPIDAOUtil.getProgramBrAPIBaseUrl(program.getId())) - .newBuilder() - .addPathSegment("observationlevelnames") - .build(); - JsonObject levelJson = new JsonObject(); - levelJson.addProperty("levelName", levelName); - if (levelOrder != null) { - levelJson.addProperty("levelOrder", levelOrder.getValue()); - } - if (programDbId != null) { - levelJson.addProperty("programDbId", programDbId); - } - JsonArray bodyArray = new JsonArray(); - bodyArray.add(levelJson); - RequestBody body = RequestBody.create(gson.toJson(bodyArray), JSON_MEDIA_TYPE); - var request = new Request.Builder() - .url(url) - .post(body) - .addHeader("Content-Type", "application/json") - .build(); - return brAPIDAOUtil.makeCall(request); - } - public BrAPIObservationUnitHierarchyLevel createLevelName(Program program, String programDbId, String levelName, diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 3c115afd9..b15f1c1b0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -34,6 +34,8 @@ import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; import org.breedinginsight.model.*; +import org.breedinginsight.model.delta.DeltaEntityFactory; +import org.breedinginsight.model.delta.Experiment; import org.breedinginsight.services.TraitService; import org.breedinginsight.services.exceptions.AlreadyExistsException; import org.breedinginsight.services.exceptions.DoesNotExistException; @@ -79,6 +81,7 @@ public class BrAPITrialService { private final DistributedLockService lockService; private static final String SHEET_NAME = "Data"; private final DatasetService datasetService; + private final DeltaEntityFactory deltaEntityFactory; private final BrAPIObservationLevelService observationLevelService; @Inject @@ -96,6 +99,7 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin FileMappingUtil fileMappingUtil, DistributedLockService lockService, DatasetService datasetService, + DeltaEntityFactory deltaEntityFactory, BrAPIObservationLevelService observationLevelService) { this.referenceSource = referenceSource; @@ -112,6 +116,7 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin this.fileMappingUtil = fileMappingUtil; this.lockService = lockService; this.datasetService = datasetService; + this.deltaEntityFactory = deltaEntityFactory; this.observationLevelService = observationLevelService; } @@ -430,6 +435,35 @@ public List getDatasetsMetadata(Program program, UUID experimen return datasets; } + /** + * Returns list of recommended sub entity names based on observation levels for the program that exclude + * level names already used in the experiment and is deduplicated for same name at multiple levels + * @param program Program + * @param experimentId Experiment Id + * @return list of dataset name recommendations + * @throws DoesNotExistException If trial does not exist + * @throws ApiException If BrAPI trial retrieval fails + */ + public List getRecommendedSubEntityDatasetNames(Program program, UUID experimentId) throws DoesNotExistException, ApiException { + BrAPITrial experiment = trialDAO.getTrialById(program.getId(), experimentId).orElseThrow(() -> new DoesNotExistException("Trial does not exist")); + Experiment deltaExperiment = deltaEntityFactory.makeExperimentBean(experiment); + // set to eliminate possible duplicates like plant for exp unit and sub unit + Set currentExperimentDatasetNames = deltaExperiment.getDatasetsMetadata() + .stream() + .map(DatasetMetadata::getName) + .filter(Objects::nonNull) + .map(String::toLowerCase) + .collect(Collectors.toSet()); + + return getProgramObservationLevelNames(program).stream() + .filter(Objects::nonNull) + .map(String::toLowerCase) + .filter(name -> !currentExperimentDatasetNames.contains(name)) + .distinct() + .sorted() + .collect(Collectors.toList()); + } + /** * Creates sub-entity dataset * TODO: Handle compensating transactions in event of failure. Currently brapi server does not support @@ -796,6 +830,12 @@ public int deleteExperiment(Program program, UUID experimentId, boolean hard) th return existingObservations.size(); } + private List getProgramObservationLevelNames(Program program) { + String programDbId = program.getBrapiProgram() != null ? program.getBrapiProgram().getProgramDbId() : null; + List levelNames = observationLevelService.getProgrammaticLevelNames(program, programDbId); + return levelNames.stream().map(BrAPIObservationUnitHierarchyLevel::getLevelName).collect(Collectors.toList()); + } + private Map createExportRow( BrAPITrial experiment, Program program, diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 21358e69d..5677dc0b0 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -14,6 +14,7 @@ import io.reactivex.Flowable; import lombok.SneakyThrows; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; @@ -53,6 +54,7 @@ import tech.tablesaw.api.Table; import javax.inject.Inject; import java.io.*; +import java.math.BigDecimal; import java.time.OffsetDateTime; import java.util.*; import java.util.stream.Collectors; @@ -62,7 +64,6 @@ @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ExperimentControllerIntegrationTest extends BrAPITest { private Program program; @@ -213,12 +214,21 @@ void setup() throws Exception { // Create an experiment with no observations. private String uploadExperimentWithoutObs() throws Exception { + return uploadExperimentWithoutObs("Without Obs", "Plot"); + } + + private String uploadExperimentWithoutObs(String title, String expUnit) throws Exception { + return uploadExperimentWithoutObs(program, title, expUnit); + } + + private String uploadExperimentWithoutObs(Program targetProgram, String title, String expUnit) throws Exception { ImportTestUtils importTestUtils = new ImportTestUtils(); List> expRows = new ArrayList<>(); // Make test experiment import. - Map row1 = makeExpImportRow("Without Obs", "NewEnv1"); - Map row2 = makeExpImportRow("Without Obs", "NewEnv2"); + String envBase = title.replaceAll("\\s+", ""); + Map row1 = makeExpImportRow(title, envBase + "1", expUnit); + Map row2 = makeExpImportRow(title, envBase + "2", expUnit); expRows.add(row1); expRows.add(row2); @@ -229,7 +239,7 @@ private String uploadExperimentWithoutObs() throws Exception { null, true, client, - program, + targetProgram, mappingId, newExperimentWorkflowId); String expId = importResult @@ -313,14 +323,14 @@ void downloadDatasets(boolean includeTimestamps, String extension, int numberOfE List> filteredRows = rows.stream() .filter(row -> file.getName().contains(row.get(ExperimentObservation.Columns.ENV).toString())) .collect(Collectors.toList()); - parseAndCheck(fileStream, extension, true, filteredRows, includeTimestamps, expectedColNumber); + parseAndCheck(fileStream, extension, true, filteredRows, includeTimestamps, expectedColNumber, "Plot ObsUnitID", filteredRows.size()); } } else { assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); // All (both) rows when 0 or 2 envs sent, first row when 1 env sent as query param. List> filteredRows = numberOfEnvsRequested == 1 ? List.of(rows.get(0)) : rows; - parseAndCheck(bodyStream, extension, numberOfEnvsRequested > 0, filteredRows, includeTimestamps, expectedColNumber); + parseAndCheck(bodyStream, extension, numberOfEnvsRequested > 0, filteredRows, includeTimestamps, expectedColNumber, "Plot ObsUnitID", filteredRows.size()); } // Remove temp directory after each test run. FileUtils.deleteDirectory(new File(tempDir)); @@ -333,14 +343,19 @@ void downloadDatasets(boolean includeTimestamps, String extension, int numberOfE @ParameterizedTest @CsvSource(value = {"CSV", "XLSX", "XLS"}) @SneakyThrows - @Disabled // disabled for now until we re-enable subentity support void downloadSubEntityDataset(String extension) { + Program subEntityProgram = createSeededProgram("SubEntity Download"); + String subEntityExperimentTitle = "SubEntity Download " + extension; + List> topLevelRows = buildObservedRows(subEntityExperimentTitle); + String subEntityExperimentId = uploadExperimentWithObs(subEntityProgram, subEntityExperimentTitle, topLevelRows); + String plantDatasetName = "plant" + extension.toLowerCase(Locale.ROOT); + String plantObservationLevel = StringUtils.capitalize(plantDatasetName); // Create sub-entity dataset. Flowable> postCall = client.exchange( POST(String.format("/programs/%s/experiments/%s/dataset", - program.getId().toString(), experimentId), - "{\"name\":\"Plant\",\"repeatedMeasures\":3}") + subEntityProgram.getId().toString(), subEntityExperimentId), + String.format("{\"name\":\"%s\",\"repeatedMeasures\":3}", plantDatasetName)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class); HttpResponse postResponse = postCall.blockingFirst(); @@ -349,11 +364,11 @@ void downloadSubEntityDataset(String extension) { assertEquals(HttpStatus.OK, postResponse.getStatus()); // Get top-level datasetId to include in export request. - BrAPITrial experiment = experimentService.getTrialDataByUUID(program.getId(), UUID.fromString(experimentId), false); + BrAPITrial experiment = experimentService.getTrialDataByUUID(subEntityProgram.getId(), UUID.fromString(subEntityExperimentId), false); String topLevelDatasetId = DatasetUtil.getTopLevelDataset(experiment).getId().toString(); Flowable> topLevelExportCall = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", - program.getId().toString(), experimentId, extension, topLevelDatasetId)) + subEntityProgram.getId().toString(), subEntityExperimentId, extension, topLevelDatasetId)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class ); HttpResponse topLevelResponse = topLevelExportCall.blockingFirst(); @@ -371,13 +386,22 @@ void downloadSubEntityDataset(String extension) { // Check file contents. ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(topLevelResponse.body())); - parseAndCheck(bodyStream, extension, false, rows, false, 25); + parseAndCheck( + bodyStream, + extension, + false, + topLevelRows, + false, + getExpectedExportColumnCount(false, traits.size(), false), + "Plot ObsUnitID", + topLevelRows.size() + ); // Make sub-entity dataset export request. - String plantDatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), "Plant"); + String plantDatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), plantDatasetName); Flowable> plantExportCall = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", - program.getId().toString(), experimentId, extension, plantDatasetId)) + subEntityProgram.getId().toString(), subEntityExperimentId, extension, plantDatasetId)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class ); HttpResponse plantResponse = plantExportCall.blockingFirst(); @@ -389,11 +413,89 @@ void downloadSubEntityDataset(String extension) { assertEquals(mediaTypeByExtension.get(extension), plantResponse.getHeaders().getContentType().orElseThrow(Exception::new)); // The expected contents of the exported Plant dataset (3 sub-obs units for each top-level unit were requested). - List> plantRows = buildSubEntityRows(rows, "Plant", 3); + List> plantRows = buildSubEntityRows(topLevelRows, plantObservationLevel, 3); // Check file contents. ByteArrayInputStream plantBodyStream = new ByteArrayInputStream(Objects.requireNonNull(plantResponse.body())); - parseAndCheck(plantBodyStream, extension, false, plantRows, false, 23); + parseAndCheck( + plantBodyStream, + extension, + false, + plantRows, + false, + getExpectedExportColumnCount(true, experimentService.getDatasetObsVars(plantDatasetId, subEntityProgram).size(), false), + plantObservationLevel + " ObsUnitID", + plantRows.size() + ); + } + + @Test + public void createSubEntityDatasetRejectsExpUnitNameAlreadyUsedInSameExperiment() throws Exception { + Program testProgram = createSeededProgram("Reject Already Used"); + String plantExperimentId = uploadExperimentWithoutObs(testProgram, "Plant Same Experiment", "Plant"); + + Flowable> call = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", testProgram.getId(), plantExperimentId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + + HttpClientResponseException e = assertThrows(HttpClientResponseException.class, call::blockingFirst); + assertEquals(HttpStatus.CONFLICT, e.getStatus()); + } + + @Test + public void createSubEntityDatasetAllowsExpUnitNameUsedInOtherExperiment() throws Exception { + Program testProgram = createSeededProgram("Allow Other Experiment"); + uploadExperimentWithoutObs(testProgram, "Plant Source Experiment", "Plant"); + String recipientExperimentId = uploadExperimentWithoutObs(testProgram, "Plot Recipient Experiment", "Plot"); + + Flowable> call = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", testProgram.getId(), recipientExperimentId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + } + + @Test + public void recommendedSubEntityDatasetNamesIncludeExpUnitNamesFromOtherExperiments() throws Exception { + Program testProgram = createSeededProgram("Recommended Names"); + uploadExperimentWithoutObs(testProgram, "Plant Autocomplete Source", "Plant"); + String recipientExperimentId = uploadExperimentWithoutObs(testProgram, "Autocomplete Recipient", "Plot"); + + List recommendedNames = getRecommendedSubEntityDatasetNames(testProgram, recipientExperimentId); + + assertTrue(recommendedNames.stream().anyMatch(name -> name.equalsIgnoreCase("plant"))); + assertFalse(recommendedNames.stream().anyMatch(name -> name.equalsIgnoreCase("plot"))); + } + + @Test + public void recommendedSubEntityDatasetNamesDeDuplicateExpUnitAndSubUnitNamesAcrossExperiments() throws Exception { + Program testProgram = createSeededProgram("Dedup Across Experiments"); + uploadExperimentWithoutObs(testProgram, "Plant Exp Unit Source", "Plant"); + String subEntitySourceExperimentId = uploadExperimentWithoutObs(testProgram, "Plant Sub Unit Source", "Plot"); + String recipientExperimentId = uploadExperimentWithoutObs(testProgram, "Plant Unique Recipient", "Plot"); + + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", testProgram.getId(), subEntitySourceExperimentId), + "{\"name\":\"Plant\",\"repeatedMeasures\":2}") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + HttpResponse postResponse = postCall.blockingFirst(); + assertEquals(HttpStatus.OK, postResponse.getStatus()); + + List recommendedNames = getRecommendedSubEntityDatasetNames(testProgram, recipientExperimentId); + + assertEquals(1L, recommendedNames.stream().filter(name -> name.equalsIgnoreCase("plant")).count()); } /** @@ -752,21 +854,20 @@ public void deleteExperimentInvalid() { @CsvSource(value = {"true,true", "false,true", "true,false", "false,false"}) @SneakyThrows public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations) { + Program deleteProgram = createSeededProgram("Delete Experiment"); + String deleteExperimentTitle = "Delete Experiment " + UUID.randomUUID(); // Set up a test trial and get the trialDbId. String trialDbId; if (withObservations) { - JsonArray beforeData = getProgramTrials(program.getId().toString()); - - // The trial created by setup has observations. - trialDbId = beforeData.get(0).getAsJsonObject().get("trialDbId").getAsString(); + trialDbId = uploadExperimentWithObs(deleteProgram, deleteExperimentTitle, buildObservedRows(deleteExperimentTitle)); } else { // Create a trial without observations. - trialDbId = uploadExperimentWithoutObs(); + trialDbId = uploadExperimentWithoutObs(deleteProgram, deleteExperimentTitle, "Plot"); } // A DELETE request should delete an experiment with observations unless there are observations and hardDelete = true. Flowable> deleteCall = client.exchange( - DELETE(String.format("/programs/%s/experiments/%s?hard=%s", program.getId().toString(), trialDbId, hardDelete)) + DELETE(String.format("/programs/%s/experiments/%s?hard=%s", deleteProgram.getId().toString(), trialDbId, hardDelete)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class ); @@ -780,15 +881,15 @@ public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations assertEquals(HttpStatus.CONFLICT, e.getStatus()); // Check that the trial was not deleted. - JsonArray trials = getProgramTrials(program.getId().toString()); + JsonArray trials = getProgramTrials(deleteProgram.getId().toString()); assertEquals(1, trials.size()); // Check that the studies were not deleted. - JsonArray studies = getProgramStudies(program.getId().toString()); + JsonArray studies = getProgramStudies(deleteProgram.getId().toString()); assertEquals(2, studies.size()); // Check that lists were not deleted. - JsonArray lists = getProgramObsVarLists(program.getId().toString()); + JsonArray lists = getProgramObsVarLists(deleteProgram.getId().toString()); assertEquals(1, lists.size()); } else { HttpResponse deleteResponse = deleteCall.blockingFirst(); @@ -796,20 +897,55 @@ public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations assertEquals(HttpStatus.NO_CONTENT, deleteResponse.getStatus()); // Check that the trial was deleted. - JsonArray trials = getProgramTrials(program.getId().toString()); + JsonArray trials = getProgramTrials(deleteProgram.getId().toString()); assertEquals(0, trials.size()); // Check that the studies were deleted. - JsonArray studies = getProgramStudies(program.getId().toString()); + JsonArray studies = getProgramStudies(deleteProgram.getId().toString()); assertEquals(0, studies.size()); // Check that the BrAPI lists were deleted. - JsonArray lists = getProgramObsVarLists(program.getId().toString()); + JsonArray lists = getProgramObsVarLists(deleteProgram.getId().toString()); assertEquals(0, lists.size()); } } + private List> buildObservedRows(String title) { + List> observedRows = new ArrayList<>(); + String envBase = title.replaceAll("\\s+", ""); + Map row1 = makeExpImportRow(title, envBase + "1"); + Map row2 = makeExpImportRow(title, envBase + "2"); + + for (int i = 0; i < traits.size(); i++) { + row1.put(traits.get(i).getObservationVariableName(), (float) (i + 1)); + } + + observedRows.add(row1); + observedRows.add(row2); + return observedRows; + } + + private String uploadExperimentWithObs(Program targetProgram, String title, List> expRows) throws Exception { + ImportTestUtils importTestUtils = new ImportTestUtils(); + + JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( + importTestUtils.writeExperimentDataToFile(expRows, traits, false, false, null), + null, + true, + client, + targetProgram, + mappingId, + newExperimentWorkflowId); + + return importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + } + private List> buildSubEntityRows(List> topLevelRows, String entityName, int repeatedMeasures) { List> plantRows = new ArrayList<>(); for (Map row : topLevelRows) { @@ -817,8 +953,8 @@ private List> buildSubEntityRows(List> t // Deep copy map entries. Map plantRow = new HashMap<>(row); - plantRow.put("Exp Unit", entityName); - plantRow.put("Exp Unit ID", i.toString()); + plantRow.put(ExperimentObservation.Columns.SUB_OBS_UNIT, entityName); + plantRow.put(ExperimentObservation.Columns.SUB_UNIT_ID, i.toString()); plantRow.remove("tt_test_1"); plantRow.remove("tt_test_2"); plantRows.add(plantRow); @@ -827,6 +963,14 @@ private List> buildSubEntityRows(List> t return plantRows; } + private int getExpectedExportColumnCount(boolean isSubEntity, int obsVarCount, boolean includeTimestamps) { + int baseColumnCount = isSubEntity + ? ExperimentFileColumns.getOrderedColumnsSubEntity().size() + 2 + : ExperimentFileColumns.getOrderedColumns().size() + 1; + int timestampColumnCount = includeTimestamps ? obsVarCount : 0; + return baseColumnCount + obsVarCount + timestampColumnCount; + } + private File writeDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); @@ -845,12 +989,76 @@ private File writeDataToFile(List> data, List traits) return file; } + private List getRecommendedSubEntityDatasetNames(String targetExperimentId) { + return getRecommendedSubEntityDatasetNames(program, targetExperimentId); + } + + private List getRecommendedSubEntityDatasetNames(Program targetProgram, String targetExperimentId) { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/experiments/%s/recommended-sub-entity-dataset-names", + targetProgram.getId(), targetExperimentId)) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + String.class + ); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonArray result = JsonParser.parseString(Objects.requireNonNull(response.body())) + .getAsJsonObject() + .getAsJsonArray("result"); + + List recommendedNames = new ArrayList<>(); + result.forEach(name -> recommendedNames.add(name.getAsString())); + return recommendedNames; + } + + private Program createSeededProgram(String prefix) throws Exception { + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + String suffix = UUID.randomUUID().toString().replaceAll("[^a-fA-F]", "").toUpperCase(Locale.ROOT); + while (suffix.length() < 4) { + suffix += UUID.randomUUID().toString().replaceAll("[^a-fA-F]", "").toUpperCase(Locale.ROOT); + } + suffix = suffix.substring(0, 4); + ProgramRequest programRequest = ProgramRequest.builder() + .name(prefix + " " + suffix) + .abbreviation("OT" + suffix) + .documentationUrl("localhost:8080") + .objective("To test ordered experiment scenarios") + .species(SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build()) + .key("OT" + suffix) + .build(); + Program seededProgram = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), seededProgram.getId()); + + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + ontologyService.createTraits(seededProgram.getId(), createTraits(2), user, false); + + List germplasm = createGermplasm(1); + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + newReference.setReferenceID(seededProgram.getId().toString()); + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + germplasmDAO.createBrAPIGermplasm(germplasm, seededProgram.getId(), null); + + return seededProgram; + } + private Map makeExpImportRow(String title, String environment) { + return makeExpImportRow(title, environment, "Plot"); + } + + private Map makeExpImportRow(String title, String environment, String expUnit) { Map row = new HashMap<>(); row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); row.put(ExperimentObservation.Columns.EXP_TITLE, title); - row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_UNIT, expUnit); //row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, ""); row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); row.put(ExperimentObservation.Columns.ENV, environment); @@ -927,7 +1135,9 @@ private void parseAndCheck(InputStream stream, boolean requestEnv, List> rows, boolean includeTimestamps, - Integer expectedColNumber) throws ParsingException { + Integer expectedColNumber, + String expectedObsUnitIdColumn, + Integer expectedObsUnitIdUniqueCount) throws ParsingException { Table download = Table.create(); if (extension.equals("CSV")) { download = FileUtil.parseTableFromCsv(stream); @@ -937,7 +1147,16 @@ private void parseAndCheck(InputStream stream, } // Assert import/export fidelity and presence of observation units in export - checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension, expectedColNumber); + checkDownloadTable( + requestEnv, + rows, + download, + includeTimestamps, + extension, + expectedColNumber, + expectedObsUnitIdColumn, + expectedObsUnitIdUniqueCount + ); } private void checkDownloadTable( @@ -946,12 +1165,15 @@ private void checkDownloadTable( Table table, boolean includeTimestamps, String extension, - Integer expectedColNumber) { + Integer expectedColNumber, + String expectedObsUnitIdColumn, + Integer expectedObsUnitIdUniqueCount) { // Filename is correct: _Observation Dataset [-]__ List expectedEnvNames = requestedImportRows.stream() .map(row -> row.get(ExperimentObservation.Columns.ENV).toString()).collect(Collectors.toList()); assertEquals(expectedColNumber, table.columnCount()); + assertEquals(requestedImportRows.size(), table.rowCount()); // Check that requested envs are present. expectedEnvNames.forEach(envName -> assertTrue(table.stringColumn("Env").contains(envName))); @@ -968,14 +1190,24 @@ private void checkDownloadTable( String gid = ExperimentObservation.Columns.GERMPLASM_GID; String env = ExperimentObservation.Columns.ENV; String expUnitId = ExperimentObservation.Columns.EXP_UNIT_ID; + String subObsUnit = ExperimentObservation.Columns.SUB_OBS_UNIT; + String subUnitId = ExperimentObservation.Columns.SUB_UNIT_ID; + boolean subObsUnitMatches = !row.containsKey(subObsUnit) || + row.get(subObsUnit).equals(downloadRow.getString(subObsUnit)); + boolean subUnitIdMatches = !row.containsKey(subUnitId) || + row.get(subUnitId).equals(downloadRow.getObject(subUnitId).toString()); if (extension.equalsIgnoreCase(FileType.CSV.getName())) { return Integer.parseInt(row.get(gid).toString()) == downloadRow.getInt(gid) && row.get(env).equals(downloadRow.getString(env)) && - row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); + row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()) && + subObsUnitMatches && + subUnitIdMatches; } else { return row.get(gid).equals(downloadRow.getString(gid)) && row.get(env).equals(downloadRow.getString(env)) && - row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); + row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()) && + subObsUnitMatches && + subUnitIdMatches; } }).findAny(); assertTrue(matchingImportRow.isPresent() && !matchingImportRow.get().isEmpty()); @@ -987,58 +1219,74 @@ private void checkDownloadTable( } assertEquals(requestedImportRows.size(),matchingImportRows.size()); - //Observation level for tests should be "Plot" // Observation units populated. - assertEquals(0, table.column("Plot ObsUnitID").countMissing()); + assertTrue(table.columnNames().contains(expectedObsUnitIdColumn)); + assertEquals(0, table.column(expectedObsUnitIdColumn).countMissing()); // Observation Unit IDs are assigned. - assertEquals(requestedImportRows.size(), table.column("Plot ObsUnitID").countUnique()); + assertEquals(expectedObsUnitIdUniqueCount, table.column(expectedObsUnitIdColumn).countUnique()); } private boolean isMatchedRow(Map importRow, Row downloadRow) { - System.out.println("Validating row: " + downloadRow.getRowNumber()); - System.out.println("import columns: " + importRow.size()); return importRow.entrySet().stream().filter(e -> { String header = e.getKey(); - List importColumns = columns - .stream() - .filter(col -> header.equals(col.getValue())).collect(Collectors.toList()); - if (importColumns.size() != 1) { - return false; - } - Object expectedVal = null; - Object downloadedVal = null; - boolean doCompare = false; - - if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.STRING)) { - expectedVal = e.getValue().toString(); - downloadedVal = downloadRow.getString(e.getKey()); - doCompare = true; - } - if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.INTEGER)) { - expectedVal = Integer.parseInt(e.getValue().toString()); - downloadedVal = downloadRow.getInt(e.getKey()); - doCompare = true; - } - if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.DOUBLE)) { - expectedVal = Double.parseDouble(e.getValue().toString()); - downloadedVal = downloadRow.getDouble(e.getKey()); - doCompare = true; - } - if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.FLOAT)) { - expectedVal = e.getValue(); - downloadedVal = downloadRow.getFloat(e.getKey()); - doCompare = true; - } - System.out.println("Column: "+e.getKey()+", Expected: '"+ expectedVal +"', Received: '" + downloadedVal+"'"); - if(doCompare) { - assertEquals(expectedVal, downloadedVal); - return expectedVal.equals(downloadedVal); - } else { + if (!downloadRow.columnNames().contains(header)) { return false; } + + Object expectedVal = e.getValue(); + Object downloadedVal = getDownloadedValue(downloadRow, header); + assertValuesMatch(header, expectedVal, downloadedVal); + return true; }).count() == importRow.size(); } + private Object getDownloadedValue(Row downloadRow, String header) { + ColumnType columnType = downloadRow.getColumnType(header); + if (columnType.equals(ColumnType.STRING)) { + return downloadRow.getString(header); + } + if (columnType.equals(ColumnType.INTEGER)) { + return downloadRow.getInt(header); + } + if (columnType.equals(ColumnType.DOUBLE)) { + return downloadRow.getDouble(header); + } + if (columnType.equals(ColumnType.FLOAT)) { + return downloadRow.getFloat(header); + } + return downloadRow.getObject(header); + } + + private void assertValuesMatch(String header, Object expectedVal, Object downloadedVal) { + if (isNumeric(expectedVal) && isNumeric(downloadedVal)) { + assertEquals( + 0, + toBigDecimal(expectedVal).compareTo(toBigDecimal(downloadedVal)), + "Column " + header + " mismatch"); + return; + } + assertEquals( + Objects.toString(expectedVal, null), + Objects.toString(downloadedVal, null), + "Column " + header + " mismatch"); + } + + private boolean isNumeric(Object value) { + if (value == null) { + return false; + } + try { + new BigDecimal(value.toString()); + return true; + } catch (NumberFormatException e) { + return false; + } + } + + private BigDecimal toBigDecimal(Object value) { + return new BigDecimal(value.toString()); + } + private String getEnvId(JsonObject result, int index) { return result .get("preview").getAsJsonObject()