diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/measure/MeasureBundler.java b/tooling/src/main/java/org/opencds/cqf/tooling/measure/MeasureBundler.java index 7a91a3df4..a04f0b482 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/measure/MeasureBundler.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/measure/MeasureBundler.java @@ -3,24 +3,21 @@ import ca.uhn.fhir.context.FhirContext; import org.hl7.fhir.instance.model.api.IBaseResource; import org.opencds.cqf.tooling.processor.AbstractBundler; -import org.opencds.cqf.tooling.utilities.HttpClientUtils; import org.opencds.cqf.tooling.utilities.IOUtils; -import org.opencds.cqf.tooling.utilities.IOUtils.Encoding; -import java.io.File; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; public class MeasureBundler extends AbstractBundler { public static final String ResourcePrefix = "measure-"; + protected CopyOnWriteArrayList identifiers; public static String getId(String baseId) { return ResourcePrefix + baseId; } + @Override protected String getSourcePath(FhirContext fhirContext, Map.Entry resourceEntry) { return IOUtils.getMeasurePathMap(fhirContext).get(resourceEntry.getKey()); @@ -41,67 +38,4 @@ protected Set getPaths(FhirContext fhirContext) { return IOUtils.getMeasurePaths(fhirContext); } - //so far only the Measure Bundle process needs to persist extra files: - @Override - protected int persistFilesFolder(String bundleDestPath, String libraryName, Encoding encoding, FhirContext fhirContext, String fhirUri) { - //persist tests-* before group-* files and make a record of which files were tracked: - List persistedFiles = persistTestFilesWithPriority(bundleDestPath, libraryName, encoding, fhirContext, fhirUri); - persistedFiles.addAll(persistEverythingElse(bundleDestPath, libraryName, encoding, fhirContext, fhirUri, persistedFiles)); - - return persistedFiles.size(); - } - - private List persistTestFilesWithPriority(String bundleDestPath, String libraryName, Encoding encoding, FhirContext fhirContext, String fhirUri) { - List persistedResources = new ArrayList<>(); - String filesLoc = bundleDestPath + File.separator + libraryName + "-files"; - File directory = new File(filesLoc); - if (directory.exists()) { - File[] filesInDir = directory.listFiles(); - if (!(filesInDir == null || filesInDir.length == 0)) { - for (File file : filesInDir) { - if (file.getName().toLowerCase().startsWith("tests-")) { - try { - IBaseResource resource = IOUtils.readResource(file.getAbsolutePath(), fhirContext, true); - HttpClientUtils.post(fhirUri, resource, encoding, fhirContext, file.getAbsolutePath(), true); - persistedResources.add(file.getAbsolutePath()); - } catch (Exception e) { - //resource is likely not IBaseResource - logger.error("MeasureBundler.persistTestFilesWithPriority", e); - } - } - } - } - } - return persistedResources; - } - - private List persistEverythingElse(String bundleDestPath, String libraryName, Encoding encoding, FhirContext fhirContext, String fhirUri, List alreadyPersisted) { - List persistedResources = new ArrayList<>(); - String filesLoc = bundleDestPath + File.separator + libraryName + "-files"; - File directory = new File(filesLoc); - if (directory.exists()) { - - File[] filesInDir = directory.listFiles(); - - if (!(filesInDir == null || filesInDir.length == 0)) { - for (File file : filesInDir) { - //don't post what has already been processed - if (alreadyPersisted.contains(file.getAbsolutePath())) { - continue; - } - if (file.getName().toLowerCase().endsWith(".json") || file.getName().toLowerCase().endsWith(".xml")) { - try { - IBaseResource resource = IOUtils.readResource(file.getAbsolutePath(), fhirContext, true); - HttpClientUtils.post(fhirUri, resource, encoding, fhirContext, file.getAbsolutePath(), false); - persistedResources.add(file.getAbsolutePath()); - } catch (Exception e) { - //resource is likely not IBaseResource - logger.error("persistEverythingElse", e); - } - } - } - } - } - return persistedResources; - } } diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/plandefinition/PlanDefinitionBundler.java b/tooling/src/main/java/org/opencds/cqf/tooling/plandefinition/PlanDefinitionBundler.java index 9e9852ded..e6b9acb93 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/plandefinition/PlanDefinitionBundler.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/plandefinition/PlanDefinitionBundler.java @@ -37,8 +37,4 @@ protected String getResourceBundlerType() { return TYPE_PLAN_DEFINITION; } - @Override - protected int persistFilesFolder(String bundleDestPath, String libraryName, IOUtils.Encoding encoding, FhirContext fhirContext, String fhirUri) { - return 0; - } } diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/processor/AbstractBundler.java b/tooling/src/main/java/org/opencds/cqf/tooling/processor/AbstractBundler.java index 64c10f842..2cd0c4b14 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/processor/AbstractBundler.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/processor/AbstractBundler.java @@ -130,7 +130,7 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L final Map> cqlTranslatorErrorMessages = new ConcurrentHashMap<>(); //used to summarize file count user can expect to see in POST queue for each resource: - final Map persistedFileReport = new ConcurrentHashMap<>(); + final List persistedFileReport = new CopyOnWriteArrayList<>(); //build list of executable tasks to be sent to thread pool: List> tasks = new ArrayList<>(); @@ -142,12 +142,12 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L final Map libraryPathMap = new ConcurrentHashMap<>(IOUtils.getLibraryPathMap(fhirContext)); if (resourcesMap.isEmpty()) { - logger.info("[INFO] No " + getResourceBundlerType() + "s found. Continuing..."); + logger.info("\n\r" + "[INFO] No " + getResourceBundlerType() + "s found. Continuing...\n\r"); return; } for (Map.Entry resourceEntry : resourcesMap.entrySet()) { - String resourceId; + String resourceId = ""; if (resourceEntry.getValue() != null) { resourceId = resourceEntry.getValue() @@ -169,7 +169,7 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L tasks.add(() -> { //check if resourceSourcePath has been processed before: if (processedResources.contains(resourceSourcePath)) { - logger.info(getResourceBundlerType() + " processed already: " + resourceSourcePath); + logger.info("\n\r" + getResourceBundlerType() + " processed already: " + resourceSourcePath); return null; } String resourceName = FilenameUtils.getBaseName(resourceSourcePath).replace(getResourcePrefix(), ""); @@ -203,10 +203,20 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L shouldPersist = shouldPersist & ResourceUtils.safeAddResource(primaryLibrarySourcePath, resources, fhirContext); + String cqlFileName = IOUtils.formatFileName(primaryLibraryName, IOUtils.Encoding.CQL, fhirContext); + + String cqlLibrarySourcePath = IOUtils.getCqlLibrarySourcePath(primaryLibraryName, cqlFileName, binaryPaths); + + if (cqlLibrarySourcePath == null) { + failedExceptionMessages.put(resourceSourcePath, String.format("Could not determine CqlLibrarySource path for library %s", primaryLibraryName)); + //exit from task: + return null; + } + if (includeTerminology) { //throws CQLTranslatorException if failed with severe errors, which will be logged and reported it in the final summary try { - ValueSetsProcessor.bundleValueSets(primaryLibrary, fhirContext, resources, encoding, includeDependencies); + ValueSetsProcessor.bundleValueSets(cqlLibrarySourcePath, igPath, fhirContext, resources, encoding, includeDependencies, includeVersion); } catch (CqlTranslatorException cqlTranslatorException) { cqlTranslatorErrorMessages.put(primaryLibraryName, cqlTranslatorException.getErrors()); } @@ -215,7 +225,8 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L if (includeDependencies) { if (libraryProcessor == null) libraryProcessor = new LibraryProcessor(); try { - libraryProcessor.bundleLibraryDependencies(primaryLibrary, fhirContext, resources, encoding, includeVersion); + // libraryProcessor.bundleLibraryDependencies(primaryLibrarySourcePath, fhirContext, resources, encoding, includeVersion); + libraryProcessor.bundleLibraryDependencies(primaryLibrary, fhirContext, resources, encoding, includeVersion); } catch (Exception bre) { failedExceptionMessages.put(resourceSourcePath, getResourceBundlerType() + " will not be bundled because Library Dependency bundling failed: " + bre.getMessage()); //exit from task: @@ -236,19 +247,13 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L if (shouldPersist) { String bundleDestPath = FilenameUtils.concat(FilenameUtils.concat(IGProcessor.getBundlesPath(igPath), getResourceTestGroupName()), resourceName); - persistBundle(bundleDestPath, resourceName, encoding, fhirContext, new ArrayList(resources.values()), fhirUri, addBundleTimestamp); - - // It's not clear at all why this is happening... we've already persisted the bundle? Why write out all the bundle files?? - // And if we _do_ need to write out the bundle files, why go through the whole assembling process again? Just write out the resources in the bundle we already have, right? - //bundleFiles(igPath, bundleDestPath, resourceName, binaryPaths, resourceSourcePath, - // primaryLibrarySourcePath, fhirContext, encoding, includeTerminology, includeDependencies, includePatientScenarios, - // includeVersion, addBundleTimestamp, cqlTranslatorErrorMessages); + bundleFiles(igPath, bundleDestPath, resourceName, binaryPaths, resourceSourcePath, + primaryLibrarySourcePath, fhirContext, encoding, includeTerminology, includeDependencies, includePatientScenarios, + includeVersion, addBundleTimestamp, cqlTranslatorErrorMessages); //If user supplied a fhir server url, inform them of total # of files to be persisted to the server: if (fhirUri != null && !fhirUri.isEmpty()) { - persistedFileReport.put(resourceName, - //+1 to account for -bundle - persistFilesFolder(bundleDestPath, resourceName, encoding, fhirContext, fhirUri) + 1); + persistedFileReport.add(resourceName); } if (cdsHooksProcessor != null) { @@ -256,12 +261,14 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L cdsHooksProcessor.addActivityDefinitionFilesToBundle(igPath, bundleDestPath, activityDefinitionPaths, fhirContext, encoding); } + persistBundle(bundleDestPath, resourceName, encoding, fhirContext, new ArrayList<>(resources.values()), fhirUri, addBundleTimestamp); + bundledResources.add(resourceSourcePath); } } catch (Exception e) { - String failMsg; + String failMsg = ""; if (e.getMessage() != null) { failMsg = e.getMessage(); } else { @@ -288,7 +295,7 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L //Output final report: String summaryOutput = generateBundleProcessSummary(refreshedLibraryNames, fhirContext, fhirUri, verboseMessaging, persistedFileReport, bundledResources, failedExceptionMessages, cqlTranslatorErrorMessages).toString(); - logger.info(summaryOutput); + logger.info("\n\r" + summaryOutput); } /** @@ -307,7 +314,7 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L * @return A StringBuilder containing the generated summary message. */ private StringBuilder generateBundleProcessSummary(List refreshedLibraryNames, FhirContext fhirContext, - String fhirUri, Boolean verboseMessaging, Map persistedFileReport, + String fhirUri, Boolean verboseMessaging, List persistedFileReport, List bundledResources, Map failedExceptionMessages, Map> cqlTranslatorErrorMessages) { @@ -315,45 +322,8 @@ private StringBuilder generateBundleProcessSummary(List refreshedLibrary //Give user a snapshot of the files each resource will have persisted to their FHIR server (if fhirUri is provided) final int persistCount = persistedFileReport.size(); - if (persistCount > 0) { - String fileDisplay = " File(s): "; - summaryMessage.append(NEWLINE).append(persistCount).append(" ").append(getResourceBundlerType()).append("(s) have POST tasks in the queue for ").append(fhirUri).append(": "); - int totalQueueCount = 0; - List persistMessages = new ArrayList<>(); - for (String library : persistedFileReport.keySet()) { - totalQueueCount = totalQueueCount + persistedFileReport.get(library); - persistMessages.add(NEWLINE_INDENT - + persistedFileReport.get(library) - + fileDisplay - + library); - } - - //anon comparator class to sort by the file count for better presentation - persistMessages.sort(new Comparator<>() { - @Override - public int compare(String displayFileCount1, String displayFileCount2) { - int count1 = getFileCountFromString(displayFileCount1); - int count2 = getFileCountFromString(displayFileCount2); - return Integer.compare(count1, count2); - } - - private int getFileCountFromString(String fileName) { - int endIndex = fileName.indexOf(fileDisplay); - if (endIndex != -1) { - String countString = fileName.substring(0, endIndex).trim(); - return Integer.parseInt(countString); - } - return 0; - } - }); - - for (String persistMessage : persistMessages) { - summaryMessage.append(persistMessage); - } - summaryMessage.append(NEWLINE_INDENT) - .append("Total: ") - .append(totalQueueCount) - .append(" File(s)"); + if (!persistedFileReport.isEmpty()) { + summaryMessage.append(NEWLINE).append(persistCount).append(" ").append(getResourceBundlerType()).append("(s) have HTTP request tasks in the queue for ").append(fhirUri); } @@ -444,19 +414,16 @@ private void persistBundle(String bundleDestPath, String libraryName, IOUtils.Encoding encoding, FhirContext fhirContext, List resources, String fhirUri, Boolean addBundleTimestamp) throws IOException { - IOUtils.initializeDirectory(bundleDestPath); Object bundle = BundleUtils.bundleArtifacts(libraryName, resources, fhirContext, addBundleTimestamp, this.getIdentifiers()); IOUtils.writeBundle(bundle, bundleDestPath, encoding, fhirContext); if (fhirUri != null && !fhirUri.isEmpty()) { String resourceWriteLocation = bundleDestPath + separator + libraryName + "-bundle." + encoding; - HttpClientUtils.post(fhirUri, (IBaseResource) bundle, encoding, fhirContext, resourceWriteLocation, true); + //give resource the highest priority (0): + HttpClientUtils.sendToServer(fhirUri, (IBaseResource) bundle, encoding, fhirContext, resourceWriteLocation); } } - - protected abstract int persistFilesFolder(String bundleDestPath, String libraryName, IOUtils.Encoding encoding, FhirContext fhirContext, String fhirUri); - private void bundleFiles(String igPath, String bundleDestPath, String primaryLibraryName, List binaryPaths, String resourceFocusSourcePath, String librarySourcePath, FhirContext fhirContext, IOUtils.Encoding encoding, Boolean includeTerminology, Boolean includeDependencies, Boolean includePatientScenarios, Boolean includeVersion, Boolean addBundleTimestamp, Map> translatorWarningMessages) { @@ -468,7 +435,7 @@ private void bundleFiles(String igPath, String bundleDestPath, String primaryLib IOUtils.copyFile(librarySourcePath, FilenameUtils.concat(bundleDestFilesPath, FilenameUtils.getName(librarySourcePath))); String cqlFileName = IOUtils.formatFileName(FilenameUtils.getBaseName(librarySourcePath), IOUtils.Encoding.CQL, fhirContext); - if (cqlFileName.toLowerCase().startsWith("library-")) { + if (cqlFileName.toLowerCase().startsWith("library-") && !cqlFileName.toLowerCase().startsWith("library-deps-")) { cqlFileName = cqlFileName.substring(8); } String cqlLibrarySourcePath = IOUtils.getCqlLibrarySourcePath(primaryLibraryName, cqlFileName, binaryPaths); @@ -501,4 +468,4 @@ private void bundleFiles(String igPath, String bundleDestPath, String primaryLib } } -} +} \ No newline at end of file diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/processor/IGBundleProcessor.java b/tooling/src/main/java/org/opencds/cqf/tooling/processor/IGBundleProcessor.java index 67bd26df5..54e326801 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/processor/IGBundleProcessor.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/processor/IGBundleProcessor.java @@ -2,8 +2,10 @@ import ca.uhn.fhir.context.FhirContext; import org.opencds.cqf.tooling.library.LibraryProcessor; +import org.opencds.cqf.tooling.measure.MeasureBundler; import org.opencds.cqf.tooling.packaging.PackageMeasures; import org.opencds.cqf.tooling.packaging.PackagePlanDefinitions; +import org.opencds.cqf.tooling.plandefinition.PlanDefinitionBundler; import org.opencds.cqf.tooling.questionnaire.QuestionnaireBundler; import org.opencds.cqf.tooling.utilities.HttpClientUtils; import org.opencds.cqf.tooling.utilities.IOUtils; @@ -24,38 +26,48 @@ public class IGBundleProcessor { LibraryProcessor libraryProcessor; CDSHooksProcessor cdsHooksProcessor; - public IGBundleProcessor(Boolean verboseMessaging, LibraryProcessor libraryProcessor, CDSHooksProcessor cdsHooksProcessor) { + public IGBundleProcessor(Boolean verboseMessaging, LibraryProcessor libraryProcessor, + CDSHooksProcessor cdsHooksProcessor) { this.verboseMessaging = verboseMessaging; this.libraryProcessor = libraryProcessor; this.cdsHooksProcessor = cdsHooksProcessor; } - public void bundleIg(List refreshedLibraryNames, String igPath, List binaryPaths, Encoding encoding, Boolean includeELM, - Boolean includeDependencies, Boolean includeTerminology, Boolean includePatientScenarios, Boolean versioned, Boolean addBundleTimestamp, - FhirContext fhirContext, String fhirUri) { + public void bundleIg(List refreshedLibraryNames, String igPath, List binaryPaths, Encoding encoding, + Boolean includeELM, + Boolean includeDependencies, Boolean includeTerminology, Boolean includePatientScenarios, Boolean versioned, + Boolean addBundleTimestamp, + FhirContext fhirContext, String fhirUri) { + + // new MeasureBundler().bundleResources(refreshedLibraryNames, + // igPath, binaryPaths, includeDependencies, includeTerminology, + // includePatientScenarios, versioned, addBundleTimestamp, fhirContext, + // fhirUri, encoding, verboseMessaging); + + new PackageMeasures(igPath, fhirContext, includeDependencies, includeTerminology, includePatientScenarios, + fhirUri); + + // new PlanDefinitionBundler(this.libraryProcessor, this.cdsHooksProcessor).bundleResources(refreshedLibraryNames, + // igPath, binaryPaths, includeDependencies, includeTerminology, + // includePatientScenarios, versioned, addBundleTimestamp, fhirContext, + // fhirUri, encoding, verboseMessaging); + + new PackagePlanDefinitions(igPath, fhirContext, includeDependencies, includeTerminology, + includePatientScenarios, fhirUri); -// new MeasureBundler().bundleResources(refreshedLibraryNames, -// igPath, binaryPaths, includeDependencies, includeTerminology, -// includePatientScenarios, versioned, addBundleTimestamp, fhirContext, -// fhirUri, encoding, verboseMessaging); - new PackageMeasures(igPath, fhirContext, includeDependencies, includeTerminology, includePatientScenarios, fhirUri); -// new PlanDefinitionBundler(this.libraryProcessor, this.cdsHooksProcessor).bundleResources(refreshedLibraryNames, -// igPath, binaryPaths, includeDependencies, includeTerminology, -// includePatientScenarios, versioned, addBundleTimestamp, fhirContext, -// fhirUri, encoding, verboseMessaging); - new PackagePlanDefinitions(igPath, fhirContext, includeDependencies, includeTerminology, includePatientScenarios, fhirUri); new QuestionnaireBundler(this.libraryProcessor).bundleResources(refreshedLibraryNames, igPath, binaryPaths, includeDependencies, includeTerminology, includePatientScenarios, versioned, addBundleTimestamp, fhirContext, fhirUri, encoding, verboseMessaging); - //run collected post calls last: - if (HttpClientUtils.hasPostTasksInQueue()) { - logger.info("[Persisting Files to {}]", fhirUri); - HttpClientUtils.postTaskCollection(); + // run collected post calls last: + if (HttpClientUtils.hasHttpRequestTasksInQueue()) { + logger.info("\n\r[Persisting Files to " + fhirUri + "]\n\r"); + HttpClientUtils.executeHttpRequestTaskCollection(); } - // run cleanup (maven runs all ci tests sequentially and static member variables could retain values from previous tests) + // run cleanup (maven runs all ci tests sequentially and static member variables + // could retain values from previous tests) IOUtils.cleanUp(); ResourceUtils.cleanUp(); } diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/processor/PostBundlesInDirProcessor.java b/tooling/src/main/java/org/opencds/cqf/tooling/processor/PostBundlesInDirProcessor.java index 61a2ea366..db6ce2856 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/processor/PostBundlesInDirProcessor.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/processor/PostBundlesInDirProcessor.java @@ -63,15 +63,15 @@ public static void PostBundlesInDir(PostBundlesInDirParameters params) { List> resources = BundleUtils.getBundlesInDir(params.directoryPath, fhirContext); resources.forEach(entry -> postBundleToFhirUri(fhirUri, encoding, fhirContext, entry.getValue())); - if (HttpClientUtils.hasPostTasksInQueue()){ - HttpClientUtils.postTaskCollection(); + if (HttpClientUtils.hasHttpRequestTasksInQueue()){ + HttpClientUtils.executeHttpRequestTaskCollection(); } } private static void postBundleToFhirUri(String fhirUri, Encoding encoding, FhirContext fhirContext, IBaseResource bundle) { - if (fhirUri != null && !fhirUri.equals("")) { + if (fhirUri != null && !fhirUri.isEmpty()) { try { - HttpClientUtils.post(fhirUri, bundle, encoding, fhirContext, null); + HttpClientUtils.sendToServer(fhirUri, bundle, encoding, fhirContext, null); logger.info("Resource successfully posted to FHIR server ({}): {}", fhirUri, bundle.getIdElement().getIdPart()); } catch (Exception e) { logger.error("Error occurred for element {}: {}",bundle.getIdElement().getIdPart(), e.getMessage()); diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/questionnaire/QuestionnaireBundler.java b/tooling/src/main/java/org/opencds/cqf/tooling/questionnaire/QuestionnaireBundler.java index d605c9694..b3af38911 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/questionnaire/QuestionnaireBundler.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/questionnaire/QuestionnaireBundler.java @@ -35,11 +35,6 @@ protected String getResourceBundlerType() { return TYPE_QUESTIONNAIRE; } - @Override - protected int persistFilesFolder(String bundleDestPath, String libraryName, IOUtils.Encoding encoding, FhirContext fhirContext, String fhirUri) { - //do nothing - return 0; - } @Override protected Set getPaths(FhirContext fhirContext) { diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/utilities/BundleUtils.java b/tooling/src/main/java/org/opencds/cqf/tooling/utilities/BundleUtils.java index 8b3eb8935..6f4f28b7b 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/utilities/BundleUtils.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/utilities/BundleUtils.java @@ -279,4 +279,77 @@ public static boolean resourceIsTransactionBundle(IBaseResource inputResource) { return false; } + + public static List getResourceIdsFromTransactionBundle(IBaseResource inputResource) { + List returnedIDs = new ArrayList<>(); + + if (inputResource instanceof org.hl7.fhir.dstu3.model.Bundle) { + org.hl7.fhir.dstu3.model.Bundle bundle = (org.hl7.fhir.dstu3.model.Bundle) inputResource; + if (bundle.getType() == org.hl7.fhir.dstu3.model.Bundle.BundleType.TRANSACTION) { + for (org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { + if (entry.getResource() != null && entry.getResource().getIdElement() != null) { + if (entry.getResource() != null) { + String resourceId = entry.getResource().getIdElement().getIdPart(); + if (resourceId != null) { + returnedIDs.add(resourceId); + } else { + // Log a message if ID is missing (optional) + System.out.println("Resource in transaction bundle has no ID (likely a POST operation)."); + } + } + } + } + } + } else if (inputResource instanceof org.hl7.fhir.r4.model.Bundle) { + org.hl7.fhir.r4.model.Bundle bundle = (org.hl7.fhir.r4.model.Bundle) inputResource; + if (bundle.getType() == org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION) { + for (org.hl7.fhir.r4.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { + if (entry.getResource() != null && entry.getResource().getIdElement() != null) { + if (entry.getResource() != null) { + String resourceId = entry.getResource().getIdElement().getIdPart(); + if (resourceId != null) { + returnedIDs.add(resourceId); + } else { + // Log a message if ID is missing (optional) + System.out.println("Resource in transaction bundle has no ID (likely a POST operation)."); + } + } + } + } + } + } + + return returnedIDs; + } + + public static List getResourcesFromTransactionBundle(IBaseResource inputResource) { + List returnedResources = new ArrayList<>(); + + if (inputResource instanceof org.hl7.fhir.dstu3.model.Bundle) { + org.hl7.fhir.dstu3.model.Bundle bundle = (org.hl7.fhir.dstu3.model.Bundle) inputResource; + if (bundle.getType() == org.hl7.fhir.dstu3.model.Bundle.BundleType.TRANSACTION) { + for (org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { + if (entry.getResource() != null && entry.getResource().getIdElement() != null) { + if (entry.getResource() != null) { + returnedResources.add(entry.getResource()); + } + } + } + } + } else if (inputResource instanceof org.hl7.fhir.r4.model.Bundle) { + org.hl7.fhir.r4.model.Bundle bundle = (org.hl7.fhir.r4.model.Bundle) inputResource; + if (bundle.getType() == org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION) { + for (org.hl7.fhir.r4.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { + if (entry.getResource() != null && entry.getResource().getIdElement() != null) { + if (entry.getResource() != null) { + returnedResources.add(entry.getResource()); + } + } + } + } + } + + return returnedResources; + } + } diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/utilities/HttpClientUtils.java b/tooling/src/main/java/org/opencds/cqf/tooling/utilities/HttpClientUtils.java index 5f10fc86f..4e1152370 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/utilities/HttpClientUtils.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/utilities/HttpClientUtils.java @@ -1,6 +1,9 @@ package org.opencds.cqf.tooling.utilities; import ca.uhn.fhir.context.FhirContext; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.apache.commons.lang3.tuple.Pair; import org.apache.http.Header; @@ -10,11 +13,15 @@ import org.apache.http.client.ClientProtocolException; import org.apache.http.client.ResponseHandler; import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; import org.hl7.fhir.instance.model.api.IBaseResource; import org.slf4j.Logger; @@ -28,10 +35,11 @@ import java.util.function.Function; /** - * A utility class for collecting HTTP requests to a FHIR server and executing them collectively. + * A utility class for collecting HTTP requests to a FHIR server and executing + * them collectively. */ public class HttpClientUtils { - //60 second timeout + // 60 second timeout protected static final RequestConfig requestConfig = RequestConfig.custom() .setSocketTimeout(60000) .setConnectTimeout(60000) @@ -43,37 +51,60 @@ public class HttpClientUtils { private static final String ENCODING_TYPE = "Encoding Type"; private static final String FHIR_CONTEXT = "FHIR Context"; - //This is not to maintain a thread count, but rather to maintain the maximum number of POST calls that can simultaneously be waiting for a response from the server. - //This gives us some control over how many POSTs we're making so we don't crash the server. - //possible TODO:Allow users to specify this value on their own with arg passed into operation so that more robust servers can process post list faster - private static final int MAX_SIMULTANEOUS_POST_COUNT = 10; - - //failedPostCalls needs to maintain the details built in the FAILED message, as well as a copy of the inputs for a retry by the user on failed posts. - private static Queue> failedPostCalls = new ConcurrentLinkedQueue<>(); - private static List successfulPostCalls = new CopyOnWriteArrayList<>(); + // This is not to maintain a thread count, but rather to maintain the maximum + // number of HTTP calls that can simultaneously be waiting for a response from + // the server. + // This gives us some control over how many HTTP requests we're making so we + // don't crash the server. + // possible TODO:Allow users to specify this value on their own with arg passed + // into operation so that more robust servers can process put list faster + private static final int MAX_SIMULTANEOUS_REQUEST_COUNT = 10; + + // failedHttptCalls needs to maintain the details built in the FAILED message, + // as well as a copy of the inputs for a retry by the user on failed puts. + private static Queue> failedHttpCalls = new ConcurrentLinkedQueue<>(); + private static List successfulHttpCalls = new CopyOnWriteArrayList<>(); private static Map> tasks = new ConcurrentHashMap<>(); - private static Map> initialTasks = new ConcurrentHashMap<>(); - private static List runningPostTaskList = new CopyOnWriteArrayList<>(); - private static int processedPostCounter = 0; + // Parent map uses resourceType as key so that resourceTypes can have their + // tasks called in a specific order: + + private static List runningRequestTaskList = new CopyOnWriteArrayList<>(); + private static int processedRequestCounter = 0; + + private static final CloseableHttpClient httpClient; + + static { + PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); + connectionManager.setMaxTotal(50); // Total max connections + connectionManager.setDefaultMaxPerRoute(10); // Max connections per route + + httpClient = HttpClients.custom() + .setConnectionManager(connectionManager) + .build(); + } private HttpClientUtils() { } - public static boolean hasPostTasksInQueue() { + public static boolean hasHttpRequestTasksInQueue() { return !tasks.isEmpty(); } /** - * Initiates an HTTP POST request to a FHIR server with the specified parameters. + * Initiates an HTTP request to a FHIR server with the specified parameters. * - * @param fhirServerUrl The URL of the FHIR server to which the POST request will be sent. - * @param resource The FHIR resource to be posted. + * @param fhirServerUrl The URL of the FHIR server to which the request will be + * sent. + * @param resource The FHIR resource to be sent. * @param encoding The encoding type of the resource. * @param fhirContext The FHIR context for the resource. - * @param fileLocation Optional fileLocation indicator for identifying resources by raw filename + * @param fileLocation Optional fileLocation indicator for identifying + * resources by raw filename * @throws IOException If an I/O error occurs during the request. */ - public static void post(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext, String fileLocation, boolean withPriority) throws IOException { + public static void sendToServer(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, + FhirContext fhirContext, String fileLocation) throws IOException { + List missingValues = new ArrayList<>(); List values = new ArrayList<>(); validateAndAddValue(fhirServerUrl, FHIR_SERVER_URL, missingValues, values); @@ -83,24 +114,33 @@ public static void post(String fhirServerUrl, IBaseResource resource, IOUtils.En if (!missingValues.isEmpty()) { String missingValueString = String.join(", ", missingValues); - logger.error("An invalid HTTP POST call was attempted with a null value for: " + missingValueString + - (!values.isEmpty() ? "\\nRemaining values are: " + String.join(", ", values) : "")); + + if (!values.isEmpty()) { + String remainingValuesString = String.join(", ", values); + logger.error( + "An invalid HTTP request was attempted with a null value for: {}\nRemaining values are: {}", + missingValueString, remainingValuesString); + } else { + logger.error("An invalid HTTP request was attempted with a null value for: {}", + missingValueString); + } + return; } - createPostTask(fhirServerUrl, resource, encoding, fhirContext, fileLocation, withPriority); - } - - public static void post(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext, String fileLocation) throws IOException { - post(fhirServerUrl, resource, encoding, fhirContext, fileLocation, false); + createHttpRequestTask(fhirServerUrl, resource, encoding, fhirContext, fileLocation); } /** - * Validates a value and adds its representation to the provided lists using a custom value-to-string function. + * Validates a value and adds its representation to the provided lists using a + * custom value-to-string function. *

- * This method checks if the given value is null. If the value is not null, it is converted to a string using the provided - * value-to-string function, and the value along with its label is added to the 'values' list. If the value is null, the - * label is added to the 'missingValues' list to indicate a missing or invalid value. + * This method checks if the given value is null. If the value is not null, it + * is converted to a string using the provided + * value-to-string function, and the value along with its label is added to the + * 'values' list. If the value is null, the + * label is added to the 'missingValues' list to indicate a missing or invalid + * value. * * @param value The value to be validated and added. * @param label A label describing the value (e.g., parameter name). @@ -109,7 +149,8 @@ public static void post(String fhirServerUrl, IBaseResource resource, IOUtils.En * @param valueToString A custom function to convert the value to a string. * @param The type of the value to be validated. */ - private static void validateAndAddValue(T value, String label, List missingValues, List values, Function valueToString) { + private static void validateAndAddValue(T value, String label, List missingValues, List values, + Function valueToString) { if (value == null) { missingValues.add(label); } else { @@ -117,182 +158,233 @@ private static void validateAndAddValue(T value, String label, List } } - private static void validateAndAddValue(T value, String label, List missingValues, List values) { + private static void validateAndAddValue(T value, String label, List missingValues, + List values) { validateAndAddValue(value, label, missingValues, values, Object::toString); } /** - * Creates a task for handling an HTTP POST request to a FHIR server with the specified parameters. + * Creates a task for handling an HTTP request to a FHIR server with the + * specified parameters. *

- * This method is responsible for creating a task that prepares and executes an HTTP POST request to the provided FHIR server - * with the given FHIR resource, encoding type, and FHIR context. It adds the task to the queue of tasks for later execution. - * If any exceptions occur during task creation or configuration, an error message is logged. + * This method is responsible for creating a task that prepares and executes an + * HTTP request to the provided FHIR server + * with the given FHIR resource, encoding type, and FHIR context. It adds the + * task to the queue of tasks for later execution. + * If any exceptions occur during task creation or configuration, an error + * message is logged. * - * @param fhirServerUrl The URL of the FHIR server to which the POST request will be sent. - * @param resource The FHIR resource to be posted. + * @param fhirServerUrl The URL of the FHIR server to which the request will be + * sent. * @param encoding The encoding type of the resource. * @param fhirContext The FHIR context for the resource. */ - private static void createPostTask(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext, String fileLocation, boolean withPriority) { + private static void createHttpRequestTask(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, + FhirContext fhirContext, String fileLocation) { try { - PostComponent postPojo = new PostComponent(fhirServerUrl, resource, encoding, fhirContext, fileLocation, withPriority); - HttpPost post = configureHttpPost(fhirServerUrl, resource, encoding, fhirContext); - if (withPriority) { - initialTasks.put(resource, createPostCallable(post, postPojo)); - } else { - tasks.put(resource, createPostCallable(post, postPojo)); - } + HttpRequestComponent httpRequestPojo = new HttpRequestComponent(fhirServerUrl, resource, encoding, + fhirContext, fileLocation); + HttpEntityEnclosingRequestBase httpRequest = configureHttpRequest(fhirServerUrl, resource, encoding, + fhirContext); + tasks.put(resource, createHttpRequestCallable(httpRequest, httpRequestPojo)); } catch (Exception e) { - logger.error("Error while submitting the POST request: " + e.getMessage(), e); + logger.error("Error while submitting the HTTP request: " + e.getMessage(), e); } } /** - * Configures and prepares an HTTP POST request with the specified parameters. + * Configures and prepares an HTTP request with the specified parameters. *

- * This method creates and configures an HTTP POST request to be used for posting a FHIR resource to the given FHIR server. - * It sets the request's headers, encodes the FHIR resource, and sets request timeouts. If an unsupported encoding type is + * This method creates and configures an HTTP request to be used for sending a + * FHIR resource to the given FHIR server. + * It sets the request's headers, encodes the FHIR resource, and sets request + * timeouts. If an unsupported encoding type is * encountered, it throws a runtime exception. * - * @param fhirServerUrl The URL of the FHIR server to which the POST request will be sent. - * @param resource The FHIR resource to be posted. + * @param fhirServerUrl The URL of the FHIR server to which the request will be + * sent. + * @param resource The FHIR resource to be sent. * @param encoding The encoding type of the resource. * @param fhirContext The FHIR context for the resource. - * @return An HTTP POST request configured for the FHIR server and resource. + * @return An HTTP request configured for the FHIR server and resource. */ - private static HttpPost configureHttpPost(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext) { - - //Transaction bundles get posted to /fhir but other resources get posted to /fhir/resourceType ie fhir/Group - String fhirServer = fhirServerUrl; - if (!BundleUtils.resourceIsTransactionBundle(resource)) { - fhirServer = fhirServer + - (fhirServerUrl.endsWith("/") ? resource.fhirType() - : "/" + resource.fhirType()); - } + private static HttpEntityEnclosingRequestBase configureHttpRequest(String fhirServerUrl, IBaseResource resource, + IOUtils.Encoding encoding, FhirContext fhirContext) { + + // Transaction bundles get POST to /fhir but other resources get PUT to + // /fhir/resourceType/id ie fhir/Group/Group-123456 + String fhirUrl = fhirServerUrl.endsWith("/") ? fhirServerUrl : fhirServerUrl + "/"; + + if (BundleUtils.resourceIsTransactionBundle(resource)) { + HttpPost post = new HttpPost(fhirUrl); + post.addHeader("content-type", "application/" + encoding.toString()); + String resourceString = IOUtils.encodeResourceAsString(resource, encoding, fhirContext); + StringEntity input; + try { + input = new StringEntity(resourceString); + } catch (UnsupportedEncodingException e) { + throw new InvalidHttpRequestException(e); + } + post.setEntity(input); + post.setConfig(requestConfig); - HttpPost post = new HttpPost(fhirServer); - post.addHeader("content-type", "application/" + encoding.toString()); - String resourceString = IOUtils.encodeResourceAsString(resource, encoding, fhirContext); - StringEntity input; - try { - input = new StringEntity(resourceString); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - post.setEntity(input); - post.setConfig(requestConfig); + return post; + + } else { - return post; + String fhirServer = fhirUrl + "/" + resource.fhirType() + "/" + resource.getIdElement().getIdPart(); + + HttpPut put = new HttpPut(fhirServer); + put.addHeader("content-type", "application/" + encoding.toString()); + String resourceString = IOUtils.encodeResourceAsString(resource, encoding, fhirContext); + StringEntity input; + try { + input = new StringEntity(resourceString); + } catch (UnsupportedEncodingException e) { + throw new InvalidHttpRequestException(e); + } + put.setEntity(input); + put.setConfig(requestConfig); + + return put; + + } } /** - * Creates a callable task for executing an HTTP POST request and handling the response. + * Creates a callable task for executing an HTTP PUT request and handling the + * response. *

* This method constructs a callable task that performs the following steps: - * 1. Executes an HTTP POST request using the provided parameters. + * 1. Executes an HTTP PUT request using the provided parameters. * 2. Processes the HTTP response, checking the status code and reason phrase. * 3. Logs success or failure messages based on the response status. * 4. Handles exceptions related to the request and response. - * 5. Updates the progress and status of the post task. + * 5. Updates the progress and status of the put task. * - * @param post The HTTP POST request to be executed. - * @param postComponent A data object containing additional information about the POST request. - * @return A callable task for executing the HTTP POST request. + * @param request The HTTP PUT request to be executed. + * @param httpRequestComponent A data object containing additional information + * about the PUT request. + * @return A callable task for executing the HTTP PUT request. */ - private static Callable createPostCallable(HttpPost post, PostComponent postComponent) { + private static Callable createHttpRequestCallable(HttpEntityEnclosingRequestBase request, + HttpRequestComponent httpRequestComponent) { return () -> { - String resourceIdentifier = (postComponent.fileLocation != null ? - Paths.get(postComponent.fileLocation).getFileName().toString() - : - postComponent.resource.getIdElement().getIdPart()); - try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) { + String resourceIdentifier = (httpRequestComponent.fileLocation != null + ? Paths.get(httpRequestComponent.fileLocation).getFileName().toString() + : httpRequestComponent.resource.getIdElement().getIdPart()); + try { - HttpResponse response = httpClient.execute(post); + HttpResponse response = httpClient.execute(request); StatusLine statusLine = response.getStatusLine(); int statusCode = statusLine.getStatusCode(); String diagnosticString = getDiagnosticString(EntityUtils.toString(response.getEntity())); if (statusCode >= 200 && statusCode < 300) { - successfulPostCalls.add(buildSuccessMessage(postComponent.fhirServerUrl, resourceIdentifier)); - }else if (statusCode == 301){ - //redirected, find new location: + successfulHttpCalls + .add(buildSuccessMessage(httpRequestComponent.fhirServerUrl, resourceIdentifier)); + } else if (statusCode == 301) { + // redirected, find new location: Header locationHeader = response.getFirstHeader("Location"); if (locationHeader != null) { - postComponent.redirectFhirServerUrl = locationHeader.getValue(); - HttpPost redirectedPost = configureHttpPost(postComponent.redirectFhirServerUrl, postComponent.resource, postComponent.encoding, postComponent.fhirContext); - String redirectLocationIdentifier = postComponent.redirectFhirServerUrl - + "(redirected from " + postComponent.fhirServerUrl + ")"; - //attempt to post at location specified in redirect response: - try (CloseableHttpClient redirectHttpClient = HttpClientBuilder.create().build()) { - HttpResponse redirectResponse = redirectHttpClient.execute(redirectedPost); + httpRequestComponent.redirectFhirServerUrl = locationHeader.getValue(); + HttpEntityEnclosingRequestBase redirectedHttpRequest = configureHttpRequest( + httpRequestComponent.redirectFhirServerUrl, httpRequestComponent.resource, + httpRequestComponent.encoding, httpRequestComponent.fhirContext); + String redirectLocationIdentifier = httpRequestComponent.redirectFhirServerUrl + + "(redirected from " + httpRequestComponent.fhirServerUrl + ")"; + // attempt to put at location specified in redirect response: + try { + HttpResponse redirectResponse = httpClient.execute(redirectedHttpRequest); StatusLine redirectStatusLine = redirectResponse.getStatusLine(); int redirectStatusCode = redirectStatusLine.getStatusCode(); - String redirectDiagnosticString = getDiagnosticString(EntityUtils.toString(redirectResponse.getEntity())); + String redirectDiagnosticString = getDiagnosticString( + EntityUtils.toString(redirectResponse.getEntity())); - //treat new response same as we would before: + // treat new response same as we would before: if (redirectStatusCode >= 200 && redirectStatusCode < 300) { - successfulPostCalls.add(buildSuccessMessage(redirectLocationIdentifier, resourceIdentifier)); + successfulHttpCalls + .add(buildSuccessMessage(redirectLocationIdentifier, resourceIdentifier)); } else { - failedPostCalls.add(buildFailedPostMessage(postComponent, redirectStatusCode, redirectLocationIdentifier, resourceIdentifier, redirectDiagnosticString)); + failedHttpCalls.add(buildFailedHTTPMessage(httpRequestComponent, redirectStatusCode, + redirectLocationIdentifier, resourceIdentifier, redirectDiagnosticString)); } } catch (Exception e) { - failedPostCalls.add(buildExceptionMessage(postComponent, e, resourceIdentifier, redirectLocationIdentifier)); + failedHttpCalls.add(buildExceptionMessage(httpRequestComponent, e, resourceIdentifier, + redirectLocationIdentifier)); } } else { - //failed to extract a location from redirect message: - failedPostCalls.add(Pair.of("[FAIL] Exception during " + resourceIdentifier + " POST request execution to " - + postComponent.fhirServerUrl + ": Redirect, but no new location specified", postComponent)); + // failed to extract a location from redirect message: + failedHttpCalls.add( + Pair.of("[FAIL] Exception during " + resourceIdentifier + " HTTP request execution to " + + httpRequestComponent.fhirServerUrl + + ": Redirect, but no new location specified", httpRequestComponent)); } } else { - failedPostCalls.add(buildFailedPostMessage(postComponent, statusCode, postComponent.fhirServerUrl, resourceIdentifier, diagnosticString)); + failedHttpCalls.add(buildFailedHTTPMessage(httpRequestComponent, statusCode, + httpRequestComponent.fhirServerUrl, resourceIdentifier, diagnosticString)); } } catch (Exception e) { - failedPostCalls.add(buildExceptionMessage(postComponent, e, resourceIdentifier, postComponent.fhirServerUrl)); + failedHttpCalls.add(buildExceptionMessage(httpRequestComponent, e, resourceIdentifier, + httpRequestComponent.fhirServerUrl)); } - runningPostTaskList.remove(postComponent.resource); + runningRequestTaskList.remove(httpRequestComponent.resource); reportProgress(); return null; }; } - private static Pair buildExceptionMessage(PostComponent postComponent, Exception e, String resourceIdentifier, String locationIdentifier) { - return Pair.of("[FAIL] Exception during " + resourceIdentifier + " POST request execution to " + locationIdentifier + ": " + e.getMessage(), postComponent); + private static Pair buildExceptionMessage(HttpRequestComponent requestComponent, + Exception e, String resourceIdentifier, String locationIdentifier) { + return Pair.of("[FAIL] Exception during " + resourceIdentifier + " PUT request execution to " + + locationIdentifier + ": " + e.getMessage(), requestComponent); } - private static Pair buildFailedPostMessage(PostComponent postComponent, int statusCode, String locationIdentifier, String resourceIdentifier, String diagnosticString) { - return Pair.of("[FAIL] Error " + statusCode + " from " + locationIdentifier + ": " + resourceIdentifier + ": " + diagnosticString, postComponent); + private static Pair buildFailedHTTPMessage(HttpRequestComponent requestComponent, + int statusCode, String locationIdentifier, String resourceIdentifier, String diagnosticString) { + return Pair.of("[FAIL] Error " + statusCode + " from " + locationIdentifier + ": " + resourceIdentifier + ": " + + diagnosticString, requestComponent); } private static String buildSuccessMessage(String locationIdentifier, String resourceIdentifier) { - return "[SUCCESS] Resource successfully posted to " + locationIdentifier + ": " + resourceIdentifier; + return "[SUCCESS] Resource successfully sent to " + locationIdentifier + ": " + resourceIdentifier; } /** - * This method takes in a json string from the endpoint that might look like this: + * This method takes in a json string from the endpoint that might look like + * this: * { * "resourceType": "OperationOutcome", * "issue": [ { * "severity": "error", * "code": "processing", - * "diagnostics": "HAPI-1094: Resource Condition/delivery-of-singleton-f83c not found, specified in path: Encounter.diagnosis.condition" + * "diagnostics": "HAPI-1094: Resource Condition/delivery-of-singleton-f83c not + * found, specified in path: Encounter.diagnosis.condition" * } ] * } * It extracts the diagnostics and returns a string appendable to the response */ private static String getDiagnosticString(String jsonString) { try { - // Get the "diagnostics" property - return JsonParser.parseString(jsonString) - .getAsJsonObject() - .getAsJsonArray("issue") - .get(0) + JsonArray issues = JsonParser.parseString(jsonString) .getAsJsonObject() - .getAsJsonPrimitive("diagnostics") - .getAsString(); + .getAsJsonArray("issue"); + + StringBuilder diagnostics = new StringBuilder(); + for (JsonElement issueElement : issues) { + if (diagnostics.length() > 0) { + diagnostics.append("\n"); + } + JsonObject issueObject = issueElement.getAsJsonObject(); + String diagnostic = issueObject.getAsJsonPrimitive("diagnostics").getAsString(); + diagnostics.append(diagnostic); + } + + return diagnostics.toString(); } catch (Exception e) { return ""; } @@ -301,124 +393,127 @@ private static String getDiagnosticString(String jsonString) { /** * Reports the progress of HTTP POST calls and the current thread pool size. *

- * This method updates and prints the progress of HTTP POST calls by calculating the percentage of completed tasks - * relative to the total number of tasks. It also displays the current size of the running thread pool. The progress + * This method updates and prints the progress of HTTP POST calls by calculating + * the percentage of completed tasks + * relative to the total number of tasks. It also displays the current size of + * the running thread pool. The progress * and pool size information is printed to the standard output. */ private static void reportProgress() { - int currentCounter = processedPostCounter++; + int currentCounter = processedRequestCounter++; double percentage = (double) currentCounter / getTotalTaskCount() * 100; - System.out.print("\rPOST calls: " + String.format("%.2f%%", percentage) + " processed. POST response pool size: " + runningPostTaskList.size() + ". "); + System.out.print("\rPOST calls: " + String.format("%.2f%%", percentage) + + " processed. POST response pool size: " + runningRequestTaskList.size() + ". "); } private static int getTotalTaskCount() { - return tasks.size() + initialTasks.size(); + return tasks.size(); } /** - * Posts a collection of tasks to execute HTTP POST requests to a FHIR server. + * Puts a collection of tasks to execute HTTP requests to a FHIR server. *

- * This method orchestrates the execution of a collection of HTTP POST requests, each represented as a task. + * This method orchestrates the execution of a collection of HTTP requests, each + * represented as a task. * The method performs the following steps: * 1. Creates a thread pool using a fixed number of threads (usually 1). - * 2. Initiates the HTTP POST tasks for FHIR resources and monitors their progress. + * 2. Initiates the HTTP tasks for FHIR resources and monitors their progress. * 3. Collects and logs success or failure messages for each task. - * 4. Sorts and reports the results of the post tasks, both successful and failed. + * 4. Sorts and reports the results of the tasks, both successful and failed. * 5. Offers the option to retry failed tasks, if desired by the user. * 6. Cleans up resources and shuts down the thread pool when finished. *

- * This method serves as the entry point for posting tasks and provides progress monitoring and result reporting. + * This method serves as the entry point for sending tasks and provides progress + * monitoring and result reporting. */ - public static void postTaskCollection() { + public static void executeHttpRequestTaskCollection() { + ExecutorService executorService = Executors.newFixedThreadPool(1); try { - logger.info(getTotalTaskCount() + " POST calls to be made. Starting now. Please wait..."); - double percentage = 0; - System.out.print("\rPOST: " + String.format("%.2f%%", percentage) + " done. "); + logger.info("{} POST calls to be made. Starting now. Please wait...", getTotalTaskCount()); - //execute any tasks marked as having priority: - executeTasks(executorService, initialTasks); - - //execute the remaining tasks: + double percentage = 0; + String percentageDisplay = String.format("%.2f%%", percentage); + logger.info("\rPOST: {} done. ", percentageDisplay); + // execute the remaining tasks: executeTasks(executorService, tasks); reportProgress(); logger.info("Processing results..."); - Collections.sort(successfulPostCalls); + Collections.sort(successfulHttpCalls); StringBuilder message = new StringBuilder(); - for (String successPost : successfulPostCalls) { + for (String successPost : successfulHttpCalls) { message.append("\n").append(successPost); } - message.append("\r\n").append(successfulPostCalls.size()).append(" resources successfully posted."); - logger.info(message.toString()); - successfulPostCalls = new ArrayList<>(); + message.append("\r\n").append(successfulHttpCalls.size()).append(" resources successfully posted."); + String messageString = message.toString(); + logger.info(messageString); + successfulHttpCalls = new ArrayList<>(); - if (!failedPostCalls.isEmpty()) { - logger.info(failedPostCalls.size() + " tasks failed to POST. Retry these failed posts? (Y/N)"); + if (!failedHttpCalls.isEmpty()) { + logger.info("{} tasks failed to POST. Retry these failed posts? (Y/N)", failedHttpCalls.size()); Scanner scanner = new Scanner(System.in); String userInput = scanner.nextLine().trim().toLowerCase(); - + scanner.close(); if (userInput.equalsIgnoreCase("y")) { - List> failedPostCallList = new ArrayList<>(failedPostCalls); - cleanUp(); //clear the queue, reset the counter, start fresh + List> failedPostCallList = new ArrayList<>(failedHttpCalls); + cleanUp(); // clear the queue, reset the counter, start fresh - for (Pair pair : failedPostCallList) { - PostComponent postComponent = pair.getRight(); + for (Pair pair : failedPostCallList) { + HttpRequestComponent postComponent = pair.getRight(); try { - post(postComponent.fhirServerUrl, + sendToServer(postComponent.fhirServerUrl, postComponent.resource, postComponent.encoding, postComponent.fhirContext, - postComponent.fileLocation, - postComponent.hasPriority); + postComponent.fileLocation); } catch (IOException e) { - throw new RuntimeException(e); + throw new InvalidHttpRequestException(e); } } - //execute any tasks marked as having priority: - executeTasks(executorService, initialTasks); + - //execute the remaining tasks: + // execute the remaining tasks: executeTasks(executorService, tasks); reportProgress(); - if (failedPostCalls.isEmpty()) { + if (failedHttpCalls.isEmpty()) { logger.info("\r\nRetry successful, all tasks successfully posted"); } } } - if (!successfulPostCalls.isEmpty()) { + if (!successfulHttpCalls.isEmpty()) { message = new StringBuilder(); - for (String successPost : successfulPostCalls) { + for (String successPost : successfulHttpCalls) { message.append("\n").append(successPost); } - message.append("\r\n").append(successfulPostCalls.size()).append(" resources successfully posted."); - logger.info(message.toString()); - successfulPostCalls = new ArrayList<>(); + message.append("\r\n").append(successfulHttpCalls.size()).append(" resources successfully posted."); + messageString = message.toString(); + logger.info(messageString); + successfulHttpCalls = new ArrayList<>(); } - if (!failedPostCalls.isEmpty()) { + if (!failedHttpCalls.isEmpty()) { List failedMessages = new ArrayList<>(); - for (Pair pair : failedPostCalls) { + for (Pair pair : failedHttpCalls) { failedMessages.add(pair.getLeft()); } Collections.sort(failedMessages); message = new StringBuilder(); - for (String failedPost : failedMessages) { message.append("\n").append(failedPost); } - message.append("\r\n").append(failedMessages.size()).append(" resources failed to post."); - logger.info(message.toString()); + messageString = message.toString(); + logger.info(messageString); - writeFailedPostAttemptsToLog(failedMessages); + writeFailedHttpRequestAttemptsToLog(failedMessages); } } finally { @@ -428,32 +523,36 @@ public static void postTaskCollection() { } /** - * Gives the user a log file containing failed POST attempts during postTaskCollection() + * Gives the user a log file containing failed attempts during + * requestTaskCollection() + * * @param failedMessages */ - private static void writeFailedPostAttemptsToLog(List failedMessages) { + private static void writeFailedHttpRequestAttemptsToLog(List failedMessages) { if (!failedMessages.isEmpty()) { - //generate a unique filename based on simple timestamp: - String httpFailLogFilename = "http_post_fail_" + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + ".log"; + // generate a unique filename based on simple timestamp: + String httpFailLogFilename = "http_request_fail_" + + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + ".log"; try (BufferedWriter writer = new BufferedWriter(new FileWriter(httpFailLogFilename))) { for (String str : failedMessages) { writer.write(str + "\n"); } - logger.info("\r\nRecorded failed POST tasks to log file: " + new File(httpFailLogFilename).getAbsolutePath() + "\r\n"); + logger.info("\r\nRecorded failed HTTP tasks to log file: {} \r\n", new File(httpFailLogFilename).getAbsolutePath()); } catch (IOException e) { - logger.info("\r\nRecording of failed POST tasks to log failed with exception: " + e.getMessage() + "\r\n"); + logger.info("\r\nRecording of failed HTTP tasks to log failed with exception: {}\r\n", e.getMessage()); } } } + private static void executeTasks(ExecutorService executorService, + Map> executableTasksMap) { - private static void executeTasks(ExecutorService executorService, Map> executableTasksMap) { List> futures = new ArrayList<>(); List resources = new ArrayList<>(executableTasksMap.keySet()); for (int i = 0; i < resources.size(); i++) { IBaseResource thisResource = resources.get(i); - if (runningPostTaskList.size() < MAX_SIMULTANEOUS_POST_COUNT) { - runningPostTaskList.add(thisResource); + if (runningRequestTaskList.size() < MAX_SIMULTANEOUS_REQUEST_COUNT) { + runningRequestTaskList.add(thisResource); futures.add(executorService.submit(executableTasksMap.get(thisResource))); } else { threadSleep(10); @@ -472,7 +571,8 @@ private static void executeTasks(ExecutorService executorService, Map - * This method is responsible for resetting various data structures used during the processing of HTTP POST tasks. It performs the following actions: - * 1. Clears the queue of failed POST calls. - * 2. Clears the list of successful POST call results. + * This method is responsible for resetting various data structures used during + * the processing of HTTP PUT tasks. It performs the following actions: + * 1. Clears the queue of failed PUT calls. + * 2. Clears the list of successful PUT call results. * 3. Resets the map of tasks to be executed. - * 4. Resets the counter that tracks the number of processed POST calls. - * 5. Clears the list of resources currently being posted. + * 4. Resets the counter that tracks the number of processed PUT calls. + * 5. Clears the list of resources currently being puted. *

- * This method ensures a clean state and prepares the system for potential subsequent POST calls or retries. + * This method ensures a clean state and prepares the system for potential + * subsequent PUT calls or retries. */ private static void cleanUp() { - failedPostCalls = new ConcurrentLinkedQueue<>(); - successfulPostCalls = new CopyOnWriteArrayList<>(); - tasks = new ConcurrentHashMap<>(); - initialTasks = new ConcurrentHashMap<>(); - processedPostCounter = 0; - runningPostTaskList = new CopyOnWriteArrayList<>(); + failedHttpCalls = new ConcurrentLinkedQueue<>(); + successfulHttpCalls = new CopyOnWriteArrayList<>(); + processedRequestCounter = 0; + runningRequestTaskList = new CopyOnWriteArrayList<>(); } public static String get(String path) throws IOException { @@ -525,26 +626,28 @@ public static String getResponse(HttpResponse response) throws IOException { } /** - * A data class representing information needed for HTTP POST requests. + * A data class representing information needed for HTTP requests. *

- * The PostComponent class encapsulates the essential information required for making an HTTP POST request to a FHIR server. - * It includes the FHIR server URL, the FHIR resource to be posted, the encoding type, and the FHIR context. + * The httpRequestComponent class encapsulates the essential information + * required for making an HTTP request to a FHIR server. + * It includes the FHIR server URL, the FHIR resource to be sent, the encoding + * type, and the FHIR context. */ - private static class PostComponent { + private static class HttpRequestComponent { private final String fhirServerUrl; private String redirectFhirServerUrl; private final IBaseResource resource; private final IOUtils.Encoding encoding; private final FhirContext fhirContext; private final String fileLocation; - private final boolean hasPriority; - public PostComponent(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext, String fileLocation, boolean hasPriority) { + + public HttpRequestComponent(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, + FhirContext fhirContext, String fileLocation) { this.fhirServerUrl = fhirServerUrl; this.resource = resource; this.encoding = encoding; this.fhirContext = fhirContext; this.fileLocation = fileLocation; - this.hasPriority = hasPriority; } } @@ -559,4 +662,14 @@ public static ResponseHandler getDefaultResponseHandler() { } }; } -} \ No newline at end of file + + public static class InvalidHttpRequestException extends RuntimeException { + public InvalidHttpRequestException(Exception e) { + super(e); + } + + public InvalidHttpRequestException(String message) { + super(message); + } + } +} diff --git a/tooling/src/test/java/org/opencds/cqf/tooling/operation/RefreshIGOperationTest.java b/tooling/src/test/java/org/opencds/cqf/tooling/operation/RefreshIGOperationTest.java index 47f464b54..581814a63 100644 --- a/tooling/src/test/java/org/opencds/cqf/tooling/operation/RefreshIGOperationTest.java +++ b/tooling/src/test/java/org/opencds/cqf/tooling/operation/RefreshIGOperationTest.java @@ -6,6 +6,7 @@ import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.common.Json; +import com.github.tomakehurst.wiremock.matching.UrlPathPattern; import com.google.gson.*; import org.apache.commons.io.FileUtils; import org.hl7.fhir.instance.model.api.IBaseResource; @@ -34,75 +35,78 @@ import java.util.Map; import static org.testng.Assert.*; + public class RefreshIGOperationTest extends RefreshTest { - protected final Logger logger = LoggerFactory.getLogger(this.getClass()); - public RefreshIGOperationTest() { - super(FhirContext.forCached(FhirVersionEnum.R4)); - } + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); - private static final String EXCEPTIONS_OCCURRED_LOADING_IG_FILE = "Exceptions occurred loading IG file"; - private static final String EXCEPTIONS_OCCURRED_INITIALIZING_REFRESH_FROM_INI_FILE = "Exceptions occurred initializing refresh from ini file"; - private final String ID = "id"; - private final String ENTRY = "entry"; - private final String RESOURCE = "resource"; - private final String RESOURCE_TYPE = "resourceType"; - private final String BUNDLE_TYPE = "Bundle"; - private final String LIB_TYPE = "Library"; - private final String MEASURE_TYPE = "Measure"; + public RefreshIGOperationTest() { + super(FhirContext.forCached(FhirVersionEnum.R4)); + } - private final String INI_LOC = Path.of("target","refreshIG","ig.ini").toString(); + private static final String EXCEPTIONS_OCCURRED_LOADING_IG_FILE = "Exceptions occurred loading IG file"; + private static final String EXCEPTIONS_OCCURRED_INITIALIZING_REFRESH_FROM_INI_FILE = "Exceptions occurred initializing refresh from ini file"; + private final String ID = "id"; + private final String ENTRY = "entry"; + private final String RESOURCE = "resource"; + private final String RESOURCE_TYPE = "resourceType"; + private final String BUNDLE_TYPE = "Bundle"; + private final String LIB_TYPE = "Library"; + private final String MEASURE_TYPE = "Measure"; - private static final String[] NEW_REFRESH_IG_LIBRARY_FILE_NAMES = { - "GMTPInitialExpressions.json", "GMTPInitialExpressions.json", - "MBODAInitialExpressions.json", "USCoreCommon.json", "USCoreElements.json", "USCoreTests.json" - }; + private final String INI_LOC = Path.of("target", "refreshIG", "ig.ini").toString(); + + private static final String[] NEW_REFRESH_IG_LIBRARY_FILE_NAMES = { + "GMTPInitialExpressions.json", "GMTPInitialExpressions.json", + "MBODAInitialExpressions.json", "USCoreCommon.json", "USCoreElements.json", "USCoreTests.json" + }; private static final String TARGET_OUTPUT_FOLDER_PATH = "target" + separator + "NewRefreshIG"; - private static final String TARGET_OUTPUT_IG_CQL_FOLDER_PATH = TARGET_OUTPUT_FOLDER_PATH + separator + "input" + separator + "cql"; - private static final String TARGET_OUTPUT_IG_LIBRARY_FOLDER_PATH = TARGET_OUTPUT_FOLDER_PATH + separator + "input" + separator + "resources" + separator + "library"; - - // Store the original standard out before changing it. - private final PrintStream originalStdOut = System.out; - private ByteArrayOutputStream console = new ByteArrayOutputStream(); - - @BeforeClass - public void init() { - // This overrides the default max string length for Jackson (which wiremock uses under the hood). - var constraints = StreamReadConstraints.builder().maxStringLength(Integer.MAX_VALUE).build(); - Json.getObjectMapper().getFactory().setStreamReadConstraints(constraints); - } + private static final String TARGET_OUTPUT_IG_CQL_FOLDER_PATH = TARGET_OUTPUT_FOLDER_PATH + separator + "input" + separator + "cql"; + private static final String TARGET_OUTPUT_IG_LIBRARY_FOLDER_PATH = TARGET_OUTPUT_FOLDER_PATH + separator + "input" + separator + "resources" + separator + "library"; + + // Store the original standard out before changing it. + private final PrintStream originalStdOut = System.out; + private ByteArrayOutputStream console = new ByteArrayOutputStream(); + + @BeforeClass + public void init() { + // This overrides the default max string length for Jackson (which wiremock uses under the hood). + var constraints = StreamReadConstraints.builder().maxStringLength(Integer.MAX_VALUE).build(); + Json.getObjectMapper().getFactory().setStreamReadConstraints(constraints); + } - @BeforeMethod - public void setUp() throws Exception { - IOUtils.resourceDirectories = new ArrayList(); - IOUtils.clearDevicePaths(); - System.setOut(new PrintStream(this.console)); + @BeforeMethod + public void setUp() throws Exception { + IOUtils.resourceDirectories = new ArrayList(); + IOUtils.clearDevicePaths(); + System.setOut(new PrintStream(this.console)); - // Delete directories - deleteDirectory("target" + File.separator + "refreshIG"); - deleteDirectory("target" + File.separator + "NewRefreshIG"); + // Delete directories + deleteDirectory("target" + File.separator + "refreshIG"); + deleteDirectory("target" + File.separator + "NewRefreshIG"); - deleteTempINI(); - } + deleteTempINI(); + } - /** - * Attempts to delete a directory if it exists. - * @param path The path to the directory to delete. - */ - private void deleteDirectory(String path) { - File dir = new File(path); - if (dir.exists()) { - try { - FileUtils.deleteDirectory(dir); - } catch (IOException e) { - System.err.println("Failed to delete directory: " + path + " - " + e.getMessage()); - } - } - } + /** + * Attempts to delete a directory if it exists. + * + * @param path The path to the directory to delete. + */ + private void deleteDirectory(String path) { + File dir = new File(path); + if (dir.exists()) { + try { + FileUtils.deleteDirectory(dir); + } catch (IOException e) { + System.err.println("Failed to delete directory: " + path + " - " + e.getMessage()); + } + } + } - @Test - public void testNewRefreshOperation() throws IOException { - copyResourcesToTargetDir(TARGET_OUTPUT_FOLDER_PATH, "testfiles/NewRefreshIG"); + @Test + public void testNewRefreshOperation() throws Exception { + copyResourcesToTargetDir(TARGET_OUTPUT_FOLDER_PATH, "testfiles/NewRefreshIG"); File folder = new File(TARGET_OUTPUT_FOLDER_PATH); assertTrue(folder.exists(), "Folder should be present"); File jsonFile = new File(folder, "ig.ini");