diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/measure/MeasureBundler.java b/tooling/src/main/java/org/opencds/cqf/tooling/measure/MeasureBundler.java index 7a91a3df4..a04f0b482 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/measure/MeasureBundler.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/measure/MeasureBundler.java @@ -3,24 +3,21 @@ import ca.uhn.fhir.context.FhirContext; import org.hl7.fhir.instance.model.api.IBaseResource; import org.opencds.cqf.tooling.processor.AbstractBundler; -import org.opencds.cqf.tooling.utilities.HttpClientUtils; import org.opencds.cqf.tooling.utilities.IOUtils; -import org.opencds.cqf.tooling.utilities.IOUtils.Encoding; -import java.io.File; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; public class MeasureBundler extends AbstractBundler { public static final String ResourcePrefix = "measure-"; + protected CopyOnWriteArrayList identifiers; public static String getId(String baseId) { return ResourcePrefix + baseId; } + @Override protected String getSourcePath(FhirContext fhirContext, Map.Entry resourceEntry) { return IOUtils.getMeasurePathMap(fhirContext).get(resourceEntry.getKey()); @@ -41,67 +38,4 @@ protected Set getPaths(FhirContext fhirContext) { return IOUtils.getMeasurePaths(fhirContext); } - //so far only the Measure Bundle process needs to persist extra files: - @Override - protected int persistFilesFolder(String bundleDestPath, String libraryName, Encoding encoding, FhirContext fhirContext, String fhirUri) { - //persist tests-* before group-* files and make a record of which files were tracked: - List persistedFiles = persistTestFilesWithPriority(bundleDestPath, libraryName, encoding, fhirContext, fhirUri); - persistedFiles.addAll(persistEverythingElse(bundleDestPath, libraryName, encoding, fhirContext, fhirUri, persistedFiles)); - - return persistedFiles.size(); - } - - private List persistTestFilesWithPriority(String bundleDestPath, String libraryName, Encoding encoding, FhirContext fhirContext, String fhirUri) { - List persistedResources = new ArrayList<>(); - String filesLoc = bundleDestPath + File.separator + libraryName + "-files"; - File directory = new File(filesLoc); - if (directory.exists()) { - File[] filesInDir = directory.listFiles(); - if (!(filesInDir == null || filesInDir.length == 0)) { - for (File file : filesInDir) { - if (file.getName().toLowerCase().startsWith("tests-")) { - try { - IBaseResource resource = IOUtils.readResource(file.getAbsolutePath(), fhirContext, true); - HttpClientUtils.post(fhirUri, resource, encoding, fhirContext, file.getAbsolutePath(), true); - persistedResources.add(file.getAbsolutePath()); - } catch (Exception e) { - //resource is likely not IBaseResource - logger.error("MeasureBundler.persistTestFilesWithPriority", e); - } - } - } - } - } - return persistedResources; - } - - private List persistEverythingElse(String bundleDestPath, String libraryName, Encoding encoding, FhirContext fhirContext, String fhirUri, List alreadyPersisted) { - List persistedResources = new ArrayList<>(); - String filesLoc = bundleDestPath + File.separator + libraryName + "-files"; - File directory = new File(filesLoc); - if (directory.exists()) { - - File[] filesInDir = directory.listFiles(); - - if (!(filesInDir == null || filesInDir.length == 0)) { - for (File file : filesInDir) { - //don't post what has already been processed - if (alreadyPersisted.contains(file.getAbsolutePath())) { - continue; - } - if (file.getName().toLowerCase().endsWith(".json") || file.getName().toLowerCase().endsWith(".xml")) { - try { - IBaseResource resource = IOUtils.readResource(file.getAbsolutePath(), fhirContext, true); - HttpClientUtils.post(fhirUri, resource, encoding, fhirContext, file.getAbsolutePath(), false); - persistedResources.add(file.getAbsolutePath()); - } catch (Exception e) { - //resource is likely not IBaseResource - logger.error("persistEverythingElse", e); - } - } - } - } - } - return persistedResources; - } } diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/plandefinition/PlanDefinitionBundler.java b/tooling/src/main/java/org/opencds/cqf/tooling/plandefinition/PlanDefinitionBundler.java index 9e9852ded..e6b9acb93 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/plandefinition/PlanDefinitionBundler.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/plandefinition/PlanDefinitionBundler.java @@ -37,8 +37,4 @@ protected String getResourceBundlerType() { return TYPE_PLAN_DEFINITION; } - @Override - protected int persistFilesFolder(String bundleDestPath, String libraryName, IOUtils.Encoding encoding, FhirContext fhirContext, String fhirUri) { - return 0; - } } diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/processor/AbstractBundler.java b/tooling/src/main/java/org/opencds/cqf/tooling/processor/AbstractBundler.java index 870737f62..57a7fd117 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/processor/AbstractBundler.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/processor/AbstractBundler.java @@ -130,7 +130,7 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L final Map> cqlTranslatorErrorMessages = new ConcurrentHashMap<>(); //used to summarize file count user can expect to see in POST queue for each resource: - final Map persistedFileReport = new ConcurrentHashMap<>(); + final List persistedFileReport = new CopyOnWriteArrayList<>(); //build list of executable tasks to be sent to thread pool: List> tasks = new ArrayList<>(); @@ -142,7 +142,7 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L final Map libraryPathMap = new ConcurrentHashMap<>(IOUtils.getLibraryPathMap(fhirContext)); if (resourcesMap.isEmpty()) { - logger.info("[INFO] No " + getResourceBundlerType() + "s found. Continuing..."); + logger.info("\n\r" + "[INFO] No " + getResourceBundlerType() + "s found. Continuing...\n\r"); return; } @@ -169,7 +169,7 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L tasks.add(() -> { //check if resourceSourcePath has been processed before: if (processedResources.contains(resourceSourcePath)) { - logger.info(getResourceBundlerType() + " processed already: " + resourceSourcePath); + logger.info("\n\r" + getResourceBundlerType() + " processed already: " + resourceSourcePath); return null; } String resourceName = FilenameUtils.getBaseName(resourceSourcePath).replace(getResourcePrefix(), ""); @@ -246,17 +246,13 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L if (shouldPersist) { String bundleDestPath = FilenameUtils.concat(FilenameUtils.concat(IGProcessor.getBundlesPath(igPath), getResourceTestGroupName()), resourceName); - persistBundle(bundleDestPath, resourceName, encoding, fhirContext, new ArrayList(resources.values()), fhirUri, addBundleTimestamp); - bundleFiles(igPath, bundleDestPath, resourceName, binaryPaths, resourceSourcePath, primaryLibrarySourcePath, fhirContext, encoding, includeTerminology, includeDependencies, includePatientScenarios, includeVersion, addBundleTimestamp, cqlTranslatorErrorMessages); //If user supplied a fhir server url, inform them of total # of files to be persisted to the server: if (fhirUri != null && !fhirUri.isEmpty()) { - persistedFileReport.put(resourceName, - //+1 to account for -bundle - persistFilesFolder(bundleDestPath, resourceName, encoding, fhirContext, fhirUri) + 1); + persistedFileReport.add(resourceName); } if (cdsHooksProcessor != null) { @@ -264,6 +260,8 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L cdsHooksProcessor.addActivityDefinitionFilesToBundle(igPath, bundleDestPath, activityDefinitionPaths, fhirContext, encoding); } + persistBundle(bundleDestPath, resourceName, encoding, fhirContext, new ArrayList<>(resources.values()), fhirUri, addBundleTimestamp); + bundledResources.add(resourceSourcePath); } @@ -296,7 +294,7 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L //Output final report: String summaryOutput = generateBundleProcessSummary(refreshedLibraryNames, fhirContext, fhirUri, verboseMessaging, persistedFileReport, bundledResources, failedExceptionMessages, cqlTranslatorErrorMessages).toString(); - logger.info(summaryOutput); + logger.info("\n\r" + summaryOutput); } /** @@ -315,7 +313,7 @@ public void bundleResources(List refreshedLibraryNames, String igPath, L * @return A StringBuilder containing the generated summary message. */ private StringBuilder generateBundleProcessSummary(List refreshedLibraryNames, FhirContext fhirContext, - String fhirUri, Boolean verboseMessaging, Map persistedFileReport, + String fhirUri, Boolean verboseMessaging, List persistedFileReport, List bundledResources, Map failedExceptionMessages, Map> cqlTranslatorErrorMessages) { @@ -323,45 +321,8 @@ private StringBuilder generateBundleProcessSummary(List refreshedLibrary //Give user a snapshot of the files each resource will have persisted to their FHIR server (if fhirUri is provided) final int persistCount = persistedFileReport.size(); - if (persistCount > 0) { - String fileDisplay = " File(s): "; - summaryMessage.append(NEWLINE).append(persistCount).append(" ").append(getResourceBundlerType()).append("(s) have POST tasks in the queue for ").append(fhirUri).append(": "); - int totalQueueCount = 0; - List persistMessages = new ArrayList<>(); - for (String library : persistedFileReport.keySet()) { - totalQueueCount = totalQueueCount + persistedFileReport.get(library); - persistMessages.add(NEWLINE_INDENT - + persistedFileReport.get(library) - + fileDisplay - + library); - } - - //anon comparator class to sort by the file count for better presentation - persistMessages.sort(new Comparator<>() { - @Override - public int compare(String displayFileCount1, String displayFileCount2) { - int count1 = getFileCountFromString(displayFileCount1); - int count2 = getFileCountFromString(displayFileCount2); - return Integer.compare(count1, count2); - } - - private int getFileCountFromString(String fileName) { - int endIndex = fileName.indexOf(fileDisplay); - if (endIndex != -1) { - String countString = fileName.substring(0, endIndex).trim(); - return Integer.parseInt(countString); - } - return 0; - } - }); - - for (String persistMessage : persistMessages) { - summaryMessage.append(persistMessage); - } - summaryMessage.append(NEWLINE_INDENT) - .append("Total: ") - .append(totalQueueCount) - .append(" File(s)"); + if (!persistedFileReport.isEmpty()) { + summaryMessage.append(NEWLINE).append(persistCount).append(" ").append(getResourceBundlerType()).append("(s) have HTTP request tasks in the queue for ").append(fhirUri); } @@ -452,19 +413,16 @@ private void persistBundle(String bundleDestPath, String libraryName, IOUtils.Encoding encoding, FhirContext fhirContext, List resources, String fhirUri, Boolean addBundleTimestamp) throws IOException { - IOUtils.initializeDirectory(bundleDestPath); Object bundle = BundleUtils.bundleArtifacts(libraryName, resources, fhirContext, addBundleTimestamp, this.getIdentifiers()); IOUtils.writeBundle(bundle, bundleDestPath, encoding, fhirContext); if (fhirUri != null && !fhirUri.isEmpty()) { String resourceWriteLocation = bundleDestPath + separator + libraryName + "-bundle." + encoding; - HttpClientUtils.post(fhirUri, (IBaseResource) bundle, encoding, fhirContext, resourceWriteLocation, true); + //give resource the highest priority (0): + HttpClientUtils.sendToServer(fhirUri, (IBaseResource) bundle, encoding, fhirContext, resourceWriteLocation); } } - - protected abstract int persistFilesFolder(String bundleDestPath, String libraryName, IOUtils.Encoding encoding, FhirContext fhirContext, String fhirUri); - private void bundleFiles(String igPath, String bundleDestPath, String primaryLibraryName, List binaryPaths, String resourceFocusSourcePath, String librarySourcePath, FhirContext fhirContext, IOUtils.Encoding encoding, Boolean includeTerminology, Boolean includeDependencies, Boolean includePatientScenarios, Boolean includeVersion, Boolean addBundleTimestamp, Map> translatorWarningMessages) { @@ -476,7 +434,7 @@ private void bundleFiles(String igPath, String bundleDestPath, String primaryLib IOUtils.copyFile(librarySourcePath, FilenameUtils.concat(bundleDestFilesPath, FilenameUtils.getName(librarySourcePath))); String cqlFileName = IOUtils.formatFileName(FilenameUtils.getBaseName(librarySourcePath), IOUtils.Encoding.CQL, fhirContext); - if (cqlFileName.toLowerCase().startsWith("library-")) { + if (cqlFileName.toLowerCase().startsWith("library-") && !cqlFileName.toLowerCase().startsWith("library-deps-")) { cqlFileName = cqlFileName.substring(8); } String cqlLibrarySourcePath = IOUtils.getCqlLibrarySourcePath(primaryLibraryName, cqlFileName, binaryPaths); diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/processor/IGBundleProcessor.java b/tooling/src/main/java/org/opencds/cqf/tooling/processor/IGBundleProcessor.java index 369767b93..3c3136e92 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/processor/IGBundleProcessor.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/processor/IGBundleProcessor.java @@ -48,9 +48,9 @@ public void bundleIg(List refreshedLibraryNames, String igPath, List> resources = BundleUtils.getBundlesInDir(params.directoryPath, fhirContext); resources.forEach(entry -> postBundleToFhirUri(fhirUri, encoding, fhirContext, entry.getValue())); - if (HttpClientUtils.hasPostTasksInQueue()){ - HttpClientUtils.postTaskCollection(); + if (HttpClientUtils.hasHttpRequestTasksInQueue()){ + HttpClientUtils.executeHttpRequestTaskCollection(); } } private static void postBundleToFhirUri(String fhirUri, Encoding encoding, FhirContext fhirContext, IBaseResource bundle) { - if (fhirUri != null && !fhirUri.equals("")) { + if (fhirUri != null && !fhirUri.isEmpty()) { try { - HttpClientUtils.post(fhirUri, bundle, encoding, fhirContext, null); + HttpClientUtils.sendToServer(fhirUri, bundle, encoding, fhirContext, null); logger.info("Resource successfully posted to FHIR server ({}): {}", fhirUri, bundle.getIdElement().getIdPart()); } catch (Exception e) { logger.error("Error occurred for element {}: {}",bundle.getIdElement().getIdPart(), e.getMessage()); diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/questionnaire/QuestionnaireBundler.java b/tooling/src/main/java/org/opencds/cqf/tooling/questionnaire/QuestionnaireBundler.java index d605c9694..b3af38911 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/questionnaire/QuestionnaireBundler.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/questionnaire/QuestionnaireBundler.java @@ -35,11 +35,6 @@ protected String getResourceBundlerType() { return TYPE_QUESTIONNAIRE; } - @Override - protected int persistFilesFolder(String bundleDestPath, String libraryName, IOUtils.Encoding encoding, FhirContext fhirContext, String fhirUri) { - //do nothing - return 0; - } @Override protected Set getPaths(FhirContext fhirContext) { diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/utilities/BundleUtils.java b/tooling/src/main/java/org/opencds/cqf/tooling/utilities/BundleUtils.java index 26663c132..00ec22ebc 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/utilities/BundleUtils.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/utilities/BundleUtils.java @@ -278,4 +278,77 @@ public static boolean resourceIsTransactionBundle(IBaseResource inputResource) { return false; } + + public static List getResourceIdsFromTransactionBundle(IBaseResource inputResource) { + List returnedIDs = new ArrayList<>(); + + if (inputResource instanceof org.hl7.fhir.dstu3.model.Bundle) { + org.hl7.fhir.dstu3.model.Bundle bundle = (org.hl7.fhir.dstu3.model.Bundle) inputResource; + if (bundle.getType() == org.hl7.fhir.dstu3.model.Bundle.BundleType.TRANSACTION) { + for (org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { + if (entry.getResource() != null && entry.getResource().getIdElement() != null) { + if (entry.getResource() != null) { + String resourceId = entry.getResource().getIdElement().getIdPart(); + if (resourceId != null) { + returnedIDs.add(resourceId); + } else { + // Log a message if ID is missing (optional) + System.out.println("Resource in transaction bundle has no ID (likely a POST operation)."); + } + } + } + } + } + } else if (inputResource instanceof org.hl7.fhir.r4.model.Bundle) { + org.hl7.fhir.r4.model.Bundle bundle = (org.hl7.fhir.r4.model.Bundle) inputResource; + if (bundle.getType() == org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION) { + for (org.hl7.fhir.r4.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { + if (entry.getResource() != null && entry.getResource().getIdElement() != null) { + if (entry.getResource() != null) { + String resourceId = entry.getResource().getIdElement().getIdPart(); + if (resourceId != null) { + returnedIDs.add(resourceId); + } else { + // Log a message if ID is missing (optional) + System.out.println("Resource in transaction bundle has no ID (likely a POST operation)."); + } + } + } + } + } + } + + return returnedIDs; + } + + public static List getResourcesFromTransactionBundle(IBaseResource inputResource) { + List returnedResources = new ArrayList<>(); + + if (inputResource instanceof org.hl7.fhir.dstu3.model.Bundle) { + org.hl7.fhir.dstu3.model.Bundle bundle = (org.hl7.fhir.dstu3.model.Bundle) inputResource; + if (bundle.getType() == org.hl7.fhir.dstu3.model.Bundle.BundleType.TRANSACTION) { + for (org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { + if (entry.getResource() != null && entry.getResource().getIdElement() != null) { + if (entry.getResource() != null) { + returnedResources.add(entry.getResource()); + } + } + } + } + } else if (inputResource instanceof org.hl7.fhir.r4.model.Bundle) { + org.hl7.fhir.r4.model.Bundle bundle = (org.hl7.fhir.r4.model.Bundle) inputResource; + if (bundle.getType() == org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION) { + for (org.hl7.fhir.r4.model.Bundle.BundleEntryComponent entry : bundle.getEntry()) { + if (entry.getResource() != null && entry.getResource().getIdElement() != null) { + if (entry.getResource() != null) { + returnedResources.add(entry.getResource()); + } + } + } + } + } + + return returnedResources; + } + } diff --git a/tooling/src/main/java/org/opencds/cqf/tooling/utilities/HttpClientUtils.java b/tooling/src/main/java/org/opencds/cqf/tooling/utilities/HttpClientUtils.java index 5f10fc86f..70d3adf66 100644 --- a/tooling/src/main/java/org/opencds/cqf/tooling/utilities/HttpClientUtils.java +++ b/tooling/src/main/java/org/opencds/cqf/tooling/utilities/HttpClientUtils.java @@ -1,6 +1,9 @@ package org.opencds.cqf.tooling.utilities; import ca.uhn.fhir.context.FhirContext; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.apache.commons.lang3.tuple.Pair; import org.apache.http.Header; @@ -10,11 +13,15 @@ import org.apache.http.client.ClientProtocolException; import org.apache.http.client.ResponseHandler; import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; import org.hl7.fhir.instance.model.api.IBaseResource; import org.slf4j.Logger; @@ -43,37 +50,124 @@ public class HttpClientUtils { private static final String ENCODING_TYPE = "Encoding Type"; private static final String FHIR_CONTEXT = "FHIR Context"; - //This is not to maintain a thread count, but rather to maintain the maximum number of POST calls that can simultaneously be waiting for a response from the server. - //This gives us some control over how many POSTs we're making so we don't crash the server. - //possible TODO:Allow users to specify this value on their own with arg passed into operation so that more robust servers can process post list faster - private static final int MAX_SIMULTANEOUS_POST_COUNT = 10; + //This is not to maintain a thread count, but rather to maintain the maximum number of HTTP calls that can simultaneously be waiting for a response from the server. + //This gives us some control over how many HTTP requests we're making so we don't crash the server. + //possible TODO:Allow users to specify this value on their own with arg passed into operation so that more robust servers can process put list faster + private static final int MAX_SIMULTANEOUS_REQUEST_COUNT = 10; - //failedPostCalls needs to maintain the details built in the FAILED message, as well as a copy of the inputs for a retry by the user on failed posts. - private static Queue> failedPostCalls = new ConcurrentLinkedQueue<>(); - private static List successfulPostCalls = new CopyOnWriteArrayList<>(); - private static Map> tasks = new ConcurrentHashMap<>(); - private static Map> initialTasks = new ConcurrentHashMap<>(); - private static List runningPostTaskList = new CopyOnWriteArrayList<>(); - private static int processedPostCounter = 0; + //failedHttptCalls needs to maintain the details built in the FAILED message, as well as a copy of the inputs for a retry by the user on failed puts. + private static Queue> failedHttpCalls = new ConcurrentLinkedQueue<>(); + private static List successfulHttpCalls = new CopyOnWriteArrayList<>(); + + //Parent map uses resourceType as key so that resourceTypes can have their tasks called in a specific order: + private static ConcurrentHashMap>> mappedTasksByPriorityRank = new ConcurrentHashMap<>(); + + private static List runningRequestTaskList = new CopyOnWriteArrayList<>(); + private static int processedRequestCounter = 0; + + private static final CloseableHttpClient httpClient; + + static { + PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); + connectionManager.setMaxTotal(50); // Total max connections + connectionManager.setDefaultMaxPerRoute(10); // Max connections per route + + httpClient = HttpClients.custom() + .setConnectionManager(connectionManager) + .build(); + } + + + /** + * used for sending requests ordered by resource type (for reference validation): + * Terminology First: + * CODESYSTEM and VALUESET define critical terminologies that other resources may reference. For example, a Condition might reference a code from a CODESYSTEM. + * Logic Resources Next: + * LIBRARY resources are often foundational for clinical decision support or quality measures, and they might be referenced by PLANDEFINITION or MEASURE resources. + * Bundles: + * Transaction bundles need to be processed early to ensure their atomic operations complete and establish any interdependent resources before other resources reference them. + * Dependent Resources: + * Resources like PATIENT, GROUP, and CONDITION often serve as references for subsequent resources like OBSERVATION or CAREPLAN. + */ + private static final List resourceTypeOrder = new ArrayList<>(Arrays.asList( + HttpRequestResourceType.CODESYSTEM, + HttpRequestResourceType.VALUESET, + HttpRequestResourceType.LIBRARY, + HttpRequestResourceType.MEASURE, + HttpRequestResourceType.PLANDEFINITION, + HttpRequestResourceType.PATIENT, + HttpRequestResourceType.GROUP, + HttpRequestResourceType.CONDITION, + HttpRequestResourceType.OBSERVATION, + HttpRequestResourceType.PROCEDURE, + HttpRequestResourceType.CAREPLAN, + HttpRequestResourceType.GOAL, + HttpRequestResourceType.SERVICEREQUEST, + HttpRequestResourceType.MEDICATION, + HttpRequestResourceType.MEDICATIONREQUEST, + HttpRequestResourceType.DIAGNOSTICREPORT, + HttpRequestResourceType.IMAGINGSTUDY, + HttpRequestResourceType.OTHER, + HttpRequestResourceType.TRANSACTION_BUNDLE, + HttpRequestResourceType.BUNDLE + )); + + public enum HttpRequestResourceType { + CODESYSTEM("Code Systems"), + VALUESET("Value Sets"), + LIBRARY("Libraries"), + + MEASURE("Measures"), + PLANDEFINITION("Plan Definitions"), + PATIENT("Patients"), + GROUP("Groups"), + CONDITION("Conditions"), + OBSERVATION("Observations"), + PROCEDURE("Procedures"), + CAREPLAN("Care Plans"), + GOAL("Goals"), + SERVICEREQUEST("Service Requests"), + MEDICATION("Medications"), + MEDICATIONREQUEST("Medication Requests"), + DIAGNOSTICREPORT("Diagnostic Reports"), + IMAGINGSTUDY("Imaging Studies"), + OTHER("Everything Else"), + TRANSACTION_BUNDLE("Transaction Bundles"), + BUNDLE("Bundles"); + private final String displayName; + + HttpRequestResourceType(String displayName) { + this.displayName = displayName; + } + + public String getDisplayName() { + return displayName; + } + } private HttpClientUtils() { } - public static boolean hasPostTasksInQueue() { - return !tasks.isEmpty(); + public static boolean hasHttpRequestTasksInQueue() { + return !mappedTasksByPriorityRank.isEmpty(); } /** - * Initiates an HTTP POST request to a FHIR server with the specified parameters. + * Initiates an HTTP request to a FHIR server with the specified parameters. * - * @param fhirServerUrl The URL of the FHIR server to which the POST request will be sent. - * @param resource The FHIR resource to be posted. + * @param fhirServerUrl The URL of the FHIR server to which the request will be sent. + * @param resource The FHIR resource to be sent. * @param encoding The encoding type of the resource. * @param fhirContext The FHIR context for the resource. * @param fileLocation Optional fileLocation indicator for identifying resources by raw filename * @throws IOException If an I/O error occurs during the request. */ - public static void post(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext, String fileLocation, boolean withPriority) throws IOException { + public static void sendToServer(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext, String fileLocation) throws IOException { + + // A key in our mappedTasksByPriorityRank using HttpRequestResourceType. Allows specification of + // the resource type for priority ordering at execute. + HttpRequestResourceType priorityRank = categorizeResource(resource); + List missingValues = new ArrayList<>(); List values = new ArrayList<>(); validateAndAddValue(fhirServerUrl, FHIR_SERVER_URL, missingValues, values); @@ -83,16 +177,12 @@ public static void post(String fhirServerUrl, IBaseResource resource, IOUtils.En if (!missingValues.isEmpty()) { String missingValueString = String.join(", ", missingValues); - logger.error("An invalid HTTP POST call was attempted with a null value for: " + missingValueString + + logger.error("An invalid HTTP request was attempted with a null value for: " + missingValueString + (!values.isEmpty() ? "\\nRemaining values are: " + String.join(", ", values) : "")); return; } - createPostTask(fhirServerUrl, resource, encoding, fhirContext, fileLocation, withPriority); - } - - public static void post(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext, String fileLocation) throws IOException { - post(fhirServerUrl, resource, encoding, fhirContext, fileLocation, false); + createHttpRequestTask(fhirServerUrl, resource, encoding, fhirContext, fileLocation, priorityRank); } /** @@ -122,153 +212,180 @@ private static void validateAndAddValue(T value, String label, List } /** - * Creates a task for handling an HTTP POST request to a FHIR server with the specified parameters. + * Creates a task for handling an HTTP request to a FHIR server with the specified parameters. *

- * This method is responsible for creating a task that prepares and executes an HTTP POST request to the provided FHIR server + * This method is responsible for creating a task that prepares and executes an HTTP request to the provided FHIR server * with the given FHIR resource, encoding type, and FHIR context. It adds the task to the queue of tasks for later execution. * If any exceptions occur during task creation or configuration, an error message is logged. * - * @param fhirServerUrl The URL of the FHIR server to which the POST request will be sent. - * @param resource The FHIR resource to be posted. + * @param fhirServerUrl The URL of the FHIR server to which the request will be sent. * @param encoding The encoding type of the resource. * @param fhirContext The FHIR context for the resource. */ - private static void createPostTask(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext, String fileLocation, boolean withPriority) { + private static void createHttpRequestTask(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext, String fileLocation, HttpRequestResourceType priorityRank) { + + if (priorityRank == null) { + //added to last in the list: + priorityRank = HttpRequestResourceType.OTHER; + } + ConcurrentHashMap> theseTasks = new ConcurrentHashMap<>(); + try { - PostComponent postPojo = new PostComponent(fhirServerUrl, resource, encoding, fhirContext, fileLocation, withPriority); - HttpPost post = configureHttpPost(fhirServerUrl, resource, encoding, fhirContext); - if (withPriority) { - initialTasks.put(resource, createPostCallable(post, postPojo)); - } else { - tasks.put(resource, createPostCallable(post, postPojo)); - } + HttpRequestComponent httpRequestPojo = new HttpRequestComponent(fhirServerUrl, resource, encoding, fhirContext, fileLocation, priorityRank); + HttpEntityEnclosingRequestBase httpRequest = configureHttpRequest(fhirServerUrl, resource, encoding, fhirContext); + theseTasks.put(resource, createHttpRequestCallable(httpRequest, httpRequestPojo)); } catch (Exception e) { - logger.error("Error while submitting the POST request: " + e.getMessage(), e); + logger.error("Error while submitting the HTTP request: " + e.getMessage(), e); + } + + //check if a callable tasks map has started based on this resourceType: + if (mappedTasksByPriorityRank.containsKey(priorityRank)) { + mappedTasksByPriorityRank.get(priorityRank).putAll(theseTasks); + } else { + mappedTasksByPriorityRank.put(priorityRank, theseTasks); } } /** - * Configures and prepares an HTTP POST request with the specified parameters. + * Configures and prepares an HTTP request with the specified parameters. *

- * This method creates and configures an HTTP POST request to be used for posting a FHIR resource to the given FHIR server. + * This method creates and configures an HTTP request to be used for sending a FHIR resource to the given FHIR server. * It sets the request's headers, encodes the FHIR resource, and sets request timeouts. If an unsupported encoding type is * encountered, it throws a runtime exception. * - * @param fhirServerUrl The URL of the FHIR server to which the POST request will be sent. - * @param resource The FHIR resource to be posted. + * @param fhirServerUrl The URL of the FHIR server to which the request will be sent. + * @param resource The FHIR resource to be sent. * @param encoding The encoding type of the resource. * @param fhirContext The FHIR context for the resource. - * @return An HTTP POST request configured for the FHIR server and resource. + * @return An HTTP request configured for the FHIR server and resource. */ - private static HttpPost configureHttpPost(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext) { - - //Transaction bundles get posted to /fhir but other resources get posted to /fhir/resourceType ie fhir/Group - String fhirServer = fhirServerUrl; - if (!BundleUtils.resourceIsTransactionBundle(resource)) { - fhirServer = fhirServer + - (fhirServerUrl.endsWith("/") ? resource.fhirType() - : "/" + resource.fhirType()); - } + private static HttpEntityEnclosingRequestBase configureHttpRequest(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext) { - HttpPost post = new HttpPost(fhirServer); - post.addHeader("content-type", "application/" + encoding.toString()); - String resourceString = IOUtils.encodeResourceAsString(resource, encoding, fhirContext); - StringEntity input; - try { - input = new StringEntity(resourceString); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - post.setEntity(input); - post.setConfig(requestConfig); + //Transaction bundles get POST to /fhir but other resources get PUT to /fhir/resourceType/id ie fhir/Group/Group-123456 + String fhirUrl = fhirServerUrl.endsWith("/") ? fhirServerUrl : fhirServerUrl + "/"; + + if (BundleUtils.resourceIsTransactionBundle(resource)) { + HttpPost post = new HttpPost(fhirUrl); + post.addHeader("content-type", "application/" + encoding.toString()); + String resourceString = IOUtils.encodeResourceAsString(resource, encoding, fhirContext); + StringEntity input; + try { + input = new StringEntity(resourceString); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + post.setEntity(input); + post.setConfig(requestConfig); + + return post; + + } else { + + String fhirServer = fhirUrl + "/" + resource.fhirType() + "/" + resource.getIdElement().getIdPart(); + + HttpPut put = new HttpPut(fhirServer); + put.addHeader("content-type", "application/" + encoding.toString()); + String resourceString = IOUtils.encodeResourceAsString(resource, encoding, fhirContext); + StringEntity input; + try { + input = new StringEntity(resourceString); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + put.setEntity(input); + put.setConfig(requestConfig); - return post; + return put; + + + } } + /** - * Creates a callable task for executing an HTTP POST request and handling the response. + * Creates a callable task for executing an HTTP PUT request and handling the response. *

* This method constructs a callable task that performs the following steps: - * 1. Executes an HTTP POST request using the provided parameters. + * 1. Executes an HTTP PUT request using the provided parameters. * 2. Processes the HTTP response, checking the status code and reason phrase. * 3. Logs success or failure messages based on the response status. * 4. Handles exceptions related to the request and response. - * 5. Updates the progress and status of the post task. + * 5. Updates the progress and status of the put task. * - * @param post The HTTP POST request to be executed. - * @param postComponent A data object containing additional information about the POST request. - * @return A callable task for executing the HTTP POST request. + * @param request The HTTP PUT request to be executed. + * @param httpRequestComponent A data object containing additional information about the PUT request. + * @return A callable task for executing the HTTP PUT request. */ - private static Callable createPostCallable(HttpPost post, PostComponent postComponent) { + private static Callable createHttpRequestCallable(HttpEntityEnclosingRequestBase request, HttpRequestComponent httpRequestComponent) { return () -> { - String resourceIdentifier = (postComponent.fileLocation != null ? - Paths.get(postComponent.fileLocation).getFileName().toString() + String resourceIdentifier = (httpRequestComponent.fileLocation != null ? + Paths.get(httpRequestComponent.fileLocation).getFileName().toString() : - postComponent.resource.getIdElement().getIdPart()); - try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) { + httpRequestComponent.resource.getIdElement().getIdPart()); + try { - HttpResponse response = httpClient.execute(post); + HttpResponse response = httpClient.execute(request); StatusLine statusLine = response.getStatusLine(); int statusCode = statusLine.getStatusCode(); String diagnosticString = getDiagnosticString(EntityUtils.toString(response.getEntity())); if (statusCode >= 200 && statusCode < 300) { - successfulPostCalls.add(buildSuccessMessage(postComponent.fhirServerUrl, resourceIdentifier)); - }else if (statusCode == 301){ + successfulHttpCalls.add(buildSuccessMessage(httpRequestComponent.fhirServerUrl, resourceIdentifier)); + } else if (statusCode == 301) { //redirected, find new location: Header locationHeader = response.getFirstHeader("Location"); if (locationHeader != null) { - postComponent.redirectFhirServerUrl = locationHeader.getValue(); - HttpPost redirectedPost = configureHttpPost(postComponent.redirectFhirServerUrl, postComponent.resource, postComponent.encoding, postComponent.fhirContext); - String redirectLocationIdentifier = postComponent.redirectFhirServerUrl - + "(redirected from " + postComponent.fhirServerUrl + ")"; - //attempt to post at location specified in redirect response: - try (CloseableHttpClient redirectHttpClient = HttpClientBuilder.create().build()) { - HttpResponse redirectResponse = redirectHttpClient.execute(redirectedPost); + httpRequestComponent.redirectFhirServerUrl = locationHeader.getValue(); + HttpEntityEnclosingRequestBase redirectedHttpRequest = configureHttpRequest(httpRequestComponent.redirectFhirServerUrl, httpRequestComponent.resource, httpRequestComponent.encoding, httpRequestComponent.fhirContext); + String redirectLocationIdentifier = httpRequestComponent.redirectFhirServerUrl + + "(redirected from " + httpRequestComponent.fhirServerUrl + ")"; + //attempt to put at location specified in redirect response: + try { + HttpResponse redirectResponse = httpClient.execute(redirectedHttpRequest); StatusLine redirectStatusLine = redirectResponse.getStatusLine(); int redirectStatusCode = redirectStatusLine.getStatusCode(); String redirectDiagnosticString = getDiagnosticString(EntityUtils.toString(redirectResponse.getEntity())); //treat new response same as we would before: if (redirectStatusCode >= 200 && redirectStatusCode < 300) { - successfulPostCalls.add(buildSuccessMessage(redirectLocationIdentifier, resourceIdentifier)); + successfulHttpCalls.add(buildSuccessMessage(redirectLocationIdentifier, resourceIdentifier)); } else { - failedPostCalls.add(buildFailedPostMessage(postComponent, redirectStatusCode, redirectLocationIdentifier, resourceIdentifier, redirectDiagnosticString)); + failedHttpCalls.add(buildFailedHTTPMessage(httpRequestComponent, redirectStatusCode, redirectLocationIdentifier, resourceIdentifier, redirectDiagnosticString)); } } catch (Exception e) { - failedPostCalls.add(buildExceptionMessage(postComponent, e, resourceIdentifier, redirectLocationIdentifier)); + failedHttpCalls.add(buildExceptionMessage(httpRequestComponent, e, resourceIdentifier, redirectLocationIdentifier)); } } else { //failed to extract a location from redirect message: - failedPostCalls.add(Pair.of("[FAIL] Exception during " + resourceIdentifier + " POST request execution to " - + postComponent.fhirServerUrl + ": Redirect, but no new location specified", postComponent)); + failedHttpCalls.add(Pair.of("[FAIL] Exception during " + resourceIdentifier + " HTTP request execution to " + + httpRequestComponent.fhirServerUrl + ": Redirect, but no new location specified", httpRequestComponent)); } } else { - failedPostCalls.add(buildFailedPostMessage(postComponent, statusCode, postComponent.fhirServerUrl, resourceIdentifier, diagnosticString)); + failedHttpCalls.add(buildFailedHTTPMessage(httpRequestComponent, statusCode, httpRequestComponent.fhirServerUrl, resourceIdentifier, diagnosticString)); } } catch (Exception e) { - failedPostCalls.add(buildExceptionMessage(postComponent, e, resourceIdentifier, postComponent.fhirServerUrl)); + failedHttpCalls.add(buildExceptionMessage(httpRequestComponent, e, resourceIdentifier, httpRequestComponent.fhirServerUrl)); } - runningPostTaskList.remove(postComponent.resource); - reportProgress(); + runningRequestTaskList.remove(httpRequestComponent.resource); + reportProgress(httpRequestComponent.type); return null; }; } - private static Pair buildExceptionMessage(PostComponent postComponent, Exception e, String resourceIdentifier, String locationIdentifier) { - return Pair.of("[FAIL] Exception during " + resourceIdentifier + " POST request execution to " + locationIdentifier + ": " + e.getMessage(), postComponent); + private static Pair buildExceptionMessage(HttpRequestComponent requestComponent, Exception e, String resourceIdentifier, String locationIdentifier) { + return Pair.of("[FAIL] Exception during " + resourceIdentifier + " PUT request execution to " + locationIdentifier + ": " + e.getMessage(), requestComponent); } - private static Pair buildFailedPostMessage(PostComponent postComponent, int statusCode, String locationIdentifier, String resourceIdentifier, String diagnosticString) { - return Pair.of("[FAIL] Error " + statusCode + " from " + locationIdentifier + ": " + resourceIdentifier + ": " + diagnosticString, postComponent); + private static Pair buildFailedHTTPMessage(HttpRequestComponent requestComponent, int statusCode, String locationIdentifier, String resourceIdentifier, String diagnosticString) { + return Pair.of("[FAIL] Error " + statusCode + " from " + locationIdentifier + ": " + resourceIdentifier + ": " + diagnosticString, requestComponent); } private static String buildSuccessMessage(String locationIdentifier, String resourceIdentifier) { - return "[SUCCESS] Resource successfully posted to " + locationIdentifier + ": " + resourceIdentifier; + return "[SUCCESS] Resource successfully sent to " + locationIdentifier + ": " + resourceIdentifier; } /** @@ -285,142 +402,209 @@ private static String buildSuccessMessage(String locationIdentifier, String reso */ private static String getDiagnosticString(String jsonString) { try { - // Get the "diagnostics" property - return JsonParser.parseString(jsonString) - .getAsJsonObject() - .getAsJsonArray("issue") - .get(0) + JsonArray issues = JsonParser.parseString(jsonString) .getAsJsonObject() - .getAsJsonPrimitive("diagnostics") - .getAsString(); + .getAsJsonArray("issue"); + + StringBuilder diagnostics = new StringBuilder(); + for (JsonElement issueElement : issues) { + if (diagnostics.length() > 0) { + diagnostics.append("\n"); + } + JsonObject issueObject = issueElement.getAsJsonObject(); + String diagnostic = issueObject.getAsJsonPrimitive("diagnostics").getAsString(); + diagnostics.append(diagnostic); + } + + return diagnostics.toString(); } catch (Exception e) { return ""; } } + private static String getSectionProgress(HttpRequestResourceType httpRequestResourceType) { + //processedPutCounter holds the count we're at. + //logically all preceding items up until this point are processed + //add all sections until we get to this type, subtract total processed from other type count + //get percentage from that using (processedPutCounter - theseTypesCount) / currentTypeProcessedCount + + int otherTypesProcessedCount = 0; + + for (int i = 0; i < resourceTypeOrder.size(); i++) { + HttpRequestResourceType iterType = resourceTypeOrder.get(i); + if (iterType == httpRequestResourceType) { + break; + } + if (mappedTasksByPriorityRank.containsKey(iterType)) { + otherTypesProcessedCount = otherTypesProcessedCount + (mappedTasksByPriorityRank.get(iterType)).size(); + } + } + + int thisTypeProcessedCount = mappedTasksByPriorityRank.get(httpRequestResourceType).size(); + int currentCounter = processedRequestCounter - otherTypesProcessedCount; + + double percentage = (double) currentCounter / thisTypeProcessedCount * 100; + + return currentCounter + "/" + thisTypeProcessedCount + " (" + String.format("%.2f%%", percentage) + ")"; + } + /** - * Reports the progress of HTTP POST calls and the current thread pool size. + * Reports the progress of HTTP calls and the current thread pool size. *

- * This method updates and prints the progress of HTTP POST calls by calculating the percentage of completed tasks + * This method updates and prints the progress of HTTP calls by calculating the percentage of completed tasks * relative to the total number of tasks. It also displays the current size of the running thread pool. The progress * and pool size information is printed to the standard output. */ + private static void reportProgress(HttpRequestResourceType httpRequestResourceType) { + int currentCounter = processedRequestCounter++; + + String fileGroup = ""; + if (httpRequestResourceType != null) { + fileGroup = " | Sending: " + httpRequestResourceType.getDisplayName() + " " + getSectionProgress(httpRequestResourceType); + } + + int taskCount = getTotalTaskCount(); + double percentage = (double) currentCounter / taskCount * 100; + String percentOutput = String.format("%.2f%%", percentage); + + String progressStr = "\rProgress: " + percentOutput + " (" + currentCounter + "/" + taskCount + ")" + + fileGroup + + " | Response pool: " + runningRequestTaskList.size(); + + + String repeatedString = " ".repeat(progressStr.length() * 2); + System.out.print(repeatedString); + + System.out.print(progressStr); + } + + private static void reportProgress() { - int currentCounter = processedPostCounter++; - double percentage = (double) currentCounter / getTotalTaskCount() * 100; - System.out.print("\rPOST calls: " + String.format("%.2f%%", percentage) + " processed. POST response pool size: " + runningPostTaskList.size() + ". "); + reportProgress(null); } private static int getTotalTaskCount() { - return tasks.size() + initialTasks.size(); + int totalCount = 0; + for (ConcurrentHashMap> map : mappedTasksByPriorityRank.values()) { + totalCount += map.size(); + } + return totalCount; + } + + private static String getPresentTypesAndCounts() { + StringBuilder output = new StringBuilder(); + for (int i = 0; i < resourceTypeOrder.size(); i++) { + //execute the tasks in order specified + if (mappedTasksByPriorityRank.containsKey(resourceTypeOrder.get(i))) { + output.append("\n\r - ") + .append(resourceTypeOrder.get(i).displayName) + .append(": ") + .append(mappedTasksByPriorityRank.get(resourceTypeOrder.get(i)).size()); + } + } + + return output.toString(); } /** - * Posts a collection of tasks to execute HTTP POST requests to a FHIR server. + * Puts a collection of tasks to execute HTTP requests to a FHIR server. *

- * This method orchestrates the execution of a collection of HTTP POST requests, each represented as a task. + * This method orchestrates the execution of a collection of HTTP requests, each represented as a task. * The method performs the following steps: * 1. Creates a thread pool using a fixed number of threads (usually 1). - * 2. Initiates the HTTP POST tasks for FHIR resources and monitors their progress. + * 2. Initiates the HTTP tasks for FHIR resources and monitors their progress. * 3. Collects and logs success or failure messages for each task. - * 4. Sorts and reports the results of the post tasks, both successful and failed. + * 4. Sorts and reports the results of the tasks, both successful and failed. * 5. Offers the option to retry failed tasks, if desired by the user. * 6. Cleans up resources and shuts down the thread pool when finished. *

- * This method serves as the entry point for posting tasks and provides progress monitoring and result reporting. + * This method serves as the entry point for sending tasks and provides progress monitoring and result reporting. */ - public static void postTaskCollection() { + public static void executeHttpRequestTaskCollection() { + ExecutorService executorService = Executors.newFixedThreadPool(1); try { - logger.info(getTotalTaskCount() + " POST calls to be made. Starting now. Please wait..."); - double percentage = 0; - System.out.print("\rPOST: " + String.format("%.2f%%", percentage) + " done. "); - - //execute any tasks marked as having priority: - executeTasks(executorService, initialTasks); + logger.info("\n\r" + getTotalTaskCount() + + " HTTP calls to be made: " + + getPresentTypesAndCounts() + + "\n\r Executing, please wait..." + "\n\r"); - //execute the remaining tasks: - executeTasks(executorService, tasks); + runHttpRequestCalls(executorService); - reportProgress(); - logger.info("Processing results..."); - Collections.sort(successfulPostCalls); + logger.info("\n\r" + "Processing results..." + "\n\r"); + Collections.sort(successfulHttpCalls); StringBuilder message = new StringBuilder(); - for (String successPost : successfulPostCalls) { - message.append("\n").append(successPost); + for (String successRequest : successfulHttpCalls) { + message.append("\n").append(successRequest); } - message.append("\r\n").append(successfulPostCalls.size()).append(" resources successfully posted."); + message.append("\r\n").append(successfulHttpCalls.size()).append(" resources successfully sent."); logger.info(message.toString()); - successfulPostCalls = new ArrayList<>(); + successfulHttpCalls = new ArrayList<>(); - if (!failedPostCalls.isEmpty()) { - logger.info(failedPostCalls.size() + " tasks failed to POST. Retry these failed posts? (Y/N)"); + if (!failedHttpCalls.isEmpty()) { + logger.info("\n\r" + failedHttpCalls.size() + " tasks failed to send. Retry these failed requests? (Y/N)"); Scanner scanner = new Scanner(System.in); String userInput = scanner.nextLine().trim().toLowerCase(); if (userInput.equalsIgnoreCase("y")) { - List> failedPostCallList = new ArrayList<>(failedPostCalls); + List> failedPutCallList = new ArrayList<>(failedHttpCalls); cleanUp(); //clear the queue, reset the counter, start fresh - for (Pair pair : failedPostCallList) { - PostComponent postComponent = pair.getRight(); + for (Pair pair : failedPutCallList) { + HttpRequestComponent httpRequestComponent = pair.getRight(); try { - post(postComponent.fhirServerUrl, - postComponent.resource, - postComponent.encoding, - postComponent.fhirContext, - postComponent.fileLocation, - postComponent.hasPriority); + sendToServer(httpRequestComponent.fhirServerUrl, + httpRequestComponent.resource, + httpRequestComponent.encoding, + httpRequestComponent.fhirContext, + httpRequestComponent.fileLocation); } catch (IOException e) { throw new RuntimeException(e); } } - //execute any tasks marked as having priority: - executeTasks(executorService, initialTasks); - //execute the remaining tasks: - executeTasks(executorService, tasks); + runHttpRequestCalls(executorService); - reportProgress(); - if (failedPostCalls.isEmpty()) { - logger.info("\r\nRetry successful, all tasks successfully posted"); + if (failedHttpCalls.isEmpty()) { + logger.info("\r\nRetry successful, all tasks successfully sent."); } } } - if (!successfulPostCalls.isEmpty()) { + if (!successfulHttpCalls.isEmpty()) { message = new StringBuilder(); - for (String successPost : successfulPostCalls) { - message.append("\n").append(successPost); + for (String successPut : successfulHttpCalls) { + message.append("\n").append(successPut); } - message.append("\r\n").append(successfulPostCalls.size()).append(" resources successfully posted."); + message.append("\r\n").append(successfulHttpCalls.size()).append(" resources successfully sent."); logger.info(message.toString()); - successfulPostCalls = new ArrayList<>(); + successfulHttpCalls = new ArrayList<>(); } - if (!failedPostCalls.isEmpty()) { + if (!failedHttpCalls.isEmpty()) { List failedMessages = new ArrayList<>(); - for (Pair pair : failedPostCalls) { + for (Pair pair : failedHttpCalls) { failedMessages.add(pair.getLeft()); } Collections.sort(failedMessages); message = new StringBuilder(); - for (String failedPost : failedMessages) { - message.append("\n").append(failedPost); + for (String failedRequest : failedMessages) { + message.append("\n").append(failedRequest); } - message.append("\r\n").append(failedMessages.size()).append(" resources failed to post."); + message.append("\r\n").append(failedMessages.size()).append(" resources failed to send."); logger.info(message.toString()); - writeFailedPostAttemptsToLog(failedMessages); + writeFailedHttpRequestAttemptsToLog(failedMessages); } + } finally { cleanUp(); executorService.shutdown(); @@ -428,32 +612,52 @@ public static void postTaskCollection() { } /** - * Gives the user a log file containing failed POST attempts during postTaskCollection() + * Executes the tasks for each priority ranking, sorted: + * + * @param executorService + */ + private static void runHttpRequestCalls(ExecutorService executorService) { + //execute the tasks for each priority ranking, sorted: + for (int i = 0; i < resourceTypeOrder.size(); i++) { + //execute the tasks in order specified + if (mappedTasksByPriorityRank.containsKey(resourceTypeOrder.get(i))) { + executeTasks(executorService, mappedTasksByPriorityRank.get(resourceTypeOrder.get(i))); + } + } + + reportProgress(); + } + + /** + * Gives the user a log file containing failed attempts during requestTaskCollection() + * * @param failedMessages */ - private static void writeFailedPostAttemptsToLog(List failedMessages) { + private static void writeFailedHttpRequestAttemptsToLog(List failedMessages) { if (!failedMessages.isEmpty()) { //generate a unique filename based on simple timestamp: - String httpFailLogFilename = "http_post_fail_" + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + ".log"; + String httpFailLogFilename = "http_request_fail_" + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + ".log"; try (BufferedWriter writer = new BufferedWriter(new FileWriter(httpFailLogFilename))) { for (String str : failedMessages) { writer.write(str + "\n"); } - logger.info("\r\nRecorded failed POST tasks to log file: " + new File(httpFailLogFilename).getAbsolutePath() + "\r\n"); + logger.info("\r\nRecorded failed HTTP tasks to log file: " + new File(httpFailLogFilename).getAbsolutePath() + "\r\n"); } catch (IOException e) { - logger.info("\r\nRecording of failed POST tasks to log failed with exception: " + e.getMessage() + "\r\n"); + logger.info("\r\nRecording of failed HTTP tasks to log failed with exception: " + e.getMessage() + "\r\n"); } } } private static void executeTasks(ExecutorService executorService, Map> executableTasksMap) { + + List> futures = new ArrayList<>(); List resources = new ArrayList<>(executableTasksMap.keySet()); for (int i = 0; i < resources.size(); i++) { IBaseResource thisResource = resources.get(i); - if (runningPostTaskList.size() < MAX_SIMULTANEOUS_POST_COUNT) { - runningPostTaskList.add(thisResource); + if (runningRequestTaskList.size() < MAX_SIMULTANEOUS_REQUEST_COUNT) { + runningRequestTaskList.add(thisResource); futures.add(executorService.submit(executableTasksMap.get(thisResource))); } else { threadSleep(10); @@ -481,31 +685,92 @@ private static void threadSleep(int i) { try { Thread.sleep(i); } catch (InterruptedException e) { - logger.error("postTaskCollection", new RuntimeException(e)); + logger.error("httpRequestTaskCollection", new RuntimeException(e)); } } /** - * Cleans up and resets internal data structures after processing HTTP POST tasks. + * Cleans up and resets internal data structures after processing HTTP PUT tasks. *

- * This method is responsible for resetting various data structures used during the processing of HTTP POST tasks. It performs the following actions: - * 1. Clears the queue of failed POST calls. - * 2. Clears the list of successful POST call results. + * This method is responsible for resetting various data structures used during the processing of HTTP PUT tasks. It performs the following actions: + * 1. Clears the queue of failed PUT calls. + * 2. Clears the list of successful PUT call results. * 3. Resets the map of tasks to be executed. - * 4. Resets the counter that tracks the number of processed POST calls. - * 5. Clears the list of resources currently being posted. + * 4. Resets the counter that tracks the number of processed PUT calls. + * 5. Clears the list of resources currently being puted. *

- * This method ensures a clean state and prepares the system for potential subsequent POST calls or retries. + * This method ensures a clean state and prepares the system for potential subsequent PUT calls or retries. */ private static void cleanUp() { - failedPostCalls = new ConcurrentLinkedQueue<>(); - successfulPostCalls = new CopyOnWriteArrayList<>(); - tasks = new ConcurrentHashMap<>(); - initialTasks = new ConcurrentHashMap<>(); - processedPostCounter = 0; - runningPostTaskList = new CopyOnWriteArrayList<>(); + failedHttpCalls = new ConcurrentLinkedQueue<>(); + successfulHttpCalls = new CopyOnWriteArrayList<>(); + mappedTasksByPriorityRank = new ConcurrentHashMap<>(); + processedRequestCounter = 0; + runningRequestTaskList = new CopyOnWriteArrayList<>(); + } + + + public static HttpRequestResourceType categorizeResource(IBaseResource inputResource) { + if (inputResource == null) { + return HttpRequestResourceType.OTHER; + } + + // Check resource type and assign category + if (inputResource instanceof org.hl7.fhir.dstu3.model.Bundle || inputResource instanceof org.hl7.fhir.r4.model.Bundle) { + org.hl7.fhir.instance.model.api.IBaseBundle bundle = (org.hl7.fhir.instance.model.api.IBaseBundle) inputResource; + if (bundle instanceof org.hl7.fhir.dstu3.model.Bundle && + ((org.hl7.fhir.dstu3.model.Bundle) bundle).getType() == org.hl7.fhir.dstu3.model.Bundle.BundleType.TRANSACTION) { + return HttpRequestResourceType.TRANSACTION_BUNDLE; + } else if (bundle instanceof org.hl7.fhir.r4.model.Bundle && + ((org.hl7.fhir.r4.model.Bundle) bundle).getType() == org.hl7.fhir.r4.model.Bundle.BundleType.TRANSACTION) { + return HttpRequestResourceType.TRANSACTION_BUNDLE; + } else { + return HttpRequestResourceType.BUNDLE; + } + + + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.CodeSystem || inputResource instanceof org.hl7.fhir.r4.model.CodeSystem) { + return HttpRequestResourceType.CODESYSTEM; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.ValueSet || inputResource instanceof org.hl7.fhir.r4.model.ValueSet) { + return HttpRequestResourceType.VALUESET; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.Library || inputResource instanceof org.hl7.fhir.r4.model.Library) { + return HttpRequestResourceType.LIBRARY; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.Measure || inputResource instanceof org.hl7.fhir.r4.model.Measure) { + return HttpRequestResourceType.MEASURE; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.PlanDefinition || inputResource instanceof org.hl7.fhir.r4.model.PlanDefinition) { + return HttpRequestResourceType.PLANDEFINITION; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.Patient || inputResource instanceof org.hl7.fhir.r4.model.Patient) { + return HttpRequestResourceType.PATIENT; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.Group || inputResource instanceof org.hl7.fhir.r4.model.Group) { + return HttpRequestResourceType.GROUP; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.Condition || inputResource instanceof org.hl7.fhir.r4.model.Condition) { + return HttpRequestResourceType.CONDITION; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.Observation || inputResource instanceof org.hl7.fhir.r4.model.Observation) { + return HttpRequestResourceType.OBSERVATION; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.Procedure || inputResource instanceof org.hl7.fhir.r4.model.Procedure) { + return HttpRequestResourceType.PROCEDURE; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.CarePlan || inputResource instanceof org.hl7.fhir.r4.model.CarePlan) { + return HttpRequestResourceType.CAREPLAN; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.Goal || inputResource instanceof org.hl7.fhir.r4.model.Goal) { + return HttpRequestResourceType.GOAL; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.ProcedureRequest || + inputResource instanceof org.hl7.fhir.r4.model.ServiceRequest) { + return HttpRequestResourceType.SERVICEREQUEST; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.ReferralRequest) { + return HttpRequestResourceType.SERVICEREQUEST; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.Medication || inputResource instanceof org.hl7.fhir.r4.model.Medication) { + return HttpRequestResourceType.MEDICATION; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.MedicationRequest || inputResource instanceof org.hl7.fhir.r4.model.MedicationRequest) { + return HttpRequestResourceType.MEDICATIONREQUEST; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.DiagnosticReport || inputResource instanceof org.hl7.fhir.r4.model.DiagnosticReport) { + return HttpRequestResourceType.DIAGNOSTICREPORT; + } else if (inputResource instanceof org.hl7.fhir.dstu3.model.ImagingStudy || inputResource instanceof org.hl7.fhir.r4.model.ImagingStudy) { + return HttpRequestResourceType.IMAGINGSTUDY; + } + return HttpRequestResourceType.OTHER; } + public static String get(String path) throws IOException { try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) { HttpGet get = new HttpGet(path); @@ -525,29 +790,32 @@ public static String getResponse(HttpResponse response) throws IOException { } /** - * A data class representing information needed for HTTP POST requests. + * A data class representing information needed for HTTP requests. *

- * The PostComponent class encapsulates the essential information required for making an HTTP POST request to a FHIR server. - * It includes the FHIR server URL, the FHIR resource to be posted, the encoding type, and the FHIR context. + * The httpRequestComponent class encapsulates the essential information required for making an HTTP request to a FHIR server. + * It includes the FHIR server URL, the FHIR resource to be sent, the encoding type, and the FHIR context. */ - private static class PostComponent { + private static class HttpRequestComponent { private final String fhirServerUrl; private String redirectFhirServerUrl; private final IBaseResource resource; private final IOUtils.Encoding encoding; private final FhirContext fhirContext; private final String fileLocation; - private final boolean hasPriority; - public PostComponent(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext, String fileLocation, boolean hasPriority) { + + private final HttpRequestResourceType type; + + public HttpRequestComponent(String fhirServerUrl, IBaseResource resource, IOUtils.Encoding encoding, FhirContext fhirContext, String fileLocation, HttpRequestResourceType type) { this.fhirServerUrl = fhirServerUrl; this.resource = resource; this.encoding = encoding; this.fhirContext = fhirContext; this.fileLocation = fileLocation; - this.hasPriority = hasPriority; + this.type = type; } } + public static ResponseHandler getDefaultResponseHandler() { return response -> { int status = response.getStatusLine().getStatusCode(); diff --git a/tooling/src/test/java/org/opencds/cqf/tooling/operation/RefreshIGOperationTest.java b/tooling/src/test/java/org/opencds/cqf/tooling/operation/RefreshIGOperationTest.java index d5fa136bc..3bb6c74e9 100644 --- a/tooling/src/test/java/org/opencds/cqf/tooling/operation/RefreshIGOperationTest.java +++ b/tooling/src/test/java/org/opencds/cqf/tooling/operation/RefreshIGOperationTest.java @@ -6,6 +6,7 @@ import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.common.Json; +import com.github.tomakehurst.wiremock.matching.UrlPathPattern; import com.google.gson.*; import org.apache.commons.io.FileUtils; import org.hl7.fhir.instance.model.api.IBaseResource; @@ -34,75 +35,78 @@ import java.util.Map; import static org.testng.Assert.*; + public class RefreshIGOperationTest extends RefreshTest { - protected final Logger logger = LoggerFactory.getLogger(this.getClass()); - public RefreshIGOperationTest() { - super(FhirContext.forCached(FhirVersionEnum.R4)); - } - - private static final String EXCEPTIONS_OCCURRED_LOADING_IG_FILE = "Exceptions occurred loading IG file"; - private static final String EXCEPTIONS_OCCURRED_INITIALIZING_REFRESH_FROM_INI_FILE = "Exceptions occurred initializing refresh from ini file"; - private final String ID = "id"; - private final String ENTRY = "entry"; - private final String RESOURCE = "resource"; - private final String RESOURCE_TYPE = "resourceType"; - private final String BUNDLE_TYPE = "Bundle"; - private final String LIB_TYPE = "Library"; - private final String MEASURE_TYPE = "Measure"; - - private final String INI_LOC = Path.of("target","refreshIG","ig.ini").toString(); - - private static final String[] NEW_REFRESH_IG_LIBRARY_FILE_NAMES = { - "GMTPInitialExpressions.json", "GMTPInitialExpressions.json", - "MBODAInitialExpressions.json", "USCoreCommon.json", "USCoreElements.json", "USCoreTests.json" - }; + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + + public RefreshIGOperationTest() { + super(FhirContext.forCached(FhirVersionEnum.R4)); + } + + private static final String EXCEPTIONS_OCCURRED_LOADING_IG_FILE = "Exceptions occurred loading IG file"; + private static final String EXCEPTIONS_OCCURRED_INITIALIZING_REFRESH_FROM_INI_FILE = "Exceptions occurred initializing refresh from ini file"; + private final String ID = "id"; + private final String ENTRY = "entry"; + private final String RESOURCE = "resource"; + private final String RESOURCE_TYPE = "resourceType"; + private final String BUNDLE_TYPE = "Bundle"; + private final String LIB_TYPE = "Library"; + private final String MEASURE_TYPE = "Measure"; + + private final String INI_LOC = Path.of("target", "refreshIG", "ig.ini").toString(); + + private static final String[] NEW_REFRESH_IG_LIBRARY_FILE_NAMES = { + "GMTPInitialExpressions.json", "GMTPInitialExpressions.json", + "MBODAInitialExpressions.json", "USCoreCommon.json", "USCoreElements.json", "USCoreTests.json" + }; private static final String TARGET_OUTPUT_FOLDER_PATH = "target" + separator + "NewRefreshIG"; - private static final String TARGET_OUTPUT_IG_CQL_FOLDER_PATH = TARGET_OUTPUT_FOLDER_PATH + separator + "input" + separator + "cql"; - private static final String TARGET_OUTPUT_IG_LIBRARY_FOLDER_PATH = TARGET_OUTPUT_FOLDER_PATH + separator + "input" + separator + "resources" + separator + "library"; - - // Store the original standard out before changing it. - private final PrintStream originalStdOut = System.out; - private ByteArrayOutputStream console = new ByteArrayOutputStream(); - - @BeforeClass - public void init() { - // This overrides the default max string length for Jackson (which wiremock uses under the hood). - var constraints = StreamReadConstraints.builder().maxStringLength(Integer.MAX_VALUE).build(); - Json.getObjectMapper().getFactory().setStreamReadConstraints(constraints); - } - - @BeforeMethod - public void setUp() throws Exception { - IOUtils.resourceDirectories = new ArrayList(); - IOUtils.clearDevicePaths(); - System.setOut(new PrintStream(this.console)); - - // Delete directories - deleteDirectory("target" + File.separator + "refreshIG"); - deleteDirectory("target" + File.separator + "NewRefreshIG"); - - deleteTempINI(); - } - - /** - * Attempts to delete a directory if it exists. - * @param path The path to the directory to delete. - */ - private void deleteDirectory(String path) { - File dir = new File(path); - if (dir.exists()) { - try { - FileUtils.deleteDirectory(dir); - } catch (IOException e) { - System.err.println("Failed to delete directory: " + path + " - " + e.getMessage()); - } - } - } - - @Test - public void testNewRefreshOperation() throws IOException { - copyResourcesToTargetDir(TARGET_OUTPUT_FOLDER_PATH, "testfiles/NewRefreshIG"); + private static final String TARGET_OUTPUT_IG_CQL_FOLDER_PATH = TARGET_OUTPUT_FOLDER_PATH + separator + "input" + separator + "cql"; + private static final String TARGET_OUTPUT_IG_LIBRARY_FOLDER_PATH = TARGET_OUTPUT_FOLDER_PATH + separator + "input" + separator + "resources" + separator + "library"; + + // Store the original standard out before changing it. + private final PrintStream originalStdOut = System.out; + private ByteArrayOutputStream console = new ByteArrayOutputStream(); + + @BeforeClass + public void init() { + // This overrides the default max string length for Jackson (which wiremock uses under the hood). + var constraints = StreamReadConstraints.builder().maxStringLength(Integer.MAX_VALUE).build(); + Json.getObjectMapper().getFactory().setStreamReadConstraints(constraints); + } + + @BeforeMethod + public void setUp() throws Exception { + IOUtils.resourceDirectories = new ArrayList(); + IOUtils.clearDevicePaths(); + System.setOut(new PrintStream(this.console)); + + // Delete directories + deleteDirectory("target" + File.separator + "refreshIG"); + deleteDirectory("target" + File.separator + "NewRefreshIG"); + + deleteTempINI(); + } + + /** + * Attempts to delete a directory if it exists. + * + * @param path The path to the directory to delete. + */ + private void deleteDirectory(String path) { + File dir = new File(path); + if (dir.exists()) { + try { + FileUtils.deleteDirectory(dir); + } catch (IOException e) { + System.err.println("Failed to delete directory: " + path + " - " + e.getMessage()); + } + } + } + + @Test + public void testNewRefreshOperation() throws Exception { + copyResourcesToTargetDir(TARGET_OUTPUT_FOLDER_PATH, "testfiles/NewRefreshIG"); File folder = new File(TARGET_OUTPUT_FOLDER_PATH); assertTrue(folder.exists(), "Folder should be present"); File jsonFile = new File(folder, "ig.ini"); @@ -193,377 +197,390 @@ private void verifyFileContent(File file, String expectedContent) { } } - @AfterSuite - public void cleanup() { - deleteDirectory("null"); - } - - /** - * This test breaks down refreshIG's process and can verify multiple bundles - */ - @SuppressWarnings("unchecked") - @Test - //TODO: Fix separately, this is blocking a bunch of other higher priority things - public void testBundledFiles() throws IOException { - //we can assert how many bundles were posted by keeping track via WireMockServer - //first find an open port: - int availablePort = findAvailablePort(); - String fhirUri = "http://localhost:" + availablePort + "/fhir/"; - if (availablePort == -1){ - fhirUri = ""; - logger.info("No available ports to test post with. Removing mock fhir server from test."); - }else{ - System.out.println("Available port: " + availablePort + ", mock fhir server url: " + fhirUri); - } - - WireMockServer wireMockServer = null; - if (!fhirUri.isEmpty()) { - wireMockServer = new WireMockServer(availablePort); - wireMockServer.start(); - - WireMock.configureFor("localhost", availablePort); - wireMockServer.stubFor(WireMock.post(WireMock.urlPathMatching("/fhir/([a-zA-Z]*)")) - .willReturn(WireMock.aResponse() - .withStatus(200) - .withBody("Mock response"))); - } - - // Call the method under test, which should use HttpClientUtils.post - copyResourcesToTargetDir("target" + separator + "refreshIG", "testfiles/refreshIG"); - // build ini object - File iniFile = new File(INI_LOC); - String iniFileLocation = iniFile.getAbsolutePath(); - IniFile ini = new IniFile(iniFileLocation); - - String bundledFilesLocation = iniFile.getParent() + separator + "bundles" + separator + "measure" + separator; - - String[] args; - if (!fhirUri.isEmpty()) { - args = new String[]{"-RefreshIG", "-ini=" + INI_LOC, "-t", "-d", "-p", "-e=json", "-ts=false", "-fs=" + fhirUri}; - } else { - args = new String[]{"-RefreshIG", "-ini=" + INI_LOC, "-t", "-d", "-p", "-e=json", "-ts=false"}; - } - - // EXECUTE REFRESHIG WITH OUR ARGS: - new RefreshIGOperation().execute(args); - - int requestCount = WireMock.getAllServeEvents().size(); - assertEquals(requestCount, 7); //Looking for 7 resources posted (all files found in -files ending in .cql, .xml, or .json) - - if (wireMockServer != null) { - wireMockServer.stop(); - } - - // determine fhirContext for measure lookup - FhirContext fhirContext = IGProcessor.getIgFhirContext(getFhirVersion(ini)); - - // get list of measures resulting from execution - Map measures = IOUtils.getMeasures(fhirContext); - - // loop through measure, verify each has all resources from multiple files - // bundled into single file using id/resourceType as lookup: - for (String measureName : measures.keySet()) { - // location of single bundled file: - final String bundledFileResult = bundledFilesLocation + measureName + separator + measureName - + "-bundle.json"; - // multiple individual files in sub directory to loop through: - final Path dir = Paths - .get(bundledFilesLocation + separator + measureName + separator + measureName + "-files"); - - // loop through each file, determine resourceType and treat accordingly - Map resourceTypeMap = new HashMap<>(); - List groupPatientList = new ArrayList<>(); - - try (final DirectoryStream dirStream = Files.newDirectoryStream(dir)) { - dirStream.forEach(path -> { - File file = new File(path.toString()); - - //Group file testing: - if (file.getName().equalsIgnoreCase("Group-BreastCancerScreeningFHIR.json")){ - try{ - org.hl7.fhir.r4.model.Group group = (org.hl7.fhir.r4.model.Group)IOUtils.readResource(file.getAbsolutePath(), fhirContext); - assertTrue(group.hasMember()); - // Check if the group contains members - // Iterate through the members - for (Group.GroupMemberComponent member : group.getMember()) { - groupPatientList.add(member.getEntity().getDisplay()); - } - }catch (Exception e){ - fail("Group-BreastCancerScreeningFHIR.json did not parse to valid Group instance."); - } - - } - - if (file.getName().toLowerCase().endsWith(".json")) { - - Map map = this.jsonMap(file); - if (map == null) { - System.out.println("# Unable to parse " + file.getName() + " as json"); - } else { - - // ensure "resourceType" exists - if (map.containsKey(RESOURCE_TYPE)) { - String parentResourceType = (String) map.get(RESOURCE_TYPE); - // if Library, resource will be translated into "Measure" in main bundled file: - if (parentResourceType.equalsIgnoreCase(LIB_TYPE)) { - resourceTypeMap.put((String) map.get(ID), MEASURE_TYPE); - } else if (parentResourceType.equalsIgnoreCase(BUNDLE_TYPE)) { - // file is a bundle type, loop through resources in entry list, build up map of - // : - if (map.get(ENTRY) != null) { - ArrayList> entryList = (ArrayList>) map.get(ENTRY); - for (Map entry : entryList) { - if (entry.containsKey(RESOURCE)) { - Map resourceMap = (Map) entry.get(RESOURCE); - resourceTypeMap.put((String) resourceMap.get(ID), - (String) resourceMap.get(RESOURCE_TYPE)); - } - } - } - } - } - } - } - }); - - } catch (IOException e) { - logger.info(e.getMessage()); - } - - //Group file should contain two patients: - assertEquals(groupPatientList.size(), 2); - - // map out entries in the resulting single bundle file: - Map bundledJson = this.jsonMap(new File(bundledFileResult)); - Map bundledJsonResourceTypes = new HashMap<>(); - ArrayList> entryList = (ArrayList>) bundledJson.get(ENTRY); - for (Map entry : entryList) { - Map resourceMap = (Map) entry.get(RESOURCE); - bundledJsonResourceTypes.put((String) resourceMap.get(ID), (String) resourceMap.get(RESOURCE_TYPE)); - } - - // compare mappings of to ensure all bundled correctly: - assertTrue(mapsAreEqual(resourceTypeMap, bundledJsonResourceTypes)); - } - - // run cleanup (maven runs all ci tests sequentially and static member variables could retain values from previous tests) - IOUtils.cleanUp(); - ResourceUtils.cleanUp(); - } - - private static int findAvailablePort() { - for (int port = 8000; port <= 9000; port++) { - if (isPortAvailable(port)) { - return port; - } - } - return -1; - } - - private static boolean isPortAvailable(int port) { - ServerSocket ss; - try (ServerSocket serverSocket = new ServerSocket(port)) { - System.out.println("Trying " + serverSocket); - ss = serverSocket; - } catch (IOException e) { - return false; - } - System.out.println(ss + " is open."); - return true; - } - - //@Test(expectedExceptions = IllegalArgumentException.class) - //TODO: Fix separately, this is blocking a bunch of other higher priority things - public void testNullArgs() { - new RefreshIGOperation().execute(null); - } - - //@Test - //TODO: Fix separately, this is blocking a bunch of other higher priority things - public void testBlankINILoc() { - String args[] = { "-RefreshIG", "-ini=", "-t", "-d", "-p" }; - - try { - new RefreshIGOperation().execute(args); - } catch (IllegalArgumentException e) { - assertEquals(e.getMessage(), IGProcessor.IG_VERSION_REQUIRED); - assertTrue(this.console.toString().indexOf("fhir-version was not specified in the ini file.") != -1); - } - } - - - //@Test - //TODO: Fix separately, this is blocking a bunch of other higher priority things - public void testInvalidIgVersion() { - Map igProperties = new HashMap(); - igProperties.put("ig", "nonsense"); - igProperties.put("template", "nonsense"); - igProperties.put("usage-stats-opt-out", "nonsense"); - igProperties.put("fhir-version", "nonsense"); - - File iniFile = this.createTempINI(igProperties); - - String args[] = { "-RefreshIG", "-ini=" + iniFile.getAbsolutePath(), "-t", "-d", "-p" }; - - if (iniFile != null) { - try { - new RefreshIGOperation().execute(args); - } catch (Exception e) { - assertTrue(e.getClass() == IllegalArgumentException.class); - assertTrue(this.console.toString().indexOf(EXCEPTIONS_OCCURRED_INITIALIZING_REFRESH_FROM_INI_FILE) != -1); - assertTrue(this.console.toString().indexOf("Unknown Version 'nonsense'") != -1); - - assertEquals(e.getMessage(), IGProcessor.IG_VERSION_REQUIRED); - } - deleteTempINI(); - } - } - - //@Test - //TODO: Fix separately, this is blocking a bunch of other higher priority things - public void testInvalidIgInput() { - Map igProperties = new HashMap(); - igProperties.put("ig", "nonsense"); - igProperties.put("template", "nonsense"); - igProperties.put("usage-stats-opt-out", "nonsense"); - igProperties.put("fhir-version", "4.0.1"); - - File iniFile = this.createTempINI(igProperties); - - String args[] = { "-RefreshIG", "-ini=" + iniFile.getAbsolutePath(), "-t", "-d", "-p" }; - - if (iniFile != null) { - try { - new RefreshIGOperation().execute(args); - } catch (Exception e) { - assertTrue(e.getClass() == IllegalArgumentException.class); - assertEquals(e.getMessage(), IGProcessor.IG_VERSION_REQUIRED); - - assertTrue(this.console.toString().indexOf(EXCEPTIONS_OCCURRED_LOADING_IG_FILE) != -1); - assertTrue(this.console.toString().indexOf(EXCEPTIONS_OCCURRED_INITIALIZING_REFRESH_FROM_INI_FILE) != -1); - } - deleteTempINI(); - } - } - - - //@Test - //TODO: Fix separately, this is blocking a bunch of other higher priority things - public void testParamsMissingINI() { - Map igProperties = new HashMap(); - igProperties.put("ig", "nonsense"); - igProperties.put("template", "nonsense"); - igProperties.put("usage-stats-opt-out", "nonsense"); - igProperties.put("fhir-version", "4.0.1"); - - File iniFile = this.createTempINI(igProperties); - - String[] args = { "-RefreshIG", "-ini=" + iniFile.getAbsolutePath(), "-t", "-d", "-p" }; - - RefreshIGParameters params = null; - try { - params = new RefreshIGArgumentProcessor().parseAndConvert(args); - } - catch (Exception e) { - System.err.println(e.getMessage()); - System.exit(1); - } - - //override ini to be null - params.ini = null; - - try { - new IGProcessor().publishIG(params); - } catch (Exception e) { - assertEquals(e.getClass(), NullPointerException.class); - } - - deleteTempINI(); - } - - - @AfterMethod - public void afterTest() { - deleteTempINI(); - System.setOut(this.originalStdOut); - System.out.println(this.console.toString()); - this.console = new ByteArrayOutputStream(); - } - - - private File createTempINI(Map properties) { + @AfterSuite + public void cleanup() { + deleteDirectory("null"); + } + + /** + * This test breaks down refreshIG's process and can verify multiple bundles + */ + @SuppressWarnings("unchecked") + @Test + //TODO: Fix separately, this is blocking a bunch of other higher priority things + public void testBundledFiles() throws Exception { + //we can assert how many bundles were posted by keeping track via WireMockServer + //first find an open port: + int availablePort = findAvailablePort(); + String fhirUri = "http://localhost:" + availablePort + "/fhir/"; + if (availablePort == -1) { + fhirUri = ""; + logger.info("No available ports to test post with. Removing mock fhir server from test."); + } else { + System.out.println("Available port: " + availablePort + ", mock fhir server url: " + fhirUri); + } + + WireMockServer wireMockServer = null; + if (!fhirUri.isEmpty()) { + wireMockServer = new WireMockServer(availablePort); + wireMockServer.start(); + + WireMock.configureFor("localhost", availablePort); + + // Match exact base path (e.g., POST to /fhir/) + WireMock.stubFor(WireMock.post(WireMock.urlEqualTo("/fhir/")) + .willReturn(WireMock.aResponse().withStatus(200).withBody("Mock response"))); + WireMock.stubFor(WireMock.put(WireMock.urlEqualTo("/fhir/")) + .willReturn(WireMock.aResponse().withStatus(200).withBody("Mock response"))); + + // Match resource-level (e.g., POST to /fhir/Library) + WireMock.stubFor(WireMock.post(WireMock.urlPathMatching("/fhir/[a-zA-Z]+")) + .willReturn(WireMock.aResponse().withStatus(200).withBody("Mock response"))); + WireMock.stubFor(WireMock.put(WireMock.urlPathMatching("/fhir/[a-zA-Z]+")) + .willReturn(WireMock.aResponse().withStatus(200).withBody("Mock response"))); + + // Match resource + ID (e.g., PUT to /fhir/Library/SomeID) + WireMock.stubFor(WireMock.put(WireMock.urlPathMatching("/fhir/[a-zA-Z]+/[a-zA-Z0-9\\-\\.]+")) + .willReturn(WireMock.aResponse().withStatus(200).withBody("Mock response"))); + + } + + // Call the method under test, which should use HttpClientUtils.post + copyResourcesToTargetDir("target" + separator + "refreshIG", "testfiles/refreshIG"); + // build ini object + File iniFile = new File(INI_LOC); + String iniFileLocation = iniFile.getAbsolutePath(); + IniFile ini = new IniFile(iniFileLocation); + + String bundledFilesLocation = iniFile.getParent() + separator + "bundles" + separator + "measure" + separator; + + String[] args; + if (!fhirUri.isEmpty()) { + args = new String[]{"-RefreshIG", "-ini=" + INI_LOC, "-t", "-d", "-p", "-e=json", "-ts=false", "-fs=" + fhirUri}; + } else { + args = new String[]{"-RefreshIG", "-ini=" + INI_LOC, "-t", "-d", "-p", "-e=json", "-ts=false"}; + } + + // EXECUTE REFRESHIG WITH OUR ARGS: + new RefreshIGOperation().execute(args); + + int requestCount = WireMock.getAllServeEvents().size(); + assertEquals(requestCount, 1); + + if (wireMockServer != null) { + wireMockServer.stop(); + } + + // determine fhirContext for measure lookup + FhirContext fhirContext = IGProcessor.getIgFhirContext(getFhirVersion(ini)); + + // get list of measures resulting from execution + Map measures = IOUtils.getMeasures(fhirContext); + + // loop through measure, verify each has all resources from multiple files + // bundled into single file using id/resourceType as lookup: + for (String measureName : measures.keySet()) { + // location of single bundled file: + final String bundledFileResult = bundledFilesLocation + measureName + separator + measureName + + "-bundle.json"; + // multiple individual files in sub directory to loop through: + final Path dir = Paths + .get(bundledFilesLocation + separator + measureName + separator + measureName + "-files"); + + // loop through each file, determine resourceType and treat accordingly + Map resourceTypeMap = new HashMap<>(); + List groupPatientList = new ArrayList<>(); + + try (final DirectoryStream dirStream = Files.newDirectoryStream(dir)) { + dirStream.forEach(path -> { + File file = new File(path.toString()); + + //Group file testing: + if (file.getName().equalsIgnoreCase("Group-BreastCancerScreeningFHIR.json")) { + try { + org.hl7.fhir.r4.model.Group group = (org.hl7.fhir.r4.model.Group) IOUtils.readResource(file.getAbsolutePath(), fhirContext); + assertTrue(group.hasMember()); + // Check if the group contains members + // Iterate through the members + for (Group.GroupMemberComponent member : group.getMember()) { + groupPatientList.add(member.getEntity().getDisplay()); + } + } catch (Exception e) { + fail("Group-BreastCancerScreeningFHIR.json did not parse to valid Group instance."); + } + + } + + if (file.getName().toLowerCase().endsWith(".json")) { + + Map map = this.jsonMap(file); + if (map == null) { + System.out.println("# Unable to parse " + file.getName() + " as json"); + } else { + + // ensure "resourceType" exists + if (map.containsKey(RESOURCE_TYPE)) { + String parentResourceType = (String) map.get(RESOURCE_TYPE); + // if Library, resource will be translated into "Measure" in main bundled file: + if (parentResourceType.equalsIgnoreCase(LIB_TYPE)) { + resourceTypeMap.put((String) map.get(ID), MEASURE_TYPE); + } else if (parentResourceType.equalsIgnoreCase(BUNDLE_TYPE)) { + // file is a bundle type, loop through resources in entry list, build up map of + // : + if (map.get(ENTRY) != null) { + ArrayList> entryList = (ArrayList>) map.get(ENTRY); + for (Map entry : entryList) { + if (entry.containsKey(RESOURCE)) { + Map resourceMap = (Map) entry.get(RESOURCE); + resourceTypeMap.put((String) resourceMap.get(ID), + (String) resourceMap.get(RESOURCE_TYPE)); + } + } + } + } + } + } + } + }); + + } catch (IOException e) { + logger.info(e.getMessage()); + } + + //Group file should contain two patients: + assertEquals(groupPatientList.size(), 2); + + // map out entries in the resulting single bundle file: + Map bundledJson = this.jsonMap(new File(bundledFileResult)); + Map bundledJsonResourceTypes = new HashMap<>(); + ArrayList> entryList = (ArrayList>) bundledJson.get(ENTRY); + for (Map entry : entryList) { + Map resourceMap = (Map) entry.get(RESOURCE); + bundledJsonResourceTypes.put((String) resourceMap.get(ID), (String) resourceMap.get(RESOURCE_TYPE)); + } + + // compare mappings of to ensure all bundled correctly: + assertTrue(mapsAreEqual(resourceTypeMap, bundledJsonResourceTypes)); + + } + + // run cleanup (maven runs all ci tests sequentially and static member variables could retain values from previous tests) + IOUtils.cleanUp(); + ResourceUtils.cleanUp(); + } + + private static int findAvailablePort() { + for (int port = 8000; port <= 9000; port++) { + if (isPortAvailable(port)) { + return port; + } + } + return -1; + } + + private static boolean isPortAvailable(int port) { + ServerSocket ss; + try (ServerSocket serverSocket = new ServerSocket(port)) { + System.out.println("Trying " + serverSocket); + ss = serverSocket; + } catch (IOException e) { + return false; + } + System.out.println(ss + " is open."); + return true; + } + + //@Test(expectedExceptions = IllegalArgumentException.class) + //TODO: Fix separately, this is blocking a bunch of other higher priority things + public void testNullArgs() { + new RefreshIGOperation().execute(null); + } + + //@Test + //TODO: Fix separately, this is blocking a bunch of other higher priority things + public void testBlankINILoc() { + String args[] = {"-RefreshIG", "-ini=", "-t", "-d", "-p"}; + + try { + new RefreshIGOperation().execute(args); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), IGProcessor.IG_VERSION_REQUIRED); + assertTrue(this.console.toString().indexOf("fhir-version was not specified in the ini file.") != -1); + } + } + + + //@Test + //TODO: Fix separately, this is blocking a bunch of other higher priority things + public void testInvalidIgVersion() { + Map igProperties = new HashMap(); + igProperties.put("ig", "nonsense"); + igProperties.put("template", "nonsense"); + igProperties.put("usage-stats-opt-out", "nonsense"); + igProperties.put("fhir-version", "nonsense"); + + File iniFile = this.createTempINI(igProperties); + + String args[] = {"-RefreshIG", "-ini=" + iniFile.getAbsolutePath(), "-t", "-d", "-p"}; + + if (iniFile != null) { + try { + new RefreshIGOperation().execute(args); + } catch (Exception e) { + assertTrue(e.getClass() == IllegalArgumentException.class); + assertTrue(this.console.toString().indexOf(EXCEPTIONS_OCCURRED_INITIALIZING_REFRESH_FROM_INI_FILE) != -1); + assertTrue(this.console.toString().indexOf("Unknown Version 'nonsense'") != -1); + + assertEquals(e.getMessage(), IGProcessor.IG_VERSION_REQUIRED); + } + deleteTempINI(); + } + } + + //@Test + //TODO: Fix separately, this is blocking a bunch of other higher priority things + public void testInvalidIgInput() { + Map igProperties = new HashMap(); + igProperties.put("ig", "nonsense"); + igProperties.put("template", "nonsense"); + igProperties.put("usage-stats-opt-out", "nonsense"); + igProperties.put("fhir-version", "4.0.1"); + + File iniFile = this.createTempINI(igProperties); + + String args[] = {"-RefreshIG", "-ini=" + iniFile.getAbsolutePath(), "-t", "-d", "-p"}; + + if (iniFile != null) { + try { + new RefreshIGOperation().execute(args); + } catch (Exception e) { + assertTrue(e.getClass() == IllegalArgumentException.class); + assertEquals(e.getMessage(), IGProcessor.IG_VERSION_REQUIRED); + + assertTrue(this.console.toString().indexOf(EXCEPTIONS_OCCURRED_LOADING_IG_FILE) != -1); + assertTrue(this.console.toString().indexOf(EXCEPTIONS_OCCURRED_INITIALIZING_REFRESH_FROM_INI_FILE) != -1); + } + deleteTempINI(); + } + } + + + //@Test + //TODO: Fix separately, this is blocking a bunch of other higher priority things + public void testParamsMissingINI() { + Map igProperties = new HashMap(); + igProperties.put("ig", "nonsense"); + igProperties.put("template", "nonsense"); + igProperties.put("usage-stats-opt-out", "nonsense"); + igProperties.put("fhir-version", "4.0.1"); + + File iniFile = this.createTempINI(igProperties); + + String[] args = {"-RefreshIG", "-ini=" + iniFile.getAbsolutePath(), "-t", "-d", "-p"}; + + RefreshIGParameters params = null; + try { + params = new RefreshIGArgumentProcessor().parseAndConvert(args); + } catch (Exception e) { + System.err.println(e.getMessage()); + System.exit(1); + } + + //override ini to be null + params.ini = null; + + try { + new IGProcessor().publishIG(params); + } catch (Exception e) { + assertEquals(e.getClass(), NullPointerException.class); + } + + deleteTempINI(); + } + + + @AfterMethod + public void afterTest() { + deleteTempINI(); + System.setOut(this.originalStdOut); + System.out.println(this.console.toString()); + this.console = new ByteArrayOutputStream(); + } + + + private File createTempINI(Map properties) { // should look like: // [IG] // ig = input/ecqm-content-r4.xml // template = cqf.fhir.template // usage-stats-opt-out = false // fhir-version=4.0.1 - try { - File iniFile = new File("temp.ini"); - FileOutputStream fos = new FileOutputStream(iniFile); - BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fos)); - bw.write("[IG]"); - bw.newLine(); - for (String key : properties.keySet()) { - bw.write(key + " = " + properties.get(key)); - bw.newLine(); - } - - bw.close(); - return iniFile; - } catch (Exception e) { - e.printStackTrace(); - return null; - } - } - - private boolean deleteTempINI() { - try { - File iniFile = new File("temp.ini"); - if (iniFile.exists()) { - iniFile.delete(); - } - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - return true; - } - - private Map jsonMap(File file) { - Map map = null; - try { - Gson gson = new Gson(); - BufferedReader reader = new BufferedReader(new FileReader(file)); - map = gson.fromJson(reader, Map.class); - reader.close(); - } catch (Exception ex) { - // swallow exception if directory doesnt' exist - // ex.printStackTrace(); - } - return map; - } - - private boolean mapsAreEqual(Map map1, Map map2) { - System.out.println("#TEST INFO: COMPARING " + map1.getClass() + "(" + map1.size() + ") AND " + map2.getClass() - + "(" + map2.size() + ")"); - - if (map1.size() != map2.size()) { - return false; - } - boolean comparison = map1.entrySet().stream().allMatch(e -> e.getValue().equals(map2.get(e.getKey()))); - System.out.println("#TEST INFO: MATCH: " + comparison); - return comparison; - } - - private String getFhirVersion(IniFile ini) { - String specifiedFhirVersion = ini.getStringProperty("IG", "fhir-version"); - if (specifiedFhirVersion == null || specifiedFhirVersion.equals("")) { - - // TODO: Should point to global constant: - specifiedFhirVersion = "4.0.1"; - } - return specifiedFhirVersion; - } + try { + File iniFile = new File("temp.ini"); + FileOutputStream fos = new FileOutputStream(iniFile); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fos)); + bw.write("[IG]"); + bw.newLine(); + for (String key : properties.keySet()) { + bw.write(key + " = " + properties.get(key)); + bw.newLine(); + } + + bw.close(); + return iniFile; + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } + + private boolean deleteTempINI() { + try { + File iniFile = new File("temp.ini"); + if (iniFile.exists()) { + iniFile.delete(); + } + } catch (Exception e) { + e.printStackTrace(); + return false; + } + + return true; + } + + private Map jsonMap(File file) { + Map map = null; + try { + Gson gson = new Gson(); + BufferedReader reader = new BufferedReader(new FileReader(file)); + map = gson.fromJson(reader, Map.class); + reader.close(); + } catch (Exception ex) { + // swallow exception if directory doesnt' exist + // ex.printStackTrace(); + } + return map; + } + + private boolean mapsAreEqual(Map map1, Map map2) { + System.out.println("#TEST INFO: COMPARING " + map1.getClass() + "(" + map1.size() + ") AND " + map2.getClass() + + "(" + map2.size() + ")"); + + if (map1.size() != map2.size()) { + return false; + } + boolean comparison = map1.entrySet().stream().allMatch(e -> e.getValue().equals(map2.get(e.getKey()))); + System.out.println("#TEST INFO: MATCH: " + comparison); + return comparison; + } + + private String getFhirVersion(IniFile ini) { + String specifiedFhirVersion = ini.getStringProperty("IG", "fhir-version"); + if (specifiedFhirVersion == null || specifiedFhirVersion.equals("")) { + + // TODO: Should point to global constant: + specifiedFhirVersion = "4.0.1"; + } + return specifiedFhirVersion; + } }