HttpClientUtils: POST and PUT handling / Resource ingestion priority

tooling/src/main/java/org/opencds/cqf/tooling/measure/MeasureBundler.java

@@ -3,24 +3,21 @@
 import ca.uhn.fhir.context.FhirContext;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.opencds.cqf.tooling.processor.AbstractBundler;
-import org.opencds.cqf.tooling.utilities.HttpClientUtils;
 import org.opencds.cqf.tooling.utilities.IOUtils;
-import org.opencds.cqf.tooling.utilities.IOUtils.Encoding;
 
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 
 public class MeasureBundler extends AbstractBundler {
     public static final String ResourcePrefix = "measure-";
+
     protected CopyOnWriteArrayList<Object> identifiers;
 
     public static String getId(String baseId) {
         return ResourcePrefix + baseId;
     }
+
     @Override
     protected String getSourcePath(FhirContext fhirContext, Map.Entry<String, IBaseResource> resourceEntry) {
         return IOUtils.getMeasurePathMap(fhirContext).get(resourceEntry.getKey());
@@ -41,67 +38,4 @@ protected Set<String> getPaths(FhirContext fhirContext) {
         return IOUtils.getMeasurePaths(fhirContext);
     }
 
-    //so far only the Measure Bundle process needs to persist extra files:
-    @Override
-    protected int persistFilesFolder(String bundleDestPath, String libraryName, Encoding encoding, FhirContext fhirContext, String fhirUri) {
-        //persist tests-* before group-* files and make a record of which files were tracked:
-        List<String> persistedFiles = persistTestFilesWithPriority(bundleDestPath, libraryName, encoding, fhirContext, fhirUri);
-        persistedFiles.addAll(persistEverythingElse(bundleDestPath, libraryName, encoding, fhirContext, fhirUri, persistedFiles));
-
-        return persistedFiles.size();
-    }
-
-    private List<String> persistTestFilesWithPriority(String bundleDestPath, String libraryName, Encoding encoding, FhirContext fhirContext, String fhirUri) {
-        List<String> persistedResources = new ArrayList<>();
-        String filesLoc = bundleDestPath + File.separator + libraryName + "-files";
-        File directory = new File(filesLoc);
-        if (directory.exists()) {
-            File[] filesInDir = directory.listFiles();
-            if (!(filesInDir == null || filesInDir.length == 0)) {
-                for (File file : filesInDir) {
-                    if (file.getName().toLowerCase().startsWith("tests-")) {
-                        try {
-                            IBaseResource resource = IOUtils.readResource(file.getAbsolutePath(), fhirContext, true);
-                            HttpClientUtils.post(fhirUri, resource, encoding, fhirContext, file.getAbsolutePath(), true);
-                            persistedResources.add(file.getAbsolutePath());
-                        } catch (Exception e) {
-                            //resource is likely not IBaseResource
-                            logger.error("MeasureBundler.persistTestFilesWithPriority", e);
-                        }
-                    }
-                }
-            }
-        }
-        return persistedResources;
-    }
-
-    private List<String> persistEverythingElse(String bundleDestPath, String libraryName, Encoding encoding, FhirContext fhirContext, String fhirUri, List<String> alreadyPersisted) {
-        List<String> persistedResources = new ArrayList<>();
-        String filesLoc = bundleDestPath + File.separator + libraryName + "-files";
-        File directory = new File(filesLoc);
-        if (directory.exists()) {
-
-            File[] filesInDir = directory.listFiles();
-
-            if (!(filesInDir == null || filesInDir.length == 0)) {
-                for (File file : filesInDir) {
-                    //don't post what has already been processed
-                    if (alreadyPersisted.contains(file.getAbsolutePath())) {
-                        continue;
-                    }
-                    if (file.getName().toLowerCase().endsWith(".json") || file.getName().toLowerCase().endsWith(".xml")) {
-                        try {
-                            IBaseResource resource = IOUtils.readResource(file.getAbsolutePath(), fhirContext, true);
-                            HttpClientUtils.post(fhirUri, resource, encoding, fhirContext, file.getAbsolutePath(), false);
-                            persistedResources.add(file.getAbsolutePath());
-                        } catch (Exception e) {
-                            //resource is likely not IBaseResource
-                            logger.error("persistEverythingElse", e);
-                        }
-                    }
-                }
-            }
-        }
-        return persistedResources;
-    }
 }

tooling/src/main/java/org/opencds/cqf/tooling/plandefinition/PlanDefinitionBundler.java

@@ -37,8 +37,4 @@ protected String getResourceBundlerType() {
         return TYPE_PLAN_DEFINITION;
     }
 
-    @Override
-    protected int persistFilesFolder(String bundleDestPath, String libraryName, IOUtils.Encoding encoding, FhirContext fhirContext, String fhirUri) {
-        return 0;
-    }
 }

tooling/src/main/java/org/opencds/cqf/tooling/processor/AbstractBundler.java

@@ -130,7 +130,7 @@ public void bundleResources(List<String> refreshedLibraryNames, String igPath, L
         final Map<String, List<CqlCompilerException>> cqlTranslatorErrorMessages = new ConcurrentHashMap<>();
 
         //used to summarize file count user can expect to see in POST queue for each resource:
-        final Map<String, Integer> persistedFileReport = new ConcurrentHashMap<>();
+        final List<String> persistedFileReport = new CopyOnWriteArrayList<>();
 
         //build list of executable tasks to be sent to thread pool:
         List<Callable<Void>> tasks = new ArrayList<>();
@@ -142,7 +142,7 @@ public void bundleResources(List<String> refreshedLibraryNames, String igPath, L
             final Map<String, String> libraryPathMap = new ConcurrentHashMap<>(IOUtils.getLibraryPathMap(fhirContext));
 
             if (resourcesMap.isEmpty()) {
-                logger.info("[INFO] No " + getResourceBundlerType() + "s found. Continuing...");
+                logger.info("\n\r" + "[INFO] No " + getResourceBundlerType() + "s found. Continuing...\n\r");
                 return;
             }
 
@@ -169,7 +169,7 @@ public void bundleResources(List<String> refreshedLibraryNames, String igPath, L
                 tasks.add(() -> {
                     //check if resourceSourcePath has been processed before:
                     if (processedResources.contains(resourceSourcePath)) {
-                        logger.info(getResourceBundlerType() + " processed already: " + resourceSourcePath);
+                        logger.info("\n\r" + getResourceBundlerType() + " processed already: " + resourceSourcePath);
                         return null;
                     }
                     String resourceName = FilenameUtils.getBaseName(resourceSourcePath).replace(getResourcePrefix(), "");
@@ -246,24 +246,22 @@ public void bundleResources(List<String> refreshedLibraryNames, String igPath, L
                         if (shouldPersist) {
                             String bundleDestPath = FilenameUtils.concat(FilenameUtils.concat(IGProcessor.getBundlesPath(igPath), getResourceTestGroupName()), resourceName);
 
-                            persistBundle(bundleDestPath, resourceName, encoding, fhirContext, new ArrayList<IBaseResource>(resources.values()), fhirUri, addBundleTimestamp);
-
                             bundleFiles(igPath, bundleDestPath, resourceName, binaryPaths, resourceSourcePath,
                                     primaryLibrarySourcePath, fhirContext, encoding, includeTerminology, includeDependencies, includePatientScenarios,
                                     includeVersion, addBundleTimestamp, cqlTranslatorErrorMessages);
 
                             //If user supplied a fhir server url, inform them of total # of files to be persisted to the server:
                             if (fhirUri != null && !fhirUri.isEmpty()) {
-                                persistedFileReport.put(resourceName,
-                                        //+1 to account for -bundle
-                                        persistFilesFolder(bundleDestPath, resourceName, encoding, fhirContext, fhirUri) + 1);
+                                persistedFileReport.add(resourceName);
                             }
 
                             if (cdsHooksProcessor != null) {
                                 List<String> activityDefinitionPaths = CDSHooksProcessor.bundleActivityDefinitions(resourceSourcePath, fhirContext, resources, encoding, includeVersion, shouldPersist);
                                 cdsHooksProcessor.addActivityDefinitionFilesToBundle(igPath, bundleDestPath, activityDefinitionPaths, fhirContext, encoding);
                             }
 
+                            persistBundle(bundleDestPath, resourceName, encoding, fhirContext, new ArrayList<>(resources.values()), fhirUri, addBundleTimestamp);
+
                             bundledResources.add(resourceSourcePath);
                         }
 
@@ -296,7 +294,7 @@ public void bundleResources(List<String> refreshedLibraryNames, String igPath, L
         //Output final report:
         String summaryOutput = generateBundleProcessSummary(refreshedLibraryNames, fhirContext, fhirUri, verboseMessaging,
                 persistedFileReport, bundledResources, failedExceptionMessages, cqlTranslatorErrorMessages).toString();
-        logger.info(summaryOutput);
+        logger.info("\n\r" + summaryOutput);
     }
 
     /**
@@ -315,53 +313,16 @@ public void bundleResources(List<String> refreshedLibraryNames, String igPath, L
      * @return A StringBuilder containing the generated summary message.
      */
     private StringBuilder generateBundleProcessSummary(List<String> refreshedLibraryNames, FhirContext fhirContext,
-                                                       String fhirUri, Boolean verboseMessaging, Map<String, Integer> persistedFileReport,
+                                                       String fhirUri, Boolean verboseMessaging, List<String> persistedFileReport,
                                                        List<String> bundledResources, Map<String, String> failedExceptionMessages,
                                                        Map<String, List<CqlCompilerException>> cqlTranslatorErrorMessages) {
 
         StringBuilder summaryMessage = new StringBuilder(NEWLINE);
 
         //Give user a snapshot of the files each resource will have persisted to their FHIR server (if fhirUri is provided)
         final int persistCount = persistedFileReport.size();
-        if (persistCount > 0) {
-            String fileDisplay = " File(s): ";
-            summaryMessage.append(NEWLINE).append(persistCount).append(" ").append(getResourceBundlerType()).append("(s) have POST tasks in the queue for ").append(fhirUri).append(": ");
-            int totalQueueCount = 0;
-            List<String> persistMessages = new ArrayList<>();
-            for (String library : persistedFileReport.keySet()) {
-                totalQueueCount = totalQueueCount + persistedFileReport.get(library);
-                persistMessages.add(NEWLINE_INDENT
-                        + persistedFileReport.get(library)
-                        + fileDisplay
-                        + library);
-            }
-
-            //anon comparator class to sort by the file count for better presentation
-            persistMessages.sort(new Comparator<>() {
-                @Override
-                public int compare(String displayFileCount1, String displayFileCount2) {
-                    int count1 = getFileCountFromString(displayFileCount1);
-                    int count2 = getFileCountFromString(displayFileCount2);
-                    return Integer.compare(count1, count2);
-                }
-
-                private int getFileCountFromString(String fileName) {
-                    int endIndex = fileName.indexOf(fileDisplay);
-                    if (endIndex != -1) {
-                        String countString = fileName.substring(0, endIndex).trim();
-                        return Integer.parseInt(countString);
-                    }
-                    return 0;
-                }
-            });
-
-            for (String persistMessage : persistMessages) {
-                summaryMessage.append(persistMessage);
-            }
-            summaryMessage.append(NEWLINE_INDENT)
-                    .append("Total: ")
-                    .append(totalQueueCount)
-                    .append(" File(s)");
+        if (!persistedFileReport.isEmpty()) {
+            summaryMessage.append(NEWLINE).append(persistCount).append(" ").append(getResourceBundlerType()).append("(s) have HTTP request tasks in the queue for ").append(fhirUri);
         }
 
 
@@ -452,19 +413,16 @@ private void persistBundle(String bundleDestPath, String libraryName,
                                IOUtils.Encoding encoding, FhirContext fhirContext,
                                List<IBaseResource> resources, String fhirUri,
                                Boolean addBundleTimestamp) throws IOException {
-        IOUtils.initializeDirectory(bundleDestPath);
         Object bundle = BundleUtils.bundleArtifacts(libraryName, resources, fhirContext, addBundleTimestamp, this.getIdentifiers());
         IOUtils.writeBundle(bundle, bundleDestPath, encoding, fhirContext);
 
         if (fhirUri != null && !fhirUri.isEmpty()) {
             String resourceWriteLocation = bundleDestPath + separator + libraryName + "-bundle." + encoding;
-            HttpClientUtils.post(fhirUri, (IBaseResource) bundle, encoding, fhirContext, resourceWriteLocation, true);
+            //give resource the highest priority (0):
+            HttpClientUtils.sendToServer(fhirUri, (IBaseResource) bundle, encoding, fhirContext, resourceWriteLocation);
         }
     }
 
-
-    protected abstract int persistFilesFolder(String bundleDestPath, String libraryName, IOUtils.Encoding encoding, FhirContext fhirContext, String fhirUri);
-
     private void bundleFiles(String igPath, String bundleDestPath, String primaryLibraryName, List<String> binaryPaths, String resourceFocusSourcePath,
                              String librarySourcePath, FhirContext fhirContext, IOUtils.Encoding encoding, Boolean includeTerminology, Boolean includeDependencies, Boolean includePatientScenarios,
                              Boolean includeVersion, Boolean addBundleTimestamp, Map<String, List<CqlCompilerException>> translatorWarningMessages) {
@@ -476,7 +434,7 @@ private void bundleFiles(String igPath, String bundleDestPath, String primaryLib
         IOUtils.copyFile(librarySourcePath, FilenameUtils.concat(bundleDestFilesPath, FilenameUtils.getName(librarySourcePath)));
 
         String cqlFileName = IOUtils.formatFileName(FilenameUtils.getBaseName(librarySourcePath), IOUtils.Encoding.CQL, fhirContext);
-        if (cqlFileName.toLowerCase().startsWith("library-")) {
+        if (cqlFileName.toLowerCase().startsWith("library-") && !cqlFileName.toLowerCase().startsWith("library-deps-")) {
             cqlFileName = cqlFileName.substring(8);
         }
         String cqlLibrarySourcePath = IOUtils.getCqlLibrarySourcePath(primaryLibraryName, cqlFileName, binaryPaths);

tooling/src/main/java/org/opencds/cqf/tooling/processor/IGBundleProcessor.java

@@ -48,9 +48,9 @@ public void bundleIg(List<String> refreshedLibraryNames, String igPath, List<Str
                 fhirUri, encoding, verboseMessaging);
 
         //run collected post calls last:
-        if (HttpClientUtils.hasPostTasksInQueue()) {
-            logger.info("[Persisting Files to {}]", fhirUri);
-            HttpClientUtils.postTaskCollection();
+        if (HttpClientUtils.hasHttpRequestTasksInQueue()) {
+            logger.info("\n\r[Persisting Files to " + fhirUri + "]\n\r");
+            HttpClientUtils.executeHttpRequestTaskCollection();
         }
 
         // run cleanup (maven runs all ci tests sequentially and static member variables could retain values from previous tests)

tooling/src/main/java/org/opencds/cqf/tooling/processor/PostBundlesInDirProcessor.java

@@ -63,15 +63,15 @@ public static void PostBundlesInDir(PostBundlesInDirParameters params) {
         List<Map.Entry<String, IBaseResource>> resources = BundleUtils.getBundlesInDir(params.directoryPath, fhirContext);
         resources.forEach(entry -> postBundleToFhirUri(fhirUri, encoding, fhirContext, entry.getValue()));
 
-        if (HttpClientUtils.hasPostTasksInQueue()){
-            HttpClientUtils.postTaskCollection();
+        if (HttpClientUtils.hasHttpRequestTasksInQueue()){
+            HttpClientUtils.executeHttpRequestTaskCollection();
         }
     }
 
     private static void postBundleToFhirUri(String fhirUri, Encoding encoding, FhirContext fhirContext, IBaseResource bundle) {
-        if (fhirUri != null && !fhirUri.equals("")) {
+        if (fhirUri != null && !fhirUri.isEmpty()) {
             try {
-                HttpClientUtils.post(fhirUri, bundle, encoding, fhirContext, null);
+                HttpClientUtils.sendToServer(fhirUri, bundle, encoding, fhirContext, null);
                 logger.info("Resource successfully posted to FHIR server ({}): {}", fhirUri, bundle.getIdElement().getIdPart());
             } catch (Exception e) {
                 logger.error("Error occurred for element {}: {}",bundle.getIdElement().getIdPart(), e.getMessage());

tooling/src/main/java/org/opencds/cqf/tooling/questionnaire/QuestionnaireBundler.java

@@ -35,11 +35,6 @@ protected String getResourceBundlerType() {
         return TYPE_QUESTIONNAIRE;
     }
 
-    @Override
-    protected int persistFilesFolder(String bundleDestPath, String libraryName, IOUtils.Encoding encoding, FhirContext fhirContext, String fhirUri) {
-        //do nothing
-        return 0;
-    }
 
     @Override
     protected Set<String> getPaths(FhirContext fhirContext) {