Adds "ignoreinvalid" configuration option

FHIR/src/Microsoft.Health.Fhir.Anonymizer.Shared.AzureDataFactoryPipeline/src/DataFactoryCustomActivity.cs

@@ -163,23 +163,27 @@ private async Task AnonymizeSingleBlobInJsonFormatAsync(BlobClient inputBlobClie
                     };
                     string output = engine.AnonymizeJson(input, settings);
 
-                    using (MemoryStream outputStream = new MemoryStream(reader.CurrentEncoding.GetBytes(output)))
-                    {
-                        await OperationExecutionHelper.InvokeWithTimeoutRetryAsync(async () =>
+                    if (output != string.Empty) {
+                        using (MemoryStream outputStream = new MemoryStream(reader.CurrentEncoding.GetBytes(output)))
                         {
-                            outputStream.Position = 0;
-                            using MemoryStream stream = new MemoryStream();
-                            await outputStream.CopyToAsync(stream).ConfigureAwait(false);
-                            stream.Position = 0;
-
-                            return await outputBlobClient.UploadAsync(stream).ConfigureAwait(false);
-                        }, 
-                        TimeSpan.FromSeconds(FhirAzureConstants.DefaultBlockUploadTimeoutInSeconds), 
-                        FhirAzureConstants.DefaultBlockUploadTimeoutRetryCount,
-                        isRetrableException: OperationExecutionHelper.IsRetrableException).ConfigureAwait(false);
+                            await OperationExecutionHelper.InvokeWithTimeoutRetryAsync(async () =>
+                            {
+                                outputStream.Position = 0;
+                                using MemoryStream stream = new MemoryStream();
+                                await outputStream.CopyToAsync(stream).ConfigureAwait(false);
+                                stream.Position = 0;
+
+                                return await outputBlobClient.UploadAsync(stream).ConfigureAwait(false);
+                            }, 
+                            TimeSpan.FromSeconds(FhirAzureConstants.DefaultBlockUploadTimeoutInSeconds), 
+                            FhirAzureConstants.DefaultBlockUploadTimeoutRetryCount,
+                            isRetrableException: OperationExecutionHelper.IsRetrableException).ConfigureAwait(false);
+                        }
+
+                        Console.WriteLine($"[{blobName}]: Anonymize completed.");
+                    } else {
+                        Console.WriteLine($"[{blobName}]: Anonymize skipped due to invalid JSON.");
                     }
-
-                    Console.WriteLine($"[{blobName}]: Anonymize completed.");
                 }
             }
             catch (Exception ex)
@@ -227,6 +231,11 @@ private async Task AnonymizeSingleBlobInNdJsonFormatAsync(BlobClient inputBlobCl
             };
 
             await executor.ExecuteAsync(CancellationToken.None, progress).ConfigureAwait(false);
+
+            // If nothing was successfully processed, do not write the file.
+            if (processedCount == 0) {
+              await outputBlobClient.DeleteIfExistsAsync().ConfigureAwait(false);
+            }
         }
 
         private string GetBlobPrefixFromFolderPath(string folderPath)

FHIR/src/Microsoft.Health.Fhir.Anonymizer.Shared.CommandLineTool/FilesAnonymizerForJsonFormatResource.cs

@@ -101,7 +101,12 @@ public async Task<string> FileAnonymize(string fileName)
                     ValidateOutput = _options.ValidateOutput
                 };
                 var resourceResult = engine.AnonymizeJson(resourceJson, settings);
-                await File.WriteAllTextAsync(resourceOutputFileName, resourceResult).ConfigureAwait(false);
+                if (resourceResult != string.Empty) {
+                  await File.WriteAllTextAsync(resourceOutputFileName, resourceResult).ConfigureAwait(false);
+                } else {
+                  Console.WriteLine($"Skip processing on file {fileName} due to invalid input.");
+                  return string.Empty;
+                }
                 return resourceResult;
             }
             catch (Exception innerException)

FHIR/src/Microsoft.Health.Fhir.Anonymizer.Shared.Core/AnonymizerConfigurations/ProcessingErrorsOption.cs

@@ -4,6 +4,6 @@ public enum ProcessingErrorsOption
     {
         Raise, // Invalid processing will raise an exception.
         Skip,  // Invalid processing will return empty element.
-        // Ignore Invalid processing will return input.
+        IgnoreInvalid, // Ignore Invalid processing will return empty element.
     }
 }

FHIR/src/Microsoft.Health.Fhir.Anonymizer.Shared.Core/AnonymizerEngine.cs

@@ -78,7 +78,7 @@ public ITypedElement AnonymizeElement(ITypedElement element, AnonymizerSettings
             }
             catch (AnonymizerProcessingException)
             {
-                if(_configurationManager.Configuration.processingErrors == ProcessingErrorsOption.Skip)
+                if(_configurationManager.Configuration.processingErrors == ProcessingErrorsOption.Skip || _configurationManager.Configuration.processingErrors == ProcessingErrorsOption.IgnoreInvalid)
                 {
                     // Return empty resource.
                     return new EmptyElement(element.InstanceType);
@@ -102,9 +102,23 @@ public Resource AnonymizeResource(Resource resource, AnonymizerSettings settings
         public string AnonymizeJson(string json, AnonymizerSettings settings = null)
         {
             EnsureArg.IsNotNullOrEmpty(json, nameof(json));
+            ITypedElement element;
+            ITypedElement anonymizedElement;
 
-            var element = ParseJsonToTypedElement(json);
-            var anonymizedElement = AnonymizeElement(element);
+            try
+            {
+                element = ParseJsonToTypedElement(json);
+                anonymizedElement = AnonymizeElement(element);
+            }
+            catch (InvalidInputException) {
+                if(_configurationManager.Configuration.processingErrors == ProcessingErrorsOption.IgnoreInvalid)
+                {
+                    // Return empty string to indicate that nothing should be written.
+                    return string.Empty;
+                }
+
+                throw;
+            }
 
             var serializationSettings = new FhirJsonSerializationSettings
             {

FHIR/src/Microsoft.Health.Fhir.Anonymizer.Shared.Core/PartitionedExecution/FhirPartitionedExecutor.cs

@@ -112,7 +112,7 @@ private async Task<IEnumerable<TResult>> AnonymizeAsync(List<TSource> batchData,
                     }
 
                     TResult anonymizedResult = await AnonymizerFunctionAsync(content);
-                    if (EmptyElement.IsEmptyElement(anonymizedResult))
+                    if (EmptyElement.IsEmptyElement(anonymizedResult) || anonymizedResult.ToString() == string.Empty)
                     {
                         batchAnonymizeProgressDetail.ProcessSkipped++;
                     }

FHIR/src/Microsoft.Health.Fhir.Anonymizer.Shared.FunctionalTests/CollectionResourceTests.cs

@@ -21,6 +21,20 @@ public void GivenAResourceWithContained_WhenAnonymizing_ThenAnonymizedJsonShould
             FunctionalTestUtility.VerifySingleJsonResourceFromFile(engine, CollectionResourceTestsFile("contained-basic.json"), CollectionResourceTestsFile("contained-basic-target.json"));
         }
 
+        [Fact]
+        public void GivenAResourceWithContained_WhenAnonymizingWithBadFile_IfIgnore_EmptyResultWillBeReturned()
+        {
+            AnonymizerEngine engine = new AnonymizerEngine(Path.Combine("Configurations", "configuration-ignoreinvalid-processing-error.json"));
+            FunctionalTestUtility.VerifyEmptyStringFromFile(engine, CollectionResourceTestsFile("invalid-resource.json"));
+        }
+
+        [Fact]
+        public void GivenAResourceWithContained_WhenAnonymizingWithBadJsonFile_IfIgnore_EmptyResultWillBeReturned()
+        {
+            AnonymizerEngine engine = new AnonymizerEngine(Path.Combine("Configurations", "configuration-ignoreinvalid-processing-error.json"));
+            FunctionalTestUtility.VerifyEmptyStringFromFile(engine, CollectionResourceTestsFile("invalid-json-resource.json"));
+        }
+
         [Fact]
         public void GivenAResourceWithContained_WhenAnonymizingWithProcessingError_IfSkip_EmptyResultWillBeReturned()
         {

FHIR/src/Microsoft.Health.Fhir.Anonymizer.Shared.FunctionalTests/Configurations/configuration-ignoreinvalid-processing-error.json

@@ -0,0 +1,52 @@
+{
+  "fhirVersion": "",
+  "processingErrors": "ignoreinvalid",
+  "fhirPathRules": [
+    {
+        "path": "nodesByType('HumanName').family",
+        "method": "generalize",
+        "cases": {
+            "$this>=0 and $this<20": "20"
+        }
+    },
+    {
+      "path": "TestResource",
+      "method": "redact"
+    },
+    {
+      "path": "nodesByType('HumanName')",
+      "method": "redact"
+    },
+    {
+      "path": "Resource",
+      "method": "keep"
+    },
+    {
+      "path": "Device.where(id.exists()).id",
+      "method": "keep"
+    }
+  ],
+  "parameters": {
+    "dateShiftKey": "",
+    "cryptoHashKey": "",
+    "enablePartialAgesForRedact": true,
+    "enablePartialDatesForRedact": true,
+    "enablePartialZipCodesForRedact": true,
+    "restrictedZipCodeTabulationAreas": [
+      "036",
+      "059",
+      "102",
+      "203",
+      "205",
+      "369",
+      "556",
+      "692",
+      "821",
+      "823",
+      "878",
+      "879",
+      "884",
+      "893"
+    ]
+  }
+}

FHIR/src/Microsoft.Health.Fhir.Anonymizer.Shared.FunctionalTests/FunctionalTestUtility.cs

@@ -18,6 +18,14 @@ public static void VerifySingleJsonResourceFromFile(AnonymizerEngine engine, str
             Assert.Equal(Standardize(targetContent), Standardize(resultAfterAnonymize));
         } 
 
+        public static void VerifyEmptyStringFromFile(AnonymizerEngine engine, string testFile)
+        {
+            Console.WriteLine($"VerifyEmptyStringFromFile. TestFile: {testFile}");
+            string testContent = File.ReadAllText(testFile);
+            string resultAfterAnonymize = engine.AnonymizeJson(testContent);
+            Assert.Equal(string.Empty, resultAfterAnonymize);
+        }
+
         private static string Standardize(string jsonContent)
         {
             var resource = new FhirJsonParser().Parse<Resource>(jsonContent);

FHIR/src/Microsoft.Health.Fhir.Anonymizer.Shared.FunctionalTests/Microsoft.Health.Fhir.Anonymizer.Shared.FunctionalTests.projitems

@@ -18,6 +18,9 @@
     <None Include="$(MSBuildThisFileDirectory)Configurations\configuration-raise-processing-error.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Include="$(MSBuildThisFileDirectory)Configurations\configuration-ignoreinvalid-processing-error.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="$(MSBuildThisFileDirectory)Configurations\configuration-skip-processing-error.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
@@ -96,6 +99,12 @@
     <None Include="$(MSBuildThisFileDirectory)TestResources\contained-redact-all-target.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Include="$(MSBuildThisFileDirectory)TestResources\invalid-resource.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Include="$(MSBuildThisFileDirectory)TestResources\invalid-json-resource.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="$(MSBuildThisFileDirectory)TestResources\patient-generalize-target.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
@@ -242,4 +251,4 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
-</Project>
+</Project>

FHIR/src/Microsoft.Health.Fhir.Anonymizer.Shared.FunctionalTests/TestResources/invalid-json-resource.json

@@ -0,0 +1 @@
+{"resourceType": "Patient", "badField": "NotAField"}

FHIR/src/Microsoft.Health.Fhir.Anonymizer.Shared.FunctionalTests/TestResources/invalid-resource.json

@@ -0,0 +1 @@
+BAD BAD BAD

docs/FHIR-anonymization.md

@@ -253,8 +253,9 @@ Since _AnonymizationProcessingException_ may caused by a specific FHIR resource,
 
 |processingErrors|Description|
 |----|----|
-|raise|Raise _AnonymizationProcessingException_ with program failed and stopped.|
-|skip| Skip _AnonymizationProcessingException_ and return an empty FHIR resource with program continued. |
+|raise|Raise _AnonymizerProcessingException_ with program failed and stopped.|
+|skip| Skip _AnonymizerProcessingException_ and return an empty FHIR resource with program continued. |
+|ignoreinvalid| Skip both _AnonymizerProcessingException_ and _InvalidInputException_ errors, and return an empty FHIR resource with program continued. |
 
 Here is the structure of empty FHIR resource for patient:
 ```