Add support for array of jsons

go.mod

@@ -1,4 +1,4 @@
-module scrub-pii
+module scrub
 
 go 1.19
 

main.go

@@ -21,12 +21,12 @@ func main() {
 	prettyPtr := flag.Bool("pretty", true, "display pretty output; otherwise do: -pretty=false")
 	flag.Parse()
 	if len(os.Args) < 2 {
-		fmt.Println("Usage: scrub-pii <input json file> <sensitive fields file>")
+		fmt.Println("Usage: scrub <sensitive fields file> <input json file>")
 		os.Exit(1)
 	}
 
-	inputPath := flag.Args()[0]
-	sensitiveFieldsPath := flag.Args()[1]
+	sensitiveFieldsPath := flag.Args()[0]
+	inputPath := flag.Args()[1]
 
 	// scrub the input file for the given sensitive fields
 	output, err := ScrubPersonalInfo(inputPath, sensitiveFieldsPath)

scrub.go

@@ -12,15 +12,8 @@ import (
 	"reflect"
 	"regexp"
 	"strings"
-
-	"github.com/perimeterx/marshmallow"
 )
 
-// the only known property in the input json ahead of time
-type userID struct {
-	ID int `json:"id"`
-}
-
 // return a json string with scrubbed sensitive information from the provided input and sensitive fields files
 func ScrubPersonalInfo(inputPath, sensitiveFieldsPath string) (string, error) {
 	if inputPath == "" {
@@ -33,8 +26,8 @@ func ScrubPersonalInfo(inputPath, sensitiveFieldsPath string) (string, error) {
 	inputFile, _ := ioutil.ReadFile(inputPath)
 	sensitiveFields, _ := getSensitiveFields(sensitiveFieldsPath)
 
-	uid := userID{}
-	input, err := marshmallow.Unmarshal([]byte(inputFile), &uid)
+	var input interface{}
+	err := json.Unmarshal([]byte(inputFile), &input)
 	if err != nil {
 		log.Fatal(err)
 		return "", err
@@ -51,7 +44,7 @@ func ScrubPersonalInfo(inputPath, sensitiveFieldsPath string) (string, error) {
 	b, _ = json.Marshal(input)
 
 	// reset all scrubbed values back to their original values
-	scrubRecursive(input, "", sensitiveFields, &savedValues, false /* unmask */, false /* doScrub */)
+	scrubRecursive(&input, "", sensitiveFields, &savedValues, false /* unmask */, false /* doScrub */)
 
 	// return the scrubbed string
 	return string(b), nil
@@ -80,19 +73,16 @@ func scrubRecursive(field interface{}, fieldName string, sensitiveFields map[str
 		return
 	}
 
-	// skip if no field names
-	if fieldName == "" {
-		return
-	}
 	// skip these types
 	if !fieldValue.CanSet() || fieldValue.IsZero() {
 		return
 	}
 
 	if fieldType.Kind() == reflect.Interface {
 		_, doFieldScrub := sensitiveFields[strings.ToLower(fieldName)]
-		// if parent field is sensitive field, scrub all children (sensitive or not), track in doScrub
-		ok := doScrub || doFieldScrub
+		// skip if no field name and if parent field is a sensitive field, scrub all children
+		// (sensitive or not), track in doScrub
+		ok := fieldName != "" && (doScrub || doFieldScrub)
 
 		// scrub leaf nodes; otherwise, continue recursing
 		switch fValue := fieldValue.Interface().(type) {

tests/00_basic_array/input.json

@@ -0,0 +1,12 @@
+[{
+  "name": "Kelly Doe",
+  "email": "kdoe@example.com",
+  "id": 12324,
+  "phone": "5551234567"
+},
+{
+  "name": "Kelly Doe",
+  "email": "kdoe@example.com",
+  "id": 12324,
+  "phone": "5551234567"
+}]

tests/00_basic_array/output.json

@@ -0,0 +1,12 @@
+[{
+  "name": "***** ***",
+  "email": "****@*******.***",
+  "id": 12324,
+  "phone": "**********"
+},
+{
+  "name": "***** ***",
+  "email": "****@*******.***",
+  "id": 12324,
+  "phone": "**********"
+}]

tests/00_basic_array/sensitive_fields.txt

@@ -0,0 +1,3 @@
+name
+email
+phone

tests/01_alphanumeric_array/input.json

@@ -0,0 +1,12 @@
+[{
+  "name": "Kelly Doe",
+  "email": "k.doe@example.com",
+  "id": 12324,
+  "phone": "(555) 123 - 4567"
+},
+{
+  "name": "Kelly Doe",
+  "email": "k.doe@example.com",
+  "id": 12324,
+  "phone": "(555) 123 - 4567"
+}]

tests/01_alphanumeric_array/output.json

@@ -0,0 +1,12 @@
+[{
+  "name": "***** ***",
+  "email": "*.***@*******.***",
+  "id": 12324,
+  "phone": "(***) *** - ****"
+},
+{
+  "name": "***** ***",
+  "email": "*.***@*******.***",
+  "id": 12324,
+  "phone": "(***) *** - ****"
+}]

tests/01_alphanumeric_array/sensitive_fields.txt

@@ -0,0 +1,3 @@
+name
+email
+phone

tests/02_array_array/input.json

@@ -0,0 +1,12 @@
+[{
+  "name": "Kelly Doe",
+  "email": ["kdoe@example.com", "kelly@gmail.com", "kelly@doe.net"],
+  "id": 12324,
+  "phone": "5551234567"
+},
+{
+  "name": "Kelly Doe",
+  "email": ["kdoe@example.com", "kelly@gmail.com", "kelly@doe.net"],
+  "id": 12324,
+  "phone": "5551234567"
+}]

tests/02_array_array/output.json

@@ -0,0 +1,12 @@
+[{
+  "name": "***** ***",
+  "email": ["****@*******.***", "*****@*****.***", "*****@***.***"],
+  "id": 12324,
+  "phone": "**********"
+},
+{
+  "name": "***** ***",
+  "email": ["****@*******.***", "*****@*****.***", "*****@***.***"],
+  "id": 12324,
+  "phone": "**********"
+}]

tests/02_array_array/sensitive_fields.txt

@@ -0,0 +1,3 @@
+name
+email
+phone

tests/03_booleans_array/input.json

@@ -0,0 +1,16 @@
+[{
+  "name": "Kelly Doe",
+  "email": "kdoe@example.com",
+  "id": 12324,
+  "phone": "5551234567",
+  "us_citizen": false,
+  "admin": false
+},
+{
+  "name": "Kelly Doe",
+  "email": "kdoe@example.com",
+  "id": 12324,
+  "phone": "5551234567",
+  "us_citizen": false,
+  "admin": false
+}]

tests/03_booleans_array/output.json

@@ -0,0 +1,16 @@
+[{
+  "name": "***** ***",
+  "email": "****@*******.***",
+  "id": 12324,
+  "phone": "**********",
+  "us_citizen": "-",
+  "admin": false
+},
+{
+  "name": "***** ***",
+  "email": "****@*******.***",
+  "id": 12324,
+  "phone": "**********",
+  "us_citizen": "-",
+  "admin": false
+}]

tests/03_booleans_array/sensitive_fields.txt

@@ -0,0 +1,4 @@
+name
+email
+phone
+us_citizen

tests/04_numbers_array/input.json

@@ -0,0 +1,12 @@
+[{
+  "name": "Kelly Doe",
+  "email": "kdoe@example.com",
+  "id": 12324,
+  "phone": 5551234567
+},
+{
+  "name": "Kelly Doe",
+  "email": "kdoe@example.com",
+  "id": 12324,
+  "phone": 5551234567
+}]

tests/04_numbers_array/output.json

@@ -0,0 +1,12 @@
+[{
+  "name": "***** ***",
+  "email": "****@*******.***",
+  "id": 12324,
+  "phone": "**********"
+},
+{
+  "name": "***** ***",
+  "email": "****@*******.***",
+  "id": 12324,
+  "phone": "**********"
+}]

tests/04_numbers_array/sensitive_fields.txt

@@ -0,0 +1,3 @@
+name
+email
+phone

tests/05_floats_array/input.json

@@ -0,0 +1,16 @@
+[{
+  "name": "Kelly Doe",
+  "email": "kdoe@example.com",
+  "id": 12324,
+  "phone": "5551234567",
+  "account_balance": 1234.56,
+  "title": "manager"
+},
+{
+  "name": "Kelly Doe",
+  "email": "kdoe@example.com",
+  "id": 12324,
+  "phone": "5551234567",
+  "account_balance": 1234.56,
+  "title": "manager"
+}]

tests/05_floats_array/output.json

@@ -0,0 +1,16 @@
+[{
+  "name": "***** ***",
+  "email": "****@*******.***",
+  "id": 12324,
+  "phone": "**********",
+  "account_balance": "****.**",
+  "title": "manager"
+},
+{
+  "name": "***** ***",
+  "email": "****@*******.***",
+  "id": 12324,
+  "phone": "**********",
+  "account_balance": "****.**",
+  "title": "manager"
+}]

tests/05_floats_array/sensitive_fields.txt

@@ -0,0 +1,4 @@
+name
+email
+phone
+account_balance

tests/06_nested_object_array/input.json

@@ -0,0 +1,17 @@
+[{
+  "name": "Kelly Doe",
+  "id": 12324,
+  "contact": {
+    "email": "kdoe@example.com",
+    "phone": "5551234567"
+  }
+},
+{
+  "name": "Kelly Doe",
+  "id": 12324,
+  "contact": {
+    "email": "kdoe@example.com",
+    "phone": "5551234567"
+  }
+}]
+

tests/06_nested_object_array/output.json

@@ -0,0 +1,16 @@
+[{
+  "name": "***** ***",
+  "id": 12324,
+  "contact": {
+    "email": "****@*******.***",
+    "phone": "**********"
+  }
+},
+{
+  "name": "***** ***",
+  "id": 12324,
+  "contact": {
+    "email": "****@*******.***",
+    "phone": "**********"
+  }
+}]

tests/06_nested_object_array/sensitive_fields.txt

@@ -0,0 +1,3 @@
+name
+email
+phone

tests/07_mixed_type_arrays_array/input.json

@@ -0,0 +1,46 @@
+[{
+  "name": "Kelly Doe",
+  "email": "kdoe@example.com",
+  "id": 12324,
+  "phone": "5551234567",
+  "contacts": [{
+      "name": "Bob Doe",
+      "us_citizen": false
+    },
+    12345,
+    "bob@example.com",
+    {
+      "id": 2343,
+      "name": "John Smith",
+      "email": "john.smith@yahoo.com"
+    },
+    {
+      "phone": "(555) 234-2343",
+      "name": "Joe Schmoe",
+      "email": "jschmoe@aol.com"
+    }
+  ]
+},
+{
+  "name": "Kelly Doe",
+  "email": "kdoe@example.com",
+  "id": 12324,
+  "phone": "5551234567",
+  "contacts": [{
+      "name": "Bob Doe",
+      "us_citizen": false
+    },
+    12345,
+    "bob@example.com",
+    {
+      "id": 2343,
+      "name": "John Smith",
+      "email": "john.smith@yahoo.com"
+    },
+    {
+      "phone": "(555) 234-2343",
+      "name": "Joe Schmoe",
+      "email": "jschmoe@aol.com"
+    }
+  ]
+}]