Merge pull request #86 from michaeladler/perf/parse-from-rdf

perf: skip redundant n-quads duplicate checks when parsing RDF

Author: kazarena

ld/api_from_rdf.go

@@ -158,7 +158,7 @@ func (api *JsonLdApi) FromRDF(dataset *RDFDataset, opts *JsonLdOptions) ([]inter
 			}
 
 			// 3.5.6+7)
-			MergeValue(node.Values, predicate, value)
+			mergeValue(node.Values, predicate, value, dataset.parsedWithoutDuplicates)
 
 			// 3.5.8)
 			if IsBlankNode(object) || IsIRI(object) {

ld/rdf_dataset.go

@@ -98,6 +98,9 @@ type RDFDataset struct {
 	Graphs map[string][]*Quad
 
 	context map[string]string
+
+	// indicates that this dataset has had no duplicates (guaranteed by the serializer) at the time it's been parsed
+	parsedWithoutDuplicates bool
 }
 
 // RDFSerializer can serialize and de-serialize RDFDatasets.

ld/serialize_nquads.go

@@ -239,6 +239,7 @@ func ParseNQuadsFrom(o interface{}) (*RDFDataset, error) {
 
 	// maintain a set of triples for each graph to check for duplicates
 	triplesByGraph := make(map[string]map[Quad]struct{})
+	dataset.parsedWithoutDuplicates = true // the following code ensures that no duplicate quads are added
 
 	scanner, err := newScannerFor(o)
 	if err != nil {

ld/utils.go

@@ -130,6 +130,10 @@ func deepContains(values []interface{}, value interface{}) bool {
 
 // MergeValue adds a value to a subject. If the value is an array, all values in the array will be added.
 func MergeValue(obj map[string]interface{}, key string, value interface{}) {
+	mergeValue(obj, key, value, false)
+}
+
+func mergeValue(obj map[string]interface{}, key string, value interface{}, assumeNoDuplicates bool) {
 	if obj == nil {
 		return
 	}
@@ -140,7 +144,7 @@ func MergeValue(obj map[string]interface{}, key string, value interface{}) {
 	}
 	valueMap, isMap := value.(map[string]interface{})
 	_, valueContainsList := valueMap["@list"]
-	if key == "@list" || (isMap && valueContainsList) || !deepContains(values, value) {
+	if key == "@list" || (isMap && valueContainsList) || assumeNoDuplicates || !deepContains(values, value) {
 		values = append(values, value)
 	}
 	obj[key] = values