Merge pull request #361 from metafacture/350-subfields

Support HTML attribute values as subfields

Author: fsteeg

metafacture-html/src/main/java/org/metafacture/html/HtmlDecoder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020 Fabian Steeg, hbz
+ * Copyright 2020, 2021 Fabian Steeg, hbz
  *
  * Licensed under the Apache License, Version 2.0 the "License";
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,11 @@
 
 import java.io.IOException;
 import java.io.Reader;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.UUID;
 
 import org.apache.commons.io.IOUtils;
@@ -38,12 +43,28 @@
  * @author Fabian Steeg (fsteeg)
  *
  */
-@Description("Decode HTML to metadata events")
+@Description("Decode HTML to metadata events. The attrValsAsSubfields option can be used to override "
+        + "the default attribute values to be used as subfields (e.g. by default "
+        + "`link rel=\"canonical\" href=\"http://example.org\"` becomes `link.canonical`). "
+        + "It expects an HTTP-style query string specifying as key the attributes whose value should "
+        + "be used as a subfield, and as value the attribute whose value should be the subfield value, "
+        + "e.g. the default contains `link.rel=href`. To use the HTML element text as the value "
+        + "(instead of another attribute), omit the value of the query-string key-value pair, "
+        + "e.g. `title.lang`. To add to the defaults, instead of replacing them, start with an `&`, "
+        + "e.g. `&h3.class`")
 @In(Reader.class)
 @Out(StreamReceiver.class)
 @FluxCommand("decode-html")
 public class HtmlDecoder extends DefaultObjectPipe<Reader, StreamReceiver> {
 
+    private static final String DEFAULT_ATTR_VALS_AS_SUBFIELDS = //
+            "meta.name=content&meta.property=content&link.rel=href&a.rel=href";
+    private Map<String, String> attrValsAsSubfields;
+
+    public HtmlDecoder() {
+        setAttrValsAsSubfields(DEFAULT_ATTR_VALS_AS_SUBFIELDS);
+    }
+
     @Override
     public void process(final Reader reader) {
         try {
@@ -61,18 +82,54 @@ private void process(Element parent, StreamReceiver receiver) {
         for (Element element : parent.children()) {
             receiver.startEntity(element.nodeName());
             Attributes attributes = element.attributes();
+            boolean addedValueAsSubfield = false;
             for (Attribute attribute : attributes) {
+                addedValueAsSubfield = handleAttributeValuesAsSubfields(receiver, element, attributes, attribute);
                 receiver.literal(attribute.getKey(), attribute.getValue());
             }
             if (element.children().isEmpty()) {
                 String text = element.text().trim();
                 String value = text.isEmpty() ? element.data() : text;
-                if (!value.isEmpty()) {
+                if (!value.isEmpty() && !addedValueAsSubfield) {
                     receiver.literal("value", value);
                 }
             }
             process(element, receiver);
             receiver.endEntity();
         }
     }
+
+    private boolean handleAttributeValuesAsSubfields(StreamReceiver receiver, Element element,
+            Attributes attributes, Attribute attribute) {
+        String fullFieldKey = element.nodeName() + "." + attribute.getKey();
+        if (attrValsAsSubfields.containsKey(fullFieldKey)) {
+            String configValue = attrValsAsSubfields.get(fullFieldKey);
+            if (configValue.trim().isEmpty()) {
+                receiver.literal(attribute.getValue(), element.text().trim());
+                return true;
+            } else {
+                String value = attributes.get(configValue);
+                receiver.literal(attribute.getValue(), value);
+            }
+        }
+        return false;
+    }
+
+    public void setAttrValsAsSubfields(String mapString) {
+        this.attrValsAsSubfields = new HashMap<String, String>();
+        String input = mapString.startsWith("&") ? DEFAULT_ATTR_VALS_AS_SUBFIELDS + mapString
+                : mapString;
+        for (String nameValuePair : input.split("&")) {
+            String[] nameValue = nameValuePair.split("=");
+            try {
+                String utf8 = StandardCharsets.UTF_8.name();
+                String key = URLDecoder.decode(nameValue[0], utf8);
+                String val = nameValue.length > 1 ? URLDecoder.decode(nameValue[1], utf8) : "";
+                attrValsAsSubfields.put(key, val);
+            } catch (UnsupportedEncodingException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+
 }

metafacture-html/src/test/java/org/metafacture/html/HtmlDecoderTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020 Fabian Steeg, hbz
+ * Copyright 2020, 2021 Fabian Steeg, hbz
  *
  * Licensed under the Apache License, Version 2.0 the "License";
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 
 import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.never;
 
 import java.io.StringReader;
 
@@ -97,4 +98,39 @@ public void htmlScriptElementData() {
         ordered.verify(receiver, times(4)).endEntity();
     }
 
+    @Test
+    public void htmlAttributesAsSubfieldsDefault() {
+        htmlDecoder.process(new StringReader("<meta name=\"language\" content=\"DE\"/>"));
+        final InOrder ordered = inOrder(receiver);
+        ordered.verify(receiver).startEntity("meta");
+        ordered.verify(receiver).literal("language", "DE");
+        ordered.verify(receiver).literal("name", "language");
+        ordered.verify(receiver).literal("content", "DE");
+        ordered.verify(receiver, times(4)).endEntity();
+    }
+
+    @Test
+    public void htmlAttributesAsSubfieldsCustom() {
+        htmlDecoder.setAttrValsAsSubfields("mods:url.access");
+        htmlDecoder.process(new StringReader("<mods:url access=\"preview\">file:///img.png</mods:url>"));
+        final InOrder ordered = inOrder(receiver);
+        ordered.verify(receiver).startEntity("mods:url");
+        ordered.verify(receiver).literal("preview", "file:///img.png");
+        ordered.verify(receiver, never()).literal("value", "file:///img.png");
+        ordered.verify(receiver, times(3)).endEntity();
+    }
+
+    @Test
+    public void htmlAttributesAsSubfieldsDefaultPlusCustom() {
+        htmlDecoder.setAttrValsAsSubfields("&mods:url.access");
+        htmlDecoder.process(new StringReader("<meta name=\"language\" content=\"DE\"/>"
+                + "<mods:url access=\"preview\">file:///img.png</mods:url>"));
+        final InOrder ordered = inOrder(receiver);
+        ordered.verify(receiver).startEntity("meta");
+        ordered.verify(receiver).literal("language", "DE");
+        ordered.verify(receiver).startEntity("mods:url");
+        ordered.verify(receiver).literal("preview", "file:///img.png");
+        ordered.verify(receiver, never()).literal("value", "file:///img.png");
+        ordered.verify(receiver, times(3)).endEntity();
+    }
 }