CSVW metadata download: Part 2

datahost-ld-openapi/deps.edn

@@ -19,7 +19,7 @@
 				camel-snake-kebab/camel-snake-kebab {:mvn/version "0.4.3"}
 				metosin/jsonista {:mvn/version "0.3.7"}
 				org.glassfish/jakarta.json {:mvn/version "2.0.1"}
-				com.apicatalog/titanium-json-ld {:mvn/version "1.3.2"}
+				com.apicatalog/titanium-json-ld {:mvn/version "1.3.3"}
 				scicloj/tablecloth {:mvn/version "7.000-beta-50"}
 				net.openhft/zero-allocation-hashing {:mvn/version "0.16"}
 				metrics-clojure/metrics-clojure {:mvn/version "2.10.0"}

datahost-ld-openapi/hurl-scripts/int-002.hurl

@@ -115,3 +115,14 @@ Accept: text/csv
 HTTP 200
 [Asserts]
 body matches /max,44/
+
+
+GET {{scheme}}://{{host_name}}/doc/{{series}}/release/release-1.csv-metadata.json
+Authorization: {{auth_token}}
+
+HTTP 200
+[Asserts]
+jsonpath "$.['@context']" == "http://www.w3.org/ns/csvw"
+jsonpath "$.['http://purl.org/dc/terms/title']" == "Fun schema"
+jsonpath "$.['tableSchema'].columns" count == 2
+jsonpath "$.['url']" == "/doc/{{series}}/release/release-1.csv"

datahost-ld-openapi/hurl-scripts/int-011.hurl

@@ -102,15 +102,15 @@ Accept: application/json
 Authorization: {{auth_token}}
 
 HTTP 302
-Location: /data/{{series}}/release/release-1.json
+Location: /doc/{{series}}/release/release-1.json
 
 # Release requested as CSV via ACCEPTS header gets redirected to correct route
 GET {{scheme}}://{{host_name}}/data/{{series}}/release/release-1
 Accept: text/csv
 Authorization: {{auth_token}}
 
 HTTP 302
-Location: /data/{{series}}/release/release-1.csv
+Location: /doc/{{series}}/release/release-1.csv
 
 
 #appends to the revision
@@ -296,13 +296,15 @@ header "Location" == "{{revision2_url}}"
 
 
 
-GET {{scheme}}://{{host_name}}/data/{{series}}/release/release-1-metadata.json
-#Accept: application/json
-#Content-Type: application/json
+GET {{scheme}}://{{host_name}}/doc/{{series}}/release/release-1.csv-metadata.json
 Authorization: {{auth_token}}
+
 HTTP 200
 [Asserts]
-body == "{\"@context\": [\"http://www.w3.org/ns/csvw\", {\"@language\": \"en\"}]}"
+jsonpath "$.['@context']" == "http://www.w3.org/ns/csvw"
+jsonpath "$.['http://purl.org/dc/terms/title']" == "Dummy Schema"
+jsonpath "$.['tableSchema'].columns" count == 8
+jsonpath "$.['url']" == "/doc/{{series}}/release/release-1.csv"
 
 
 #ensure path header contains a URL when requesting revision 2

datahost-ld-openapi/hurl-scripts/int-012.hurl

@@ -56,14 +56,6 @@ Accept: text/csv
 HTTP 200
 
 
-GET {{scheme}}://{{host_name}}/data/{{series}}/release/release-1/revision/1-metadata.json
-
-
-HTTP 200
-[Asserts]
-body == "{\"@context\": [\"http://www.w3.org/ns/csvw\", {\"@language\": \"en\"}]}"
-
-
 POST {{scheme}}://{{host_name}}/data/{{series}}/release/release-1/schema
 Content-Type: application/json
 Authorization: {{auth_token}}
@@ -247,16 +239,6 @@ variable "path" contains "rel=\"describedBy\""
 header "Location" == "{{revision2_url}}"
 
 
-
-GET {{scheme}}://{{host_name}}/data/{{series}}/release/release-1-metadata.json
-Accept: application/json 
-Content-Type: application/json
-Authorization: {{auth_token}}
-HTTP 200
-[Asserts]
-body == "{\"@context\": [\"http://www.w3.org/ns/csvw\", {\"@language\": \"en\"}]}"
-
-
 #ensure path header contains a URL when requesting revision 2
 GET {{scheme}}://{{host_name}}{{revision2_url}}
 Accept: text/csv

datahost-ld-openapi/src/tpximpact/datahost/ldapi/handlers.clj

@@ -1,5 +1,7 @@
 (ns tpximpact.datahost.ldapi.handlers
   (:require
+   [clojure.string :as str]
+   [clojure.set :as cs]
    [clojure.tools.logging :as log]
    [grafter.matcha.alpha :as matcha]
    [ring.util.io :as ring-io]
@@ -19,7 +21,7 @@
    [tpximpact.datahost.ldapi.schemas.api :as s.api]
    [tpximpact.datahost.ldapi.util.data.validation :as data.validation]
    [tpximpact.datahost.ldapi.util.triples
-    :refer [triples->ld-resource triples->ld-resource-collection]]
+    :refer [triples->ld-resource triples->ld-resource-collection triples->csvw-resource]]
    [clojure.java.io :as io])
   (:import
    (java.net URI)
@@ -31,6 +33,9 @@
 (defn- as-json-ld [response]
   (assoc-in response [:headers "content-type"] "application/ld+json"))
 
+(defn- as-csvm [response]
+  (assoc-in response [:headers "content-type"] "application/csvm+json"))
+
 (defn get-api-params [{:keys [path-params query-params]}]
   (-> query-params (update-keys keyword) (merge path-params)))
 
@@ -84,7 +89,7 @@
 
 (defn get-release
   [triplestore _change-store system-uris
-   {{:keys [extension] :as path-params} :path-params
+   {{:keys [extension]} :path-params
     {release-uri :dh/Release} :datahost.request/uris
     :as request}]
   (if-let [release (->> release-uri
@@ -100,7 +105,7 @@
           :else (let [rev-uri ^URI (:rev change-info)]
                   (-> (.getPath rev-uri)
                       (util.response/redirect)
-                      (shared/set-csvm-header request)))))
+                      (shared/set-csvm-link-header request)))))
       (as-json-ld {:status 200
                    :body (-> (json-ld/compact release (json-ld/simple-context system-uris))
                              (.toString))}))
@@ -154,6 +159,28 @@
                                (.toString))}))
       (errors/not-found-response request))))
 
+(defn get-release-csvw-metadata
+  [triplestore system-uris {path-params :path-params request-uri :uri :as request}]
+  (let [release-uri (su/dataset-release-uri* system-uris path-params)
+        matcha-db (matcha/index-triples (db/get-release-schema-statements triplestore release-uri))]
+    (if-let [schema-id (get-schema-id matcha-db)]
+      (let [csvw-number-uri (cmp/expand :csvw/number)
+            schema-resource (-> (triples->csvw-resource matcha-db schema-id)
+                                (assoc (cmp/expand :csvw/url)
+                                       (str/replace request-uri #"-metadata.json" ""))
+                                (update (cmp/expand :dh/columns)
+                                        (fn [schema-cols]
+                                          (->> (box schema-cols)
+                                               (map #(triples->csvw-resource matcha-db %))
+                                               (sort-by #(get % csvw-number-uri))
+                                               (hash-map (cmp/expand :csvw/columns)))))
+                                (cs/rename-keys {(cmp/expand :dh/columns) (cmp/expand :csvw/tableSchema)}))]
+        (as-csvm {:status 200
+                  :body (-> (json-ld/compact schema-resource json-ld/datahost-csvw-vocab-context)
+                            (json-ld/update-with-csvw-context)
+                            (.toString))}))
+      (errors/not-found-response request))))
+
 (defn- revision-number
   "Returns a number or throws."
   [rev-id]
@@ -185,7 +212,7 @@
                       (when (nil? key)
                         (throw (ex-info "No snapshot reference for revision" {:revision revision})))
                       (ring-io/piped-input-stream (partial change-store-to-ring-io-writer change-store key)))))}
-          (shared/set-csvm-header request))
+          (shared/set-csvm-link-header request))
 
       (as-json-ld {:status 200
                    :body (-> (json-ld/compact revision-ld (json-ld/simple-context system-uris))
@@ -289,7 +316,6 @@
    change-kind
    {router :reitit.core/router
     {:keys [series-slug release-slug revision-id] :as path-params} :path-params
-    query-params :query-params
     appends :body
     {release-uri :dh/Release :as request-uris} :datahost.request/uris
     :as request}]

datahost-ld-openapi/src/tpximpact/datahost/ldapi/json_ld.clj

@@ -3,6 +3,8 @@
             [tpximpact.datahost.system-uris :as su])
   (:import (com.apicatalog.jsonld JsonLd)
            (com.apicatalog.jsonld.document JsonDocument)
+           (com.apicatalog.jsonld.json JsonMapBuilder JsonUtils)
+           (jakarta.json JsonObject)
            (java.io StringReader)))
 
 (defn simple-context [system-uris]
@@ -21,6 +23,12 @@
          {:contents {"@id" "dh:collection-contents",
                      "@container" "@set"}}))
 
+(def datahost-csvw-vocab-context
+  "Default CSVW @vocab, used during JSON-LD processing steps & later replaced by
+  correct CSVW @context during final stages of processing, because CSVW isn't JSON-LD"
+  {"@vocab" "http://www.w3.org/ns/csvw#"
+   "columns" {"@container" "@set"}})
+
 (defn ->json-document
   ^JsonDocument [edn]
   (-> edn
@@ -38,3 +46,10 @@
       (.compactArrays true)
       (.compactToRelative true)
       (.get)))
+
+(defn update-with-csvw-context
+  "Once JSON-LD processing is complete. @context can be updated with legal CSVW context."
+  [^JsonObject json-obj]
+  (let [builder (JsonMapBuilder/create json-obj)]
+    (.put builder "@context" (JsonUtils/toJsonValue "http://www.w3.org/ns/csvw"))
+    (.build builder)))

datahost-ld-openapi/src/tpximpact/datahost/ldapi/router.clj

@@ -345,7 +345,17 @@ specifications for each route.")
         {:post (routes.rev/post-revision-retractions-route-config triplestore change-store system-uris)}]
 
        ["/:revision-id/corrections"
-        {:post (routes.rev/post-revision-corrections-route-config triplestore change-store system-uris)}]]]]]
+        {:post (routes.rev/post-revision-corrections-route-config triplestore change-store system-uris)}]]]]
+
+    ["/doc" {:muuntaja leave-keys-alone-muuntaja-coercer}
+     ["/:series-slug/release"
+
+      ;; /doc/dataset-series/release/:release-id.csv-metadata.json
+      ["/{release-slug}.csv-metadata.{extension}"
+       {:get (routes.rel/get-release-csvw-metadata-config triplestore system-uris)}]
+
+      ["/{release-slug}.{extension}"
+       {:get (routes.rel/get-release-route-config triplestore change-store system-uris)}]]]]
 
    {;;:reitit.middleware/transform dev/print-request-diffs ;; pretty diffs
     ;;:validate spec/validate ;; enable spec validation for route data

datahost-ld-openapi/src/tpximpact/datahost/ldapi/routes/middleware.clj

@@ -6,6 +6,7 @@
    [reitit.coercion :as coercion]
    [reitit.impl :as impl]
    [reitit.spec :as rs]
+   [muuntaja.parse :as m.parse]
    [ring.middleware.multipart-params :as multipart-params]
    [ring.util.http-status :as status]
    [tpximpact.datahost.ldapi.db :as db]
@@ -197,41 +198,21 @@
 (def accepts->extension {"text/csv" "csv"
                          "application/json" "json"})
 
-(defn- find-first-accept-header
-  "Simple selection of single accept http header mime type. Significant
-  assumptions are made; most notably that the accept headers are already
-  sorted such that the most desired accept type is first in the collection.
-
-  Risk of incorrect mime type selection is greatly reduced because it's
-  quite likely that only one accept header type will ever be sent by API
-  clients.
-
-  Q values are ignored."
-  [accept-headers]
-  (let [prioritized-accept-mimes (str/split accept-headers #",")]
-    (some->> prioritized-accept-mimes
-             (filter (fn [prioritized-mime]
-                       (some #(.contains prioritized-mime %)
-                             (keys accepts->extension))))
-
-             (first)
-             (#(str/split % #";"))
-             (first))))
-
-(defn accepts->extension-redirect-middleware
+(defn accepts->file-doc-middleware
   "When a route using this middleware receives an allowed Accept: text/csv header, for
   example, instead of serving the content from its route it redirects users to the same
-  route but with the corresponding .csv file extension. Same for application/json
-  (redirects with .json extension)"
+  route but with the corresponding doc route with a .csv file extension. Same for
+  application/json (redirects with .json extension)"
   [handler _id]
   (fn [{{:strs [accept]} :headers
         {:keys [path path-params]} :reitit.core/match
         :as request}]
-    (if-let [first-accept (find-first-accept-header accept)]
+    (if-let [first-accept (first (m.parse/parse-accept accept))]
       (if (and (nil? (:extension path-params))
                (contains? accepts->extension first-accept))
         {:status 302
-         :headers {"Location" (str path "." (get accepts->extension first-accept))}}
+         :headers {"Location" (-> (str path "." (get accepts->extension first-accept))
+                                  (str/replace #"^\/data" "/doc"))}}
         (handler request))
       (handler request))))
 

datahost-ld-openapi/src/tpximpact/datahost/ldapi/routes/release.clj

@@ -22,7 +22,7 @@
   {:summary (str "Retrieve data or metadata for an existing release via ACCEPT header."
                  "Redirects to correct file route with file extension when allowed ACCEPT"
                  "header is provided. e.g. text/csv")
-   :middleware [[(partial middleware/accepts->extension-redirect-middleware) :accepts-redirect]]
+   :middleware [[(partial middleware/accepts->file-doc-middleware) :accepts-redirect]]
    :handler identity
    :parameters {:path [:map
                        routes-shared/series-slug-param-spec
@@ -74,8 +74,7 @@ It is currently considered an error to put data into a revision which
 has been succeeded by another.
 "
    :handler (partial handlers/get-release triplestore change-store system-uris)
-   :middleware [[(partial middleware/csvm-request-response triplestore system-uris) :csvm-response]
-                [(partial middleware/entity-uris-from-path system-uris #{:dh/Release}) :entity-uris]]
+   :middleware [[(partial middleware/entity-uris-from-path system-uris #{:dh/Release}) :entity-uris]]
    :coercion (rcm/create {:transformers {}, :validate false})
    :parameters {:path [:map
                        routes-shared/series-slug-param-spec
@@ -131,7 +130,7 @@ set `dcterms:modified` times for you."
   {:summary "Retrieve release schema"
    :description "Returns the Datahost TableSchema associated with the release.
 
-NOTE: Datahost tableschemas are extended subsets of CSVW:TableSchema."
+NOTE: Datahost tableSchemas are extended subsets of CSVW:TableSchema."
    :handler (partial handlers/get-release-schema triplestore system-uris)
    :parameters {:path [:map
                        routes-shared/series-slug-param-spec
@@ -141,6 +140,19 @@ NOTE: Datahost tableschemas are extended subsets of CSVW:TableSchema."
                404 {:body routes-shared/NotFoundErrorBody}}
    :tags ["Consumer API"]})
 
+(defn get-release-csvw-metadata-config
+  [triplestore system-uris]
+  {:summary "Retrieve release CSVW metadata as JSON"
+   :description "Returns the Datahost CSVW metadata schema associated with the release."
+   :handler (partial handlers/get-release-csvw-metadata triplestore system-uris)
+   :parameters {:path [:map
+                       routes-shared/series-slug-param-spec
+                       routes-shared/release-slug-param-spec]}
+   :responses {200 {:description "Release CSVW metadata schema successfully retrieved"
+                    :content {"application/csvm+json" string?}}
+               404 {:body routes-shared/NotFoundErrorBody}}
+   :tags ["Consumer API"]})
+
 (defn post-release-ld-schema-config
   [clock triplestore system-uris]
   {:summary "Create schema for a release"

datahost-ld-openapi/src/tpximpact/datahost/ldapi/routes/shared.clj

@@ -150,7 +150,7 @@
 (defn base-url [request]
   (request/request-url (select-keys request [:scheme :headers :uri])))
 
-(defn set-csvm-header [response request]
+(defn set-csvm-link-header [response request]
   (let [csvm-url (-> request (update :uri str "-metadata.json") base-url)]
     (update response :headers assoc "link" (str "<" csvm-url ">; "
                                                 "rel=\"describedBy\"; "

datahost-ld-openapi/src/tpximpact/datahost/ldapi/util/triples.clj

@@ -1,7 +1,8 @@
 (ns tpximpact.datahost.ldapi.util.triples
   (:require
    [grafter.matcha.alpha :as matcha]
-   [grafter.vocabularies.rdf :as vocab.rdf]))
+   [grafter.vocabularies.rdf :as vocab.rdf]
+   [tpximpact.datahost.ldapi.compact :as compact]))
 
 (defn triples->ld-resource
   "Given triples returned from a DB query, transform them into a single resource
@@ -44,3 +45,28 @@
                        (matcha/optional [[?s vocab.rdf/rdf:a ?t]])]
                       matcha-db)
         (map #(dissoc % vocab.rdf/rdf:a)))))
+
+(defn- dissoc-non-csvw-uris [resource-map]
+  (dissoc resource-map
+          vocab.rdf/rdf:a
+          vocab.rdf/rdf:type
+          :grafter.rdf/uri
+          (compact/expand :dh/appliesToRelease)
+          (compact/expand :appropriate-csvw/modeling-of-dialect)))
+
+(defn triples->csvw-resource
+  "Given triples returned from a DB query, transform them into a single resource
+   map (e.g. Release or Revision) that's ready for CSVW JSON metadata serialization"
+  ([matcha-db]
+   (-> (matcha/build-1 ?s
+                       {?p ?o}
+                       [[?s ?p ?o]]
+                       matcha-db)
+       (dissoc-non-csvw-uris)))
+  ([matcha-db subject]
+   (-> (matcha/build-1 ?s
+                       {?p ?o}
+                       [[?s ?p ?o]
+                        (matcha/values ?s [subject])]
+                       matcha-db)
+       (dissoc-non-csvw-uris))))