Content pipeline

SPEC.md

@@ -27,13 +27,22 @@ set for the tasks using task options.
 
 ## Post metadata
 
-All posts have a filename which is used as a key to identify the post.
+All posts have a path which is used as a key to identify the post.
 
 - **:title**
     - Required by: *atom-feed*
     - Used by: *rss* either this or description is required
 - **:content** The post content
-    - Set by: *markdown*
+    - Populated from file content
+    - Only set in the `:entry` value of the map passed to `renderer` functions
+- **:has-content** Flag indicating that this file contains page content
+    - Set by input parsing tasks (like `markdown`) on output files
+    - Passed through by rendering tasks
+- **:original-path** The path for the input file from which this entry is descended
+    - Set by input parsing tasks (like `markdown`) on output files
+    - Passed through by rendering tasks
+- **:parsed** Contains the parsed file content
+    - Set by input parsing tasks (like `markdown`) on input files
 - **:description**
     - Used by: *rss* either this or title is required
 - **:slug**

src/io/perun/atom.clj

@@ -32,8 +32,7 @@
        [:link {:href (str base-url filename) :rel "self"}]
        [:link {:href base-url :type "text/html"}]
        [:updated (->> (take 10 posts)
-                      (map updated)
-                      (map iso-datetime)
+                      (map (comp iso-datetime updated first))
                       sort
                       reverse
                       first)]
@@ -44,7 +43,7 @@
           [:name (:author global-metadata)]
           [:email (:author-email global-metadata)]])
 
-       (for [{:keys [uuid canonical-url content title author author-email] :as post} (take 10 posts)
+       (for [[{:keys [uuid canonical-url title author author-email] :as post} content] (take 10 posts)
              :let [author (or author (:author global-metadata))
                    author-email (or author-email (:author-email global-metadata))]]
          (do

src/io/perun/markdown.clj

@@ -55,7 +55,7 @@
   (let [file-content (-> file :full-path io/file slurp)
         md-metadata (merge (:meta options) (yaml/parse-file-metadata file-content))
         html (markdown-to-html file-content (:options options))]
-    (merge md-metadata {:content html} file)))
+    (merge md-metadata {:parsed html} file)))
 
 (defn parse-markdown [markdown-files options]
   (let [updated-files (doall (map #(process-file % options) markdown-files))]

src/io/perun/meta.clj

@@ -4,19 +4,24 @@
 
 (def +meta-key+ :io.perun)
 
+(defn meta-from-file
+  [tmpfile]
+  (when-let [meta (+meta-key+ tmpfile)]
+    (assoc meta :path (:path tmpfile))))
+
 (defn get-meta
   "Return metadata on files. Files metadata is a list.
    Internally it's stored as a map indexed by `:path`"
   [fileset]
-  (keep +meta-key+ (vals (:tree fileset))))
+  (keep meta-from-file (vals (:tree fileset))))
 
 (defn key-meta [data]
   (into {} (for [d data] [(:path d) d])))
 
 (defn set-meta
   "Update `+meta-key+` metadata for files in `data` and return updated fileset"
   [fileset data]
-  (boot/add-meta fileset (into {} (for [d data] [(:path d) {+meta-key+ d}]))))
+  (boot/add-meta fileset (into {} (for [d data] [(:path d) {+meta-key+ (dissoc d :path)}]))))
 
 (defn merge-meta* [m1 m2]
   (vals (merge-with merge (key-meta m1) (key-meta m2))))

src/io/perun/ttr.clj

@@ -2,12 +2,11 @@
   (:require [io.perun.core     :as perun]
             [time-to-read.core :as time-to-read]))
 
-(defn add-ttr [file]
-  (if-let [content (:content file)]
-    (assoc file :ttr (time-to-read/estimate-for-text content))
-    file))
+(defn add-ttr [[meta content]]
+  (when content
+    (assoc meta :ttr (time-to-read/estimate-for-text content))))
 
-(defn calculate-ttr [files]
-  (let [updated-files (map add-ttr files)]
-    (perun/report-info "ttr" "added TTR to %s files" (count updated-files))
-    updated-files))
+(defn calculate-ttr [meta-contents]
+  (let [metas (keep add-ttr meta-contents)]
+    (perun/report-info "ttr" "added TTR to %s files" (count metas))
+    metas))

src/io/perun/word_count.clj

@@ -1,12 +1,11 @@
 (ns io.perun.word-count
-  (:require [io.perun.core     :as perun]))
+  (:require [io.perun.core :as perun]))
 
-(defn add-word-count [file]
-  (if-let [content (:content file)]
-    (assoc file :word-count (count (clojure.string/split content #"\s")))
-    file))
+(defn add-word-count [[meta content]]
+  (when content
+    (assoc meta :word-count (count (clojure.string/split content #"\s")))))
 
-(defn count-words [files]
-  (let [updated-files (map add-word-count files)]
-    (perun/report-info "word-count" "added word-count to %s files" (count updated-files))
-    updated-files))
+(defn count-words [meta-contents]
+  (let [metas (map add-word-count meta-contents)]
+    (perun/report-info "word-count" "added word-count to %s files" (count metas))
+    metas))

test/io/perun_test.clj

@@ -18,6 +18,12 @@
   (or (some #{val} (-> file pm/+meta-key+ key))
       (contains? (-> file pm/+meta-key+ key) val)))
 
+(deftask prn-meta-key
+  [p path PATH str "path of the file to test"
+   k key  KEY  kw  "the key to prn"]
+  (boot/with-pass-thru fileset
+    (->> path (boot/tmp-get fileset) pm/+meta-key+ key prn)))
+
 (deftask key-test
   [p path PATH str "path of the file to test"
    k key  KEY  kw  "the key to test"
@@ -34,6 +40,15 @@
     (let [file (boot/tmp-get fileset path)]
       (is (and (not (nil? file)) (value-fn file)) msg))))
 
+(deftask content-test
+  [p path    PATH    str  "path of the file to test"
+   c content CONTENT str  "The content of the file"
+   n negate?         bool "true to check if file doesn't exist"
+   m msg     MSG     str  "message shown on failure"]
+  (boot/with-pass-thru fileset
+    (let [f (if negate? not identity)]
+      (is (f (.contains (slurp (boot/tmp-file (boot/tmp-get fileset path))) content))))))
+
 (deftask file-exists?
   [p path    PATH str  "path of the image to add"
    n negate?      bool "true to check if file doesn't exist"
@@ -105,6 +120,10 @@ author: Testy McTesterson
 
 This --- be ___markdown___.")
 
+(def parsed-md-basic "<h1><a href=\"#hello-there\" name=\"hello-there\"></a>Hello there</h1>\n<p>This --- be <strong><em>markdown</em></strong>.</p>")
+
+(def parsed-md-smarts "<h1><a href=\"#hello-there\" name=\"hello-there\"></a>Hello there</h1>\n<p>This &mdash; be <strong><em>markdown</em></strong>.</p>")
+
 (def js-content "(function somejs() { console.log('foo'); })();")
 
 (deftesttask global-metadata-test []
@@ -117,7 +136,7 @@ This --- be ___markdown___.")
   [data]
   (str "<body>" (:content (:entry data)) "</body>"))
 
-(deftesttask default-test []
+(deftesttask default-tests []
   (comp (add-txt-file :path "2017-01-01-test.md" :content md-content)
         (boot/with-pre-wrap fileset
           (pm/set-global-meta fileset {:base-url "http://example.com/"
@@ -126,51 +145,54 @@ This --- be ___markdown___.")
         (p/markdown)
         (testing "markdown"
           (value-test :path "2017-01-01-test.md"
-                      :value-fn #(meta= % :content "<h1><a href=\"#hello-there\" name=\"hello-there\"></a>Hello there</h1>\n<p>This --- be <strong><em>markdown</em></strong>.</p>")
-                      :msg "`markdown` should set `:content` metadata on markdown file"))
+                      :value-fn #(meta= % :parsed parsed-md-basic)
+                      :msg "`markdown` should set `:parsed` metadata on markdown file")
+          (content-test :path "2017-01-01-test.html"
+                        :content parsed-md-basic
+                        :msg "`markdown` should populate HTML file with parsed content"))
 
         (p/ttr)
         (testing "ttr"
-          (value-test :path "2017-01-01-test.md"
+          (value-test :path "2017-01-01-test.html"
                       :value-fn #(meta= % :ttr 1)
                       :msg "`ttr` should set `:ttr` metadata"))
 
         (p/word-count)
         (testing "word-count"
-          (value-test :path "2017-01-01-test.md"
-                      :value-fn #(meta= % :word-count 8)
+          (value-test :path "2017-01-01-test.html"
+                      :value-fn #(meta= % :word-count 19)
                       :msg "`word-count` should set `:word-count` metadata"))
 
         (p/gravatar :source-key :email :target-key :gravatar)
         (testing "gravatar"
-          (value-test :path "2017-01-01-test.md"
+          (value-test :path "2017-01-01-test.html"
                       :value-fn
                       #(meta= % :gravatar "http://www.gravatar.com/avatar/a1a361f6c96acb1e31ad4b3bbf7aa444")
                       :msg "`gravatar` should set `:gravatar` metadata"))
 
         (p/build-date)
         (testing "build-date"
-          (key-test :path "2017-01-01-test.md"
+          (key-test :path "2017-01-01-test.html"
                     :key :date-build
                     :msg "`build-date` should set `:date-build` metadata"))
 
         (p/slug)
         (testing "slug"
-          (value-test :path "2017-01-01-test.md"
+          (value-test :path "2017-01-01-test.html"
                       :value-fn #(meta= % :slug "test")
-                      :msg "`:slug` should set `:slug` metadata"))
+                      :msg "`slug` should set `:slug` metadata"))
 
         (p/permalink)
         (testing "permalink"
-          (value-test :path "2017-01-01-test.md"
+          (value-test :path "2017-01-01-test.html"
                       :value-fn #(meta= % :permalink "/test/index.html")
-                      :msg "`:permalink` should set `:permalink` metadata"))
+                      :msg "`permalink` should set `:permalink` metadata"))
 
         (p/canonical-url)
         (testing "canonical-url"
-          (value-test :path "2017-01-01-test.md"
+          (value-test :path "2017-01-01-test.html"
                       :value-fn #(meta= % :canonical-url "http://example.com/test/index.html")
-                      :msg "`:canonical-url` should set `:canonical-url` metadata"))
+                      :msg "`canonical-url` should set `:canonical-url` metadata"))
 
         (p/sitemap)
         (testing "sitemap"
@@ -192,14 +214,13 @@ This --- be ___markdown___.")
         (add-txt-file :path "test.js" :content js-content)
         (p/inject-scripts :scripts #{"test.js"})
         (testing "inject-scripts"
-          (boot/with-pass-thru fileset
-            (is (.contains (slurp (boot/tmp-file (boot/tmp-get fileset "public/test/index.html")))
-                           (str "<script>" js-content "</script>"))
-                "`inject-scripts` should alter the contents of a file")))
+          (content-test :path "public/test/index.html"
+                        :content (str "<script>" js-content "</script>")
+                        :msg "`inject-scripts` should alter the contents of a file"))
 
         (p/draft)
         (testing "draft"
-          (file-exists? :path "2017-01-01-test.md"
+          (file-exists? :path "2017-01-01-test.html"
                         :negate? true
                         :msg "`draft` should remove files"))))
 
@@ -212,8 +233,11 @@ This --- be ___markdown___.")
         (p/markdown :meta {:markdown-set :metadata} :options {:extensions {:smarts true}})
         (testing "markdown"
           (value-test :path "test.md"
-                      :value-fn #(meta= % :content "<h1><a href=\"#hello-there\" name=\"hello-there\"></a>Hello there</h1>\n<p>This &mdash; be <strong><em>markdown</em></strong>.</p>")
-                      :msg "`markdown` should set `:content` metadata on markdown file"))
+                      :value-fn #(meta= % :parsed parsed-md-smarts)
+                      :msg "`markdown` should set `:parsed` metadata on markdown file")
+          (content-test :path "test.html"
+                        :content parsed-md-smarts
+                        :msg "`markdown` should populate HTML file with parsed content"))
 
         (p/ttr :filterer :markdown-set)
         (testing "ttr"
@@ -224,7 +248,7 @@ This --- be ___markdown___.")
         (p/word-count :filterer :markdown-set)
         (testing "word-count"
           (value-test :path "test.md"
-                      :value-fn #(meta= % :word-count 8)
+                      :value-fn #(meta= % :word-count 19)
                       :msg "`word-count` should set `:word-count` metadata"))
 
         (p/gravatar :source-key :email :target-key :gravatar :filterer :markdown-set)
@@ -298,10 +322,43 @@ This --- be ___markdown___.")
         (p/inject-scripts :scripts #{"test.js"} :filter #{#"foo"})
         (p/inject-scripts :scripts #{"test.js"} :remove #{#"baz"})
         (testing "inject-scripts"
-          (boot/with-pass-thru fileset
-            (is (.contains (slurp (boot/tmp-file (boot/tmp-get fileset "bar/foo.html")))
-                           (str "<script>" js-content "</script>"))
-                "`inject-scripts` should alter the contents of an included file")
-            (is (not (.contains (slurp (boot/tmp-file (boot/tmp-get fileset "baz.html")))
-                                (str "<script>" js-content "</script>")))
-                "`inject-scripts` should not alter the contents of a removed file")))))
+          (content-test :path "bar/foo.html"
+                        :content (str "<script>" js-content "</script>")
+                        :msg "`inject-scripts` should alter the contents of a file")
+          (content-test :path "baz.html"
+                        :content (str "<script>" js-content "</script>")
+                        :negate? true
+                        :msg "`inject-scripts` should not alter the contents of a removed file"))))
+
+(deftesttask content-tests []
+  (comp (testing "Collection works without input files" ;; #77
+          (p/collection :renderer 'io.perun-test/render))
+
+        (add-txt-file :path "test.md" :content md-content)
+        (p/markdown) ;; render once
+
+        (add-txt-file :path "test.md" :content (str/replace md-content #"Hello" "Salutations"))
+        (p/markdown)
+        (testing "detecting content changes"
+          (content-test :path "test.html" :content "Salutations"))
+
+        (add-txt-file :path "test.md" :content (str/replace md-content #"draft: true" "draft: false"))
+        (p/markdown)
+        (testing "detecting metadata changes"
+          (value-test :path "test.html" :value-fn #(meta= % :draft false)))
+
+        (add-txt-file :path "test.md" :content (str/replace md-content #"draft: true" "draft: true\nfoo: bar"))
+        (p/markdown)
+        (testing "detecting metadata additions"
+          (value-test :path "test.html" :value-fn #(meta= % :foo "bar")))
+
+        (add-txt-file :path "test.md" :content md-content)
+        (p/markdown)
+        (testing "detecting metadata deletions"
+          (value-test :path "test.html" :value-fn #(meta= % :foo nil)))
+
+        (add-txt-file :path "test2.md" :content md-content)
+        (p/markdown)
+        (testing "detecting new files"
+          (content-test :path "test2.html" :content parsed-md-basic)
+          (value-test :path "test2.md" :value-fn #(meta= % :parsed parsed-md-basic)))))