From 55412cc9ba1802cbc3a8d6e36ac7c3e176856331 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sun, 4 Dec 2016 19:26:59 -0600 Subject: [PATCH 01/31] Allow more flexibility and composability based around file content Making Perun more file-based than metadata-based makes composing arbitrary rendering pipelines easier, by leveraging Boot's primary abstraction, the fileset. This opens up more possibilities of using third party tasks that don't know anything about Perun, and makes Perun's own tasks more useful. Summary of changes: - Tasks no longer store content in the :content metadata key, but instead use the actual file's content - `markdown` now sets `:parsed` instead of `:content` - In addition to setting `:parsed`, `markdown` also writes an html file that contains the parsed content, for processing downstream. - Rather than setting `:content` on the file that `markdown` writes, a new flag, `:has-content` is set to indicate to downstream tasks that this file is available for further processing - Default filterers are set to `:has-content` - Most of `markdown`'s functionality is wrapped by `content-pre-wrap`, which can be reused by future input parsing tasks - `collection` can now be used as a kind of input-parsing task that produces a page fragment in the same manner that `markdown` does. This fragment can then be picked up by subsequent `render` tasks and used in the same way as the output from `markdown`. The original usage of `collection` to produce a complete page is also still supported, with no changes. - `collection` no longer sets `:date-build` or `:canonical-url`, because you can now use the `build-date` or `canonical-url` tasks to set these keys, just as you would with any other content. - You can now chain multiple `render`s together in the same task pipeline. This allows you to split your rendering needs into as many logical parts as necessary. --- src/io/perun.clj | 168 +++++++++++++++++++++++++------------- src/io/perun/markdown.clj | 2 +- 2 files changed, 110 insertions(+), 60 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index f9fda7fe..01036192 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -4,6 +4,7 @@ [boot.pod :as pod] [boot.util :as u] [clojure.java.io :as io] + [clojure.set :as set] [clojure.string :as string] [clojure.edn :as edn] [io.perun.core :as perun])) @@ -113,6 +114,56 @@ (perun/set-meta fileset updated-files) (commit fileset tmp)))) +(defn content-pre-wrap + [parse-form file-exts tracer options] + (let [tmp (boot/tmp-dir!) + prev-fs (atom nil)] + (boot/with-pre-wrap fileset + (let [changed-files (->> (boot/fileset-diff @prev-fs fileset :hash) + boot/user-files + (boot/by-ext file-exts) + add-filedata) + fs-files (->> fileset + boot/user-files + (boot/by-ext file-exts) + set) + prev-fs-files (if @prev-fs + (->> @prev-fs + boot/user-files + (boot/by-ext file-exts) + set) + #{}) + all-files (set/intersection fs-files prev-fs-files) + unchanged-metadata (->> changed-files + set + (set/difference all-files) + (keep perun/+meta-key+)) + input-metadata (->> + (if-let [pod (:pod options)] + (pod/with-call-in @pod ~(parse-form changed-files)) + (eval (parse-form changed-files))) + (trace tracer)) + output-metadata (doall + (for [{:keys [path parsed filename] :as entry*} (concat unchanged-metadata + input-metadata)] + (let [page-filepath (string/replace path #"(?i).[a-z]+$" ".html") + entry (-> entry* + (assoc :has-content true) + (assoc :path page-filepath) + (assoc :filename (string/replace filename + #"(?i).[a-z]+$" ".html")) + (dissoc :parsed :extension :file-type :parent-path + :full-path :mime-type :original))] + (perun/create-file tmp page-filepath parsed) + entry))) + final-metadata (perun/merge-meta* (perun/get-meta @prev-fs) + (concat input-metadata output-metadata)) + new-fs (-> fileset + (commit tmp) + (perun/set-meta final-metadata))] + (reset! prev-fs new-fs) + new-fs)))) + (def ^:private markdown-deps '[[org.pegdown/pegdown "1.6.0"] [circleci/clj-yaml "0.5.5"]]) @@ -126,26 +177,17 @@ "Parse markdown files This task will look for files ending with `md` or `markdown` - and add a `:content` key to their metadata containing the + and add a `:parsed` key to their metadata containing the HTML resulting from processing markdown file's content" [m meta META edn "metadata to set on each entry; keys here will be overridden by metadata in each file" o options OPTS edn "options to be passed to the markdown parser"] (let [pod (create-pod markdown-deps) - prev-fs (atom nil)] - (boot/with-pre-wrap fileset - (let [options (merge +markdown-defaults+ *opts*) - md-files (->> fileset - (boot/fileset-diff @prev-fs) - boot/user-files - (boot/by-ext ["md" "markdown"]) - add-filedata) - updated-files (trace :io.perun/markdown - (pod/with-call-in @pod - (io.perun.markdown/parse-markdown ~md-files ~options))) - final-metadata (perun/merge-meta* (perun/get-meta @prev-fs) updated-files) - new-fs (perun/set-meta fileset final-metadata)] - (reset! prev-fs new-fs) - new-fs)))) + options (merge +markdown-defaults+ {:pod pod} *opts*)] + (content-pre-wrap + (fn [files] `(io.perun.markdown/parse-markdown ~files ~(dissoc options :pod))) + ["md" "markdown"] + :io.perun/markdown + options))) (deftask global-metadata "Read global metadata from `perun.base.edn` or configured file. @@ -247,12 +289,12 @@ drop-last (string/join "-") string/lower-case)) - :filterer :content}) + :filterer :has-content}) (deftask slug "Adds :slug key to files metadata. Slug is derived from filename." [s slug-fn SLUGFN code "function to build slug from filename" - _ filterer FILTER code "predicate to use for selecting entries (default: `:content`)"] + _ filterer FILTER code "predicate to use for selecting entries (default: `:has-content`)"] (boot/with-pre-wrap fileset (let [options (merge +slug-defaults+ *opts*) slug-fn (:slug-fn options) @@ -266,14 +308,14 @@ (def ^:private +permalink-defaults+ {:permalink-fn (fn [m] (perun/absolutize-url (str (:slug m) "/index.html"))) - :filterer :content}) + :filterer :has-content}) (deftask permalink "Adds :permalink key to files metadata. Value of key will determine target path. Make files permalinked. E.x. about.html will become about/index.html" [p permalink-fn PERMALINKFN code "function to build permalink from TmpFile metadata" - _ filterer FILTER code "predicate to use for selecting entries (default: `:content`)"] + _ filterer FILTER code "predicate to use for selecting entries (default: `:has-content`)"] (boot/with-pre-wrap fileset (let [options (merge +permalink-defaults+ *opts*) files (filter (:filterer options) (perun/get-meta fileset)) @@ -286,14 +328,14 @@ (perun/merge-meta fileset updated-files)))) (def ^:private +canonical-url-defaults+ - {:filterer :content}) + {:filterer :has-content}) (deftask canonical-url "Adds :canonical-url key to files metadata. The url is concatenation of :base-url in global metadata and files' permaurl. The base-url must end with '/'." - [_ filterer FILTER code "predicate to use for selecting entries (default: `:content`)"] + [_ filterer FILTER code "predicate to use for selecting entries (default: `:has-content`)"] (boot/with-pre-wrap fileset (let [options (merge +canonical-url-defaults+ *opts*) files (filter (:filterer options) (perun/get-meta fileset)) @@ -315,13 +357,13 @@ (def ^:private +sitemap-defaults+ {:filename "sitemap.xml" - :filterer :content + :filterer :has-content :out-dir "public"}) (deftask sitemap "Generate sitemap" [f filename FILENAME str "generated sitemap filename" - _ filterer FILTER code "predicate to use for selecting entries (default: `:content`)" + _ filterer FILTER code "predicate to use for selecting entries (default: `:has-content`)" o out-dir OUTDIR str "the output directory" u url URL str "base URL"] (let [pod (create-pod sitemap-deps) @@ -402,7 +444,7 @@ (def ^:private +render-defaults+ {:out-dir "public" - :filterer :content}) + :filterer :has-content}) (def ^:private render-deps '[[org.clojure/tools.namespace "0.3.0-alpha3"]]) @@ -418,8 +460,8 @@ - `:render-data` the map argument that `renderer` will be called with - `:entry` the metadata for the item being rendered - All `:entry`s will be returned, after having their `:content` set to the - rendering result" + All `:entry`s will be returned, with their `:path`s set, `:has-content` + set to `true`, and `tracer` added to `io.perun/trace`." [data renderer tmp tracer] (pod/with-call-in @render-pod (io.perun.render/update!)) @@ -428,7 +470,9 @@ (for [[path {:keys [render-data entry]}] data] (let [content (render-in-pod @render-pod renderer render-data)] (perun/create-file tmp path content) - (assoc entry :content content)))))) + (assoc entry + :path path + :has-content true)))))) (defn render-pre-wrap "Handles common rendering task orchestration @@ -439,15 +483,26 @@ Returns a boot `with-pre-wrap` result" [render-paths-fn options tracer] - (let [tmp (boot/tmp-dir!)] + (let [tmp (boot/tmp-dir!)] (boot/with-pre-wrap fileset - (let [new-metadata (-> fileset - (render-paths-fn options) - (render-to-paths (:renderer options) tmp tracer))] + (let [render-paths (render-paths-fn fileset options) + new-metadata (render-to-paths render-paths (:renderer options) tmp tracer) + rm-files (keep #(boot/tmp-get fileset (-> % :entry :path)) (vals render-paths))] (-> fileset + (boot/rm rm-files) (commit tmp) (perun/merge-meta new-metadata)))))) +(defn- make-path + [out-dir permalink path] + (perun/create-filepath + out-dir + ; If permalink ends in slash, append index.html as filename + (or (some-> permalink + (string/replace #"/$" "/index.html") + perun/url-to-path) + path))) + (deftask render "Render individual pages for entries in perun data. @@ -465,28 +520,24 @@ If permalink ends in slash, index.html is used as filename. If permalink is not set, the original filename is used with file extension set to html." [o out-dir OUTDIR str "the output directory (default: \"public\")" - _ filterer FILTER code "predicate to use for selecting entries (default: `:content`)" + _ filterer FILTER code "predicate to use for selecting entries (default: `:has-content`)" r renderer RENDERER sym "page renderer (fully qualified symbol which resolves to a function)" m meta META edn "metadata to set on each entry"] (let [options (merge +render-defaults+ *opts*)] (letfn [(render-paths [fileset options] (let [entries (filter (:filterer options) (perun/get-meta fileset)) paths (reduce - (fn [result {:keys [path] :as entry*}] - (let [entry (merge meta entry*) + (fn [result {:keys [path permalink] :as entry}] + (let [content (slurp (boot/tmp-file (boot/tmp-get fileset path))) + new-path (make-path (:out-dir options) permalink path) + meta-entry (merge meta entry) + content-entry (assoc meta-entry :content content) render-data {:meta (perun/get-global-meta fileset) :entries (vec entries) - :entry entry} - page-filepath (perun/create-filepath - (:out-dir options) - ; If permalink ends in slash, append index.html as filename - (or (some-> (:permalink entry) - (string/replace #"/$" "/index.html") - perun/url-to-path) - (string/replace path #"(?i).[a-z]+$" ".html")))] + :entry content-entry}] (perun/report-debug "render" "rendered page for path" path) - (assoc result page-filepath {:render-data render-data - :entry entry}))) + (assoc result new-path {:render-data render-data + :entry meta-entry}))) {} entries)] (perun/report-info "render" "rendered %s pages" (count paths)) @@ -504,24 +555,22 @@ (filter (:filterer options)) grouper (reduce - (fn [result [path {:keys [entries group-meta]}]] - (let [sorted (sort-by (:sortby options) (:comparator options) entries) - page-filepath (perun/create-filepath (:out-dir options) path) - new-entry (merge {:path page-filepath - :canonical-url (str (:base-url global-meta) path) - :date-build (:date-build global-meta)} - group-meta) - render-data {:meta global-meta - :entry new-entry - :entries (vec sorted)}] + (fn [result [path {:keys [entries group-meta permalink]}]] + (let [sorted (sort-by (:sortby options) (:comparator options) entries) + new-path (make-path (:out-dir options) permalink path) + new-entry (merge group-meta {:path new-path + :filename path}) + render-data {:meta global-meta + :entry new-entry + :entries (vec sorted)}] (perun/report-info task-name (str "rendered " task-name " " path)) - (assoc result page-filepath {:render-data render-data - :entry new-entry}))) + (assoc result new-path {:render-data render-data + :entry new-entry}))) {})))) (def ^:private +collection-defaults+ {:out-dir "public" - :filterer :content + :filterer :has-content :sortby (fn [file] (:date-published file)) :comparator (fn [i1 i2] (compare i2 i1))}) @@ -530,6 +579,7 @@ The symbol supplied as `renderer` should resolve to a function which will be called with a map containing the following keys: - `:meta`, global perun metadata + - `:entry`, the metadata for this collection - `:entries`, all entries Entries can optionally be filtered by supplying a function @@ -539,7 +589,7 @@ before rendering as well as rendering groups of entries to different pages." [o out-dir OUTDIR str "the output directory" r renderer RENDERER sym "page renderer (fully qualified symbol resolving to a function)" - _ filterer FILTER code "predicate to use for selecting entries (default: `:content`)" + _ filterer FILTER code "predicate to use for selecting entries (default: `:has-content`)" s sortby SORTBY code "sort entries by function" g groupby GROUPBY code "group posts by function, keys are filenames, values are to-be-rendered entries" c comparator COMPARATOR code "sort by comparator function" diff --git a/src/io/perun/markdown.clj b/src/io/perun/markdown.clj index 9ee929e0..114e40e3 100644 --- a/src/io/perun/markdown.clj +++ b/src/io/perun/markdown.clj @@ -55,7 +55,7 @@ (let [file-content (-> file :full-path io/file slurp) md-metadata (merge (:meta options) (yaml/parse-file-metadata file-content)) html (markdown-to-html file-content (:options options))] - (merge md-metadata {:content html} file))) + (merge md-metadata {:parsed html} file))) (defn parse-markdown [markdown-files options] (let [updated-files (doall (map #(process-file % options) markdown-files))] From c43171154d12b0c0cf0d07f29ba6101094b2dbe3 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sun, 4 Dec 2016 22:34:09 -0600 Subject: [PATCH 02/31] Simplify finding unchanged content files --- src/io/perun.clj | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 01036192..619ea16d 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -123,29 +123,25 @@ boot/user-files (boot/by-ext file-exts) add-filedata) - fs-files (->> fileset - boot/user-files - (boot/by-ext file-exts) - set) + input-metadata (->> + (if-let [pod (:pod options)] + (pod/with-call-in @pod ~(parse-form changed-files)) + (eval (parse-form changed-files))) + (trace tracer)) prev-fs-files (if @prev-fs (->> @prev-fs boot/user-files (boot/by-ext file-exts) set) #{}) - all-files (set/intersection fs-files prev-fs-files) unchanged-metadata (->> changed-files set - (set/difference all-files) + (set/difference prev-fs-files) + (filter #(boot/tmp-get fileset %)) (keep perun/+meta-key+)) - input-metadata (->> - (if-let [pod (:pod options)] - (pod/with-call-in @pod ~(parse-form changed-files)) - (eval (parse-form changed-files))) - (trace tracer)) output-metadata (doall - (for [{:keys [path parsed filename] :as entry*} (concat unchanged-metadata - input-metadata)] + (for [{:keys [path parsed filename] :as entry*} (concat input-metadata + unchanged-metadata)] (let [page-filepath (string/replace path #"(?i).[a-z]+$" ".html") entry (-> entry* (assoc :has-content true) From 03eec4e50422c4b9dcb427cf831c9f619ca8d04e Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sun, 4 Dec 2016 22:34:32 -0600 Subject: [PATCH 03/31] Include `:parent-path` in parsed file metadata --- src/io/perun.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 619ea16d..057616d5 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -148,7 +148,7 @@ (assoc :path page-filepath) (assoc :filename (string/replace filename #"(?i).[a-z]+$" ".html")) - (dissoc :parsed :extension :file-type :parent-path + (dissoc :parsed :extension :file-type :full-path :mime-type :original))] (perun/create-file tmp page-filepath parsed) entry))) From 88e5ff5d7e8ad18ffe0fe2ad105f0485438843b1 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sun, 4 Dec 2016 22:35:19 -0600 Subject: [PATCH 04/31] Exclude `:include-rss` and `:include-atom` from original content file metadata --- src/io/perun.clj | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 057616d5..58800bb3 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -152,8 +152,12 @@ :full-path :mime-type :original))] (perun/create-file tmp page-filepath parsed) entry))) + input-metadata* (map #(-> % + (dissoc :include-rss) + (dissoc :include-atom)) + input-metadata) final-metadata (perun/merge-meta* (perun/get-meta @prev-fs) - (concat input-metadata output-metadata)) + (concat input-metadata* output-metadata)) new-fs (-> fileset (commit tmp) (perun/set-meta final-metadata))] From 7f40dc8f2cae4afa759760e57431c590be81068d Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sun, 4 Dec 2016 23:53:32 -0600 Subject: [PATCH 05/31] Get original, not fully-rendered, content into atom xml --- src/io/perun.clj | 1 + src/io/perun/atom.clj | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 58800bb3..14a84adb 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -145,6 +145,7 @@ (let [page-filepath (string/replace path #"(?i).[a-z]+$" ".html") entry (-> entry* (assoc :has-content true) + (assoc :original-content parsed) (assoc :path page-filepath) (assoc :filename (string/replace filename #"(?i).[a-z]+$" ".html")) diff --git a/src/io/perun/atom.clj b/src/io/perun/atom.clj index 8d0bea29..9ced78ef 100644 --- a/src/io/perun/atom.clj +++ b/src/io/perun/atom.clj @@ -44,7 +44,7 @@ [:name (:author global-metadata)] [:email (:author-email global-metadata)]]) - (for [{:keys [uuid canonical-url content title author author-email] :as post} (take 10 posts) + (for [{:keys [uuid canonical-url original-content title author author-email] :as post} (take 10 posts) :let [author (or author (:author global-metadata)) author-email (or author-email (:author-email global-metadata))]] (do @@ -58,7 +58,7 @@ [:published (iso-datetime (published post))] [:updated (iso-datetime (updated post))] ;; FIXME: plain text on xml:base property - [:content {:type "html"} (str content)] + [:content {:type "html"} (str original-content)] [:author [:name author] (if author-email [:email author-email])] From b36bac8bbcf85b81ef2b17369ca16b7efc28eca0 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Mon, 5 Dec 2016 01:15:50 -0600 Subject: [PATCH 06/31] Update spec document with `:has-content`, `:original-content`, `:parsed` --- SPEC.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/SPEC.md b/SPEC.md index 39a23d1d..41086249 100644 --- a/SPEC.md +++ b/SPEC.md @@ -27,13 +27,22 @@ set for the tasks using task options. ## Post metadata -All posts have a filename which is used as a key to identify the post. +All posts have a path which is used as a key to identify the post. - **:title** - Required by: *atom-feed* - Used by: *rss* either this or description is required - **:content** The post content - - Set by: *markdown* + - Populated from file content + - Only set in the `:entry` value of the map passed to `renderer` functions +- **:has-content** Flag indicating that this file contains page content + - Set by input parsing tasks (like `markdown`) on output files + - Passed through by rendering tasks +- **:original-content** Contains the content from the original parsing task + - Set by input parsing tasks (like `markdown`) on output files + - Passed through by rendering tasks +- **:parsed** Contains the parsed file content + - Set by input parsing tasks (like `markdown`) on input files - **:description** - Used by: *rss* either this or title is required - **:slug** From 14a7e1b46b461895e869b8b6824d2a4d89f70082 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Mon, 5 Dec 2016 07:43:37 -0600 Subject: [PATCH 07/31] Small `content-pre-wrap` refactor; pass only `pod` instead of `options`, and make it optional --- src/io/perun.clj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 14a84adb..1198bebf 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -115,7 +115,7 @@ (commit fileset tmp)))) (defn content-pre-wrap - [parse-form file-exts tracer options] + [parse-form file-exts tracer & [pod]] (let [tmp (boot/tmp-dir!) prev-fs (atom nil)] (boot/with-pre-wrap fileset @@ -124,7 +124,7 @@ (boot/by-ext file-exts) add-filedata) input-metadata (->> - (if-let [pod (:pod options)] + (if pod (pod/with-call-in @pod ~(parse-form changed-files)) (eval (parse-form changed-files))) (trace tracer)) @@ -183,12 +183,12 @@ [m meta META edn "metadata to set on each entry; keys here will be overridden by metadata in each file" o options OPTS edn "options to be passed to the markdown parser"] (let [pod (create-pod markdown-deps) - options (merge +markdown-defaults+ {:pod pod} *opts*)] + options (merge +markdown-defaults+ *opts*)] (content-pre-wrap - (fn [files] `(io.perun.markdown/parse-markdown ~files ~(dissoc options :pod))) + (fn [files] `(io.perun.markdown/parse-markdown ~files ~options)) ["md" "markdown"] :io.perun/markdown - options))) + pod))) (deftask global-metadata "Read global metadata from `perun.base.edn` or configured file. From f17e211866550158e088987250166fe695db8a2b Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Mon, 5 Dec 2016 08:42:48 -0600 Subject: [PATCH 08/31] Add docstrings to `content-pre-wrap` and `make-path` --- src/io/perun.clj | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/io/perun.clj b/src/io/perun.clj index 1198bebf..2333b0d0 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -115,6 +115,10 @@ (commit fileset tmp)))) (defn content-pre-wrap + "Wrapper for input parsing tasks. Calls `parse-form` on new or changed + files with extensions in `file-exts`, adds `tracer` to `:io.perun/trace` + and writes html files for subsequent tasks to process, if desired. Pass + `pod` if one is needed for parsing" [parse-form file-exts tracer & [pod]] (let [tmp (boot/tmp-dir!) prev-fs (atom nil)] @@ -495,6 +499,8 @@ (perun/merge-meta new-metadata)))))) (defn- make-path + "Encapsulates common logic for deciding where to write a file, + based on the source's metadata" [out-dir permalink path] (perun/create-filepath out-dir From 4563918507c3b45beef156e88fd47a5618ba9142 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Mon, 5 Dec 2016 17:31:24 -0600 Subject: [PATCH 09/31] Update new default filterers, `:content` -> `:has-content` --- src/io/perun.clj | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index e62229b7..3131ad03 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -218,11 +218,11 @@ '[[time-to-read "0.1.0"]]) (def ^:private +ttr-defaults+ - {:filterer :content}) + {:filterer :has-content}) (deftask ttr "Calculate time to read for each file. Add `:ttr` key to the files' meta" - [_ filterer FILTER code "predicate to use for selecting entries (default: `:content`)"] + [_ filterer FILTER code "predicate to use for selecting entries (default: `:has-content`)"] (let [pod (create-pod ttr-deps) options (merge +ttr-defaults+ *opts*)] (boot/with-pre-wrap fileset @@ -234,11 +234,11 @@ (perun/set-meta fileset updated-files))))) (def ^:private +word-count-defaults+ - {:filterer :content}) + {:filterer :has-content}) (deftask word-count "Count words in each file. Add `:word-count` key to the files' meta" - [_ filterer FILTER code "predicate to use for selecting entries (default: `:content`)"] + [_ filterer FILTER code "predicate to use for selecting entries (default: `:has-content`)"] (let [pod (create-pod ttr-deps) options (merge +word-count-defaults+ *opts*)] (boot/with-pre-wrap fileset @@ -253,13 +253,13 @@ '[[gravatar "0.1.0"]]) (def ^:private +gravatar-defaults+ - {:filterer :content}) + {:filterer :has-content}) (deftask gravatar "Find gravatar urls using emails" [s source-key SOURCE-PROP kw "email property used to lookup gravatar url" t target-key TARGET-PROP kw "property name to store gravatar url" - _ filterer FILTER code "predicate to use for selecting entries (default: `:content`)"] + _ filterer FILTER code "predicate to use for selecting entries (default: `:has-content`)"] (let [pod (create-pod gravatar-deps) options (merge +gravatar-defaults+ *opts*)] (boot/with-pre-wrap fileset @@ -283,11 +283,11 @@ (perun/set-meta fileset updated-files)))) (def ^:private +build-date-defaults+ - {:filterer :content}) + {:filterer :has-content}) (deftask build-date "Add :date-build attribute to each file metadata and also to the global meta" - [_ filterer FILTER code "predicate to use for selecting entries (default: `:content`)"] + [_ filterer FILTER code "predicate to use for selecting entries (default: `:has-content`)"] (boot/with-pre-wrap fileset (let [options (merge +build-date-defaults+ *opts*) files (filter (:filterer options) (perun/get-meta fileset)) @@ -408,7 +408,7 @@ (deftask rss "Generate RSS feed" [f filename FILENAME str "generated RSS feed filename" - _ filterer FILTER code "predicate to use for selecting entries (default: `:content`)" + _ filterer FILTER code "predicate to use for selecting entries (default: `:include-rss`)" o out-dir OUTDIR str "the output directory" t site-title TITLE str "feed title" p description DESCRIPTION str "feed description" @@ -436,7 +436,7 @@ (deftask atom-feed "Generate Atom feed" [f filename FILENAME str "generated Atom feed filename" - _ filterer FILTER code "predicate to use for selecting entries (default: `:content`)" + _ filterer FILTER code "predicate to use for selecting entries (default: `:include-atom`)" o out-dir OUTDIR str "the output directory" t site-title TITLE str "feed title" s subtitle SUBTITLE str "feed subtitle" From d0acca3514b681a7ab78cb11fcc030e4b6d54815 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Tue, 6 Dec 2016 07:53:14 -0600 Subject: [PATCH 10/31] Don't store `:original-content` in meta; get it on demand --- src/io/perun.clj | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 3131ad03..ab1f6540 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -149,7 +149,7 @@ (let [page-filepath (string/replace path #"(?i).[a-z]+$" ".html") entry (-> entry* (assoc :has-content true) - (assoc :original-content parsed) + (assoc :original-path path) (assoc :path page-filepath) (assoc :filename (string/replace filename #"(?i).[a-z]+$" ".html")) @@ -447,7 +447,13 @@ (boot/with-pre-wrap fileset (let [global-meta (perun/get-global-meta fileset) options (merge +atom-defaults+ global-meta *opts*) - files (filter (:filterer options) (perun/get-meta fileset))] + files (->> fileset + perun/get-meta + (filter (:filterer options)) + (map #(let [f (boot/tmp-get fileset (:original-path %)) + oc (or (-> f perun/+meta-key+ :parsed) + (-> f boot/tmp-file slurp))] + (assoc % :original-content oc))))] (perun/assert-base-url (:base-url options)) (pod/with-call-in @pod (io.perun.atom/generate-atom ~(.getPath tmp) ~files ~(dissoc options :filterer))) From ba71a04a0c988e923bc27e6fb43a80a7faa004a7 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Tue, 6 Dec 2016 10:30:57 -0600 Subject: [PATCH 11/31] Update spec document, `:original-content` -> `:original-path` --- SPEC.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPEC.md b/SPEC.md index 41086249..ffb87836 100644 --- a/SPEC.md +++ b/SPEC.md @@ -38,7 +38,7 @@ All posts have a path which is used as a key to identify the post. - **:has-content** Flag indicating that this file contains page content - Set by input parsing tasks (like `markdown`) on output files - Passed through by rendering tasks -- **:original-content** Contains the content from the original parsing task +- **:original-path** The path for the input file from which this entry is descended - Set by input parsing tasks (like `markdown`) on output files - Passed through by rendering tasks - **:parsed** Contains the parsed file content From e2ba0e38a0438796d7c4f497df97d348450172f8 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Tue, 6 Dec 2016 17:13:04 -0600 Subject: [PATCH 12/31] Make atom content more flexible Rather than only using the original parsed input, if that isn't available, fall back to input content, parsed current file, or current file content, in that order. --- src/io/perun.clj | 10 ++++++---- src/io/perun/atom.clj | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index ab1f6540..ae119b4e 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -450,10 +450,12 @@ files (->> fileset perun/get-meta (filter (:filterer options)) - (map #(let [f (boot/tmp-get fileset (:original-path %)) - oc (or (-> f perun/+meta-key+ :parsed) - (-> f boot/tmp-file slurp))] - (assoc % :original-content oc))))] + (map #(let [file (if-let [original-path (:original-path %)] + (boot/tmp-get fileset original-path) + (boot/tmp-get fileset (:path %))) + content (or (-> file perun/+meta-key+ :parsed) + (-> file boot/tmp-file slurp))] + (assoc % :content content))))] (perun/assert-base-url (:base-url options)) (pod/with-call-in @pod (io.perun.atom/generate-atom ~(.getPath tmp) ~files ~(dissoc options :filterer))) diff --git a/src/io/perun/atom.clj b/src/io/perun/atom.clj index 9ced78ef..8d0bea29 100644 --- a/src/io/perun/atom.clj +++ b/src/io/perun/atom.clj @@ -44,7 +44,7 @@ [:name (:author global-metadata)] [:email (:author-email global-metadata)]]) - (for [{:keys [uuid canonical-url original-content title author author-email] :as post} (take 10 posts) + (for [{:keys [uuid canonical-url content title author author-email] :as post} (take 10 posts) :let [author (or author (:author global-metadata)) author-email (or author-email (:author-email global-metadata))]] (do @@ -58,7 +58,7 @@ [:published (iso-datetime (published post))] [:updated (iso-datetime (updated post))] ;; FIXME: plain text on xml:base property - [:content {:type "html"} (str original-content)] + [:content {:type "html"} (str content)] [:author [:name author] (if author-email [:email author-email])] From cf07276cff79551369ed36de0ee64ca6cb6d986d Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Wed, 7 Dec 2016 18:00:14 -0600 Subject: [PATCH 13/31] Correctly handle `:include-rss` and `:include-atom` on markdown file metadata --- src/io/perun.clj | 78 ++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index ae119b4e..fec9ebe0 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -4,7 +4,6 @@ [boot.pod :as pod] [boot.util :as u] [clojure.java.io :as io] - [clojure.set :as set] [clojure.string :as string] [clojure.edn :as edn] [io.perun.core :as perun])) @@ -114,6 +113,13 @@ (perun/set-meta fileset updated-files) (commit fileset tmp)))) +(defn meta-by-ext + [fileset file-exts] + (->> fileset + boot/user-files + (boot/by-ext file-exts) + (keep perun/+meta-key+))) + (defn content-pre-wrap "Wrapper for input parsing tasks. Calls `parse-form` on new or changed files with extensions in `file-exts`, adds `tracer` to `:io.perun/trace` @@ -127,45 +133,39 @@ boot/user-files (boot/by-ext file-exts) add-filedata) - input-metadata (->> - (if pod - (pod/with-call-in @pod ~(parse-form changed-files)) - (eval (parse-form changed-files))) - (trace tracer)) - prev-fs-files (if @prev-fs - (->> @prev-fs - boot/user-files - (boot/by-ext file-exts) - set) - #{}) - unchanged-metadata (->> changed-files - set - (set/difference prev-fs-files) - (filter #(boot/tmp-get fileset %)) - (keep perun/+meta-key+)) - output-metadata (doall - (for [{:keys [path parsed filename] :as entry*} (concat input-metadata - unchanged-metadata)] - (let [page-filepath (string/replace path #"(?i).[a-z]+$" ".html") - entry (-> entry* - (assoc :has-content true) - (assoc :original-path path) - (assoc :path page-filepath) - (assoc :filename (string/replace filename - #"(?i).[a-z]+$" ".html")) - (dissoc :parsed :extension :file-type - :full-path :mime-type :original))] - (perun/create-file tmp page-filepath parsed) - entry))) - input-metadata* (map #(-> % - (dissoc :include-rss) - (dissoc :include-atom)) - input-metadata) - final-metadata (perun/merge-meta* (perun/get-meta @prev-fs) - (concat input-metadata* output-metadata)) - new-fs (-> fileset + changed-meta (->> (if pod + (pod/with-call-in @pod ~(parse-form changed-files)) + (eval (parse-form changed-files))) + (map #(let [{:keys [include-rss include-atom]} %] + (-> % + (assoc :include-rss* include-rss + :include-atom* include-atom) + (dissoc :include-rss :include-atom)))) + (trace tracer)) + input-fs (-> (if @prev-fs + (perun/set-meta fileset (meta-by-ext @prev-fs file-exts)) + fileset) + (perun/set-meta changed-meta)) + input-meta (meta-by-ext input-fs file-exts) + output-meta (doall + (for [{:keys [path parsed filename + include-rss* include-atom*] :as entry*} input-meta] + (let [page-filepath (string/replace path #"(?i).[a-z]+$" ".html") + entry (-> entry* + (assoc :has-content true + :original-path path + :path page-filepath + :filename (string/replace filename + #"(?i).[a-z]+$" ".html") + :include-rss include-rss* + :include-atom include-atom*) + (dissoc :parsed :extension :file-type :full-path + :mime-type :original :include-rss* :include-atom*))] + (perun/create-file tmp page-filepath parsed) + entry))) + new-fs (-> input-fs (commit tmp) - (perun/set-meta final-metadata))] + (perun/set-meta output-meta))] (reset! prev-fs new-fs) new-fs)))) From 0ea04a4e9b029434c4eb0d6d3a37805a1ec65073 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sun, 11 Dec 2016 20:56:41 -0600 Subject: [PATCH 14/31] Use correct namespace for `get-global-meta` --- src/io/perun.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index b95b577d..1ec3b08b 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -582,7 +582,7 @@ "Produces path maps of the shape required by `render-to-paths`, based on the provided `fileset` and `options`." [task-name fileset options] - (let [global-meta (perun/get-global-meta fileset) + (let [global-meta (pm/get-global-meta fileset) grouper (:grouper options)] (->> fileset pm/get-meta From 01516fbd23e28aed3689b6e03073c9a2de994311 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Mon, 12 Dec 2016 07:13:17 -0600 Subject: [PATCH 15/31] Add `extensions` parameter to `atom-feed` and `rss` Adding this allows users to more precisely control what ends up in their feeds, and also avoids some tedious work in `content-pre-wrap` --- src/io/perun.clj | 62 +++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 1ec3b08b..2e773218 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -136,34 +136,26 @@ boot/user-files (boot/by-ext file-exts) add-filedata) - changed-meta (->> (if pod - (pod/with-call-in @pod ~(parse-form changed-files)) - (eval (parse-form changed-files))) - (map #(let [{:keys [include-rss include-atom]} %] - (-> % - (assoc :include-rss* include-rss - :include-atom* include-atom) - (dissoc :include-rss :include-atom)))) - (trace tracer)) + changed-meta (trace tracer + (if pod + (pod/with-call-in @pod ~(parse-form changed-files)) + (eval (parse-form changed-files)))) input-fs (-> (if @prev-fs (pm/set-meta fileset (meta-by-ext @prev-fs file-exts)) fileset) (pm/set-meta changed-meta)) input-meta (meta-by-ext input-fs file-exts) output-meta (doall - (for [{:keys [path parsed filename - include-rss* include-atom*] :as entry*} input-meta] + (for [{:keys [path parsed filename] :as entry*} input-meta] (let [page-filepath (string/replace path #"(?i).[a-z]+$" ".html") entry (-> entry* (assoc :has-content true :original-path path :path page-filepath :filename (string/replace filename - #"(?i).[a-z]+$" ".html") - :include-rss include-rss* - :include-atom include-atom*) + #"(?i).[a-z]+$" ".html")) (dissoc :parsed :extension :file-type :full-path - :mime-type :original :include-rss* :include-atom*))] + :mime-type :original))] (perun/create-file tmp page-filepath parsed) entry))) new-fs (-> input-fs @@ -406,22 +398,28 @@ (def ^:private +rss-defaults+ {:filename "feed.rss" :filterer :include-rss + :extensions [".html"] :out-dir "public"}) (deftask rss "Generate RSS feed" - [f filename FILENAME str "generated RSS feed filename" - _ filterer FILTER code "predicate to use for selecting entries (default: `:include-rss`)" - o out-dir OUTDIR str "the output directory" - t site-title TITLE str "feed title" - p description DESCRIPTION str "feed description" - l base-url LINK str "feed link"] + [f filename FILENAME str "generated RSS feed filename" + _ filterer FILTER code "predicate to use for selecting entries (default: `:include-rss`)" + e extensions EXTENSIONS [str] "extensions of files to include in the feed" + o out-dir OUTDIR str "the output directory" + t site-title TITLE str "feed title" + p description DESCRIPTION str "feed description" + l base-url LINK str "feed link"] (let [pod (create-pod rss-deps) tmp (boot/tmp-dir!)] (boot/with-pre-wrap fileset (let [global-meta (pm/get-global-meta fileset) options (merge +rss-defaults+ global-meta *opts*) - files (filter (:filterer options) (pm/get-meta fileset))] + files (->> fileset + boot/user-files + (boot/by-ext (:extensions options)) + (keep pm/+meta-key+) + (filter (:filterer options)))] (perun/assert-base-url (:base-url options)) (pod/with-call-in @pod (io.perun.rss/generate-rss ~(.getPath tmp) ~files ~(dissoc options :filterer))) @@ -434,24 +432,28 @@ (def ^:private +atom-defaults+ {:filename "atom.xml" :filterer :include-atom + :extensions [".html"] :out-dir "public"}) (deftask atom-feed "Generate Atom feed" - [f filename FILENAME str "generated Atom feed filename" - _ filterer FILTER code "predicate to use for selecting entries (default: `:include-atom`)" - o out-dir OUTDIR str "the output directory" - t site-title TITLE str "feed title" - s subtitle SUBTITLE str "feed subtitle" - p description DESCRIPTION str "feed description" - l base-url LINK str "feed link"] + [f filename FILENAME str "generated Atom feed filename" + _ filterer FILTER code "predicate to use for selecting entries (default: `:include-atom`)" + e extensions EXTENSIONS [str] "extensions of files to include in the feed" + o out-dir OUTDIR str "the output directory" + t site-title TITLE str "feed title" + s subtitle SUBTITLE str "feed subtitle" + p description DESCRIPTION str "feed description" + l base-url LINK str "feed link"] (let [pod (create-pod atom-deps) tmp (boot/tmp-dir!)] (boot/with-pre-wrap fileset (let [global-meta (pm/get-global-meta fileset) options (merge +atom-defaults+ global-meta *opts*) files (->> fileset - pm/get-meta + boot/output-files + (boot/by-ext (:extensions options)) + (keep pm/+meta-key+) (filter (:filterer options)) (map #(let [file (if-let [original-path (:original-path %)] (boot/tmp-get fileset original-path) From c5f6ff7f84a7329ca5b4b4d11b9471f4b02c0685 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Mon, 12 Dec 2016 23:18:51 -0600 Subject: [PATCH 16/31] Set `:content\` key for `\ttr\` and \`word-count\` tasks --- src/io/perun.clj | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 2e773218..1a2b8091 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -209,6 +209,15 @@ (perun/report-info "global-metadata" "read global metadata from %s" meta-file) (pm/set-global-meta fileset global-meta)))) +(defn- set-content-from-meta + [fileset meta] + (let [content (->> meta + :path + (boot/tmp-get fileset) + boot/tmp-file + slurp)] + (assoc meta :content content))) + (def ^:private ttr-deps '[[time-to-read "0.1.0"]]) @@ -221,7 +230,10 @@ (let [pod (create-pod ttr-deps) options (merge +ttr-defaults+ *opts*)] (boot/with-pre-wrap fileset - (let [files (filter (:filterer options) (pm/get-meta fileset)) + (let [files (->> fileset + pm/get-meta + (filter (:filterer options)) + (map #(set-content-from-meta fileset %))) updated-files (trace :io.perun/ttr (pod/with-call-in @pod (io.perun.ttr/calculate-ttr ~files)))] @@ -237,7 +249,10 @@ (let [pod (create-pod ttr-deps) options (merge +word-count-defaults+ *opts*)] (boot/with-pre-wrap fileset - (let [files (filter (:filterer options) (pm/get-meta fileset)) + (let [files (->> fileset + pm/get-meta + (filter (:filterer options)) + (map #(set-content-from-meta fileset %))) updated-files (trace :io.perun/word-count (pod/with-call-in @pod (io.perun.word-count/count-words ~files)))] From a4014235ad58dde030a1b7832756c28e60fd5575 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Mon, 12 Dec 2016 23:38:07 -0600 Subject: [PATCH 17/31] Use output-files for rss --- src/io/perun.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 1a2b8091..824c92e0 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -431,7 +431,7 @@ (let [global-meta (pm/get-global-meta fileset) options (merge +rss-defaults+ global-meta *opts*) files (->> fileset - boot/user-files + boot/output-files (boot/by-ext (:extensions options)) (keep pm/+meta-key+) (filter (:filterer options)))] From 0432a3c80c3dcfeee84f435fe54875f81c293b96 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Mon, 12 Dec 2016 23:38:46 -0600 Subject: [PATCH 18/31] Use full extensions, with leading dot, for extension lookups --- src/io/perun.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 824c92e0..8d6ce74d 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -81,7 +81,7 @@ (let [pod (create-pod images-dimensions-deps) files (->> fileset boot/user-files - (boot/by-ext ["png" "jpeg" "jpg"]) + (boot/by-ext [".png" ".jpeg" ".jpg"]) add-filedata (trace :io.perun/images-dimensions)) updated-files (pod/with-call-in @pod @@ -107,7 +107,7 @@ pod (create-pod images-resize-deps) files (->> fileset boot/user-files - (boot/by-ext ["png" "jpeg" "jpg"]) + (boot/by-ext [".png" ".jpeg" ".jpg"]) add-filedata (trace :io.perun/images-resize)) updated-files (pod/with-call-in @pod @@ -185,7 +185,7 @@ options (merge +markdown-defaults+ *opts*)] (content-pre-wrap (fn [files] `(io.perun.markdown/parse-markdown ~files ~options)) - ["md" "markdown"] + [".md" ".markdown"] :io.perun/markdown pod))) From 72b4bfaf74db29ca5ca2f83a5d7351b5347802bd Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sat, 17 Dec 2016 22:38:14 -0600 Subject: [PATCH 19/31] Add `:content` key to collection `:entries` --- src/io/perun.clj | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 8d6ce74d..4e5645c4 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -600,24 +600,34 @@ on the provided `fileset` and `options`." [task-name fileset options] (let [global-meta (pm/get-global-meta fileset) - grouper (:grouper options)] - (->> fileset - pm/get-meta - (filter (:filterer options)) - grouper - (reduce - (fn [result [path {:keys [entries group-meta permalink]}]] - (let [sorted (sort-by (:sortby options) (:comparator options) entries) - new-path (make-path (:out-dir options) permalink path) - new-entry (merge group-meta {:path new-path - :filename path}) - render-data {:meta global-meta - :entry new-entry - :entries (vec sorted)}] - (perun/report-info task-name (str "rendered " task-name " " path)) - (assoc result new-path {:render-data render-data - :entry new-entry}))) - {})))) + grouper (:grouper options) + paths (->> fileset + pm/get-meta + (filter (:filterer options)) + grouper)] + (if (seq paths) + (reduce + (fn [result [path {:keys [entries group-meta permalink]}]] + (let [sorted (->> entries + (sort-by (:sortby options) (:comparator options)) + (map #(assoc % :content (->> (:path %) + (boot/tmp-get fileset) + boot/tmp-file + slurp)))) + new-path (make-path (:out-dir options) permalink path) + new-entry (merge group-meta {:path new-path + :filename path}) + render-data {:meta global-meta + :entry new-entry + :entries (vec sorted)}] + (perun/report-info task-name (str "rendered " task-name " " path)) + (assoc result new-path {:render-data render-data + :entry new-entry}))) + {} + paths) + (do + (perun/report-info task-name (str task-name " found nothing to render")) + [])))) (def ^:private +collection-defaults+ {:out-dir "public" From ea7939f998e0a74488f1494410f0f2877438c461 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sat, 17 Dec 2016 22:48:45 -0600 Subject: [PATCH 20/31] Tighten up the data that `render-paths-fn`s return --- src/io/perun.clj | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 4e5645c4..069d6256 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -505,9 +505,9 @@ the result to `tmp`. `data` should be a map with keys that are fileset paths, and - values that are themselves maps with these keys: - - `:render-data` the map argument that `renderer` will be called with - - `:entry` the metadata for the item being rendered + values that are the map argument that `renderer` will be called with. + The values must be maps, with the required key `:entry`, representing + the page being rendered. All `:entry`s will be returned, with their `:path`s set, `:has-content` set to `true`, and `tracer` added to `io.perun/trace`." @@ -516,10 +516,10 @@ (io.perun.render/update!)) (doall (trace tracer - (for [[path {:keys [render-data entry]}] data] + (for [[path render-data] data] (let [content (render-in-pod @render-pod renderer render-data)] (perun/create-file tmp path content) - (assoc entry + (assoc (:entry render-data) :path path :has-content true)))))) @@ -582,13 +582,11 @@ (let [content (slurp (boot/tmp-file (boot/tmp-get fileset path))) new-path (make-path (:out-dir options) permalink path) meta-entry (merge meta entry) - content-entry (assoc meta-entry :content content) - render-data {:meta (pm/get-global-meta fileset) - :entries (vec entries) - :entry content-entry}] + content-entry (assoc meta-entry :content content)] (perun/report-debug "render" "rendered page for path" path) - (assoc result new-path {:render-data render-data - :entry meta-entry}))) + (assoc result new-path {:meta (pm/get-global-meta fileset) + :entries (vec entries) + :entry content-entry}))) {} entries)] (perun/report-info "render" "rendered %s pages" (count paths)) @@ -616,13 +614,11 @@ slurp)))) new-path (make-path (:out-dir options) permalink path) new-entry (merge group-meta {:path new-path - :filename path}) - render-data {:meta global-meta - :entry new-entry - :entries (vec sorted)}] + :filename path})] (perun/report-info task-name (str "rendered " task-name " " path)) - (assoc result new-path {:render-data render-data - :entry new-entry}))) + (assoc result new-path {:meta global-meta + :entry new-entry + :entries (vec sorted)}))) {} paths) (do From 61c14f112d1d07a4057e164b95ece3ebb72ab92f Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sat, 7 Jan 2017 16:07:16 -0600 Subject: [PATCH 21/31] Avoid setting `:content` key on file metadata; only use for rendering --- src/io/perun.clj | 62 ++++++++++++++++--------------------- src/io/perun/atom.clj | 5 ++- src/io/perun/ttr.clj | 15 +++++---- src/io/perun/word_count.clj | 17 +++++----- 4 files changed, 44 insertions(+), 55 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 069d6256..d6e1f11b 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -209,14 +209,11 @@ (perun/report-info "global-metadata" "read global metadata from %s" meta-file) (pm/set-global-meta fileset global-meta)))) -(defn- set-content-from-meta - [fileset meta] - (let [content (->> meta - :path - (boot/tmp-get fileset) - boot/tmp-file - slurp)] - (assoc meta :content content))) +(defn- get-meta-contents + [fileset filterer] + (->> (vals (:tree fileset)) + (filter (comp filterer pm/+meta-key+)) + (map (juxt pm/+meta-key+ (comp slurp boot/tmp-file))))) (def ^:private ttr-deps '[[time-to-read "0.1.0"]]) @@ -230,15 +227,12 @@ (let [pod (create-pod ttr-deps) options (merge +ttr-defaults+ *opts*)] (boot/with-pre-wrap fileset - (let [files (->> fileset - pm/get-meta - (filter (:filterer options)) - (map #(set-content-from-meta fileset %))) - updated-files (trace :io.perun/ttr + (let [meta-contents (get-meta-contents fileset (:filterer options)) + updated-metas (trace :io.perun/ttr (pod/with-call-in @pod - (io.perun.ttr/calculate-ttr ~files)))] - (perun/report-debug "ttr" "generated time-to-read" (map :ttr updated-files)) - (pm/set-meta fileset updated-files))))) + (io.perun.ttr/calculate-ttr ~meta-contents)))] + (perun/report-debug "ttr" "generated time-to-read" (map :ttr updated-metas)) + (pm/set-meta fileset updated-metas))))) (def ^:private +word-count-defaults+ {:filterer :has-content}) @@ -246,18 +240,16 @@ (deftask word-count "Count words in each file. Add `:word-count` key to the files' meta" [_ filterer FILTER code "predicate to use for selecting entries (default: `:has-content`)"] - (let [pod (create-pod ttr-deps) - options (merge +word-count-defaults+ *opts*)] + (let [options (merge +word-count-defaults+ *opts*)] (boot/with-pre-wrap fileset - (let [files (->> fileset - pm/get-meta - (filter (:filterer options)) - (map #(set-content-from-meta fileset %))) - updated-files (trace :io.perun/word-count - (pod/with-call-in @pod - (io.perun.word-count/count-words ~files)))] - (perun/report-debug "word-count" "counted words" (map :word-count updated-files)) - (pm/set-meta fileset updated-files))))) + (let [meta-contents (get-meta-contents fileset (:filterer options)) + updated-metas (trace :io.perun/word-count + ;; word count doesn't have any special dependencies, + ;; so we can just reuse the filedata pod + (pod/with-call-in @filedata-pod + (io.perun.word-count/count-words ~meta-contents)))] + (perun/report-debug "word-count" "counted words" (map :word-count updated-metas)) + (pm/set-meta fileset updated-metas))))) (def ^:private gravatar-deps '[[gravatar "0.1.0"]]) @@ -465,20 +457,20 @@ (boot/with-pre-wrap fileset (let [global-meta (pm/get-global-meta fileset) options (merge +atom-defaults+ global-meta *opts*) - files (->> fileset + meta-contents (->> fileset boot/output-files (boot/by-ext (:extensions options)) - (keep pm/+meta-key+) - (filter (:filterer options)) - (map #(let [file (if-let [original-path (:original-path %)] + (filter (comp (:filterer options) pm/+meta-key+)) + (map #(let [meta (pm/+meta-key+ %) + file (if-let [original-path (:original-path meta)] (boot/tmp-get fileset original-path) - (boot/tmp-get fileset (:path %))) + %) content (or (-> file pm/+meta-key+ :parsed) (-> file boot/tmp-file slurp))] - (assoc % :content content))))] + [meta content])))] (perun/assert-base-url (:base-url options)) (pod/with-call-in @pod - (io.perun.atom/generate-atom ~(.getPath tmp) ~files ~(dissoc options :filterer))) + (io.perun.atom/generate-atom ~(.getPath tmp) ~meta-contents ~(dissoc options :filterer))) (commit fileset tmp))))) (defn- assert-renderer [sym] @@ -519,7 +511,7 @@ (for [[path render-data] data] (let [content (render-in-pod @render-pod renderer render-data)] (perun/create-file tmp path content) - (assoc (:entry render-data) + (assoc (dissoc (:entry render-data) :content) :path path :has-content true)))))) diff --git a/src/io/perun/atom.clj b/src/io/perun/atom.clj index 8d0bea29..edfd18f3 100644 --- a/src/io/perun/atom.clj +++ b/src/io/perun/atom.clj @@ -32,8 +32,7 @@ [:link {:href (str base-url filename) :rel "self"}] [:link {:href base-url :type "text/html"}] [:updated (->> (take 10 posts) - (map updated) - (map iso-datetime) + (map (comp iso-datetime updated first)) sort reverse first)] @@ -44,7 +43,7 @@ [:name (:author global-metadata)] [:email (:author-email global-metadata)]]) - (for [{:keys [uuid canonical-url content title author author-email] :as post} (take 10 posts) + (for [[{:keys [uuid canonical-url title author author-email] :as post} content] (take 10 posts) :let [author (or author (:author global-metadata)) author-email (or author-email (:author-email global-metadata))]] (do diff --git a/src/io/perun/ttr.clj b/src/io/perun/ttr.clj index ea4f8150..2e2cead9 100644 --- a/src/io/perun/ttr.clj +++ b/src/io/perun/ttr.clj @@ -2,12 +2,11 @@ (:require [io.perun.core :as perun] [time-to-read.core :as time-to-read])) -(defn add-ttr [file] - (if-let [content (:content file)] - (assoc file :ttr (time-to-read/estimate-for-text content)) - file)) +(defn add-ttr [[meta content]] + (when content + (assoc meta :ttr (time-to-read/estimate-for-text content)))) -(defn calculate-ttr [files] - (let [updated-files (map add-ttr files)] - (perun/report-info "ttr" "added TTR to %s files" (count updated-files)) - updated-files)) +(defn calculate-ttr [meta-contents] + (let [metas (keep add-ttr meta-contents)] + (perun/report-info "ttr" "added TTR to %s files" (count metas)) + metas)) diff --git a/src/io/perun/word_count.clj b/src/io/perun/word_count.clj index 2a1f351d..6cc2c826 100644 --- a/src/io/perun/word_count.clj +++ b/src/io/perun/word_count.clj @@ -1,12 +1,11 @@ (ns io.perun.word-count - (:require [io.perun.core :as perun])) + (:require [io.perun.core :as perun])) -(defn add-word-count [file] - (if-let [content (:content file)] - (assoc file :word-count (count (clojure.string/split content #"\s"))) - file)) +(defn add-word-count [[meta content]] + (when content + (assoc meta :word-count (count (clojure.string/split content #"\s"))))) -(defn count-words [files] - (let [updated-files (map add-word-count files)] - (perun/report-info "word-count" "added word-count to %s files" (count updated-files)) - updated-files)) +(defn count-words [meta-contents] + (let [metas (map add-word-count meta-contents)] + (perun/report-info "word-count" "added word-count to %s files" (count metas)) + metas)) From 40fd58d4b2307864b6f3ee8039f2540ad0ed6385 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sat, 7 Jan 2017 18:20:30 -0600 Subject: [PATCH 22/31] Count words in original markdown, rather than parsed or rendered content --- src/io/perun.clj | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 0a8aa855..0c6e0004 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -209,12 +209,6 @@ (perun/report-info "global-metadata" "read global metadata from %s" meta-file) (pm/set-global-meta fileset global-meta)))) -(defn- get-meta-contents - [fileset filterer] - (->> (vals (:tree fileset)) - (filter (comp filterer pm/+meta-key+)) - (map (juxt pm/+meta-key+ (comp slurp boot/tmp-file))))) - (def ^:private ttr-deps '[[time-to-read "0.1.0"]]) @@ -227,7 +221,9 @@ (let [pod (create-pod ttr-deps) options (merge +ttr-defaults+ *opts*)] (boot/with-pre-wrap fileset - (let [meta-contents (get-meta-contents fileset (:filterer options)) + (let [meta-contents (->> (vals (:tree fileset)) + (filter (comp (:filterer options) pm/+meta-key+)) + (map (juxt pm/+meta-key+ (comp slurp boot/tmp-file)))) updated-metas (trace :io.perun/ttr (pod/with-call-in @pod (io.perun.ttr/calculate-ttr ~meta-contents)))] @@ -242,7 +238,14 @@ [_ filterer FILTER code "predicate to use for selecting entries (default: `:has-content`)"] (let [options (merge +word-count-defaults+ *opts*)] (boot/with-pre-wrap fileset - (let [meta-contents (get-meta-contents fileset (:filterer options)) + (let [meta-contents (->> (vals (:tree fileset)) + (filter (comp (:filterer options) pm/+meta-key+)) + (map #(let [meta (pm/+meta-key+ %) + file (if-let [original-path (:original-path meta)] + (boot/tmp-get fileset original-path) + %) + content (-> file boot/tmp-file slurp)] + [meta content]))) updated-metas (trace :io.perun/word-count ;; word count doesn't have any special dependencies, ;; so we can just reuse the filedata pod From 96fef7e9e33de2129771bf5339f642fd463cb222 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sun, 8 Jan 2017 11:23:19 -0600 Subject: [PATCH 23/31] Use `:path` from TmpFile, rather than maintaining in perun metadata This change prevents perun's `:path` from getting out of sync with where the file actually is. For instance, if a 3rd party task moves a file without perun's knowledge, this change allows perun to still get the correct path. --- src/io/perun.clj | 22 +++++++++++----------- src/io/perun/meta.clj | 9 +++++++-- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 0c6e0004..47cc992f 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -56,7 +56,7 @@ (defn add-filedata [tmp-files] (pod/with-call-in @filedata-pod (io.perun.filedata/filedatas - ~(vec (map (juxt boot/tmp-path #(.getPath (boot/tmp-file %)) pm/+meta-key+) tmp-files))))) + ~(vec (map (juxt boot/tmp-path #(.getPath (boot/tmp-file %)) pm/meta-from-file) tmp-files))))) (deftask base "Add some basic information to the perun metadata and @@ -121,7 +121,7 @@ (->> fileset boot/user-files (boot/by-ext file-exts) - (keep pm/+meta-key+))) + (keep pm/meta-from-file))) (defn content-pre-wrap "Wrapper for input parsing tasks. Calls `parse-form` on new or changed @@ -222,8 +222,8 @@ options (merge +ttr-defaults+ *opts*)] (boot/with-pre-wrap fileset (let [meta-contents (->> (vals (:tree fileset)) - (filter (comp (:filterer options) pm/+meta-key+)) - (map (juxt pm/+meta-key+ (comp slurp boot/tmp-file)))) + (filter (comp (:filterer options) pm/meta-from-file)) + (map (juxt pm/meta-from-file (comp slurp boot/tmp-file)))) updated-metas (trace :io.perun/ttr (pod/with-call-in @pod (io.perun.ttr/calculate-ttr ~meta-contents)))] @@ -239,8 +239,8 @@ (let [options (merge +word-count-defaults+ *opts*)] (boot/with-pre-wrap fileset (let [meta-contents (->> (vals (:tree fileset)) - (filter (comp (:filterer options) pm/+meta-key+)) - (map #(let [meta (pm/+meta-key+ %) + (filter (comp (:filterer options) pm/meta-from-file)) + (map #(let [meta (pm/meta-from-file %) file (if-let [original-path (:original-path meta)] (boot/tmp-get fileset original-path) %) @@ -280,7 +280,7 @@ "Exclude draft files" [] (boot/with-pre-wrap fileset - (let [draft-files (filter #(-> % pm/+meta-key+ :draft) (vals (:tree fileset)))] + (let [draft-files (filter #(-> % pm/meta-from-file :draft) (vals (:tree fileset)))] (perun/report-info "draft" "removed %s draft files" (count draft-files)) (boot/rm fileset draft-files)))) @@ -425,7 +425,7 @@ files (->> fileset boot/output-files (boot/by-ext (:extensions options)) - (keep pm/+meta-key+) + (keep pm/meta-from-file) (filter (:filterer options)))] (perun/assert-base-url (:base-url options)) (pod/with-call-in @pod @@ -460,12 +460,12 @@ meta-contents (->> fileset boot/output-files (boot/by-ext (:extensions options)) - (filter (comp (:filterer options) pm/+meta-key+)) - (map #(let [meta (pm/+meta-key+ %) + (filter (comp (:filterer options) pm/meta-from-file)) + (map #(let [meta (pm/meta-from-file %) file (if-let [original-path (:original-path meta)] (boot/tmp-get fileset original-path) %) - content (or (-> file pm/+meta-key+ :parsed) + content (or (-> file pm/meta-from-file :parsed) (-> file boot/tmp-file slurp))] [meta content])))] (perun/assert-base-url (:base-url options)) diff --git a/src/io/perun/meta.clj b/src/io/perun/meta.clj index 93f6c8b5..3b06c0ee 100644 --- a/src/io/perun/meta.clj +++ b/src/io/perun/meta.clj @@ -4,11 +4,16 @@ (def +meta-key+ :io.perun) +(defn meta-from-file + [tmpfile] + (when-let [meta (+meta-key+ tmpfile)] + (assoc meta :path (:path tmpfile)))) + (defn get-meta "Return metadata on files. Files metadata is a list. Internally it's stored as a map indexed by `:path`" [fileset] - (keep +meta-key+ (vals (:tree fileset)))) + (keep meta-from-file (vals (:tree fileset)))) (defn key-meta [data] (into {} (for [d data] [(:path d) d]))) @@ -16,7 +21,7 @@ (defn set-meta "Update `+meta-key+` metadata for files in `data` and return updated fileset" [fileset data] - (boot/add-meta fileset (into {} (for [d data] [(:path d) {+meta-key+ d}])))) + (boot/add-meta fileset (into {} (for [d data] [(:path d) {+meta-key+ (dissoc d :path)}])))) (defn merge-meta* [m1 m2] (vals (merge-with merge (key-meta m1) (key-meta m2)))) From c8b10235609c2fded9403643d8f8dc0357a4e986 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Fri, 13 Jan 2017 19:49:06 -0600 Subject: [PATCH 24/31] Build regex from `file-exts` parameter to `content-pre-wrap` --- src/io/perun.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 47cc992f..20c20768 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -147,13 +147,14 @@ input-meta (meta-by-ext input-fs file-exts) output-meta (doall (for [{:keys [path parsed filename] :as entry*} input-meta] - (let [page-filepath (string/replace path #"(?i).[a-z]+$" ".html") + (let [ext-pattern (re-pattern (str "(" (string/join "|" file-exts) ")$")) + page-filepath (string/replace path ext-pattern ".html") entry (-> entry* (assoc :has-content true :original-path path :path page-filepath :filename (string/replace filename - #"(?i).[a-z]+$" ".html")) + ext-pattern ".html")) (dissoc :parsed :extension :file-type :full-path :mime-type :original))] (perun/create-file tmp page-filepath parsed) From 51e7fef43096c4a328ccd7bcd0e95fd55b88a3f3 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Fri, 13 Jan 2017 20:12:17 -0600 Subject: [PATCH 25/31] Set new filedata on files created by `content-pre-wrap` --- src/io/perun.clj | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 20c20768..4548b551 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -155,13 +155,14 @@ :path page-filepath :filename (string/replace filename ext-pattern ".html")) - (dissoc :parsed :extension :file-type :full-path - :mime-type :original))] + (dissoc :parsed :original))] (perun/create-file tmp page-filepath parsed) entry))) - new-fs (-> input-fs - (commit tmp) - (pm/set-meta output-meta))] + new-fs* (-> input-fs + (commit tmp) + (pm/set-meta output-meta)) + filedata (add-filedata (boot/by-path (map :path output-meta) (boot/ls new-fs*))) + new-fs (pm/set-meta new-fs* filedata)] (reset! prev-fs new-fs) new-fs)))) From 51fe571b114f5a53c4a5e41ace0acf4ea55f0db7 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Fri, 13 Jan 2017 20:14:20 -0600 Subject: [PATCH 26/31] Amend `markdown` docstring to note it writes HTML files --- src/io/perun.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index 4548b551..fee1b46c 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -180,7 +180,8 @@ This task will look for files ending with `md` or `markdown` and add a `:parsed` key to their metadata containing the - HTML resulting from processing markdown file's content" + HTML resulting from processing markdown file's content. Also + writes an HTML file that contains the same content as `:parsed`" [m meta META edn "metadata to set on each entry; keys here will be overridden by metadata in each file" o options OPTS edn "options to be passed to the markdown parser"] (let [pod (create-pod markdown-deps) From 5c642967602cecc96d7146ba95ea3cc6305eabc3 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Fri, 13 Jan 2017 20:15:54 -0600 Subject: [PATCH 27/31] Add debug statement before file removal during rendering --- src/io/perun.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/io/perun.clj b/src/io/perun.clj index fee1b46c..e1e55995 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -532,6 +532,7 @@ (let [render-paths (render-paths-fn fileset options) new-metadata (render-to-paths render-paths (:renderer options) tmp tracer) rm-files (keep #(boot/tmp-get fileset (-> % :entry :path)) (vals render-paths))] + (perun/report-debug "render-pre-wrap" "removing files" rm-files) (-> fileset (boot/rm rm-files) (commit tmp) From 919f16feabeb59544d26c20c2856458122d53e49 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Fri, 13 Jan 2017 20:22:31 -0600 Subject: [PATCH 28/31] Move rendering debug and info statements to more logical places --- src/io/perun.clj | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/io/perun.clj b/src/io/perun.clj index e1e55995..bb7c6011 100644 --- a/src/io/perun.clj +++ b/src/io/perun.clj @@ -506,7 +506,7 @@ All `:entry`s will be returned, with their `:path`s set, `:has-content` set to `true`, and `tracer` added to `io.perun/trace`." - [data renderer tmp tracer] + [task-name data renderer tmp tracer] (pod/with-call-in @render-pod (io.perun.render/update!)) (doall @@ -514,6 +514,7 @@ (for [[path render-data] data] (let [content (render-in-pod @render-pod renderer render-data)] (perun/create-file tmp path content) + (perun/report-debug task-name "rendered page for path" path) (assoc (dissoc (:entry render-data) :content) :path path :has-content true)))))) @@ -526,13 +527,14 @@ that are required by `render-paths-fn`. Returns a boot `with-pre-wrap` result" - [render-paths-fn options tracer] + [task-name render-paths-fn options tracer] (let [tmp (boot/tmp-dir!)] (boot/with-pre-wrap fileset (let [render-paths (render-paths-fn fileset options) - new-metadata (render-to-paths render-paths (:renderer options) tmp tracer) + new-metadata (render-to-paths task-name render-paths (:renderer options) tmp tracer) rm-files (keep #(boot/tmp-get fileset (-> % :entry :path)) (vals render-paths))] - (perun/report-debug "render-pre-wrap" "removing files" rm-files) + (perun/report-info task-name "rendered %s pages" (count render-paths)) + (perun/report-debug task-name "removing files" rm-files) (-> fileset (boot/rm rm-files) (commit tmp) @@ -579,15 +581,13 @@ new-path (make-path (:out-dir options) permalink path) meta-entry (merge meta entry) content-entry (assoc meta-entry :content content)] - (perun/report-debug "render" "rendered page for path" path) (assoc result new-path {:meta (pm/get-global-meta fileset) :entries (vec entries) :entry content-entry}))) {} entries)] - (perun/report-info "render" "rendered %s pages" (count paths)) paths))] - (render-pre-wrap render-paths options :io.perun/render)))) + (render-pre-wrap "render" render-paths options :io.perun/render)))) (defn- grouped-paths "Produces path maps of the shape required by `render-to-paths`, based @@ -674,7 +674,7 @@ (u/fail "collection task :sortby option value should implement IFn\n") :else (let [collection-paths (partial grouped-paths "collection")] - (render-pre-wrap collection-paths options :io.perun/collection))))) + (render-pre-wrap "collection" collection-paths options :io.perun/collection))))) (deftask inject-scripts "Inject JavaScript scripts into html files. From 5a48a1f664b555faf6efe6557c94e15bde9e308f Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sat, 14 Jan 2017 08:22:43 -0600 Subject: [PATCH 29/31] Add test for #77 - collection renders without input files --- test/io/perun_test.clj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/io/perun_test.clj b/test/io/perun_test.clj index 69fc0912..c7ff2f53 100644 --- a/test/io/perun_test.clj +++ b/test/io/perun_test.clj @@ -122,7 +122,9 @@ This be ___markdown___.") (str "" (:content (:entry data)) "")) (deftesttask markdown-test [] - (comp (add-txt-file :path "2017-01-01-test.md" :content md-content) + (comp (p/collection :renderer 'io.perun-test/render) ;; test #77, collection works without input files + + (add-txt-file :path "2017-01-01-test.md" :content md-content) (boot/with-pre-wrap fileset (pm/set-global-meta fileset {:base-url "http://example.com/" :site-title "Test Title" From f677bd81f9c826baf55b0bc5d0345df40a2586c5 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sat, 14 Jan 2017 14:28:29 -0600 Subject: [PATCH 30/31] Add content tests --- test/io/perun_test.clj | 84 +++++++++++++++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 18 deletions(-) diff --git a/test/io/perun_test.clj b/test/io/perun_test.clj index c7ff2f53..5ed5f5cc 100644 --- a/test/io/perun_test.clj +++ b/test/io/perun_test.clj @@ -2,6 +2,7 @@ (:require [boot.core :as boot :refer [deftask]] [boot.test :as boot-test :refer [deftesttask]] [clojure.java.io :as io] + [clojure.string :as str] [clojure.test :refer [deftest testing is]] [io.perun :as p] [io.perun.meta :as pm]) @@ -121,53 +122,100 @@ This be ___markdown___.") [data] (str "" (:content (:entry data)) "")) -(deftesttask markdown-test [] - (comp (p/collection :renderer 'io.perun-test/render) ;; test #77, collection works without input files - - (add-txt-file :path "2017-01-01-test.md" :content md-content) +(deftesttask default-tests [] + (comp (add-txt-file :path "2017-01-01-test.md" :content md-content) (boot/with-pre-wrap fileset (pm/set-global-meta fileset {:base-url "http://example.com/" :site-title "Test Title" :description "Test Desc"})) (p/markdown) - (content-test :path "2017-01-01-test.html" :content parsed-md) - (value-test :path "2017-01-01-test.md" :value-fn #(meta= % :parsed parsed-md)) + (testing "markdown" + (content-test :path "2017-01-01-test.html" :content parsed-md) + (value-test :path "2017-01-01-test.md" :value-fn #(meta= % :parsed parsed-md))) (p/ttr) - (value-test :path "2017-01-01-test.html" :value-fn #(meta= % :ttr 1)) + (testing "ttr" + (value-test :path "2017-01-01-test.html" :value-fn #(meta= % :ttr 1))) (p/word-count) - (value-test :path "2017-01-01-test.html" :value-fn #(meta= % :word-count 18)) + (testing "word-count" + (value-test :path "2017-01-01-test.html" :value-fn #(meta= % :word-count 18))) (p/gravatar :source-key :email :target-key :gravatar) - (value-test :path "2017-01-01-test.html" :value-fn #(meta= % :gravatar "http://www.gravatar.com/avatar/a1a361f6c96acb1e31ad4b3bbf7aa444")) + (testing "gravatar" + (value-test :path "2017-01-01-test.html" + :value-fn + #(meta= % :gravatar "http://www.gravatar.com/avatar/a1a361f6c96acb1e31ad4b3bbf7aa444"))) (p/build-date) - (key-test :path "2017-01-01-test.html" :key :date-build) + (testing "build-date" + (key-test :path "2017-01-01-test.html" :key :date-build)) (p/slug) - (value-test :path "2017-01-01-test.html" :value-fn #(meta= % :slug "test")) + (testing "slug" + (value-test :path "2017-01-01-test.html" :value-fn #(meta= % :slug "test"))) (p/permalink) - (value-test :path "2017-01-01-test.html" :value-fn #(meta= % :permalink "/test/index.html")) + (testing "permalink" + (value-test :path "2017-01-01-test.html" :value-fn #(meta= % :permalink "/test/index.html"))) (p/canonical-url) - (value-test :path "2017-01-01-test.html" :value-fn #(meta= % :canonical-url "http://example.com/test/index.html")) + (testing "canonical-url" + (value-test :path "2017-01-01-test.html" + :value-fn #(meta= % :canonical-url "http://example.com/test/index.html"))) (p/sitemap) - (file-exists? :path "public/sitemap.xml") + (testing "sitemap" + (file-exists? :path "public/sitemap.xml")) (p/rss) - (file-exists? :path "public/feed.rss") + (testing "rss" + (file-exists? :path "public/feed.rss")) (p/atom-feed) - (file-exists? :path "public/atom.xml") + (testing "atom-feed" + (file-exists? :path "public/atom.xml")) (p/render :renderer 'io.perun-test/render) (add-txt-file :path "test.js" :content js-content) (p/inject-scripts :scripts #{"test.js"}) - (content-test :path "public/test/index.html" :content (str "")) + (testing "inject-scripts" + (content-test :path "public/test/index.html" :content (str ""))) (p/draft) - (file-exists? :path "2017-01-01-test.html" :negate? true))) + (testing "draft" + (file-exists? :path "2017-01-01-test.html" :negate? true)))) + +(deftesttask content-tests [] + (comp (testing "Collection works without input files" ;; #77 + (p/collection :renderer 'io.perun-test/render)) + + (add-txt-file :path "test.md" :content md-content) + (p/markdown) ;; render once + + (add-txt-file :path "test.md" :content (str/replace md-content #"Hello" "Salutations")) + (p/markdown) + (testing "detecting content changes" + (content-test :path "test.html" :content "Salutations")) + + (add-txt-file :path "test.md" :content (str/replace md-content #"draft: true" "draft: false")) + (p/markdown) + (testing "detecting metadata changes" + (value-test :path "test.html" :value-fn #(meta= % :draft false))) + + (add-txt-file :path "test.md" :content (str/replace md-content #"draft: true" "draft: true\nfoo: bar")) + (p/markdown) + (testing "detecting metadata additions" + (value-test :path "test.html" :value-fn #(meta= % :foo "bar"))) + + (add-txt-file :path "test.md" :content md-content) + (p/markdown) + (testing "detecting metadata deletions" + (value-test :path "test.html" :value-fn #(meta= % :foo nil))) + + (add-txt-file :path "test2.md" :content md-content) + (p/markdown) + (testing "detecting new files" + (content-test :path "test2.html" :content parsed-md) + (value-test :path "test2.md" :value-fn #(meta= % :parsed parsed-md))))) From d7cb61b362e7d74c64befcda5086af82667b2125 Mon Sep 17 00:00:00 2001 From: Brent Hagany Date: Sun, 15 Jan 2017 15:31:54 -0600 Subject: [PATCH 31/31] Add testing util task --- test/io/perun_test.clj | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/io/perun_test.clj b/test/io/perun_test.clj index c561e8e8..defa1d87 100644 --- a/test/io/perun_test.clj +++ b/test/io/perun_test.clj @@ -18,6 +18,12 @@ (or (some #{val} (-> file pm/+meta-key+ key)) (contains? (-> file pm/+meta-key+ key) val))) +(deftask prn-meta-key + [p path PATH str "path of the file to test" + k key KEY kw "the key to prn"] + (boot/with-pass-thru fileset + (->> path (boot/tmp-get fileset) pm/+meta-key+ key prn))) + (deftask key-test [p path PATH str "path of the file to test" k key KEY kw "the key to test"