Concept of an application which components run within

src/stoic/bootstrap.clj

@@ -3,15 +3,14 @@
   (:require [com.stuartsierra.component :as component]
             [stoic.components.foo]
             [stoic.protocols.config-supplier :as cs]
-            [stoic.config.zk]
             [stoic.config.file :as file]
             [stoic.config.curator]
             [clojure.tools.logging :as log]))
 
-(defn choose-supplier []
+(defn build-config-supplier [system-name]
   (if (file/enabled?)
-    (file/config-supplier)
-    (stoic.config.curator/config-supplier)))
+    (file/config-supplier system-name)
+    (stoic.config.curator/config-supplier system-name)))
 
 (defn- inject-components
   "Inject components associating in the respective settings as an atom.
@@ -21,14 +20,18 @@
          (reduce into []
                  (for [[k c] system]
                    [k (or (and (:settings c) c)
+                          (when-not (associative? c) c)
                           (assoc c :settings (get component-settings k)))]))))
 
 (defn- fetch-settings
   "Fetch settings from the config supplier and wrap in atoms."
   [config-supplier system]
-  (into {} (for [[k c] system
-                 :when (not (:settings c))]
-             [k (atom (cs/fetch config-supplier k))])))
+  (let [system-settings (when-let [system-name (:name system)]
+                          {system-name (cs/fetch config-supplier (:name system))})]
+    (into {} (for [[k c] system
+                   :when (not (:settings c))]
+               [k (atom (merge system-settings
+                               (cs/fetch config-supplier k)))]))))
 
 (defn- bounce-component! [config-supplier k c settings-atom]
   (let [settings (cs/fetch config-supplier k)]
@@ -51,7 +54,7 @@
    Components will be bounced when their respective settings change.
    Returns a SystemMap with Stoic config attached."
   ([system]
-     (bootstrap (choose-supplier) system))
+     (bootstrap (build-config-supplier (:name system)) system))
   ([config-supplier system]
      (let [config-supplier-component (component/start config-supplier)
            component-settings (fetch-settings config-supplier-component system)

src/stoic/config/curator.clj

@@ -4,6 +4,7 @@
             [stoic.protocols.config-supplier]
             [stoic.config.data :refer :all]
             [stoic.config.env :refer :all]
+            [stoic.merge :refer [deep-merge]]
             [com.stuartsierra.component :as component]
             [clojure.tools.logging :as log]
             [stoic.config.exhibitor :refer :all])
@@ -32,7 +33,21 @@
   (.. client checkExists watched (usingWatcher watcher)
       (forPath path)))
 
-(defrecord CuratorConfigSupplier [root]
+(defn- safe-read [client path]
+  (when-not (.. client checkExists (forPath path))
+    (.. client create (forPath path nil)))
+  (read-from-zk client path))
+
+(defn- register-watch [client path watcher-fn]
+  (watch-path client path
+              (reify CuratorWatcher
+                (process [this event]
+                  (when (= :NodeDataChanged (keyword (.. event getType name)))
+                    (log/info "Data changed, firing watcher" event)
+                    (watcher-fn)
+                    (watch-path client path this))))))
+
+(defrecord CuratorConfigSupplier [root system-name]
   stoic.protocols.config-supplier/ConfigSupplier
   component/Lifecycle
 
@@ -46,20 +61,10 @@
     this)
 
   (fetch [{:keys [client]} k]
-    (let [path (path-for root k)]
-      (when-not (.. client checkExists (forPath path))
-        (.. client create (forPath path nil)))
-      (read-from-zk client path)))
+    (safe-read client (path-for root k)))
 
   (watch! [{:keys [client]} k watcher-fn]
-    (let [path (path-for root k)]
-      (watch-path client path
-                  (reify CuratorWatcher
-                    (process [this event]
-                      (when (= :NodeDataChanged (keyword (.. event getType name)))
-                        (log/info "Data changed, firing watcher" event)
-                        (watcher-fn)
-                        (watch-path client path this))))))))
+    (register-watch client (path-for root k) watcher-fn)))
 
-(defn config-supplier []
-  (CuratorConfigSupplier. (zk-root)))
+(defn config-supplier [system-name]
+  (CuratorConfigSupplier. (zk-root) system-name))

src/stoic/config/file.clj

@@ -2,6 +2,7 @@
   "Loads stoic config from a file containing edn.The file is watched for changes and config is reloaded when they occur."
   (:import (java.nio.file AccessDeniedException))
   (:require [stoic.protocols.config-supplier]
+            [stoic.merge :refer [deep-merge]]
             [clojure.tools.logging :as log]
             [clojure.edn :as edn]
             [clojure.java.io :as io]
@@ -25,41 +26,49 @@
     true
     (catch IllegalArgumentException e false)))
 
-(defn read-config [file-path]
+(defn read-config [config-files]
   "Reads the edn based config from the specified file"
-  (log/info "Reading config from file: " file-path)
-  (let [config (edn/read-string (slurp file-path))]
+  (log/info "Reading config from files: " (map #(.getAbsolutePath %) config-files))
+  (let [config (apply deep-merge (map #(edn/read-string (slurp %)) config-files))]
     (log/debug "Config: " config)
     config))
 
-(defn- config-file-change? [config-path f]
+(defn- config-file-change? [config-files f]
   "Filter out all file system events that do not match the stoic config file modification or creation"
-  (and (= config-path (.getAbsolutePath (:file f))) (not= (:action f) :delete)))
+  (and (contains? (set config-files) (:file f)) (not= (:action f) :delete)))
 
-(defn reload-config! [config-atom config-path watch-fn-atom file-events]
+(defn reload-config! [config-atom config-files watch-fn-atom file-events]
   "If the stoic config file has changed, reload the config and call the optional watch function"
-  (when (not-empty (filter (partial config-file-change? config-path) file-events))
+  (when (not-empty (filter (partial config-file-change? config-files) file-events))
     (log/info "Reloading config...")
-    (reset! config-atom (read-config config-path))
+    (reset! config-atom (read-config config-files))
     (when @watch-fn-atom
       (@watch-fn-atom))))
 
-(defrecord FileConfigSupplier []
+(defn- config-files [config-paths system-name]
+  (for [path config-paths
+        :let [f (io/file path)]]
+    (if (.isDirectory f)
+      (let [system-settings (io/file f (str (name system-name) ".edn"))]
+        (when (.exists system-settings) system-settings))
+      f)))
+
+(defrecord FileConfigSupplier [system-name]
   stoic.protocols.config-supplier/ConfigSupplier
   component/Lifecycle
 
   (start [this]
-    (let [config-path (*read-config-path*)
-          config-dir (.getParentFile (io/as-file config-path))]
+    (let [config-paths (string/split (*read-config-path*) #":")
+          config-files (config-files config-paths system-name)
+          config-dirs  (set (map #(.getParentFile %) config-files))]
       (try
-        (let [
-               config-atom (atom (read-config config-path))
-               watch-fn-atom (atom nil)
-               config-watcher (watch-dir (partial reload-config! config-atom config-path watch-fn-atom) config-dir)]
+        (let [config-atom (atom (read-config config-files))
+              watch-fn-atom (atom nil)
+              config-watcher (apply watch-dir (partial reload-config! config-atom config-files watch-fn-atom) config-dirs)]
           (assoc this :config config-atom :config-watcher config-watcher :watch-fn watch-fn-atom))
         (catch AccessDeniedException e
           (do
-            (log/fatal "Unable to assign watcher to directory " (.getAbsolutePath config-dir) " check permissions")
+            (log/fatal "Unable to assign watcher to directories " (map #(.getAbsolutePath %) config-dirs) " check permissions")
             (throw e))))))
 
   (stop [this]
@@ -73,5 +82,5 @@
   (watch! [{:keys [watch-fn]} k watcher-function]
     (reset! watch-fn watcher-function)))
 
-(defn config-supplier []
-  (FileConfigSupplier.))
+(defn config-supplier [system-name]
+  (FileConfigSupplier. system-name))

src/stoic/merge.clj

@@ -0,0 +1,6 @@
+(ns stoic.merge)
+
+(defn deep-merge [& maps]
+  (if (every? map? maps)
+    (apply merge-with deep-merge maps)
+    (last maps)))

test-resources/config/test-application.edn

@@ -0,0 +1,2 @@
+{:test-application {:pqr :stu}
+ :http-kit {:threads 16}}

test-resources/config/test.edn

@@ -1,6 +1,2 @@
-{:sauron
-           {:consumer-portal-url "http://test.com:8080",
-            :storm-ui-url "http://test2.com:8080",
-            :media-url-prefix "http://images.test.com/"},
- :http-kit {:port 8080, :threads 4}
- }
+{:ptth-tik {:abc :def}
+ :http-kit {:port 8080 :threads 4}}

test/stoic/bootstrap_test.clj

@@ -1,6 +1,7 @@
 (ns stoic.bootstrap-test
   (:use [clojure.core.async :only [chan timeout >!! <!! buffer alts!!]])
-  (:require [stoic.config.zk :as stoic-zk]
+  (:require [stoic.config.curator :as stoic-zk]
+            [stoic.config.env :refer [zk-root]]
             [clojure.test :refer :all]
             [stoic.protocols.config-supplier :as cs]
             [com.stuartsierra.component :as component]
@@ -25,7 +26,7 @@
 
 (deftest can-bounce-component-on-config-change
   (harness
-   (stoic-zk/add-to-zk client (path-for :default :test) {:a :initial-value})
+   (stoic-zk/add-to-zk client (path-for (zk-root) :test) {:a :initial-value})
 
    (let [starts (chan (buffer 1))
          stops (chan (buffer 1))]
@@ -35,19 +36,19 @@
 
      (is (= :initial-value (first (alts!! [(timeout 2000) starts]))))
 
-     (stoic-zk/add-to-zk client (path-for :default :test) {:a :b})
+     (stoic-zk/add-to-zk client (path-for (zk-root) :test) {:a :b})
 
      (is (= :initial-value (first (alts!! [(timeout 2000) stops]))))
      (is (= :b (first (alts!! [(timeout 2000) starts]))))
 
-     (stoic-zk/add-to-zk client (path-for :default :test) {:a :c})
+     (stoic-zk/add-to-zk client (path-for (zk-root) :test) {:a :c})
 
      (is (= :b (first (alts!! [(timeout 2000) stops]))))
      (is (= :c (first (alts!! [(timeout 2000) starts])))))))
 
 (deftest can-not-bounce-component-if-doesnt-change
   (harness
-   (stoic-zk/add-to-zk client (path-for :default :test) {:a :initial-value})
+   (stoic-zk/add-to-zk client (path-for (zk-root) :test) {:a :initial-value})
 
    (let [starts (chan (buffer 1))
          stops (chan (buffer 1))]
@@ -57,7 +58,7 @@
 
      (is (= :initial-value (first (alts!! [(timeout 2000) starts]))))
 
-     (stoic-zk/add-to-zk client (path-for :default :test) {:a :initial-value})
+     (stoic-zk/add-to-zk client (path-for (zk-root) :test) {:a :initial-value})
 
      (let [t-c (timeout 2000)
            [v c] (alts!! [t-c stops])]
@@ -77,8 +78,8 @@
 
 (deftest component-with-dependencies-get-injected
   (harness
-   (stoic-zk/add-to-zk client (path-for :default :test1) {:a :test-1-value})
-   (stoic-zk/add-to-zk client (path-for :default :test2) {:a :test-2-value})
+   (stoic-zk/add-to-zk client (path-for (zk-root) :test1) {:a :test-1-value})
+   (stoic-zk/add-to-zk client (path-for (zk-root) :test2) {:a :test-2-value})
 
    (let [system (component/start
                  (b/bootstrap
@@ -97,15 +98,30 @@
 
 (deftest components-are-shutdown-safely-if-component-fails-during-startup
   (harness
-   (stoic-zk/add-to-zk client (path-for :default :test1) {:a :initial-value})
+   (stoic-zk/add-to-zk client (path-for (zk-root) :test1) {:a :initial-value})
    (let [starts (chan (buffer 1))
          stops (chan (buffer 1))
          system
          (b/start-safely
           (b/bootstrap
-           (component/system-map :test1  (->TestAsyncComponent starts stops)
+           (component/system-map :test1 (->TestAsyncComponent starts stops)
                                  :test2 (component/using
                                          (map->TestBarfingComponent {})
                                          [:test1]))))]
      (is (= :initial-value (first (alts!! [(timeout 2000) starts]))))
      (is (= :initial-value (first (alts!! [(timeout 2000) stops])))))))
+
+(deftest system-with-a-name-has-application-settings-merged
+  (harness
+   (stoic-zk/add-to-zk client (path-for (zk-root) :test1) {:a :test-1-value})
+   (stoic-zk/add-to-zk client (path-for (zk-root) :test-application) {:y :z})
+
+   (testing "Application settings are merged with component settings"
+     (let [system (component/start
+                   (b/bootstrap
+                    (component/system-map :name :test-application
+                                          :test1 (map->TestComponent {}))))]
+
+       (is (= :test-application (-> system :name)))
+       (is (= {:a :test-1-value
+               :test-application {:y :z}} (-> system :test1 :settings deref)))))))

test/stoic/config/curator_test.clj

@@ -0,0 +1,20 @@
+(ns stoic.config.curator-test
+  (:require [stoic.config.curator :refer :all]
+            [stoic.config.env :refer :all]
+            [stoic.config.data :refer :all]
+            [clojure.test :refer :all]
+            [stoic.protocols.config-supplier :as cs]
+            [com.stuartsierra.component :as component]))
+
+(deftest can-write-and-read-from-zookeeper
+  (let [expected {:a :b}
+        zk (component/start (config-supplier :foo))]
+    (add-to-zk (connect) (path-for (zk-root) :foo) expected)
+    (is (= expected (cs/fetch zk :foo)))))
+
+#_(deftest deep-merges-multiple-paths-in-config
+  (let [config (:config (component/start (->FileConfigSupplier :test-application)))]
+    (is (= {:ptth-tik {:abc :def}
+            :http-kit {:port 8080 :threads 16}
+            :test-application {:pqr :stu}}
+           @config))))