diff --git a/package.json b/package.json index 4647393..db58163 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "inquirer": "^6.2.2", "mkdirp": "^0.5.1", "open": "6.4.0", - "rimraf": "^3.0.0" + "rimraf": "^3.0.0", + "yauzl": "3.2.0" } } diff --git a/src/sharetribe/flex_cli/api/client.cljs b/src/sharetribe/flex_cli/api/client.cljs index 03412a7..b545124 100644 --- a/src/sharetribe/flex_cli/api/client.cljs +++ b/src/sharetribe/flex_cli/api/client.cljs @@ -1,12 +1,15 @@ (ns sharetribe.flex-cli.api.client - (:require [cognitect.transit :as t] - [clojure.core.async :as async :refer [put! chan chan req :query :marketplace) api-key-suffix (->> req :client @@ -66,7 +69,12 @@ [:span "Use " (bold (str cli-info/bin " login")) " to relogin if needed." :line]) :else - (error-page [:span "API call failed. Status: " (str status) ", reason: " (or (-> response :errors first :title) "Unspecified") :line])))) + (error-page (cond-> + [:span "API call failed. Status: " (str status) + ", reason: " (or (-> response :errors first :title) original-text "Unspecified")] + ;; Enable for debugging + #_#_(-> response :errors first :details) (conj (str ", details: " (-> response :errors first :details))) + true (conj :line)))))) (defmethod exception/format-exception :api/error [_ _ data] (default-error-format data)) @@ -111,84 +119,105 @@ (.set (.-searchParams url) (name k) (to-query-param v))) url)) -(defn read-transit [s] +(defn- content-type [response] (try - {:success true - :data (t/read transit-reader s)} + (util/MIMEType. (-> response .-headers (.get "content-type"))) (catch js/Error _e - {:success false}))) - -(defn- handle-response [out-chan response req] - (-> - (.text response) - (.then - (fn [body-str] - (let [parsed (read-transit body-str)] - (put! out-chan - (if (.-ok response) - (:data parsed) - (let [parsed (read-transit body-str)] - (handle-error req - (cond-> {:status (.-status response) - :status-text (.-statusText response)} - (:success parsed) (assoc :response (:data parsed)) - (not (:success parsed)) (assoc :original-text body-str))))))))))) - -(defn do-get [client path query] - (let [c (chan)] - (-> (js/fetch - (construct-url (str (config/value :api-base-url) path) query) - (clj->js - {:method "GET" - :headers {"Authorization" (str "Apikey " (::api-key client)) - "User-Agent" user-agent - "Accept" "application/transit+json"}})) - (.then (fn [response] - (handle-response c response {:client client - :path path - :query query}))) - ;; Unknown failure - (.catch (fn [error] (put! c (exception/exception :client/api-request-failed error))))) - c)) + nil))) + +(defn- parse-text [response] + (go + (try + {:success true + :type "text/plain" + :content (chan (.text response)))} + (catch js/Error _e + {:success false})))) + +(defn- parse-transit [response] + (go + (try + {:success true + :type "application/transit+json" + :content (t/read transit-reader ( {:status (.-status response) + :status-text (.-statusText response)} + + ;; Assoc :response only if parsing was + ;; successful and response content type + ;; was what we expected. If we for + ;; example use Accept: + ;; application/transit+json, but get + ;; back 500 with text/plain, we don't + ;; assoc it to response + (and accepted? success) (assoc :response content) + + (= "text/plain" type) + (assoc :original-text content))))))) + +(defn- do-request [client path method query body opts] + (go-try + (let [accept (or (::accept opts) "application/transit+json") + res (chan (js/fetch + (construct-url (str (config/value :api-base-url) path) query) + (clj->js + (cond-> + {:method method + :headers (cond-> {"Authorization" (str "Apikey " (::api-key client)) + "User-Agent" user-agent + "Accept" accept} + (::content-type opts) (assoc "Content-Type" (::content-type opts)))} + body (assoc :body body))))))] + + ( (js/fetch - (construct-url (str (config/value :api-base-url) path) query) - (clj->js - {:method "POST" - :body (t/write transit-writer body) - :headers {"Authorization" (str "Apikey " (::api-key client)) - "Content-Type" "application/transit+json" - "User-Agent" user-agent - "Accept" "application/transit+json"}})) - (.then (fn [response] - (handle-response c response {:client client - :path path - :query query}))) - - ;; Unknown failure - (.catch (fn [error] (put! c (exception/exception :client/api-request-failed error))))) - c)) + (do-request client path "POST" query (t/write transit-writer body) {::content-type "application/transit+json"})) (defn do-multipart-post [client path query form-data] - (let [c (chan)] - (-> (js/fetch - (construct-url (str (config/value :api-base-url) path) query) - (clj->js - {:method "POST" - :body form-data - :headers {"Authorization" (str "Apikey " (::api-key client)) - "User-Agent" user-agent - "Accept" "application/transit+json"}})) - (.then (fn [response] - (handle-response c response {:client client - :path path - :query query}))) - - ;; Unknown failure - (.catch (fn [error] (put! c (exception/exception :client/api-request-failed error))))) - c)) + (do-request client path "POST" query form-data nil)) (defn new-client [api-key] {::api-key api-key}) diff --git a/src/sharetribe/flex_cli/async_util.cljs b/src/sharetribe/flex_cli/async_util.cljs index fc34588..6521e34 100644 --- a/src/sharetribe/flex_cli/async_util.cljs +++ b/src/sharetribe/flex_cli/async_util.cljs @@ -25,3 +25,12 @@ (if (read-port? x) x (go x))) + +(defn ->chan [p] + (let [c (async/promise-chan)] + (-> p + (.then (fn [v] (if (some? v) + (async/put! c v) + (async/close! c)))) + (.catch (fn [err] (async/put! c err)))) + c)) diff --git a/src/sharetribe/flex_cli/commands/assets.cljs b/src/sharetribe/flex_cli/commands/assets.cljs index 083dfe4..0c347ad 100644 --- a/src/sharetribe/flex_cli/commands/assets.cljs +++ b/src/sharetribe/flex_cli/commands/assets.cljs @@ -1,6 +1,6 @@ (ns sharetribe.flex-cli.commands.assets "Commands for managing assets." - (:require [clojure.core.async :as async :refer [go res :meta :version) - (-> res :meta :aliased-version)) - local-assets (when prune (io-util/list-assets path)) + (:version new-asset-meta) + (:aliased-version new-asset-meta)) deleted-paths (when prune (set/difference (into #{} (map :path local-assets)) - (into #{} (map :path (:data res))))) + (into #{} (map :path (:assets new-asset-meta))))) updated? (not= old-version new-version)] (if (or updated? (seq deleted-paths)) (do - (when updated? - (io-util/write-assets path (:data res))) (when (seq deleted-paths) (io-util/remove-assets path deleted-paths)) (io-util/write-asset-meta path (assoc asset-meta :version new-version - :assets (into [] - (map #(dissoc % :data-raw)) - (:data res)))) + :assets (:assets new-asset-meta))) (io-util/ppd [:span "Version " new-version " successfully pulled."])) @@ -180,13 +242,12 @@ (let [query-params {:marketplace marketplace} body-params (to-multipart-form-data {:current-version (if version version "nil") ;; stringify nil as initial version - :assets (concat changed-assets delete-assets)}) - + :assets (concat changed-assets delete-assets)}) res (try ( res :data :version)] (if new-version diff --git a/src/sharetribe/flex_cli/io_util.cljs b/src/sharetribe/flex_cli/io_util.cljs index 0bb2d98..74f4a5d 100644 --- a/src/sharetribe/flex_cli/io_util.cljs +++ b/src/sharetribe/flex_cli/io_util.cljs @@ -8,7 +8,7 @@ [fipp.engine :as fipp] [clojure.edn :as edn] [clojure.string :as str] - [clojure.core.async :as async :refer [go ! go-loop]] #_[cljs-time.format :refer [formatter unparse]] #_[cljs-time.coerce :refer [to-date-time]] [chalk] @@ -16,6 +16,10 @@ [os] #_[sharetribe.util.money :as util.money] [sharetribe.flex-cli.exception :as exception] + [sharetribe.flex-cli.async-util :refer [chan]] + [yauzl :as yauzl] + ["fs" :as node-fs] + ["stream/promises" :refer [pipeline]] ["crypto" :as crypto] ["mkdirp" :rename {sync mkdirp-sync}] ["rimraf" :rename {sync rmrf-sync}])) @@ -68,16 +72,6 @@ (catch js/Error e (exception/throw! :io/write-failed {:path path}))))) -(defn save-file-binary - "Save the content to the given file path. If content is a Buffer, i.e. binary - data, does not do eny encoding." - ([path content] (save-file-binary path content nil)) - ([path content opts] - (try - (fs/writeFile path content opts) - (catch js/Error e - (exception/throw! :io/write-failed {:path path}))))) - (defn dir? "Check if the given path is a directory." [path] @@ -114,6 +108,74 @@ [& parts] (apply fs/path.join (remove nil? parts))) +(defn tmp-file-path [file] + (join (os/tmpdir) file)) + +(defn pipe-stream-to-file + "Takes `stream` and `path` and pipes the content of the stream to disk to the + given `path`. Uses `mkdirp` to create the parent directories of `path`. + Returns channel." + [stream path] + (mkdirp (dirname path)) + (->chan + (pipeline + stream + (node-fs/createWriteStream path)))) + +(defn unzip-entries + "Takes a `zipfile-path` and returns a channel. Each zip file entry is put to the + channel separately. The value put the the channel is a map of + `{:read-stream, :filename, :next}`. + + - `:read-stream` the stream of the entry content + - `:filename` the filename + - `:next` callback function which the consumer has to call after processing the entry. + + Errors are put to channel. + " + [zipfile-path] + (let [c (chan)] + (yauzl/open + zipfile-path + (clj->js {:lazyEntries true}) + (fn [err, ^js zipfile] + (when err (put! c err)) + ;; Skip directory check + (.readEntry zipfile) + (.on zipfile "entry" + (fn [entry] + (.openReadStream + zipfile + entry + (fn [err ^js read-stream] + (when err (put! c err)) + (put! c {:read-stream read-stream + :filename (.-fileName entry) + :next #(.readEntry zipfile)}))))) + (.on zipfile "end" (fn [] (close! c))))) + c)) + +(defn stream->string + "Takes a stream and reads it to memory as string which is put to a channel. In + case of error, the error object is put to channel." + [^js readable] + (let [c (chan) + chunks (array)] + (.on readable "data" (fn [chunk] (.push chunks chunk))) + (.on readable "end" + (fn [] + (put! c (.toString (js/Buffer.concat chunks) "utf8")))) + (.on readable "error" (fn [err] (put! c err))) + + c)) + +(defn parse-edn-stream + "Takes a ReadableStream of EDN and parses the EDN. Returns channel. In case of + error, the error is put in the channel." + [read-stream] + (go-try + (edn/read-string (string read-stream))))) + (defn process-file-path [path] (join path process-filename)) @@ -276,26 +338,6 @@ :content-hash (derive-content-hash data) :file-stream (streams/FileInputStream full-path)}))))) -(defn write-assets - [asset-dir-path assets] - (if-not (fs/dir? asset-dir-path) - nil - (doseq [{:keys [data-raw path type]} assets] - (let [file-path (join asset-dir-path path) - dir-path (dirname file-path)] - (mkdirp dir-path) - ;; TODO no EOL conversion in the CLI atm. Perhaps CLI should behave like - ;; git with core.autocrlf=true: convert unix2dos on pull, convert - ;; dos2unix on push - (case type - ;; For JSON assets, data-raw is a string. - :json (save-file file-path data-raw) - ;; Image asset data is served as a byte array in Transit, which gets - ;; turned into a JS Buffer automatically by the JS transit - ;; implementation. So, we are passing a Buffer to save-file-binary and - ;; it saves the file without attempting any string encoding. - :image (save-file-binary file-path data-raw)))))) - (defn kw->title "Create a title from a (unqualified) keyword by replacing dashes with spaces and capitalizing the first word." diff --git a/yarn.lock b/yarn.lock index 38fc558..8bfe3a7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -41,11 +41,6 @@ assert@^1.1.1: object.assign "^4.1.4" util "^0.10.4" -asynckit@^0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" - integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== - available-typed-arrays@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz#a5cc375d6a03c2efc87a553f3e0b1522def14846" @@ -149,6 +144,11 @@ browserify-zlib@^0.2.0: dependencies: pako "~1.0.5" +buffer-crc32@~0.2.3: + version "0.2.13" + resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" + integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ== + buffer-xor@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" @@ -240,13 +240,6 @@ color-name@1.1.3: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== -combined-stream@^1.0.8: - version "1.0.8" - resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" - integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== - dependencies: - delayed-stream "~1.0.0" - concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -344,11 +337,6 @@ define-properties@^1.2.1: has-property-descriptors "^1.0.0" object-keys "^1.1.1" -delayed-stream@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" - integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== - des.js@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/des.js/-/des.js-1.1.0.tgz#1d37f5766f3bbff4ee9638e871a8768c173b81da" @@ -410,16 +398,6 @@ es-object-atoms@^1.0.0, es-object-atoms@^1.1.1: dependencies: es-errors "^1.3.0" -es-set-tostringtag@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz#f31dbbe0c183b00a6d26eb6325c810c0fd18bd4d" - integrity sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA== - dependencies: - es-errors "^1.3.0" - get-intrinsic "^1.2.6" - has-tostringtag "^1.0.2" - hasown "^2.0.2" - escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" @@ -461,17 +439,6 @@ for-each@^0.3.5: dependencies: is-callable "^1.2.7" -form-data@^2.5.1: - version "2.5.3" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.3.tgz#f9bcf87418ce748513c0c3494bb48ec270c97acc" - integrity sha512-XHIrMD0NpDrNM/Ckf7XJiBbLl57KEhT3+i3yY+eWm+cqYZJQTZrKo8Y8AWKnuV5GT4scfuUGt9LzNoIx3dU1nQ== - dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.8" - es-set-tostringtag "^2.1.0" - mime-types "^2.1.35" - safe-buffer "^5.2.1" - fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -482,7 +449,7 @@ function-bind@^1.1.2: resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== -get-intrinsic@^1.2.4, get-intrinsic@^1.2.5, get-intrinsic@^1.2.6, get-intrinsic@^1.3.0: +get-intrinsic@^1.2.4, get-intrinsic@^1.2.5, get-intrinsic@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz#743f0e3b6964a93a5491ed1bffaae054d7f98d01" integrity sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ== @@ -713,18 +680,6 @@ miller-rabin@^4.0.0: bn.js "^4.0.0" brorand "^1.0.1" -mime-db@1.52.0: - version "1.52.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" - integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== - -mime-types@^2.1.35: - version "2.1.35" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" - integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== - dependencies: - mime-db "1.52.0" - mimic-fn@^1.0.0: version "1.2.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022" @@ -885,6 +840,11 @@ pbkdf2@^3.1.2: sha.js "^2.4.11" to-buffer "^1.2.0" +pend@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" + integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg== + possible-typed-array-names@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz#93e3582bc0e5426586d9d07b79ee40fc841de4ae" @@ -1304,12 +1264,15 @@ ws@^7.4.6: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9" integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ== -xmlhttprequest@^1.8.0: - version "1.8.0" - resolved "https://registry.yarnpkg.com/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz#67fe075c5c24fef39f9d65f5f7b7fe75171968fc" - integrity sha512-58Im/U0mlVBLM38NdZjHyhuMtCqa61469k2YP/AaPbvCoV9aQGUpbJBj1QRm2ytRiVQBD/fsw7L2bJGDVQswBA== - xtend@^4.0.0: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== + +yauzl@3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-3.2.0.tgz#7b6cb548f09a48a6177ea0be8ece48deb7da45c0" + integrity sha512-Ow9nuGZE+qp1u4JIPvg+uCiUr7xGQWdff7JQSk5VGYTAZMDe2q8lxJ10ygv10qmSj031Ty/6FNJpLO4o1Sgc+w== + dependencies: + buffer-crc32 "~0.2.3" + pend "~1.2.0"