diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5f91b140..d90af28c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -61,10 +61,7 @@ jobs: with: ssh-private-key: ${{ secrets.NEXTJOURNAL_CI_SSH_KEY }} - - name: Build Notebooks - run: clojure -X:nextjournal/devdocs - - - name: Compile ClojureScript (snapshot devcards/stories build) + - name: Compile ClojureScript (snapshot build) env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: bb release:cljs @@ -72,12 +69,14 @@ jobs: - name: Compile CSS run: bb build:css - - name: Build static devcards - run: ./bin/build_static_snapshot + - name: Build Notebooks + run: bb build:devdocs + + - name: Copy Asseets + run: bb build:copy-assets - - name: Copy build to bucket under SHA - run: | - gsutil cp -r build gs://nextjournal-snapshots/viewers/build/${{ github.sha }} + - name: Store in CAS + run: echo "cas_url=$(bb devdocs:cas)" >> $GITHUB_ENV - name: Add status flag to the devcards build uses: Sibz/github-status-action@v1 @@ -87,4 +86,4 @@ jobs: description: 'Ready' state: 'success' sha: ${{github.event.pull_request.head.sha || github.sha}} - target_url: https://storage.googleapis.com/nextjournal-snapshots/viewers/build/${{ github.sha }}/index.html + target_url: "${{ env.cas_url }}" diff --git a/.gitignore b/.gitignore index 8df568b1..5640c402 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ node_modules # VSCode/Calva .calva/ /build/ +public/*.edn diff --git a/bb.edn b/bb.edn index 2744da1f..10d5119a 100644 --- a/bb.edn +++ b/bb.edn @@ -1,6 +1,8 @@ -{:tasks - {:init (def tailwind-build-cmd "npx tailwindcss --input resources/css/viewer.css --output public/css/viewer.css") - +{:deps {io.github.nextjournal/cas-client {:git/sha "e34901f4a29d54954544313ae1a0bb859fe3931f"}} + :tasks + {:init (do (def tailwind-build-cmd "npx tailwindcss --input resources/css/viewer.css --output public/css/viewer.css") + (def devdocs-build-path "build/devdocs")) + :requires ([babashka.fs :as fs] [nextjournal.cas-client :as cas]) yarn-install {:doc "Installs and updates npm dependencies" :task (shell "yarn install")} @@ -16,7 +18,23 @@ :task (shell tailwind-build-cmd) :depends [yarn-install]} - -dev {:depends [watch:cljs watch:css]} + build:copy-assets {:doc "Moves assets into devdocs out path prior to uploading them" + :task (do + (fs/create-dirs (str devdocs-build-path "/js")) + (fs/create-dirs (str devdocs-build-path "/css")) + (fs/copy "public/index.html" (str devdocs-build-path "/index.html") {:replace-existing true}) + (fs/copy "public/js/viewer.js" (str devdocs-build-path "/js/") {:replace-existing true}) + (fs/copy "public/css/viewer.css" (str devdocs-build-path "/css/") {:replace-existing true}))} + + build:devdocs {:doc "Build Devdocs" :task (clojure "-X:nextjournal/devdocs")} + + build:devdocs:dev {:doc "Build Devdocs to be served by shadow dev http server" + :task (clojure "-X:nextjournal/devdocs :out-path '\"public\"'")} + + devdocs:cas {:doc "Store devdocs into CAS" + :task (print (get-in (cas/put {:path devdocs-build-path}) ["manifest" "index.html"]))} + + -dev {:depends [watch:cljs watch:css build:devdocs:dev]} dev {:doc "Start app in dev mode, compiles cljs and css, starts an nrepl " :task (run '-dev {:parallel true})} diff --git a/examples/examples.cljs b/examples/examples.cljs index f0ec2d91..abf9e8c0 100644 --- a/examples/examples.cljs +++ b/examples/examples.cljs @@ -2,6 +2,7 @@ (:require ["react-dom/client" :as react-client] [clojure.string :as str] [nextjournal.devcards.routes :as devcards.routes] + [nextjournal.devdocs :as devdocs] [nextjournal.devdocs.demo :as devdocs.demo] [nextjournal.devcards-ui :as devcards-ui] [nextjournal.clerk-sci-env] ;; sets up clerk sci env extensions @@ -83,6 +84,8 @@ (react-client/createRoot el))) (defn ^:export ^:dev/after-load init [] - (re-frame/dispatch-sync [:init-commands (commands.state/empty-db!)]) - (rfe/start! router #(reset! match %1) {:use-fragment true}) - (.render react-root (reagent/as-element [view]))) + (.then (devdocs/init!) + (fn [_] + (re-frame/dispatch-sync [:init-commands (commands.state/empty-db!)]) + (rfe/start! router #(reset! match %1) {:use-fragment true}) + (.render react-root (reagent/as-element [view]))))) diff --git a/examples/nextjournal/devdocs/demo.cljs b/examples/nextjournal/devdocs/demo.cljs index ba19c0bc..4dae35f2 100644 --- a/examples/nextjournal/devdocs/demo.cljs +++ b/examples/nextjournal/devdocs/demo.cljs @@ -4,13 +4,6 @@ [reitit.frontend :as rf]) (:require-macros [nextjournal.devdocs :as devdocs])) -(reset! devdocs/registry (assoc (devdocs/build-registry {:paths ["README.md" - "docs/**.{clj,md}"]}) - :navbar/theme - {:back "text-[12px] text-slate-300 hover:bg-white/10 font-normal px-5 py-1" - :icon "text-slate-400"})) - - (defonce router (rf/router ["/docs" ["/*path" {:name :devdocs/show}]])) (defn view [match] diff --git a/modules/devdocs/deps.edn b/modules/devdocs/deps.edn index c72b4583..326c3126 100644 --- a/modules/devdocs/deps.edn +++ b/modules/devdocs/deps.edn @@ -4,7 +4,6 @@ io.github.nextjournal/clerk.render {:git/url "https://github.com/nextjournal/clerk" :git/sha "711a1b2fae3c212d2c8c2323cf4d53c178766114" :deps/root "render"} - io.github.nextjournal/cas-client {:git/sha "84ab35c3321c1e51a589fddbeee058aecd055bf8"} re-frame/re-frame {:git/url "https://github.com/nextjournal/freerange" :git/sha "8cf68c30722a4c6f8f948a134c900d7a656ecad4"} com.lambdaisland/deja-fu {:mvn/version "0.3.33"}}} diff --git a/modules/devdocs/src/nextjournal/devdocs.clj b/modules/devdocs/src/nextjournal/devdocs.clj index 6c940224..37665095 100644 --- a/modules/devdocs/src/nextjournal/devdocs.clj +++ b/modules/devdocs/src/nextjournal/devdocs.clj @@ -5,7 +5,6 @@ [clojure.java.shell :as sh] [clojure.stacktrace :as stacktrace] [clojure.string :as str] - [nextjournal.cas-client :as cas] [nextjournal.clerk :as clerk] [nextjournal.clerk.builder :as builder] [nextjournal.clerk.eval :as eval] @@ -15,13 +14,11 @@ (def build-path "build/devdocs") -(defn doc-path->edn-path [path] - (str build-path "/" (str/replace (str path) fs/file-separator "|") ".edn")) +(defn doc-path->edn-path [path] (str (str/replace (str path) fs/file-separator "|") ".edn")) #_(doc-path->edn-path "docs/clerk/clerk.md") #_(doc-path->edn-path "docs/clerk/clerk.clj") #_(doc-path->edn-path "manifest") -#_(str (fs/relativize build-path (doc-path->edn-path "docs/clerk/clerk.clj"))) (defn path->title [path] (-> path fs/file-name (str/replace #"\.(clj(c?)|md)$" "") (str/split #"_") @@ -56,18 +53,14 @@ (defn guard [p? val] (when (p? val) val)) (defn assoc-when-missing [m k v] (cond-> m (not (contains? m k)) (assoc k v))) -(defn file->doc-info [{:keys [cas-manifest]} path] +(defn file->doc-info [path] (-> (parser/parse-file {:doc? true} (fs/file path)) (select-keys [:title :doc]) (assoc-when-missing :title (path->title path)) (assoc :path (str path) + :edn-url (doc-path->edn-path path) :last-modified (when-some [ts (-> (sh/sh "git" "log" "-1" "--format=%ct" (str path)) :out str/trim not-empty)] - (* (Long/parseLong ts) 1000))) - (as-> info - (if-some [cas-url (get cas-manifest (str (fs/relativize build-path (doc-path->edn-path path))))] - (assoc info :edn-cas-url cas-url) - (do (println (str "No EDN data url found for notebook: '" path "'. Falling back to notebook viewer data with no cell results.")) - (assoc info :edn-doc (doc-viewer-edn {:path path}))))))) + (* (Long/parseLong ts) 1000))))) #_(file->doc-info "docs/clerk/clerk.clj") #_(file->doc-info "docs/clerk/missing_title.clj") @@ -106,53 +99,50 @@ "README.md" "modules/devdocs/src/nextjournal/devdocs.clj"]) -(defmacro build-registry +(defn build-registry "Populates a nested registry of parsed notebooks given a set of paths." [{:keys [paths]}] - (let [cas-manifest (when (fs/exists? (doc-path->edn-path "manifest")) - (clojure.edn/read-string (slurp (doc-path->edn-path "manifest"))))] - (->> paths - expand-paths - (map (partial file->doc-info {:cas-manifest cas-manifest})) - (group-by (comp str fs/parent :path)) - (sort-by first) - (reduce add-collection {})))) - -#_(letfn [(strip-edn [coll] (-> coll - (update :devdocs (partial into [] (map #(dissoc % :edn-doc)))) - (update :collections (partial into [] (map strip-edn)))))] - (strip-edn - (build-registry {:paths ["docs/**/*.{clj,md}" - "README.md" - "modules/devdocs/src/nextjournal/devdocs.clj"]}))) - -(defn write-edn-results [_opts docs] + (->> paths + expand-paths + (map file->doc-info) + (group-by (comp str fs/parent :path)) + (sort-by first) + (reduce add-collection {}))) + +(defn write-edn-results [{:keys [out-path]} docs] (doseq [{:as _doc :keys [viewer file]} docs] - (let [edn-path (doc-path->edn-path file)] + (let [edn-path (str (fs/path out-path (doc-path->edn-path file)))] (when-not (fs/exists? (fs/parent edn-path)) (fs/create-dirs (fs/parent edn-path))) (spit edn-path (viewer/->edn viewer))))) (defn build! "Expand paths and evals resulting notebooks with clerk. Persists EDN results to fs at conventional path (see `doc-path->cached-edn-path`)." - [{:as opts :keys [paths]}] + [{:as opts :keys [paths out-path] :navbar/keys [theme] :or {out-path build-path}}] + ;; build Clerk static app (in EDN format) one file per notebook (with-redefs [builder/write-static-app! write-edn-results] - (builder/build-static-app! {:paths (expand-paths paths)})) - ;; store in CAS - (let [{:as response :strs [manifest]} (cas/put (-> opts (select-keys [:cas-host]) (assoc :path build-path)))] - (when-not manifest - (throw (ex-info "Unable to upload to CAS" {:response response}))) - (spit (doc-path->edn-path "manifest") (pr-str manifest)))) + (builder/build-static-app! {:paths (expand-paths paths) :out-path out-path})) + ;; Assemble a registry respecting fs hierarchy + (println "🗂 Building devdocs registry...") + (spit (fs/file out-path "registry.edn") + (pr-str (cond-> (build-registry {:paths paths}) + theme (assoc :navbar/theme theme)))) + (println "done.")) (comment (shadow.cljs.devtools.api/repl :browser) (fs/delete-tree "build/devdocs") (fs/list-dir "build/devdocs") - (cas/put {:path "build/devdocs"}) + ;; get manifest - (clojure.edn/read-string (slurp (doc-path->edn-path "manifest"))) + (clojure.edn/read-string (slurp (str build-path "/registry.edn"))) (clerk/clear-cache!) + ;; build devdocs for results to appear in notebooks - (build! {:paths ["docs/**.{clj,md}"]}) + (build! {:paths ["docs/**.{clj,md}"] + ;; #_#_ + :out-path "public" + :navbar/theme {:back "text-[12px] text-slate-300 hover:bg-white/10 font-normal px-5 py-1" + :icon "text-slate-400"}}) ;; registry (build-registry {:paths ["docs/**.{clj,md}"]})) diff --git a/modules/devdocs/src/nextjournal/devdocs.cljs b/modules/devdocs/src/nextjournal/devdocs.cljs index 97c6666a..ecfbfb8c 100644 --- a/modules/devdocs/src/nextjournal/devdocs.cljs +++ b/modules/devdocs/src/nextjournal/devdocs.cljs @@ -21,6 +21,13 @@ (defonce registry (reagent/atom [])) +(defn init! + ([] (init! {})) + ([{:keys [registry-url] :or {registry-url "registry.edn"}}] + (.. (js/fetch registry-url) + (then #(.text %)) + (then #(reset! registry (render/read-string %)))))) + ;; TODO: maybe compile into reitit router (defn find-coll [reg path] (some #(and (str/starts-with? path (:path %)) %) (:items reg))) (defn find-doc [{:keys [items]} path] (some #(and (= path (:path %)) %) items)) @@ -42,11 +49,11 @@ (declare collection-inner-view) -(defn item-view [{:as item :keys [title edn-cas-url edn-doc path last-modified items]}] +(defn item-view [{:as item :keys [title edn-url path last-modified items]}] [:div (cond ;; doc - (or edn-cas-url edn-doc) + edn-url [:div.mb-2 [:a.hover:underline.font-bold {:href (rfe/href :devdocs/show {:path path}) :title path} @@ -75,11 +82,8 @@ [:h1.pt-8 "Devdocs"] [collection-inner-view collection]]]) -(defn devdoc-view [{:as doc :keys [edn-doc edn-cas-url fragment]}] - (let [edn (render.hooks/use-promise - (if edn-cas-url - (.then (js/fetch edn-cas-url) #(.text %)) - (js/Promise.resolve edn-doc)))] +(defn devdoc-view [{:as doc :keys [edn-url fragment]}] + (let [edn (render.hooks/use-promise (.then (js/fetch edn-url) #(.text %)))] [:div.overflow-y-auto.bg-white.dark:bg-gray-900.flex-auto.relative.font-sans (cond-> {:style {:padding-top 45 :padding-bottom 70}} fragment (assoc :ref #(scroll-to-fragment fragment))) @@ -121,8 +125,8 @@ [navbar/panel !state [navbar/navbar !state]] (if (or (nil? path) (contains? #{"" "/"} path)) [collection-view @registry] - (let [{:as node :keys [edn-doc edn-cas-url]} (lookup @registry path)] - (when (or edn-cas-url edn-doc) + (let [{:as node :keys [edn-url]} (lookup @registry path)] + (when edn-url ^{:key path} [devdoc-view node])))])) (defn devdoc-commands