Add mechanism for data migrations

src/computersandblues/lodestone/app.cljs

@@ -4,6 +4,7 @@
             [clojure.string :as str]
             [clojure.pprint :as pprint]
             [computersandblues.lodestone.database :as db]
+            [computersandblues.lodestone.interop :refer [promise-all promise-resolve promise-reject]]
             [computersandblues.lodestone.match :refer [query->matching-xform]]
             [applied-science.js-interop :as j]))
 
@@ -34,15 +35,6 @@
 
 ;; Mastodon API helpers
 
-(defn- promise-all [xs]
-  (js/Promise.all (apply array xs)))
-
-(defn- promise-resolve [val]
-  (js/Promise.resolve val))
-
-(defn- promise-reject [val]
-  (js/Promise.reject val))
-
 (defn- now []
   (js/Date.now))
 
@@ -560,26 +552,45 @@
 
 ;; database
 
-(def db-version 3)
+;;; we provide two different kinds of migrations
+;;;
+;;; - structural migrations
+;;; - data migrations
+;;;
+;;; this is necessary due to the constraints imposed on us by indexeddb.
+;;;
+;;; structural migrations run in a callback after opening the database. they run
+;;; synchronously and do things like creates stores and indices. because they run
+;;; synchronously, they cannot modify any of the stored data.
+;;;
+;;; data migrations can do that! they run after all structural migrations for
+;;; the current database version have succeeded, and work on the open database.
+;;;
+;;; note that for each db-version, there needs to be at least one structural or
+;;; data migration. it is fine to have both for a single version, in which case
+;;; the structural migration will run first.
 
-(def migrations
+(def db-version 4)
-  {1 (fn migration-0001 [db _]
+
+(def structural-migrations
+  {1 (fn =0001-create-stores [db _]
        (db/create-object-store! db ::db/application {:keyPath "id"})
        (db/create-object-store! db ::db/posts {:keyPath "id"}))
-   2 (fn migration-0002 [_ txn]
+   2 (fn =0002-add-post-created-at-idx [_ txn]
        (-> (db/open-store txn ::db/posts "readwrite")
            (db/create-index! ::db/post-created-at "created_at" {:unique false})))
-   3 (fn migration-0003 [_ txn]
+   3 (fn =0003-add-post-internal-id-idx [_ txn]
        (-> (db/open-store txn ::db/posts "readwrite")
            (db/create-index! ::db/post-internal-id "internal_id" {:unique false})))})
 
-(defn- convert-internal-ids! []
+(def data-migrations
-  ; TODO figure out when we can remove this again
+  {4 (fn =0004-convert-internal-ids [db]
-  (let [cursor (db/open-cursor ::db/posts ::db/all)
+       (let [cursor (db/open-cursor db ::db/posts ::db/post-internal-id ::db/all)
-        result (db/transduce-cursor (filter (comp string? (j/get :internal_id))) cursor)]
+             result (db/transduce-cursor (filter (comp string? (j/get :internal_id))) cursor)]
-    (.then result (fn [rows]
+         (.then result
-                    (doseq [row rows]
+                (fn [rows]
-                      (db/put! ::db/posts (j/update! row :internal_id parse-long)))))))
+                  (promise-all (for [row rows]
+                                 (db/put! ::db/posts (j/update! row :internal_id parse-long))))))))})
 
 ;; go go go
 
@@ -589,8 +600,7 @@
 (defn init []
   (swap! state assoc :root (rd/create-root (js/document.querySelector "#root")))
   (render)
-  (-> (db/setup! ::database db-version migrations)
+  (-> (db/setup! ::database db-version structural-migrations data-migrations)
       (.then #(setup-application!))
-      (.then #(convert-internal-ids!))
       (.catch (fn [err]
                 (js/console.error ::db/setup! err)))))

src/computersandblues/lodestone/database.cljs

@@ -1,11 +1,14 @@
 (ns computersandblues.lodestone.database
-  (:refer-clojure :exclude [count get]))
+  (:refer-clojure :exclude [count get])
+  (:require [computersandblues.lodestone.interop :refer [promise-seq]]))
 
 (defonce +db+ (atom nil))
 
-(defn setup! [namespace db-version migrations]
+(defn setup! [namespace db-version migrations data-migrations]
-  (assert (some? (migrations db-version)) "Will not increase db version as no migration is found")
+  (assert (some? (or (migrations db-version) (data-migrations db-version)))
-  (let [request (js/indexedDB.open (str namespace) db-version)]
+          "Will not increase db version as no matching migration is found")
+  (let [request (js/indexedDB.open (str namespace) db-version)
+        outstanding-data-migrations (volatile! [])]
     (js/Promise.
      (fn [resolve reject]
        (doto request
@@ -17,8 +20,10 @@
                                                    (fn []
                                                      (.. request -result close)
                                                      (js/alert "Database is outdated! Please reload the browser window.")))
-                                (reset! +db+ db)
+                                (. (promise-seq @outstanding-data-migrations)
-                                (resolve @+db+ ev))))
+                                   (then (fn [_]
+                                           (reset! +db+ db)
+                                           (resolve @+db+ ev)))))))
          (.addEventListener "upgradeneeded"
                             (fn [ev]
                               (let [db (.-result request)
@@ -26,9 +31,17 @@
                                     txn (-> ev .-target .-transaction)]
                                 (js/console.log ::upgradeneeded ev db)
                                 (doseq [version (range (inc old-version) (inc db-version))
-                                        :let [migration (migrations version)]]
+                                        :let [migration (migrations version)]
-                                  (migration db txn)))))
+                                        :when migration]
-                                        ; we don't add a listener for "blocked" events because we handle "versionchange" above
+                                  (migration db txn))
+                                (vreset! outstanding-data-migrations
+                                         (for [version (range (inc old-version) (inc db-version))
+                                               :let [migration (data-migrations version)]
+                                               :when migration]
+                                           ; we're not passing `txn` because it has already finished
+                                           ; by the time this migration is run
+                                           (migration db))))))
+         ; we don't add a listener for "blocked" events because we handle "versionchange" above
          (.addEventListener "error" (fn [ev] (reject (.-result request) ev))))))))
 
 (def transaction? (partial instance? js/IDBTransaction))

src/computersandblues/lodestone/interop.cljs

@@ -0,0 +1,39 @@
+(ns computersandblues.lodestone.interop)
+
+(defn promise-all [promises]
+  (js/Promise.all (into-array promises)))
+
+(reduce (fn [a b]
+          [a b]) [:foo])
+
+(defn promise-seq
+  "Takes a sequence of promises, and runs them all. This is similar to
+  `js/Promise.all`, but if any of the promises passed is wrapped in a function,
+  it will be executed. This allows ordered and delayed execution of a sequence
+  of promises. "
+  [promises]
+  (js/Promise. (fn [resolve _]
+                 (let [results (volatile! (transient []))]
+                   (cond
+                     (and (seq promises) (> (count promises) 2))
+                     (. (reduce (fn [current next]
+                                  (.then current
+                                         (fn [res]
+                                           (vswap! results conj! res)
+                                           (if (fn? next) (next) next))))
+                                promises)
+                        (then (fn [res]
+                                (vswap! results conj! res)
+                                (resolve (persistent! @results)))))
+
+                     (seq promises)
+                     (. (first promises)
+                        (then resolve))
+
+                     :else (resolve []))))))
+
+(defn promise-resolve [val]
+  (js/Promise.resolve val))
+
+(defn promise-reject [val]
+  (js/Promise.reject val))

src/computersandblues/lodestone/interop_test.cljs

@@ -0,0 +1,43 @@
+(ns computersandblues.lodestone.interop-test
+  (:require [computersandblues.lodestone.interop :as sut]
+            [cljs.test :refer-macros [deftest testing is async]]))
+
+(deftest promise-seq-empty
+  (testing "should invoke the callback when passed seq of promises is empty"
+    (async done (. (sut/promise-seq []) (then done)))))
+
+(deftest promise-seq-single-element
+  (testing "should invoke the callback when passed seq contains only a single element"
+    (async done (. (sut/promise-seq [(sut/promise-resolve true)]) (then done)))))
+
+(deftest promise-seq-order
+  (testing "should keep promise order"
+    (async done
+      (. (sut/promise-seq (map sut/promise-resolve (range 10)))
+         (then (fn [result]
+                 (is (= (vec (range 10)) result))
+                 (done)))))))
+
+(deftest promise-seq-lazyness
+  (testing "should invoke functions when passes as promise promise order"
+    (async done
+      (. (sut/promise-seq [(sut/promise-resolve :a)
+                           (fn [] :b)
+                           (fn [] (sut/promise-resolve :c))])
+         (then (fn [result]
+                 (is (= [:a :b :c] result))
+                 (done)))))))
+
+(deftest promise-seq-in-sequence
+  (testing "should invoke all promises in sequence"
+    (async done
+      ; we create a range of promises that all resolve within the same timeout,
+      ; but because we're wrapping them in a function they will only get created
+      ; once the sequence has processed far enough
+      (. (sut/promise-seq (map (fn [_]
+                                 (js/Promise. (fn [resolve _]
+                                                (js/setTimeout #(resolve (js/Date.now)) 10))))
+                               (range 5)))
+         (then (fn [result]
+                 (is (true? (reduce < result)))
+                 (done)))))))