Fix "fetch more" button

This sent a lot of unnecessary requests, because the `min_id` / `max_id`
parameters that were fetched from the database were way off. No idea why
exactly that happened, but aggregating the result manually seems to work
fine.

src/computersandblues/lodestone/app.cljs

@@ -27,7 +27,6 @@
                         ; TODO: Handle other lists
                         :section/posts posts-init-state}))
 
-; TODO Ensure that cached data is up to date
 ; TODO Handle 429
 ; TODO Search for tags (`#foo`) and handles (`@bar`)
 ; TODO Explain which kind of search currently is possible
@@ -156,7 +155,7 @@
 (declare debounced-refresh!)
 
 (defn- fetch-application-settings []
-  (->> (db/open-cursor! ::db/application db/all)
+  (->> (db/open-cursor! ::db/application ::db/all)
        (db/first-result (map #(js->clj % :keywordize-keys true)))))
 
 (defn setup-application!
@@ -253,7 +252,8 @@
        (swap! state update-in [:section/posts :loading] conj req-id)
        (-> (mastodon-request! {:url url :bearer-token bearer-token})
            (.then (fn [response]
-                    (let [next-url (link-header "next" (:raw response))]
+                    (let [url-params (url->search-params (.-url (:raw response)))
+                          next-url (link-header (if (.get url-params "min_id") "prev" "next") (:raw response))]
                       (swap! state update-in [:section/posts :loading] disj req-id)
                       (on-response response)
                       (when (and (continue? response) next-url)
@@ -381,7 +381,9 @@
         refresh-id (js/Date.now)]
     (swap! state update-in [:section/posts :loading] conj refresh-id)
     (. (promise-all [(db/count! ::db/posts)
-                     (->> (db/open-cursor! ::db/posts ::db/post-created-at db/all "prev")
+                     (->> (db/open-cursor! ::db/posts ::db/all
+                                           {:index ::db/post-created-at
+                                            :direction :desc})
                           (db/transduce-cursor xform))])
        (then (fn [[total displayed-posts]]
                (swap! state update :section/posts #(-> (assoc % :total total)
@@ -395,6 +397,7 @@
                   :on-response (fn [response]
                                  (let [url-params (url->search-params (.-url (:raw response)))]
                                    (doseq [post (:body response)]
+                                     ; this returns a promise, but we don't care if these updates happen in sequence
                                      (db/put! ::db/posts (cond-> post
                                                            ; these IDs are internal server ids and it looks like
                                                            ; they are not returned in any response; they are
@@ -403,28 +406,41 @@
                                                            ; if outer circumstances decide so (for example if the
                                                            ; tab is closed)
                                                            (.get url-params "max_id")
-                                                           (assoc :internal_id (.get url-params "max_id"))
+                                                           (assoc :internal_id (parse-long (.get url-params "max_id")))
 
                                                            (.get url-params "min_id")
-                                                           (assoc :internal_id (.get url-params "min_id"))))))
+                                                           (assoc :internal_id (parse-long (.get url-params "min_id")))))))
                                  (debounced-refresh! (:section/posts @state)))}]
     (paginate-posts! (merge defaults opts))))
 
-(defn- internal-post-id [max-or-min]
+(defn- internal-post-id
-  (->> (db/open-cursor! ::db/posts ::db/post-created-at db/all (if (= max-or-min :min)
+  "Returns a promise which resolves to the smallest or largest internal post id.
-                                                                 "next"
+  This is useful to continue interrupted paginated requests."
-                                                                 "prev"))
+  [max-or-min]
-       (db/first-result (keep (j/get :internal_id)))))
+  ; this may look like a horribly inefficient way to get these numbers, and it is!
+  ; there was an earlier version that simply used an index on :internal_id and
+  ; retrieved the first result in ascending or descending order; however, the
+  ; result that was returned was unexpected and unreliable. this may very well
+  ; have been my fault, because i'm only learning how the indexeddb api works as
+  ; i implement all of this here.
+  (let [xform (comp (keep (j/get :internal_id)))
+        rf (if (= max-or-min :min) min max)
+        init (if (= max-or-min :min) js/Number.POSITIVE_INFINITY js/Number.NEGATIVE_INFINITY)]
+    (. (->> (db/open-cursor! ::db/posts ::db/all)
+            (db/transduce-cursor xform rf init))
+       (then (fn [result]
+               (when-not (infinite? result)
+                 result))))))
 
 (defn fetch-more-posts! [e]
   (.preventDefault e)
   (. (promise-all [(fetch-application-settings) (internal-post-id :min) (internal-post-id :max)])
      (then (fn [[application min-id max-id]]
-             (when min-id
+             (when max-id
                (fetch-posts! {:instance-url (:instance_url application)
                               :bearer-token (:bearer_token application)
                               :min-id max-id}))
-             (when max-id
+             (when min-id
                (fetch-posts! {:instance-url (:instance_url application)
                               :bearer-token (:bearer_token application)
                               :max-id min-id}))
@@ -513,6 +529,14 @@
        (-> (db/open-store txn ::db/posts "readwrite")
            (db/create-index! ::db/post-internal-id "internal_id" {:unique false})))})
 
+(defn- convert-internal-ids! []
+  ; TODO figure out when we can remove this again
+  (. (->> (db/open-cursor! ::db/posts ::db/all)
+          (db/transduce-cursor (filter (comp string? (j/get :internal_id)))))
+     (then (fn [rows]
+              (doseq [row rows]
+                (db/put! ::db/posts (j/update! row :internal_id parse-long)))))))
+
 ;; go go go
 
 (defn ^:dev/after-load render []
@@ -523,5 +547,6 @@
   (render)
   (-> (db/setup! ::database db-version migrations)
       (.then #(setup-application!))
+      (.then #(convert-internal-ids!))
       (.catch (fn [err]
-                (js/console.log ::db/setup! err)))))
+                (js/console.error ::db/setup! err)))))

src/computersandblues/lodestone/database.cljs

@@ -30,14 +30,17 @@
                                         ; we don't add a listener for "blocked" events because we handle "versionchange" above
          (.addEventListener "error" (fn [ev] (reject (.-result request) ev))))))))
 
+(def transaction? (partial instance? js/IDBTransaction))
+(def store? (partial instance? js/IDBObjectStore))
+
 (defn open-store
-  ([db-or-txn store-id]
+  ([store-id permissions]
-   (open-store db-or-txn store-id "readonly"))
+   (open-store @+db+ store-id permissions))
-  ([db-or-txn store-id permissions]
+  ([db store-id permissions]
    (let [store-id (str store-id) ; simplifies using keywords as store identifiers
-         txn (if (instance? js/IDBTransaction db-or-txn)
+         txn (if (transaction? db)
-               db-or-txn
+               db
-               (.transaction db-or-txn store-id permissions))]
+               (.transaction db store-id permissions))]
      (.objectStore txn store-id))))
 
 (defn create-object-store! [db store-id key-opts]
@@ -55,39 +58,69 @@
                    (.addEventListener "success" (fn [] (resolve (.-result request))))
                    (.addEventListener "error" (fn [] (reject (.-error request))))))))
 
-(defn add! [store-id object]
+(defn add!
-  (let [store (open-store @+db+ store-id "readwrite")
+  ([store-id object] (add! @+db+ store-id object))
-        request (.add store (clj->js object))]
+  ([db store-id object]
-    (promisify request)))
+   (let [store (open-store db store-id "readwrite")
+         request (.add store (clj->js object))]
+     (promisify request))))
 
-(defn put! [store-id object]
+(defn put!
-  (let [store (open-store @+db+ store-id "readwrite")
+  ([store-id object] (put! @+db+ store-id object))
-        request (.put store (clj->js object))]
+  ([db store-id object]
-    (promisify request)))
+   (let [store (open-store db store-id "readwrite")
+         request (.put store (clj->js object))]
+     (promisify request))))
 
-(defn get! [store-id k]
+(defn get!
-  (let [store (open-store @+db+ store-id)
+  ([store-id k] (get! @+db+ store-id k))
-        request (.get store k)]
+  ([db store-id k]
-    (promisify request)))
+   (let [store (open-store db store-id "readonly")
+         request (.get store k)]
+     (promisify request))))
 
-(defn get-all! [store-id key-range]
+(defn get-all!
-  (let [store (open-store @+db+ store-id)
+  ([store-id key-range] (get-all! @+db+ store-id key-range))
-        request (.getAll store key-range)]
+  ([db store-id key-range]
-    (promisify request)))
+   (let [store (open-store db store-id "readonly")
+         request (.getAll store key-range)]
+     (promisify request))))
+
+(defn clear!
+  ([store-id] (clear! @+db+ store-id))
+  ([db store-id]
+   (let [store (open-store db store-id "readwrite")
+         request (.clear store)]
+     (promisify request))))
+
+(defn delete!
+  ([store-id k] (get! @+db+ store-id k))
+  ([db store-id k]
+   (let [store (open-store db store-id "readwrite")
+         request (.delete store k)]
+     (promisify request))))
+
+(defn count!
+  ([store-id] (count! @+db+ store-id))
+  ([db store-id]
+   (let [store (open-store db store-id "readonly")
+         request (.count store)]
+     (promisify request))))
 
-(defn clear! [store-id]
-  (let [store (open-store @+db+ store-id "readwrite")
-        request (.clear store)]
-    (promisify request)))
 
 (defn open-cursor!
-  ([store-id key-range] (open-cursor! store-id key-range "next"))
+  ([store-id key-range] (open-cursor! @+db+ store-id key-range {:direction :asc}))
-  ([store-id key-range direction]
+  ([store-id key-range opts] (open-cursor! @+db+ store-id key-range opts))
-   (let [store (open-store @+db+ store-id)]
+  ([db store-id key-range {:keys [direction index]}]
-     (.openCursor store key-range direction)))
+   (let [store (open-store db store-id "readonly")
-  ([store-id idx-name key-range direction]
+         key-range (if (= key-range ::all)
-   (let [store (open-store @+db+ store-id)]
+                     (js/IDBKeyRange.lowerBound "")
-     (.openCursor (.index store (str idx-name)) key-range direction))))
+                     key-range)
+         direction ({:asc "next" :desc "prev"} direction direction)]
+     (if index
+       (.openCursor (.index store (str index)) key-range direction)
+       (.openCursor store key-range direction)))))
+
 
 #_(defn logging [f]
   (let [n (volatile! 0)]
@@ -130,13 +163,11 @@
 (defn first-result [xform cursor-req]
   (transduce-cursor (comp xform (take 1)) (fn [_ x] x) cursor-req))
 
-(def all (js/IDBKeyRange.lowerBound ""))
-
 (comment
 
   (let [re (js/RegExp. "user" "i")]
     (do (print "starting…" (js/Date.))
-        (-> (open-cursor! ::posts all)
+        (-> (open-cursor! ::posts ::all)
             (transduce-cursor (comp (filter #(re-find re (.-content %)))
                                     (take 50)
                                     (map #(js->clj % :keywordize-keys true))))
@@ -145,13 +176,3 @@
                      (js/console.log (first result)))))))
 
   )
-
-(defn delete! [store-id k]
-  (let [store (open-store @+db+ store-id)
-        request (.delete store k)]
-    (promisify request)))
-
-(defn count! [store-id]
-  (let [store (open-store @+db+ store-id)
-        request (.count store)]
-    (promisify request)))