diff --git a/src/computersandblues/lodestone/app.cljs b/src/computersandblues/lodestone/app.cljs index a619735..4a8e271 100644 --- a/src/computersandblues/lodestone/app.cljs +++ b/src/computersandblues/lodestone/app.cljs @@ -27,7 +27,6 @@ ; TODO: Handle other lists :section/posts posts-init-state})) -; TODO Ensure that cached data is up to date ; TODO Handle 429 ; TODO Search for tags (`#foo`) and handles (`@bar`) ; TODO Explain which kind of search currently is possible @@ -156,7 +155,7 @@ (declare debounced-refresh!) (defn- fetch-application-settings [] - (->> (db/open-cursor! ::db/application db/all) + (->> (db/open-cursor! ::db/application ::db/all) (db/first-result (map #(js->clj % :keywordize-keys true))))) (defn setup-application! @@ -253,7 +252,8 @@ (swap! state update-in [:section/posts :loading] conj req-id) (-> (mastodon-request! {:url url :bearer-token bearer-token}) (.then (fn [response] - (let [next-url (link-header "next" (:raw response))] + (let [url-params (url->search-params (.-url (:raw response))) + next-url (link-header (if (.get url-params "min_id") "prev" "next") (:raw response))] (swap! state update-in [:section/posts :loading] disj req-id) (on-response response) (when (and (continue? response) next-url) @@ -381,7 +381,9 @@ refresh-id (js/Date.now)] (swap! state update-in [:section/posts :loading] conj refresh-id) (. (promise-all [(db/count! ::db/posts) - (->> (db/open-cursor! ::db/posts ::db/post-created-at db/all "prev") + (->> (db/open-cursor! ::db/posts ::db/all + {:index ::db/post-created-at + :direction :desc}) (db/transduce-cursor xform))]) (then (fn [[total displayed-posts]] (swap! state update :section/posts #(-> (assoc % :total total) @@ -395,6 +397,7 @@ :on-response (fn [response] (let [url-params (url->search-params (.-url (:raw response)))] (doseq [post (:body response)] + ; this returns a promise, but we don't care if these updates happen in sequence (db/put! ::db/posts (cond-> post ; these IDs are internal server ids and it looks like ; they are not returned in any response; they are @@ -403,28 +406,41 @@ ; if outer circumstances decide so (for example if the ; tab is closed) (.get url-params "max_id") - (assoc :internal_id (.get url-params "max_id")) + (assoc :internal_id (parse-long (.get url-params "max_id"))) (.get url-params "min_id") - (assoc :internal_id (.get url-params "min_id")))))) + (assoc :internal_id (parse-long (.get url-params "min_id"))))))) (debounced-refresh! (:section/posts @state)))}] (paginate-posts! (merge defaults opts)))) -(defn- internal-post-id [max-or-min] - (->> (db/open-cursor! ::db/posts ::db/post-created-at db/all (if (= max-or-min :min) - "next" - "prev")) - (db/first-result (keep (j/get :internal_id))))) +(defn- internal-post-id + "Returns a promise which resolves to the smallest or largest internal post id. + This is useful to continue interrupted paginated requests." + [max-or-min] + ; this may look like a horribly inefficient way to get these numbers, and it is! + ; there was an earlier version that simply used an index on :internal_id and + ; retrieved the first result in ascending or descending order; however, the + ; result that was returned was unexpected and unreliable. this may very well + ; have been my fault, because i'm only learning how the indexeddb api works as + ; i implement all of this here. + (let [xform (comp (keep (j/get :internal_id))) + rf (if (= max-or-min :min) min max) + init (if (= max-or-min :min) js/Number.POSITIVE_INFINITY js/Number.NEGATIVE_INFINITY)] + (. (->> (db/open-cursor! ::db/posts ::db/all) + (db/transduce-cursor xform rf init)) + (then (fn [result] + (when-not (infinite? result) + result)))))) (defn fetch-more-posts! [e] (.preventDefault e) (. (promise-all [(fetch-application-settings) (internal-post-id :min) (internal-post-id :max)]) (then (fn [[application min-id max-id]] - (when min-id + (when max-id (fetch-posts! {:instance-url (:instance_url application) :bearer-token (:bearer_token application) :min-id max-id})) - (when max-id + (when min-id (fetch-posts! {:instance-url (:instance_url application) :bearer-token (:bearer_token application) :max-id min-id})) @@ -513,6 +529,14 @@ (-> (db/open-store txn ::db/posts "readwrite") (db/create-index! ::db/post-internal-id "internal_id" {:unique false})))}) +(defn- convert-internal-ids! [] + ; TODO figure out when we can remove this again + (. (->> (db/open-cursor! ::db/posts ::db/all) + (db/transduce-cursor (filter (comp string? (j/get :internal_id))))) + (then (fn [rows] + (doseq [row rows] + (db/put! ::db/posts (j/update! row :internal_id parse-long))))))) + ;; go go go (defn ^:dev/after-load render [] @@ -523,5 +547,6 @@ (render) (-> (db/setup! ::database db-version migrations) (.then #(setup-application!)) + (.then #(convert-internal-ids!)) (.catch (fn [err] - (js/console.log ::db/setup! err))))) + (js/console.error ::db/setup! err))))) diff --git a/src/computersandblues/lodestone/database.cljs b/src/computersandblues/lodestone/database.cljs index 119bc6a..9930386 100644 --- a/src/computersandblues/lodestone/database.cljs +++ b/src/computersandblues/lodestone/database.cljs @@ -30,14 +30,17 @@ ; we don't add a listener for "blocked" events because we handle "versionchange" above (.addEventListener "error" (fn [ev] (reject (.-result request) ev)))))))) +(def transaction? (partial instance? js/IDBTransaction)) +(def store? (partial instance? js/IDBObjectStore)) + (defn open-store - ([db-or-txn store-id] - (open-store db-or-txn store-id "readonly")) - ([db-or-txn store-id permissions] + ([store-id permissions] + (open-store @+db+ store-id permissions)) + ([db store-id permissions] (let [store-id (str store-id) ; simplifies using keywords as store identifiers - txn (if (instance? js/IDBTransaction db-or-txn) - db-or-txn - (.transaction db-or-txn store-id permissions))] + txn (if (transaction? db) + db + (.transaction db store-id permissions))] (.objectStore txn store-id)))) (defn create-object-store! [db store-id key-opts] @@ -55,39 +58,69 @@ (.addEventListener "success" (fn [] (resolve (.-result request)))) (.addEventListener "error" (fn [] (reject (.-error request)))))))) -(defn add! [store-id object] - (let [store (open-store @+db+ store-id "readwrite") - request (.add store (clj->js object))] - (promisify request))) +(defn add! + ([store-id object] (add! @+db+ store-id object)) + ([db store-id object] + (let [store (open-store db store-id "readwrite") + request (.add store (clj->js object))] + (promisify request)))) -(defn put! [store-id object] - (let [store (open-store @+db+ store-id "readwrite") - request (.put store (clj->js object))] - (promisify request))) +(defn put! + ([store-id object] (put! @+db+ store-id object)) + ([db store-id object] + (let [store (open-store db store-id "readwrite") + request (.put store (clj->js object))] + (promisify request)))) -(defn get! [store-id k] - (let [store (open-store @+db+ store-id) - request (.get store k)] - (promisify request))) +(defn get! + ([store-id k] (get! @+db+ store-id k)) + ([db store-id k] + (let [store (open-store db store-id "readonly") + request (.get store k)] + (promisify request)))) -(defn get-all! [store-id key-range] - (let [store (open-store @+db+ store-id) - request (.getAll store key-range)] - (promisify request))) +(defn get-all! + ([store-id key-range] (get-all! @+db+ store-id key-range)) + ([db store-id key-range] + (let [store (open-store db store-id "readonly") + request (.getAll store key-range)] + (promisify request)))) + +(defn clear! + ([store-id] (clear! @+db+ store-id)) + ([db store-id] + (let [store (open-store db store-id "readwrite") + request (.clear store)] + (promisify request)))) + +(defn delete! + ([store-id k] (get! @+db+ store-id k)) + ([db store-id k] + (let [store (open-store db store-id "readwrite") + request (.delete store k)] + (promisify request)))) + +(defn count! + ([store-id] (count! @+db+ store-id)) + ([db store-id] + (let [store (open-store db store-id "readonly") + request (.count store)] + (promisify request)))) -(defn clear! [store-id] - (let [store (open-store @+db+ store-id "readwrite") - request (.clear store)] - (promisify request))) (defn open-cursor! - ([store-id key-range] (open-cursor! store-id key-range "next")) - ([store-id key-range direction] - (let [store (open-store @+db+ store-id)] - (.openCursor store key-range direction))) - ([store-id idx-name key-range direction] - (let [store (open-store @+db+ store-id)] - (.openCursor (.index store (str idx-name)) key-range direction)))) + ([store-id key-range] (open-cursor! @+db+ store-id key-range {:direction :asc})) + ([store-id key-range opts] (open-cursor! @+db+ store-id key-range opts)) + ([db store-id key-range {:keys [direction index]}] + (let [store (open-store db store-id "readonly") + key-range (if (= key-range ::all) + (js/IDBKeyRange.lowerBound "") + key-range) + direction ({:asc "next" :desc "prev"} direction direction)] + (if index + (.openCursor (.index store (str index)) key-range direction) + (.openCursor store key-range direction))))) + #_(defn logging [f] (let [n (volatile! 0)] @@ -130,13 +163,11 @@ (defn first-result [xform cursor-req] (transduce-cursor (comp xform (take 1)) (fn [_ x] x) cursor-req)) -(def all (js/IDBKeyRange.lowerBound "")) - (comment (let [re (js/RegExp. "user" "i")] (do (print "starting…" (js/Date.)) - (-> (open-cursor! ::posts all) + (-> (open-cursor! ::posts ::all) (transduce-cursor (comp (filter #(re-find re (.-content %))) (take 50) (map #(js->clj % :keywordize-keys true)))) @@ -145,13 +176,3 @@ (js/console.log (first result))))))) ) - -(defn delete! [store-id k] - (let [store (open-store @+db+ store-id) - request (.delete store k)] - (promisify request))) - -(defn count! [store-id] - (let [store (open-store @+db+ store-id) - request (.count store)] - (promisify request)))