Move to reagent
This commit is contained in:
parent
3d2075764a
commit
6bfa94a2ca
8 changed files with 322 additions and 75 deletions
|
|
@ -1,75 +1,27 @@
|
|||
(ns heyarne.all-my-friends.core
|
||||
(:require ["@tensorflow/tfjs-core" :as tf]
|
||||
["@tensorflow-models/facemesh" :as facemesh]
|
||||
[applied-science.js-interop :as j]))
|
||||
(:require [applied-science.js-interop :as j]
|
||||
[reagent.core :as r]
|
||||
[reagent.dom :as dom]
|
||||
[heyarne.all-my-friends.views :as views]))
|
||||
|
||||
(defonce state (atom {}))
|
||||
(defonce state (r/atom {:status :welcome-message}))
|
||||
|
||||
(defn draw-results [elem predictions]
|
||||
(let [canvas (.querySelector js/document "canvas#result")
|
||||
ctx (. canvas (getContext "2d"))]
|
||||
;; remove previous results
|
||||
(.clearRect ctx 0 0 (.. ctx -canvas -width) (.. ctx -canvas -height))
|
||||
(defn draw-results [ctx predictions]
|
||||
;; remove previous results
|
||||
(.clearRect ctx 0 0 (.. ctx -canvas -width) (.. ctx -canvas -height))
|
||||
|
||||
;; draw and append new results
|
||||
(set! (.-strokeStyle ctx) "pink")
|
||||
(doseq [prediction predictions
|
||||
:let [[[t-x t-y]] (j/get-in prediction [:boundingBox :topLeft])
|
||||
[[b-x b-y]] (j/get-in prediction [:boundingBox :bottomRight])]]
|
||||
;; draw and append new results
|
||||
(set! (.-strokeStyle ctx) "pink")
|
||||
(doseq [prediction predictions]
|
||||
(doseq [[x y _] (j/get prediction :scaledMesh)]
|
||||
(.beginPath ctx)
|
||||
(.arc ctx x y 1 0 (* 2 Math/PI))
|
||||
(.stroke ctx))))
|
||||
|
||||
(doseq [[x y _] (j/get prediction :scaledMesh)]
|
||||
(.beginPath ctx)
|
||||
(.arc ctx x y 1 0 (* 2 Math/PI))
|
||||
(.stroke ctx)))))
|
||||
|
||||
(defn detect-faces [model elem]
|
||||
(.. model
|
||||
(estimateFaces elem)
|
||||
(then (fn [predictions]
|
||||
(set! (.-predictions js/window) predictions)
|
||||
#_(println "Predictions" predictions)
|
||||
(draw-results elem predictions))))
|
||||
(js/requestAnimationFrame #(detect-faces model elem)))
|
||||
|
||||
(defn start-capture [video-elem]
|
||||
;; set up webcam
|
||||
(.. js/navigator
|
||||
-mediaDevices
|
||||
(getUserMedia #js {:audio false
|
||||
:video #js {:facingMode "user"
|
||||
:width 320
|
||||
:height 320}})
|
||||
(then (fn [stream]
|
||||
(set! (.-srcObject video-elem) stream))))
|
||||
;; return promise
|
||||
(js/Promise.
|
||||
(fn [resolve]
|
||||
(set! (.-onloadedmetadata video-elem) #(resolve video-elem)))))
|
||||
|
||||
;; TODO: Handle rejected permission request
|
||||
;; TODO: Initialize model in the background
|
||||
|
||||
(defn init []
|
||||
(defn ^:dev/after-load init []
|
||||
(println "Initializing…")
|
||||
(swap! state ::status :webcam-init)
|
||||
(let [video (.querySelector js/document "video#capture")
|
||||
canvas (.querySelector js/document "canvas#result")]
|
||||
(-> (start-capture video)
|
||||
(.then (fn [video]
|
||||
(.play video)
|
||||
(println "video.videoWidth" (.-videoWidth video)
|
||||
"video.videoHeight" (.-videoHeight video))
|
||||
|
||||
;; initialize canvas
|
||||
(set! (.-width canvas) (.-videoWidth video))
|
||||
(set! (.-height canvas) (.-videoHeight video))
|
||||
#_(.scale ctx (/ (.-clientWidth video) (.-videoWidth video)) (/ (.-clientHeight video) (.-videoHeight video)))
|
||||
|
||||
;; initalize model
|
||||
(swap! state ::status :model-init)
|
||||
(.. tf
|
||||
(setBackend "webgl")
|
||||
(then #(.load facemesh #js {:maxFaces 1}))
|
||||
(then #(detect-faces % video))))))))
|
||||
(dom/render [views/app {:state state
|
||||
:on-faces-detected draw-results}]
|
||||
(.querySelector js/document "#app")))
|
||||
|
||||
(defonce initialize (init))
|
||||
|
|
|
|||
78
src/heyarne/all_my_friends/facemesh.cljs
Normal file
78
src/heyarne/all_my_friends/facemesh.cljs
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
(ns heyarne.all-my-friends.facemesh
|
||||
(:require ["@tensorflow/tfjs-core" :as tf]
|
||||
["@tensorflow-models/facemesh" :as facemesh]
|
||||
[reagent.core :as r]
|
||||
[reagent.dom :as dom]))
|
||||
|
||||
;; these two init functions take care of our tensorflow model and webcam stream
|
||||
;; they accept classic node-style callbacks like (fn [err result])
|
||||
|
||||
(defn init-model [on-model-init]
|
||||
(->
|
||||
(.setBackend tf "webgl")
|
||||
(.then #(.load facemesh #js {:maxFaces 1}))
|
||||
(.then #(on-model-init nil %) #_#(detect-faces % video))))
|
||||
|
||||
(defn init-webcam [on-stream-init]
|
||||
(-> (.-mediaDevices js/navigator)
|
||||
(.getUserMedia #js {:audio false
|
||||
:video #js {:facingMode "user"
|
||||
:width 320
|
||||
:height 320}})
|
||||
(.then #(on-stream-init nil %) #_(fn [stream]
|
||||
(set! (.-srcObject video-elem) stream)))
|
||||
(.catch on-stream-init)))
|
||||
|
||||
(defn- promisify
|
||||
"Resolves a promise as soon as `callback` calls the function that is
|
||||
passed as it's"
|
||||
[cb-fn]
|
||||
(js/Promise. (fn [resolve reject]
|
||||
(cb-fn (fn [err result]
|
||||
(if err
|
||||
(reject err)
|
||||
(resolve result)))))))
|
||||
|
||||
;; this function will be repeatedly called on the video stream to detect faces
|
||||
(defn detect-faces [model video on-faces-detected]
|
||||
(-> (.estimateFaces model video)
|
||||
(.then (fn [predictions]
|
||||
(on-faces-detected predictions))))
|
||||
(js/requestAnimationFrame #(detect-faces model video on-faces-detected)))
|
||||
|
||||
(defn webcam-facemesh [{:keys [on-webcam-rejected
|
||||
on-faces-detected]}]
|
||||
(r/create-class
|
||||
{:display-name "webcam-facemesh"
|
||||
|
||||
:reagent-render
|
||||
(fn [_ _]
|
||||
[:div.capture-container
|
||||
[:canvas#result]
|
||||
[:video#capture]])
|
||||
|
||||
:component-did-mount
|
||||
(fn [this]
|
||||
;; this function does the following
|
||||
;; - set up the tensorflow model
|
||||
;; - request access to the user's webcam
|
||||
;; - continuously detect faces in the webcam feed
|
||||
(let [container (dom/dom-node this)
|
||||
video (.querySelector container "#capture")
|
||||
canvas (.querySelector container "#result")
|
||||
ctx (.getContext canvas "2d")
|
||||
model (promisify init-model)
|
||||
stream (promisify init-webcam)]
|
||||
(-> (js/Promise.all #js [model stream])
|
||||
(.then (fn [[model stream]]
|
||||
(println "model and stream initialized")
|
||||
(js/console.log model stream)
|
||||
(set! (.-srcObject video) stream)
|
||||
(set! (.-onloadedmetadata video)
|
||||
(fn [_]
|
||||
(.play video)
|
||||
(set! (.-width canvas) (.-videoWidth video))
|
||||
(set! (.-height canvas) (.-videoHeight video))
|
||||
;; detect-faces will continously be called via requestAnimationFrame
|
||||
;; on-faces-detected receives the canvas context as first param and detected predictions as second
|
||||
(detect-faces model video (partial (:on-faces-detected (r/props this)) ctx)))))))))}))
|
||||
21
src/heyarne/all_my_friends/views.cljs
Normal file
21
src/heyarne/all_my_friends/views.cljs
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
(ns heyarne.all-my-friends.views
|
||||
(:require [reagent.core :as r]
|
||||
[reagent.dom :as dom]
|
||||
[heyarne.all-my-friends.facemesh :refer [webcam-facemesh]]))
|
||||
|
||||
(defn welcome-message [{:keys [hidden?]}]
|
||||
[:section.welcome-message
|
||||
{:hidden false}
|
||||
[:h1 "Hi Freund!"]
|
||||
[:p "Ich möchte dir kurz erklären, was dich hier erwartet:
|
||||
Seit der globalen Covid19-Pandemie sind wir alle dazu gezwungen, auf physischen Kontakt weitgehend zu verzichten. Ein Großteil der Zeit, die ich mit euch verbringe, hat sich ins Digitale verlagert."]
|
||||
[:p "Das fühlt sich sicher bald komplett normal an -- vorher möchte ich aber gerne irgendwas mit dem komischen Gefühl machen, das das hinterlässt."]
|
||||
[:p "Ich würde mich freuen, wenn du mir dabei hilfst. Folge dazu einfach den Anweisungen. Das Ergebnis wird hoffentlich eine schöne Sammlung von Webcambildern und 3D-Modellen eurer Köpfe" [:sup "1"] "."]
|
||||
[:button "Weiter"]])
|
||||
|
||||
(defn app [{:keys [state on-faces-detected]}]
|
||||
(let [status (:status @state)
|
||||
viewing-welcome-message? (= :welcome-message status)]
|
||||
[:div#app
|
||||
[welcome-message {:hidden? viewing-welcome-message?}]
|
||||
[webcam-facemesh {:on-faces-detected on-faces-detected}]]))
|
||||
Loading…
Add table
Add a link
Reference in a new issue