near/heyarne.all-my-friends.app
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Change to video capture (which solves bad scaledPrediction)

Browse source
This commit is contained in:
heyarne 2020-05-02 20:37:55 +02:00
commit 1783b00d2c
2 changed files with 90 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

3
resources/public/index.html
View file

@ -2,7 +2,8 @@
<html> <html>
<head><meta charset="utf-8" /></head> <head><meta charset="utf-8" /></head>
<body> <body>
<img id="example-pic" src="example.jpg" alt="Example webcam picture" /> <canvas id="result" style="position: absolute; z-index: 10; transform: scale(-1, 1)"></canvas>
<video id="capture" playsinline style="transform: scale(-1, 1);"></video>
</body> </body>
<script src="js/main.js"></script> <script src="js/main.js"></script>
</html> </html>

118
src/heyarne/all_my_friends/core.cljs
View file

@ -1,48 +1,104 @@
(ns heyarne.all-my-friends.core (ns heyarne.all-my-friends.core
(:require ["@tensorflow/tfjs-core" :as tf] (:require ["@tensorflow/tfjs-core" :as tf]
["@tensorflow-models/facemesh" :as facemesh] ["@tensorflow-models/facemesh" :as facemesh]
[applied-science.js-interop :as j] [applied-science.js-interop :as j]))
[goog.dom :as dom]))
(def img (js/document.querySelector "img")) (defonce state (atom {}))
(defn draw-results [img predictions] (defn draw-results [elem predictions]
(let [bounds (j/lookup (.getBoundingClientRect img)) (let [canvas (.querySelector js/document "canvas#result")
parent (dom/getParentElement img)
canvas (doto (dom/createElement "canvas")
(.setAttribute "width" (:width bounds))
(.setAttribute "height" (:height bounds))
(.setAttribute "style" (str "display: block; "
"position: absolute; "
"top: " (:top bounds) "px; "
"left: " (:left bounds) "px")))
ctx (. canvas (getContext "2d"))] ctx (. canvas (getContext "2d"))]
;; remove previous results ;; remove previous results
(doseq [result-canvas (.querySelectorAll parent "canvas")] (.clearRect ctx 0 0 (.. ctx -canvas -width) (.. ctx -canvas -height))
(dom/removeNode result-canvas))
;; draw and append new results
(set! (.-strokeStyle ctx) "white")
(doseq [prediction predictions
[x y _] (j/get prediction :scaledMesh)]
(.beginPath ctx)
(.ellipse ctx x y 2 2 0 0 (* 2 Math/PI))
(.stroke ctx))
(dom/appendChild parent canvas)))
(defn detect-faces [model] ;; draw and append new results
(println "Facemesh loaded")
(doseq [prediction predictions
:let [[[t-x t-y]] (j/get-in prediction [:boundingBox :topLeft])
[[b-x b-y]] (j/get-in prediction [:boundingBox :bottomRight])]]
(set! (.-strokeStyle ctx) "pink")
(comment
(.setLineDash ctx #js [2 2])
(.beginPath ctx)
(.moveTo ctx t-x t-y)
(.lineTo ctx t-x b-y)
(.lineTo ctx b-x b-y)
(.lineTo ctx b-x t-y)
(.lineTo ctx t-x t-y)
(.stroke ctx)
(.setLineDash ctx #js [])
(.beginPath ctx)
(.arc ctx b-x b-y 2 0 (* 2 Math/PI))
(.stroke ctx))
(doseq [[x y _] (j/get prediction :scaledMesh)]
(.beginPath ctx)
(.arc ctx x y 1 0 (* 2 Math/PI))
(.stroke ctx))
(comment
(doseq [[x y _] (j/get prediction :mesh)]
(.beginPath ctx)
(.arc ctx x y 2 0 (* 2 Math/PI))
(.stroke ctx))))))
(defn detect-faces [model elem]
(.. model (.. model
(estimateFaces img) (estimateFaces elem)
(then (fn [predictions] (then (fn [predictions]
(set! (.-predictions js/window) predictions) (set! (.-predictions js/window) predictions)
#_(println "Predictions" predictions) #_(println "Predictions" predictions)
(draw-results img predictions))))) (draw-results elem predictions))))
(js/requestAnimationFrame #(detect-faces model elem)))
(defonce (defn start-capture [video-elem]
init ;; set up webcam
(do (.. js/navigator
-mediaDevices
(getUserMedia #js {:audio false
:video #js {:facingMode "user"
:width 320
:height 320}})
(then (fn [stream]
(set! (.-srcObject video-elem) stream))))
;; return promise
(js/Promise.
(fn [resolve]
(set! (.-onloadedmetadata video-elem) #(resolve video-elem)))))
;; TODO: Handle rejected permission request
(defn init []
(swap! state ::status :webcam-init)
(let [video (.querySelector js/document "video#capture")
canvas (.querySelector js/document "canvas#result")
ctx (.getContext canvas "2d")]
(-> (start-capture video)
(.then (fn [video]
(.play video)
(println "video.videoWidth" (.-videoWidth video)
"video.videoHeight" (.-videoHeight video))
;; initialize canvas
(set! (.-width canvas) (.-videoWidth video))
(set! (.-height canvas) (.-videoHeight video))
#_(.translate ctx (.-width canvas) 0)
#_(.scale ctx -1 1)
;; initalize model
(swap! state ::status :model-init)
(.. tf
(setBackend "webgl")
(then #(.load facemesh #js {:maxFaces 1}))
(then #(detect-faces % video))))))))
(defonce initialize (init)
#_(do
(println "Initializing…") (println "Initializing…")
(.. tf (.. tf
(setBackend "webgl") (setBackend "webgl")
(then #(.load facemesh)) (then #(.load facemesh #js {:maxFaces 1}))
(then detect-faces)))) (then detect-faces))))