Move to reagent

This commit is contained in:
heyarne 2020-05-03 10:01:03 +02:00
commit 6bfa94a2ca
8 changed files with 322 additions and 75 deletions

View file

@ -1,3 +1,4 @@
{:paths ["src"]
:deps {thheller/shadow-cljs {:mvn/version "2.8.109"}
appliedscience/js-interop {:mvn/version "0.2.5"}}}
appliedscience/js-interop {:mvn/version "0.2.5"}
reagent {:mvn/version "1.0.0-alpha1"}}}

166
package-lock.json generated
View file

@ -55,6 +55,11 @@
"resolved": "https://registry.npmjs.org/@types/webgl2/-/webgl2-0.0.4.tgz",
"integrity": "sha512-PACt1xdErJbMUOUweSrbVM7gSIYm1vTncW2hF6Os/EeWi6TXYAYMPp+8v6rzHmypE5gHrxaxZNXgMkJVIdZpHw=="
},
"asap": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
"integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY="
},
"base64-js": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
@ -69,25 +74,186 @@
"ieee754": "^1.1.4"
}
},
"core-js": {
"version": "1.2.7",
"resolved": "https://registry.npmjs.org/core-js/-/core-js-1.2.7.tgz",
"integrity": "sha1-ZSKUwUZR2yj6k70tX/KYOk8IxjY="
},
"create-react-class": {
"version": "15.6.3",
"resolved": "https://registry.npmjs.org/create-react-class/-/create-react-class-15.6.3.tgz",
"integrity": "sha512-M+/3Q6E6DLO6Yx3OwrWjwHBnvfXXYA7W+dFjt/ZDBemHO1DDZhsalX/NUtnTYclN6GfnBDRh4qRHjcDHmlJBJg==",
"requires": {
"fbjs": "^0.8.9",
"loose-envify": "^1.3.1",
"object-assign": "^4.1.1"
}
},
"encoding": {
"version": "0.1.12",
"resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz",
"integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=",
"requires": {
"iconv-lite": "~0.4.13"
}
},
"fbjs": {
"version": "0.8.17",
"resolved": "https://registry.npmjs.org/fbjs/-/fbjs-0.8.17.tgz",
"integrity": "sha1-xNWY6taUkRJlPWWIsBpc3Nn5D90=",
"requires": {
"core-js": "^1.0.0",
"isomorphic-fetch": "^2.1.1",
"loose-envify": "^1.0.0",
"object-assign": "^4.1.0",
"promise": "^7.1.1",
"setimmediate": "^1.0.5",
"ua-parser-js": "^0.7.18"
}
},
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
},
"ieee754": {
"version": "1.1.13",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
"integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
},
"is-stream": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
"integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
},
"isomorphic-fetch": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz",
"integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=",
"requires": {
"node-fetch": "^1.0.1",
"whatwg-fetch": ">=0.10.0"
},
"dependencies": {
"node-fetch": {
"version": "1.7.3",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz",
"integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==",
"requires": {
"encoding": "^0.1.11",
"is-stream": "^1.0.1"
}
}
}
},
"js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="
},
"loose-envify": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
"integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",
"requires": {
"js-tokens": "^3.0.0 || ^4.0.0"
}
},
"node-fetch": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz",
"integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U="
},
"object-assign": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
},
"process": {
"version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
"integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI="
},
"promise": {
"version": "7.3.1",
"resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz",
"integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==",
"requires": {
"asap": "~2.0.3"
}
},
"prop-types": {
"version": "15.7.2",
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz",
"integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==",
"requires": {
"loose-envify": "^1.4.0",
"object-assign": "^4.1.1",
"react-is": "^16.8.1"
}
},
"react": {
"version": "16.13.1",
"resolved": "https://registry.npmjs.org/react/-/react-16.13.1.tgz",
"integrity": "sha512-YMZQQq32xHLX0bz5Mnibv1/LHb3Sqzngu7xstSM+vrkE5Kzr9xE0yMByK5kMoTK30YVJE61WfbxIFFvfeDKT1w==",
"requires": {
"loose-envify": "^1.1.0",
"object-assign": "^4.1.1",
"prop-types": "^15.6.2"
}
},
"react-dom": {
"version": "16.13.1",
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-16.13.1.tgz",
"integrity": "sha512-81PIMmVLnCNLO/fFOQxdQkvEq/+Hfpv24XNJfpyZhTRfO0QcmQIF/PgCa1zCOj2w1hrn12MFLyaJ/G0+Mxtfag==",
"requires": {
"loose-envify": "^1.1.0",
"object-assign": "^4.1.1",
"prop-types": "^15.6.2",
"scheduler": "^0.19.1"
}
},
"react-is": {
"version": "16.13.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
"integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="
},
"safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"scheduler": {
"version": "0.19.1",
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.19.1.tgz",
"integrity": "sha512-n/zwRWRYSUj0/3g/otKDRPMh6qv2SYMWNq85IEa8iZyAv8od9zDYpGSnpBEjNgcMNq6Scbu5KfIPxNF72R/2EA==",
"requires": {
"loose-envify": "^1.1.0",
"object-assign": "^4.1.1"
}
},
"seedrandom": {
"version": "2.4.3",
"resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-2.4.3.tgz",
"integrity": "sha1-JDhQTa0zkXMUv/GKxNeU8W1qrsw="
},
"setimmediate": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
"integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU="
},
"ua-parser-js": {
"version": "0.7.21",
"resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.21.tgz",
"integrity": "sha512-+O8/qh/Qj8CgC6eYBVBykMrNtp5Gebn4dlGD/kKXVkJNDwyrAwSIqwz8CDf+tsAIWVycKcku6gIXJ0qwx/ZXaQ=="
},
"whatwg-fetch": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz",
"integrity": "sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q=="
}
}
}

View file

@ -13,6 +13,9 @@
"@tensorflow/tfjs-converter": "^1.7.4",
"@tensorflow/tfjs-core": "^1.7.4",
"buffer": "^5.6.0",
"process": "^0.11.10"
"create-react-class": "^15.6.3",
"process": "^0.11.10",
"react": "^16.13.1",
"react-dom": "^16.13.1"
}
}

View file

@ -3,10 +3,12 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="style.css" />
</head>
<body>
<canvas id="result" style="position: absolute; z-index: 10; transform: scale(-1, 1)"></canvas>
<video id="capture" playsinline style="transform: scale(-1, 1); max-width: 100%; height: auto; op"></video>
<div id="app"></div>
<!-- <canvas id="result"></canvas>
<video id="capture" playsinline></video> -->
</body>
<script src="js/main.js"></script>
</html>

View file

@ -0,0 +1,24 @@
*,
*::before,
*::after {
box-sizing: border-box
}
body {
color: #222
}
canvas#result {
position: absolute;
z-index: 10;
}
video#capture {
max-width: 100%;
height: auto;
}
canvas#result,
video#capture {
transform: scale(-1, 1)
}

View file

@ -1,75 +1,27 @@
(ns heyarne.all-my-friends.core
(:require ["@tensorflow/tfjs-core" :as tf]
["@tensorflow-models/facemesh" :as facemesh]
[applied-science.js-interop :as j]))
(:require [applied-science.js-interop :as j]
[reagent.core :as r]
[reagent.dom :as dom]
[heyarne.all-my-friends.views :as views]))
(defonce state (atom {}))
(defonce state (r/atom {:status :welcome-message}))
(defn draw-results [elem predictions]
(let [canvas (.querySelector js/document "canvas#result")
ctx (. canvas (getContext "2d"))]
(defn draw-results [ctx predictions]
;; remove previous results
(.clearRect ctx 0 0 (.. ctx -canvas -width) (.. ctx -canvas -height))
;; draw and append new results
(set! (.-strokeStyle ctx) "pink")
(doseq [prediction predictions
:let [[[t-x t-y]] (j/get-in prediction [:boundingBox :topLeft])
[[b-x b-y]] (j/get-in prediction [:boundingBox :bottomRight])]]
(doseq [prediction predictions]
(doseq [[x y _] (j/get prediction :scaledMesh)]
(.beginPath ctx)
(.arc ctx x y 1 0 (* 2 Math/PI))
(.stroke ctx)))))
(.stroke ctx))))
(defn detect-faces [model elem]
(.. model
(estimateFaces elem)
(then (fn [predictions]
(set! (.-predictions js/window) predictions)
#_(println "Predictions" predictions)
(draw-results elem predictions))))
(js/requestAnimationFrame #(detect-faces model elem)))
(defn start-capture [video-elem]
;; set up webcam
(.. js/navigator
-mediaDevices
(getUserMedia #js {:audio false
:video #js {:facingMode "user"
:width 320
:height 320}})
(then (fn [stream]
(set! (.-srcObject video-elem) stream))))
;; return promise
(js/Promise.
(fn [resolve]
(set! (.-onloadedmetadata video-elem) #(resolve video-elem)))))
;; TODO: Handle rejected permission request
;; TODO: Initialize model in the background
(defn init []
(defn ^:dev/after-load init []
(println "Initializing…")
(swap! state ::status :webcam-init)
(let [video (.querySelector js/document "video#capture")
canvas (.querySelector js/document "canvas#result")]
(-> (start-capture video)
(.then (fn [video]
(.play video)
(println "video.videoWidth" (.-videoWidth video)
"video.videoHeight" (.-videoHeight video))
;; initialize canvas
(set! (.-width canvas) (.-videoWidth video))
(set! (.-height canvas) (.-videoHeight video))
#_(.scale ctx (/ (.-clientWidth video) (.-videoWidth video)) (/ (.-clientHeight video) (.-videoHeight video)))
;; initalize model
(swap! state ::status :model-init)
(.. tf
(setBackend "webgl")
(then #(.load facemesh #js {:maxFaces 1}))
(then #(detect-faces % video))))))))
(dom/render [views/app {:state state
:on-faces-detected draw-results}]
(.querySelector js/document "#app")))
(defonce initialize (init))

View file

@ -0,0 +1,78 @@
(ns heyarne.all-my-friends.facemesh
(:require ["@tensorflow/tfjs-core" :as tf]
["@tensorflow-models/facemesh" :as facemesh]
[reagent.core :as r]
[reagent.dom :as dom]))
;; these two init functions take care of our tensorflow model and webcam stream
;; they accept classic node-style callbacks like (fn [err result])
(defn init-model [on-model-init]
(->
(.setBackend tf "webgl")
(.then #(.load facemesh #js {:maxFaces 1}))
(.then #(on-model-init nil %) #_#(detect-faces % video))))
(defn init-webcam [on-stream-init]
(-> (.-mediaDevices js/navigator)
(.getUserMedia #js {:audio false
:video #js {:facingMode "user"
:width 320
:height 320}})
(.then #(on-stream-init nil %) #_(fn [stream]
(set! (.-srcObject video-elem) stream)))
(.catch on-stream-init)))
(defn- promisify
"Resolves a promise as soon as `callback` calls the function that is
passed as it's"
[cb-fn]
(js/Promise. (fn [resolve reject]
(cb-fn (fn [err result]
(if err
(reject err)
(resolve result)))))))
;; this function will be repeatedly called on the video stream to detect faces
(defn detect-faces [model video on-faces-detected]
(-> (.estimateFaces model video)
(.then (fn [predictions]
(on-faces-detected predictions))))
(js/requestAnimationFrame #(detect-faces model video on-faces-detected)))
(defn webcam-facemesh [{:keys [on-webcam-rejected
on-faces-detected]}]
(r/create-class
{:display-name "webcam-facemesh"
:reagent-render
(fn [_ _]
[:div.capture-container
[:canvas#result]
[:video#capture]])
:component-did-mount
(fn [this]
;; this function does the following
;; - set up the tensorflow model
;; - request access to the user's webcam
;; - continuously detect faces in the webcam feed
(let [container (dom/dom-node this)
video (.querySelector container "#capture")
canvas (.querySelector container "#result")
ctx (.getContext canvas "2d")
model (promisify init-model)
stream (promisify init-webcam)]
(-> (js/Promise.all #js [model stream])
(.then (fn [[model stream]]
(println "model and stream initialized")
(js/console.log model stream)
(set! (.-srcObject video) stream)
(set! (.-onloadedmetadata video)
(fn [_]
(.play video)
(set! (.-width canvas) (.-videoWidth video))
(set! (.-height canvas) (.-videoHeight video))
;; detect-faces will continously be called via requestAnimationFrame
;; on-faces-detected receives the canvas context as first param and detected predictions as second
(detect-faces model video (partial (:on-faces-detected (r/props this)) ctx)))))))))}))

View file

@ -0,0 +1,21 @@
(ns heyarne.all-my-friends.views
(:require [reagent.core :as r]
[reagent.dom :as dom]
[heyarne.all-my-friends.facemesh :refer [webcam-facemesh]]))
(defn welcome-message [{:keys [hidden?]}]
[:section.welcome-message
{:hidden false}
[:h1 "Hi Freund!"]
[:p "Ich möchte dir kurz erklären, was dich hier erwartet:
Seit der globalen Covid19-Pandemie sind wir alle dazu gezwungen, auf physischen Kontakt weitgehend zu verzichten. Ein Großteil der Zeit, die ich mit euch verbringe, hat sich ins Digitale verlagert."]
[:p "Das fühlt sich sicher bald komplett normal an -- vorher möchte ich aber gerne irgendwas mit dem komischen Gefühl machen, das das hinterlässt."]
[:p "Ich würde mich freuen, wenn du mir dabei hilfst. Folge dazu einfach den Anweisungen. Das Ergebnis wird hoffentlich eine schöne Sammlung von Webcambildern und 3D-Modellen eurer Köpfe" [:sup "1"] "."]
[:button "Weiter"]])
(defn app [{:keys [state on-faces-detected]}]
(let [status (:status @state)
viewing-welcome-message? (= :welcome-message status)]
[:div#app
[welcome-message {:hidden? viewing-welcome-message?}]
[webcam-facemesh {:on-faces-detected on-faces-detected}]]))