diff --git a/deps.edn b/deps.edn index e422438..bf2c59a 100644 --- a/deps.edn +++ b/deps.edn @@ -1,3 +1,4 @@ {:paths ["src"] :deps {thheller/shadow-cljs {:mvn/version "2.8.109"} - appliedscience/js-interop {:mvn/version "0.2.5"}}} + appliedscience/js-interop {:mvn/version "0.2.5"} + reagent {:mvn/version "1.0.0-alpha1"}}} diff --git a/package-lock.json b/package-lock.json index 4f32c55..8c67c62 100644 --- a/package-lock.json +++ b/package-lock.json @@ -55,6 +55,11 @@ "resolved": "https://registry.npmjs.org/@types/webgl2/-/webgl2-0.0.4.tgz", "integrity": "sha512-PACt1xdErJbMUOUweSrbVM7gSIYm1vTncW2hF6Os/EeWi6TXYAYMPp+8v6rzHmypE5gHrxaxZNXgMkJVIdZpHw==" }, + "asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=" + }, "base64-js": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", @@ -69,25 +74,186 @@ "ieee754": "^1.1.4" } }, + "core-js": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-1.2.7.tgz", + "integrity": "sha1-ZSKUwUZR2yj6k70tX/KYOk8IxjY=" + }, + "create-react-class": { + "version": "15.6.3", + "resolved": "https://registry.npmjs.org/create-react-class/-/create-react-class-15.6.3.tgz", + "integrity": "sha512-M+/3Q6E6DLO6Yx3OwrWjwHBnvfXXYA7W+dFjt/ZDBemHO1DDZhsalX/NUtnTYclN6GfnBDRh4qRHjcDHmlJBJg==", + "requires": { + "fbjs": "^0.8.9", + "loose-envify": "^1.3.1", + "object-assign": "^4.1.1" + } + }, + "encoding": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz", + "integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=", + "requires": { + "iconv-lite": "~0.4.13" + } + }, + "fbjs": { + "version": "0.8.17", + "resolved": "https://registry.npmjs.org/fbjs/-/fbjs-0.8.17.tgz", + "integrity": "sha1-xNWY6taUkRJlPWWIsBpc3Nn5D90=", + "requires": { + "core-js": "^1.0.0", + "isomorphic-fetch": "^2.1.1", + "loose-envify": "^1.0.0", + "object-assign": "^4.1.0", + "promise": "^7.1.1", + "setimmediate": "^1.0.5", + "ua-parser-js": "^0.7.18" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, "ieee754": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" + }, + "isomorphic-fetch": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz", + "integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=", + "requires": { + "node-fetch": "^1.0.1", + "whatwg-fetch": ">=0.10.0" + }, + "dependencies": { + "node-fetch": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", + "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", + "requires": { + "encoding": "^0.1.11", + "is-stream": "^1.0.1" + } + } + } + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + }, + "loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "requires": { + "js-tokens": "^3.0.0 || ^4.0.0" + } + }, "node-fetch": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz", "integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U=" }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, "process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=" }, + "promise": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", + "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", + "requires": { + "asap": "~2.0.3" + } + }, + "prop-types": { + "version": "15.7.2", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz", + "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==", + "requires": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.8.1" + } + }, + "react": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react/-/react-16.13.1.tgz", + "integrity": "sha512-YMZQQq32xHLX0bz5Mnibv1/LHb3Sqzngu7xstSM+vrkE5Kzr9xE0yMByK5kMoTK30YVJE61WfbxIFFvfeDKT1w==", + "requires": { + "loose-envify": "^1.1.0", + "object-assign": "^4.1.1", + "prop-types": "^15.6.2" + } + }, + "react-dom": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-16.13.1.tgz", + "integrity": "sha512-81PIMmVLnCNLO/fFOQxdQkvEq/+Hfpv24XNJfpyZhTRfO0QcmQIF/PgCa1zCOj2w1hrn12MFLyaJ/G0+Mxtfag==", + "requires": { + "loose-envify": "^1.1.0", + "object-assign": "^4.1.1", + "prop-types": "^15.6.2", + "scheduler": "^0.19.1" + } + }, + "react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "scheduler": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.19.1.tgz", + "integrity": "sha512-n/zwRWRYSUj0/3g/otKDRPMh6qv2SYMWNq85IEa8iZyAv8od9zDYpGSnpBEjNgcMNq6Scbu5KfIPxNF72R/2EA==", + "requires": { + "loose-envify": "^1.1.0", + "object-assign": "^4.1.1" + } + }, "seedrandom": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-2.4.3.tgz", "integrity": "sha1-JDhQTa0zkXMUv/GKxNeU8W1qrsw=" + }, + "setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" + }, + "ua-parser-js": { + "version": "0.7.21", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.21.tgz", + "integrity": "sha512-+O8/qh/Qj8CgC6eYBVBykMrNtp5Gebn4dlGD/kKXVkJNDwyrAwSIqwz8CDf+tsAIWVycKcku6gIXJ0qwx/ZXaQ==" + }, + "whatwg-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz", + "integrity": "sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q==" } } } diff --git a/package.json b/package.json index 63e65be..98a593b 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,9 @@ "@tensorflow/tfjs-converter": "^1.7.4", "@tensorflow/tfjs-core": "^1.7.4", "buffer": "^5.6.0", - "process": "^0.11.10" + "create-react-class": "^15.6.3", + "process": "^0.11.10", + "react": "^16.13.1", + "react-dom": "^16.13.1" } } diff --git a/resources/public/index.html b/resources/public/index.html index 75f5050..032ac86 100644 --- a/resources/public/index.html +++ b/resources/public/index.html @@ -3,10 +3,12 @@ + - - +
+ diff --git a/resources/public/style.css b/resources/public/style.css new file mode 100644 index 0000000..832142d --- /dev/null +++ b/resources/public/style.css @@ -0,0 +1,24 @@ +*, +*::before, +*::after { + box-sizing: border-box +} + +body { + color: #222 +} + +canvas#result { + position: absolute; + z-index: 10; +} + +video#capture { + max-width: 100%; + height: auto; +} + +canvas#result, +video#capture { + transform: scale(-1, 1) +} diff --git a/src/heyarne/all_my_friends/core.cljs b/src/heyarne/all_my_friends/core.cljs index 3fd347f..3a762f3 100644 --- a/src/heyarne/all_my_friends/core.cljs +++ b/src/heyarne/all_my_friends/core.cljs @@ -1,75 +1,27 @@ (ns heyarne.all-my-friends.core - (:require ["@tensorflow/tfjs-core" :as tf] - ["@tensorflow-models/facemesh" :as facemesh] - [applied-science.js-interop :as j])) + (:require [applied-science.js-interop :as j] + [reagent.core :as r] + [reagent.dom :as dom] + [heyarne.all-my-friends.views :as views])) -(defonce state (atom {})) +(defonce state (r/atom {:status :welcome-message})) -(defn draw-results [elem predictions] - (let [canvas (.querySelector js/document "canvas#result") - ctx (. canvas (getContext "2d"))] - ;; remove previous results - (.clearRect ctx 0 0 (.. ctx -canvas -width) (.. ctx -canvas -height)) +(defn draw-results [ctx predictions] + ;; remove previous results + (.clearRect ctx 0 0 (.. ctx -canvas -width) (.. ctx -canvas -height)) - ;; draw and append new results - (set! (.-strokeStyle ctx) "pink") - (doseq [prediction predictions - :let [[[t-x t-y]] (j/get-in prediction [:boundingBox :topLeft]) - [[b-x b-y]] (j/get-in prediction [:boundingBox :bottomRight])]] + ;; draw and append new results + (set! (.-strokeStyle ctx) "pink") + (doseq [prediction predictions] + (doseq [[x y _] (j/get prediction :scaledMesh)] + (.beginPath ctx) + (.arc ctx x y 1 0 (* 2 Math/PI)) + (.stroke ctx)))) - (doseq [[x y _] (j/get prediction :scaledMesh)] - (.beginPath ctx) - (.arc ctx x y 1 0 (* 2 Math/PI)) - (.stroke ctx))))) - -(defn detect-faces [model elem] - (.. model - (estimateFaces elem) - (then (fn [predictions] - (set! (.-predictions js/window) predictions) - #_(println "Predictions" predictions) - (draw-results elem predictions)))) - (js/requestAnimationFrame #(detect-faces model elem))) - -(defn start-capture [video-elem] - ;; set up webcam - (.. js/navigator - -mediaDevices - (getUserMedia #js {:audio false - :video #js {:facingMode "user" - :width 320 - :height 320}}) - (then (fn [stream] - (set! (.-srcObject video-elem) stream)))) - ;; return promise - (js/Promise. - (fn [resolve] - (set! (.-onloadedmetadata video-elem) #(resolve video-elem))))) - -;; TODO: Handle rejected permission request -;; TODO: Initialize model in the background - -(defn init [] +(defn ^:dev/after-load init [] (println "Initializing…") - (swap! state ::status :webcam-init) - (let [video (.querySelector js/document "video#capture") - canvas (.querySelector js/document "canvas#result")] - (-> (start-capture video) - (.then (fn [video] - (.play video) - (println "video.videoWidth" (.-videoWidth video) - "video.videoHeight" (.-videoHeight video)) - - ;; initialize canvas - (set! (.-width canvas) (.-videoWidth video)) - (set! (.-height canvas) (.-videoHeight video)) - #_(.scale ctx (/ (.-clientWidth video) (.-videoWidth video)) (/ (.-clientHeight video) (.-videoHeight video))) - - ;; initalize model - (swap! state ::status :model-init) - (.. tf - (setBackend "webgl") - (then #(.load facemesh #js {:maxFaces 1})) - (then #(detect-faces % video)))))))) + (dom/render [views/app {:state state + :on-faces-detected draw-results}] + (.querySelector js/document "#app"))) (defonce initialize (init)) diff --git a/src/heyarne/all_my_friends/facemesh.cljs b/src/heyarne/all_my_friends/facemesh.cljs new file mode 100644 index 0000000..e2cf90a --- /dev/null +++ b/src/heyarne/all_my_friends/facemesh.cljs @@ -0,0 +1,78 @@ +(ns heyarne.all-my-friends.facemesh + (:require ["@tensorflow/tfjs-core" :as tf] + ["@tensorflow-models/facemesh" :as facemesh] + [reagent.core :as r] + [reagent.dom :as dom])) + +;; these two init functions take care of our tensorflow model and webcam stream +;; they accept classic node-style callbacks like (fn [err result]) + +(defn init-model [on-model-init] + (-> + (.setBackend tf "webgl") + (.then #(.load facemesh #js {:maxFaces 1})) + (.then #(on-model-init nil %) #_#(detect-faces % video)))) + +(defn init-webcam [on-stream-init] + (-> (.-mediaDevices js/navigator) + (.getUserMedia #js {:audio false + :video #js {:facingMode "user" + :width 320 + :height 320}}) + (.then #(on-stream-init nil %) #_(fn [stream] + (set! (.-srcObject video-elem) stream))) + (.catch on-stream-init))) + +(defn- promisify + "Resolves a promise as soon as `callback` calls the function that is + passed as it's" + [cb-fn] + (js/Promise. (fn [resolve reject] + (cb-fn (fn [err result] + (if err + (reject err) + (resolve result))))))) + +;; this function will be repeatedly called on the video stream to detect faces +(defn detect-faces [model video on-faces-detected] + (-> (.estimateFaces model video) + (.then (fn [predictions] + (on-faces-detected predictions)))) + (js/requestAnimationFrame #(detect-faces model video on-faces-detected))) + +(defn webcam-facemesh [{:keys [on-webcam-rejected + on-faces-detected]}] + (r/create-class + {:display-name "webcam-facemesh" + + :reagent-render + (fn [_ _] + [:div.capture-container + [:canvas#result] + [:video#capture]]) + + :component-did-mount + (fn [this] + ;; this function does the following + ;; - set up the tensorflow model + ;; - request access to the user's webcam + ;; - continuously detect faces in the webcam feed + (let [container (dom/dom-node this) + video (.querySelector container "#capture") + canvas (.querySelector container "#result") + ctx (.getContext canvas "2d") + model (promisify init-model) + stream (promisify init-webcam)] + (-> (js/Promise.all #js [model stream]) + (.then (fn [[model stream]] + (println "model and stream initialized") + (js/console.log model stream) + (set! (.-srcObject video) stream) + (set! (.-onloadedmetadata video) + (fn [_] + (.play video) + (set! (.-width canvas) (.-videoWidth video)) + (set! (.-height canvas) (.-videoHeight video)) + ;; detect-faces will continously be called via requestAnimationFrame + ;; on-faces-detected receives the canvas context as first param and detected predictions as second + (detect-faces model video (partial (:on-faces-detected (r/props this)) ctx)))))))))})) diff --git a/src/heyarne/all_my_friends/views.cljs b/src/heyarne/all_my_friends/views.cljs new file mode 100644 index 0000000..b5b516e --- /dev/null +++ b/src/heyarne/all_my_friends/views.cljs @@ -0,0 +1,21 @@ +(ns heyarne.all-my-friends.views + (:require [reagent.core :as r] + [reagent.dom :as dom] + [heyarne.all-my-friends.facemesh :refer [webcam-facemesh]])) + +(defn welcome-message [{:keys [hidden?]}] + [:section.welcome-message + {:hidden false} + [:h1 "Hi Freund!"] + [:p "Ich möchte dir kurz erklären, was dich hier erwartet: +Seit der globalen Covid19-Pandemie sind wir alle dazu gezwungen, auf physischen Kontakt weitgehend zu verzichten. Ein Großteil der Zeit, die ich mit euch verbringe, hat sich ins Digitale verlagert."] + [:p "Das fühlt sich sicher bald komplett normal an -- vorher möchte ich aber gerne irgendwas mit dem komischen Gefühl machen, das das hinterlässt."] + [:p "Ich würde mich freuen, wenn du mir dabei hilfst. Folge dazu einfach den Anweisungen. Das Ergebnis wird hoffentlich eine schöne Sammlung von Webcambildern und 3D-Modellen eurer Köpfe" [:sup "1"] "."] + [:button "Weiter"]]) + +(defn app [{:keys [state on-faces-detected]}] + (let [status (:status @state) + viewing-welcome-message? (= :welcome-message status)] + [:div#app + [welcome-message {:hidden? viewing-welcome-message?}] + [webcam-facemesh {:on-faces-detected on-faces-detected}]]))