Implemented image recognition

3 files changed, 50 insertions, 5 deletions

diff --git a/client/package.json b/client/package.json
index d3596b3..0ebc93b 100644
--- a/client/package.json
+++ b/client/package.json
@@ -8,6 +8,8 @@
     "@mui/icons-material": "^5.8.4",
     "@mui/material": "^5.9.1",
     "@mui/styles": "^5.9.3",
+    "@tensorflow-models/coco-ssd": "^2.2.2",
+    "@tensorflow/tfjs": "^3.19.0",
     "axios": "^0.27.2",
     "country-state-city": "^3.0.1",
     "i18next": "^21.9.0",
diff --git a/client/src/pages/Enrollment/Enrollment.jsx b/client/src/pages/Enrollment/Enrollment.jsx
index 88bdf47..53d64d5 100644
--- a/client/src/pages/Enrollment/Enrollment.jsx
+++ b/client/src/pages/Enrollment/Enrollment.jsx
@@ -1,4 +1,4 @@
-import React, { useState } from 'react'
+import React, { useState, useEffect } from 'react'
 import Address from './Address/Address'
 import Agreement from './Agreement/Agreement'
 import DocumentScanner from './DocumentScanner/DocumentScanner'
@@ -25,6 +25,8 @@ import { userContext } from '../../context/User'
 import { useNavigate } from 'react-router-dom'
 import BackButton from '../../components/BackButton/BackButton'
 import { initialUserData } from '../../constants/userData'
+import * as cocoSsd from '@tensorflow-models/coco-ssd'
+import * as tf from '@tensorflow/tfjs'
 
 const Enrollment = () => {
   const { t } = useTranslation()
@@ -33,6 +35,35 @@ const Enrollment = () => {
   const navigate = useNavigate()
   const [unverified, setUnverified] = useState(true)
 
+  const [model, setModel] = useState()
+
+  async function loadModel () {
+    try {
+      const model = await cocoSsd.load()
+      setModel(model)
+      console.log('setloadedModel')
+    } catch (err) {
+      console.log(err)
+      console.log('failed load model')
+    }
+  }
+
+  useEffect(() => {
+    tf.ready().then(() => {
+      loadModel()
+    })
+  }, [])
+
+  let predictions = []
+  async function predictionFunction () {
+    predictions = await model.detect(document.getElementById('img'))
+    if (predictions.length > 0) {
+      console.log(predictions)
+      console.log(predictions[0])
+    }
+    return predictions
+  }
+
   const { mutate } = useMutation((payload) => createUser(payload), {
     onSuccess: (data) => {
       setConfirm.mutate({
@@ -108,11 +139,23 @@ const Enrollment = () => {
         setPage(page + 1)
       }
     } else if (page === 3) {
-      if (!userData.photo) {
-        toast.error(t('PLEASE_CAPTURE_PHOTOGRAPH'))
-      } else {
+      predictionFunction()
+       if (predictions.length === 0) {
+         toast.warning(t('PLEASE_WAIT'), {
+          timeout: 1000
+         })
+       }
+      setTimeout(() => {
+        if (predictions.length > 0) {
+       if (!userData.photo) {
+         toast.error(t('PLEASE_CAPTURE_PHOTOGRAPH'))
+       } else if (predictions[0].class === 'person' && predictions[0].score > 0.5) {
         setPage(page + 1)
+       } else {
+        toast.error(t('PLEASE_CAPTURE_CLEAR_PHOTOGRAPH'))
+       }
       }
+      }, 1000)
     } else if (page === 4) {
       if (!userData.documents.POI) {
         toast.error(t('SCAN_YOUR_DOCUMENT'))
diff --git a/client/src/pages/Enrollment/PhotoCapture/PhotoCapture.jsx b/client/src/pages/Enrollment/PhotoCapture/PhotoCapture.jsx
index 4e4a2b3..2430016 100644
--- a/client/src/pages/Enrollment/PhotoCapture/PhotoCapture.jsx
+++ b/client/src/pages/Enrollment/PhotoCapture/PhotoCapture.jsx
@@ -58,7 +58,7 @@ const PhotoCapture = () => {
             }}
           />
         ) : (
-          <img src={userData.photo} />
+          <img id="img" src={userData.photo} />
         )}
       </div>
       <Grid container columnSpacing={10} justifyContent="center">