Merge pull request #29 from Capstone-Projects-2024-Spring/gesture-ticket

Basic page for detecting gestures and controlling a light

Author: bryanreiter

app/app.py

@@ -1,5 +1,6 @@
 from flask import Flask, render_template, Response, jsonify
 import cv2
+import requests
 import time
 import cv2
 import  asyncio, os
@@ -12,20 +13,12 @@
 #so you will need to install it with pip install python_weather
 #queue to find the right gesture
 from collections import deque
+deviceChoice = None
+deviceStatus = None
 
+#https://colab.research.google.com/github/googlesamples/mediapipe/blob/main/examples/gesture_recognizer/python/gesture_recognizer.ipynb#scrollTo=TUfAcER1oUS6
+#https://developers.google.com/mediapipe/solutions/vision/gesture_recognizer/python#video
 
-def thumbClassifier(results):
-    res=results.multi_hand_landmarks[0].landmark
-    GestureObject = hand(results.multi_hand_landmarks[0])
-    
-    # print('Thumb angle: ', thumb.angle)
-    # print('Ring Finger angle: ', ringFinger.angle)
-    # print('Middle Finger angle: ', middleFinger.angle)
-    # print('Index Finger angle: ', indexFinger.angle)
-    # print('Pinky Finger angle: ', pinkyFinger.angle)
-    # print(wrist.x, wrist.y, wrist.z)
-
-    return GestureObject.gesture
 
 
 def detectHand(hands, img, ASLModel):
@@ -49,8 +42,10 @@ def detectHand(hands, img, ASLModel):
 
 
 
-def detect_motion(last_frame, current_frame, threshold=50):
+def detect_motion(last_frame, current_frame, threshold=20):
     # Convert frames to grayscale
+    if last_frame is None:
+        last_frame = current_frame
     gray_last = cv2.cvtColor(last_frame, cv2.COLOR_BGR2GRAY)
     gray_current = cv2.cvtColor(current_frame, cv2.COLOR_BGR2GRAY)
 
@@ -68,71 +63,115 @@ def detect_motion(last_frame, current_frame, threshold=50):
     contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
 
     # Return True if contours are found
-    return len(contours) > 0
-
-def detection(cap,queue):
-    while True:
-        success, img = cap.read()
-        if not success:
-            return False, None
-        detected, frame = detectHand(hands,img, '')
-        print(detected)
-        if detected: queue.append(detected)
-        if len(queue) ==30 and len(set(queue))==1:
-            global firstGesture
-            firstGesture = set(queue).pop()
-            queue.clear()
-            return firstGesture
-        
-        
+    print('checking for motion', len(contours))
+    return len(contours) > 0, current_frame
+
+def toggle_light():
+    #action = "turn_on" if state else "turn_off"
+    url = f"http://localhost:8123/api/services/switch/toggle"
+    headers = {
+        "Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiIyOGU3ZDZmNTg5MjE0MzAxOWQwNTVjZWI5MThmYTcyMCIsImlhdCI6MTcxMjM0NDQ1MywiZXhwIjoyMDI3NzA0NDUzfQ.AXaP5ndD3QFtxhYxfXwT93x6qBh3GacCKmgiTHU6g7A", 
+        "Content-Type": "application/json",
+    }
+    data = {"entity_id": "switch.living_room_light_1"}
+    print('toggling light',data)
+    response = requests.post(url, json=data, headers=headers)
+    if response.status_code == 200:
+        # Get the new state of the light
+        time.sleep(1)
+        light_state = requests.get(f"http://localhost:8123/api/states/{data['entity_id']}", headers=headers).json()
+        return light_state['state'] == 'on' 
+    return None
+
+def black_image(img):
+    black_screen = np.zeros_like(img)
+    img = black_screen 
+    print('no motion')
+    ret, buffer = cv2.imencode('.jpg', img)
+    img = buffer.tobytes()
+    
+    yield (b'--frame\r\n'
+        b'Content-Type: image/jpeg\r\n\r\n' + img + b'\r\n')
 
 def gen_frames(cap): 
     inMotion = False
     last_frame = None
+    last_motion = None
+    global deviceStatus
+    global deviceChoice
+    #loop to keep the iterations of the model going 
     while True:
-        
         success, img = cap.read()
         if not success:
+            print('failed to read frame')
             break
+        
+        inMotion,last_frame = detect_motion(last_frame, img)
+        
+        if not inMotion: 
+            if last_motion and time.time()-last_motion > 2:
+                print('no motion detected, black screen being shown.')
+                img = np.zeros_like(img)
+                ret, buffer = cv2.imencode('.jpg', img)
+                img = buffer.tobytes()
+                yield (b'--frame\r\n'
+                    b'Content-Type: image/jpeg\r\n\r\n' + img + b'\r\n')
+                continue
         else:
-            
-            detected, frame = detectHand(hands,img, '')
-            print(detected)
-            if detected: firstQueue.append(detected)
-            if len(firstQueue) ==30 and len(set(firstQueue))==1:
-                print("first gesture detected")
-                global firstGesture
-                firstGesture = set(firstQueue).pop()
-                firstQueue.clear()
-                while True:
-                    print('made it into second loop')
-                    success, img = cap.read()
-                    if not success:
-                        break
-                    else:
-                        detected, frame = detectHand(hands,img, '')
-                    if detected: secondQueue.append(detected)
-                    
-                    if len(secondQueue)== 30 and len(set(secondQueue))==1:
-                        global secondGesture
-                        secondGesture = set(secondQueue).pop()
-                        print('both gestures are',firstGesture,secondGesture)
-                        time.sleep(2)
-                        firstGesture,secondGesture = 'No gesture detected','No gesture detected'
-                        secondQueue.clear()
-                        
-                        break
-                    ret, buffer = cv2.imencode('.jpg', img)
-                    img = buffer.tobytes()
-                    yield (b'--frame\r\n'
-                        b'Content-Type: image/jpeg\r\n\r\n' + img + b'\r\n')
-            #cv2.putText(img,'put text on the frame', (10,130), cv2.FONT_HERSHEY_PLAIN, 3, (100,50,100), 3)
-            ret, buffer = cv2.imencode('.jpg', img)
-            img = buffer.tobytes()
-            
-            yield (b'--frame\r\n'
-                   b'Content-Type: image/jpeg\r\n\r\n' + img + b'\r\n')
-
+            last_motion = time.time()
+        
+        detected, frame = detectHand(hands,img, '')
+        print(detected)
+        if detected: firstQueue.append(detected)
+        if len(firstQueue) ==30 and len(set(firstQueue))==1 and set(firstQueue).pop() != 'No gesture detected':
+            print("first gesture detected")
+            global firstGesture
+            firstGesture = set(firstQueue).pop()
+            firstQueue.clear()
+
+            while True:
+                print('made it into second loop')
+                success, img = cap.read()
+                if not success:
+                    break
+                else:
+                    detected, frame = detectHand(hands,img, '')
+                if detected: secondQueue.append(detected)
+                
+                if len(secondQueue)== 30 and len(set(secondQueue))==1 and set(secondQueue).pop() != 'No gesture detected':
+                    global secondGesture
+                    secondGesture = set(secondQueue).pop()
+                    print('both gestures are',firstGesture,secondGesture)
+                    if firstGesture == 'thumbs up' and secondGesture == 'thumbs up':
+                        print('both thumbs up detected')
+                        deviceChoice = 'light'
+                        print('device choice is', deviceChoice)
+                        try:   
+                            lightState = toggle_light()
+                            if lightState is True:
+                                deviceStatus = 'on'
+                            elif lightState is False:
+                                deviceStatus = 'off'
+                            print('Device Status is', deviceStatus)
+                        except:
+                            print('toggle light didnt work')
+                    time.sleep(5)
+                    deviceChoice, deviceStatus = 'N/A','N/A'
+                    firstGesture,secondGesture = 'No gesture detected','No gesture detected'
+                    secondQueue.clear()
+                    break
+                
+                #writing the image in the second gesture loop, shouldn't be changed
+                ret, buffer = cv2.imencode('.jpg', img)
+                img = buffer.tobytes()
+                yield (b'--frame\r\n'
+                    b'Content-Type: image/jpeg\r\n\r\n' + img + b'\r\n')
+        
+        #writing the image in the first gesture loop, shouldn't be changed
+        ret, buffer = cv2.imencode('.jpg', img)
+        img = buffer.tobytes()
+        yield (b'--frame\r\n'
+                b'Content-Type: image/jpeg\r\n\r\n' + img + b'\r\n')
 
 app = Flask(__name__)
 #comment this out if mediapipe doesnt work
@@ -144,8 +183,8 @@ def gen_frames(cap):
 #until here
 #comment the next line in if mediapipe doesn't work
 #hands = ""
-latest_gesture = 'No gesture detected'
-firstGesture, secondGesture = 'No gesture detected yet','No gesture detected'
+
+latest_gesture, firstGesture, secondGesture = 'No gesture detected yet','No gesture detected','No gesture detected'
 firstQueue,secondQueue = deque(maxlen=30),deque(maxlen=30)
 
 
@@ -155,16 +194,17 @@ def video_feed():
     return Response(gen_frames(cv2.VideoCapture(0)),
                     mimetype='multipart/x-mixed-replace; boundary=frame')
 
+
+    
 @app.route('/')
 def index():
-    """Video streaming home page."""
+    
     return render_template('index.html')
 
 @app.route('/current_gesture')
 def current_gesture():
-    return jsonify(gesture=latest_gesture, firstGesture = firstGesture, secondGesture = secondGesture)
+    
+    return jsonify(gesture=latest_gesture, firstGesture = firstGesture, secondGesture = secondGesture, deviceChoice=deviceChoice, deviceStatus=deviceStatus)
 
 if __name__ == "__main__":
-    
-    
-    app.run(debug=True) 
+    app.run(debug=True) 

app/methods.py

@@ -74,6 +74,22 @@ def calculate_angle(self,p1, p2, p3):
 
 
 
+def thumbClassifier(results):
+    res=results.multi_hand_landmarks[0].landmark
+    GestureObject = hand(results.multi_hand_landmarks[0])
+    
+    # print('Thumb angle: ', thumb.angle)
+    # print('Ring Finger angle: ', ringFinger.angle)
+    # print('Middle Finger angle: ', middleFinger.angle)
+    # print('Index Finger angle: ', indexFinger.angle)
+    # print('Pinky Finger angle: ', pinkyFinger.angle)
+    # print(wrist.x, wrist.y, wrist.z)
+
+    return GestureObject.gesture
+
+
+
+    
 def createSquare(results, img):
     h, w, c = img.shape
     min_x, min_y = w, h

app/readme.md

@@ -4,15 +4,17 @@
 
 *To run this project, you will most likely need to be in the Professor's office, as our demo will be using our Raspberry Pi device.*
 1. The first step of running our project is to turn on and connect the Raspberry Pi to the wifi, screen, camera, and light. *If you're using our setup, you only need to connect to the wifi, and connect the light to any power source through a micro-USB. Follow the steps outlined [here](#connecting-the-pi-and-the-light-to-the-same-network) to connect*
-2. Once the Raspberry Pi is connected and configured, navigate to the `app` subfolder of the `Project-Intelligest-Smart-Home` repository.
+2. In the root directory, enable the virtual environment with `source venv/bin/activate`
+3. In the `Models` directory, install the requirements with `pip install -r requirements.txt`. *You may need to install additional requirements that don't install due to unknown issues with the requirments file, manually install them with `pip install opencv-python`, `pip install flask`, `pip install requests`, `pip install mediapipe`*
+5. Once the Raspberry Pi is connected and configured, navigate to the `app` subfolder of the `Project-Intelligest-Smart-Home` repository.
 This can be done using the `cd` command if you are using the terminal and have opened the repository
-3. Once in the `app` subfolder, run the command `python -m flask run` to start the program. The program should take a few seconds to start up.
-4. Once the program has output the link to the flask server, open the link in a web browser.
-5. The web page should display a live feed from the camera, and the program should be running.
-6. To test the program, make a thumbs up, thumbs down, and thumbs flat gesture to make sure the model is correctly detecting gestures.
-7. After this test, hold your index finger up for the camera until the gesture is detected as your first gesture.
-8. Next, hold a thumbs-up gesture in front of the camera. After this gesture has been held for a few seconds, the web page should show the gestures you used.
-9. Once these gestures are detected, the light should turn on through the home assistant API.
+6. Once in the `app` subfolder, run the command `python -m flask run` to start the program. The program should take a few seconds to start up.
+7. Once the program has output the link to the flask server, open the link in a web browser and enter full-screen mode.
+8. The web page should display a live feed from the camera, and the program should be running.
+9. To test the program, make a thumbs up, thumbs down, and thumbs flat gesture to make sure the model is correctly detecting gestures.
+10. After this test, hold your index finger up for the camera until the gesture is detected as your first gesture.
+11. Next, hold a thumbs-up gesture in front of the camera. After this gesture has been held for a few seconds, the web page should show the gestures you used.
+12. Once these gestures are detected, the light should turn on through the home assistant API.
 
 ## Connecting the Pi and the Light to the Same Network
 With Home Assistant, any devices in the system _must_ be connected to the same network as the device running Home Assistant. For demo and testing purposes, you will need to host this network yourself.

app/templates/index.html

@@ -76,10 +76,11 @@
                 <p>Current Gesture: <span id="currentGesture">Waiting...</span></p>
                 <p>First Gesture: <span id="firstGesture">Waiting...</span></p>
                 <p>Second Gesture: <span id="secondGesture">Waiting...</span></p>
+                <p>Device Choice: <span id="deviceChoice">N/A</p>
+                <p>Device Status: <span id="deviceStatus">N/A</p>
             </div>
         </div>
 
-        <!-- Blocks Container (already correctly styled) -->
         <div class="blocks-container">
             <div class="block">
                 <img src="{{ url_for('static', filename='images/light.png') }}" alt="Light">
@@ -108,6 +109,8 @@
                 document.getElementById('currentGesture').textContent = data.gesture || 'No gesture detected yet';
                 document.getElementById('firstGesture').textContent = data.firstGesture || 'No gesture detected yet';
                 document.getElementById('secondGesture').textContent = data.secondGesture || 'No gesture detected yet';
+                document.getElementById('deviceChoice').textContent = data.deviceChoice || 'N/A';
+                document.getElementById('deviceStatus').textContent = data.deviceStatus || 'N/A';
 
             });
     }, 1000);

app/toDoList.md

@@ -1,6 +1,6 @@
-Make images of hands into buttons for the instructions(light, weather etc)
-Make sure web page loads and looks okay on raspberry pi screen
-Add in method to turn on and off light based on first and second gesture.
-Get the available devices displayed in a grid format after the first gesture. Must have for demo after this one
-Make html look more official, use css to make it look more like home assistant
-
+- Make images of hands into buttons for the instructions(light, weather etc)
+- Make sure web page loads and looks okay on raspberry pi screen
+- Add in method to turn on and off light based on first and second gesture.
+- Get the available devices displayed in a grid format after the first gesture. Must have for demo after this one
+- Make html look more official, use css to make it look more like home assistant
+- Make screen go dark when inmotion is false. Add a different and better way to