Merge pull request #26 from Capstone-Projects-2024-Spring/gesture-ticket

Gesture ticket

Author: pdarsh58

app/app.py

@@ -28,78 +28,75 @@ def thumbClassifier(results):
     return GestureObject.gesture
 
 
-
-def preprocessHandRegion(handRegion):
-    #resize the image to the same resolution used in the dataset
-    resized_hand = cv2.resize(handRegion, (224,224))
-    normalized_hand = resized_hand / 255.0
-    
-    reshaped_hand = np.reshape(normalized_hand, (224,224, 3))
-    batch_hand = np.expand_dims(reshaped_hand, axis=0)
-    return batch_hand
-
-
-
-def detectHand(hands, img, cTime, pTime, ASLModel, colors):
-    
+def detectHand(hands, img, ASLModel):
+    #comment this in if meidapipe doesnt work
+    #return "thumbs up", img
     gestureName=""
-    #success, img = cap.read()
-    cv2.putText(img, "looking for ASL gestures", (int(img.shape[1]/2),20), cv2.FONT_HERSHEY_PLAIN, 1, (255, 255, 255), 2)
-    if img is None:
-        print("empty camera frame!!!!!")
-        
+    if img is None: print("empty camera frame!!!!!")
+       
     results = hands.process(img)
     if results.multi_hand_landmarks:
-        #get the dimensions for the cropped image
+        
         minX,minY,maxX,maxY=createSquare(results,img)
-        # Draw the square bounding box
-        cv2.rectangle(img, (minX, minY), (maxX, maxY), (colors, colors, colors), 2)
-        if minX < maxX and minY < maxY:
-            handRegion = img[minY:maxY, minX:maxX]
-            #Preprocess the hand region for the ASL model
-            preprocessedHand = preprocessHandRegion(handRegion) 
-            #Predict the ASL gesture given by user
-            #asl_prediction = ASLModel.predict(preprocessedHand) 
-            asl_prediction = 1
-            #turning the gesture from clas number to real name and adding to video feed
-            #gestureName = "Detected Gesture: " + IdentifyGesture(np.argmax(asl_prediction)) 
-            gestureName = thumbClassifier(results)
-    cTime = time.time()
-    fps = 1/(cTime-pTime)
-    pTime = cTime
-    
-    #adding all the text before displaying the image
-    cv2.putText(img, gestureName, (10, 130), cv2.FONT_HERSHEY_PLAIN, 2, (colors, colors, colors), 2)
-    cv2.putText(img,str(int(fps)), (10,70), cv2.FONT_HERSHEY_PLAIN, 3, (colors,50,colors), 3)
-    #cv2.imshow("Image", img)
-    cv2.waitKey(1)
-    global latest_gesture  # Declare it as global inside the function if you're updating it
-    # Update this line accordingly in your detectHand function
+        cv2.rectangle(img, (minX, minY), (maxX, maxY), (155, 155, 155), 2)
+        gestureName = thumbClassifier(results)
+    else:
+        gestureName ='No gesture detected'
+    global latest_gesture 
     latest_gesture = gestureName  # gestureName is the detected gesture
-    return pTime, cTime, gestureName,img
+    return gestureName,img
+
 
-app = Flask(__name__)
-mpHands = mp.solutions.hands
-hands = mpHands.Hands(static_image_mode=False,
-                    max_num_hands=1,
-                    min_detection_confidence=0.5,
-                    min_tracking_confidence=0.5)
-latest_gesture = 'No gesture detected yet'
-firstGesture, secondGesture = 'No gesture detected yet','No gesture detected yet'
-firstQueue,secondQueue = deque(maxlen=30),deque(maxlen=30)
 
-# Your existing code modified for Flask will go here
 
+def detect_motion(last_frame, current_frame, threshold=50):
+    # Convert frames to grayscale
+    gray_last = cv2.cvtColor(last_frame, cv2.COLOR_BGR2GRAY)
+    gray_current = cv2.cvtColor(current_frame, cv2.COLOR_BGR2GRAY)
+    
+    # Apply Gaussian Blur to reduce noise and detail
+    gray_last = cv2.GaussianBlur(gray_last, (21, 21), 0)
+    gray_current = cv2.GaussianBlur(gray_current, (21, 21), 0)
+    
+    # Compute the absolute difference between the current frame and reference frame
+    frame_diff = cv2.absdiff(gray_last, gray_current)
+    
+    # Threshold the difference
+    _, thresh = cv2.threshold(frame_diff, threshold, 255, cv2.THRESH_BINARY)
+    
+    # Find contours to see if there are significant changes
+    contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    
+    # Return True if contours are found
+    return len(contours) > 0
 
-def gen_frames(cap):  # Generator function
+def detection(cap,queue):
     while True:
+        success, img = cap.read()
+        if not success:
+            return False, None
+        detected, frame = detectHand(hands,img, '')
+        print(detected)
+        if detected: queue.append(detected)
+        if len(queue) ==30 and len(set(queue))==1:
+            global firstGesture
+            firstGesture = set(queue).pop()
+            queue.clear()
+            return firstGesture
+        
+        
+
+def gen_frames(cap): 
+    inMotion = False
+    last_frame = None
+    while True:
+        
         success, img = cap.read()
         if not success:
             break
         else:
-            # Your hand detection and instructions code integrated here
-            # Instead of cv2.imshow, convert frame to bytes and yield
-            pTime,cTime, detected, frame = detectHand(hands,img, 0,0, '', 155)
+            
+            detected, frame = detectHand(hands,img, '')
             print(detected)
             if detected: firstQueue.append(detected)
             if len(firstQueue) ==30 and len(set(firstQueue))==1:
@@ -113,16 +110,20 @@ def gen_frames(cap):  # Generator function
                     if not success:
                         break
                     else:
-                        pTime,cTime, detected, frame = detectHand(hands,img, pTime,cTime, '', 155)
+                        detected, frame = detectHand(hands,img, '')
                     if detected: secondQueue.append(detected)
-                    ret, buffer = cv2.imencode('.jpg', img)
-                    img = buffer.tobytes()
+                    
                     if len(secondQueue)== 30 and len(set(secondQueue))==1:
                         global secondGesture
                         secondGesture = set(secondQueue).pop()
                         print('both gestures are',firstGesture,secondGesture)
-                        time.sleep(3)
-                        quit()
+                        time.sleep(2)
+                        firstGesture,secondGesture = 'No gesture detected','No gesture detected'
+                        secondQueue.clear()
+                        
+                        break
+                    ret, buffer = cv2.imencode('.jpg', img)
+                    img = buffer.tobytes()
                     yield (b'--frame\r\n'
                         b'Content-Type: image/jpeg\r\n\r\n' + img + b'\r\n')
             #cv2.putText(img,'put text on the frame', (10,130), cv2.FONT_HERSHEY_PLAIN, 3, (100,50,100), 3)
@@ -132,6 +133,22 @@ def gen_frames(cap):  # Generator function
             yield (b'--frame\r\n'
                    b'Content-Type: image/jpeg\r\n\r\n' + img + b'\r\n')
 
+
+app = Flask(__name__)
+#comment this out if mediapipe doesnt work
+mpHands = mp.solutions.hands
+hands = mpHands.Hands(static_image_mode=False,
+                    max_num_hands=1,
+                    min_detection_confidence=0.5,
+                    min_tracking_confidence=0.5)
+#until here
+#comment the next line in if mediapipe doesn't work
+#hands = ""
+latest_gesture = 'No gesture detected'
+firstGesture, secondGesture = 'No gesture detected yet','No gesture detected'
+firstQueue,secondQueue = deque(maxlen=30),deque(maxlen=30)
+
+
 @app.route('/video_feed')
 def video_feed():
     # Assuming 'cap' is your cv2.VideoCapture object
@@ -148,4 +165,6 @@ def current_gesture():
     return jsonify(gesture=latest_gesture, firstGesture = firstGesture, secondGesture = secondGesture)
 
 if __name__ == "__main__":
+    
+    
     app.run(debug=True) 

app/methods.py

@@ -1,5 +1,5 @@
 import math,cv2, time
-
+import numpy as np
 
 class hand:
     def __init__(self,hand):
@@ -143,6 +143,15 @@ def thumbClassifier(results):
     return GestureObject.gesture
 
 
+def preprocessHandRegion(handRegion):
+    #resize the image to the same resolution used in the dataset
+    resized_hand = cv2.resize(handRegion, (224,224))
+    normalized_hand = resized_hand / 255.0
+    
+    reshaped_hand = np.reshape(normalized_hand, (224,224, 3))
+    batch_hand = np.expand_dims(reshaped_hand, axis=0)
+    return batch_hand
+
 
 def InstructionCommand(hands, img, cTime, pTime,firstDetected):
     result = ""

app/readme.md

@@ -0,0 +1,29 @@
+# IntelliGest Smart Home
+
+## How to Run the Project
+
+*To run this project, you will most likely need to be in the Professor's office, as our demo will be using our Raspberry Pi device.*
+1. The first step of running our project is to turn on and connect the Raspberry Pi to the wifi, screen, camera, and light. *If you're using our setup, you only need to connect to the wifi, and connect the light to any power source through a micro-USB. Follow the steps outlined [here](#connecting-the-pi-and-the-light-to-the-same-network) to connect*
+2. Once the Raspberry Pi is connected and configured, navigate to the `app` subfolder of the `Project-Intelligest-Smart-Home` repository.
+This can be done using the `cd` command if you are using the terminal and have opened the repository
+3. Once in the `app` subfolder, run the command `python -m flask run` to start the program. The program should take a few seconds to start up.
+4. Once the program has output the link to the flask server, open the link in a web browser.
+5. The web page should display a live feed from the camera, and the program should be running.
+6. To test the program, make a thumbs up, thumbs down, and thumbs flat gesture to make sure the model is correctly detecting gestures.
+7. After this test, hold your index finger up for the camera until the gesture is detected as your first gesture.
+8. Next, hold a thumbs-up gesture in front of the camera. After this gesture has been held for a few seconds, the web page should show the gestures you used.
+9. Once these gestures are detected, the light should turn on through the home assistant API.
+
+## Connecting the Pi and the Light to the Same Network
+With Home Assistant, any devices in the system _must_ be connected to the same network as the device running Home Assistant. For demo and testing purposes, you will need to host this network yourself.
+We recommend utilizing a laptop to provide "shared internet". Both macOS and Windows have this feature.
+
+For minimal setup, configure the network as follows:
+- SSID (network name): "showthesign"
+- Password: "f334p8ofpehgb"
+- Channel: 1, 6, or 11 (if required, it *MUST* be one of these channels since the light communicates over 2.4GHz band)
+- WPA2/WPA3 Security
+
+With this configuration, the Raspberry Pi already recognizes this connection and so do the ESPs running the light. To configure your own network with your own name and password, first edit `livingroom.yaml` to include your chosen ssid and password and then you will need to reflash the ESP by connecting it to your laptop and running `esphome run livingroom.yaml` through the terminal. ***We highly recommend just setting up the internet share with the defaults we provide***
+
+Once both the ESP and the main device are connected to the same network, you are good to start using the project, following the steps outlined above!

app/templates/index.html

@@ -1,67 +1,111 @@
-<!DOCTYPE html>
+</html><!DOCTYPE html>
 <html>
 <head>
     <title>Video Stream</title>
     <style>
+        html, body {
+            margin: 0; /* Removes the margin from the body */
+            padding: 0; /* Removes padding from the body */
+            overflow: auto; /* Prevents scrolling */
+            height: 100%; /* Ensures the body takes full viewport height */
+        }
+    
         .content-container {
             display: flex;
-            justify-content: space-between;
-            align-items: flex-start;
+            font-size = 20px;
+            flex-direction: row; /* Adjusts direction to row to place video and blocks side by side */
+            align-items: start;
+            justify-content: center;
+            flex-wrap: nowrap; /* Prevents wrapping to ensure no scrolling */
+            height: 100vh; /* Sets the container height to fill the viewport */
         }
-
+    
         .video-container {
-            margin-right: 20px;
+            flex: 1; /* Allows the video container to grow and fill available space */
+            max-width: 40%; /* Limits the maximum width of the video container */
+            /* Adjusts margin and padding as needed */
+        }
+    
+        .blocks-container {
             display: flex;
             flex-direction: column;
+            justify-content: start;
+            margin-left: auto;
+            flex-grow: 0;
+            width: 20%;
+            height: 100vh;
+            overflow-y: hidden;
+            padding: 10px 0; /* Adds padding to the top and bottom */
         }
-
-        .all-gestures-container {
+        
+        .block {
+            margin: 5px 10px; /* Adjust margins for left and right */
             display: flex;
             flex-direction: column;
+            justify-content: space-between; /* Ensures space is distributed between img and text */
+            align-items: center;
+            flex-grow: 1; /* Allows block to grow as needed */
+            min-height: 100px; /* Sets a minimum height */
         }
-
-        .video-container img,
-        .all-gestures-container img {
-            width: auto;
+        
+        .block img {
+            width: auto; /* Allows width to adjust based on height */
+            max-height: 60%; /* Limits image height to leave space for text */
+            object-fit: contain; /* Ensures image is scaled properly without distortion */
         }
+        
+        .block p {
+            text-align: center;
+            margin-top: 5px; /* Ensures a small space between the image and the text */
+            flex-shrink: 0; /* Prevents text from shrinking */
+            width: 100%; /* Ensures text width matches block width */
+            overflow: hidden; /* Hides overflow text */
+            text-overflow: ellipsis; /* Adds an ellipsis to text that overflows */
+            white-space: nowrap; /* Keeps text in a single line */
+        }
+        /* Your existing .block and .block img styles */
     </style>
 </head>
 <body>
     <div class="content-container">
         <!-- Video Stream -->
-        <div class="video-container" justify-content= "left">
-            <h1>Video Stream</h1>
-            <img src="{{ url_for('video_feed') }}" alt="Video Feed" style="width: 500px;">
-            <!--<img src="{{ url_for('static', filename='images/all-gestures.png') }}" alt="Video Feed" style="width: 500px;">-->
-            <div id="gesture">Current Gesture: No gesture detected yet</div>
+        <div class="video-container">
+            <img src="{{ url_for('video_feed') }}" alt="Video Feed" style="width: 100%; max-width: 640px; height: auto;">
+            
             <div id="gestures">
+                <p>Current Gesture: <span id="currentGesture">Waiting...</span></p>
                 <p>First Gesture: <span id="firstGesture">Waiting...</span></p>
                 <p>Second Gesture: <span id="secondGesture">Waiting...</span></p>
             </div>
         </div>
 
         <!-- Blocks Container (already correctly styled) -->
-        <div class="blocks-container" justify-content= "right">
+        <div class="blocks-container">
             <div class="block">
-                <img src="{{ url_for('static', filename='images/all-gestures.png') }}" alt="Gestures" style="width: 500px;">
+                <img src="{{ url_for('static', filename='images/light.png') }}" alt="Light">
                 <p>Light Gesture</p>
             </div>
-            <!--<div class="block">
+            <div class="block">
                 <img src="{{ url_for('static', filename='images/news.png') }}" alt="News">
                 <p>News Gesture</p>
             </div>
             <div class="block">
                 <img src="{{ url_for('static', filename='images/weather.png') }}" alt="Weather">
                 <p>Weather Gesture</p>
-            </div> -->
+            </div>
+            <div class="block">
+                <img src="{{ url_for('static', filename='images/all-gestures.png') }}" alt="Gestures" style="width: 500px;">
+                <p>All Gestures</p>
+            </div>
+            
         </div>
     </div>
     <script>
     setInterval(function() {
         fetch('/current_gesture')
             .then(response => response.json())
             .then(data => {
-                document.getElementById('gesture').innerHTML = 'Current Gesture: ' + data.gesture;
+                document.getElementById('currentGesture').textContent = data.gesture || 'No gesture detected yet';
                 document.getElementById('firstGesture').textContent = data.firstGesture || 'No gesture detected yet';
                 document.getElementById('secondGesture').textContent = data.secondGesture || 'No gesture detected yet';
 

app/toDoList.md

@@ -0,0 +1,6 @@
+Make images of hands into buttons for the instructions(light, weather etc)
+Make sure web page loads and looks okay on raspberry pi screen
+Add in method to turn on and off light based on first and second gesture.
+Get the available devices displayed in a grid format after the first gesture. Must have for demo after this one
+Make html look more official, use css to make it look more like home assistant
+