wrote note detection code, changed contour to ellipse -- need to tune…

… for monochrome

src/main/python/calibrate_hsv.py

@@ -0,0 +1,65 @@
+import cv2
+import numpy as np
+
+"""interactive tool for finding appropriate HSV color range to detect orange objects via a video stream
+    key: green represents a contour that has been found, blue represents the biggest contour
+    - try to get it so contours only appear around notes"""
+
+# creates window to adjust the lower and upper bounds
+cv2.namedWindow("Trackbars", cv2.WINDOW_NORMAL)  # Use WINDOW_NORMAL to allow resizing
+cv2.resizeWindow("Trackbars", 600, 300)  # Set the size of the window (width, height)
+
+# creates trackbars
+cv2.createTrackbar("Hue Lower", "Trackbars", 1, 179, lambda x: None)
+cv2.createTrackbar("Saturation Lower", "Trackbars", 80, 255, lambda x: None)
+cv2.createTrackbar("Value Lower", "Trackbars", 130, 255, lambda x: None)
+cv2.createTrackbar("Hue Upper", "Trackbars", 6, 179, lambda x: None)
+cv2.createTrackbar("Saturation Upper", "Trackbars", 255, 255, lambda x: None)
+cv2.createTrackbar("Value Upper", "Trackbars", 255, 255, lambda x: None)
+
+cap = cv2.VideoCapture(0)
+
+while True:
+    ret, frame = cap.read()
+
+    if ret:
+        # convert frame from BGR to HSV color space
+        hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
+
+        # get current trackbar positions
+        hue_lower = cv2.getTrackbarPos("Hue Lower", "Trackbars")
+        saturation_lower = cv2.getTrackbarPos("Saturation Lower", "Trackbars")
+        value_lower = cv2.getTrackbarPos("Value Lower", "Trackbars")
+        hue_upper = cv2.getTrackbarPos("Hue Upper", "Trackbars")
+        saturation_upper = cv2.getTrackbarPos("Saturation Upper", "Trackbars")
+        value_upper = cv2.getTrackbarPos("Value Upper", "Trackbars")
+
+        # define lower and upper bounds for orange color in HSV
+        lower_orange = np.array([hue_lower, saturation_lower, value_lower])
+        upper_orange = np.array([hue_upper, saturation_upper, value_upper])
+
+        # threshold the HSV image to get only orange colors
+        mask = cv2.inRange(hsv, lower_orange, upper_orange)
+
+        # find contours in the mask
+        contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+        # find the largest contour (clump) of orange pixels
+        if contours:
+            # draws everything else it's detecting
+            cv2.drawContours(frame, contours, -1, [0, 255, 0], 1)
+            # gets the largest contour and draws it on
+            largest_contour = max(contours, key=cv2.contourArea)
+            cv2.drawContours(frame, [largest_contour], 0, [255, 0, 0], 2)
+
+        cv2.imshow("frame", frame)
+
+        # break when 'q' pressed
+        if cv2.waitKey(1) & 0xFF == ord("q"):
+            break
+    else:
+        print("Error: Unable to capture frame")
+        break
+
+cap.release()
+cv2.destroyAllWindows()

src/main/python/detector.py

@@ -5,44 +5,122 @@
 import skimage.color
 import time
 
+# define orange hsv values (opencv range is h:0-180, s:0-255, v:0-255)
+# NOTE: BELOW VALS REPRESENT RED ACTUALLY BC I ONLY HAD A RED SPONGE TO USE 
+LOWER_ORANGE_HSV = np.array([3, 80, 80])
+UPPER_ORANGE_HSV = np.array([6, 255, 255])
+
+LOW_THRESHOLD = 90
+HIGH_THRESHOLD = 150
+
+
+# orange hsv values
+# LOWER_ORANGE_HSV = np.array([3, 80, 80])
+# UPPER_ORANGE_HSV = np.array([6, 255, 255])
+
+# minimum contour area to detect a note
+MINIMUM_CONTOUR_AREA = 400
+# threshold for a contour to be considered a disk
+CONTOUR_DISK_THRESHOLD = 0.9
+
 class Detector:
 
     def __init__(self):
         pass
 
+    # TODO purpose of this function?
     def bgr_to_rgb(self, image):
         return image[:,:,::-1]
+
+    def find_largest_orange_contour(self, grayscale_image: np.ndarray) -> np.ndarray:
+        """
+        finds the largest orange contour in an HSV image
+        input: hsv image (np array)
+        output: largest contour (np array)
+        """
+        # threshold the HSV image to filter only orange, creates a binary mask - white 255, black 0
+        """NOTE: to threshold with monochrome image (single-channel, grayscale) to create a binary mask, 
+                can specify a SINGLE scalar value for lower/upper bounds
+                returns: binary image (single-channel, 8-bit)"""
+        #mask = cv2.inRange(grayscale_image, LOW_THRESHOLD, HIGH_THRESHOLD)
+        orange_mask = np.where(LOW_THRESHOLD < grayscale_image, 255, 0).astype(np.uint8)
+
+        # displays cv2 video stream 
+        cv2.imshow('after thresholding', orange_mask)
+
+        # find contours in the mask:  (* ignore second return value *)
+        # cv2.RETR_EXTERNAL retrieves external contours only
+        # cv2.CHAIN_APPROX_SIMPLE compresses horizontal, vertical, and diagonal segments and leaves only their end points 
+        # input has to be binary image (CV_8UC1)
+
+        contours, _ = cv2.findContours(orange_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
 
-    def detectGameElement(self, frame, objectsToDetect: list):
+        # returns contour with max area
+        # if contours:
+        #     # draws everything else it's detecting
+        #     for i in range(len(contours)):
+        #         if len(contours[i]) >= 5:
+        #             print("contours with greater than 5 points")
+        #             cv2.drawContours(orange_mask, contours, -1, [0, 255, 0], 1)
+        #             # gets the largest contour and draws it on
+        #             largest_contour = max(contours, key=cv2.contourArea)
+        #             cv2.drawContours(orange_mask, [largest_contour], 0, [255, 0, 0], 2)
 
-        results = dict(zip(objectsToDetect, [None for i in range(len(objectsToDetect))]))
+        return max(contours, key=cv2.contourArea)
 
-        for object in objectsToDetect:
+    def contour_is_note(self, contour: np.ndarray) -> bool:
+        """
+        checks if the contour is shaped like a note
+        input: contour (np array)
+        output: if contour is a ring (boolean)
+        """
+        # makes sure the contour isn't some random small spec of noise
+        if cv2.contourArea(contour) < MINIMUM_CONTOUR_AREA:
+            return False
+
+        # gets the convex hull: smallest convex polygon that can fit around the contour
+        contour_hull = cv2.convexHull(contour)
+
+        # fits an ellipse to the hull, and gets its area
+        ellipse = cv2.fitEllipse(contour_hull)
+
+        # area formula: pi * semi-major axis * semi-minor axis
+        best_fit_ellipse_area = np.pi * (ellipse[1][0] / 2) * (ellipse[1][1] / 2)
+
+        """compares area of the hull to area of the best-fit ellipse, if the ratio is greater than a certain threshold, 
+        returns true & indicates that the contour is likely shaped like a note"""
+        return cv2.contourArea(contour_hull) / best_fit_ellipse_area > CONTOUR_DISK_THRESHOLD
+
+    # def detectGameElement(self, frame, objectsToDetect: list):
+
+    #     results = dict(zip(objectsToDetect, [None for i in range(len(objectsToDetect))]))
+
+    #     for object in objectsToDetect:
 
-            #The following three functinos edits the mask in order to remove potential discrepencies in the frame
-            kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (12, 12))
-            morph = cv2.morphologyEx(frame, cv2.MORPH_CLOSE, kernel)
-            # mask = cv2.medianBlur(mask, 5)
+    #         #The following three functinos edits the mask in order to remove potential discrepencies in the frame
+    #         kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (12, 12))
+    #         morph = cv2.morphologyEx(frame, cv2.MORPH_CLOSE, kernel)
+    #         #blurred = cv2.GaussianBlur(mask, 5)
 
 
 
-            #The below code runs to detect if there is a ring in the given frame.
-            if (object == "RING"):
-                contours, hier = cv2.findContours(morph, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
-                contours = sorted(contours, key=cv2.contourArea)
-                contours = [contour for contour in contours if contour.size > 1000]
-                #last detection is supposed be the biggest because of the sorting function above
-                if (len(contours) > 0):
-                    tx,ty,tw,th = cv2.boundingRect(contours[len(contours) -1])
-                    cv2.rectangle(frame, (tx, ty), (tx + tw, ty + th),
-                                         (0, 0, 255), 2)
-
-        if (len(contours) > 0):
-            results[object] = Target(contours[len(contours) -1], object)
-            return results
-        return None
+    #         #The below code runs to detect if there is a ring in the given frame.
+    #         if (object == "RING"):
+    #             contours, hier = cv2.findContours(morph, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
+    #             contours = sorted(contours, key=cv2.contourArea)
+    #             contours = [contour for contour in contours if contour.size > 1000]
+    #             #last detection is supposed be the biggest because of the sorting function above
+    #             if (len(contours) > 0):
+    #                 tx,ty,tw,th = cv2.boundingRect(contours[len(contours) -1])
+    #                 cv2.rectangle(frame, (tx, ty), (tx + tw, ty + th),
+    #                                      (0, 0, 255), 2)
+
+    #     if (len(contours) > 0):
+    #         results[object] = Target(contours[len(contours) -1], object)
+    #         return results
+    #     return None
 
 
     def detectOrange(self, frame, threshold):
-        return np.where(frame > threshold, 255, frame)
+        return np.where(frame > threshold, 255, 0).astype(np.uint8)        
 

src/main/python/main.py

@@ -17,6 +17,8 @@
 input = VisionInput(FOV, RES, CAM_HEIGHT, CAM_ANGLE)
 cnt = 0
 p = 0
+
+
 while True:
     p = time.time()
     try:

src/main/python/test.py

@@ -2,48 +2,57 @@
 from detector import Detector
 import numpy as np
 import os
+import time
 
-#Showing output from arducam
-# open video0
-cap = cv2.VideoCapture(1)
-
-threshold = 40
+cap = cv2.VideoCapture(0)
 counter = 0
 
 while(True):
-    cap.set(cv2.CAP_PROP_AUTO_EXPOSURE, 1)
+    # turns off auto exposure (for mac: 0.25 is OFF, 0.75 is on)
+    #cap.set(cv2.CAP_PROP_AUTO_EXPOSURE, 0.25)
     # set exposure time
-    cap.set(cv2.CAP_PROP_EXPOSURE, -12)
-    # Capture frame-by-frame
+    # time.sleep(1)
+    #cap.set(cv2.CAP_PROP_EXPOSURE, -12)
+
     ret, frame = cap.read()
+    assert ret
+
+    # reduce noise
+    blurred_frame = cv2.GaussianBlur(frame, (5, 5), 0)
+
+    # import matplotlib.pyplot as plt
+    # plt.figure()
+    # # need a 2D array at least for an image
+    # plt.imshow(frame[:,:,0])
+    # #plt.imshow(frame)
+    # plt.show()
 
     # if cv2.waitKey(1) & 0xFF == ord('q'):
     #     break
     d = Detector()
-    frame = d.detectOrange(frame[:,:,0], threshold)
-    cv2.imshow('frame', frame)
 
-    results = d.detectGameElement(np.asarray(frame), ["RING"])
-    print(results)
+    # convert tuple from (height, width, # of channels) to just (height, width)
+    frame_single_channel = frame[:,:,0]
+
+    contour = d.find_largest_orange_contour(frame_single_channel)
+    #cv2.drawContours(frame, contour, 0, [255, 0, 0], 2)
+    if contour is not None and d.contour_is_note(contour):
+        print("ellipse error being thrown AFTER contour detected")
+        cv2.ellipse(frame, cv2.fitEllipse(contour), (255, 0, 255), 2)
+
+    # TODO: ALTER THRESHOLD VALS HERE - define orange based on intensity thresholds? (exposure, brightness..)
+    # orange_mask = d.detectOrange(frame_single_channel, threshold=100)
+    #cv2.imshow("frame", frame)
+
     key = cv2.waitKey(1)
-    # Check if 'a' key is pressed
-    #automated way of changing the values of threshold
-    '''
-    if key == ord('a'):
-        # Pause and allow the user to change the threshold value
-        threshold = int(input("Enter a new threshold value: "))
-        print("new threshold value: " + str(threshold))
-    elif key == ord('q'):  # Quit if 'q' key is pressed
-        break
-    '''
 
     filename = f"frame_"+ str(counter) + ".jpg"  # Customize filename format if needed
     counter +=1
     # Assuming your Desktop path, construct the path to save the image
-    image_path = os.path.join(os.path.expanduser('~'), 'Desktop', filename)
-    cv2.imwrite(image_path, frame)
+    # image_path = os.path.join(os.path.expanduser('~'), 'Desktop', filename)
+    # cv2.imwrite(image_path, frame)
 
-    print(f"Frame saved successfully to: {image_path}")
+    # print(f"Frame saved successfully to: {image_path}")
     if key == ord('q'):  # Quit if 'q' key is pressed
 
         break

src/main/python/vision_input.py

@@ -9,7 +9,9 @@ class VisionInput:
     def __init__(self, fov, res: tuple, height, angle):
         self.w = res[0]
         self.h = res[1]
+        # self.cap = cv2.VideoCapture(0)
         self.cap = cv2.VideoCapture(0)
+
         Target.FOV = fov
         Target.RES = res
         Target.CAM_HEIGHT = height
@@ -26,8 +28,13 @@ def getFrame(self):
         ret, frame = self.cap.read()
 
         if not ret:
-            print('frame malf')
-        exit
+            print('frame malf foo')
+            exit()
+
+        # from matplotlib import pyplot as plt
+        # plt.figure()
+        # plt.imshow(frame)
+        # plt.show()
 
         fr = cv2.resize(frame, (self.w, self.h), interpolation=cv2.INTER_AREA)
         return fr

src/main/python/webcam.py

@@ -0,0 +1,31 @@
+# import the necessary packages
+from threading import Thread
+import cv2
+
+class WebcamVideoStream:
+	def __init__(self, src=0):
+		# initialize the video camera stream and read the first frame
+		# from the stream
+		self.stream = cv2.VideoCapture(src)
+		(self.grabbed, self.frame) = self.stream.read()
+		# initialize the variable used to indicate if the thread should
+		# be stopped
+		self.stopped = False
+	def start(self):
+		# start the thread to read frames from the video stream
+		Thread(target=self.update, args=()).start()
+		return self
+	def update(self):
+		# keep looping infinitely until the thread is stopped
+		while True:
+			# if the thread indicator variable is set, stop the thread
+			if self.stopped:
+				return
+			# otherwise, read the next frame from the stream
+			(self.grabbed, self.frame) = self.stream.read()
+	def read(self):
+		# return the frame most recently read
+		return self.frame
+	def stop(self):
+		# indicate that the thread should be stopped
+		self.stopped = True