Smile detection in OpenCV

Detecting smiles in images or video streams can be achieved using OpenCV in conjunction with pre-trained Haar cascades. Haar cascades are machine-learning based classifiers that are trained to detect specific objects in images. OpenCV provides several trained Haar cascades, including ones for detecting faces, eyes, and smiles.

In this tutorial, we'll implement a simple smile detection algorithm using OpenCV.

Steps to Detect Smile using OpenCV:

1. Setup:

Ensure you have OpenCV installed:

pip install opencv-python

Import necessary libraries:

import cv2

2. Load Haar Cascades:

OpenCV provides Haar cascades for detecting faces and smiles. Load them using the CascadeClassifier class:

face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
smile_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_smile.xml')

3. Detect Smiles:

Define a function that takes an image, detects faces first, and then detects smiles within those faces:

def detect_smile(img):
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    faces = face_cascade.detectMultiScale(gray, 1.3, 5)

    for (x,y,w,h) in faces:
        cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)
        roi_gray = gray[y:y+h, x:x+w]
        roi_color = img[y:y+h, x:x+w]
        
        smiles = smile_cascade.detectMultiScale(roi_gray, 1.8, 20)
        
        for (sx, sy, sw, sh) in smiles:
            cv2.rectangle(roi_color, (sx, sy), (sx+sw, sy+sh), (0, 255, 0), 2)

    return img

4. Apply Smile Detection to Images or Video Stream:

For an image:

img = cv2.imread('path_to_image.jpg')
result = detect_smile(img)
cv2.imshow('Smile Detection', result)
cv2.waitKey(0)
cv2.destroyAllWindows()

For a video stream (e.g., from a webcam):

cap = cv2.VideoCapture(0)

while True:
    ret, frame = cap.read()
    if not ret:
        break

    result = detect_smile(frame)
    
    cv2.imshow('Smile Detection', result)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()

Note:

The parameters for the detectMultiScale function (like the scaling factor and neighbors) might need adjustments depending on your specific use-case.
Haar cascades can have false positives or false negatives, and they may not work as effectively in complex real-world scenarios. For more robust solutions, consider deep learning-based methods like CNNs with datasets specifically prepared for smile detection.

Smile detection with OpenCV in Python:

import cv2

# Load the pre-trained Haar cascade for smile detection
smile_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_smile.xml')

# Read an image
img = cv2.imread('face_smile.jpg')
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

# Detect smiles in the image
smiles = smile_cascade.detectMultiScale(gray, scaleFactor=1.8, minNeighbors=20)

# Draw rectangles around the detected smiles
for (x, y, w, h) in smiles:
    cv2.rectangle(img, (x, y), (x+w, y+h), (0, 255, 0), 2)

# Display the result
cv2.imshow('Smile Detection', img)
cv2.waitKey(0)
cv2.destroyAllWindows()

Smile detection in real-time video with OpenCV:

import cv2

smile_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_smile.xml')
cap = cv2.VideoCapture(0)  # Use webcam, or replace with video path

while True:
    ret, frame = cap.read()
    if not ret:
        break

    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    smiles = smile_cascade.detectMultiScale(gray, scaleFactor=1.8, minNeighbors=20)

    for (x, y, w, h) in smiles:
        cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)

    cv2.imshow('Smile Detection', frame)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()

Smile detection using machine learning in OpenCV: For machine learning-based smile detection, you can use a pre-trained model. Here's an example using the Dlib library:

import cv2
import dlib

detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')
smile_model = dlib.shape_predictor('smile_predictor.dat')

cap = cv2.VideoCapture(0)  # Use webcam, or replace with video path

while True:
    ret, frame = cap.read()
    if not ret:
        break

    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    faces = detector(gray)

    for face in faces:
        landmarks = predictor(gray, face)
        smile = smile_model(gray, landmarks)

        if smile > 0.5:  # Adjust the threshold as needed
            # Draw rectangle or perform other actions

    cv2.imshow('Smile Detection', frame)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()

Smile detection using deep learning in OpenCV: You can use pre-trained deep learning models for facial expression recognition. For example, using a model from the OpenCV deep learning module:

import cv2

net = cv2.dnn.readNetFromTensorflow('opencv_face_detector_uint8.pb', 'opencv_face_detector.pbtxt')
smile_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_smile.xml')

cap = cv2.VideoCapture(0)  # Use webcam, or replace with video path

while True:
    ret, frame = cap.read()
    if not ret:
        break

    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # Use deep learning for face detection
    faces = cv2.dnn.blobFromImage(gray, 1.0, (300, 300), [104, 117, 123], False, False)
    net.setInput(faces)
    detections = net.forward()

    for i in range(detections.shape[2]):
        confidence = detections[0, 0, i, 2]
        if confidence > 0.5:  # Adjust the confidence threshold
            box = detections[0, 0, i, 3:7] * np.array([frame.shape[1], frame.shape[0], frame.shape[1], frame.shape[0]])
            (x, y, w, h) = box.astype(int)

            # Perform smile detection within the detected face region
            roi_gray = gray[y:y+h, x:x+w]
            smiles = smile_cascade.detectMultiScale(roi_gray, scaleFactor=1.8, minNeighbors=20)

            for (sx, sy, sw, sh) in smiles:
                cv2.rectangle(frame, (x+sx, y+sy), (x+sx+sw, y+sy+sh), (0, 255, 0), 2)

    cv2.imshow('Smile Detection', frame)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()