Tag Archives: Artificial intelligence

In this exciting project, we combine a Raspberry Pi 5, the Fusion AI Lab Kit, a Pi Camera, and a remote IP camera to generate a stunning real-time composite video. Watch as a glowing, translucent MediaPipe face mesh of my face hovers magically over live video of the Mighty River Nice scenery captured by an IP camera. The effect looks futuristic and professional — perfect for creative video effects, interactive installations, or just blowing your mind with computer vision! Using Picamera2 for high-frame-rate local capture and OpenCV with an RTSP stream from the river camera, we process everything in real time. MediaPipe’s Face Mesh detects and tracks facial landmarks, which we draw as beautiful cyan/teal contours with glowing irises. Then we create a clean mask, separate the mesh foreground from the river background, and blend them seamlessly into one composite frame. You’ll see every debugging layer live on screen too — meshLayer, mask, inverted mask, riverBG, and meshFG — so you can understand exactly how the magic happens.This tutorial is beginner-to-intermediate friendly and packed with practical OpenCV + MediaPipe techniques you can adapt for your own augmented reality projects. Whether you’re a longtime follower of the Paul McWhorter channel or new to the Fusion AI Kit, you’ll walk away inspired and ready to build your own hovering effects, overlays, or interactive displays.Grab the full code from the video description, fire up your Pi 5, and start creating jaw-dropping computer vision projects today. Drop a comment and let me know what you’d like to overlay next — another face mesh, hand tracking, or something completely different? Let’s keep pushing the limits of what we can do with affordable AI hardware!

import cv2
import time
from picamera2 import Picamera2
import mediapipe as mp
import numpy as np
import secret
W=1280
H=720
tStart = time.time()
fps = 0

piCam = Picamera2(1)
piCam.preview_configuration.main.size = (W, H)
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate = 60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

cam = cv2.VideoCapture(secret.RTSP1,cv2.CAP_FFMPEG)
cam.set(cv2.CAP_PROP_BUFFERSIZE,0)
cam.set(cv2.CAP_PROP_POS_FRAMES,0)

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(
    
    max_num_faces=1,
    refine_landmarks=True,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)

cv2.namedWindow('Composite',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('Composite',W,H)
cv2.moveWindow('Composite',0,65)

cv2.namedWindow('meshLayer',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('meshLayer',320,180)
cv2.moveWindow('meshLayer',0,H+65)

cv2.namedWindow('grayLayer',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('grayLayer',320,180)
cv2.moveWindow('grayLayer',320,H+65)

cv2.namedWindow('maskLayer',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('maskLayer',320,180)
cv2.moveWindow('maskLayer',320*2,H+65)

cv2.namedWindow('invLayer',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('invLayer',320,180)
cv2.moveWindow('invLayer',320*3,H+65)

cv2.namedWindow('riverBG',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('riverBG',320,180)
cv2.moveWindow('riverBG',W,65)

cv2.namedWindow('meshFG',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('meshFG',320,180)
cv2.moveWindow('meshFG',W,180+65+20)
while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame = piCam.capture_array()
    frame = cv2.flip(frame, -1)
    
    _, river = cam.read()
    river = cv2.resize(river,(W,H))
    meshLayer = frame.copy()
    meshLayer[0:H-1,0:W-1] = [255,255,255]
    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = faceMesh.process(rgb)

    if results.multi_face_landmarks:
        for faceLandmarks in results.multi_face_landmarks:
            # Optional second pass for glow effect
            mp.solutions.drawing_utils.draw_landmarks(
                image=meshLayer,
                landmark_list=faceLandmarks,
                connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,
                landmark_drawing_spec=None,
                connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
                    color=(0, 150, 200),
                    thickness=2
                )
            )
            # === Best looking translucent mesh ===
            mp.solutions.drawing_utils.draw_landmarks(
                image=meshLayer,
                landmark_list=faceLandmarks,
                connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,
                landmark_drawing_spec=None,                    # Hide dots
                connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
                    color=(0, 0, 180),      # Bright cyan/teal
                    thickness=7
                )
            )

            # Optional: Extra detail on irises (makes eyes look better)
            mp.solutions.drawing_utils.draw_landmarks(
                image=meshLayer,
                landmark_list=faceLandmarks,
                connections=mp.solutions.face_mesh.FACEMESH_IRISES,
                landmark_drawing_spec=None,
                connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
                    color=(255, 0, 0),
                    thickness=7
                )
            )
    gray = cv2.cvtColor(meshLayer,cv2.COLOR_BGR2GRAY)

    _ , mask =cv2.threshold(gray, 250, 255, cv2.THRESH_BINARY)
    invLayer = cv2.bitwise_not(mask)
    riverBG =cv2.bitwise_and(river, river, mask = mask)
    meshFG = cv2.bitwise_and(meshLayer,meshLayer, mask=invLayer)
    composite = cv2.add(riverBG,meshFG)
    
    myText = "FPS: "+str(round(fps,1))
    cv2.putText(composite,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

    cv2.imshow("Composite", composite)
    cv2.imshow("meshLayer",meshLayer)
    cv2.imshow("grayLayer",gray)
    cv2.imshow("maskLayer",mask)
    cv2.imshow("invLayer",invLayer)
    cv2.imshow("riverBG",riverBG)
    cv2.imshow("meshFG",meshFG)
    if cv2.waitKey(1) == ord('q'):
        break

cv2.destroyAllWindows()
piCam.stop()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

import cv2

import time

from picamera2 import Picamera2

import mediapipe as mp

import numpy as np

import secret

W=1280

H=720

tStart = time.time()

fps = 0

piCam = Picamera2(1)

piCam.preview_configuration.main.size = (W, H)

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate = 60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

cam = cv2.VideoCapture(secret.RTSP1,cv2.CAP_FFMPEG)

cam.set(cv2.CAP_PROP_BUFFERSIZE,0)

cam.set(cv2.CAP_PROP_POS_FRAMES,0)

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(

max_num_faces=1,

refine_landmarks=True,

min_detection_confidence=0.5,

min_tracking_confidence=0.5)

cv2.namedWindow('Composite',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('Composite',W,H)

cv2.moveWindow('Composite',0,65)

cv2.namedWindow('meshLayer',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('meshLayer',320,180)

cv2.moveWindow('meshLayer',0,H+65)

cv2.namedWindow('grayLayer',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('grayLayer',320,180)

cv2.moveWindow('grayLayer',320,H+65)

cv2.namedWindow('maskLayer',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('maskLayer',320,180)

cv2.moveWindow('maskLayer',320*2,H+65)

cv2.namedWindow('invLayer',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('invLayer',320,180)

cv2.moveWindow('invLayer',320*3,H+65)

cv2.namedWindow('riverBG',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('riverBG',320,180)

cv2.moveWindow('riverBG',W,65)

cv2.namedWindow('meshFG',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('meshFG',320,180)

cv2.moveWindow('meshFG',W,180+65+20)

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame = piCam.capture_array()

frame = cv2.flip(frame, -1)

_, river = cam.read()

river = cv2.resize(river,(W,H))

meshLayer = frame.copy()

meshLayer[0:H-1,0:W-1] = [255,255,255]

rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

results = faceMesh.process(rgb)

if results.multi_face_landmarks:

for faceLandmarks in results.multi_face_landmarks:

# Optional second pass for glow effect

mp.solutions.drawing_utils.draw_landmarks(

image=meshLayer,

landmark_list=faceLandmarks,

connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,

landmark_drawing_spec=None,

connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

color=(0, 150, 200),

thickness=2

)

# === Best looking translucent mesh ===

mp.solutions.drawing_utils.draw_landmarks(

image=meshLayer,

landmark_list=faceLandmarks,

connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,

landmark_drawing_spec=None, # Hide dots

connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

color=(0, 0, 180), # Bright cyan/teal

thickness=7

)

# Optional: Extra detail on irises (makes eyes look better)

mp.solutions.drawing_utils.draw_landmarks(

image=meshLayer,

landmark_list=faceLandmarks,

connections=mp.solutions.face_mesh.FACEMESH_IRISES,

landmark_drawing_spec=None,

connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

color=(255, 0, 0),

thickness=7

)

gray = cv2.cvtColor(meshLayer,cv2.COLOR_BGR2GRAY)

_ , mask =cv2.threshold(gray, 250, 255, cv2.THRESH_BINARY)

invLayer = cv2.bitwise_not(mask)

riverBG =cv2.bitwise_and(river, river, mask = mask)

meshFG = cv2.bitwise_and(meshLayer,meshLayer, mask=invLayer)

composite = cv2.add(riverBG,meshFG)

myText = "FPS: "+str(round(fps,1))

cv2.putText(composite,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Composite", composite)

cv2.imshow("meshLayer",meshLayer)

cv2.imshow("grayLayer",gray)

cv2.imshow("maskLayer",mask)

cv2.imshow("invLayer",invLayer)

cv2.imshow("riverBG",riverBG)

cv2.imshow("meshFG",meshFG)

if cv2.waitKey(1) == ord('q'):

break

cv2.destroyAllWindows()

piCam.stop()

AI On the Edge, Raspberry Pi

AI on the Edge LESSON 41: Creating FaceMesh Using MediaPipe in OpenCV

August 13, 2026 admin

In this project, I demonstrate how to create a smooth, real-time face mesh overlay using the Raspberry Pi 5, the official Pi Camera, MediaPipe, and OpenCV. The program captures live video from the camera and draws a detailed, colorful mesh that follows every movement of the face with high accuracy. The result is a visually appealing augmented reality-style effect that runs efficiently even on a single-board computer.

The goal of this project is to build a responsive face tracking system that detects and draws 468 facial landmarks in real time. This creates a striking mesh that highlights the contours of the face, eyes, lips, and jawline, making it an excellent foundation for more advanced computer vision projects like virtual filters, AR effects, or interactive installations.

The program follows a straightforward but efficient real-time vision pipeline. First, it initializes the Raspberry Pi Camera using the modern picamera2 library, configured for 1280×720 resolution at 60 frames per second. It then sets up MediaPipe’s Face Mesh solution with landmark refinement enabled for better eye tracking.

In the main loop, the program continuously grabs a frame from the camera, corrects its orientation, and converts it from BGR to RGB format since MediaPipe expects RGB input. The frame is then passed to the Face Mesh model for processing. When a face is detected, the program draws multiple layers of graphics on top of the image: a fine tesselation mesh across the entire face, thick and vibrant contours around the major facial features, and special highlighting on the irises. Finally, the processed frame is displayed in an OpenCV window, creating a smooth and engaging real-time visualization.

This approach works particularly well on the Raspberry Pi 5 because it balances visual quality with performance. By limiting detection to a single face and using efficient drawing methods, the application maintains high frame rates while producing a professional-looking result. The multi-layer drawing technique (tesselation + contours + irises) gives the mesh depth and visual appeal that single-pass drawings often lack.

The project makes use of several powerful technologies: picamera2 for fast camera access, Google’s MediaPipe for high-speed machine learning-based landmark detection, OpenCV for image handling and display, and NumPy for efficient array operations.

This face mesh project serves as an excellent stepping stone into real-time AI and computer vision on embedded hardware. Once you have the basic mesh working, it becomes much easier to expand into creative applications such as face filters, gesture recognition, or overlaying the mesh onto other video sources.

The code developed in the video lesson is presented below:

# ====================================================================
# DISCLAIMER:
# This code is provided as-is for educational and experimental 
# purposes only. The author makes no representations or warranties of 
# any kind concerning the safety, suitability, or accuracy of this 
# code. Use at your own risk. The author assumes no liability for any 
# damages, system failures, security breaches, or network issues 
# resulting from the use or implementation of this script.
# ====================================================================
 
import cv2
import time
from picamera2 import Picamera2
import mediapipe as mp
import numpy as np
W=1280
H=720
tStart = time.time()
fps = 0

piCam = Picamera2(1)
piCam.preview_configuration.main.size = (W, H)
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate = 60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(
    
    max_num_faces=1,
    refine_landmarks=True,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)

cv2.namedWindow('Camera',cv2.WINDOW_GUI_NORMAL)
cv2.moveWindow('Camera',0,65)
cv2.resizeWindow('Camera',W,H)
while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame = piCam.capture_array()
    frame = cv2.flip(frame, -1)

    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = faceMesh.process(rgb)

    if results.multi_face_landmarks:
        for faceLandmarks in results.multi_face_landmarks:
            # Optional second pass for glow effect
            mp.solutions.drawing_utils.draw_landmarks(
                image=frame,
                landmark_list=faceLandmarks,
                connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,
                landmark_drawing_spec=None,
                connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
                    color=(100, 100, 100),
                    thickness=1
                )
            )
            # === Best looking translucent mesh ===
            mp.solutions.drawing_utils.draw_landmarks(
                image=frame,
                landmark_list=faceLandmarks,
                connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,
                landmark_drawing_spec=None,                    # Hide dots
                connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
                    color=(0, 0, 180),      # Bright cyan/teal
                    thickness=7
                )
            )

            # Optional: Extra detail on irises (makes eyes look better)
            mp.solutions.drawing_utils.draw_landmarks(
                image=frame,
                landmark_list=faceLandmarks,
                connections=mp.solutions.face_mesh.FACEMESH_IRISES,
                landmark_drawing_spec=None,
                connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
                    color=(255, 0, 0),
                    thickness=7
                )
            )

    myText = "FPS: "+str(round(fps,1))
    cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

    cv2.imshow("Camera", frame)

    if cv2.waitKey(1) == ord('q'):
        break

cv2.destroyAllWindows()
piCam.stop()

100

101

# ====================================================================

# DISCLAIMER:

# This code is provided as-is for educational and experimental

# purposes only. The author makes no representations or warranties of

# any kind concerning the safety, suitability, or accuracy of this

# code. Use at your own risk. The author assumes no liability for any

# damages, system failures, security breaches, or network issues

# resulting from the use or implementation of this script.

# ====================================================================

import cv2

import time

from picamera2 import Picamera2

import mediapipe as mp

import numpy as np

W=1280

H=720

tStart = time.time()

fps = 0

piCam = Picamera2(1)

piCam.preview_configuration.main.size = (W, H)

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate = 60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(

max_num_faces=1,

refine_landmarks=True,

min_detection_confidence=0.5,

min_tracking_confidence=0.5)

cv2.namedWindow('Camera',cv2.WINDOW_GUI_NORMAL)

cv2.moveWindow('Camera',0,65)

cv2.resizeWindow('Camera',W,H)

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame = piCam.capture_array()

frame = cv2.flip(frame, -1)

rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

results = faceMesh.process(rgb)

if results.multi_face_landmarks:

for faceLandmarks in results.multi_face_landmarks:

# Optional second pass for glow effect

mp.solutions.drawing_utils.draw_landmarks(

image=frame,

landmark_list=faceLandmarks,

connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,

landmark_drawing_spec=None,

connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

color=(100, 100, 100),

thickness=1

)

# === Best looking translucent mesh ===

mp.solutions.drawing_utils.draw_landmarks(

image=frame,

landmark_list=faceLandmarks,

connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,

landmark_drawing_spec=None, # Hide dots

connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

color=(0, 0, 180), # Bright cyan/teal

thickness=7

)

# Optional: Extra detail on irises (makes eyes look better)

mp.solutions.drawing_utils.draw_landmarks(

image=frame,

landmark_list=faceLandmarks,

connections=mp.solutions.face_mesh.FACEMESH_IRISES,

landmark_drawing_spec=None,

connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

color=(255, 0, 0),

thickness=7

)

myText = "FPS: "+str(round(fps,1))

cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Camera", frame)

if cv2.waitKey(1) == ord('q'):

break

cv2.destroyAllWindows()

piCam.stop()

AI On the Edge, Raspberry Pi

AI on the Edge 38: Using MediaPipe for Face Recognition on the Raspberry Pi 5

August 2, 2026 admin

In this video lesson we introduce you to MediaPipe. The OS we had you flash in LESSON 1 already has the MediaPipe framework installed, and all the needed and working dependencies. If you have installed that OS and not modified it, this and future lessons will work. If you find dependency errors, you might need to reflash the original OS.

MediaPipe is a free, open-source framework developed by Google that makes it much easier to add advanced computer vision and AI features to your Python programs. It is especially popular among developers who use OpenCV because it works seamlessly with it and delivers excellent real-time performance, even on devices like the Raspberry Pi 5.

With MediaPipe, you can quickly add powerful capabilities such as face detection, face mesh (detailed facial landmarks), hand tracking, body pose estimation, and more — all without having to write complex deep learning code from scratch. It comes with pre-trained machine learning models that are optimized for speed, allowing your programs to run smoothly at 30 frames per second or higher.

The biggest advantage for Python + OpenCV users is its simplicity. You capture video frames using OpenCV or picamera2, pass them to MediaPipe for processing, and then draw the results (such as bounding boxes or landmarks) back onto your image using normal OpenCV functions. This combination gives developers a fast and straightforward way to build interactive computer vision projects like face trackers, gesture-controlled robots, or smart camera applications.

In short, MediaPipe acts as a powerful, easy-to-use toolkit that bridges the gap between OpenCV and modern AI vision technology.

We introduce you to MediaPipe using a simple example where we create a faceFinder on our Raspberry Pi 5.

# ====================================================================
# DISCLAIMER:
# This code is provided as-is for educational and experimental 
# purposes only. The author makes no representations or warranties of 
# any kind concerning the safety, suitability, or accuracy of this 
# code. Use at your own risk. The author assumes no liability for any 
# damages, system failures, security breaches, or network issues 
# resulting from the use or implementation of this script.
# ====================================================================
import cv2
import time
from picamera2 import Picamera2
import mediapipe as mp
piCam = Picamera2()
W=1280
H=720
tStart = time.time()
fps = 0
RES = (W,H)
piCam.preview_configuration.main.size = RES
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate=60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

faceFinder = mp.solutions.face_detection.FaceDetection(
    model_selection = 0,
    min_detection_confidence = .5)
mpDraw = mp.solutions.drawing_utils

while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame= piCam.capture_array()
    #frame=cv2.flip(frame,-1)
    
    rgbFrame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
    results = faceFinder.process(rgbFrame)
    if results.detections:
        for detection in results.detections:
            mpDraw.draw_detection(frame,detection)
    
    myText = "FPS: "+str(round(fps,1))
    cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)
    cv2.imshow("Camera", frame)
    cv2.moveWindow("Camera",0,60)
    if cv2.waitKey(1)==ord('q'):
        break
cv2.destroyAllWindows()
print('Program Terminated')

# ====================================================================

# DISCLAIMER:

# This code is provided as-is for educational and experimental

# purposes only. The author makes no representations or warranties of

# any kind concerning the safety, suitability, or accuracy of this

# code. Use at your own risk. The author assumes no liability for any

# damages, system failures, security breaches, or network issues

# resulting from the use or implementation of this script.

# ====================================================================

import cv2

import time

from picamera2 import Picamera2

import mediapipe as mp

piCam = Picamera2()

W=1280

H=720

tStart = time.time()

fps = 0

RES = (W,H)

piCam.preview_configuration.main.size = RES

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate=60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

faceFinder = mp.solutions.face_detection.FaceDetection(

model_selection = 0,

min_detection_confidence = .5)

mpDraw = mp.solutions.drawing_utils

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame= piCam.capture_array()

#frame=cv2.flip(frame,-1)

rgbFrame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results = faceFinder.process(rgbFrame)

if results.detections:

for detection in results.detections:

mpDraw.draw_detection(frame,detection)

myText = "FPS: "+str(round(fps,1))

cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Camera", frame)

cv2.moveWindow("Camera",0,60)

if cv2.waitKey(1)==ord('q'):

break

cv2.destroyAllWindows()

print('Program Terminated')

NVIDIA, NVIDIA Jetson Orin Nano

Local Voice Control of NVIDIA Jetson Orin Nano with STT: Getting Started with Vosk

July 22, 2026 admin

Engineering Your Own Local Voice Assistant: No Cloud, No Compromise

Most “smart” voice assistants are just glorified remote controls for someone else’s server. Today, we’re changing that. We are going to build a local, offline voice command pipeline. This isn’t just about saving data; it’s about ownership. When you can control your hardware—like opening and closing a farm gate—without an internet connection, you have built a system that is robust, private, and yours to control forever. Today you are going to get Speech to Text up and running on your NVIDIA Jetson Orin Nano running under Jetpack 7.2.

The “Why” Behind the Setup

You might ask, “Why not just use a cloud API?” Because cloud APIs are fragile. They rely on internet stability, external servers, and privacy-invasive data logging. By running Vosk locally, we keep the processing on your hardware (like the NVIDIA Jetson). It’s faster, it works in the middle of a power-isolated homestead, and it’s 100% secure.

Part 1: Preparing the Environment

Before we can make the machine listen, we have to prepare the battlefield. We aren’t just downloading files; we are setting up a stable environment where your dependencies won’t conflict with your OS.

# Create a dedicated directory so our work stays organized
mkdir -p ~/STT
cd ~/STT

# We need the PortAudio headers. Why? Because Python’s 'pyaudio' is just a wrapper.
# Under the hood, it talks to the C-based PortAudio library. If the headers aren't there,
# the library won't compile, and your microphone will never wake up.
sudo apt update
sudo apt install -y portaudio19-dev wget unzip

# Download the lightweight, low-latency English model weights from Vosk
wget https://alphacephei.com/vosk/models/vosk-model-small-en-us-0.15.zip

# Unzip the archive and rename the folder cleanly to 'model' so the Python script finds it
unzip vosk-model-small-en-us-0.15.zip
mv vosk-model-small-en-us-0.15 model

# Clean up the downloaded zip file to keep the folder pristine
rm vosk-model-small-en-us-0.15.zip

# Setting up a Virtual Environment (venv) is the hallmark of a pro.
# It traps our project dependencies inside this folder so we don't break our system Python.
python3 -m venv sttVenv
source sttVenv/bin/activate
pip install --upgrade pip

# Vosk does the heavy lifting for speech recognition, and Pyaudio grabs the raw data.
pip install vosk pyaudio

# Create a dedicated directory so our work stays organized

mkdir -p ~/STT

cd ~/STT

# We need the PortAudio headers. Why? Because Python’s 'pyaudio' is just a wrapper.

# Under the hood, it talks to the C-based PortAudio library. If the headers aren't there,

# the library won't compile, and your microphone will never wake up.

sudo apt update

sudo apt install -y portaudio19-dev wget unzip

# Download the lightweight, low-latency English model weights from Vosk

wget https://alphacephei.com/vosk/models/vosk-model-small-en-us-0.15.zip

# Unzip the archive and rename the folder cleanly to 'model' so the Python script finds it

unzip vosk-model-small-en-us-0.15.zip

mv vosk-model-small-en-us-0.15 model

# Clean up the downloaded zip file to keep the folder pristine

rm vosk-model-small-en-us-0.15.zip

# Setting up a Virtual Environment (venv) is the hallmark of a pro.

# It traps our project dependencies inside this folder so we don't break our system Python.

python3 -m venv sttVenv

source sttVenv/bin/activate

pip install --upgrade pip

# Vosk does the heavy lifting for speech recognition, and Pyaudio grabs the raw data.

pip install vosk pyaudio

This is IMPORTANT!

Now you can post the code below. You also have to point Thonny to run in the virtual environment. Open Thonny, and under run – select interpreter. Then you must point it to /home/yourUserName/STT/ttsVenv/bin/python3. For me, my username is pjm, but you put in your user name in path above. Here is what mine looked like:

Part 2: Solving the PipeWire Challenge

The biggest headache in modern Linux audio is PipeWire. If you try to open a microphone stream using a hardcoded sample rate that doesn’t match your hardware, your program won’t just fail—it will segfault. We use the validation script below to programmatically query the hardware, asking it: “What sample rate are you running at?” before we even try to open a stream.

# Import the PyAudio library to handle real-time microphone recording and audio stream manipulation.
import pyaudio
# Import the built-in JSON module to parse the structured text results returned by the speech recognition engine.
import json
# Import the audioop library to perform real-time mathematical operations and downsampling on raw audio byte data.
import audioop
# Import the core Vosk objects: Model loads the AI brain/weights, and KaldiRecognizer handles the acoustic processing.
from vosk import Model, KaldiRecognizer
# Import the standard OS module to reliably build absolute system file paths across different user directories.
import os
state=None

# Define a function to discover the specific hardware index and sample rate of the active system sound server.
def get_pipewire_index():
    # Instantiate a PyAudio session object to query the operating system's hardware configuration list.
    p = pyaudio.PyAudio()
    # Loop through every single available audio device reported by the operating system kernel.
    for i in range(p.get_device_count()):
        # Retrieve a descriptive configuration dictionary for the current audio device index.
        dev = p.get_device_info_by_index(i)
        # Check if the name of the device contains 'pipewire' to target the modern Linux audio subsystem.
        if "pipewire" in dev.get('name', '').lower():
            # Return the valid device index integer and its hardware default sample rate, converted cleanly to an integer.
            return i, int(dev.get('defaultSampleRate'))
    # Return empty values if the loop completes without finding an active PipeWire sound server profile.
    return None, None

# Execute the discovery function to capture both the correct system hardware device index and its native sample rate.
dev_index, native_rate = get_pipewire_index()

# Check if the hardware discovery handshake failed to locate an active PipeWire audio interface.
if dev_index is None:
    # Alert the user on screen that the script cannot communicate with the system's active audio server layer.
    print("Error: Could not find 'pipewire' device.")
    # Stop the script execution immediately since it is impossible to record audio without a valid device hook.
    exit()

# Instantiate a fresh PyAudio session specifically dedicated to opening our incoming recording channel.
p = pyaudio.PyAudio()
# Open the active system recording stream channel using explicit audio hardware parameter configurations.
# PARAMETER MANUAL:
# - format=pyaudio.paInt16: Sets 16-bit Signed Integer PCM format. (LEAVE ALONE - required by the Vosk engine).
# - channels=1: Sets recording to Mono channel audio. (LEAVE ALONE - Vosk processes single-channel input).
# - rate=native_rate: Sets the sample rate to match the mic exactly. (LEAVE ALONE - prevents driver hardware failure).
# - input=True: Configures this stream as an audio capture device. (LEAVE ALONE - required to wake up the microphone).
# - input_device_index=dev_index: Points to our discovered target device. (LEAVE ALONE - ensures we grab the right mic).
# - frames_per_buffer=4096: Controls audio chunk size. (PLAY WITH: Lower values like 2048 reduce latency but risk buffer overflows; higher values like 8192 increase safety but introduce slight latency).
stream = p.open(format=pyaudio.paInt16, channels=1, rate=native_rate, 
                input=True, input_device_index=dev_index, frames_per_buffer=4096)

# Generate an absolute directory path targeting the exact folder containing your extracted acoustic model weights.
# PARAMETER MANUAL:
# - "STT", "model": Represents folder names. (PLAY WITH: Change these only if your physical folder layout uses different names).
model_path = os.path.join(os.path.expanduser("~"), "STT", "model")
# Load the heavy machine learning weights into system memory to initialize the Vosk voice processing brain.
model = Model(model_path)
# Initialize the primary speech recognizer object using the loaded model weights and target processing frequency.
# PARAMETER MANUAL:
# - model: The initialized AI brain variable. (LEAVE ALONE).
# - 16000: Specifies the target processing sample rate in Hertz. (LEAVE ALONE - Vosk models are mathematically trained for exactly 16000Hz).
recognizer = KaldiRecognizer(model, 16000)

# Print a status message to the terminal showing that the handshake succeeded and revealing the mic's operating frequency.
print(f"Listening on PipeWire at {native_rate}Hz...")
# Turn on the hardware recording valve to begin filling up the system input data buffers.
stream.start_stream()

# Set up an exception safety block to intercept manual user exits gracefully without leaving system resources locked up.
try:
    # Launch the infinite real-time audio capture loop to continuously monitor the microphone stream.
    while True:
        # Pull raw binary audio frame bytes directly out of the incoming hardware stream buffer memory block.
        # PARAMETER MANUAL:
        # - 4096: The chunk size to read. (LEAVE ALONE - must perfectly match your stream's frames_per_buffer setting).
        # - exception_on_overflow=False: Prevents Python crashes if the Orin momentarily falls behind. (LEAVE ALONE).
        data = stream.read(4096, exception_on_overflow=False)
        # Downsample the raw audio bytes mathematically to 16000Hz using digital filtering algorithms.
        # PARAMETER MANUAL:
        # - data: Raw input audio byte string. (LEAVE ALONE).
        # - 2: Size of sample width in bytes for 16-bit sound. (LEAVE ALONE).
        # - 1: Input audio channel count. (LEAVE ALONE).
        # - native_rate: The native frequency of your mic. (LEAVE ALONE).
        # - 16000: The required downsampled output frequency. (LEAVE ALONE).
        # - state: The digital filtering conversion state history object. (LEAVE ALONE).
        # CRITICAL NOTE: The 'state' variable must be initialized as 'state = None' before entering this 'try:' block, 
        # otherwise Python will throw a NameError on the very first loop iteration because 'state' does not exist yet.
        data_16k, state = audioop.ratecv(data, 2, 1, native_rate, 16000, state)
        
        # Feed the processed 16kHz audio data block into Vosk to evaluate if a complete spoken phrase has finished.
        if recognizer.AcceptWaveform(data_16k):
            # Parse the structured JSON text string returned by the acoustic model into a standard Python dictionary.
            res = json.loads(recognizer.Result())
            # Check the parsed result dictionary to ensure that the detected text sequence is not completely blank.
            if res.get("text"):
                # Cleanly output the decoded text command straight to the standard system terminal display screen.
                print("Command: "+res['text'])
# Intercept the user pressing Ctrl+C in the terminal to stop the continuous script loop cleanly.
except KeyboardInterrupt:
    # Print a clean closing status notification on a new line to confirm the signal was intercepted.
    print("\nStopped.")
# Execute mandatory clean up instructions that are guaranteed to run no matter how the script terminates or crashes.
finally:
    # Turn off the active recording stream data valve to stop pulling raw information from the audio driver layer.
    stream.stop_stream()
    # Close the recording stream channel completely to drop the active hardware pointer binding handles.
    stream.close()
    # Terminate the parent PyAudio engine instance to release all allocated audio device resources back to the OS.
    p.terminate()

100

101

102

103

104

105

106

107

108

# Import the PyAudio library to handle real-time microphone recording and audio stream manipulation.

import pyaudio

# Import the built-in JSON module to parse the structured text results returned by the speech recognition engine.

import json

# Import the audioop library to perform real-time mathematical operations and downsampling on raw audio byte data.

import audioop

# Import the core Vosk objects: Model loads the AI brain/weights, and KaldiRecognizer handles the acoustic processing.

from vosk import Model, KaldiRecognizer

# Import the standard OS module to reliably build absolute system file paths across different user directories.

import os

state=None

# Define a function to discover the specific hardware index and sample rate of the active system sound server.

def get_pipewire_index():

# Instantiate a PyAudio session object to query the operating system's hardware configuration list.

p = pyaudio.PyAudio()

# Loop through every single available audio device reported by the operating system kernel.

for i in range(p.get_device_count()):

# Retrieve a descriptive configuration dictionary for the current audio device index.

dev = p.get_device_info_by_index(i)

# Check if the name of the device contains 'pipewire' to target the modern Linux audio subsystem.

if "pipewire" in dev.get('name', '').lower():

# Return the valid device index integer and its hardware default sample rate, converted cleanly to an integer.

return i, int(dev.get('defaultSampleRate'))

# Return empty values if the loop completes without finding an active PipeWire sound server profile.

return None, None

# Execute the discovery function to capture both the correct system hardware device index and its native sample rate.

dev_index, native_rate = get_pipewire_index()

# Check if the hardware discovery handshake failed to locate an active PipeWire audio interface.

if dev_index is None:

# Alert the user on screen that the script cannot communicate with the system's active audio server layer.

print("Error: Could not find 'pipewire' device.")

# Stop the script execution immediately since it is impossible to record audio without a valid device hook.

exit()

# Instantiate a fresh PyAudio session specifically dedicated to opening our incoming recording channel.

p = pyaudio.PyAudio()

# Open the active system recording stream channel using explicit audio hardware parameter configurations.

# PARAMETER MANUAL:

# - format=pyaudio.paInt16: Sets 16-bit Signed Integer PCM format. (LEAVE ALONE - required by the Vosk engine).

# - channels=1: Sets recording to Mono channel audio. (LEAVE ALONE - Vosk processes single-channel input).

# - rate=native_rate: Sets the sample rate to match the mic exactly. (LEAVE ALONE - prevents driver hardware failure).

# - input=True: Configures this stream as an audio capture device. (LEAVE ALONE - required to wake up the microphone).

# - input_device_index=dev_index: Points to our discovered target device. (LEAVE ALONE - ensures we grab the right mic).

# - frames_per_buffer=4096: Controls audio chunk size. (PLAY WITH: Lower values like 2048 reduce latency but risk buffer overflows; higher values like 8192 increase safety but introduce slight latency).

stream = p.open(format=pyaudio.paInt16, channels=1, rate=native_rate,

input=True, input_device_index=dev_index, frames_per_buffer=4096)

# Generate an absolute directory path targeting the exact folder containing your extracted acoustic model weights.

# PARAMETER MANUAL:

# - "STT", "model": Represents folder names. (PLAY WITH: Change these only if your physical folder layout uses different names).

model_path = os.path.join(os.path.expanduser("~"), "STT", "model")

# Load the heavy machine learning weights into system memory to initialize the Vosk voice processing brain.

model = Model(model_path)

# Initialize the primary speech recognizer object using the loaded model weights and target processing frequency.

# PARAMETER MANUAL:

# - model: The initialized AI brain variable. (LEAVE ALONE).

# - 16000: Specifies the target processing sample rate in Hertz. (LEAVE ALONE - Vosk models are mathematically trained for exactly 16000Hz).

recognizer = KaldiRecognizer(model, 16000)

# Print a status message to the terminal showing that the handshake succeeded and revealing the mic's operating frequency.

print(f"Listening on PipeWire at {native_rate}Hz...")

# Turn on the hardware recording valve to begin filling up the system input data buffers.

stream.start_stream()

# Set up an exception safety block to intercept manual user exits gracefully without leaving system resources locked up.

try:

# Launch the infinite real-time audio capture loop to continuously monitor the microphone stream.

while True:

# Pull raw binary audio frame bytes directly out of the incoming hardware stream buffer memory block.

# PARAMETER MANUAL:

# - 4096: The chunk size to read. (LEAVE ALONE - must perfectly match your stream's frames_per_buffer setting).

# - exception_on_overflow=False: Prevents Python crashes if the Orin momentarily falls behind. (LEAVE ALONE).

data = stream.read(4096, exception_on_overflow=False)

# Downsample the raw audio bytes mathematically to 16000Hz using digital filtering algorithms.

# PARAMETER MANUAL:

# - data: Raw input audio byte string. (LEAVE ALONE).

# - 2: Size of sample width in bytes for 16-bit sound. (LEAVE ALONE).

# - 1: Input audio channel count. (LEAVE ALONE).

# - native_rate: The native frequency of your mic. (LEAVE ALONE).

# - 16000: The required downsampled output frequency. (LEAVE ALONE).

# - state: The digital filtering conversion state history object. (LEAVE ALONE).

# CRITICAL NOTE: The 'state' variable must be initialized as 'state = None' before entering this 'try:' block,

# otherwise Python will throw a NameError on the very first loop iteration because 'state' does not exist yet.

data_16k, state = audioop.ratecv(data, 2, 1, native_rate, 16000, state)

# Feed the processed 16kHz audio data block into Vosk to evaluate if a complete spoken phrase has finished.

if recognizer.AcceptWaveform(data_16k):

# Parse the structured JSON text string returned by the acoustic model into a standard Python dictionary.

res = json.loads(recognizer.Result())

# Check the parsed result dictionary to ensure that the detected text sequence is not completely blank.

if res.get("text"):

# Cleanly output the decoded text command straight to the standard system terminal display screen.

print("Command: "+res['text'])

# Intercept the user pressing Ctrl+C in the terminal to stop the continuous script loop cleanly.

except KeyboardInterrupt:

# Print a clean closing status notification on a new line to confirm the signal was intercepted.

print("\nStopped.")

# Execute mandatory clean up instructions that are guaranteed to run no matter how the script terminates or crashes.

finally:

# Turn off the active recording stream data valve to stop pulling raw information from the audio driver layer.

stream.stop_stream()

# Close the recording stream channel completely to drop the active hardware pointer binding handles.

stream.close()

# Terminate the parent PyAudio engine instance to release all allocated audio device resources back to the OS.

p.terminate()

Homework: Your Gate Controller

You now have a system that identifies audio input, resamples it to 16kHz, and outputs text. Your assignment: Transform this text output into an action.

I want you to add a conditional statement to the main loop. If the recognized text is “open”, print an ASCII art representation of an open gate. If it’s “close”, print the closed version. This is the first step in closing the loop between your AI and the physical world. Go get ’em, and don’t just copy the code—understand how the data flows from the microphone to your decision logic!

AI On the Edge, Raspberry Pi

AI on the Edge LESSON 30: Tune Object Tracker with Mouse Selected ROI

July 3, 2026 admin

AI on the Edge LESSON 30: Tune Object Tracker with Mouse Selected ROI

Welcome, Makers!

Well, hello there! It is absolutely fantastic to have you back. I’m Paul McWhorter, and today, we are taking a massive step forward in our AI on the Edge journey.

Up until now, we’ve been hard-coding our color thresholds (those pesky Lower Color and Upper Color values) to tell our camera what to look for. That’s fine for a science experiment, but it’s not exactly “smart,” is it? If the lighting changes, or if we want to track a different colored object, we have to go back into the code and manually edit those numbers.

Not anymore!

In today’s lesson, we are building a tool that lets us teach the AI. We’re going to use the mouse to draw a Region of Interest (ROI) right on our camera feed. The system will look at the pixels inside that box, calculate the average Hue, Saturation, and Value, and automatically set our tracking range for us.

This is the kind of professional-level functionality that turns a hobby project into a true, intelligent machine.

The Concept: From Hard-Coding to Dynamic Learning

The magic happens in our mouseAction function. Instead of just reading pixel values, we are now implementing a “click-and-drag” system:

Click and Hold: We capture the startX and startY coordinates.
Drag: We draw a rectangle in real-time so we can see exactly what area we are selecting.
Release: We take that specific slice of the image, convert it to the HSV color space, and use the cv2.mean() function to find the average color properties.
Auto-Tune: We set our LC (Lower Color) and UC (Upper Color) based on that average.

By doing this, the system learns what “object” we want to track on the fly. It’s elegant, it’s powerful, and it feels like real magic when you see those servos snap onto your target after a quick mouse drag.

What We’ve Accomplished

By the end of this lesson, you will have a system that:

Visually selects an object using the mouse.
Automatically calculates the optimal HSV thresholds for that specific object.
Updates the tracking behavior immediately without needing to stop or re-run the code.
Maintains that professional “Edge” feel, giving you real-time feedback on your FPS and mouse position data.

A Note on the “Edge”

Remember, we aren’t just running code; we are running on hardware. When we calculate the mean of the ROI, we are doing real image processing on the fly. You’ll notice the Composite and Mask windows updated immediately, giving you a visual confirmation that your “teacher” (you!) has successfully guided the “student” (the AI).

This is the power of working with OpenCV and the Raspberry Pi. You are building a system that observes, thinks, and reacts—all in real-time.

Get Ready to Build

Grab your Pi, make sure your servos are ready to go, and let’s get that camera calibrated. You’ve put in the work to get this far, and today is where all that effort starts to feel really rewarding.

I’m incredibly proud of how far you’ve come. Let’s dive in and start building!

Are you ready to see how accurately your Pi can “see” once you’ve given it the ability to learn from your selections?

In the video lesson we developed the following code.

# ====================================================================
# DISCLAIMER:
# This code is provided as-is for educational and experimental 
# purposes only. The author makes no representations or warranties of 
# any kind concerning the safety, suitability, or accuracy of this 
# code. Use at your own risk. The author assumes no liability for any 
# damages, system failures, security breaches, or network issues 
# resulting from the use or implementation of this script.
# ====================================================================
 
import cv2
import time
from picamera2 import Picamera2
from fusion_hat.pwm import PWM
from fusion_hat.servo import Servo
panPin = 2
tiltPin = 3
panServo = Servo(panPin)
tiltServo = Servo(tiltPin)
panAngle = 0
tiltAngle = 0
panServo.angle(panAngle)
tiltServo.angle(tiltAngle)

piCam = Picamera2()
W=1280
H=720
tStart = time.time()
fps = 0

xFrameC = int((W-1)/2)
yFrameC = int((H-1)/2)
xBoxC = xFrameC
yBoxC = yFrameC


redPin = 5
greenPin = 6
bluePin = 7
redLED = PWM(redPin)
greenLED = PWM(greenPin)
blueLED = PWM(bluePin)

RES = (W,H)
piCam.preview_configuration.main.size = RES
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate=60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

textLowerLeft = (int(W*.01),int(H*.06))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)
xPos = 0
textLowerLeft1 = (int(W*.01),int(H*.06)*2)
textLowerLeft2 = (int(W*.01),int(H*.06)*3)
yPos = 0
valR = 0
valG = 0
valB = 0

Hue = 0
Sat = 0
Val = 0

LC = (25,100,90)
UC = (35,255,255)

frame = None
track = False
drawing = False
makeROI = False
boxColor = (255,255,0)
boxThick = 3
def mouseAction(event, x, y, flags, param):
    global frame, xPos, yPos, Hue, Sat, Val
    global track,startX,startY,endX,endY,makeROI,drawing
    if event == 0:
        xPos = x
        yPos = y
        if frame is not None:
            valB, valG, valR = frame[y,x]
            redLED.pulse_width_percent(int(valR/255*100))
            greenLED.pulse_width_percent(int(valG/255*100/2))
            blueLED.pulse_width_percent(int(valB/255*100/4))
            frameHSV = cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)
            Hue, Sat, Val =frameHSV[y,x]
            if Hue == 60:
                track = True
            if Hue == 0:
                track = False
        if drawing == True:
            endX = x
            endY = y
    if event == 1:
        drawing = True
        makeROI = False
        startX = x
        startY = y
        endX = x
        endY = y 
    if event == 4:
        drawing = False
        makeROI = True
        endX = x
        endY = y                    
cv2.namedWindow('Camera',cv2.WINDOW_GUI_NORMAL)
cv2.moveWindow('Camera',0,65)
cv2.resizeWindow('Camera',W,H)

cv2.namedWindow('Mask',cv2.WINDOW_GUI_NORMAL)
cv2.moveWindow('Mask',W,65)
cv2.resizeWindow('Mask',int(W/2),int(H/2))

cv2.namedWindow('Composite',cv2.WINDOW_GUI_NORMAL)
cv2.moveWindow('Composite',W,65+int(H/2)+25)
cv2.resizeWindow('Composite',int(W/2),int(H/2))

cv2.setMouseCallback('Camera',mouseAction)

while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame= piCam.capture_array()
    frame=cv2.flip(frame,-1)
    
    if drawing == True:
        cv2.rectangle(frame,(startX,startY),(endX,endY),boxColor,boxThick)
    
    if makeROI == True:
        ROI = frame[startY:endY,startX:endX]
        ROIhsv = cv2.cvtColor(ROI,cv2.COLOR_BGR2HSV)
        AVGhsv = cv2.mean(ROIhsv)
        print(AVGhsv)
        LC = (AVGhsv[0]-15,AVGhsv[1]-50,AVGhsv[2]-50)
        UC = (AVGhsv[0]+15,255,255)
        cv2.imshow("ROI",ROI)
        cv2.moveWindow("ROI",W,65)
        makeROI = False

    
    cv2.circle(frame,(int(.9*W),int(.1*H)),20,(0,255,0),-1)
    cv2.circle(frame,(int(.95*W),int(.1*H)),20,(0,0,255),-1)    
    frameHSV = cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)
    mask=cv2.inRange(frameHSV,LC,UC)
    composite = cv2.bitwise_and(frame, frame, mask=mask)
    
    contours, _ =cv2.findContours(mask,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
    if contours:
        #cv2.drawContours(frame,contours,-1,(255,0,0),3)
        largestContour = max(contours, key = cv2.contourArea)
        area = cv2.contourArea(largestContour)
        if area>100:
            #cv2.drawContours(frame,largestContour,-1,(255,0,0),3)
            x, y, w, h = cv2.boundingRect(largestContour)
            cv2.rectangle(frame, (x,y),(x+w,y+h),(0,255,255),3)
            xBoxC  =  x + int(w/2)
            yBoxC  =  y + int(h/2)
            
            xError = xBoxC - xFrameC
            yError = yBoxC - yFrameC
            
            cv2.circle(frame,(xFrameC,yFrameC),10,(255,0,0),-1)
            cv2.circle(frame,(xBoxC,yBoxC), 30,(0,0,255),3)
            if track == True:
                panAngle = panAngle - xError/50/2
                panServo.angle(int(panAngle))
                time.sleep(.02)
                tiltAngle=tiltAngle + yError/50/2
                tiltServo.angle(int(tiltAngle))
                time.sleep(.02)
                           
    myText = "FPS: "+str(round(fps,1))
    cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)
    
    text1 = "Mouse Pos: "+str((xPos,yPos))
    text2 = "Pixel Color: "+str((Hue,Sat,Val))
    cv2.putText(frame,text1,textLowerLeft1,fontFace,fontScale,fontColor,fontThickness)    
    cv2.putText(frame,text2,textLowerLeft2,fontFace,fontScale,fontColor,fontThickness)    
    cv2.imshow("Camera", frame)
    cv2.imshow("Composite",composite)
    cv2.imshow("Mask",mask)

    if cv2.waitKey(1)==ord('q'):
        break
cv2.destroyAllWindows()
redLED.pulse_width_percent(0)
greenLED.pulse_width_percent(0)
blueLED.pulse_width_percent(0)
print('Program Terminated')

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

# ====================================================================

# DISCLAIMER:

# This code is provided as-is for educational and experimental

# purposes only. The author makes no representations or warranties of

# any kind concerning the safety, suitability, or accuracy of this

# code. Use at your own risk. The author assumes no liability for any

# damages, system failures, security breaches, or network issues

# resulting from the use or implementation of this script.

# ====================================================================

import cv2

import time

from picamera2 import Picamera2

from fusion_hat.pwm import PWM

from fusion_hat.servo import Servo

panPin = 2

tiltPin = 3

panServo = Servo(panPin)

tiltServo = Servo(tiltPin)

panAngle = 0

tiltAngle = 0

panServo.angle(panAngle)

tiltServo.angle(tiltAngle)

piCam = Picamera2()

W=1280

H=720

tStart = time.time()

fps = 0

xFrameC = int((W-1)/2)

yFrameC = int((H-1)/2)

xBoxC = xFrameC

yBoxC = yFrameC

redPin = 5

greenPin = 6

bluePin = 7

redLED = PWM(redPin)

greenLED = PWM(greenPin)

blueLED = PWM(bluePin)

RES = (W,H)

piCam.preview_configuration.main.size = RES

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate=60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

textLowerLeft = (int(W*.01),int(H*.06))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

xPos = 0

textLowerLeft1 = (int(W*.01),int(H*.06)*2)

textLowerLeft2 = (int(W*.01),int(H*.06)*3)

yPos = 0

valR = 0

valG = 0

valB = 0

Hue = 0

Sat = 0

Val = 0

LC = (25,100,90)

UC = (35,255,255)

frame = None

track = False

drawing = False

makeROI = False

boxColor = (255,255,0)

boxThick = 3

def mouseAction(event, x, y, flags, param):

global frame, xPos, yPos, Hue, Sat, Val

global track,startX,startY,endX,endY,makeROI,drawing

if event == 0:

xPos = x

yPos = y

if frame is not None:

valB, valG, valR = frame[y,x]

redLED.pulse_width_percent(int(valR/255*100))

greenLED.pulse_width_percent(int(valG/255*100/2))

blueLED.pulse_width_percent(int(valB/255*100/4))

frameHSV = cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)

Hue, Sat, Val =frameHSV[y,x]

if Hue == 60:

track = True

if Hue == 0:

track = False

if drawing == True:

endX = x

endY = y

if event == 1:

drawing = True

makeROI = False

startX = x

startY = y

endX = x

endY = y

if event == 4:

drawing = False

makeROI = True

endX = x

endY = y

cv2.namedWindow('Camera',cv2.WINDOW_GUI_NORMAL)

cv2.moveWindow('Camera',0,65)

cv2.resizeWindow('Camera',W,H)

cv2.namedWindow('Mask',cv2.WINDOW_GUI_NORMAL)

cv2.moveWindow('Mask',W,65)

cv2.resizeWindow('Mask',int(W/2),int(H/2))

cv2.namedWindow('Composite',cv2.WINDOW_GUI_NORMAL)

cv2.moveWindow('Composite',W,65+int(H/2)+25)

cv2.resizeWindow('Composite',int(W/2),int(H/2))

cv2.setMouseCallback('Camera',mouseAction)

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame= piCam.capture_array()

frame=cv2.flip(frame,-1)

if drawing == True:

cv2.rectangle(frame,(startX,startY),(endX,endY),boxColor,boxThick)

if makeROI == True:

ROI = frame[startY:endY,startX:endX]

ROIhsv = cv2.cvtColor(ROI,cv2.COLOR_BGR2HSV)

AVGhsv = cv2.mean(ROIhsv)

print(AVGhsv)

LC = (AVGhsv[0]-15,AVGhsv[1]-50,AVGhsv[2]-50)

UC = (AVGhsv[0]+15,255,255)

cv2.imshow("ROI",ROI)

cv2.moveWindow("ROI",W,65)

makeROI = False

cv2.circle(frame,(int(.9*W),int(.1*H)),20,(0,255,0),-1)

cv2.circle(frame,(int(.95*W),int(.1*H)),20,(0,0,255),-1)

frameHSV = cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)

mask=cv2.inRange(frameHSV,LC,UC)

composite = cv2.bitwise_and(frame, frame, mask=mask)

contours, _ =cv2.findContours(mask,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)

if contours:

#cv2.drawContours(frame,contours,-1,(255,0,0),3)

largestContour = max(contours, key = cv2.contourArea)

area = cv2.contourArea(largestContour)

if area>100:

#cv2.drawContours(frame,largestContour,-1,(255,0,0),3)

x, y, w, h = cv2.boundingRect(largestContour)

cv2.rectangle(frame, (x,y),(x+w,y+h),(0,255,255),3)

xBoxC = x + int(w/2)

yBoxC = y + int(h/2)

xError = xBoxC - xFrameC

yError = yBoxC - yFrameC

cv2.circle(frame,(xFrameC,yFrameC),10,(255,0,0),-1)

cv2.circle(frame,(xBoxC,yBoxC), 30,(0,0,255),3)

if track == True:

panAngle = panAngle - xError/50/2

panServo.angle(int(panAngle))

time.sleep(.02)

tiltAngle=tiltAngle + yError/50/2

tiltServo.angle(int(tiltAngle))

time.sleep(.02)

myText = "FPS: "+str(round(fps,1))

cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

text1 = "Mouse Pos: "+str((xPos,yPos))

text2 = "Pixel Color: "+str((Hue,Sat,Val))

cv2.putText(frame,text1,textLowerLeft1,fontFace,fontScale,fontColor,fontThickness)

cv2.putText(frame,text2,textLowerLeft2,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Camera", frame)

cv2.imshow("Composite",composite)

cv2.imshow("Mask",mask)

if cv2.waitKey(1)==ord('q'):

break

cv2.destroyAllWindows()

redLED.pulse_width_percent(0)

greenLED.pulse_width_percent(0)

blueLED.pulse_width_percent(0)

print('Program Terminated')

Technology Tutorials

Tag Archives: Artificial intelligence

AI on the Edge LESSON 42: Create Composite Images Using Masks in OpenCV and MediaPipe

AI on the Edge LESSON 41: Creating FaceMesh Using MediaPipe in OpenCV

AI on the Edge 38: Using MediaPipe for Face Recognition on the Raspberry Pi 5

Local Voice Control of NVIDIA Jetson Orin Nano with STT: Getting Started with Vosk

Engineering Your Own Local Voice Assistant: No Cloud, No Compromise

The “Why” Behind the Setup

Part 1: Preparing the Environment

This is IMPORTANT!

Part 2: Solving the PipeWire Challenge

Homework: Your Gate Controller

AI on the Edge LESSON 30: Tune Object Tracker with Mouse Selected ROI

AI on the Edge LESSON 30: Tune Object Tracker with Mouse Selected ROI

Welcome, Makers!

The Concept: From Hard-Coding to Dynamic Learning

What We’ve Accomplished

A Note on the “Edge”

Get Ready to Build

Making The World a Better Place One High Tech Project at a Time. Enjoy!