Tag Archives: Fusion AI Lab Kit

Hey everyone, Paul McWhorter here!

Welcome back to the AI on the Edge series!

In today’s lesson, we’re going to take an important next step in computer vision. We’re going to learn how to create, position, resize, and manage multiple windows at the same time using OpenCV on the Raspberry Pi.

This might sound simple, but it’s actually a very big deal. Once you can comfortably work with multiple windows, you can start building much more powerful vision applications — like having a main camera view, a processed view, zoomed-in sections, and debug windows all running at once.

In this lesson we create:

One large main camera window
A smaller color preview
A small grayscale version
Five tiny grayscale windows stacked on the side

This gives you a clean, organized workspace while the camera is running.

What You Learned in This Lesson

How to create multiple named windows with cv2.namedWindow()
How to resize windows using cv2.resizeWindow()
How to precisely position windows on your screen with cv2.moveWindow()
How to work with different resolutions of the same image (full size, half size, quarter size)
Converting between color and grayscale while running live video
Keeping everything running smoothly with good FPS

Mastering multiple windows is one of those foundational skills that separates basic OpenCV projects from more professional and useful vision systems.

Pro Tip: Play around with the window positions and sizes after you get it working. Try making one window much larger, or experiment with different layouts. This is your workspace — make it comfortable!

Ready for the next step? In the next lesson, we’re going to start doing something really cool — we’ll begin combining live video with drawn graphics and start creating interactive vision projects.

Keep building, keep learning, and I’ll see you in the next video!

In the lesson, we develop the code below:

import cv2
import time
from picamera2 import Picamera2
piCam = Picamera2()
W=640
H=360
tStart = time.time()
fps = 0
RES = (W,H)
piCam.preview_configuration.main.size = RES
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate=60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

topBar = 65
windowWaste = 25

cv2.namedWindow("Camera", cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow("Camera", W, H)
cv2.moveWindow("Camera",0,topBar)

cv2.namedWindow("Camera Small", cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow("Camera Small",  int(W/2), int(H/2))
cv2.moveWindow("Camera Small",0,topBar+windowWaste+H)
    
cv2.namedWindow("Gray Small", cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow("Gray Small", int(W/2), int(H/2))
cv2.moveWindow("Gray Small",int(W/2),topBar+windowWaste+H)

grays = ["Gray1","Gray2","Gray3","Gray4","Gray5"]
x = 0
for gray in grays:
    print(gray)
    cv2.namedWindow(gray,cv2.WINDOW_GUI_NORMAL)
    cv2.resizeWindow(gray,int(W/4),int(H/4))
    cv2.moveWindow(gray,W,topBar + x*(int(H/4) + windowWaste))
    x=x+1

while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame= piCam.capture_array()
    frame=cv2.flip(frame,-1)
    frameSmall=cv2.resize(frame,(int(W/2),int(H/2)))
    frameGraySmall = cv2.cvtColor(frameSmall,cv2.COLOR_BGR2GRAY)

    frameTiny1 = cv2.resize(frameGraySmall,(int(W/4),int(H/4)))
    frameTiny2 = cv2.resize(frameGraySmall,(int(W/4),int(H/4)))
    frameTiny3= cv2.resize(frameGraySmall,(int(W/4),int(H/4)))
    frameTiny4 = cv2.resize(frameGraySmall,(int(W/4),int(H/4)))
    frameTiny5 = cv2.resize(frameGraySmall,(int(W/4),int(H/4)))
    

    myText = "FPS: "+str(round(fps,1))
    cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)
    cv2.imshow("Camera", frame)
    cv2.imshow("Camera Small",frameSmall)
    cv2.imshow("Gray Small",frameGraySmall)
    
    cv2.imshow("Gray1",frameTiny1)
    cv2.imshow("Gray2",frameTiny2)
    cv2.imshow("Gray3",frameTiny3)
    cv2.imshow("Gray4",frameTiny4)
    cv2.imshow("Gray5",frameTiny5)


    if cv2.waitKey(1)==ord('q'):
        break
cv2.destroyAllWindows()
print('Program Terminated')

import cv2

import time

from picamera2 import Picamera2

piCam = Picamera2()

W=640

H=360

tStart = time.time()

fps = 0

RES = (W,H)

piCam.preview_configuration.main.size = RES

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate=60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

topBar = 65

windowWaste = 25

cv2.namedWindow("Camera", cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow("Camera", W, H)

cv2.moveWindow("Camera",0,topBar)

cv2.namedWindow("Camera Small", cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow("Camera Small", int(W/2), int(H/2))

cv2.moveWindow("Camera Small",0,topBar+windowWaste+H)

cv2.namedWindow("Gray Small", cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow("Gray Small", int(W/2), int(H/2))

cv2.moveWindow("Gray Small",int(W/2),topBar+windowWaste+H)

grays = ["Gray1","Gray2","Gray3","Gray4","Gray5"]

x = 0

for gray in grays:

print(gray)

cv2.namedWindow(gray,cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow(gray,int(W/4),int(H/4))

cv2.moveWindow(gray,W,topBar + x*(int(H/4) + windowWaste))

x=x+1

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame= piCam.capture_array()

frame=cv2.flip(frame,-1)

frameSmall=cv2.resize(frame,(int(W/2),int(H/2)))

frameGraySmall = cv2.cvtColor(frameSmall,cv2.COLOR_BGR2GRAY)

frameTiny1 = cv2.resize(frameGraySmall,(int(W/4),int(H/4)))

frameTiny2 = cv2.resize(frameGraySmall,(int(W/4),int(H/4)))

frameTiny3= cv2.resize(frameGraySmall,(int(W/4),int(H/4)))

frameTiny4 = cv2.resize(frameGraySmall,(int(W/4),int(H/4)))

frameTiny5 = cv2.resize(frameGraySmall,(int(W/4),int(H/4)))

myText = "FPS: "+str(round(fps,1))

cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Camera", frame)

cv2.imshow("Camera Small",frameSmall)

cv2.imshow("Gray Small",frameGraySmall)

cv2.imshow("Gray1",frameTiny1)

cv2.imshow("Gray2",frameTiny2)

cv2.imshow("Gray3",frameTiny3)

cv2.imshow("Gray4",frameTiny4)

cv2.imshow("Gray5",frameTiny5)

if cv2.waitKey(1)==ord('q'):

break

cv2.destroyAllWindows()

print('Program Terminated')

AI On the Edge, Tutorial

AI on the Edge LESSON 20: Resizing, Moving, Converting and Tiling Video frames in OpenCV

June 1, 2026 admin

Welcome back to the AI on the Edge class series! In this lesson, we are diving deep into some of the most critical foundational skills you need when working with video streams on edge devices: Resizing, Moving, Converting, and Tiling video frames using OpenCV.

When you are developing real-world AI applications on the edge, you rarely just display a single camera feed. You often need to manipulate frames to feed them into your AI models, look at grayscale versions for edge detection, or arrange multiple windows on your desktop neatly so you can monitor your data visually.

If you want to follow along exactly as we do in the video, make sure you have your Raspberry Pi 5 set up with your Camera Module.

What We Cover in This Lesson

Fixed FPS Estimation: We continue using our robust low-pass filter formula to track smooth, non-jittery frames-per-second data directly on the video frame.
Creating Named Windows: Understanding how cv2.namedWindow() combined with cv2.WINDOW_GUI_NORMAL gives you absolute programmatic control over the placement of your displays.
Resizing & Moving Windows: How to accurately position multiple OpenCV windows on your screen using specific coordinates while accounting for operating system taskbars and window decorative margins.
Frame Manipulation: Using cv2.resize() to scale down video frames and cv2.cvtColor() to transform the color space from BGR to grayscale.
Window Tiling: Arranging a main camera view, a scaled-down color view, and a scaled-down grayscale view in a perfect grid layout on your desktop.

The Complete Lesson 20 Code

Below is the complete Python code we developed during this lesson. It sets up your hardware camera stream, calculates running performance metrics, processes three distinct variations of the video feed, and tiles them cleanly on your screen.

import cv2
import time
from picamera2 import Picamera2
piCam = Picamera2()
W=640
H=360
tStart = time.time()
fps = 0
RES = (W,H)
piCam.preview_configuration.main.size = RES
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate=60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

topBar = 65
windowWaste = 25

cv2.namedWindow("Camera", cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow("Camera", W, H)
cv2.moveWindow("Camera",0,topBar)

cv2.namedWindow("Camera Small", cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow("Camera Small",  int(W/2), int(H/2))
cv2.moveWindow("Camera Small",0,topBar+windowWaste+H)
    
cv2.namedWindow("Gray Small", cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow("Gray Small", int(W/2), int(H/2))
cv2.moveWindow("Gray Small",int(W/2),topBar+windowWaste+H)


while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame= piCam.capture_array()
    frame=cv2.flip(frame,-1)
    frameSmall=cv2.resize(frame,(int(W/2),int(H/2)))
    frameGraySmall = cv2.cvtColor(frameSmall,cv2.COLOR_BGR2GRAY)

    myText = "FPS: "+str(round(fps,1))
    cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)
    cv2.imshow("Camera", frame)
    cv2.imshow("Camera Small",frameSmall)
    cv2.imshow("Gray Small",frameGraySmall)

    if cv2.waitKey(1)==ord('q'):
        break
cv2.destroyAllWindows()
print('Program Terminated')

import cv2

import time

from picamera2 import Picamera2

piCam = Picamera2()

W=640

H=360

tStart = time.time()

fps = 0

RES = (W,H)

piCam.preview_configuration.main.size = RES

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate=60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

topBar = 65

windowWaste = 25

cv2.namedWindow("Camera", cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow("Camera", W, H)

cv2.moveWindow("Camera",0,topBar)

cv2.namedWindow("Camera Small", cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow("Camera Small", int(W/2), int(H/2))

cv2.moveWindow("Camera Small",0,topBar+windowWaste+H)

cv2.namedWindow("Gray Small", cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow("Gray Small", int(W/2), int(H/2))

cv2.moveWindow("Gray Small",int(W/2),topBar+windowWaste+H)

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame= piCam.capture_array()

frame=cv2.flip(frame,-1)

frameSmall=cv2.resize(frame,(int(W/2),int(H/2)))

frameGraySmall = cv2.cvtColor(frameSmall,cv2.COLOR_BGR2GRAY)

myText = "FPS: "+str(round(fps,1))

cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Camera", frame)

cv2.imshow("Camera Small",frameSmall)

cv2.imshow("Gray Small",frameGraySmall)

if cv2.waitKey(1)==ord('q'):

break

cv2.destroyAllWindows()

print('Program Terminated')

AI On the Edge, Tutorial

AI on the Edge LESSON 18: Display Frames Per Second (FPS) on openCV Video Window

May 27, 2026 admin

In today’s lesson, we add a clean, real-time Frames Per Second (FPS) counter directly onto our live OpenCV video window. Displaying FPS on screen is an essential tool for anyone working with camera-based AI projects on the Raspberry Pi. It gives you immediate feedback on your actual processing performance, helps with optimization, and makes your projects look more professional and polished.

In this lesson, we configure the Picamera2 library to run at 1280×720 resolution with a target of 60 frames per second. We then implement a smoothed FPS calculation using a weighted rolling average, which prevents the displayed value from jumping around wildly. Finally, we overlay the FPS text in the lower-left corner of the video frame using OpenCV’s putText() function, with font size and thickness that scale appropriately with the resolution.

This technique forms an important foundation for future lessons, as we will continue adding more information and graphics directly onto the live video stream. Understanding how to efficiently display performance metrics is key to developing responsive and practical edge AI applications.

In this lesson, this is the code which we develop:

import cv2
import time
from picamera2 import Picamera2
piCam = Picamera2()
W=1280
H=720
tStart = time.time()
fps = 0
RES = (W,H)
piCam.preview_configuration.main.size = RES
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate=60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame= piCam.capture_array()
    frame=cv2.flip(frame,-1)
    myText = "FPS: "+str(round(fps,1))
    cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)
    cv2.imshow("Camera", frame)
    cv2.moveWindow("Camera",0,60)
    if cv2.waitKey(1)==ord('q'):
        break
cv2.destroyAllWindows()
print('Program Terminated')

import cv2

import time

from picamera2 import Picamera2

piCam = Picamera2()

W=1280

H=720

tStart = time.time()

fps = 0

RES = (W,H)

piCam.preview_configuration.main.size = RES

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate=60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame= piCam.capture_array()

frame=cv2.flip(frame,-1)

myText = "FPS: "+str(round(fps,1))

cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Camera", frame)

cv2.moveWindow("Camera",0,60)

if cv2.waitKey(1)==ord('q'):

break

cv2.destroyAllWindows()

print('Program Terminated')

AI On the Edge

AI on the Edge LESSON 17: Decorating and Annotating Video Frames in openCV

May 22, 2026 admin

Welcome to AI on the Edge – Lesson 17: Decorating and Annotating Video Frames in OpenCV. In this lesson we take our live video stream from the Raspberry Pi camera and start making it really useful and professional-looking. Now that we can grab frames and display them, it’s time to learn how to draw directly on top of those frames. We’re talking rectangles, lines, arrows, circles, and crisp text overlays — all the visual elements you’ll need when you start adding real AI like face detection or object recognition.

You’ll see exactly how to use OpenCV’s drawing functions to create clean, scalable annotations that look great whether you’re running at 320×180 for maximum speed or higher resolutions like 1280×720. We cover how to control line thickness, use filled shapes, position text properly, and most importantly, how to make all your drawings scale automatically with your chosen resolution so everything stays nicely proportioned.

By the end of this lesson you’ll have the skills to draw bounding boxes around detected objects, add confidence scores, label people or items, draw tracking lines — basically anything you need to show what your AI is seeing. This is one of those foundational skills that you’ll use over and over again in your computer vision projects.As always, I encourage you to type the code along with me in the video, then start playing with colors, sizes, positions, and messages. Change things around, break it, and make it your own. That’s the best way to really learn this stuff.

So fire up your Raspberry Pi 5, grab that camera, and let’s start turning raw video frames into clear, informative, and great-looking annotated output!

import cv2
from picamera2 import Picamera2
piCam = Picamera2()
W=1280
H=720
# W= 640
# H= 360
W=320
H=180
RES = (W,H)
piCam.preview_configuration.main.size = RES
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate=60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()
while True:
    frame= piCam.capture_array()
    frame=cv2.flip(frame,-1)

    upperLeft = (int((W-1)*.1),int((H-1)*.1))
    lowerRight = (int((W-1)*.9),int((H-1)*.9))
    myColor = (255,0,0)
    myThick = int(W/250)
    cv2.rectangle(frame,upperLeft,lowerRight,myColor,myThick)
    
    upperLeft = (int((W-1)*.3),int((H-1)*.3))
    lowerRight = (int((W-1)*.7),int((H-1)*.7))
    myColor = (0,0,255)
    myThick = int(W/250)
    myThick = -1
    cv2.rectangle(frame,upperLeft,lowerRight,myColor,myThick)
 
    startingPoint = (int((W-1)*.1),int((H-1)*.1))
    endingPoint = (int((W-1)*.9),int((H-1)*.9))
    myColor = (255,0,0)
    myThick = int(W/250)
    cv2.line(frame,startingPoint,endingPoint,myColor,myThick)
 
    startingPoint = (int((W-1)*.7),int((H-1)*.3))
    endingPoint = (int((W-1)*.3),int((H-1)*.7))
    myColor = (0,255,0)
    myThick = int(W/250)
    cv2.arrowedLine(frame,startingPoint,endingPoint,myColor,myThick)
 
    myCenter = (int((W-1)/2),int((H-1)/2))
    myRadius = int(.1*(H-1))
    myThick = int(W/250)
    myColor = (0,255,0)
    cv2.circle(frame,myCenter,myRadius,myColor,myThick)
    
    myCenter1 = (int((W-1)*.3),int((H-1)*.3))
    myCenter2 = (int((W-1)*.7),int((H-1)*.7))
    myCenter3 = (int((W-1)*.7),int((H-1)*.3))
    myCenter4 = (int((W-1)*.3),int((H-1)*.7))
    myRadius = int((W-1)*.05)
    myColor = (255,0,0)
    myThick= -1
    cv2.circle(frame,myCenter1,myRadius,myColor,myThick)
    cv2.circle(frame,myCenter2,myRadius,myColor,myThick)
    cv2.circle(frame,myCenter3,myRadius,myColor,myThick)
    cv2.circle(frame,myCenter4,myRadius,myColor,myThick)  

    myText = "My Excellent Label"
    textLowerLeft= (int(W*.01),int(H*.05))
    fontFace = cv2.FONT_HERSHEY_SIMPLEX
    fontThickness = int(W/450)
    fontScale = H*.0015
    fontColor = (255,0,255)
    cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale, fontColor, fontThickness)


    cv2.imshow("Camera", frame)
    

    cv2.moveWindow("Camera",0,60)
    if cv2.waitKey(1)==ord('q'):
        break
cv2.destroyAllWindows()
print('Program Terminated')

import cv2

from picamera2 import Picamera2

piCam = Picamera2()

W=1280

H=720

# W= 640

# H= 360

W=320

H=180

RES = (W,H)

piCam.preview_configuration.main.size = RES

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate=60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

while True:

frame= piCam.capture_array()

frame=cv2.flip(frame,-1)

upperLeft = (int((W-1)*.1),int((H-1)*.1))

lowerRight = (int((W-1)*.9),int((H-1)*.9))

myColor = (255,0,0)

myThick = int(W/250)

cv2.rectangle(frame,upperLeft,lowerRight,myColor,myThick)

upperLeft = (int((W-1)*.3),int((H-1)*.3))

lowerRight = (int((W-1)*.7),int((H-1)*.7))

myColor = (0,0,255)

myThick = int(W/250)

myThick = -1

cv2.rectangle(frame,upperLeft,lowerRight,myColor,myThick)

startingPoint = (int((W-1)*.1),int((H-1)*.1))

endingPoint = (int((W-1)*.9),int((H-1)*.9))

myColor = (255,0,0)

myThick = int(W/250)

cv2.line(frame,startingPoint,endingPoint,myColor,myThick)

startingPoint = (int((W-1)*.7),int((H-1)*.3))

endingPoint = (int((W-1)*.3),int((H-1)*.7))

myColor = (0,255,0)

myThick = int(W/250)

cv2.arrowedLine(frame,startingPoint,endingPoint,myColor,myThick)

myCenter = (int((W-1)/2),int((H-1)/2))

myRadius = int(.1*(H-1))

myThick = int(W/250)

myColor = (0,255,0)

cv2.circle(frame,myCenter,myRadius,myColor,myThick)

myCenter1 = (int((W-1)*.3),int((H-1)*.3))

myCenter2 = (int((W-1)*.7),int((H-1)*.7))

myCenter3 = (int((W-1)*.7),int((H-1)*.3))

myCenter4 = (int((W-1)*.3),int((H-1)*.7))

myRadius = int((W-1)*.05)

myColor = (255,0,0)

myThick= -1

cv2.circle(frame,myCenter1,myRadius,myColor,myThick)

cv2.circle(frame,myCenter2,myRadius,myColor,myThick)

cv2.circle(frame,myCenter3,myRadius,myColor,myThick)

cv2.circle(frame,myCenter4,myRadius,myColor,myThick)

myText = "My Excellent Label"

textLowerLeft= (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/450)

fontScale = H*.0015

fontColor = (255,0,255)

cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale, fontColor, fontThickness)

cv2.imshow("Camera", frame)

cv2.moveWindow("Camera",0,60)

if cv2.waitKey(1)==ord('q'):

break

cv2.destroyAllWindows()

print('Program Terminated')

AI On the Edge, Tutorial

AI on the Edge LESSON 16: Control Pan/Tilt Camera Position Using Voice Commands

May 20, 2026 admin

In AI on the Edge Lesson 16, we take a big step forward by combining voice recognition with physical motion. In this project, you will build a voice-controlled pan/tilt camera system. Using simple spoken commands such as “right,” “left,” “up,” “down,” and “quit,” you can move the Raspberry Pi camera in real time. This lesson brings together the Fusion HAT+ servo control, the Speech-to-Text (STT) capabilities we explored earlier, live video streaming with picamera2 and OpenCV, and multithreading to keep everything running smoothly.

The hardware setup is straightforward. We connect two servos to the Fusion HAT+ — one for pan (horizontal movement) on pin 2 and one for tilt (vertical movement) on pin 3. The Raspberry Pi Camera is mounted on a pan/tilt mechanism so it can physically follow your voice commands. We start the camera at a neutral position (pan = 0°, tilt = -20°) and define step sizes so the movement feels responsive but controlled.

The Python code uses two main threads: one for continuous voice listening and another for displaying the live video feed. In the listening thread, we create an STT object and continuously wait for voice input. When a command is recognized, we adjust the pan or tilt angle accordingly and immediately send the new position to the appropriate servo. The main loop captures frames from the Pi Camera, flips them for correct orientation, displays them in an OpenCV window, and checks for the ‘q’ key to exit gracefully.

This project demonstrates several important concepts working together: real-time voice command processing, servo motor control, camera streaming with picamera2 at 1280×720 resolution and 60 fps, and proper use of threading so that listening and video display do not block each other. You will also notice how we use global variables carefully to share the current pan and tilt positions between the threads.

By the end of this lesson, you will have a working voice-controlled camera that you can point anywhere you want just by talking to it. This is an excellent foundation for more advanced projects such as voice-controlled object tracking, security cameras, or interactive AI assistants that can both see and move.The complete code is provided below, along with explanations of the key sections. Feel free to experiment with different step sizes (xDelta and yDelta), starting angles, or even add new voice commands once you are comfortable with the basic version.

This is the code developed in the video lesson:

from fusion_hat.servo import Servo
from fusion_hat.stt import STT
import threading
import cv2
from picamera2 import Picamera2
piCam = Picamera2()

panPin = 2
tiltPin = 3

panServo = Servo(panPin)
tiltServo = Servo(tiltPin)

x = 0
y =-20

xDelta = 10
yDelta = 5

panServo.angle(x)
tiltServo.angle(y)

W=1280
H=720
RES = (W,H)
piCam.preview_configuration.main.size = RES
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate=60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

running = True
    
def listenThread():
    global running, x, y
    stt = STT(language = "en-us")
    while running:
        print("Listening . . .")
        result = stt.listen(stream = False)
        command = result.strip()
        print("Command was: ", command)
        if command == "right":
            x = x + xDelta
            panServo.angle(x)
        if command == "left":
            x= x - xDelta
            panServo.angle(x)
        if command == "up":
            y = y - yDelta
            tiltServo.angle(y)
        if command == "down":
            y = y + yDelta
            tiltServo.angle(y)
        if command == "quit":
            running = False
            break
    print("Thread Terminated")
myThread = threading.Thread(target = listenThread, daemon = True)
myThread.start()
while running:
    frame= piCam.capture_array()
    frame=cv2.flip(frame,-1)
    cv2.imshow("Camera", frame)
    cv2.moveWindow("Camera",0,60)
    if cv2.waitKey(1)==ord('q'):
        running = False
        break
cv2.destroyAllWindows()
piCam.stop()
print('Program Terminated')

from fusion_hat.servo import Servo

from fusion_hat.stt import STT

import threading

import cv2

from picamera2 import Picamera2

piCam = Picamera2()

panPin = 2

tiltPin = 3

panServo = Servo(panPin)

tiltServo = Servo(tiltPin)

x = 0

y =-20

xDelta = 10

yDelta = 5

panServo.angle(x)

tiltServo.angle(y)

W=1280

H=720

RES = (W,H)

piCam.preview_configuration.main.size = RES

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate=60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

running = True

def listenThread():

global running, x, y

stt = STT(language = "en-us")

while running:

print("Listening . . .")

result = stt.listen(stream = False)

command = result.strip()

print("Command was: ", command)

if command == "right":

x = x + xDelta

panServo.angle(x)

if command == "left":

x= x - xDelta

panServo.angle(x)

if command == "up":

y = y - yDelta

tiltServo.angle(y)

if command == "down":

y = y + yDelta

tiltServo.angle(y)

if command == "quit":

running = False

break

print("Thread Terminated")

myThread = threading.Thread(target = listenThread, daemon = True)

myThread.start()

while running:

frame= piCam.capture_array()

frame=cv2.flip(frame,-1)

cv2.imshow("Camera", frame)

cv2.moveWindow("Camera",0,60)

if cv2.waitKey(1)==ord('q'):

running = False

break

cv2.destroyAllWindows()

piCam.stop()

print('Program Terminated')

Technology Tutorials

Tag Archives: Fusion AI Lab Kit

AI on the Edge LESSON 21: Managing Multiple Windows in OpenCV on the Raspberry Pi

What You Learned in This Lesson

AI on the Edge LESSON 20: Resizing, Moving, Converting and Tiling Video frames in OpenCV

What We Cover in This Lesson

The Complete Lesson 20 Code

AI on the Edge LESSON 18: Display Frames Per Second (FPS) on openCV Video Window

AI on the Edge LESSON 17: Decorating and Annotating Video Frames in openCV

AI on the Edge LESSON 16: Control Pan/Tilt Camera Position Using Voice Commands

Making The World a Better Place One High Tech Project at a Time. Enjoy!