Tag Archives: Mediapipe

AI on the Edge LESSON 39: Understanding MediaPipe Data Structures

August 5, 2026 admin

In this video lesson I show you how to understand the data structures returned by MediaPipe. I show you how to peel the data structure back, to get at the useful information.

When you run face detection with MediaPipe, the results object it returns is not a normal dictionary or list. It is a special custom object called SolutionOutputs. The easiest way to explore it is to start by checking the main attribute: results.detections. This is a Python list that contains one entry for every face detected in the current frame. If no faces are found, results.detections will be None or an empty list.

To extract useful information, you loop through results.detections. Each item in that list is a Detection object. From this object, you can access two main things: the confidence score using detection.score[0], and the location data using detection.location_data. Inside location_data, you will find relative_bounding_box (which gives you xmin, ymin, width, and height as values between 0 and 1) and relative_keypoints (a list of 6 facial points such as eyes, nose, and mouth).

The standard method is to first get the frame’s height and width, then multiply the normalized values (like xmin and width) by the actual pixel dimensions of the image to convert them into usable pixel coordinates. You can then use these coordinates with OpenCV functions such as cv2.rectangle() for the box or cv2.circle() for the keypoints.

By using simple print(type()), print(dir()), and print() statements on results, results.detections, and individual detection objects, you can quickly discover the full structure. This step-by-step approach — starting from results → detections → individual detection → location_data — lets you reliably reach all the useful information MediaPipe provides.

Below is the code we developed in the video.

# ====================================================================
# DISCLAIMER:
# This code is provided as-is for educational and experimental 
# purposes only. The author makes no representations or warranties of 
# any kind concerning the safety, suitability, or accuracy of this 
# code. Use at your own risk. The author assumes no liability for any 
# damages, system failures, security breaches, or network issues 
# resulting from the use or implementation of this script.
# ====================================================================

import cv2
import time
from picamera2 import Picamera2
import mediapipe as mp
piCam = Picamera2(1)
W=1280
H=720
tStart = time.time()
fps = 0
RES = (W,H)
piCam.preview_configuration.main.size = RES
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate=60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

faceFinder = mp.solutions.face_detection.FaceDetection(
    model_selection = 0,
    min_detection_confidence = .5)
mpDraw = mp.solutions.drawing_utils

while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame= piCam.capture_array()
    frame=cv2.flip(frame,-1)
    
    rgbFrame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
    results = faceFinder.process(rgbFrame)
    if results.detections:
        for detection in results.detections:
            #print(detection.location_data.relative_bounding_box)
            #print(detection)
            bbox =detection.location_data.relative_bounding_box
            x = int(bbox.xmin*W)
            y = int(bbox.ymin*H)
            w = int(bbox.width*W)
            h = int(bbox.height*H)
            cv2.rectangle(frame,
                         (x,y),
                         (x+w,y+h),
                         (0,255,255),
                         3)
            keypoints =detection.location_data.relative_keypoints
            #print(keypoints)
            for kp in keypoints:
                kpX = int(kp.x*W)
                kpY = int(kp.y*H)
                cv2.circle(frame,
                           (kpX,kpY),
                           15,
                           (255,0,0),
                           -1)
    myText = "FPS: "+str(round(fps,1))
    cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)
    cv2.imshow("Camera", frame)
    cv2.moveWindow("Camera",0,60)
    if cv2.waitKey(1)==ord('q'):
        break
cv2.destroyAllWindows()
print('Program Terminated')

# ====================================================================

# DISCLAIMER:

# This code is provided as-is for educational and experimental

# purposes only. The author makes no representations or warranties of

# any kind concerning the safety, suitability, or accuracy of this

# code. Use at your own risk. The author assumes no liability for any

# damages, system failures, security breaches, or network issues

# resulting from the use or implementation of this script.

# ====================================================================

import cv2

import time

from picamera2 import Picamera2

import mediapipe as mp

piCam = Picamera2(1)

W=1280

H=720

tStart = time.time()

fps = 0

RES = (W,H)

piCam.preview_configuration.main.size = RES

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate=60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

faceFinder = mp.solutions.face_detection.FaceDetection(

model_selection = 0,

min_detection_confidence = .5)

mpDraw = mp.solutions.drawing_utils

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame= piCam.capture_array()

frame=cv2.flip(frame,-1)

rgbFrame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results = faceFinder.process(rgbFrame)

if results.detections:

for detection in results.detections:

#print(detection.location_data.relative_bounding_box)

#print(detection)

bbox =detection.location_data.relative_bounding_box

x = int(bbox.xmin*W)

y = int(bbox.ymin*H)

w = int(bbox.width*W)

h = int(bbox.height*H)

cv2.rectangle(frame,

(x,y),

(x+w,y+h),

(0,255,255),

keypoints =detection.location_data.relative_keypoints

#print(keypoints)

for kp in keypoints:

kpX = int(kp.x*W)

kpY = int(kp.y*H)

cv2.circle(frame,

(kpX,kpY),

15,

(255,0,0),

-1)

myText = "FPS: "+str(round(fps,1))

cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Camera", frame)

cv2.moveWindow("Camera",0,60)

if cv2.waitKey(1)==ord('q'):

break

cv2.destroyAllWindows()

print('Program Terminated')

AI On the Edge, Raspberry Pi

AI on the Edge 38: Using MediaPipe for Face Recognition on the Raspberry Pi 5

August 2, 2026 admin

In this video lesson we introduce you to MediaPipe. The OS we had you flash in LESSON 1 already has the MediaPipe framework installed, and all the needed and working dependencies. If you have installed that OS and not modified it, this and future lessons will work. If you find dependency errors, you might need to reflash the original OS.

MediaPipe is a free, open-source framework developed by Google that makes it much easier to add advanced computer vision and AI features to your Python programs. It is especially popular among developers who use OpenCV because it works seamlessly with it and delivers excellent real-time performance, even on devices like the Raspberry Pi 5.

With MediaPipe, you can quickly add powerful capabilities such as face detection, face mesh (detailed facial landmarks), hand tracking, body pose estimation, and more — all without having to write complex deep learning code from scratch. It comes with pre-trained machine learning models that are optimized for speed, allowing your programs to run smoothly at 30 frames per second or higher.

The biggest advantage for Python + OpenCV users is its simplicity. You capture video frames using OpenCV or picamera2, pass them to MediaPipe for processing, and then draw the results (such as bounding boxes or landmarks) back onto your image using normal OpenCV functions. This combination gives developers a fast and straightforward way to build interactive computer vision projects like face trackers, gesture-controlled robots, or smart camera applications.

In short, MediaPipe acts as a powerful, easy-to-use toolkit that bridges the gap between OpenCV and modern AI vision technology.

We introduce you to MediaPipe using a simple example where we create a faceFinder on our Raspberry Pi 5.

# ====================================================================
# DISCLAIMER:
# This code is provided as-is for educational and experimental 
# purposes only. The author makes no representations or warranties of 
# any kind concerning the safety, suitability, or accuracy of this 
# code. Use at your own risk. The author assumes no liability for any 
# damages, system failures, security breaches, or network issues 
# resulting from the use or implementation of this script.
# ====================================================================
import cv2
import time
from picamera2 import Picamera2
import mediapipe as mp
piCam = Picamera2()
W=1280
H=720
tStart = time.time()
fps = 0
RES = (W,H)
piCam.preview_configuration.main.size = RES
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate=60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

faceFinder = mp.solutions.face_detection.FaceDetection(
    model_selection = 0,
    min_detection_confidence = .5)
mpDraw = mp.solutions.drawing_utils

while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame= piCam.capture_array()
    #frame=cv2.flip(frame,-1)
    
    rgbFrame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
    results = faceFinder.process(rgbFrame)
    if results.detections:
        for detection in results.detections:
            mpDraw.draw_detection(frame,detection)
    
    myText = "FPS: "+str(round(fps,1))
    cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)
    cv2.imshow("Camera", frame)
    cv2.moveWindow("Camera",0,60)
    if cv2.waitKey(1)==ord('q'):
        break
cv2.destroyAllWindows()
print('Program Terminated')

# ====================================================================

# DISCLAIMER:

# This code is provided as-is for educational and experimental

# purposes only. The author makes no representations or warranties of

# any kind concerning the safety, suitability, or accuracy of this

# code. Use at your own risk. The author assumes no liability for any

# damages, system failures, security breaches, or network issues

# resulting from the use or implementation of this script.

# ====================================================================

import cv2

import time

from picamera2 import Picamera2

import mediapipe as mp

piCam = Picamera2()

W=1280

H=720

tStart = time.time()

fps = 0

RES = (W,H)

piCam.preview_configuration.main.size = RES

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate=60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

faceFinder = mp.solutions.face_detection.FaceDetection(

model_selection = 0,

min_detection_confidence = .5)

mpDraw = mp.solutions.drawing_utils

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame= piCam.capture_array()

#frame=cv2.flip(frame,-1)

rgbFrame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results = faceFinder.process(rgbFrame)

if results.detections:

for detection in results.detections:

mpDraw.draw_detection(frame,detection)

myText = "FPS: "+str(round(fps,1))

cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Camera", frame)

cv2.moveWindow("Camera",0,60)

if cv2.waitKey(1)==ord('q'):

break

cv2.destroyAllWindows()

print('Program Terminated')

Artificial Intelligence, OpenCV

AI for Everyone LESSON 29: Control of Real World Objects with Gesture Recognition in Mediapipe

February 24, 2022 admin

In this video lesson we show you how you can control objects in the real world using OpenCV, Python, Mediapipa and our old friend, the Arduino. On the Python side, we recognize hand gestures, and then we pass the recognized gesture to Arduino and Arduino lights LED in response to what hand signal is seen. This is a simple example, but a very powerful method. Instead of LED, you could operate servos, stepper motors or relays to control any manner of different devices. For your convenience, this is the code we used on the Arduino side:

String cmd;
int LED1=9;
int LED2=6;
int LED3=5;
int LED4=3;
int LED5=10;
void setup() {
  pinMode(LED1,OUTPUT);
  pinMode(LED2,OUTPUT);
  pinMode(LED3,OUTPUT);
   pinMode(LED4,OUTPUT);
   pinMode(LED5,OUTPUT);
 Serial.begin(115200);

}
void loop() {
 while (Serial.available()==0){
 
 }
 cmd = Serial.readStringUntil('\r');
if (cmd=="One"){
  digitalWrite(LED1,HIGH);
  digitalWrite(LED2,LOW);
  digitalWrite(LED3,LOW);
  digitalWrite(LED4,LOW);
  digitalWrite(LED5,LOW);
}
if (cmd=="Two"){
  digitalWrite(LED1,HIGH);
  digitalWrite(LED2,HIGH);
  digitalWrite(LED3,LOW);
  digitalWrite(LED4,LOW);
  digitalWrite(LED5,LOW);
}
 if (cmd=="Three"){
  digitalWrite(LED1,HIGH);
  digitalWrite(LED2,HIGH);
  digitalWrite(LED3,HIGH);
  digitalWrite(LED4,LOW);
  digitalWrite(LED5,LOW);
 }
 if (cmd=="Four"){
  digitalWrite(LED1,HIGH);
  digitalWrite(LED2,HIGH);
  digitalWrite(LED3,HIGH);
  digitalWrite(LED4,HIGH);
  digitalWrite(LED5,LOW);
 }
 if (cmd=="Five"){
  digitalWrite(LED1,HIGH);
  digitalWrite(LED2,HIGH);
  digitalWrite(LED3,HIGH);
  digitalWrite(LED4,HIGH);
  digitalWrite(LED5,HIGH);

 }

 if (cmd=="Pinky"){
  digitalWrite(LED1,LOW);
  digitalWrite(LED2,LOW);
  digitalWrite(LED3,LOW);
  digitalWrite(LED4,HIGH);
  digitalWrite(LED5,LOW);

 }
  if (cmd=="Thumb"){
  digitalWrite(LED1,LOW);
  digitalWrite(LED2,LOW);
  digitalWrite(LED3,LOW);
  digitalWrite(LED4,LOW);
  digitalWrite(LED5,HIGH);

 }
  if (cmd=="Inside"){
  digitalWrite(LED1,LOW);
  digitalWrite(LED2,HIGH);
  digitalWrite(LED3,HIGH);
  digitalWrite(LED4,LOW);
  digitalWrite(LED5,LOW);

 }
   if (cmd=="Outside"){
  digitalWrite(LED1,HIGH);
  digitalWrite(LED2,LOW);
  digitalWrite(LED3,LOW);
  digitalWrite(LED4,HIGH);
  digitalWrite(LED5,LOW);

 }
    if (cmd=="Unknown"){
  digitalWrite(LED1,LOW);
  digitalWrite(LED2,LOW);
  digitalWrite(LED3,LOW);
  digitalWrite(LED4,LOW);
  digitalWrite(LED5,LOW);

 }
}

String cmd;

int LED1=9;

int LED2=6;

int LED3=5;

int LED4=3;

int LED5=10;

void setup() {

pinMode(LED1,OUTPUT);

pinMode(LED2,OUTPUT);

pinMode(LED3,OUTPUT);

pinMode(LED4,OUTPUT);

pinMode(LED5,OUTPUT);

Serial.begin(115200);

}

void loop() {

while (Serial.available()==0){

}

cmd = Serial.readStringUntil('\r');

if (cmd=="One"){

digitalWrite(LED1,HIGH);

digitalWrite(LED2,LOW);

digitalWrite(LED3,LOW);

digitalWrite(LED4,LOW);

digitalWrite(LED5,LOW);

}

if (cmd=="Two"){

digitalWrite(LED1,HIGH);

digitalWrite(LED2,HIGH);

digitalWrite(LED3,LOW);

digitalWrite(LED4,LOW);

digitalWrite(LED5,LOW);

}

if (cmd=="Three"){

digitalWrite(LED1,HIGH);

digitalWrite(LED2,HIGH);

digitalWrite(LED3,HIGH);

digitalWrite(LED4,LOW);

digitalWrite(LED5,LOW);

}

if (cmd=="Four"){

digitalWrite(LED1,HIGH);

digitalWrite(LED2,HIGH);

digitalWrite(LED3,HIGH);

digitalWrite(LED4,HIGH);

digitalWrite(LED5,LOW);

}

if (cmd=="Five"){

digitalWrite(LED1,HIGH);

digitalWrite(LED2,HIGH);

digitalWrite(LED3,HIGH);

digitalWrite(LED4,HIGH);

digitalWrite(LED5,HIGH);

}

if (cmd=="Pinky"){

digitalWrite(LED1,LOW);

digitalWrite(LED2,LOW);

digitalWrite(LED3,LOW);

digitalWrite(LED4,HIGH);

digitalWrite(LED5,LOW);

}

if (cmd=="Thumb"){

digitalWrite(LED1,LOW);

digitalWrite(LED2,LOW);

digitalWrite(LED3,LOW);

digitalWrite(LED4,LOW);

digitalWrite(LED5,HIGH);

}

if (cmd=="Inside"){

digitalWrite(LED1,LOW);

digitalWrite(LED2,HIGH);

digitalWrite(LED3,HIGH);

digitalWrite(LED4,LOW);

digitalWrite(LED5,LOW);

}

if (cmd=="Outside"){

digitalWrite(LED1,HIGH);

digitalWrite(LED2,LOW);

digitalWrite(LED3,LOW);

digitalWrite(LED4,HIGH);

digitalWrite(LED5,LOW);

}

if (cmd=="Unknown"){

digitalWrite(LED1,LOW);

digitalWrite(LED2,LOW);

digitalWrite(LED3,LOW);

digitalWrite(LED4,LOW);

digitalWrite(LED5,LOW);

}

And on the python side, we used the following code.

import time
import cv2
print(cv2.__version__)
import numpy as np
import pickle
import serial

arduinoData = serial.Serial('COM3',115200)

class mpHands:
    import mediapipe as mp
    def __init__(self,maxHands=2,tol1=.5,tol2=.5):
        self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)
    def Marks(self,frame):
        myHands=[]
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.hands.process(frameRGB)
        if results.multi_hand_landmarks != None:
            for handLandMarks in results.multi_hand_landmarks:
                myHand=[]
                for landMark in handLandMarks.landmark:
                    myHand.append((int(landMark.x*width),int(landMark.y*height)))
                myHands.append(myHand)
        return myHands
def findDistances(handData):
    distMatrix=np.zeros([len(handData),len(handData)],dtype='float')
    palmSize=((handData[0][0]-handData[9][0])**2+(handData[0][1]-handData[9][1])**2)**(1./2.)
    for row in range(0,len(handData)):
        for column in range(0,len(handData)):
            distMatrix[row][column]=(((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.))/palmSize
    return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):
    error=0
    for row in keyPoints:
        for column in keyPoints:
            error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])
    print(error)
    return error
def findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol):
    errorArray=[]
    for i in range(0,len(gestNames),1):
        error=findError(knownGestures[i],unknownGesture,keyPoints)
        errorArray.append(error)
    errorMin=errorArray[0]
    minIndex=0
    for i in range(0,len(errorArray),1):
        if errorArray[i]<errorMin:
            errorMin=errorArray[i]
            minIndex=i
    if errorMin<tol:
        gesture=gestNames[minIndex]
    if errorMin>=tol:
        gesture='Unknown'
    return gesture

width=1280
height=720
cam=cv2.VideoCapture(3,cv2.CAP_DSHOW)
cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)
cam.set(cv2.CAP_PROP_FPS, 30)
cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))
findHands=mpHands(1)
time.sleep(5)
keyPoints=[0,4,5,9,13,17,8,12,16,20]

train=int(input('Enter 1 to Train, Enter 0 to Recognize '))
if train==1:
    trainCnt=0
    knownGestures=[]
    numGest=int(input('How Many Gestures Do You Want? '))
    gestNames=[]
    for i in range(0,numGest,1):
        prompt='Name of Gesture #'+str(i+1)+' '
        name=input(prompt)
        gestNames.append(name)
    print(gestNames)
    trainName=input('Filename for training data? (Press Enter for Default) ')
    if trainName=='':
        trainName='default'
    trainName=trainName+'.pkl'
if train==0:
    trainName=input('What Training Data Do You Want to Use? (Press Enter for Default) ')
    if trainName=='':
        trainName='default'
    trainName=trainName+'.pkl'
    with open(trainName,'rb') as f:
        gestNames=pickle.load(f)
        knownGestures=pickle.load(f)

tol=10

while True:
    ignore,  frame = cam.read()
    frame=cv2.resize(frame,(width,height))
    handData=findHands.Marks(frame)
    if train==1:
        if handData!=[]:
            print('Please Show Gesture ',gestNames[trainCnt],': Press t when Ready')
            if cv2.waitKey(1) & 0xff==ord('t'):
                knownGesture=findDistances(handData[0])
                knownGestures.append(knownGesture)
                trainCnt=trainCnt+1
                if trainCnt==numGest:
                    train=0
                    with open(trainName,'wb') as f:
                        pickle.dump(gestNames,f)
                        pickle.dump(knownGestures,f)
    if train == 0:
        if handData!=[]:
            unknownGesture=findDistances(handData[0])
            myGesture=findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol)
            #error=findError(knownGesture,unknownGesture,keyPoints)
            cv2.putText(frame,myGesture,(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)
            myGesture=myGesture+'\r'
            arduinoData.write(myGesture.encode())
    for hand in handData:
        for ind in keyPoints:
            cv2.circle(frame,hand[ind],25,(255,0,255),3)
    cv2.imshow('my WEBcam', frame)
    cv2.moveWindow('my WEBcam',0,0)
    if cv2.waitKey(1) & 0xff ==ord('q'):
        break
cam.release()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

import time

import cv2

print(cv2.__version__)

import numpy as np

import pickle

import serial

arduinoData = serial.Serial('COM3',115200)

class mpHands:

import mediapipe as mp

def __init__(self,maxHands=2,tol1=.5,tol2=.5):

self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)

def Marks(self,frame):

myHands=[]

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.hands.process(frameRGB)

if results.multi_hand_landmarks != None:

for handLandMarks in results.multi_hand_landmarks:

myHand=[]

for landMark in handLandMarks.landmark:

myHand.append((int(landMark.x*width),int(landMark.y*height)))

myHands.append(myHand)

return myHands

def findDistances(handData):

distMatrix=np.zeros([len(handData),len(handData)],dtype='float')

palmSize=((handData[0][0]-handData[9][0])**2+(handData[0][1]-handData[9][1])**2)**(1./2.)

for row in range(0,len(handData)):

for column in range(0,len(handData)):

distMatrix[row][column]=(((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.))/palmSize

return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):

error=0

for row in keyPoints:

for column in keyPoints:

error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])

print(error)

return error

def findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol):

errorArray=[]

for i in range(0,len(gestNames),1):

error=findError(knownGestures[i],unknownGesture,keyPoints)

errorArray.append(error)

errorMin=errorArray[0]

minIndex=0

for i in range(0,len(errorArray),1):

if errorArray[i]<errorMin:

errorMin=errorArray[i]

minIndex=i

if errorMin<tol:

gesture=gestNames[minIndex]

if errorMin>=tol:

gesture='Unknown'

return gesture

width=1280

height=720

cam=cv2.VideoCapture(3,cv2.CAP_DSHOW)

cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)

cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)

cam.set(cv2.CAP_PROP_FPS, 30)

cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(1)

time.sleep(5)

keyPoints=[0,4,5,9,13,17,8,12,16,20]

train=int(input('Enter 1 to Train, Enter 0 to Recognize '))

if train==1:

trainCnt=0

knownGestures=[]

numGest=int(input('How Many Gestures Do You Want? '))

gestNames=[]

for i in range(0,numGest,1):

prompt='Name of Gesture #'+str(i+1)+' '

name=input(prompt)

gestNames.append(name)

print(gestNames)

trainName=input('Filename for training data? (Press Enter for Default) ')

if trainName=='':

trainName='default'

trainName=trainName+'.pkl'

if train==0:

trainName=input('What Training Data Do You Want to Use? (Press Enter for Default) ')

if trainName=='':

trainName='default'

trainName=trainName+'.pkl'

with open(trainName,'rb') as f:

gestNames=pickle.load(f)

knownGestures=pickle.load(f)

tol=10

while True:

ignore, frame = cam.read()

frame=cv2.resize(frame,(width,height))

handData=findHands.Marks(frame)

if train==1:

if handData!=[]:

print('Please Show Gesture ',gestNames[trainCnt],': Press t when Ready')

if cv2.waitKey(1) & 0xff==ord('t'):

knownGesture=findDistances(handData[0])

knownGestures.append(knownGesture)

trainCnt=trainCnt+1

if trainCnt==numGest:

train=0

with open(trainName,'wb') as f:

pickle.dump(gestNames,f)

pickle.dump(knownGestures,f)

if train == 0:

if handData!=[]:

unknownGesture=findDistances(handData[0])

myGesture=findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol)

#error=findError(knownGesture,unknownGesture,keyPoints)

cv2.putText(frame,myGesture,(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)

myGesture=myGesture+'\r'

arduinoData.write(myGesture.encode())

for hand in handData:

for ind in keyPoints:

cv2.circle(frame,hand[ind],25,(255,0,255),3)

cv2.imshow('my WEBcam', frame)

cv2.moveWindow('my WEBcam',0,0)

if cv2.waitKey(1) & 0xff ==ord('q'):

break

cam.release()

Artificial Intelligence, OpenCV, Python, Tutorial

Improved Gesture Recognition in Python and MediaPipe

February 10, 2022 admin

In this video lesson we show you how you can improve the accuracy of your gesture recognition program developed in the last lesson. We do this by normalizing the hand landmarks distance matrix to a standard size. By doing this, you get accurate results independent of the distance your hand is from the camera. For your convenience, I include the code below which we develop in this lesson. Enjoy!

import time
import cv2
print(cv2.__version__)
import numpy as np

class mpHands:
    import mediapipe as mp
    def __init__(self,maxHands=2,tol1=.5,tol2=.5):
        self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)
    def Marks(self,frame):
        myHands=[]
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.hands.process(frameRGB)
        if results.multi_hand_landmarks != None:
            for handLandMarks in results.multi_hand_landmarks:
                myHand=[]
                for landMark in handLandMarks.landmark:
                    myHand.append((int(landMark.x*width),int(landMark.y*height)))
                myHands.append(myHand)
        return myHands
def findDistances(handData):
    distMatrix=np.zeros([len(handData),len(handData)],dtype='float')
    palmSize=((handData[0][0]-handData[9][0])**2+(handData[0][1]-handData[9][1])**2)**(1./2.)
    for row in range(0,len(handData)):
        for column in range(0,len(handData)):
            distMatrix[row][column]=(((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.))/palmSize
    return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):
    error=0
    for row in keyPoints:
        for column in keyPoints:
            error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])
    print(error)
    return error
def findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol):
    errorArray=[]
    for i in range(0,len(gestNames),1):
        error=findError(knownGestures[i],unknownGesture,keyPoints)
        errorArray.append(error)
    errorMin=errorArray[0]
    minIndex=0
    for i in range(0,len(errorArray),1):
        if errorArray[i]<errorMin:
            errorMin=errorArray[i]
            minIndex=i
    if errorMin<tol:
        gesture=gestNames[minIndex]
    if errorMin>=tol:
        gesture='Unknown'
    return gesture


width=1280
height=720
cam=cv2.VideoCapture(4,cv2.CAP_DSHOW)
cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)
cam.set(cv2.CAP_PROP_FPS, 30)
cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))
findHands=mpHands(1)
time.sleep(5)
keyPoints=[0,4,5,9,13,17,8,12,16,20]
train=True
tol=10
trainCnt=0
knownGestures=[]

numGest=int(input('How Many Gestures Do You Want? '))

gestNames=[]

for i in range(0,numGest,1):
    prompt='Name of Gesture #'+str(i+1)+' '
    name=input(prompt)
    gestNames.append(name)
print(gestNames)

while True:
    ignore,  frame = cam.read()
    frame=cv2.resize(frame,(width,height))
    handData=findHands.Marks(frame)
    if train==True:
        if handData!=[]:
            print('Please Show Gesture ',gestNames[trainCnt],': Press t when Ready')
            if cv2.waitKey(1) & 0xff==ord('t'):
                knownGesture=findDistances(handData[0])
                knownGestures.append(knownGesture)
                trainCnt=trainCnt+1
                if trainCnt==numGest:
                    train=False
    if train == False:
        if handData!=[]:
            unknownGesture=findDistances(handData[0])
            myGesture=findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol)
            #error=findError(knownGesture,unknownGesture,keyPoints)
            cv2.putText(frame,myGesture,(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)
    for hand in handData:
        for ind in keyPoints:
            cv2.circle(frame,hand[ind],25,(255,0,255),3)
    cv2.imshow('my WEBcam', frame)
    cv2.moveWindow('my WEBcam',0,0)
    if cv2.waitKey(1) & 0xff ==ord('q'):
        break
cam.release()

100

101

102

103

104

105

import time

import cv2

print(cv2.__version__)

import numpy as np

class mpHands:

import mediapipe as mp

def __init__(self,maxHands=2,tol1=.5,tol2=.5):

self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)

def Marks(self,frame):

myHands=[]

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.hands.process(frameRGB)

if results.multi_hand_landmarks != None:

for handLandMarks in results.multi_hand_landmarks:

myHand=[]

for landMark in handLandMarks.landmark:

myHand.append((int(landMark.x*width),int(landMark.y*height)))

myHands.append(myHand)

return myHands

def findDistances(handData):

distMatrix=np.zeros([len(handData),len(handData)],dtype='float')

palmSize=((handData[0][0]-handData[9][0])**2+(handData[0][1]-handData[9][1])**2)**(1./2.)

for row in range(0,len(handData)):

for column in range(0,len(handData)):

distMatrix[row][column]=(((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.))/palmSize

return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):

error=0

for row in keyPoints:

for column in keyPoints:

error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])

print(error)

return error

def findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol):

errorArray=[]

for i in range(0,len(gestNames),1):

error=findError(knownGestures[i],unknownGesture,keyPoints)

errorArray.append(error)

errorMin=errorArray[0]

minIndex=0

for i in range(0,len(errorArray),1):

if errorArray[i]<errorMin:

errorMin=errorArray[i]

minIndex=i

if errorMin<tol:

gesture=gestNames[minIndex]

if errorMin>=tol:

gesture='Unknown'

return gesture

width=1280

height=720

cam=cv2.VideoCapture(4,cv2.CAP_DSHOW)

cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)

cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)

cam.set(cv2.CAP_PROP_FPS, 30)

cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(1)

time.sleep(5)

keyPoints=[0,4,5,9,13,17,8,12,16,20]

train=True

tol=10

trainCnt=0

knownGestures=[]

numGest=int(input('How Many Gestures Do You Want? '))

gestNames=[]

for i in range(0,numGest,1):

prompt='Name of Gesture #'+str(i+1)+' '

name=input(prompt)

gestNames.append(name)

print(gestNames)

while True:

ignore, frame = cam.read()

frame=cv2.resize(frame,(width,height))

handData=findHands.Marks(frame)

if train==True:

if handData!=[]:

print('Please Show Gesture ',gestNames[trainCnt],': Press t when Ready')

if cv2.waitKey(1) & 0xff==ord('t'):

knownGesture=findDistances(handData[0])

knownGestures.append(knownGesture)

trainCnt=trainCnt+1

if trainCnt==numGest:

train=False

if train == False:

if handData!=[]:

unknownGesture=findDistances(handData[0])

myGesture=findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol)

#error=findError(knownGesture,unknownGesture,keyPoints)

cv2.putText(frame,myGesture,(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)

for hand in handData:

for ind in keyPoints:

cv2.circle(frame,hand[ind],25,(255,0,255),3)

cv2.imshow('my WEBcam', frame)

cv2.moveWindow('my WEBcam',0,0)

if cv2.waitKey(1) & 0xff ==ord('q'):

break

cam.release()

s lesson. Enjoy!

Artificial Intelligence, Python

AI for Everyone LESSON 26: Accurate Gesture Recognition using Python and MediaPipe

February 3, 2022 admin

In this lesson we demonstrate how to use mediaPipe and Python to create an AI system that can accurately recognize hand gestures. This is follow on work to what we developed in Lesson 25. The code we developed is presented below.

import time
import cv2
print(cv2.__version__)
import numpy as np

class mpHands:
    import mediapipe as mp
    def __init__(self,maxHands=2,tol1=.5,tol2=.5):
        self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)
    def Marks(self,frame):
        myHands=[]
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.hands.process(frameRGB)
        if results.multi_hand_landmarks != None:
            for handLandMarks in results.multi_hand_landmarks:
                myHand=[]
                for landMark in handLandMarks.landmark:
                    myHand.append((int(landMark.x*width),int(landMark.y*height)))
                myHands.append(myHand)
        return myHands
def findDistances(handData):
    distMatrix=np.zeros([len(handData),len(handData)],dtype='float')
    for row in range(0,len(handData)):
        for column in range(0,len(handData)):
            distMatrix[row][column]=((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.)
    return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):
    error=0
    for row in keyPoints:
        for column in keyPoints:
            error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])
    return error
def findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol):
    errorArray=[]
    for i in range(0,len(gestNames),1):
        error=findError(knownGestures[i],unknownGesture,keyPoints)
        errorArray.append(error)
    errorMin=errorArray[0]
    minIndex=0
    for i in range(0,len(errorArray),1):
        if errorArray[i]<errorMin:
            errorMin=errorArray[i]
            minIndex=i
    if errorMin<tol:
        gesture=gestNames[minIndex]
    if errorMin>=tol:
        gesture='Unknown'
    return gesture


width=1280
height=720
cam=cv2.VideoCapture(0,cv2.CAP_DSHOW)
cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)
cam.set(cv2.CAP_PROP_FPS, 30)
cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))
findHands=mpHands(1)
time.sleep(5)
keyPoints=[0,4,5,9,13,17,8,12,16,20]
train=True
tol=1500
trainCnt=0
knownGestures=[]

numGest=int(input('How Many Gestures Do You Want? '))

gestNames=[]

for i in range(0,numGest,1):
    prompt='Name of Gesture #'+str(i+1)+' '
    name=input(prompt)
    gestNames.append(name)
print(gestNames)

while True:
    ignore,  frame = cam.read()
    frame=cv2.resize(frame,(width,height))
    handData=findHands.Marks(frame)
    if train==True:
        if handData!=[]:
            print('Please Show Gesture ',gestNames[trainCnt],': Press t when Ready')
            if cv2.waitKey(1) & 0xff==ord('t'):
                knownGesture=findDistances(handData[0])
                knownGestures.append(knownGesture)
                trainCnt=trainCnt+1
                if trainCnt==numGest:
                    train=False
    if train == False:
        if handData!=[]:
            unknownGesture=findDistances(handData[0])
            myGesture=findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol)
            #error=findError(knownGesture,unknownGesture,keyPoints)
            cv2.putText(frame,myGesture,(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)
    for hand in handData:
        for ind in keyPoints:
            cv2.circle(frame,hand[ind],25,(255,0,255),3)
    cv2.imshow('my WEBcam', frame)
    cv2.moveWindow('my WEBcam',0,0)
    if cv2.waitKey(1) & 0xff ==ord('q'):
        break
cam.release()

100

101

102

103

import time

import cv2

print(cv2.__version__)

import numpy as np

class mpHands:

import mediapipe as mp

def __init__(self,maxHands=2,tol1=.5,tol2=.5):

self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)

def Marks(self,frame):

myHands=[]

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.hands.process(frameRGB)

if results.multi_hand_landmarks != None:

for handLandMarks in results.multi_hand_landmarks:

myHand=[]

for landMark in handLandMarks.landmark:

myHand.append((int(landMark.x*width),int(landMark.y*height)))

myHands.append(myHand)

return myHands

def findDistances(handData):

distMatrix=np.zeros([len(handData),len(handData)],dtype='float')

for row in range(0,len(handData)):

for column in range(0,len(handData)):

distMatrix[row][column]=((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.)

return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):

error=0

for row in keyPoints:

for column in keyPoints:

error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])

return error

def findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol):

errorArray=[]

for i in range(0,len(gestNames),1):

error=findError(knownGestures[i],unknownGesture,keyPoints)

errorArray.append(error)

errorMin=errorArray[0]

minIndex=0

for i in range(0,len(errorArray),1):

if errorArray[i]<errorMin:

errorMin=errorArray[i]

minIndex=i

if errorMin<tol:

gesture=gestNames[minIndex]

if errorMin>=tol:

gesture='Unknown'

return gesture

width=1280

height=720

cam=cv2.VideoCapture(0,cv2.CAP_DSHOW)

cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)

cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)

cam.set(cv2.CAP_PROP_FPS, 30)

cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(1)

time.sleep(5)

keyPoints=[0,4,5,9,13,17,8,12,16,20]

train=True

tol=1500

trainCnt=0

knownGestures=[]

numGest=int(input('How Many Gestures Do You Want? '))

gestNames=[]

for i in range(0,numGest,1):

prompt='Name of Gesture #'+str(i+1)+' '

name=input(prompt)

gestNames.append(name)

print(gestNames)

while True:

ignore, frame = cam.read()

frame=cv2.resize(frame,(width,height))

handData=findHands.Marks(frame)

if train==True:

if handData!=[]:

print('Please Show Gesture ',gestNames[trainCnt],': Press t when Ready')

if cv2.waitKey(1) & 0xff==ord('t'):

knownGesture=findDistances(handData[0])

knownGestures.append(knownGesture)

trainCnt=trainCnt+1

if trainCnt==numGest:

train=False

if train == False:

if handData!=[]:

unknownGesture=findDistances(handData[0])

myGesture=findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol)

#error=findError(knownGesture,unknownGesture,keyPoints)

cv2.putText(frame,myGesture,(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)

for hand in handData:

for ind in keyPoints:

cv2.circle(frame,hand[ind],25,(255,0,255),3)

cv2.imshow('my WEBcam', frame)

cv2.moveWindow('my WEBcam',0,0)

if cv2.waitKey(1) & 0xff ==ord('q'):

break

cam.release()

Technology Tutorials

Tag Archives: Mediapipe

AI on the Edge LESSON 39: Understanding MediaPipe Data Structures

AI on the Edge 38: Using MediaPipe for Face Recognition on the Raspberry Pi 5

AI for Everyone LESSON 29: Control of Real World Objects with Gesture Recognition in Mediapipe

Improved Gesture Recognition in Python and MediaPipe

AI for Everyone LESSON 26: Accurate Gesture Recognition using Python and MediaPipe

Making The World a Better Place One High Tech Project at a Time. Enjoy!