In this lesson we describe what our approach will be to creating a gesture detection and recognition algorithm using mediaPipe. We will demonstrate the algorithm with some simple code demos. In future lessons we will further develop the algorithm. The code below is the initial algorithm developed in this video. Enjoy!

import cv2
print(cv2.__version__)
import numpy as np

class mpHands:
    import mediapipe as mp
    def __init__(self,maxHands=2,tol1=.5,tol2=.5):
        self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)
    def Marks(self,frame):
        myHands=[]
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.hands.process(frameRGB)
        if results.multi_hand_landmarks != None:
            for handLandMarks in results.multi_hand_landmarks:
                myHand=[]
                for landMark in handLandMarks.landmark:
                    myHand.append((int(landMark.x*width),int(landMark.y*height)))
                myHands.append(myHand)
        return myHands
def findDistances(handData):
    distMatrix=np.zeros([len(handData),len(handData)],dtype='float')
    for row in range(0,len(handData)):
        for column in range(0,len(handData)):
            distMatrix[row][column]=((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.)
    return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):
    error=0
    for row in keyPoints:
        for column in keyPoints:
            error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])
    return error


width=1280
height=720
cam=cv2.VideoCapture(0,cv2.CAP_DSHOW)
cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)
cam.set(cv2.CAP_PROP_FPS, 30)
cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))
findHands=mpHands(1)

keyPoints=[0,4,5,9,13,17,8,12,16,20]
train=True
while True:
    ignore,  frame = cam.read()
    frame=cv2.resize(frame,(width,height))
    handData=findHands.Marks(frame)
    if train==True:
        if handData!=[]:
            print('Show your Gesture, press t when Ready')
            if cv2.waitKey(1) & 0xff==ord('t'):
                knownGesture=findDistances(handData[0])
                train=False
                print(knownGesture)
    if train == False:
        if handData!=[]:
            unknownGesture=findDistances(handData[0])
            error=findError(knownGesture,unknownGesture,keyPoints)
            cv2.putText(frame,str(int(error,)),(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)
    for hand in handData:
        for ind in keyPoints:
            cv2.circle(frame,hand[ind],25,(255,0,255),3)
    cv2.imshow('my WEBcam', frame)
    cv2.moveWindow('my WEBcam',0,0)
    if cv2.waitKey(1) & 0xff ==ord('q'):
        break
cam.release()

import cv2

print(cv2.__version__)

import numpy as np

class mpHands:

import mediapipe as mp

def __init__(self,maxHands=2,tol1=.5,tol2=.5):

self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)

def Marks(self,frame):

myHands=[]

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.hands.process(frameRGB)

if results.multi_hand_landmarks != None:

for handLandMarks in results.multi_hand_landmarks:

myHand=[]

for landMark in handLandMarks.landmark:

myHand.append((int(landMark.x*width),int(landMark.y*height)))

myHands.append(myHand)

return myHands

def findDistances(handData):

distMatrix=np.zeros([len(handData),len(handData)],dtype='float')

for row in range(0,len(handData)):

for column in range(0,len(handData)):

distMatrix[row][column]=((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.)

return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):

error=0

for row in keyPoints:

for column in keyPoints:

error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])

return error

width=1280

height=720

cam=cv2.VideoCapture(0,cv2.CAP_DSHOW)

cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)

cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)

cam.set(cv2.CAP_PROP_FPS, 30)

cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(1)

keyPoints=[0,4,5,9,13,17,8,12,16,20]

train=True

while True:

ignore, frame = cam.read()

frame=cv2.resize(frame,(width,height))

handData=findHands.Marks(frame)

if train==True:

if handData!=[]:

print('Show your Gesture, press t when Ready')

if cv2.waitKey(1) & 0xff==ord('t'):

knownGesture=findDistances(handData[0])

train=False

print(knownGesture)

if train == False:

if handData!=[]:

unknownGesture=findDistances(handData[0])

error=findError(knownGesture,unknownGesture,keyPoints)

cv2.putText(frame,str(int(error,)),(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)

for hand in handData:

for ind in keyPoints:

cv2.circle(frame,hand[ind],25,(255,0,255),3)

cv2.imshow('my WEBcam', frame)

cv2.moveWindow('my WEBcam',0,0)

if cv2.waitKey(1) & 0xff ==ord('q'):

break

cam.release()

Artificial Intelligence, OpenCV, Python

Parsing Mediapipe Data for Face Mesh, Pose, and Hand Landmarks

December 9, 2021 admin

We all love the incredibly powerful features of the Mediapipe AI library. But, lets face it, the data is very hard to interpret, and the returned data structures are complex, and poorly documented. The purpose of this lesson is to show you how to parse the complex data structures coming from Mediapipe, and to create simple, intuitive arrays for the landmark data. We create classes in python which do the parsing, and the classes are easy to work with. With these classes, you can use mediapipe simply, and get simple to understand data structures back. The video explains what we are doing and how we are doing it. For your convenience the resulting code is posted below.

import cv2
print(cv2.__version__)

class mpFaceMesh:
    import mediapipe as mp
    def __init__(self,still=False,numFaces=3,tol1=.5,tol2=.5,drawMesh=True):
        self.myFaceMesh=self.mp.solutions.face_mesh.FaceMesh(still,numFaces,tol1,tol2)
        self.myDraw=self.mp.solutions.drawing_utils
        self.draw=drawMesh
    def Marks(self,frame):
        global width
        global height
        drawSpecCircle=self.myDraw.DrawingSpec(thickness=0,circle_radius=0,color=(0,0,255))
        drawSpecLine=self.myDraw.DrawingSpec(thickness=3,circle_radius=2,color=(255,0,0))
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.myFaceMesh.process(frameRGB)
        facesMeshLandmarks=[]
        if results.multi_face_landmarks !=None:
            for faceMesh in results.multi_face_landmarks:
                faceMeshLandmarks=[]
                for lm in faceMesh.landmark:
                    loc=(int(lm.x*width),int(lm.y*height))
                    faceMeshLandmarks.append(loc)
                facesMeshLandmarks.append(faceMeshLandmarks)
                if self.draw==True:
                    self.myDraw.draw_landmarks(frame,faceMesh,self.mp.solutions.face_mesh.FACE_CONNECTIONS,drawSpecCircle,drawSpecLine)
        return facesMeshLandmarks

class mpFace:
    import mediapipe as mp 
    def __init__(self):
        self.myFace=self.mp.solutions.face_detection.FaceDetection()
    def Marks(self,frame):
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.myFace.process(frameRGB)
        faceBoundBoxs=[]
        if results.detections != None:
            for face in results.detections:
                bBox=face.location_data.relative_bounding_box
                topLeft=(int(bBox.xmin*width),int(bBox.ymin*height))
                bottomRight=(int((bBox.xmin+bBox.width)*width),int((bBox.ymin+bBox.height)*height))
                faceBoundBoxs.append((topLeft,bottomRight))
        return faceBoundBoxs

class mpPose:
    import mediapipe as mp
    def __init__(self,still=False,upperBody=False, smoothData=True, tol1=.5, tol2=.5):
        self.myPose=self.mp.solutions.pose.Pose(still,upperBody,smoothData,tol1,tol2)
    def Marks(self,frame):
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.myPose.process(frameRGB)
        poseLandmarks=[]
        if results.pose_landmarks:
            for lm in results.pose_landmarks.landmark:            
                poseLandmarks.append((int(lm.x*width),int(lm.y*height)))
        return poseLandmarks

class mpHands:
    import mediapipe as mp
    def __init__(self,maxHands=2,tol1=.5,tol2=.5):
        self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)
    def Marks(self,frame):
        myHands=[]
        handsType=[]
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.hands.process(frameRGB)
        if results.multi_hand_landmarks != None:
            #print(results.multi_handedness)
            for hand in results.multi_handedness:
                #print(hand)
                #print(hand.classification)
                #print(hand.classification[0])
                handType=hand.classification[0].label
                handsType.append(handType)
            for handLandMarks in results.multi_hand_landmarks:
                myHand=[]
                for landMark in handLandMarks.landmark:
                    myHand.append((int(landMark.x*width),int(landMark.y*height)))
                myHands.append(myHand)
        return myHands,handsType

width=1280
height=720
cam=cv2.VideoCapture(3,cv2.CAP_DSHOW)
cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)
cam.set(cv2.CAP_PROP_FPS, 30)
cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(2)
findFace=mpFace()
findPose=mpPose()
findMesh=mpFaceMesh(drawMesh=True)

font=cv2.FONT_HERSHEY_SIMPLEX
fontColor=(0,0,255)
fontSize=.1
fontThick=1

cv2.namedWindow('Trackbars')
cv2.moveWindow('Trackbars',width+50,0)
cv2.resizeWindow('Trackbars',400,150)


while True:
    ignore,  frame = cam.read()
    frame=cv2.resize(frame,(width,height))
    handsLM,handsType=findHands.Marks(frame)
    faceLoc=findFace.Marks(frame)
    poseLM=findPose.Marks(frame)
    facesMeshLM=findMesh.Marks(frame)
    if poseLM != []:
        for ind in [13,14,15,16]:
            cv2.circle(frame,poseLM[ind],20,(0,255,0),-1)

    for face in faceLoc:
        cv2.rectangle(frame,face[0],face[1],(255,0,0),3)
    for hand,handType in zip(handsLM,handsType):
        if handType=='Right':
            lbl='Right'
        if handType=='Left':
            lbl='Left'
        cv2.putText(frame,lbl,hand[8],font,2,fontColor,2)
    for faceMeshLM in facesMeshLM:
        cnt=0
        for lm in faceMeshLM:
            cv2.putText(frame,str(cnt),lm,font,fontSize,fontColor,fontThick)
            cnt=cnt+1

    cv2.imshow('my WEBcam', frame)
    cv2.moveWindow('my WEBcam',0,0)
    if cv2.waitKey(1) & 0xff ==ord('q'):
        break
cam.release()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

import cv2

print(cv2.__version__)

class mpFaceMesh:

import mediapipe as mp

def __init__(self,still=False,numFaces=3,tol1=.5,tol2=.5,drawMesh=True):

self.myFaceMesh=self.mp.solutions.face_mesh.FaceMesh(still,numFaces,tol1,tol2)

self.myDraw=self.mp.solutions.drawing_utils

self.draw=drawMesh

def Marks(self,frame):

global width

global height

drawSpecCircle=self.myDraw.DrawingSpec(thickness=0,circle_radius=0,color=(0,0,255))

drawSpecLine=self.myDraw.DrawingSpec(thickness=3,circle_radius=2,color=(255,0,0))

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.myFaceMesh.process(frameRGB)

facesMeshLandmarks=[]

if results.multi_face_landmarks !=None:

for faceMesh in results.multi_face_landmarks:

faceMeshLandmarks=[]

for lm in faceMesh.landmark:

loc=(int(lm.x*width),int(lm.y*height))

faceMeshLandmarks.append(loc)

facesMeshLandmarks.append(faceMeshLandmarks)

if self.draw==True:

self.myDraw.draw_landmarks(frame,faceMesh,self.mp.solutions.face_mesh.FACE_CONNECTIONS,drawSpecCircle,drawSpecLine)

return facesMeshLandmarks

class mpFace:

import mediapipe as mp

def __init__(self):

self.myFace=self.mp.solutions.face_detection.FaceDetection()

def Marks(self,frame):

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.myFace.process(frameRGB)

faceBoundBoxs=[]

if results.detections != None:

for face in results.detections:

bBox=face.location_data.relative_bounding_box

topLeft=(int(bBox.xmin*width),int(bBox.ymin*height))

bottomRight=(int((bBox.xmin+bBox.width)*width),int((bBox.ymin+bBox.height)*height))

faceBoundBoxs.append((topLeft,bottomRight))

return faceBoundBoxs

class mpPose:

import mediapipe as mp

def __init__(self,still=False,upperBody=False, smoothData=True, tol1=.5, tol2=.5):

self.myPose=self.mp.solutions.pose.Pose(still,upperBody,smoothData,tol1,tol2)

def Marks(self,frame):

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.myPose.process(frameRGB)

poseLandmarks=[]

if results.pose_landmarks:

for lm in results.pose_landmarks.landmark:

poseLandmarks.append((int(lm.x*width),int(lm.y*height)))

return poseLandmarks

class mpHands:

import mediapipe as mp

def __init__(self,maxHands=2,tol1=.5,tol2=.5):

self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)

def Marks(self,frame):

myHands=[]

handsType=[]

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.hands.process(frameRGB)

if results.multi_hand_landmarks != None:

#print(results.multi_handedness)

for hand in results.multi_handedness:

#print(hand)

#print(hand.classification)

#print(hand.classification[0])

handType=hand.classification[0].label

handsType.append(handType)

for handLandMarks in results.multi_hand_landmarks:

myHand=[]

for landMark in handLandMarks.landmark:

myHand.append((int(landMark.x*width),int(landMark.y*height)))

myHands.append(myHand)

return myHands,handsType

width=1280

height=720

cam=cv2.VideoCapture(3,cv2.CAP_DSHOW)

cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)

cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)

cam.set(cv2.CAP_PROP_FPS, 30)

cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(2)

findFace=mpFace()

findPose=mpPose()

findMesh=mpFaceMesh(drawMesh=True)

font=cv2.FONT_HERSHEY_SIMPLEX

fontColor=(0,0,255)

fontSize=.1

fontThick=1

cv2.namedWindow('Trackbars')

cv2.moveWindow('Trackbars',width+50,0)

cv2.resizeWindow('Trackbars',400,150)

while True:

ignore, frame = cam.read()

frame=cv2.resize(frame,(width,height))

handsLM,handsType=findHands.Marks(frame)

faceLoc=findFace.Marks(frame)

poseLM=findPose.Marks(frame)

facesMeshLM=findMesh.Marks(frame)

if poseLM != []:

for ind in [13,14,15,16]:

cv2.circle(frame,poseLM[ind],20,(0,255,0),-1)

for face in faceLoc:

cv2.rectangle(frame,face[0],face[1],(255,0,0),3)

for hand,handType in zip(handsLM,handsType):

if handType=='Right':

lbl='Right'

if handType=='Left':

lbl='Left'

cv2.putText(frame,lbl,hand[8],font,2,fontColor,2)

for faceMeshLM in facesMeshLM:

cnt=0

for lm in faceMeshLM:

cv2.putText(frame,str(cnt),lm,font,fontSize,fontColor,fontThick)

cnt=cnt+1

cv2.imshow('my WEBcam', frame)

cv2.moveWindow('my WEBcam',0,0)

if cv2.waitKey(1) & 0xff ==ord('q'):

break

cam.release()

Artificial Intelligence, OpenCV, Python

Parsing Mediapipe Data for Pose Landmarks, Hand Landmarks and Face Bounding Box

December 2, 2021 admin

In this lesson we show how to create python classes to parse the data coming from Mediapipe for hand Landmarks, Pose Landmarks and the bounding boxes for found faces. Creating these classes allows the difficult parsing to be done in the class, and then have a simple way to parse and use all the data.

import cv2
print(cv2.__version__)

class mpFace:
    import mediapipe as mp 
    def __init__(self):
        self.myFace=self.mp.solutions.face_detection.FaceDetection()
    def Marks(self,frame):
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.myFace.process(frameRGB)
        faceBoundBoxs=[]
        if results.detections != None:
            for face in results.detections:
                bBox=face.location_data.relative_bounding_box
                topLeft=(int(bBox.xmin*width),int(bBox.ymin*height))
                bottomRight=(int((bBox.xmin+bBox.width)*width),int((bBox.ymin+bBox.height)*height))
                faceBoundBoxs.append((topLeft,bottomRight))
        return faceBoundBoxs

class mpPose:
    import mediapipe as mp
    def __init__(self,still=False,upperBody=False, smoothData=True, tol1=.5, tol2=.5):
        self.myPose=self.mp.solutions.pose.Pose(still,upperBody,smoothData,tol1,tol2)
    def Marks(self,frame):
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.myPose.process(frameRGB)
        poseLandmarks=[]
        if results.pose_landmarks:
            for lm in results.pose_landmarks.landmark:            
                poseLandmarks.append((int(lm.x*width),int(lm.y*height)))
        return poseLandmarks

class mpHands:
    import mediapipe as mp
    def __init__(self,maxHands=2,tol1=.5,tol2=.5):
        self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)
    def Marks(self,frame):
        myHands=[]
        handsType=[]
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.hands.process(frameRGB)
        if results.multi_hand_landmarks != None:
            #print(results.multi_handedness)
            for hand in results.multi_handedness:
                #print(hand)
                #print(hand.classification)
                #print(hand.classification[0])
                handType=hand.classification[0].label
                handsType.append(handType)
            for handLandMarks in results.multi_hand_landmarks:
                myHand=[]
                for landMark in handLandMarks.landmark:
                    myHand.append((int(landMark.x*width),int(landMark.y*height)))
                myHands.append(myHand)
        return myHands,handsType

width=1280
height=720
cam=cv2.VideoCapture(3,cv2.CAP_DSHOW)
cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)
cam.set(cv2.CAP_PROP_FPS, 30)
cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(2)
findFace=mpFace()
findPose=mpPose()

font=cv2.FONT_HERSHEY_SIMPLEX
fontColor=(0,0,255)

while True:
    ignore,  frame = cam.read()
    frame=cv2.resize(frame,(width,height))
    handsLM,handsType=findHands.Marks(frame)
    faceLoc=findFace.Marks(frame)
    poseLM=findPose.Marks(frame)
    if poseLM != []:
        for ind in [13,14,15,16]:
            cv2.circle(frame,poseLM[ind],20,(0,255,0),-1)

    for face in faceLoc:
        cv2.rectangle(frame,face[0],face[1],(255,0,0),3)
    for hand,handType in zip(handsLM,handsType):
        if handType=='Right':
            lbl='Right'
        if handType=='Left':
            lbl='Left'
        cv2.putText(frame,lbl,hand[8],font,2,fontColor,2)

    cv2.imshow('my WEBcam', frame)
    cv2.moveWindow('my WEBcam',0,0)
    if cv2.waitKey(1) & 0xff ==ord('q'):
        break
cam.release()

import cv2

print(cv2.__version__)

class mpFace:

import mediapipe as mp

def __init__(self):

self.myFace=self.mp.solutions.face_detection.FaceDetection()

def Marks(self,frame):

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.myFace.process(frameRGB)

faceBoundBoxs=[]

if results.detections != None:

for face in results.detections:

bBox=face.location_data.relative_bounding_box

topLeft=(int(bBox.xmin*width),int(bBox.ymin*height))

bottomRight=(int((bBox.xmin+bBox.width)*width),int((bBox.ymin+bBox.height)*height))

faceBoundBoxs.append((topLeft,bottomRight))

return faceBoundBoxs

class mpPose:

import mediapipe as mp

def __init__(self,still=False,upperBody=False, smoothData=True, tol1=.5, tol2=.5):

self.myPose=self.mp.solutions.pose.Pose(still,upperBody,smoothData,tol1,tol2)

def Marks(self,frame):

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.myPose.process(frameRGB)

poseLandmarks=[]

if results.pose_landmarks:

for lm in results.pose_landmarks.landmark:

poseLandmarks.append((int(lm.x*width),int(lm.y*height)))

return poseLandmarks

class mpHands:

import mediapipe as mp

def __init__(self,maxHands=2,tol1=.5,tol2=.5):

self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)

def Marks(self,frame):

myHands=[]

handsType=[]

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.hands.process(frameRGB)

if results.multi_hand_landmarks != None:

#print(results.multi_handedness)

for hand in results.multi_handedness:

#print(hand)

#print(hand.classification)

#print(hand.classification[0])

handType=hand.classification[0].label

handsType.append(handType)

for handLandMarks in results.multi_hand_landmarks:

myHand=[]

for landMark in handLandMarks.landmark:

myHand.append((int(landMark.x*width),int(landMark.y*height)))

myHands.append(myHand)

return myHands,handsType

width=1280

height=720

cam=cv2.VideoCapture(3,cv2.CAP_DSHOW)

cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)

cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)

cam.set(cv2.CAP_PROP_FPS, 30)

cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(2)

findFace=mpFace()

findPose=mpPose()

font=cv2.FONT_HERSHEY_SIMPLEX

fontColor=(0,0,255)

while True:

ignore, frame = cam.read()

frame=cv2.resize(frame,(width,height))

handsLM,handsType=findHands.Marks(frame)

faceLoc=findFace.Marks(frame)

poseLM=findPose.Marks(frame)

if poseLM != []:

for ind in [13,14,15,16]:

cv2.circle(frame,poseLM[ind],20,(0,255,0),-1)

for face in faceLoc:

cv2.rectangle(frame,face[0],face[1],(255,0,0),3)

for hand,handType in zip(handsLM,handsType):

if handType=='Right':

lbl='Right'

if handType=='Left':

lbl='Left'

cv2.putText(frame,lbl,hand[8],font,2,fontColor,2)

cv2.imshow('my WEBcam', frame)

cv2.moveWindow('my WEBcam',0,0)

if cv2.waitKey(1) & 0xff ==ord('q'):

break

cam.release()

3D Graphics with Visual Python, Python

Python 3D Graphics Tutorial 21: Understanding and Using Widgets in Vpython

December 1, 2021 admin

In this video Lesson, we show how to interact with a Python model using widgets. We show how Vpython allows use of dropdown menus, radio buttons, check boxes and slider bars. These combine to give the user great control over an animated model. In past lessons we changed parameters by editing the program. By incorporating widgets we can allow non-programmers to effectively interact with a model.

from vpython import *
from time import *
mRadius=.5
wallThickness=.1
roomWidth=12
roomDepth=20
roomHeight=15
floor=box(pos=vector(0,-roomHeight/2,0),size=vector(roomWidth,wallThickness,roomDepth), color=color.white)
ceiling=box(pos=vector(0,roomHeight/2,0),size=vector(roomWidth,wallThickness,roomDepth), color=color.white)
backWall=box(pos=vector(0,0,-roomDepth/2),size=vector(roomWidth,roomHeight,wallThickness), color=color.white)
leftWall=box(pos=vector(-roomWidth/2,0,0),size=vector(wallThickness,roomHeight,roomDepth), color=color.white)
rightWall=box(pos=vector(roomWidth/2,0,0),size=vector(wallThickness,roomHeight,roomDepth), color=color.white)
marble=sphere(radius=mRadius,color=color.blue)
deltaX=.1
deltaY=.1
deltaZ=.1

run=0
mySpeed=1

xPos=0
yPos=0
zPos=0

def ballColorRed(x):
    marble.color=color.red 
button(bind=ballColorRed,text="Red",color=color.black,background=color.red)
scene.append_to_caption(' ')

def ballColorGreen(x):
    marble.color=color.green 
button(bind=ballColorGreen,text="Green",color=color.black,background=color.green)
scene.append_to_caption(' ')

def ballColorBlue(x):
    marble.color=color.blue 
button(bind=ballColorBlue,text="Blue",color=color.black,background=color.blue)
scene.append_to_caption(' ')

def runRadio(x):
    global run
    print(x.checked)
    if x.checked==True:
        run=1
    if x.checked==False:
        run=0
radio(bind=runRadio,text='Run')

def bigBall(x):
    global mRadius
    if x.checked==True:
        mRadius=mRadius*2
        marble.radius=mRadius
    if x.checked==False:
        mRadius=mRadius/2
        marble.radius=mRadius
checkbox(bind=bigBall, text='Big Ball')
scene.append_to_caption('\n\n')

wtext(text='Ball Opacity')
scene.append_to_caption('\n\n')

def ballOpacity(x):
    op=x.value
    marble.opacity=op
slider(bind=ballOpacity,vertical=False,min=0,max=1,step=.05,value=1,text='Opacity')

scene.append_to_caption('\n\n')
wtext(text='Choose Your Speed')
scene.append_to_caption('\n\n')

def speed(x):
    global mySpeed
    if x.selected=='1':
        mySpeed=1
    if x.selected=='2':
        mySpeed=2
    if x.selected=='3':
        mySpeed=3
    if x.selected=='4':
        mySpeed=4
    if x.selected=='5':
        mySpeed=5
menu(bind=speed, choices=['1','2','3','4','5'])


while True:
    rate(25*mySpeed)

    xPos=xPos+deltaX*run
    yPos=yPos+deltaY*run
    zPos=zPos+deltaZ*run

    Xrme=xPos+mRadius
    Xlme=xPos-mRadius
    Ytme=yPos+mRadius
    Ybme=yPos-mRadius
    Zbme=zPos-mRadius
    Zfme=zPos+mRadius

    Rwe=roomWidth/2-wallThickness/2
    Lwe=-roomWidth/2+wallThickness/2
    Cwe=roomHeight/2-wallThickness/2
    FLOORwe=-roomHeight/2+wallThickness/2
    Bwe=-roomDepth/2+wallThickness/2
    Fwe=roomDepth/2-wallThickness/2

    if (Xrme>=Rwe or Xlme<=Lwe):
        deltaX=deltaX*(-1)

    if (Ytme>=Cwe or Ybme<=FLOORwe):
        deltaY=deltaY*(-1)

    if (Zfme>=Fwe or Zbme<=Bwe):
        deltaZ=deltaZ*(-1)

    marble.pos=vector(xPos,yPos,zPos)

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

from vpython import *

from time import *

mRadius=.5

wallThickness=.1

roomWidth=12

roomDepth=20

roomHeight=15

floor=box(pos=vector(0,-roomHeight/2,0),size=vector(roomWidth,wallThickness,roomDepth), color=color.white)

ceiling=box(pos=vector(0,roomHeight/2,0),size=vector(roomWidth,wallThickness,roomDepth), color=color.white)

backWall=box(pos=vector(0,0,-roomDepth/2),size=vector(roomWidth,roomHeight,wallThickness), color=color.white)

leftWall=box(pos=vector(-roomWidth/2,0,0),size=vector(wallThickness,roomHeight,roomDepth), color=color.white)

rightWall=box(pos=vector(roomWidth/2,0,0),size=vector(wallThickness,roomHeight,roomDepth), color=color.white)

marble=sphere(radius=mRadius,color=color.blue)

deltaX=.1

deltaY=.1

deltaZ=.1

run=0

mySpeed=1

xPos=0

yPos=0

zPos=0

def ballColorRed(x):

marble.color=color.red

button(bind=ballColorRed,text="Red",color=color.black,background=color.red)

scene.append_to_caption(' ')

def ballColorGreen(x):

marble.color=color.green

button(bind=ballColorGreen,text="Green",color=color.black,background=color.green)

scene.append_to_caption(' ')

def ballColorBlue(x):

marble.color=color.blue

button(bind=ballColorBlue,text="Blue",color=color.black,background=color.blue)

scene.append_to_caption(' ')

def runRadio(x):

global run

print(x.checked)

if x.checked==True:

run=1

if x.checked==False:

run=0

radio(bind=runRadio,text='Run')

def bigBall(x):

global mRadius

if x.checked==True:

mRadius=mRadius*2

marble.radius=mRadius

if x.checked==False:

mRadius=mRadius/2

marble.radius=mRadius

checkbox(bind=bigBall, text='Big Ball')

scene.append_to_caption('\n\n')

wtext(text='Ball Opacity')

scene.append_to_caption('\n\n')

def ballOpacity(x):

op=x.value

marble.opacity=op

slider(bind=ballOpacity,vertical=False,min=0,max=1,step=.05,value=1,text='Opacity')

scene.append_to_caption('\n\n')

wtext(text='Choose Your Speed')

scene.append_to_caption('\n\n')

def speed(x):

global mySpeed

if x.selected=='1':

mySpeed=1

if x.selected=='2':

mySpeed=2

if x.selected=='3':

mySpeed=3

if x.selected=='4':

mySpeed=4

if x.selected=='5':

mySpeed=5

menu(bind=speed, choices=['1','2','3','4','5'])

while True:

rate(25*mySpeed)

xPos=xPos+deltaX*run

yPos=yPos+deltaY*run

zPos=zPos+deltaZ*run

Xrme=xPos+mRadius

Xlme=xPos-mRadius

Ytme=yPos+mRadius

Ybme=yPos-mRadius

Zbme=zPos-mRadius

Zfme=zPos+mRadius

Rwe=roomWidth/2-wallThickness/2

Lwe=-roomWidth/2+wallThickness/2

Cwe=roomHeight/2-wallThickness/2

FLOORwe=-roomHeight/2+wallThickness/2

Bwe=-roomDepth/2+wallThickness/2

Fwe=roomDepth/2-wallThickness/2

if (Xrme>=Rwe or Xlme<=Lwe):

deltaX=deltaX*(-1)

if (Ytme>=Cwe or Ybme<=FLOORwe):

deltaY=deltaY*(-1)

if (Zfme>=Fwe or Zbme<=Bwe):

deltaZ=deltaZ*(-1)

marble.pos=vector(xPos,yPos,zPos)

Artificial Intelligence, OpenCV, Python

AI For Everyone: Parsing Pose and Hand Data Landmarks from Mediapipe

November 25, 2021 admin

In this video lesson we show you how to create simple Python classes to parse the data from mediapipe hand Landmarks and pose estimation. With these two classes, it is very easy to parse the most important data generated by mediapipe.

import cv2
print(cv2.__version__)

class mpPose:
    import mediapipe as mp
    def __init__(self,still=False,upperBody=False, smoothData=True, tol1=.5, tol2=.5):
        self.myPose=self.mp.solutions.pose.Pose(still,upperBody,smoothData,tol1,tol2)
    def Marks(self,frame):
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.myPose.process(frameRGB)
        poseLandmarks=[]
        if results.pose_landmarks:
            for lm in results.pose_landmarks.landmark:            poseLandmarks.append((int(lm.x*width),int(lm.y*height)))
        return poseLandmarks

class mpHands:
    import mediapipe as mp
    def __init__(self,maxHands=2,tol1=.5,tol2=.5):
        self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)
    def Marks(self,frame):
        myHands=[]
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.hands.process(frameRGB)
        if results.multi_hand_landmarks != None:
            for handLandMarks in results.multi_hand_landmarks:
                myHand=[]
                for landMark in handLandMarks.landmark:
                    myHand.append((int(landMark.x*width),int(landMark.y*height)))
                myHands.append(myHand)
        return myHands

width=1280
height=720
cam=cv2.VideoCapture(4,cv2.CAP_DSHOW)
cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)
cam.set(cv2.CAP_PROP_FPS, 30)
cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(2)
findPose=mpPose()
while True:
    ignore,  frame = cam.read()
    frame=cv2.resize(frame,(width,height))
    handData=findHands.Marks(frame)
    for hand in handData:
        for ind in [0,5,6,7,8]:
            cv2.circle(frame,hand[ind],25,(255,0,255),3)
    poseData=findPose.Marks(frame)
    if len(poseData)!=0:
        cv2.circle(frame,poseData[0],5,(0,255,0),3)

    cv2.imshow('my WEBcam', frame)
    cv2.moveWindow('my WEBcam',0,0)
    if cv2.waitKey(1) & 0xff ==ord('q'):
        break
cam.release()

import cv2

print(cv2.__version__)

class mpPose:

import mediapipe as mp

def __init__(self,still=False,upperBody=False, smoothData=True, tol1=.5, tol2=.5):

self.myPose=self.mp.solutions.pose.Pose(still,upperBody,smoothData,tol1,tol2)

def Marks(self,frame):

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.myPose.process(frameRGB)

poseLandmarks=[]

if results.pose_landmarks:

for lm in results.pose_landmarks.landmark: poseLandmarks.append((int(lm.x*width),int(lm.y*height)))

return poseLandmarks

class mpHands:

import mediapipe as mp

def __init__(self,maxHands=2,tol1=.5,tol2=.5):

self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)

def Marks(self,frame):

myHands=[]

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.hands.process(frameRGB)

if results.multi_hand_landmarks != None:

for handLandMarks in results.multi_hand_landmarks:

myHand=[]

for landMark in handLandMarks.landmark:

myHand.append((int(landMark.x*width),int(landMark.y*height)))

myHands.append(myHand)

return myHands

width=1280

height=720

cam=cv2.VideoCapture(4,cv2.CAP_DSHOW)

cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)

cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)

cam.set(cv2.CAP_PROP_FPS, 30)

cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(2)

findPose=mpPose()

while True:

ignore, frame = cam.read()

frame=cv2.resize(frame,(width,height))

handData=findHands.Marks(frame)

for hand in handData:

for ind in [0,5,6,7,8]:

cv2.circle(frame,hand[ind],25,(255,0,255),3)

poseData=findPose.Marks(frame)

if len(poseData)!=0:

cv2.circle(frame,poseData[0],5,(0,255,0),3)

cv2.imshow('my WEBcam', frame)

cv2.moveWindow('my WEBcam',0,0)

if cv2.waitKey(1) & 0xff ==ord('q'):

break

cam.release()

Technology Tutorials

AI for Everyone LESSON 25: First Attempt at Using MediaPipe for Gesture Detection

Parsing Mediapipe Data for Face Mesh, Pose, and Hand Landmarks

Parsing Mediapipe Data for Pose Landmarks, Hand Landmarks and Face Bounding Box

AI For Everyone: Parsing Pose and Hand Data Landmarks from Mediapipe

Making The World a Better Place One High Tech Project at a Time. Enjoy!