In this video lesson we connect the Arduino to a DHT11 temperature and humidity sensor. We show how to wire the device up, and then how to code the Arduino. The data is passed from the Arduino to python. We then create a live 3D thermometer model that updates as the temperature changes.

On the arduino side, this is the code which we use:

#include "DHT.h"
#define DHTPIN 2
#define DHTTYPE DHT11

float tempF;
float tempC;
float humidity;

int setTime=500;
int dt=1000;

DHT TH(DHTPIN,DHTTYPE);

void setup() {
  // put your setup code here, to run once:
Serial.begin(115200);
TH.begin();
delay(setTime);
}
void loop() {
  // put your main code here, to run repeatedly:
tempC=TH.readTemperature();
tempF=TH.readTemperature(true);
humidity=TH.readHumidity();


Serial.print(tempF);
Serial.print(",");
Serial.println(humidity);

delay(dt);
}

#include "DHT.h"

#define DHTPIN 2

#define DHTTYPE DHT11

float tempF;

float tempC;

float humidity;

int setTime=500;

int dt=1000;

DHT TH(DHTPIN,DHTTYPE);

void setup() {

// put your setup code here, to run once:

Serial.begin(115200);

TH.begin();

delay(setTime);

}

void loop() {

// put your main code here, to run repeatedly:

tempC=TH.readTemperature();

tempF=TH.readTemperature(true);

humidity=TH.readHumidity();

Serial.print(tempF);

Serial.print(",");

Serial.println(humidity);

delay(dt);

}

Then on the python side, we use the following code:

import time
import serial
from vpython import *

arduinoData=serial.Serial('com3',115200)
time.sleep(.5)
digValue=label(text='50',height=20,box=False,pos=vector(0,-2.5,2))
bulb=sphere(radius=1,color=color.red,pos=vector(0,-3,0))
cyl=cylinder(radius=.6,color=color.red,axis=vector(0,1,0),length=6,pos=vector(0,-3,0))
bulbGlass=sphere(radius=1.2,color=color.white,opacity=.25,pos=vector(0,-3,0))
cylGlass=cylinder(radius=.8,color=color.white,axis=vector(0,1,0),opacity=.25,length=6,pos=vector(0,-3,0))
for temp in range(0,115,10):
    tickPos=4.5/115*temp+1.5
    tick=cylinder(radius=.7,color=color.black,length=.1,axis=vector(0,1,0),pos=vector(0,tickPos-3,0))
    label=text(text=str(temp),color=color.white,pos=vector(-2,tickPos-3,0),height=.3)
while True:
    while arduinoData.in_waiting==0:
        pass
    dataPacket=arduinoData.readline()
    dataPacket=str(dataPacket,'utf-8')
    dataPacket=dataPacket.strip('\r\n')
    dataPacket=dataPacket.split(',')
    temp=float(dataPacket[0])
    hum=float(dataPacket[1])
    len=(4.5/115)*temp+1.5
    cyl.length=len
    digValue.text=str(temp)

import time

import serial

from vpython import *

arduinoData=serial.Serial('com3',115200)

time.sleep(.5)

digValue=label(text='50',height=20,box=False,pos=vector(0,-2.5,2))

bulb=sphere(radius=1,color=color.red,pos=vector(0,-3,0))

cyl=cylinder(radius=.6,color=color.red,axis=vector(0,1,0),length=6,pos=vector(0,-3,0))

bulbGlass=sphere(radius=1.2,color=color.white,opacity=.25,pos=vector(0,-3,0))

cylGlass=cylinder(radius=.8,color=color.white,axis=vector(0,1,0),opacity=.25,length=6,pos=vector(0,-3,0))

for temp in range(0,115,10):

tickPos=4.5/115*temp+1.5

tick=cylinder(radius=.7,color=color.black,length=.1,axis=vector(0,1,0),pos=vector(0,tickPos-3,0))

label=text(text=str(temp),color=color.white,pos=vector(-2,tickPos-3,0),height=.3)

while True:

while arduinoData.in_waiting==0:

pass

dataPacket=arduinoData.readline()

dataPacket=str(dataPacket,'utf-8')

dataPacket=dataPacket.strip('\r\n')

dataPacket=dataPacket.split(',')

temp=float(dataPacket[0])

hum=float(dataPacket[1])

len=(4.5/115)*temp+1.5

cyl.length=len

digValue.text=str(temp)

Artificial Intelligence, OpenCV

AI for Everyone LESSON 29: Control of Real World Objects with Gesture Recognition in Mediapipe

February 24, 2022 admin

In this video lesson we show you how you can control objects in the real world using OpenCV, Python, Mediapipa and our old friend, the Arduino. On the Python side, we recognize hand gestures, and then we pass the recognized gesture to Arduino and Arduino lights LED in response to what hand signal is seen. This is a simple example, but a very powerful method. Instead of LED, you could operate servos, stepper motors or relays to control any manner of different devices. For your convenience, this is the code we used on the Arduino side:

String cmd;
int LED1=9;
int LED2=6;
int LED3=5;
int LED4=3;
int LED5=10;
void setup() {
  pinMode(LED1,OUTPUT);
  pinMode(LED2,OUTPUT);
  pinMode(LED3,OUTPUT);
   pinMode(LED4,OUTPUT);
   pinMode(LED5,OUTPUT);
 Serial.begin(115200);

}
void loop() {
 while (Serial.available()==0){
 
 }
 cmd = Serial.readStringUntil('\r');
if (cmd=="One"){
  digitalWrite(LED1,HIGH);
  digitalWrite(LED2,LOW);
  digitalWrite(LED3,LOW);
  digitalWrite(LED4,LOW);
  digitalWrite(LED5,LOW);
}
if (cmd=="Two"){
  digitalWrite(LED1,HIGH);
  digitalWrite(LED2,HIGH);
  digitalWrite(LED3,LOW);
  digitalWrite(LED4,LOW);
  digitalWrite(LED5,LOW);
}
 if (cmd=="Three"){
  digitalWrite(LED1,HIGH);
  digitalWrite(LED2,HIGH);
  digitalWrite(LED3,HIGH);
  digitalWrite(LED4,LOW);
  digitalWrite(LED5,LOW);
 }
 if (cmd=="Four"){
  digitalWrite(LED1,HIGH);
  digitalWrite(LED2,HIGH);
  digitalWrite(LED3,HIGH);
  digitalWrite(LED4,HIGH);
  digitalWrite(LED5,LOW);
 }
 if (cmd=="Five"){
  digitalWrite(LED1,HIGH);
  digitalWrite(LED2,HIGH);
  digitalWrite(LED3,HIGH);
  digitalWrite(LED4,HIGH);
  digitalWrite(LED5,HIGH);

 }

 if (cmd=="Pinky"){
  digitalWrite(LED1,LOW);
  digitalWrite(LED2,LOW);
  digitalWrite(LED3,LOW);
  digitalWrite(LED4,HIGH);
  digitalWrite(LED5,LOW);

 }
  if (cmd=="Thumb"){
  digitalWrite(LED1,LOW);
  digitalWrite(LED2,LOW);
  digitalWrite(LED3,LOW);
  digitalWrite(LED4,LOW);
  digitalWrite(LED5,HIGH);

 }
  if (cmd=="Inside"){
  digitalWrite(LED1,LOW);
  digitalWrite(LED2,HIGH);
  digitalWrite(LED3,HIGH);
  digitalWrite(LED4,LOW);
  digitalWrite(LED5,LOW);

 }
   if (cmd=="Outside"){
  digitalWrite(LED1,HIGH);
  digitalWrite(LED2,LOW);
  digitalWrite(LED3,LOW);
  digitalWrite(LED4,HIGH);
  digitalWrite(LED5,LOW);

 }
    if (cmd=="Unknown"){
  digitalWrite(LED1,LOW);
  digitalWrite(LED2,LOW);
  digitalWrite(LED3,LOW);
  digitalWrite(LED4,LOW);
  digitalWrite(LED5,LOW);

 }
}

String cmd;

int LED1=9;

int LED2=6;

int LED3=5;

int LED4=3;

int LED5=10;

void setup() {

pinMode(LED1,OUTPUT);

pinMode(LED2,OUTPUT);

pinMode(LED3,OUTPUT);

pinMode(LED4,OUTPUT);

pinMode(LED5,OUTPUT);

Serial.begin(115200);

}

void loop() {

while (Serial.available()==0){

}

cmd = Serial.readStringUntil('\r');

if (cmd=="One"){

digitalWrite(LED1,HIGH);

digitalWrite(LED2,LOW);

digitalWrite(LED3,LOW);

digitalWrite(LED4,LOW);

digitalWrite(LED5,LOW);

}

if (cmd=="Two"){

digitalWrite(LED1,HIGH);

digitalWrite(LED2,HIGH);

digitalWrite(LED3,LOW);

digitalWrite(LED4,LOW);

digitalWrite(LED5,LOW);

}

if (cmd=="Three"){

digitalWrite(LED1,HIGH);

digitalWrite(LED2,HIGH);

digitalWrite(LED3,HIGH);

digitalWrite(LED4,LOW);

digitalWrite(LED5,LOW);

}

if (cmd=="Four"){

digitalWrite(LED1,HIGH);

digitalWrite(LED2,HIGH);

digitalWrite(LED3,HIGH);

digitalWrite(LED4,HIGH);

digitalWrite(LED5,LOW);

}

if (cmd=="Five"){

digitalWrite(LED1,HIGH);

digitalWrite(LED2,HIGH);

digitalWrite(LED3,HIGH);

digitalWrite(LED4,HIGH);

digitalWrite(LED5,HIGH);

}

if (cmd=="Pinky"){

digitalWrite(LED1,LOW);

digitalWrite(LED2,LOW);

digitalWrite(LED3,LOW);

digitalWrite(LED4,HIGH);

digitalWrite(LED5,LOW);

}

if (cmd=="Thumb"){

digitalWrite(LED1,LOW);

digitalWrite(LED2,LOW);

digitalWrite(LED3,LOW);

digitalWrite(LED4,LOW);

digitalWrite(LED5,HIGH);

}

if (cmd=="Inside"){

digitalWrite(LED1,LOW);

digitalWrite(LED2,HIGH);

digitalWrite(LED3,HIGH);

digitalWrite(LED4,LOW);

digitalWrite(LED5,LOW);

}

if (cmd=="Outside"){

digitalWrite(LED1,HIGH);

digitalWrite(LED2,LOW);

digitalWrite(LED3,LOW);

digitalWrite(LED4,HIGH);

digitalWrite(LED5,LOW);

}

if (cmd=="Unknown"){

digitalWrite(LED1,LOW);

digitalWrite(LED2,LOW);

digitalWrite(LED3,LOW);

digitalWrite(LED4,LOW);

digitalWrite(LED5,LOW);

}

And on the python side, we used the following code.

import time
import cv2
print(cv2.__version__)
import numpy as np
import pickle
import serial

arduinoData = serial.Serial('COM3',115200)

class mpHands:
    import mediapipe as mp
    def __init__(self,maxHands=2,tol1=.5,tol2=.5):
        self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)
    def Marks(self,frame):
        myHands=[]
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.hands.process(frameRGB)
        if results.multi_hand_landmarks != None:
            for handLandMarks in results.multi_hand_landmarks:
                myHand=[]
                for landMark in handLandMarks.landmark:
                    myHand.append((int(landMark.x*width),int(landMark.y*height)))
                myHands.append(myHand)
        return myHands
def findDistances(handData):
    distMatrix=np.zeros([len(handData),len(handData)],dtype='float')
    palmSize=((handData[0][0]-handData[9][0])**2+(handData[0][1]-handData[9][1])**2)**(1./2.)
    for row in range(0,len(handData)):
        for column in range(0,len(handData)):
            distMatrix[row][column]=(((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.))/palmSize
    return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):
    error=0
    for row in keyPoints:
        for column in keyPoints:
            error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])
    print(error)
    return error
def findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol):
    errorArray=[]
    for i in range(0,len(gestNames),1):
        error=findError(knownGestures[i],unknownGesture,keyPoints)
        errorArray.append(error)
    errorMin=errorArray[0]
    minIndex=0
    for i in range(0,len(errorArray),1):
        if errorArray[i]<errorMin:
            errorMin=errorArray[i]
            minIndex=i
    if errorMin<tol:
        gesture=gestNames[minIndex]
    if errorMin>=tol:
        gesture='Unknown'
    return gesture

width=1280
height=720
cam=cv2.VideoCapture(3,cv2.CAP_DSHOW)
cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)
cam.set(cv2.CAP_PROP_FPS, 30)
cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))
findHands=mpHands(1)
time.sleep(5)
keyPoints=[0,4,5,9,13,17,8,12,16,20]

train=int(input('Enter 1 to Train, Enter 0 to Recognize '))
if train==1:
    trainCnt=0
    knownGestures=[]
    numGest=int(input('How Many Gestures Do You Want? '))
    gestNames=[]
    for i in range(0,numGest,1):
        prompt='Name of Gesture #'+str(i+1)+' '
        name=input(prompt)
        gestNames.append(name)
    print(gestNames)
    trainName=input('Filename for training data? (Press Enter for Default) ')
    if trainName=='':
        trainName='default'
    trainName=trainName+'.pkl'
if train==0:
    trainName=input('What Training Data Do You Want to Use? (Press Enter for Default) ')
    if trainName=='':
        trainName='default'
    trainName=trainName+'.pkl'
    with open(trainName,'rb') as f:
        gestNames=pickle.load(f)
        knownGestures=pickle.load(f)

tol=10

while True:
    ignore,  frame = cam.read()
    frame=cv2.resize(frame,(width,height))
    handData=findHands.Marks(frame)
    if train==1:
        if handData!=[]:
            print('Please Show Gesture ',gestNames[trainCnt],': Press t when Ready')
            if cv2.waitKey(1) & 0xff==ord('t'):
                knownGesture=findDistances(handData[0])
                knownGestures.append(knownGesture)
                trainCnt=trainCnt+1
                if trainCnt==numGest:
                    train=0
                    with open(trainName,'wb') as f:
                        pickle.dump(gestNames,f)
                        pickle.dump(knownGestures,f)
    if train == 0:
        if handData!=[]:
            unknownGesture=findDistances(handData[0])
            myGesture=findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol)
            #error=findError(knownGesture,unknownGesture,keyPoints)
            cv2.putText(frame,myGesture,(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)
            myGesture=myGesture+'\r'
            arduinoData.write(myGesture.encode())
    for hand in handData:
        for ind in keyPoints:
            cv2.circle(frame,hand[ind],25,(255,0,255),3)
    cv2.imshow('my WEBcam', frame)
    cv2.moveWindow('my WEBcam',0,0)
    if cv2.waitKey(1) & 0xff ==ord('q'):
        break
cam.release()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

import time

import cv2

print(cv2.__version__)

import numpy as np

import pickle

import serial

arduinoData = serial.Serial('COM3',115200)

class mpHands:

import mediapipe as mp

def __init__(self,maxHands=2,tol1=.5,tol2=.5):

self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)

def Marks(self,frame):

myHands=[]

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.hands.process(frameRGB)

if results.multi_hand_landmarks != None:

for handLandMarks in results.multi_hand_landmarks:

myHand=[]

for landMark in handLandMarks.landmark:

myHand.append((int(landMark.x*width),int(landMark.y*height)))

myHands.append(myHand)

return myHands

def findDistances(handData):

distMatrix=np.zeros([len(handData),len(handData)],dtype='float')

palmSize=((handData[0][0]-handData[9][0])**2+(handData[0][1]-handData[9][1])**2)**(1./2.)

for row in range(0,len(handData)):

for column in range(0,len(handData)):

distMatrix[row][column]=(((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.))/palmSize

return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):

error=0

for row in keyPoints:

for column in keyPoints:

error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])

print(error)

return error

def findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol):

errorArray=[]

for i in range(0,len(gestNames),1):

error=findError(knownGestures[i],unknownGesture,keyPoints)

errorArray.append(error)

errorMin=errorArray[0]

minIndex=0

for i in range(0,len(errorArray),1):

if errorArray[i]<errorMin:

errorMin=errorArray[i]

minIndex=i

if errorMin<tol:

gesture=gestNames[minIndex]

if errorMin>=tol:

gesture='Unknown'

return gesture

width=1280

height=720

cam=cv2.VideoCapture(3,cv2.CAP_DSHOW)

cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)

cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)

cam.set(cv2.CAP_PROP_FPS, 30)

cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(1)

time.sleep(5)

keyPoints=[0,4,5,9,13,17,8,12,16,20]

train=int(input('Enter 1 to Train, Enter 0 to Recognize '))

if train==1:

trainCnt=0

knownGestures=[]

numGest=int(input('How Many Gestures Do You Want? '))

gestNames=[]

for i in range(0,numGest,1):

prompt='Name of Gesture #'+str(i+1)+' '

name=input(prompt)

gestNames.append(name)

print(gestNames)

trainName=input('Filename for training data? (Press Enter for Default) ')

if trainName=='':

trainName='default'

trainName=trainName+'.pkl'

if train==0:

trainName=input('What Training Data Do You Want to Use? (Press Enter for Default) ')

if trainName=='':

trainName='default'

trainName=trainName+'.pkl'

with open(trainName,'rb') as f:

gestNames=pickle.load(f)

knownGestures=pickle.load(f)

tol=10

while True:

ignore, frame = cam.read()

frame=cv2.resize(frame,(width,height))

handData=findHands.Marks(frame)

if train==1:

if handData!=[]:

print('Please Show Gesture ',gestNames[trainCnt],': Press t when Ready')

if cv2.waitKey(1) & 0xff==ord('t'):

knownGesture=findDistances(handData[0])

knownGestures.append(knownGesture)

trainCnt=trainCnt+1

if trainCnt==numGest:

train=0

with open(trainName,'wb') as f:

pickle.dump(gestNames,f)

pickle.dump(knownGestures,f)

if train == 0:

if handData!=[]:

unknownGesture=findDistances(handData[0])

myGesture=findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol)

#error=findError(knownGesture,unknownGesture,keyPoints)

cv2.putText(frame,myGesture,(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)

myGesture=myGesture+'\r'

arduinoData.write(myGesture.encode())

for hand in handData:

for ind in keyPoints:

cv2.circle(frame,hand[ind],25,(255,0,255),3)

cv2.imshow('my WEBcam', frame)

cv2.moveWindow('my WEBcam',0,0)

if cv2.waitKey(1) & 0xff ==ord('q'):

break

cam.release()

Uncategorized

Using an Arduino with Python LESSON 7: Measuring Temperature and Humidity With the DHT11

February 22, 2022 admin

In this video lesson we show you how to wire up the most excellent DHT11 Temperature and Humidity Sensor. The video above takes you through the build and coding. Below, for your convenience is the code developed in the video

#include "DHT.h"
#define DHTPIN 2
#define DHTTYPE DHT11

float tempF;
float tempC;
float humidity;

int setTime=500;
int dt=1000;

DHT TH(DHTPIN,DHTTYPE);

void setup() {
  // put your setup code here, to run once:
Serial.begin(115200);
TH.begin();
delay(setTime);
}
void loop() {
  // put your main code here, to run repeatedly:
tempC=TH.readTemperature();
tempF=TH.readTemperature(true);
humidity=TH.readHumidity();


Serial.print(tempF);
Serial.print(" Degrees F, ");
Serial.print(tempC);
Serial.print(" Degrees C, ");
Serial.print(humidity);
Serial.println("% Humidity");
delay(dt);
}

#include "DHT.h"

#define DHTPIN 2

#define DHTTYPE DHT11

float tempF;

float tempC;

float humidity;

int setTime=500;

int dt=1000;

DHT TH(DHTPIN,DHTTYPE);

void setup() {

// put your setup code here, to run once:

Serial.begin(115200);

TH.begin();

delay(setTime);

}

void loop() {

// put your main code here, to run repeatedly:

tempC=TH.readTemperature();

tempF=TH.readTemperature(true);

humidity=TH.readHumidity();

Serial.print(tempF);

Serial.print(" Degrees F, ");

Serial.print(tempC);

Serial.print(" Degrees C, ");

Serial.print(humidity);

Serial.println("% Humidity");

delay(dt);

}

Uncategorized

AI for Everyone LESSON 28: Storing and Loading Gesture Training Data in Mediapipe

February 17, 2022 admin

In this video lesson we show how to train our model for different gestures, and how to store and load our training data. For your convenience, we include the code below we develop in this video.

import time
import cv2
print(cv2.__version__)
import numpy as np
import pickle

class mpHands:
    import mediapipe as mp
    def __init__(self,maxHands=2,tol1=.5,tol2=.5):
        self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)
    def Marks(self,frame):
        myHands=[]
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.hands.process(frameRGB)
        if results.multi_hand_landmarks != None:
            for handLandMarks in results.multi_hand_landmarks:
                myHand=[]
                for landMark in handLandMarks.landmark:
                    myHand.append((int(landMark.x*width),int(landMark.y*height)))
                myHands.append(myHand)
        return myHands
def findDistances(handData):
    distMatrix=np.zeros([len(handData),len(handData)],dtype='float')
    palmSize=((handData[0][0]-handData[9][0])**2+(handData[0][1]-handData[9][1])**2)**(1./2.)
    for row in range(0,len(handData)):
        for column in range(0,len(handData)):
            distMatrix[row][column]=(((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.))/palmSize
    return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):
    error=0
    for row in keyPoints:
        for column in keyPoints:
            error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])
    print(error)
    return error
def findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol):
    errorArray=[]
    for i in range(0,len(gestNames),1):
        error=findError(knownGestures[i],unknownGesture,keyPoints)
        errorArray.append(error)
    errorMin=errorArray[0]
    minIndex=0
    for i in range(0,len(errorArray),1):
        if errorArray[i]<errorMin:
            errorMin=errorArray[i]
            minIndex=i
    if errorMin<tol:
        gesture=gestNames[minIndex]
    if errorMin>=tol:
        gesture='Unknown'
    return gesture


width=1280
height=720
cam=cv2.VideoCapture(0,cv2.CAP_DSHOW)
cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)
cam.set(cv2.CAP_PROP_FPS, 30)
cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))
findHands=mpHands(1)
time.sleep(5)
keyPoints=[0,4,5,9,13,17,8,12,16,20]

train=int(input('Enter 1 to Train, Enter 0 to Recognize '))
if train==1:
    trainCnt=0
    knownGestures=[]
    numGest=int(input('How Many Gestures Do You Want? '))
    gestNames=[]
    for i in range(0,numGest,1):
        prompt='Name of Gesture #'+str(i+1)+' '
        name=input(prompt)
        gestNames.append(name)
    print(gestNames)
    trainName=input('Filename for training data? (Press Enter for Default) ')
    if trainName=='':
        trainName='default'
    trainName=trainName+'.pkl'
if train==0:
    trainName=input('What Training Data Do You Want to Use? (Press Enter for Default) ')
    if trainName=='':
        trainName='default'
    trainName=trainName+'.pkl'
    with open(trainName,'rb') as f:
        gestNames=pickle.load(f)
        knownGestures=pickle.load(f)

tol=10

while True:
    ignore,  frame = cam.read()
    frame=cv2.resize(frame,(width,height))
    handData=findHands.Marks(frame)
    if train==1:
        if handData!=[]:
            print('Please Show Gesture ',gestNames[trainCnt],': Press t when Ready')
            if cv2.waitKey(1) & 0xff==ord('t'):
                knownGesture=findDistances(handData[0])
                knownGestures.append(knownGesture)
                trainCnt=trainCnt+1
                if trainCnt==numGest:
                    train=0
                    with open(trainName,'wb') as f:
                        pickle.dump(gestNames,f)
                        pickle.dump(knownGestures,f)
    if train == 0:
        if handData!=[]:
            unknownGesture=findDistances(handData[0])
            myGesture=findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol)
            #error=findError(knownGesture,unknownGesture,keyPoints)
            cv2.putText(frame,myGesture,(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)
    for hand in handData:
        for ind in keyPoints:
            cv2.circle(frame,hand[ind],25,(255,0,255),3)
    cv2.imshow('my WEBcam', frame)
    cv2.moveWindow('my WEBcam',0,0)
    if cv2.waitKey(1) & 0xff ==ord('q'):
        break
cam.release()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

import time

import cv2

print(cv2.__version__)

import numpy as np

import pickle

class mpHands:

import mediapipe as mp

def __init__(self,maxHands=2,tol1=.5,tol2=.5):

self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)

def Marks(self,frame):

myHands=[]

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.hands.process(frameRGB)

if results.multi_hand_landmarks != None:

for handLandMarks in results.multi_hand_landmarks:

myHand=[]

for landMark in handLandMarks.landmark:

myHand.append((int(landMark.x*width),int(landMark.y*height)))

myHands.append(myHand)

return myHands

def findDistances(handData):

distMatrix=np.zeros([len(handData),len(handData)],dtype='float')

palmSize=((handData[0][0]-handData[9][0])**2+(handData[0][1]-handData[9][1])**2)**(1./2.)

for row in range(0,len(handData)):

for column in range(0,len(handData)):

distMatrix[row][column]=(((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.))/palmSize

return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):

error=0

for row in keyPoints:

for column in keyPoints:

error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])

print(error)

return error

def findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol):

errorArray=[]

for i in range(0,len(gestNames),1):

error=findError(knownGestures[i],unknownGesture,keyPoints)

errorArray.append(error)

errorMin=errorArray[0]

minIndex=0

for i in range(0,len(errorArray),1):

if errorArray[i]<errorMin:

errorMin=errorArray[i]

minIndex=i

if errorMin<tol:

gesture=gestNames[minIndex]

if errorMin>=tol:

gesture='Unknown'

return gesture

width=1280

height=720

cam=cv2.VideoCapture(0,cv2.CAP_DSHOW)

cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)

cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)

cam.set(cv2.CAP_PROP_FPS, 30)

cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(1)

time.sleep(5)

keyPoints=[0,4,5,9,13,17,8,12,16,20]

train=int(input('Enter 1 to Train, Enter 0 to Recognize '))

if train==1:

trainCnt=0

knownGestures=[]

numGest=int(input('How Many Gestures Do You Want? '))

gestNames=[]

for i in range(0,numGest,1):

prompt='Name of Gesture #'+str(i+1)+' '

name=input(prompt)

gestNames.append(name)

print(gestNames)

trainName=input('Filename for training data? (Press Enter for Default) ')

if trainName=='':

trainName='default'

trainName=trainName+'.pkl'

if train==0:

trainName=input('What Training Data Do You Want to Use? (Press Enter for Default) ')

if trainName=='':

trainName='default'

trainName=trainName+'.pkl'

with open(trainName,'rb') as f:

gestNames=pickle.load(f)

knownGestures=pickle.load(f)

tol=10

while True:

ignore, frame = cam.read()

frame=cv2.resize(frame,(width,height))

handData=findHands.Marks(frame)

if train==1:

if handData!=[]:

print('Please Show Gesture ',gestNames[trainCnt],': Press t when Ready')

if cv2.waitKey(1) & 0xff==ord('t'):

knownGesture=findDistances(handData[0])

knownGestures.append(knownGesture)

trainCnt=trainCnt+1

if trainCnt==numGest:

train=0

with open(trainName,'wb') as f:

pickle.dump(gestNames,f)

pickle.dump(knownGestures,f)

if train == 0:

if handData!=[]:

unknownGesture=findDistances(handData[0])

myGesture=findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol)

#error=findError(knownGesture,unknownGesture,keyPoints)

cv2.putText(frame,myGesture,(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)

for hand in handData:

for ind in keyPoints:

cv2.circle(frame,hand[ind],25,(255,0,255),3)

cv2.imshow('my WEBcam', frame)

cv2.moveWindow('my WEBcam',0,0)

if cv2.waitKey(1) & 0xff ==ord('q'):

break

cam.release()

Artificial Intelligence, OpenCV, Python, Tutorial

Improved Gesture Recognition in Python and MediaPipe

February 10, 2022 admin

In this video lesson we show you how you can improve the accuracy of your gesture recognition program developed in the last lesson. We do this by normalizing the hand landmarks distance matrix to a standard size. By doing this, you get accurate results independent of the distance your hand is from the camera. For your convenience, I include the code below which we develop in this lesson. Enjoy!

import time
import cv2
print(cv2.__version__)
import numpy as np

class mpHands:
    import mediapipe as mp
    def __init__(self,maxHands=2,tol1=.5,tol2=.5):
        self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)
    def Marks(self,frame):
        myHands=[]
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.hands.process(frameRGB)
        if results.multi_hand_landmarks != None:
            for handLandMarks in results.multi_hand_landmarks:
                myHand=[]
                for landMark in handLandMarks.landmark:
                    myHand.append((int(landMark.x*width),int(landMark.y*height)))
                myHands.append(myHand)
        return myHands
def findDistances(handData):
    distMatrix=np.zeros([len(handData),len(handData)],dtype='float')
    palmSize=((handData[0][0]-handData[9][0])**2+(handData[0][1]-handData[9][1])**2)**(1./2.)
    for row in range(0,len(handData)):
        for column in range(0,len(handData)):
            distMatrix[row][column]=(((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.))/palmSize
    return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):
    error=0
    for row in keyPoints:
        for column in keyPoints:
            error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])
    print(error)
    return error
def findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol):
    errorArray=[]
    for i in range(0,len(gestNames),1):
        error=findError(knownGestures[i],unknownGesture,keyPoints)
        errorArray.append(error)
    errorMin=errorArray[0]
    minIndex=0
    for i in range(0,len(errorArray),1):
        if errorArray[i]<errorMin:
            errorMin=errorArray[i]
            minIndex=i
    if errorMin<tol:
        gesture=gestNames[minIndex]
    if errorMin>=tol:
        gesture='Unknown'
    return gesture


width=1280
height=720
cam=cv2.VideoCapture(4,cv2.CAP_DSHOW)
cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)
cam.set(cv2.CAP_PROP_FPS, 30)
cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))
findHands=mpHands(1)
time.sleep(5)
keyPoints=[0,4,5,9,13,17,8,12,16,20]
train=True
tol=10
trainCnt=0
knownGestures=[]

numGest=int(input('How Many Gestures Do You Want? '))

gestNames=[]

for i in range(0,numGest,1):
    prompt='Name of Gesture #'+str(i+1)+' '
    name=input(prompt)
    gestNames.append(name)
print(gestNames)

while True:
    ignore,  frame = cam.read()
    frame=cv2.resize(frame,(width,height))
    handData=findHands.Marks(frame)
    if train==True:
        if handData!=[]:
            print('Please Show Gesture ',gestNames[trainCnt],': Press t when Ready')
            if cv2.waitKey(1) & 0xff==ord('t'):
                knownGesture=findDistances(handData[0])
                knownGestures.append(knownGesture)
                trainCnt=trainCnt+1
                if trainCnt==numGest:
                    train=False
    if train == False:
        if handData!=[]:
            unknownGesture=findDistances(handData[0])
            myGesture=findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol)
            #error=findError(knownGesture,unknownGesture,keyPoints)
            cv2.putText(frame,myGesture,(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)
    for hand in handData:
        for ind in keyPoints:
            cv2.circle(frame,hand[ind],25,(255,0,255),3)
    cv2.imshow('my WEBcam', frame)
    cv2.moveWindow('my WEBcam',0,0)
    if cv2.waitKey(1) & 0xff ==ord('q'):
        break
cam.release()

100

101

102

103

104

105

import time

import cv2

print(cv2.__version__)

import numpy as np

class mpHands:

import mediapipe as mp

def __init__(self,maxHands=2,tol1=.5,tol2=.5):

self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)

def Marks(self,frame):

myHands=[]

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.hands.process(frameRGB)

if results.multi_hand_landmarks != None:

for handLandMarks in results.multi_hand_landmarks:

myHand=[]

for landMark in handLandMarks.landmark:

myHand.append((int(landMark.x*width),int(landMark.y*height)))

myHands.append(myHand)

return myHands

def findDistances(handData):

distMatrix=np.zeros([len(handData),len(handData)],dtype='float')

palmSize=((handData[0][0]-handData[9][0])**2+(handData[0][1]-handData[9][1])**2)**(1./2.)

for row in range(0,len(handData)):

for column in range(0,len(handData)):

distMatrix[row][column]=(((handData[row][0]-handData[column][0])**2+(handData[row][1]-handData[column][1])**2)**(1./2.))/palmSize

return distMatrix

def findError(gestureMatrix,unknownMatrix,keyPoints):

error=0

for row in keyPoints:

for column in keyPoints:

error=error+abs(gestureMatrix[row][column]-unknownMatrix[row][column])

print(error)

return error

def findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol):

errorArray=[]

for i in range(0,len(gestNames),1):

error=findError(knownGestures[i],unknownGesture,keyPoints)

errorArray.append(error)

errorMin=errorArray[0]

minIndex=0

for i in range(0,len(errorArray),1):

if errorArray[i]<errorMin:

errorMin=errorArray[i]

minIndex=i

if errorMin<tol:

gesture=gestNames[minIndex]

if errorMin>=tol:

gesture='Unknown'

return gesture

width=1280

height=720

cam=cv2.VideoCapture(4,cv2.CAP_DSHOW)

cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)

cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)

cam.set(cv2.CAP_PROP_FPS, 30)

cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(1)

time.sleep(5)

keyPoints=[0,4,5,9,13,17,8,12,16,20]

train=True

tol=10

trainCnt=0

knownGestures=[]

numGest=int(input('How Many Gestures Do You Want? '))

gestNames=[]

for i in range(0,numGest,1):

prompt='Name of Gesture #'+str(i+1)+' '

name=input(prompt)

gestNames.append(name)

print(gestNames)

while True:

ignore, frame = cam.read()

frame=cv2.resize(frame,(width,height))

handData=findHands.Marks(frame)

if train==True:

if handData!=[]:

print('Please Show Gesture ',gestNames[trainCnt],': Press t when Ready')

if cv2.waitKey(1) & 0xff==ord('t'):

knownGesture=findDistances(handData[0])

knownGestures.append(knownGesture)

trainCnt=trainCnt+1

if trainCnt==numGest:

train=False

if train == False:

if handData!=[]:

unknownGesture=findDistances(handData[0])

myGesture=findGesture(unknownGesture,knownGestures,keyPoints,gestNames,tol)

#error=findError(knownGesture,unknownGesture,keyPoints)

cv2.putText(frame,myGesture,(100,175),cv2.FONT_HERSHEY_SIMPLEX,3,(255,0,0),8)

for hand in handData:

for ind in keyPoints:

cv2.circle(frame,hand[ind],25,(255,0,255),3)

cv2.imshow('my WEBcam', frame)

cv2.moveWindow('my WEBcam',0,0)

if cv2.waitKey(1) & 0xff ==ord('q'):

break

cam.release()

s lesson. Enjoy!

Technology Tutorials

Using an Arduino with Python LESSON 8: Live Thermometer 3D Visual Using DHT11

AI for Everyone LESSON 29: Control of Real World Objects with Gesture Recognition in Mediapipe

Using an Arduino with Python LESSON 7: Measuring Temperature and Humidity With the DHT11

AI for Everyone LESSON 28: Storing and Loading Gesture Training Data in Mediapipe

Improved Gesture Recognition in Python and MediaPipe

Making The World a Better Place One High Tech Project at a Time. Enjoy!