Tag Archives: Face Mesh

Hey everybody! Welcome back to the AI on the Edge series!

In Lesson 49, we’re doing something that’s just plain cool — we’re turning that tiny little SSD1306 OLED into a live animated avatar that follows your face and hands in real time!

That’s right — no more static images, and no more being tethered to your desktop and monitor. This time your OLED is alive! Move your head, smile, raise your eyebrows, wave your hands… and the little display instantly mirrors your movements like a miniature digital twin. It’s like having a tiny version of yourself living on that 128×64 screen!

What You’ll Learn in This Lesson:

How to use MediaPipe Face Mesh to track 468 facial landmarks
How to combine it with Hand Tracking at the same time
The trick to projecting those landmarks onto a tiny OLED in real time
How to draw both points and connecting lines to create smooth, animated avatars
How to keep everything running fast on the Raspberry Pi

This is one of those projects that makes people stop and say “Wait… how is that even possible on that little screen?!”

By the end of this lesson, you’ll have a foundation for building all kinds of fun real-time avatar projects — everything from simple face mirrors to gesture-controlled characters and beyond.

Why This Is Awesome:

We’re not just displaying video on the big screen anymore — we’re compressing all that AI power down onto a cheap little OLED. This is real AI on the Edge stuff, and it looks incredibly impressive for how simple the hardware is.

Whether you want to build interactive displays, robot faces, wearable tech, or just blow your friends’ minds, this lesson gives you the core technique to make it happen.

Ready to make your OLED come alive?

Fire up the code, run it, and watch your tiny digital self start moving with you. Then start experimenting! Try drawing just the eyes and mouth, or just the hands, or maybe even turn it into a little stick figure avatar.

As always, I want to see what you create with this! Drop your versions in the comments or tag me — I love seeing the creative stuff you guys come up with. Don’t just copy and paste my code. Show you really understand the video by making the code yout own.

Enough talk, lets get this party started!

# ====================================================================
# DISCLAIMER:
# This code is provided as-is for educational and experimental 
# purposes only. The author makes no representations or warranties of 
# any kind concerning the safety, suitability, or accuracy of this 
# code. Use at your own risk. The author assumes no liability for any 
# damages, system failures, security breaches, or network issues 
# resulting from the use or implementation of this script.
# ====================================================================

import cv2
import time
from picamera2 import Picamera2
import mediapipe as mp
import numpy as np
import board
import adafruit_ssd1306
from PIL import Image, ImageDraw, ImageFont
import time

wOLED = 128
hOLED = 64
I2C_ADDRESS = 0x3C
i2c = board.I2C()
oled = adafruit_ssd1306.SSD1306_I2C(wOLED, hOLED, i2c, addr = I2C_ADDRESS)

W=1280
H=720
tStart = time.time()
fps = 0

piCam = Picamera2(1)
piCam.preview_configuration.main.size = (W, H)
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate = 60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(
    
    max_num_faces=1,
    refine_landmarks=True,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)

hands = mp.solutions.hands.Hands(
    model_complexity = 0,
    min_detection_confidence = .5,
    min_tracking_confidence = .5,
    max_num_hands=2 )

cv2.namedWindow('Camera',cv2.WINDOW_GUI_NORMAL)
cv2.moveWindow('Camera',0,65)
cv2.resizeWindow('Camera',W,H)
while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame = piCam.capture_array()
    frame = cv2.flip(frame, -1)

    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    faceResults = faceMesh.process(rgb)
    handResults = hands.process(rgb)
    
    image = Image.new("1",(wOLED,hOLED))
    draw = ImageDraw.Draw(image)
    
    if handResults.multi_hand_landmarks:
        for myHands in handResults.multi_hand_landmarks:
            landMarks = myHands.landmark
            lmAll =[]
            for lm in landMarks:
                x = int(lm.x*W)
                y = int(lm.y*H)
                cv2.circle(frame,(x,y),10,(255,0,0),-1)
                xOLED = int(lm.x*wOLED)
                yOLED = int(lm.y*hOLED)
                draw.point((xOLED,yOLED), fill =255)
                lmAll.append((xOLED,yOLED))
            #Uncomment out these lines if you want
            #Lines between the finger joints
#             for idx in mp.solutions.hands.HAND_CONNECTIONS:
#                 lineStart = idx[0]
#                 lineEnd = idx[1]
#                 pt1 = lmAll[lineStart]
#                 pt2 = lmAll[lineEnd]
#                 draw.line((pt1[0],pt1[1],pt2[0],pt2[1]),width=1,fill=255)
#                              
    if faceResults.multi_face_landmarks:
        for faceLandMarks in faceResults.multi_face_landmarks:
            #print(faceLandmarks)
            lmAll = []
            for lm in faceLandMarks.landmark:
                xOLED = int(lm.x*wOLED)
                yOLED = int(lm.y*hOLED)
                draw.point((xOLED,yOLED), fill = 255)
                #print(xOLED,yOLED)
                lmAll.append((xOLED,yOLED))
            for idx in mp.solutions.face_mesh.FACEMESH_CONTOURS:
                #print(idx)
                lineStart = idx[0]
                lineEnd = idx[1]
                pt1 = lmAll[lineStart]
                pt2 = lmAll[lineEnd]
                draw.line((pt1[0],pt1[1],pt2[0],pt2[1]), width=1, fill = 255)



    oled.image(image)
    oled.show()


    myText = "FPS: "+str(round(fps,1))
    cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

    cv2.imshow("Camera", frame)

    if cv2.waitKey(1) == ord('q'):
        break

cv2.destroyAllWindows()
piCam.stop()
oled.fill(0)
oled.show()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

# ====================================================================

# DISCLAIMER:

# This code is provided as-is for educational and experimental

# purposes only. The author makes no representations or warranties of

# any kind concerning the safety, suitability, or accuracy of this

# code. Use at your own risk. The author assumes no liability for any

# damages, system failures, security breaches, or network issues

# resulting from the use or implementation of this script.

# ====================================================================

import cv2

import time

from picamera2 import Picamera2

import mediapipe as mp

import numpy as np

import board

import adafruit_ssd1306

from PIL import Image, ImageDraw, ImageFont

import time

wOLED = 128

hOLED = 64

I2C_ADDRESS = 0x3C

i2c = board.I2C()

oled = adafruit_ssd1306.SSD1306_I2C(wOLED, hOLED, i2c, addr = I2C_ADDRESS)

W=1280

H=720

tStart = time.time()

fps = 0

piCam = Picamera2(1)

piCam.preview_configuration.main.size = (W, H)

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate = 60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(

max_num_faces=1,

refine_landmarks=True,

min_detection_confidence=0.5,

min_tracking_confidence=0.5)

hands = mp.solutions.hands.Hands(

model_complexity = 0,

min_detection_confidence = .5,

min_tracking_confidence = .5,

max_num_hands=2 )

cv2.namedWindow('Camera',cv2.WINDOW_GUI_NORMAL)

cv2.moveWindow('Camera',0,65)

cv2.resizeWindow('Camera',W,H)

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame = piCam.capture_array()

frame = cv2.flip(frame, -1)

rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

faceResults = faceMesh.process(rgb)

handResults = hands.process(rgb)

image = Image.new("1",(wOLED,hOLED))

draw = ImageDraw.Draw(image)

if handResults.multi_hand_landmarks:

for myHands in handResults.multi_hand_landmarks:

landMarks = myHands.landmark

lmAll =[]

for lm in landMarks:

x = int(lm.x*W)

y = int(lm.y*H)

cv2.circle(frame,(x,y),10,(255,0,0),-1)

xOLED = int(lm.x*wOLED)

yOLED = int(lm.y*hOLED)

draw.point((xOLED,yOLED), fill =255)

lmAll.append((xOLED,yOLED))

#Uncomment out these lines if you want

#Lines between the finger joints

# for idx in mp.solutions.hands.HAND_CONNECTIONS:

# lineStart = idx[0]

# lineEnd = idx[1]

# pt1 = lmAll[lineStart]

# pt2 = lmAll[lineEnd]

# draw.line((pt1[0],pt1[1],pt2[0],pt2[1]),width=1,fill=255)

if faceResults.multi_face_landmarks:

for faceLandMarks in faceResults.multi_face_landmarks:

#print(faceLandmarks)

lmAll = []

for lm in faceLandMarks.landmark:

xOLED = int(lm.x*wOLED)

yOLED = int(lm.y*hOLED)

draw.point((xOLED,yOLED), fill = 255)

#print(xOLED,yOLED)

lmAll.append((xOLED,yOLED))

for idx in mp.solutions.face_mesh.FACEMESH_CONTOURS:

#print(idx)

lineStart = idx[0]

lineEnd = idx[1]

pt1 = lmAll[lineStart]

pt2 = lmAll[lineEnd]

draw.line((pt1[0],pt1[1],pt2[0],pt2[1]), width=1, fill = 255)

oled.image(image)

oled.show()

myText = "FPS: "+str(round(fps,1))

cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Camera", frame)

if cv2.waitKey(1) == ord('q'):

break

cv2.destroyAllWindows()

piCam.stop()

oled.fill(0)

oled.show()

AI On the Edge, Raspberry Pi

AI on the Edge LESSON 47: Emotion Detector Using MediaPipe, OpenCV and Raspberry Pi

September 3, 2026 admin

In this exciting lesson, we build a real-time Emotion Detector that can recognize basic human emotions using just a Raspberry Pi, a camera, and the power of MediaPipe. The system watches your face and identifies whether you look Neutral, Smiling, Surprised, or Angry — and then reacts instantly by changing the color of a NeoPixel ring and displaying the detected emotion on both the camera preview and a small OLED display.Using MediaPipe’s Face Mesh solution, the program tracks 468 facial landmarks in real time. From these points, we carefully calculate key facial ratios — such as eye openness, mouth width, and mouth height relative to head width. These measurements allow us to create simple but effective rules that distinguish between different emotional expressions. For example, a wide mouth combined with raised cheeks indicates a smile, while a small mouth opening and narrowed eyes suggests anger.The project beautifully integrates several powerful technologies:

Picamera2 for high-performance camera capture
OpenCV for image processing and on-screen text display
MediaPipe Face Mesh for accurate facial landmark detection
NeoPixel RGB ring that lights up in different colors depending on the detected emotion
SSD1306 OLED display that shows both the emotion name and a simplified wireframe of your face

One of the most satisfying parts of this project is seeing the NeoPixel ring instantly change color to match your emotion — green for happy, cyan for surprised, red for angry, and yellow for neutral. The OLED also mirrors the detected emotion, making the entire system feel alive and responsive.This lesson is a fantastic step forward in understanding how to combine computer vision with physical outputs on the edge. You will learn how to extract meaningful measurements from facial landmarks, build rule-based emotion logic, and synchronize visual feedback across multiple devices (camera preview, NeoPixel, and OLED).By the end of Lesson 47, you will have a working real-time emotion recognition system running entirely on a Raspberry Pi — a great foundation for more advanced projects like mood-reactive lights, interactive robots, or even assistive technology.

# ====================================================================
# DISCLAIMER:
# This code is provided as-is for educational and experimental 
# purposes only. The author makes no representations or warranties of 
# any kind concerning the safety, suitability, or accuracy of this 
# code. Use at your own risk. The author assumes no liability for any 
# damages, system failures, security breaches, or network issues 
# resulting from the use or implementation of this script.
# ====================================================================

import cv2
import time
from picamera2 import Picamera2
import mediapipe as mp
import numpy as np
#################################
import board
import adafruit_ssd1306
from PIL import Image, ImageDraw, ImageFont
import time

import neopixel_spi as neopixel
import colorsys
spi=board.SPI()
LC = 12
PO = neopixel.GRB
strip = neopixel.NeoPixel_SPI(spi, LC,
            pixel_order=PO,auto_write=False)
strip.fill(0)
strip.show()

wOLED = 128
hOLED = 64
I2C_ADDRESS = 0x3C
i2c = board.I2C()
oled = adafruit_ssd1306.SSD1306_I2C(wOLED, hOLED, i2c, addr = I2C_ADDRESS)
fontLarge = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",14)
#################################
W=1280
H=720
tStart = time.time()
fps = 0

piCam = Picamera2(1)
piCam.preview_configuration.main.size = (W, H)
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate = 60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(
    
    max_num_faces=1,
    refine_landmarks=True,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)

cv2.namedWindow('Camera',cv2.WINDOW_GUI_NORMAL)
cv2.moveWindow('Camera',0,65)
cv2.resizeWindow('Camera',W,H)
while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame = piCam.capture_array()
    frame = cv2.flip(frame, -1)

    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = faceMesh.process(rgb)
    
    if results.multi_face_landmarks:
        for faceLandMarks in results.multi_face_landmarks:
            #print(faceLandmarks)
            image = Image.new("1",(wOLED,hOLED))
            draw = ImageDraw.Draw(image)
            lmAll = []
            j=0
            for lm in faceLandMarks.landmark:
                xOLED = int(lm.x*wOLED)
                yOLED = int(lm.y*hOLED)
                draw.point((xOLED,yOLED), fill = 255)
                #print(xOLED,yOLED)
                lmAll.append((xOLED,yOLED))
                x = int(lm.x*W)
                y = int(lm.y*H)
                #cv2.putText(frame,str(j),(x,y),fontFace, .3, (0,255,0),1)
                j = j+1
            for idx in mp.solutions.face_mesh.FACEMESH_CONTOURS:
                #print(idx)
                lineStart = idx[0]
                lineEnd = idx[1]
                pt1 = lmAll[lineStart]
                pt2 = lmAll[lineEnd]
                draw.line((pt1[0],pt1[1],pt2[0],pt2[1]), width=1, fill = 255)

            keyLM = faceLandMarks.landmark
            
            headRight = 323
            headLeft = 93
            headWidth = keyLM[headRight].x -keyLM[headLeft].x
            cv2.putText(frame,str(headRight),(int(keyLM[headRight].x*W),int(keyLM[headRight].y*H)),fontFace, .3, (0,255,0),1)
            cv2.putText(frame,str(headLeft),(int(keyLM[headLeft].x*W),int(keyLM[headLeft].y*H)),fontFace, .3, (0,255,0),1)
            
            leftEyeTop = 159
            leftEyeBottom = 145
            leftEyeHeight =keyLM[leftEyeBottom].y -keyLM[leftEyeTop].y
            leftEyeLeft = 130
            leftEyeRight = 154
            leftEyeWidth = keyLM[leftEyeRight].x -keyLM[leftEyeLeft].x
            leftEyeRatio = leftEyeHeight/leftEyeWidth*100 
            
            rightEyeTop = 385
            rightEyeBottom = 253
            rightEyeHeight = keyLM[rightEyeBottom].y -keyLM[rightEyeTop].y
            rightEyeLeft = 465
            rightEyeRight = 249
            rightEyeWidth = keyLM[rightEyeRight].x -keyLM[rightEyeLeft].x
            rightEyeRatio = rightEyeHeight/rightEyeWidth*100
            
            eyeRatio = (rightEyeRatio +leftEyeRatio)/2
            
            mouthLeft = 61
            mouthRight = 291
            mouthTop = 13
            mouthBottom = 15
            mouthWidth = keyLM[mouthRight].x-keyLM[mouthLeft].x
            mouthHeight = keyLM[mouthBottom].y-keyLM[mouthTop].y
            mouthWidthRatio = mouthWidth/headWidth *100
            mouthHeightRatio = mouthHeight/headWidth*100
            
            print(round(mouthWidthRatio,1),round(mouthHeightRatio,1),round(eyeRatio,1))
            color = (255,255,0)
            emotion = 'NEUTRAL'
            if mouthWidthRatio >37 and mouthHeightRatio>4 and eyeRatio<74:
                emotion = 'SMILE'
                color = (0,255,0)
            if mouthWidthRatio <37 and mouthHeightRatio>10 and eyeRatio>70:
                emotion = 'SURPRISED'
                color = (0,255,255)
            if mouthWidthRatio <32 and mouthHeightRatio<4 and eyeRatio<57:
                emotion = 'ANGRY'
                color = (0,0,255)               
            cv2.putText(frame, emotion, (int(W*.1),int(H*.15)),
                    fontFace, 2.2, color, 4)
            strip.fill((color[2],color[1],color[0]))
            strip.show()
            #print('XXXXXXXXXXXXXXXXXXx')
            draw.text((0,0),emotion,font=fontLarge,fill=255)
            oled.image(image)
            oled.show()
        #print()

#     if results.multi_face_landmarks:
#         for faceLandmarks in results.multi_face_landmarks:
#             # Optional second pass for glow effect
#             mp.solutions.drawing_utils.draw_landmarks(
#                 image=frame,
#                 landmark_list=faceLandmarks,
#                 connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,
#                 landmark_drawing_spec=None,
#                 connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
#                     color=(100, 100, 100),
#                     thickness=1
#                 )
#             )
#             # === Best looking translucent mesh ===
#             mp.solutions.drawing_utils.draw_landmarks(
#                 image=frame,
#                 landmark_list=faceLandmarks,
#                 connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,
#                 landmark_drawing_spec=None,                    # Hide dots
#                 connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
#                     color=(0, 0, 180),      # Bright cyan/teal
#                     thickness=7
#                 )
#             )
# 
#             # Optional: Extra detail on irises (makes eyes look better)
#             mp.solutions.drawing_utils.draw_landmarks(
#                 image=frame,
#                 landmark_list=faceLandmarks,
#                 connections=mp.solutions.face_mesh.FACEMESH_IRISES,
#                 landmark_drawing_spec=None,
#                 connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
#                     color=(255, 0, 0),
#                     thickness=7
#                 )
#             )

    myText = "FPS: "+str(round(fps,1))
    cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

    cv2.imshow("Camera", frame)

    if cv2.waitKey(1) == ord('q'):
        break

cv2.destroyAllWindows()
piCam.stop()
oled.fill(0)
oled.show()
strip.fill(0)
strip.show()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

# ====================================================================

# DISCLAIMER:

# This code is provided as-is for educational and experimental

# purposes only. The author makes no representations or warranties of

# any kind concerning the safety, suitability, or accuracy of this

# code. Use at your own risk. The author assumes no liability for any

# damages, system failures, security breaches, or network issues

# resulting from the use or implementation of this script.

# ====================================================================

import cv2

import time

from picamera2 import Picamera2

import mediapipe as mp

import numpy as np

#################################

import board

import adafruit_ssd1306

from PIL import Image, ImageDraw, ImageFont

import time

import neopixel_spi as neopixel

import colorsys

spi=board.SPI()

LC = 12

PO = neopixel.GRB

strip = neopixel.NeoPixel_SPI(spi, LC,

pixel_order=PO,auto_write=False)

strip.fill(0)

strip.show()

wOLED = 128

hOLED = 64

I2C_ADDRESS = 0x3C

i2c = board.I2C()

oled = adafruit_ssd1306.SSD1306_I2C(wOLED, hOLED, i2c, addr = I2C_ADDRESS)

fontLarge = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",14)

#################################

W=1280

H=720

tStart = time.time()

fps = 0

piCam = Picamera2(1)

piCam.preview_configuration.main.size = (W, H)

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate = 60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(

max_num_faces=1,

refine_landmarks=True,

min_detection_confidence=0.5,

min_tracking_confidence=0.5)

cv2.namedWindow('Camera',cv2.WINDOW_GUI_NORMAL)

cv2.moveWindow('Camera',0,65)

cv2.resizeWindow('Camera',W,H)

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame = piCam.capture_array()

frame = cv2.flip(frame, -1)

rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

results = faceMesh.process(rgb)

if results.multi_face_landmarks:

for faceLandMarks in results.multi_face_landmarks:

#print(faceLandmarks)

image = Image.new("1",(wOLED,hOLED))

draw = ImageDraw.Draw(image)

lmAll = []

j=0

for lm in faceLandMarks.landmark:

xOLED = int(lm.x*wOLED)

yOLED = int(lm.y*hOLED)

draw.point((xOLED,yOLED), fill = 255)

#print(xOLED,yOLED)

lmAll.append((xOLED,yOLED))

x = int(lm.x*W)

y = int(lm.y*H)

#cv2.putText(frame,str(j),(x,y),fontFace, .3, (0,255,0),1)

j = j+1

for idx in mp.solutions.face_mesh.FACEMESH_CONTOURS:

#print(idx)

lineStart = idx[0]

lineEnd = idx[1]

pt1 = lmAll[lineStart]

pt2 = lmAll[lineEnd]

draw.line((pt1[0],pt1[1],pt2[0],pt2[1]), width=1, fill = 255)

keyLM = faceLandMarks.landmark

headRight = 323

headLeft = 93

headWidth = keyLM[headRight].x -keyLM[headLeft].x

cv2.putText(frame,str(headRight),(int(keyLM[headRight].x*W),int(keyLM[headRight].y*H)),fontFace, .3, (0,255,0),1)

cv2.putText(frame,str(headLeft),(int(keyLM[headLeft].x*W),int(keyLM[headLeft].y*H)),fontFace, .3, (0,255,0),1)

leftEyeTop = 159

leftEyeBottom = 145

leftEyeHeight =keyLM[leftEyeBottom].y -keyLM[leftEyeTop].y

leftEyeLeft = 130

leftEyeRight = 154

leftEyeWidth = keyLM[leftEyeRight].x -keyLM[leftEyeLeft].x

leftEyeRatio = leftEyeHeight/leftEyeWidth*100

rightEyeTop = 385

rightEyeBottom = 253

rightEyeHeight = keyLM[rightEyeBottom].y -keyLM[rightEyeTop].y

rightEyeLeft = 465

rightEyeRight = 249

rightEyeWidth = keyLM[rightEyeRight].x -keyLM[rightEyeLeft].x

rightEyeRatio = rightEyeHeight/rightEyeWidth*100

eyeRatio = (rightEyeRatio +leftEyeRatio)/2

mouthLeft = 61

mouthRight = 291

mouthTop = 13

mouthBottom = 15

mouthWidth = keyLM[mouthRight].x-keyLM[mouthLeft].x

mouthHeight = keyLM[mouthBottom].y-keyLM[mouthTop].y

mouthWidthRatio = mouthWidth/headWidth *100

mouthHeightRatio = mouthHeight/headWidth*100

print(round(mouthWidthRatio,1),round(mouthHeightRatio,1),round(eyeRatio,1))

color = (255,255,0)

emotion = 'NEUTRAL'

if mouthWidthRatio >37 and mouthHeightRatio>4 and eyeRatio<74:

emotion = 'SMILE'

color = (0,255,0)

if mouthWidthRatio <37 and mouthHeightRatio>10 and eyeRatio>70:

emotion = 'SURPRISED'

color = (0,255,255)

if mouthWidthRatio <32 and mouthHeightRatio<4 and eyeRatio<57:

emotion = 'ANGRY'

color = (0,0,255)

cv2.putText(frame, emotion, (int(W*.1),int(H*.15)),

fontFace, 2.2, color, 4)

strip.fill((color[2],color[1],color[0]))

strip.show()

#print('XXXXXXXXXXXXXXXXXXx')

draw.text((0,0),emotion,font=fontLarge,fill=255)

oled.image(image)

oled.show()

#print()

# if results.multi_face_landmarks:

# for faceLandmarks in results.multi_face_landmarks:

# # Optional second pass for glow effect

# mp.solutions.drawing_utils.draw_landmarks(

# image=frame,

# landmark_list=faceLandmarks,

# connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,

# landmark_drawing_spec=None,

# connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

# color=(100, 100, 100),

# thickness=1

# )

# # === Best looking translucent mesh ===

# mp.solutions.drawing_utils.draw_landmarks(

# image=frame,

# landmark_list=faceLandmarks,

# connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,

# landmark_drawing_spec=None, # Hide dots

# connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

# color=(0, 0, 180), # Bright cyan/teal

# thickness=7

# )

# # Optional: Extra detail on irises (makes eyes look better)

# mp.solutions.drawing_utils.draw_landmarks(

# image=frame,

# landmark_list=faceLandmarks,

# connections=mp.solutions.face_mesh.FACEMESH_IRISES,

# landmark_drawing_spec=None,

# connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

# color=(255, 0, 0),

# thickness=7

# )

myText = "FPS: "+str(round(fps,1))

cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Camera", frame)

if cv2.waitKey(1) == ord('q'):

break

cv2.destroyAllWindows()

piCam.stop()

oled.fill(0)

oled.show()

strip.fill(0)

strip.show()

These are the schematics we are using for todays project. If you are taking the class, you should already have these components connected:

Fusion Hat Circuit Diagram — This is the circuit we will use moving forward in the class

SSD1306 OLED Connected to the Fusion AI Hat

AI On the Edge, Raspberry Pi

AI on the Edge LESSON 44: Displaying Live MediaPipe FaceMesh Avatar on SSD1306 OLED in OpenCV

August 23, 2026 admin

This is Getting Crazy Cool!

Hey guys, welcome back to the Fusion AI Lab series, AI on the Edge! In Lesson 44 we just took things up another notch. We’re now running MediaPipe Face Mesh on our Raspberry Pi 5, pulling all 468 facial landmarks in real time, and then drawing a clean, smooth face contour directly onto a tiny 128×64 OLED display!

That’s right — your face is now living on that little monochrome OLED in real time! Every eyebrow raise, smile, head tilt, and eye movement gets faithfully reproduced on the display. We’re pulling the camera feed with picamera2, processing it with MediaPipe’s powerful face mesh, scaling the landmarks down to OLED resolution, and then drawing the official FACEMESH_CONTOURS connections using PIL. The result is surprisingly smooth and fun to watch! This project really shows the power of combining modern AI vision tools with simple embedded hardware. Watching your own face rendered in real time on a tiny OLED is just plain awesome — it feels like sci-fi stuff from just a few years ago, and now we’re doing it on the Edge with our Fusion AI Lab kit!

If you’ve been following along, you’re starting to see how powerful these tools are becoming. We’re no longer just detecting faces — we’re understanding the structure of the face and visualizing it however we want. And we’re just getting started!

So fire up your Raspberry Pi, grab that OLED, and let’s keep pushing the limits. You’re doing some seriously cool stuff!

# ====================================================================
# DISCLAIMER:
# This code is provided as-is for educational and experimental 
# purposes only. The author makes no representations or warranties of 
# any kind concerning the safety, suitability, or accuracy of this 
# code. Use at your own risk. The author assumes no liability for any 
# damages, system failures, security breaches, or network issues 
# resulting from the use or implementation of this script.
# ====================================================================
 
import cv2
import time
from picamera2 import Picamera2
import mediapipe as mp
import numpy as np
#################################
import board
import adafruit_ssd1306
from PIL import Image, ImageDraw, ImageFont
import time

wOLED = 128
hOLED = 64
I2C_ADDRESS = 0x3C
i2c = board.I2C()
oled = adafruit_ssd1306.SSD1306_I2C(wOLED, hOLED, i2c, addr = I2C_ADDRESS)

#################################
W=1280
H=720
tStart = time.time()
fps = 0

piCam = Picamera2(1)
piCam.preview_configuration.main.size = (W, H)
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate = 60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(
    
    max_num_faces=1,
    refine_landmarks=True,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)

cv2.namedWindow('Camera',cv2.WINDOW_GUI_NORMAL)
cv2.moveWindow('Camera',0,65)
cv2.resizeWindow('Camera',W,H)
while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame = piCam.capture_array()
    frame = cv2.flip(frame, -1)

    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = faceMesh.process(rgb)
    
    if results.multi_face_landmarks:
        for faceLandMarks in results.multi_face_landmarks:
            #print(faceLandmarks)
            image = Image.new("1",(wOLED,hOLED))
            draw = ImageDraw.Draw(image)
            lmAll = []
            for lm in faceLandMarks.landmark:
                xOLED = int(lm.x*wOLED)
                yOLED = int(lm.y*hOLED)
                draw.point((xOLED,yOLED), fill = 255)
                #print(xOLED,yOLED)
                lmAll.append((xOLED,yOLED))
            for idx in mp.solutions.face_mesh.FACEMESH_CONTOURS:
                #print(idx)
                lineStart = idx[0]
                lineEnd = idx[1]
                pt1 = lmAll[lineStart]
                pt2 = lmAll[lineEnd]
                draw.line((pt1[0],pt1[1],pt2[0],pt2[1]), width=1, fill = 255)


            #print('XXXXXXXXXXXXXXXXXXx')
            oled.image(image)
            oled.show()
        #print()

#     if results.multi_face_landmarks:
#         for faceLandmarks in results.multi_face_landmarks:
#             # Optional second pass for glow effect
#             mp.solutions.drawing_utils.draw_landmarks(
#                 image=frame,
#                 landmark_list=faceLandmarks,
#                 connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,
#                 landmark_drawing_spec=None,
#                 connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
#                     color=(100, 100, 100),
#                     thickness=1
#                 )
#             )
#             # === Best looking translucent mesh ===
#             mp.solutions.drawing_utils.draw_landmarks(
#                 image=frame,
#                 landmark_list=faceLandmarks,
#                 connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,
#                 landmark_drawing_spec=None,                    # Hide dots
#                 connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
#                     color=(0, 0, 180),      # Bright cyan/teal
#                     thickness=7
#                 )
#             )
# 
#             # Optional: Extra detail on irises (makes eyes look better)
#             mp.solutions.drawing_utils.draw_landmarks(
#                 image=frame,
#                 landmark_list=faceLandmarks,
#                 connections=mp.solutions.face_mesh.FACEMESH_IRISES,
#                 landmark_drawing_spec=None,
#                 connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
#                     color=(255, 0, 0),
#                     thickness=7
#                 )
#             )

    myText = "FPS: "+str(round(fps,1))
    cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

    cv2.imshow("Camera", frame)

    if cv2.waitKey(1) == ord('q'):
        break

cv2.destroyAllWindows()
piCam.stop()
oled.fill(0)
oled.show()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

# ====================================================================

# DISCLAIMER:

# This code is provided as-is for educational and experimental

# purposes only. The author makes no representations or warranties of

# any kind concerning the safety, suitability, or accuracy of this

# code. Use at your own risk. The author assumes no liability for any

# damages, system failures, security breaches, or network issues

# resulting from the use or implementation of this script.

# ====================================================================

import cv2

import time

from picamera2 import Picamera2

import mediapipe as mp

import numpy as np

#################################

import board

import adafruit_ssd1306

from PIL import Image, ImageDraw, ImageFont

import time

wOLED = 128

hOLED = 64

I2C_ADDRESS = 0x3C

i2c = board.I2C()

oled = adafruit_ssd1306.SSD1306_I2C(wOLED, hOLED, i2c, addr = I2C_ADDRESS)

#################################

W=1280

H=720

tStart = time.time()

fps = 0

piCam = Picamera2(1)

piCam.preview_configuration.main.size = (W, H)

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate = 60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(

max_num_faces=1,

refine_landmarks=True,

min_detection_confidence=0.5,

min_tracking_confidence=0.5)

cv2.namedWindow('Camera',cv2.WINDOW_GUI_NORMAL)

cv2.moveWindow('Camera',0,65)

cv2.resizeWindow('Camera',W,H)

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame = piCam.capture_array()

frame = cv2.flip(frame, -1)

rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

results = faceMesh.process(rgb)

if results.multi_face_landmarks:

for faceLandMarks in results.multi_face_landmarks:

#print(faceLandmarks)

image = Image.new("1",(wOLED,hOLED))

draw = ImageDraw.Draw(image)

lmAll = []

for lm in faceLandMarks.landmark:

xOLED = int(lm.x*wOLED)

yOLED = int(lm.y*hOLED)

draw.point((xOLED,yOLED), fill = 255)

#print(xOLED,yOLED)

lmAll.append((xOLED,yOLED))

for idx in mp.solutions.face_mesh.FACEMESH_CONTOURS:

#print(idx)

lineStart = idx[0]

lineEnd = idx[1]

pt1 = lmAll[lineStart]

pt2 = lmAll[lineEnd]

draw.line((pt1[0],pt1[1],pt2[0],pt2[1]), width=1, fill = 255)

#print('XXXXXXXXXXXXXXXXXXx')

oled.image(image)

oled.show()

#print()

# if results.multi_face_landmarks:

# for faceLandmarks in results.multi_face_landmarks:

# # Optional second pass for glow effect

# mp.solutions.drawing_utils.draw_landmarks(

# image=frame,

# landmark_list=faceLandmarks,

# connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,

# landmark_drawing_spec=None,

# connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

# color=(100, 100, 100),

# thickness=1

# )

# # === Best looking translucent mesh ===

# mp.solutions.drawing_utils.draw_landmarks(

# image=frame,

# landmark_list=faceLandmarks,

# connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,

# landmark_drawing_spec=None, # Hide dots

# connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

# color=(0, 0, 180), # Bright cyan/teal

# thickness=7

# )

# # Optional: Extra detail on irises (makes eyes look better)

# mp.solutions.drawing_utils.draw_landmarks(

# image=frame,

# landmark_list=faceLandmarks,

# connections=mp.solutions.face_mesh.FACEMESH_IRISES,

# landmark_drawing_spec=None,

# connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

# color=(255, 0, 0),

# thickness=7

# )

myText = "FPS: "+str(round(fps,1))

cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Camera", frame)

if cv2.waitKey(1) == ord('q'):

break

cv2.destroyAllWindows()

piCam.stop()

oled.fill(0)

oled.show()

In this lesson we are still using our standard Fusion AI Lab Kit cirtuit from the earlier lessons:

We have also connected the OLED to the above circuit according to this diagram:

AI On the Edge, Raspberry Pi

AI on the Edge LESSON 42: Create Composite Images Using Masks in OpenCV and MediaPipe

August 16, 2026 admin

In this exciting project, we combine a Raspberry Pi 5, the Fusion AI Lab Kit, a Pi Camera, and a remote IP camera to generate a stunning real-time composite video. Watch as a glowing, translucent MediaPipe face mesh of my face hovers magically over live video of the Mighty River Nice scenery captured by an IP camera. The effect looks futuristic and professional — perfect for creative video effects, interactive installations, or just blowing your mind with computer vision! Using Picamera2 for high-frame-rate local capture and OpenCV with an RTSP stream from the river camera, we process everything in real time. MediaPipe’s Face Mesh detects and tracks facial landmarks, which we draw as beautiful cyan/teal contours with glowing irises. Then we create a clean mask, separate the mesh foreground from the river background, and blend them seamlessly into one composite frame. You’ll see every debugging layer live on screen too — meshLayer, mask, inverted mask, riverBG, and meshFG — so you can understand exactly how the magic happens.This tutorial is beginner-to-intermediate friendly and packed with practical OpenCV + MediaPipe techniques you can adapt for your own augmented reality projects. Whether you’re a longtime follower of the Paul McWhorter channel or new to the Fusion AI Kit, you’ll walk away inspired and ready to build your own hovering effects, overlays, or interactive displays.Grab the full code from the video description, fire up your Pi 5, and start creating jaw-dropping computer vision projects today. Drop a comment and let me know what you’d like to overlay next — another face mesh, hand tracking, or something completely different? Let’s keep pushing the limits of what we can do with affordable AI hardware!

# ====================================================================
# DISCLAIMER:
# This code is provided as-is for educational and experimental 
# purposes only. The author makes no representations or warranties of 
# any kind concerning the safety, suitability, or accuracy of this 
# code. Use at your own risk. The author assumes no liability for any 
# damages, system failures, security breaches, or network issues 
# resulting from the use or implementation of this script.
# ====================================================================
 
import cv2
import time
from picamera2 import Picamera2
import mediapipe as mp
import numpy as np
import secret
W=1280
H=720
tStart = time.time()
fps = 0

piCam = Picamera2(1)
piCam.preview_configuration.main.size = (W, H)
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate = 60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

cam = cv2.VideoCapture(secret.RTSP1,cv2.CAP_FFMPEG)
cam.set(cv2.CAP_PROP_BUFFERSIZE,0)
cam.set(cv2.CAP_PROP_POS_FRAMES,0)

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(
    
    max_num_faces=1,
    refine_landmarks=True,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)

cv2.namedWindow('Composite',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('Composite',W,H)
cv2.moveWindow('Composite',0,65)

cv2.namedWindow('meshLayer',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('meshLayer',320,180)
cv2.moveWindow('meshLayer',0,H+65)

cv2.namedWindow('grayLayer',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('grayLayer',320,180)
cv2.moveWindow('grayLayer',320,H+65)

cv2.namedWindow('maskLayer',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('maskLayer',320,180)
cv2.moveWindow('maskLayer',320*2,H+65)

cv2.namedWindow('invLayer',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('invLayer',320,180)
cv2.moveWindow('invLayer',320*3,H+65)

cv2.namedWindow('riverBG',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('riverBG',320,180)
cv2.moveWindow('riverBG',W,65)

cv2.namedWindow('meshFG',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('meshFG',320,180)
cv2.moveWindow('meshFG',W,180+65+20)
while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame = piCam.capture_array()
    frame = cv2.flip(frame, -1)
    
    _, river = cam.read()
    river = cv2.resize(river,(W,H))
    meshLayer = frame.copy()
    meshLayer[0:H-1,0:W-1] = [255,255,255]
    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = faceMesh.process(rgb)

    if results.multi_face_landmarks:
        for faceLandmarks in results.multi_face_landmarks:
            # Optional second pass for glow effect
            mp.solutions.drawing_utils.draw_landmarks(
                image=meshLayer,
                landmark_list=faceLandmarks,
                connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,
                landmark_drawing_spec=None,
                connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
                    color=(0, 150, 200),
                    thickness=2
                )
            )
            # === Best looking translucent mesh ===
            mp.solutions.drawing_utils.draw_landmarks(
                image=meshLayer,
                landmark_list=faceLandmarks,
                connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,
                landmark_drawing_spec=None,                    # Hide dots
                connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
                    color=(0, 0, 180),      # Bright cyan/teal
                    thickness=7
                )
            )

            # Optional: Extra detail on irises (makes eyes look better)
            mp.solutions.drawing_utils.draw_landmarks(
                image=meshLayer,
                landmark_list=faceLandmarks,
                connections=mp.solutions.face_mesh.FACEMESH_IRISES,
                landmark_drawing_spec=None,
                connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
                    color=(255, 0, 0),
                    thickness=7
                )
            )
    gray = cv2.cvtColor(meshLayer,cv2.COLOR_BGR2GRAY)

    _ , mask =cv2.threshold(gray, 250, 255, cv2.THRESH_BINARY)
    invLayer = cv2.bitwise_not(mask)
    riverBG =cv2.bitwise_and(river, river, mask = mask)
    meshFG = cv2.bitwise_and(meshLayer,meshLayer, mask=invLayer)
    composite = cv2.add(riverBG,meshFG)
    
    myText = "FPS: "+str(round(fps,1))
    cv2.putText(composite,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

    cv2.imshow("Composite", composite)
    cv2.imshow("meshLayer",meshLayer)
    cv2.imshow("grayLayer",gray)
    cv2.imshow("maskLayer",mask)
    cv2.imshow("invLayer",invLayer)
    cv2.imshow("riverBG",riverBG)
    cv2.imshow("meshFG",meshFG)
    if cv2.waitKey(1) == ord('q'):
        break

cv2.destroyAllWindows()
piCam.stop()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

# ====================================================================

# DISCLAIMER:

# This code is provided as-is for educational and experimental

# purposes only. The author makes no representations or warranties of

# any kind concerning the safety, suitability, or accuracy of this

# code. Use at your own risk. The author assumes no liability for any

# damages, system failures, security breaches, or network issues

# resulting from the use or implementation of this script.

# ====================================================================

import cv2

import time

from picamera2 import Picamera2

import mediapipe as mp

import numpy as np

import secret

W=1280

H=720

tStart = time.time()

fps = 0

piCam = Picamera2(1)

piCam.preview_configuration.main.size = (W, H)

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate = 60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

cam = cv2.VideoCapture(secret.RTSP1,cv2.CAP_FFMPEG)

cam.set(cv2.CAP_PROP_BUFFERSIZE,0)

cam.set(cv2.CAP_PROP_POS_FRAMES,0)

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(

max_num_faces=1,

refine_landmarks=True,

min_detection_confidence=0.5,

min_tracking_confidence=0.5)

cv2.namedWindow('Composite',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('Composite',W,H)

cv2.moveWindow('Composite',0,65)

cv2.namedWindow('meshLayer',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('meshLayer',320,180)

cv2.moveWindow('meshLayer',0,H+65)

cv2.namedWindow('grayLayer',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('grayLayer',320,180)

cv2.moveWindow('grayLayer',320,H+65)

cv2.namedWindow('maskLayer',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('maskLayer',320,180)

cv2.moveWindow('maskLayer',320*2,H+65)

cv2.namedWindow('invLayer',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('invLayer',320,180)

cv2.moveWindow('invLayer',320*3,H+65)

cv2.namedWindow('riverBG',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('riverBG',320,180)

cv2.moveWindow('riverBG',W,65)

cv2.namedWindow('meshFG',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('meshFG',320,180)

cv2.moveWindow('meshFG',W,180+65+20)

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame = piCam.capture_array()

frame = cv2.flip(frame, -1)

_, river = cam.read()

river = cv2.resize(river,(W,H))

meshLayer = frame.copy()

meshLayer[0:H-1,0:W-1] = [255,255,255]

rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

results = faceMesh.process(rgb)

if results.multi_face_landmarks:

for faceLandmarks in results.multi_face_landmarks:

# Optional second pass for glow effect

mp.solutions.drawing_utils.draw_landmarks(

image=meshLayer,

landmark_list=faceLandmarks,

connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,

landmark_drawing_spec=None,

connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

color=(0, 150, 200),

thickness=2

)

# === Best looking translucent mesh ===

mp.solutions.drawing_utils.draw_landmarks(

image=meshLayer,

landmark_list=faceLandmarks,

connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,

landmark_drawing_spec=None, # Hide dots

connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

color=(0, 0, 180), # Bright cyan/teal

thickness=7

)

# Optional: Extra detail on irises (makes eyes look better)

mp.solutions.drawing_utils.draw_landmarks(

image=meshLayer,

landmark_list=faceLandmarks,

connections=mp.solutions.face_mesh.FACEMESH_IRISES,

landmark_drawing_spec=None,

connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

color=(255, 0, 0),

thickness=7

)

gray = cv2.cvtColor(meshLayer,cv2.COLOR_BGR2GRAY)

_ , mask =cv2.threshold(gray, 250, 255, cv2.THRESH_BINARY)

invLayer = cv2.bitwise_not(mask)

riverBG =cv2.bitwise_and(river, river, mask = mask)

meshFG = cv2.bitwise_and(meshLayer,meshLayer, mask=invLayer)

composite = cv2.add(riverBG,meshFG)

myText = "FPS: "+str(round(fps,1))

cv2.putText(composite,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Composite", composite)

cv2.imshow("meshLayer",meshLayer)

cv2.imshow("grayLayer",gray)

cv2.imshow("maskLayer",mask)

cv2.imshow("invLayer",invLayer)

cv2.imshow("riverBG",riverBG)

cv2.imshow("meshFG",meshFG)

if cv2.waitKey(1) == ord('q'):

break

cv2.destroyAllWindows()

piCam.stop()

Artificial Intelligence, OpenCV, Python

Parsing Mediapipe Data for Face Mesh, Pose, and Hand Landmarks

December 9, 2021 admin

We all love the incredibly powerful features of the Mediapipe AI library. But, lets face it, the data is very hard to interpret, and the returned data structures are complex, and poorly documented. The purpose of this lesson is to show you how to parse the complex data structures coming from Mediapipe, and to create simple, intuitive arrays for the landmark data. We create classes in python which do the parsing, and the classes are easy to work with. With these classes, you can use mediapipe simply, and get simple to understand data structures back. The video explains what we are doing and how we are doing it. For your convenience the resulting code is posted below.

import cv2
print(cv2.__version__)

class mpFaceMesh:
    import mediapipe as mp
    def __init__(self,still=False,numFaces=3,tol1=.5,tol2=.5,drawMesh=True):
        self.myFaceMesh=self.mp.solutions.face_mesh.FaceMesh(still,numFaces,tol1,tol2)
        self.myDraw=self.mp.solutions.drawing_utils
        self.draw=drawMesh
    def Marks(self,frame):
        global width
        global height
        drawSpecCircle=self.myDraw.DrawingSpec(thickness=0,circle_radius=0,color=(0,0,255))
        drawSpecLine=self.myDraw.DrawingSpec(thickness=3,circle_radius=2,color=(255,0,0))
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.myFaceMesh.process(frameRGB)
        facesMeshLandmarks=[]
        if results.multi_face_landmarks !=None:
            for faceMesh in results.multi_face_landmarks:
                faceMeshLandmarks=[]
                for lm in faceMesh.landmark:
                    loc=(int(lm.x*width),int(lm.y*height))
                    faceMeshLandmarks.append(loc)
                facesMeshLandmarks.append(faceMeshLandmarks)
                if self.draw==True:
                    self.myDraw.draw_landmarks(frame,faceMesh,self.mp.solutions.face_mesh.FACE_CONNECTIONS,drawSpecCircle,drawSpecLine)
        return facesMeshLandmarks

class mpFace:
    import mediapipe as mp 
    def __init__(self):
        self.myFace=self.mp.solutions.face_detection.FaceDetection()
    def Marks(self,frame):
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.myFace.process(frameRGB)
        faceBoundBoxs=[]
        if results.detections != None:
            for face in results.detections:
                bBox=face.location_data.relative_bounding_box
                topLeft=(int(bBox.xmin*width),int(bBox.ymin*height))
                bottomRight=(int((bBox.xmin+bBox.width)*width),int((bBox.ymin+bBox.height)*height))
                faceBoundBoxs.append((topLeft,bottomRight))
        return faceBoundBoxs

class mpPose:
    import mediapipe as mp
    def __init__(self,still=False,upperBody=False, smoothData=True, tol1=.5, tol2=.5):
        self.myPose=self.mp.solutions.pose.Pose(still,upperBody,smoothData,tol1,tol2)
    def Marks(self,frame):
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.myPose.process(frameRGB)
        poseLandmarks=[]
        if results.pose_landmarks:
            for lm in results.pose_landmarks.landmark:            
                poseLandmarks.append((int(lm.x*width),int(lm.y*height)))
        return poseLandmarks

class mpHands:
    import mediapipe as mp
    def __init__(self,maxHands=2,tol1=.5,tol2=.5):
        self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)
    def Marks(self,frame):
        myHands=[]
        handsType=[]
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.hands.process(frameRGB)
        if results.multi_hand_landmarks != None:
            #print(results.multi_handedness)
            for hand in results.multi_handedness:
                #print(hand)
                #print(hand.classification)
                #print(hand.classification[0])
                handType=hand.classification[0].label
                handsType.append(handType)
            for handLandMarks in results.multi_hand_landmarks:
                myHand=[]
                for landMark in handLandMarks.landmark:
                    myHand.append((int(landMark.x*width),int(landMark.y*height)))
                myHands.append(myHand)
        return myHands,handsType

width=1280
height=720
cam=cv2.VideoCapture(3,cv2.CAP_DSHOW)
cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)
cam.set(cv2.CAP_PROP_FPS, 30)
cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(2)
findFace=mpFace()
findPose=mpPose()
findMesh=mpFaceMesh(drawMesh=True)

font=cv2.FONT_HERSHEY_SIMPLEX
fontColor=(0,0,255)
fontSize=.1
fontThick=1

cv2.namedWindow('Trackbars')
cv2.moveWindow('Trackbars',width+50,0)
cv2.resizeWindow('Trackbars',400,150)


while True:
    ignore,  frame = cam.read()
    frame=cv2.resize(frame,(width,height))
    handsLM,handsType=findHands.Marks(frame)
    faceLoc=findFace.Marks(frame)
    poseLM=findPose.Marks(frame)
    facesMeshLM=findMesh.Marks(frame)
    if poseLM != []:
        for ind in [13,14,15,16]:
            cv2.circle(frame,poseLM[ind],20,(0,255,0),-1)

    for face in faceLoc:
        cv2.rectangle(frame,face[0],face[1],(255,0,0),3)
    for hand,handType in zip(handsLM,handsType):
        if handType=='Right':
            lbl='Right'
        if handType=='Left':
            lbl='Left'
        cv2.putText(frame,lbl,hand[8],font,2,fontColor,2)
    for faceMeshLM in facesMeshLM:
        cnt=0
        for lm in faceMeshLM:
            cv2.putText(frame,str(cnt),lm,font,fontSize,fontColor,fontThick)
            cnt=cnt+1

    cv2.imshow('my WEBcam', frame)
    cv2.moveWindow('my WEBcam',0,0)
    if cv2.waitKey(1) & 0xff ==ord('q'):
        break
cam.release()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

import cv2

print(cv2.__version__)

class mpFaceMesh:

import mediapipe as mp

def __init__(self,still=False,numFaces=3,tol1=.5,tol2=.5,drawMesh=True):

self.myFaceMesh=self.mp.solutions.face_mesh.FaceMesh(still,numFaces,tol1,tol2)

self.myDraw=self.mp.solutions.drawing_utils

self.draw=drawMesh

def Marks(self,frame):

global width

global height

drawSpecCircle=self.myDraw.DrawingSpec(thickness=0,circle_radius=0,color=(0,0,255))

drawSpecLine=self.myDraw.DrawingSpec(thickness=3,circle_radius=2,color=(255,0,0))

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.myFaceMesh.process(frameRGB)

facesMeshLandmarks=[]

if results.multi_face_landmarks !=None:

for faceMesh in results.multi_face_landmarks:

faceMeshLandmarks=[]

for lm in faceMesh.landmark:

loc=(int(lm.x*width),int(lm.y*height))

faceMeshLandmarks.append(loc)

facesMeshLandmarks.append(faceMeshLandmarks)

if self.draw==True:

self.myDraw.draw_landmarks(frame,faceMesh,self.mp.solutions.face_mesh.FACE_CONNECTIONS,drawSpecCircle,drawSpecLine)

return facesMeshLandmarks

class mpFace:

import mediapipe as mp

def __init__(self):

self.myFace=self.mp.solutions.face_detection.FaceDetection()

def Marks(self,frame):

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.myFace.process(frameRGB)

faceBoundBoxs=[]

if results.detections != None:

for face in results.detections:

bBox=face.location_data.relative_bounding_box

topLeft=(int(bBox.xmin*width),int(bBox.ymin*height))

bottomRight=(int((bBox.xmin+bBox.width)*width),int((bBox.ymin+bBox.height)*height))

faceBoundBoxs.append((topLeft,bottomRight))

return faceBoundBoxs

class mpPose:

import mediapipe as mp

def __init__(self,still=False,upperBody=False, smoothData=True, tol1=.5, tol2=.5):

self.myPose=self.mp.solutions.pose.Pose(still,upperBody,smoothData,tol1,tol2)

def Marks(self,frame):

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.myPose.process(frameRGB)

poseLandmarks=[]

if results.pose_landmarks:

for lm in results.pose_landmarks.landmark:

poseLandmarks.append((int(lm.x*width),int(lm.y*height)))

return poseLandmarks

class mpHands:

import mediapipe as mp

def __init__(self,maxHands=2,tol1=.5,tol2=.5):

self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)

def Marks(self,frame):

myHands=[]

handsType=[]

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.hands.process(frameRGB)

if results.multi_hand_landmarks != None:

#print(results.multi_handedness)

for hand in results.multi_handedness:

#print(hand)

#print(hand.classification)

#print(hand.classification[0])

handType=hand.classification[0].label

handsType.append(handType)

for handLandMarks in results.multi_hand_landmarks:

myHand=[]

for landMark in handLandMarks.landmark:

myHand.append((int(landMark.x*width),int(landMark.y*height)))

myHands.append(myHand)

return myHands,handsType

width=1280

height=720

cam=cv2.VideoCapture(3,cv2.CAP_DSHOW)

cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)

cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)

cam.set(cv2.CAP_PROP_FPS, 30)

cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(2)

findFace=mpFace()

findPose=mpPose()

findMesh=mpFaceMesh(drawMesh=True)

font=cv2.FONT_HERSHEY_SIMPLEX

fontColor=(0,0,255)

fontSize=.1

fontThick=1

cv2.namedWindow('Trackbars')

cv2.moveWindow('Trackbars',width+50,0)

cv2.resizeWindow('Trackbars',400,150)

while True:

ignore, frame = cam.read()

frame=cv2.resize(frame,(width,height))

handsLM,handsType=findHands.Marks(frame)

faceLoc=findFace.Marks(frame)

poseLM=findPose.Marks(frame)

facesMeshLM=findMesh.Marks(frame)

if poseLM != []:

for ind in [13,14,15,16]:

cv2.circle(frame,poseLM[ind],20,(0,255,0),-1)

for face in faceLoc:

cv2.rectangle(frame,face[0],face[1],(255,0,0),3)

for hand,handType in zip(handsLM,handsType):

if handType=='Right':

lbl='Right'

if handType=='Left':

lbl='Left'

cv2.putText(frame,lbl,hand[8],font,2,fontColor,2)

for faceMeshLM in facesMeshLM:

cnt=0

for lm in faceMeshLM:

cv2.putText(frame,str(cnt),lm,font,fontSize,fontColor,fontThick)

cnt=cnt+1

cv2.imshow('my WEBcam', frame)

cv2.moveWindow('my WEBcam',0,0)

if cv2.waitKey(1) & 0xff ==ord('q'):

break

cam.release()

Technology Tutorials

Tag Archives: Face Mesh

AI on the Edge LESSON 49: Live Animated Avatars on the SSD1306 OLED With OpenCV

What You’ll Learn in This Lesson:

Why This Is Awesome:

AI on the Edge LESSON 47: Emotion Detector Using MediaPipe, OpenCV and Raspberry Pi

AI on the Edge LESSON 44: Displaying Live MediaPipe FaceMesh Avatar on SSD1306 OLED in OpenCV

AI on the Edge LESSON 42: Create Composite Images Using Masks in OpenCV and MediaPipe

Parsing Mediapipe Data for Face Mesh, Pose, and Hand Landmarks

Making The World a Better Place One High Tech Project at a Time. Enjoy!