Tag Archives: Face Mesh

This is Getting Crazy Cool!

Hey guys, welcome back to the Fusion AI Lab series, AI on the Edge! In Lesson 44 we just took things up another notch. We’re now running MediaPipe Face Mesh on our Raspberry Pi 5, pulling all 468 facial landmarks in real time, and then drawing a clean, smooth face contour directly onto a tiny 128×64 OLED display!

That’s right — your face is now living on that little monochrome OLED in real time! Every eyebrow raise, smile, head tilt, and eye movement gets faithfully reproduced on the display. We’re pulling the camera feed with picamera2, processing it with MediaPipe’s powerful face mesh, scaling the landmarks down to OLED resolution, and then drawing the official FACEMESH_CONTOURS connections using PIL. The result is surprisingly smooth and fun to watch! This project really shows the power of combining modern AI vision tools with simple embedded hardware. Watching your own face rendered in real time on a tiny OLED is just plain awesome — it feels like sci-fi stuff from just a few years ago, and now we’re doing it on the Edge with our Fusion AI Lab kit!

If you’ve been following along, you’re starting to see how powerful these tools are becoming. We’re no longer just detecting faces — we’re understanding the structure of the face and visualizing it however we want. And we’re just getting started!

So fire up your Raspberry Pi, grab that OLED, and let’s keep pushing the limits. You’re doing some seriously cool stuff!

# ====================================================================
# DISCLAIMER:
# This code is provided as-is for educational and experimental 
# purposes only. The author makes no representations or warranties of 
# any kind concerning the safety, suitability, or accuracy of this 
# code. Use at your own risk. The author assumes no liability for any 
# damages, system failures, security breaches, or network issues 
# resulting from the use or implementation of this script.
# ====================================================================
 
import cv2
import time
from picamera2 import Picamera2
import mediapipe as mp
import numpy as np
#################################
import board
import adafruit_ssd1306
from PIL import Image, ImageDraw, ImageFont
import time

wOLED = 128
hOLED = 64
I2C_ADDRESS = 0x3C
i2c = board.I2C()
oled = adafruit_ssd1306.SSD1306_I2C(wOLED, hOLED, i2c, addr = I2C_ADDRESS)

#################################
W=1280
H=720
tStart = time.time()
fps = 0

piCam = Picamera2(1)
piCam.preview_configuration.main.size = (W, H)
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate = 60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(
    
    max_num_faces=1,
    refine_landmarks=True,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)

cv2.namedWindow('Camera',cv2.WINDOW_GUI_NORMAL)
cv2.moveWindow('Camera',0,65)
cv2.resizeWindow('Camera',W,H)
while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame = piCam.capture_array()
    frame = cv2.flip(frame, -1)

    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = faceMesh.process(rgb)
    
    if results.multi_face_landmarks:
        for faceLandMarks in results.multi_face_landmarks:
            #print(faceLandmarks)
            image = Image.new("1",(wOLED,hOLED))
            draw = ImageDraw.Draw(image)
            lmAll = []
            for lm in faceLandMarks.landmark:
                xOLED = int(lm.x*wOLED)
                yOLED = int(lm.y*hOLED)
                draw.point((xOLED,yOLED), fill = 255)
                #print(xOLED,yOLED)
                lmAll.append((xOLED,yOLED))
            for idx in mp.solutions.face_mesh.FACEMESH_CONTOURS:
                #print(idx)
                lineStart = idx[0]
                lineEnd = idx[1]
                pt1 = lmAll[lineStart]
                pt2 = lmAll[lineEnd]
                draw.line((pt1[0],pt1[1],pt2[0],pt2[1]), width=1, fill = 255)


            #print('XXXXXXXXXXXXXXXXXXx')
            oled.image(image)
            oled.show()
        #print()

#     if results.multi_face_landmarks:
#         for faceLandmarks in results.multi_face_landmarks:
#             # Optional second pass for glow effect
#             mp.solutions.drawing_utils.draw_landmarks(
#                 image=frame,
#                 landmark_list=faceLandmarks,
#                 connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,
#                 landmark_drawing_spec=None,
#                 connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
#                     color=(100, 100, 100),
#                     thickness=1
#                 )
#             )
#             # === Best looking translucent mesh ===
#             mp.solutions.drawing_utils.draw_landmarks(
#                 image=frame,
#                 landmark_list=faceLandmarks,
#                 connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,
#                 landmark_drawing_spec=None,                    # Hide dots
#                 connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
#                     color=(0, 0, 180),      # Bright cyan/teal
#                     thickness=7
#                 )
#             )
# 
#             # Optional: Extra detail on irises (makes eyes look better)
#             mp.solutions.drawing_utils.draw_landmarks(
#                 image=frame,
#                 landmark_list=faceLandmarks,
#                 connections=mp.solutions.face_mesh.FACEMESH_IRISES,
#                 landmark_drawing_spec=None,
#                 connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
#                     color=(255, 0, 0),
#                     thickness=7
#                 )
#             )

    myText = "FPS: "+str(round(fps,1))
    cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

    cv2.imshow("Camera", frame)

    if cv2.waitKey(1) == ord('q'):
        break

cv2.destroyAllWindows()
piCam.stop()
oled.fill(0)
oled.show()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

# ====================================================================

# DISCLAIMER:

# This code is provided as-is for educational and experimental

# purposes only. The author makes no representations or warranties of

# any kind concerning the safety, suitability, or accuracy of this

# code. Use at your own risk. The author assumes no liability for any

# damages, system failures, security breaches, or network issues

# resulting from the use or implementation of this script.

# ====================================================================

import cv2

import time

from picamera2 import Picamera2

import mediapipe as mp

import numpy as np

#################################

import board

import adafruit_ssd1306

from PIL import Image, ImageDraw, ImageFont

import time

wOLED = 128

hOLED = 64

I2C_ADDRESS = 0x3C

i2c = board.I2C()

oled = adafruit_ssd1306.SSD1306_I2C(wOLED, hOLED, i2c, addr = I2C_ADDRESS)

#################################

W=1280

H=720

tStart = time.time()

fps = 0

piCam = Picamera2(1)

piCam.preview_configuration.main.size = (W, H)

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate = 60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(

max_num_faces=1,

refine_landmarks=True,

min_detection_confidence=0.5,

min_tracking_confidence=0.5)

cv2.namedWindow('Camera',cv2.WINDOW_GUI_NORMAL)

cv2.moveWindow('Camera',0,65)

cv2.resizeWindow('Camera',W,H)

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame = piCam.capture_array()

frame = cv2.flip(frame, -1)

rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

results = faceMesh.process(rgb)

if results.multi_face_landmarks:

for faceLandMarks in results.multi_face_landmarks:

#print(faceLandmarks)

image = Image.new("1",(wOLED,hOLED))

draw = ImageDraw.Draw(image)

lmAll = []

for lm in faceLandMarks.landmark:

xOLED = int(lm.x*wOLED)

yOLED = int(lm.y*hOLED)

draw.point((xOLED,yOLED), fill = 255)

#print(xOLED,yOLED)

lmAll.append((xOLED,yOLED))

for idx in mp.solutions.face_mesh.FACEMESH_CONTOURS:

#print(idx)

lineStart = idx[0]

lineEnd = idx[1]

pt1 = lmAll[lineStart]

pt2 = lmAll[lineEnd]

draw.line((pt1[0],pt1[1],pt2[0],pt2[1]), width=1, fill = 255)

#print('XXXXXXXXXXXXXXXXXXx')

oled.image(image)

oled.show()

#print()

# if results.multi_face_landmarks:

# for faceLandmarks in results.multi_face_landmarks:

# # Optional second pass for glow effect

# mp.solutions.drawing_utils.draw_landmarks(

# image=frame,

# landmark_list=faceLandmarks,

# connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,

# landmark_drawing_spec=None,

# connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

# color=(100, 100, 100),

# thickness=1

# )

# # === Best looking translucent mesh ===

# mp.solutions.drawing_utils.draw_landmarks(

# image=frame,

# landmark_list=faceLandmarks,

# connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,

# landmark_drawing_spec=None, # Hide dots

# connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

# color=(0, 0, 180), # Bright cyan/teal

# thickness=7

# )

# # Optional: Extra detail on irises (makes eyes look better)

# mp.solutions.drawing_utils.draw_landmarks(

# image=frame,

# landmark_list=faceLandmarks,

# connections=mp.solutions.face_mesh.FACEMESH_IRISES,

# landmark_drawing_spec=None,

# connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

# color=(255, 0, 0),

# thickness=7

# )

myText = "FPS: "+str(round(fps,1))

cv2.putText(frame,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Camera", frame)

if cv2.waitKey(1) == ord('q'):

break

cv2.destroyAllWindows()

piCam.stop()

oled.fill(0)

oled.show()

In this lesson we are still using our standard Fusion AI Lab Kit cirtuit from the earlier lessons:

Fusion Hat Circuit Diagram — This is the circuit we will use moving forward in the class

We have also connected the OLED to the above circuit according to this diagram:

AI On the Edge, Raspberry Pi

AI on the Edge LESSON 42: Create Composite Images Using Masks in OpenCV and MediaPipe

August 16, 2026 admin

In this exciting project, we combine a Raspberry Pi 5, the Fusion AI Lab Kit, a Pi Camera, and a remote IP camera to generate a stunning real-time composite video. Watch as a glowing, translucent MediaPipe face mesh of my face hovers magically over live video of the Mighty River Nice scenery captured by an IP camera. The effect looks futuristic and professional — perfect for creative video effects, interactive installations, or just blowing your mind with computer vision! Using Picamera2 for high-frame-rate local capture and OpenCV with an RTSP stream from the river camera, we process everything in real time. MediaPipe’s Face Mesh detects and tracks facial landmarks, which we draw as beautiful cyan/teal contours with glowing irises. Then we create a clean mask, separate the mesh foreground from the river background, and blend them seamlessly into one composite frame. You’ll see every debugging layer live on screen too — meshLayer, mask, inverted mask, riverBG, and meshFG — so you can understand exactly how the magic happens.This tutorial is beginner-to-intermediate friendly and packed with practical OpenCV + MediaPipe techniques you can adapt for your own augmented reality projects. Whether you’re a longtime follower of the Paul McWhorter channel or new to the Fusion AI Kit, you’ll walk away inspired and ready to build your own hovering effects, overlays, or interactive displays.Grab the full code from the video description, fire up your Pi 5, and start creating jaw-dropping computer vision projects today. Drop a comment and let me know what you’d like to overlay next — another face mesh, hand tracking, or something completely different? Let’s keep pushing the limits of what we can do with affordable AI hardware!

# ====================================================================
# DISCLAIMER:
# This code is provided as-is for educational and experimental 
# purposes only. The author makes no representations or warranties of 
# any kind concerning the safety, suitability, or accuracy of this 
# code. Use at your own risk. The author assumes no liability for any 
# damages, system failures, security breaches, or network issues 
# resulting from the use or implementation of this script.
# ====================================================================
 
import cv2
import time
from picamera2 import Picamera2
import mediapipe as mp
import numpy as np
import secret
W=1280
H=720
tStart = time.time()
fps = 0

piCam = Picamera2(1)
piCam.preview_configuration.main.size = (W, H)
piCam.preview_configuration.main.format = "RGB888"
piCam.preview_configuration.controls.FrameRate = 60
piCam.preview_configuration.align()
piCam.configure("preview")
piCam.start()

cam = cv2.VideoCapture(secret.RTSP1,cv2.CAP_FFMPEG)
cam.set(cv2.CAP_PROP_BUFFERSIZE,0)
cam.set(cv2.CAP_PROP_POS_FRAMES,0)

textLowerLeft = (int(W*.01),int(H*.05))
fontFace = cv2.FONT_HERSHEY_SIMPLEX
fontThickness = int(W/425)
fontScale = H*.0015
fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(
    
    max_num_faces=1,
    refine_landmarks=True,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5)

cv2.namedWindow('Composite',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('Composite',W,H)
cv2.moveWindow('Composite',0,65)

cv2.namedWindow('meshLayer',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('meshLayer',320,180)
cv2.moveWindow('meshLayer',0,H+65)

cv2.namedWindow('grayLayer',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('grayLayer',320,180)
cv2.moveWindow('grayLayer',320,H+65)

cv2.namedWindow('maskLayer',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('maskLayer',320,180)
cv2.moveWindow('maskLayer',320*2,H+65)

cv2.namedWindow('invLayer',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('invLayer',320,180)
cv2.moveWindow('invLayer',320*3,H+65)

cv2.namedWindow('riverBG',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('riverBG',320,180)
cv2.moveWindow('riverBG',W,65)

cv2.namedWindow('meshFG',cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow('meshFG',320,180)
cv2.moveWindow('meshFG',W,180+65+20)
while True:
    deltaT = time.time() - tStart
    tStart=time.time()
    fps = fps*.95 + (1/deltaT)*.05
    frame = piCam.capture_array()
    frame = cv2.flip(frame, -1)
    
    _, river = cam.read()
    river = cv2.resize(river,(W,H))
    meshLayer = frame.copy()
    meshLayer[0:H-1,0:W-1] = [255,255,255]
    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = faceMesh.process(rgb)

    if results.multi_face_landmarks:
        for faceLandmarks in results.multi_face_landmarks:
            # Optional second pass for glow effect
            mp.solutions.drawing_utils.draw_landmarks(
                image=meshLayer,
                landmark_list=faceLandmarks,
                connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,
                landmark_drawing_spec=None,
                connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
                    color=(0, 150, 200),
                    thickness=2
                )
            )
            # === Best looking translucent mesh ===
            mp.solutions.drawing_utils.draw_landmarks(
                image=meshLayer,
                landmark_list=faceLandmarks,
                connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,
                landmark_drawing_spec=None,                    # Hide dots
                connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
                    color=(0, 0, 180),      # Bright cyan/teal
                    thickness=7
                )
            )

            # Optional: Extra detail on irises (makes eyes look better)
            mp.solutions.drawing_utils.draw_landmarks(
                image=meshLayer,
                landmark_list=faceLandmarks,
                connections=mp.solutions.face_mesh.FACEMESH_IRISES,
                landmark_drawing_spec=None,
                connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(
                    color=(255, 0, 0),
                    thickness=7
                )
            )
    gray = cv2.cvtColor(meshLayer,cv2.COLOR_BGR2GRAY)

    _ , mask =cv2.threshold(gray, 250, 255, cv2.THRESH_BINARY)
    invLayer = cv2.bitwise_not(mask)
    riverBG =cv2.bitwise_and(river, river, mask = mask)
    meshFG = cv2.bitwise_and(meshLayer,meshLayer, mask=invLayer)
    composite = cv2.add(riverBG,meshFG)
    
    myText = "FPS: "+str(round(fps,1))
    cv2.putText(composite,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

    cv2.imshow("Composite", composite)
    cv2.imshow("meshLayer",meshLayer)
    cv2.imshow("grayLayer",gray)
    cv2.imshow("maskLayer",mask)
    cv2.imshow("invLayer",invLayer)
    cv2.imshow("riverBG",riverBG)
    cv2.imshow("meshFG",meshFG)
    if cv2.waitKey(1) == ord('q'):
        break

cv2.destroyAllWindows()
piCam.stop()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

# ====================================================================

# DISCLAIMER:

# This code is provided as-is for educational and experimental

# purposes only. The author makes no representations or warranties of

# any kind concerning the safety, suitability, or accuracy of this

# code. Use at your own risk. The author assumes no liability for any

# damages, system failures, security breaches, or network issues

# resulting from the use or implementation of this script.

# ====================================================================

import cv2

import time

from picamera2 import Picamera2

import mediapipe as mp

import numpy as np

import secret

W=1280

H=720

tStart = time.time()

fps = 0

piCam = Picamera2(1)

piCam.preview_configuration.main.size = (W, H)

piCam.preview_configuration.main.format = "RGB888"

piCam.preview_configuration.controls.FrameRate = 60

piCam.preview_configuration.align()

piCam.configure("preview")

piCam.start()

cam = cv2.VideoCapture(secret.RTSP1,cv2.CAP_FFMPEG)

cam.set(cv2.CAP_PROP_BUFFERSIZE,0)

cam.set(cv2.CAP_PROP_POS_FRAMES,0)

textLowerLeft = (int(W*.01),int(H*.05))

fontFace = cv2.FONT_HERSHEY_SIMPLEX

fontThickness = int(W/425)

fontScale = H*.0015

fontColor = (0,0,255)

faceMesh = mp.solutions.face_mesh.FaceMesh(

max_num_faces=1,

refine_landmarks=True,

min_detection_confidence=0.5,

min_tracking_confidence=0.5)

cv2.namedWindow('Composite',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('Composite',W,H)

cv2.moveWindow('Composite',0,65)

cv2.namedWindow('meshLayer',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('meshLayer',320,180)

cv2.moveWindow('meshLayer',0,H+65)

cv2.namedWindow('grayLayer',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('grayLayer',320,180)

cv2.moveWindow('grayLayer',320,H+65)

cv2.namedWindow('maskLayer',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('maskLayer',320,180)

cv2.moveWindow('maskLayer',320*2,H+65)

cv2.namedWindow('invLayer',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('invLayer',320,180)

cv2.moveWindow('invLayer',320*3,H+65)

cv2.namedWindow('riverBG',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('riverBG',320,180)

cv2.moveWindow('riverBG',W,65)

cv2.namedWindow('meshFG',cv2.WINDOW_GUI_NORMAL)

cv2.resizeWindow('meshFG',320,180)

cv2.moveWindow('meshFG',W,180+65+20)

while True:

deltaT = time.time() - tStart

tStart=time.time()

fps = fps*.95 + (1/deltaT)*.05

frame = piCam.capture_array()

frame = cv2.flip(frame, -1)

_, river = cam.read()

river = cv2.resize(river,(W,H))

meshLayer = frame.copy()

meshLayer[0:H-1,0:W-1] = [255,255,255]

rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

results = faceMesh.process(rgb)

if results.multi_face_landmarks:

for faceLandmarks in results.multi_face_landmarks:

# Optional second pass for glow effect

mp.solutions.drawing_utils.draw_landmarks(

image=meshLayer,

landmark_list=faceLandmarks,

connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,

landmark_drawing_spec=None,

connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

color=(0, 150, 200),

thickness=2

)

# === Best looking translucent mesh ===

mp.solutions.drawing_utils.draw_landmarks(

image=meshLayer,

landmark_list=faceLandmarks,

connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,

landmark_drawing_spec=None, # Hide dots

connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

color=(0, 0, 180), # Bright cyan/teal

thickness=7

)

# Optional: Extra detail on irises (makes eyes look better)

mp.solutions.drawing_utils.draw_landmarks(

image=meshLayer,

landmark_list=faceLandmarks,

connections=mp.solutions.face_mesh.FACEMESH_IRISES,

landmark_drawing_spec=None,

connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(

color=(255, 0, 0),

thickness=7

)

gray = cv2.cvtColor(meshLayer,cv2.COLOR_BGR2GRAY)

_ , mask =cv2.threshold(gray, 250, 255, cv2.THRESH_BINARY)

invLayer = cv2.bitwise_not(mask)

riverBG =cv2.bitwise_and(river, river, mask = mask)

meshFG = cv2.bitwise_and(meshLayer,meshLayer, mask=invLayer)

composite = cv2.add(riverBG,meshFG)

myText = "FPS: "+str(round(fps,1))

cv2.putText(composite,myText,textLowerLeft,fontFace,fontScale,fontColor,fontThickness)

cv2.imshow("Composite", composite)

cv2.imshow("meshLayer",meshLayer)

cv2.imshow("grayLayer",gray)

cv2.imshow("maskLayer",mask)

cv2.imshow("invLayer",invLayer)

cv2.imshow("riverBG",riverBG)

cv2.imshow("meshFG",meshFG)

if cv2.waitKey(1) == ord('q'):

break

cv2.destroyAllWindows()

piCam.stop()

Artificial Intelligence, OpenCV, Python

Parsing Mediapipe Data for Face Mesh, Pose, and Hand Landmarks

December 9, 2021 admin

We all love the incredibly powerful features of the Mediapipe AI library. But, lets face it, the data is very hard to interpret, and the returned data structures are complex, and poorly documented. The purpose of this lesson is to show you how to parse the complex data structures coming from Mediapipe, and to create simple, intuitive arrays for the landmark data. We create classes in python which do the parsing, and the classes are easy to work with. With these classes, you can use mediapipe simply, and get simple to understand data structures back. The video explains what we are doing and how we are doing it. For your convenience the resulting code is posted below.

import cv2
print(cv2.__version__)

class mpFaceMesh:
    import mediapipe as mp
    def __init__(self,still=False,numFaces=3,tol1=.5,tol2=.5,drawMesh=True):
        self.myFaceMesh=self.mp.solutions.face_mesh.FaceMesh(still,numFaces,tol1,tol2)
        self.myDraw=self.mp.solutions.drawing_utils
        self.draw=drawMesh
    def Marks(self,frame):
        global width
        global height
        drawSpecCircle=self.myDraw.DrawingSpec(thickness=0,circle_radius=0,color=(0,0,255))
        drawSpecLine=self.myDraw.DrawingSpec(thickness=3,circle_radius=2,color=(255,0,0))
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.myFaceMesh.process(frameRGB)
        facesMeshLandmarks=[]
        if results.multi_face_landmarks !=None:
            for faceMesh in results.multi_face_landmarks:
                faceMeshLandmarks=[]
                for lm in faceMesh.landmark:
                    loc=(int(lm.x*width),int(lm.y*height))
                    faceMeshLandmarks.append(loc)
                facesMeshLandmarks.append(faceMeshLandmarks)
                if self.draw==True:
                    self.myDraw.draw_landmarks(frame,faceMesh,self.mp.solutions.face_mesh.FACE_CONNECTIONS,drawSpecCircle,drawSpecLine)
        return facesMeshLandmarks

class mpFace:
    import mediapipe as mp 
    def __init__(self):
        self.myFace=self.mp.solutions.face_detection.FaceDetection()
    def Marks(self,frame):
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.myFace.process(frameRGB)
        faceBoundBoxs=[]
        if results.detections != None:
            for face in results.detections:
                bBox=face.location_data.relative_bounding_box
                topLeft=(int(bBox.xmin*width),int(bBox.ymin*height))
                bottomRight=(int((bBox.xmin+bBox.width)*width),int((bBox.ymin+bBox.height)*height))
                faceBoundBoxs.append((topLeft,bottomRight))
        return faceBoundBoxs

class mpPose:
    import mediapipe as mp
    def __init__(self,still=False,upperBody=False, smoothData=True, tol1=.5, tol2=.5):
        self.myPose=self.mp.solutions.pose.Pose(still,upperBody,smoothData,tol1,tol2)
    def Marks(self,frame):
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.myPose.process(frameRGB)
        poseLandmarks=[]
        if results.pose_landmarks:
            for lm in results.pose_landmarks.landmark:            
                poseLandmarks.append((int(lm.x*width),int(lm.y*height)))
        return poseLandmarks

class mpHands:
    import mediapipe as mp
    def __init__(self,maxHands=2,tol1=.5,tol2=.5):
        self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)
    def Marks(self,frame):
        myHands=[]
        handsType=[]
        frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        results=self.hands.process(frameRGB)
        if results.multi_hand_landmarks != None:
            #print(results.multi_handedness)
            for hand in results.multi_handedness:
                #print(hand)
                #print(hand.classification)
                #print(hand.classification[0])
                handType=hand.classification[0].label
                handsType.append(handType)
            for handLandMarks in results.multi_hand_landmarks:
                myHand=[]
                for landMark in handLandMarks.landmark:
                    myHand.append((int(landMark.x*width),int(landMark.y*height)))
                myHands.append(myHand)
        return myHands,handsType

width=1280
height=720
cam=cv2.VideoCapture(3,cv2.CAP_DSHOW)
cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)
cam.set(cv2.CAP_PROP_FPS, 30)
cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(2)
findFace=mpFace()
findPose=mpPose()
findMesh=mpFaceMesh(drawMesh=True)

font=cv2.FONT_HERSHEY_SIMPLEX
fontColor=(0,0,255)
fontSize=.1
fontThick=1

cv2.namedWindow('Trackbars')
cv2.moveWindow('Trackbars',width+50,0)
cv2.resizeWindow('Trackbars',400,150)


while True:
    ignore,  frame = cam.read()
    frame=cv2.resize(frame,(width,height))
    handsLM,handsType=findHands.Marks(frame)
    faceLoc=findFace.Marks(frame)
    poseLM=findPose.Marks(frame)
    facesMeshLM=findMesh.Marks(frame)
    if poseLM != []:
        for ind in [13,14,15,16]:
            cv2.circle(frame,poseLM[ind],20,(0,255,0),-1)

    for face in faceLoc:
        cv2.rectangle(frame,face[0],face[1],(255,0,0),3)
    for hand,handType in zip(handsLM,handsType):
        if handType=='Right':
            lbl='Right'
        if handType=='Left':
            lbl='Left'
        cv2.putText(frame,lbl,hand[8],font,2,fontColor,2)
    for faceMeshLM in facesMeshLM:
        cnt=0
        for lm in faceMeshLM:
            cv2.putText(frame,str(cnt),lm,font,fontSize,fontColor,fontThick)
            cnt=cnt+1

    cv2.imshow('my WEBcam', frame)
    cv2.moveWindow('my WEBcam',0,0)
    if cv2.waitKey(1) & 0xff ==ord('q'):
        break
cam.release()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

import cv2

print(cv2.__version__)

class mpFaceMesh:

import mediapipe as mp

def __init__(self,still=False,numFaces=3,tol1=.5,tol2=.5,drawMesh=True):

self.myFaceMesh=self.mp.solutions.face_mesh.FaceMesh(still,numFaces,tol1,tol2)

self.myDraw=self.mp.solutions.drawing_utils

self.draw=drawMesh

def Marks(self,frame):

global width

global height

drawSpecCircle=self.myDraw.DrawingSpec(thickness=0,circle_radius=0,color=(0,0,255))

drawSpecLine=self.myDraw.DrawingSpec(thickness=3,circle_radius=2,color=(255,0,0))

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.myFaceMesh.process(frameRGB)

facesMeshLandmarks=[]

if results.multi_face_landmarks !=None:

for faceMesh in results.multi_face_landmarks:

faceMeshLandmarks=[]

for lm in faceMesh.landmark:

loc=(int(lm.x*width),int(lm.y*height))

faceMeshLandmarks.append(loc)

facesMeshLandmarks.append(faceMeshLandmarks)

if self.draw==True:

self.myDraw.draw_landmarks(frame,faceMesh,self.mp.solutions.face_mesh.FACE_CONNECTIONS,drawSpecCircle,drawSpecLine)

return facesMeshLandmarks

class mpFace:

import mediapipe as mp

def __init__(self):

self.myFace=self.mp.solutions.face_detection.FaceDetection()

def Marks(self,frame):

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.myFace.process(frameRGB)

faceBoundBoxs=[]

if results.detections != None:

for face in results.detections:

bBox=face.location_data.relative_bounding_box

topLeft=(int(bBox.xmin*width),int(bBox.ymin*height))

bottomRight=(int((bBox.xmin+bBox.width)*width),int((bBox.ymin+bBox.height)*height))

faceBoundBoxs.append((topLeft,bottomRight))

return faceBoundBoxs

class mpPose:

import mediapipe as mp

def __init__(self,still=False,upperBody=False, smoothData=True, tol1=.5, tol2=.5):

self.myPose=self.mp.solutions.pose.Pose(still,upperBody,smoothData,tol1,tol2)

def Marks(self,frame):

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.myPose.process(frameRGB)

poseLandmarks=[]

if results.pose_landmarks:

for lm in results.pose_landmarks.landmark:

poseLandmarks.append((int(lm.x*width),int(lm.y*height)))

return poseLandmarks

class mpHands:

import mediapipe as mp

def __init__(self,maxHands=2,tol1=.5,tol2=.5):

self.hands=self.mp.solutions.hands.Hands(False,maxHands,tol1,tol2)

def Marks(self,frame):

myHands=[]

handsType=[]

frameRGB=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)

results=self.hands.process(frameRGB)

if results.multi_hand_landmarks != None:

#print(results.multi_handedness)

for hand in results.multi_handedness:

#print(hand)

#print(hand.classification)

#print(hand.classification[0])

handType=hand.classification[0].label

handsType.append(handType)

for handLandMarks in results.multi_hand_landmarks:

myHand=[]

for landMark in handLandMarks.landmark:

myHand.append((int(landMark.x*width),int(landMark.y*height)))

myHands.append(myHand)

return myHands,handsType

width=1280

height=720

cam=cv2.VideoCapture(3,cv2.CAP_DSHOW)

cam.set(cv2.CAP_PROP_FRAME_WIDTH, width)

cam.set(cv2.CAP_PROP_FRAME_HEIGHT,height)

cam.set(cv2.CAP_PROP_FPS, 30)

cam.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*'MJPG'))

findHands=mpHands(2)

findFace=mpFace()

findPose=mpPose()

findMesh=mpFaceMesh(drawMesh=True)

font=cv2.FONT_HERSHEY_SIMPLEX

fontColor=(0,0,255)

fontSize=.1

fontThick=1

cv2.namedWindow('Trackbars')

cv2.moveWindow('Trackbars',width+50,0)

cv2.resizeWindow('Trackbars',400,150)

while True:

ignore, frame = cam.read()

frame=cv2.resize(frame,(width,height))

handsLM,handsType=findHands.Marks(frame)

faceLoc=findFace.Marks(frame)

poseLM=findPose.Marks(frame)

facesMeshLM=findMesh.Marks(frame)

if poseLM != []:

for ind in [13,14,15,16]:

cv2.circle(frame,poseLM[ind],20,(0,255,0),-1)

for face in faceLoc:

cv2.rectangle(frame,face[0],face[1],(255,0,0),3)

for hand,handType in zip(handsLM,handsType):

if handType=='Right':

lbl='Right'

if handType=='Left':

lbl='Left'

cv2.putText(frame,lbl,hand[8],font,2,fontColor,2)

for faceMeshLM in facesMeshLM:

cnt=0

for lm in faceMeshLM:

cv2.putText(frame,str(cnt),lm,font,fontSize,fontColor,fontThick)

cnt=cnt+1

cv2.imshow('my WEBcam', frame)

cv2.moveWindow('my WEBcam',0,0)

if cv2.waitKey(1) & 0xff ==ord('q'):

break

cam.release()

Technology Tutorials

Tag Archives: Face Mesh

AI on the Edge LESSON 44: Displaying Live MediaPipe FaceMesh Avatar on SSD1306 OLED in OpenCV

AI on the Edge LESSON 42: Create Composite Images Using Masks in OpenCV and MediaPipe

Parsing Mediapipe Data for Face Mesh, Pose, and Hand Landmarks

Making The World a Better Place One High Tech Project at a Time. Enjoy!