Tag Archives: Artificial intelligence

In this video lesson we show how two Raspberry Pi cameras can independently track to different objects of interest. As a demonstration, we track on two different colors, with pan/tilt servo systems adjusting to keep the object of interest in the center of the field of view.

In this project we are using the Jetson Xavier NX, which you can pick up HERE. You will also need to of the bracket/servo kits, which you can get HERE, and then two Raspberry Pi Version two cameras, available HERE.

import cv2
import numpy as np
import time
from adafruit_servokit import ServoKit
print(cv2.__version__)
timeMark=time.time()
dtFIL=0

def nothing(x):
    pass

cv2.namedWindow('TrackBars')
cv2.moveWindow('TrackBars',1320,0)
cv2.createTrackbar('hueLower', 'TrackBars',100,179,nothing)
cv2.createTrackbar('hueUpper', 'TrackBars',116,179,nothing)
cv2.createTrackbar('satLow', 'TrackBars',160,255,nothing)
cv2.createTrackbar('satHigh', 'TrackBars',255,255,nothing)
cv2.createTrackbar('valLow', 'TrackBars',150,255,nothing)
cv2.createTrackbar('valHigh', 'TrackBars',255,255,nothing)

cv2.namedWindow('TrackBars2')
cv2.moveWindow('TrackBars2',1100,0)
cv2.createTrackbar('hueLower2', 'TrackBars2',150,179,nothing)
cv2.createTrackbar('hueUpper2', 'TrackBars2',170,179,nothing)
cv2.createTrackbar('satLow2', 'TrackBars2',160,255,nothing)
cv2.createTrackbar('satHigh2', 'TrackBars2',255,255,nothing)
cv2.createTrackbar('valLow2', 'TrackBars2',150,255,nothing)
cv2.createTrackbar('valHigh2', 'TrackBars2',255,255,nothing)


kit=ServoKit(channels=16)

tilt1=90
pan1=90
tilt2=90
pan2=90

kit.servo[0].angle=pan1
kit.servo[1].angle=tilt1
kit.servo[2].angle=pan2
kit.servo[3].angle=tilt2

width=720
height=480
flip=2
font=cv2.FONT_HERSHEY_SIMPLEX
camSet1='nvarguscamerasrc sensor-id=0 ee-mode=1 ee-strength=0 tnr-mode=2 tnr-strength=1 wbmode=3 ! video/x-raw(memory:NVMM), width=3264, height=2464, framerate=21/1,format=NV12 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(width)+', height='+str(height)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! videobalance contrast=1.3 brightness=-.2 saturation=1.2 ! appsink drop=True'
camSet2='nvarguscamerasrc sensor-id=1 ee-mode=1 ee-strength=0 tnr-mode=2 tnr-strength=1 wbmode=3 ! video/x-raw(memory:NVMM), width=3264, height=2464, framerate=21/1,format=NV12 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(width)+', height='+str(height)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! videobalance contrast=1.3 brightness=-.2 saturation=1.2 ! appsink drop=True'

#camSet='nvarguscamerasrc sensor-id=0 ! video/x-raw(memory:NVMM), width=3264, height=2464, framerate=21/1,format=NV12 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(width)+', height='+str(height)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! appsink'
#camSet ='v4l2src device=/dev/video1 ! video/x-raw,width='+str(width)+',height='+str(height)+',framerate=20/1 ! videoconvert ! appsink'
cam1=cv2.VideoCapture(camSet1)
cam2=cv2.VideoCapture(camSet2)
while True:
    _, frame1 = cam1.read()
    _, frame2 = cam2.read()

    hsv1=cv2.cvtColor(frame1,cv2.COLOR_BGR2HSV)
    hsv2=cv2.cvtColor(frame2,cv2.COLOR_BGR2HSV)

    hueLow=cv2.getTrackbarPos('hueLower', 'TrackBars')
    hueUp=cv2.getTrackbarPos('hueUpper', 'TrackBars')

    Ls=cv2.getTrackbarPos('satLow', 'TrackBars')
    Us=cv2.getTrackbarPos('satHigh', 'TrackBars')

    Lv=cv2.getTrackbarPos('valLow', 'TrackBars')
    Uv=cv2.getTrackbarPos('valHigh', 'TrackBars')

    l_b=np.array([hueLow,Ls,Lv])
    u_b=np.array([hueUp,Us,Uv])


    hueLow2=cv2.getTrackbarPos('hueLower2', 'TrackBars2')
    hueUp2=cv2.getTrackbarPos('hueUpper2', 'TrackBars2')

    Ls2=cv2.getTrackbarPos('satLow2', 'TrackBars2')
    Us2=cv2.getTrackbarPos('satHigh2', 'TrackBars2')

    Lv2=cv2.getTrackbarPos('valLow2', 'TrackBars2')
    Uv2=cv2.getTrackbarPos('valHigh2', 'TrackBars2')

    l_b2=np.array([hueLow2,Ls2,Lv2])
    u_b2=np.array([hueUp2,Us2,Uv2])

    FGmask1=cv2.inRange(hsv1,l_b,u_b)
    FGmask2=cv2.inRange(hsv2,l_b2,u_b2)

    cv2.imshow('FGmask1',FGmask1)
    cv2.moveWindow('FGmask1',0,0)

    cv2.imshow('FGmask2',FGmask2)
    cv2.moveWindow('FGmask2',350,0)

    contours1,_ = cv2.findContours(FGmask1,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
    contours1=sorted(contours1,key=lambda x:cv2.contourArea(x),reverse=True)
    for cnt in contours1:
        area=cv2.contourArea(cnt)
        (x,y,w,h)=cv2.boundingRect(cnt)
        if area>=100:
            cv2.rectangle(frame1,(x,y),(x+w,y+h),(0,255,255),3)
            objX=x+w/2
            objY=y+h/2
            errorPan1=objX-width/2
            errorTilt1=objY-height/2
            if abs(errorPan1)>15:
                pan1=pan1+errorPan1/40
            if abs(errorTilt1)>15:
                tilt1=tilt1-errorTilt1/40
            if pan1>180:
                pan1=180
                print('Pan Out of Range')
            if pan1<0:
                pan1=0
                print('Pan Out of Range')
            if tilt1>180:
                tilt1=180
                print('Tilt Out of Range')
            if tilt1<0:
                tilt1=0
            kit.servo[2].angle=pan1
            kit.servo[3].angle=tilt1
            break
    
    contours2,_ = cv2.findContours(FGmask2,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
    contours2=sorted(contours2,key=lambda x:cv2.contourArea(x),reverse=True)
    for cnt in contours2:
        area=cv2.contourArea(cnt)
        (x,y,w,h)=cv2.boundingRect(cnt)
        if area>=100:
            cv2.rectangle(frame2,(x,y),(x+w,y+h),(0,255,255),3)
            objX=x+w/2
            objY=y+h/2
            errorPan2=objX-width/2
            errorTilt2=objY-height/2
            if abs(errorPan2)>15:
                pan2=pan2+errorPan2/40
            if abs(errorTilt2)>15:
                tilt2=tilt2-errorTilt2/40
            if pan2>180:
                pan2=180
                print('Pan Out of Range')
            if pan2<0:
                pan2=0
                print('Pan Out of Range')
            if tilt2>180:
                tilt2=180
                print('Tilt Out of Range')
            if tilt2<0:
                tilt2=0
            kit.servo[0].angle=pan2
            kit.servo[1].angle=tilt2            
            break

    frame3=np.hstack((frame1,frame2))
    dt=time.time()-timeMark
    timeMark=time.time()
    dtFIL=.9*dtFIL + .1*dt
    fps=1/dtFIL
    cv2.rectangle(frame3,(0,0),(150,40),(0,0,255),-1)
    cv2.putText(frame3,'fps: '+str(round(fps,1)),(0,30),font,1,(0,255,255),2)

    #cv2.imshow('myCam1',frame1)
    #cv2.imshow('myCam2',frame2)
    cv2.imshow('comboCam',frame3)
    cv2.moveWindow('comboCam',0,450)

    if cv2.waitKey(1)==ord('q'):
        break
cam1.release()
cam2.release()
cv2.destroyAllWindows()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

import cv2

import numpy as np

import time

from adafruit_servokit import ServoKit

print(cv2.__version__)

timeMark=time.time()

dtFIL=0

def nothing(x):

pass

cv2.namedWindow('TrackBars')

cv2.moveWindow('TrackBars',1320,0)

cv2.createTrackbar('hueLower', 'TrackBars',100,179,nothing)

cv2.createTrackbar('hueUpper', 'TrackBars',116,179,nothing)

cv2.createTrackbar('satLow', 'TrackBars',160,255,nothing)

cv2.createTrackbar('satHigh', 'TrackBars',255,255,nothing)

cv2.createTrackbar('valLow', 'TrackBars',150,255,nothing)

cv2.createTrackbar('valHigh', 'TrackBars',255,255,nothing)

cv2.namedWindow('TrackBars2')

cv2.moveWindow('TrackBars2',1100,0)

cv2.createTrackbar('hueLower2', 'TrackBars2',150,179,nothing)

cv2.createTrackbar('hueUpper2', 'TrackBars2',170,179,nothing)

cv2.createTrackbar('satLow2', 'TrackBars2',160,255,nothing)

cv2.createTrackbar('satHigh2', 'TrackBars2',255,255,nothing)

cv2.createTrackbar('valLow2', 'TrackBars2',150,255,nothing)

cv2.createTrackbar('valHigh2', 'TrackBars2',255,255,nothing)

kit=ServoKit(channels=16)

tilt1=90

pan1=90

tilt2=90

pan2=90

kit.servo[0].angle=pan1

kit.servo[1].angle=tilt1

kit.servo[2].angle=pan2

kit.servo[3].angle=tilt2

width=720

height=480

flip=2

font=cv2.FONT_HERSHEY_SIMPLEX

camSet1='nvarguscamerasrc sensor-id=0 ee-mode=1 ee-strength=0 tnr-mode=2 tnr-strength=1 wbmode=3 ! video/x-raw(memory:NVMM), width=3264, height=2464, framerate=21/1,format=NV12 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(width)+', height='+str(height)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! videobalance contrast=1.3 brightness=-.2 saturation=1.2 ! appsink drop=True'

camSet2='nvarguscamerasrc sensor-id=1 ee-mode=1 ee-strength=0 tnr-mode=2 tnr-strength=1 wbmode=3 ! video/x-raw(memory:NVMM), width=3264, height=2464, framerate=21/1,format=NV12 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(width)+', height='+str(height)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! videobalance contrast=1.3 brightness=-.2 saturation=1.2 ! appsink drop=True'

#camSet='nvarguscamerasrc sensor-id=0 ! video/x-raw(memory:NVMM), width=3264, height=2464, framerate=21/1,format=NV12 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(width)+', height='+str(height)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! appsink'

#camSet ='v4l2src device=/dev/video1 ! video/x-raw,width='+str(width)+',height='+str(height)+',framerate=20/1 ! videoconvert ! appsink'

cam1=cv2.VideoCapture(camSet1)

cam2=cv2.VideoCapture(camSet2)

while True:

_, frame1 = cam1.read()

_, frame2 = cam2.read()

hsv1=cv2.cvtColor(frame1,cv2.COLOR_BGR2HSV)

hsv2=cv2.cvtColor(frame2,cv2.COLOR_BGR2HSV)

hueLow=cv2.getTrackbarPos('hueLower', 'TrackBars')

hueUp=cv2.getTrackbarPos('hueUpper', 'TrackBars')

Ls=cv2.getTrackbarPos('satLow', 'TrackBars')

Us=cv2.getTrackbarPos('satHigh', 'TrackBars')

Lv=cv2.getTrackbarPos('valLow', 'TrackBars')

Uv=cv2.getTrackbarPos('valHigh', 'TrackBars')

l_b=np.array([hueLow,Ls,Lv])

u_b=np.array([hueUp,Us,Uv])

hueLow2=cv2.getTrackbarPos('hueLower2', 'TrackBars2')

hueUp2=cv2.getTrackbarPos('hueUpper2', 'TrackBars2')

Ls2=cv2.getTrackbarPos('satLow2', 'TrackBars2')

Us2=cv2.getTrackbarPos('satHigh2', 'TrackBars2')

Lv2=cv2.getTrackbarPos('valLow2', 'TrackBars2')

Uv2=cv2.getTrackbarPos('valHigh2', 'TrackBars2')

l_b2=np.array([hueLow2,Ls2,Lv2])

u_b2=np.array([hueUp2,Us2,Uv2])

FGmask1=cv2.inRange(hsv1,l_b,u_b)

FGmask2=cv2.inRange(hsv2,l_b2,u_b2)

cv2.imshow('FGmask1',FGmask1)

cv2.moveWindow('FGmask1',0,0)

cv2.imshow('FGmask2',FGmask2)

cv2.moveWindow('FGmask2',350,0)

contours1,_ = cv2.findContours(FGmask1,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)

contours1=sorted(contours1,key=lambda x:cv2.contourArea(x),reverse=True)

for cnt in contours1:

area=cv2.contourArea(cnt)

(x,y,w,h)=cv2.boundingRect(cnt)

if area>=100:

cv2.rectangle(frame1,(x,y),(x+w,y+h),(0,255,255),3)

objX=x+w/2

objY=y+h/2

errorPan1=objX-width/2

errorTilt1=objY-height/2

if abs(errorPan1)>15:

pan1=pan1+errorPan1/40

if abs(errorTilt1)>15:

tilt1=tilt1-errorTilt1/40

if pan1>180:

pan1=180

print('Pan Out of Range')

if pan1<0:

pan1=0

print('Pan Out of Range')

if tilt1>180:

tilt1=180

print('Tilt Out of Range')

if tilt1<0:

tilt1=0

kit.servo[2].angle=pan1

kit.servo[3].angle=tilt1

break

contours2,_ = cv2.findContours(FGmask2,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)

contours2=sorted(contours2,key=lambda x:cv2.contourArea(x),reverse=True)

for cnt in contours2:

area=cv2.contourArea(cnt)

(x,y,w,h)=cv2.boundingRect(cnt)

if area>=100:

cv2.rectangle(frame2,(x,y),(x+w,y+h),(0,255,255),3)

objX=x+w/2

objY=y+h/2

errorPan2=objX-width/2

errorTilt2=objY-height/2

if abs(errorPan2)>15:

pan2=pan2+errorPan2/40

if abs(errorTilt2)>15:

tilt2=tilt2-errorTilt2/40

if pan2>180:

pan2=180

print('Pan Out of Range')

if pan2<0:

pan2=0

print('Pan Out of Range')

if tilt2>180:

tilt2=180

print('Tilt Out of Range')

if tilt2<0:

tilt2=0

kit.servo[0].angle=pan2

kit.servo[1].angle=tilt2

break

frame3=np.hstack((frame1,frame2))

dt=time.time()-timeMark

timeMark=time.time()

dtFIL=.9*dtFIL + .1*dt

fps=1/dtFIL

cv2.rectangle(frame3,(0,0),(150,40),(0,0,255),-1)

cv2.putText(frame3,'fps: '+str(round(fps,1)),(0,30),font,1,(0,255,255),2)

#cv2.imshow('myCam1',frame1)

#cv2.imshow('myCam2',frame2)

cv2.imshow('comboCam',frame3)

cv2.moveWindow('comboCam',0,450)

if cv2.waitKey(1)==ord('q'):

break

cam1.release()

cam2.release()

cv2.destroyAllWindows()

Jetson Nano

AI on the Jetson Nano LESSON 52: Improving Picture Quality of the Raspberry Pi Camera with Gstreamer

July 25, 2020 admin

In this lesson we want to pause and work on improving the image quality of the video stream coming from the Raspberry Pi camera. Right now, we are using a boilerplate Gstreamer string to launch the Raspberry Pi camera. In the video above we show how image quality can be drastically improved by tweaking the Gstreamer launch string.

Based on the Video above, we develop a greatly improved image quality by adjusting the Gstreamer launch string. Below, for you enjoyment is the code that will optimize picture quality.

First, this is the key line that results in excellent video quality:

# Gstreamer code for improvded Raspberry Pi Camera Quality
camSet='nvarguscamerasrc wbmode=3 tnr-mode=2 tnr-strength=1 ee-mode=2 ee-strength=1 ! video/x-raw(memory:NVMM), width=3264, height=2464, format=NV12, framerate=21/1 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(dispW)+', height='+str(dispH)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! videobalance contrast=1.5 brightness=-.2 saturation=1.2 ! appsink'

# Gstreamer code for improvded Raspberry Pi Camera Quality

camSet='nvarguscamerasrc wbmode=3 tnr-mode=2 tnr-strength=1 ee-mode=2 ee-strength=1 ! video/x-raw(memory:NVMM), width=3264, height=2464, format=NV12, framerate=21/1 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(dispW)+', height='+str(dispH)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! videobalance contrast=1.5 brightness=-.2 saturation=1.2 ! appsink'

And here is the overall code for running and displaying from the camera with the enhanced quality:

import cv2
print(cv2.__version__)
dispW=1280
dispH=720
flip=2
#Uncomment These next Two Line for Pi Camera
camSet='nvarguscamerasrc wbmode=3 tnr-mode=2 tnr-strength=1 ee-mode=2 ee-strength=1 !  video/x-raw(memory:NVMM), width=3264, height=2464, format=NV12, framerate=21/1 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(dispW)+', height='+str(dispH)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! videobalance contrast=1.5 brightness=-.2 saturation=1.2 !  appsink'
cam= cv2.VideoCapture(camSet)

#Or, if you have a WEB cam, uncomment the next line
#cam=cv2.VideoCapture('/dev/video1')
while True:
    ret, frame = cam.read()
    cv2.imshow('nanoCam',frame)
    cv2.moveWindow('nanoCam',0,0)
    if cv2.waitKey(1)==ord('q'):
        break
cam.release()
cv2.destroyAllWindows()

import cv2

print(cv2.__version__)

dispW=1280

dispH=720

flip=2

#Uncomment These next Two Line for Pi Camera

cam= cv2.VideoCapture(camSet)

#Or, if you have a WEB cam, uncomment the next line

#cam=cv2.VideoCapture('/dev/video1')

while True:

ret, frame = cam.read()

cv2.imshow('nanoCam',frame)

cv2.moveWindow('nanoCam',0,0)

if cv2.waitKey(1)==ord('q'):

break

cam.release()

cv2.destroyAllWindows()

Now, once we have optimized the Gstreamer launch stream, we need to consider what path to move forward. In lesson #50 we saw that we could either control the camera using the NVIDIA Jetson Utilities, or we could control the camera normally from OpenCV. The advantage of our old OpenCV method is that it gives us more control of the camera. The advantage of the Jetson Utility method is that it appears to run faster, and for the rPi camera, have less latency. Below are two code examples for the two methods above. In the video lesson above, we will figure out the best strategy by tweaking the parameters in these two programas.

OPTION #1: Launch the cameras using OpenCV

import jetson.inference
import jetson.utils
import cv2
import numpy as np
import time
width=1280
height=720
dispW=width
dispH=height
flip=2
camSet='nvarguscamerasrc !  video/x-raw(memory:NVMM), width=3264, height=2464, format=NV12, framerate=21/1 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(dispW)+', height='+str(dispH)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! videobalance  contrast=1.5 brightness=-.3 saturation=1.2 ! appsink  '
cam1=cv2.VideoCapture(camSet)
#cam1=cv2.VideoCapture('/dev/video1')
#cam1.set(cv2.CAP_PROP_FRAME_WIDTH,dispW)
#cam1.set(cv2.CAP_PROP_FRAME_HEIGHT,dispH)
net=jetson.inference.imageNet('alexnet')
font=cv2.FONT_HERSHEY_SIMPLEX
timeMark=time.time()
fpsFilter=0
while True:
    _,frame=cam1.read()
    img=cv2.cvtColor(frame,cv2.COLOR_BGR2RGBA).astype(np.float32)
    img=jetson.utils.cudaFromNumpy(img)
    classID, confidence =net.Classify(img, width, height)
    item=''
    item =net.GetClassDesc(classID)
    dt=time.time()-timeMark
    fps=1/dt
    fpsFilter=.95*fpsFilter +.05*fps
    timeMark=time.time()
    cv2.putText(frame,str(round(fpsFilter,1))+' fps '+item,(0,30),font,1,(0,0,255),2)
    cv2.imshow('recCam',frame)
    cv2.moveWindow('recCam',0,0)
    if cv2.waitKey(1)==ord('q'):
        break
cam.releast()
cv2.destroyAllWindows()

import jetson.inference

import jetson.utils

import cv2

import numpy as np

import time

width=1280

height=720

dispW=width

dispH=height

flip=2

camSet='nvarguscamerasrc ! video/x-raw(memory:NVMM), width=3264, height=2464, format=NV12, framerate=21/1 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(dispW)+', height='+str(dispH)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! videobalance contrast=1.5 brightness=-.3 saturation=1.2 ! appsink '

cam1=cv2.VideoCapture(camSet)

#cam1=cv2.VideoCapture('/dev/video1')

#cam1.set(cv2.CAP_PROP_FRAME_WIDTH,dispW)

#cam1.set(cv2.CAP_PROP_FRAME_HEIGHT,dispH)

net=jetson.inference.imageNet('alexnet')

font=cv2.FONT_HERSHEY_SIMPLEX

timeMark=time.time()

fpsFilter=0

while True:

_,frame=cam1.read()

img=cv2.cvtColor(frame,cv2.COLOR_BGR2RGBA).astype(np.float32)

img=jetson.utils.cudaFromNumpy(img)

classID, confidence =net.Classify(img, width, height)

item=''

item =net.GetClassDesc(classID)

dt=time.time()-timeMark

fps=1/dt

fpsFilter=.95*fpsFilter +.05*fps

timeMark=time.time()

cv2.putText(frame,str(round(fpsFilter,1))+' fps '+item,(0,30),font,1,(0,0,255),2)

cv2.imshow('recCam',frame)

cv2.moveWindow('recCam',0,0)

if cv2.waitKey(1)==ord('q'):

break

cam.releast()

cv2.destroyAllWindows()

OPTION # 2: Control Camera with NVIDIA Jetson Utilities Library

import jetson.inference
import jetson.utils
import time
import cv2
import numpy as np
width=1280
height=720
#cam=jetson.utils.gstCamera(width,height,'/dev/video1')
cam=jetson.utils.gstCamera(width,height,'0')
net=jetson.inference.imageNet('googlenet')
timeMark=time.time()
fpsFilter=0
timeMark=time.time()
font=cv2.FONT_HERSHEY_SIMPLEX
while True:
    frame, width, height = cam.CaptureRGBA(zeroCopy=1)
    classID, confidence = net.Classify(frame, width, height)
    item = net.GetClassDesc(classID)
    dt=time.time()-timeMark
    fps=1/dt
    fpsFilter=.95*fpsFilter+.05*fps
    timeMark=time.time()
    frame=jetson.utils.cudaToNumpy(frame,width,height,4)
    frame=cv2.cvtColor(frame, cv2.COLOR_RGBA2BGR).astype(np.uint8)
    cv2.putText(frame,str(round(fpsFilter,1))+'      '+item,(0,30),font,1,(0,0,255),2)
    cv2.imshow('webCam',frame)
    cv2.moveWindow('webCam',0,0)
    if cv2.waitKey(1)==ord('q'):
        break
cam.release()
cv2.destroyAllWindows()

import jetson.inference

import jetson.utils

import time

import cv2

import numpy as np

width=1280

height=720

#cam=jetson.utils.gstCamera(width,height,'/dev/video1')

cam=jetson.utils.gstCamera(width,height,'0')

net=jetson.inference.imageNet('googlenet')

timeMark=time.time()

fpsFilter=0

timeMark=time.time()

font=cv2.FONT_HERSHEY_SIMPLEX

while True:

frame, width, height = cam.CaptureRGBA(zeroCopy=1)

classID, confidence = net.Classify(frame, width, height)

item = net.GetClassDesc(classID)

dt=time.time()-timeMark

fps=1/dt

fpsFilter=.95*fpsFilter+.05*fps

timeMark=time.time()

frame=jetson.utils.cudaToNumpy(frame,width,height,4)

frame=cv2.cvtColor(frame, cv2.COLOR_RGBA2BGR).astype(np.uint8)

cv2.putText(frame,str(round(fpsFilter,1))+' '+item,(0,30),font,1,(0,0,255),2)

cv2.imshow('webCam',frame)

cv2.moveWindow('webCam',0,0)

if cv2.waitKey(1)==ord('q'):

break

cam.release()

cv2.destroyAllWindows()

Jetson Xavier NX

Jetson Xavier NX Lesson 10: Tracking Multiple Objects of Interest with Servos in OpenCV

July 22, 2020 admin

In this lesson we learn how to use OpenCV on the Jetson Xavier NX to track an object of interest in with two cameras on two Pan/Tilt servo brackets. The system tracks based on HSV color space, but the same basic setup could be used with other object detection algorithms. In this project we are using the Jetson Xavier NX, which you can pick up HERE. You will also need to of the bracket/servo kits, which you can get HERE, and then two Raspberry Pi Version two cameras, available HERE.

import cv2
import numpy as np
import time
from adafruit_servokit import ServoKit
print(cv2.__version__)
timeMark=time.time()
dtFIL=0

def nothing(x):
    pass

cv2.namedWindow('TrackBars')
cv2.moveWindow('TrackBars',1320,0)
cv2.createTrackbar('hueLower', 'TrackBars',100,179,nothing)
cv2.createTrackbar('hueUpper', 'TrackBars',116,179,nothing)
cv2.createTrackbar('satLow', 'TrackBars',160,255,nothing)
cv2.createTrackbar('satHigh', 'TrackBars',255,255,nothing)
cv2.createTrackbar('valLow', 'TrackBars',150,255,nothing)
cv2.createTrackbar('valHigh', 'TrackBars',255,255,nothing)
kit=ServoKit(channels=16)

tilt1=90
pan1=90
tilt2=90
pan2=90

kit.servo[0].angle=pan1
kit.servo[1].angle=tilt1
kit.servo[2].angle=pan2
kit.servo[3].angle=tilt2

width=720
height=480
flip=2
font=cv2.FONT_HERSHEY_SIMPLEX
camSet1='nvarguscamerasrc sensor-id=0 ee-mode=1 ee-strength=0 tnr-mode=2 tnr-strength=1 wbmode=3 ! video/x-raw(memory:NVMM), width=3264, height=2464, framerate=21/1,format=NV12 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(width)+', height='+str(height)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! videobalance contrast=1.3 brightness=-.2 saturation=1.2 ! appsink drop=True'
camSet2='nvarguscamerasrc sensor-id=1 ee-mode=1 ee-strength=0 tnr-mode=2 tnr-strength=1 wbmode=3 ! video/x-raw(memory:NVMM), width=3264, height=2464, framerate=21/1,format=NV12 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(width)+', height='+str(height)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! videobalance contrast=1.3 brightness=-.2 saturation=1.2 ! appsink drop=True'

#camSet='nvarguscamerasrc sensor-id=0 ! video/x-raw(memory:NVMM), width=3264, height=2464, framerate=21/1,format=NV12 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(width)+', height='+str(height)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! appsink'
#camSet ='v4l2src device=/dev/video1 ! video/x-raw,width='+str(width)+',height='+str(height)+',framerate=20/1 ! videoconvert ! appsink'
cam1=cv2.VideoCapture(camSet1)
cam2=cv2.VideoCapture(camSet2)
while True:
    _, frame1 = cam1.read()
    _, frame2 = cam2.read()

    hsv1=cv2.cvtColor(frame1,cv2.COLOR_BGR2HSV)
    hsv2=cv2.cvtColor(frame2,cv2.COLOR_BGR2HSV)

    hueLow=cv2.getTrackbarPos('hueLower', 'TrackBars')
    hueUp=cv2.getTrackbarPos('hueUpper', 'TrackBars')

    Ls=cv2.getTrackbarPos('satLow', 'TrackBars')
    Us=cv2.getTrackbarPos('satHigh', 'TrackBars')

    Lv=cv2.getTrackbarPos('valLow', 'TrackBars')
    Uv=cv2.getTrackbarPos('valHigh', 'TrackBars')

    l_b=np.array([hueLow,Ls,Lv])
    u_b=np.array([hueUp,Us,Uv])

    FGmask1=cv2.inRange(hsv1,l_b,u_b)
    FGmask2=cv2.inRange(hsv2,l_b,u_b)

    cv2.imshow('FGmask1',FGmask1)
    cv2.moveWindow('FGmask1',0,0)

    contours1,_ = cv2.findContours(FGmask1,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
    contours1=sorted(contours1,key=lambda x:cv2.contourArea(x),reverse=True)
    for cnt in contours1:
        area=cv2.contourArea(cnt)
        (x,y,w,h)=cv2.boundingRect(cnt)
        if area>=100:
            cv2.rectangle(frame1,(x,y),(x+w,y+h),(0,255,255),3)
            objX=x+w/2
            objY=y+h/2
            errorPan1=objX-width/2
            errorTilt1=objY-height/2
            if abs(errorPan1)>15:
                pan1=pan1+errorPan1/40
            if abs(errorTilt1)>15:
                tilt1=tilt1-errorTilt1/40
            if pan1>180:
                pan1=180
                print('Pan Out of Range')
            if pan1<0:
                pan1=0
                print('Pan Out of Range')
            if tilt1>180:
                tilt1=180
                print('Tilt Out of Range')
            if tilt1<0:
                tilt1=0
            kit.servo[2].angle=pan1
            kit.servo[3].angle=tilt1
            break
    
    contours2,_ = cv2.findContours(FGmask2,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
    contours2=sorted(contours2,key=lambda x:cv2.contourArea(x),reverse=True)
    for cnt in contours2:
        area=cv2.contourArea(cnt)
        (x,y,w,h)=cv2.boundingRect(cnt)
        if area>=100:
            cv2.rectangle(frame2,(x,y),(x+w,y+h),(0,255,255),3)
            objX=x+w/2
            objY=y+h/2
            errorPan2=objX-width/2
            errorTilt2=objY-height/2
            if abs(errorPan2)>15:
                pan2=pan2+errorPan2/40
            if abs(errorTilt2)>15:
                tilt2=tilt2-errorTilt2/40
            if pan2>180:
                pan2=180
                print('Pan Out of Range')
            if pan2<0:
                pan2=0
                print('Pan Out of Range')
            if tilt2>180:
                tilt2=180
                print('Tilt Out of Range')
            if tilt2<0:
                tilt2=0
            kit.servo[0].angle=pan2
            kit.servo[1].angle=tilt2            
            break

    frame3=np.hstack((frame1,frame2))
    dt=time.time()-timeMark
    timeMark=time.time()
    dtFIL=.9*dtFIL + .1*dt
    fps=1/dtFIL
    cv2.rectangle(frame3,(0,0),(150,40),(0,0,255),-1)
    cv2.putText(frame3,'fps: '+str(round(fps,1)),(0,30),font,1,(0,255,255),2)

    #cv2.imshow('myCam1',frame1)
    #cv2.imshow('myCam2',frame2)
    cv2.imshow('comboCam',frame3)
    cv2.moveWindow('comboCam',0,450)

    if cv2.waitKey(1)==ord('q'):
        break
cam1.release()
cam2.release()
cv2.destroyAllWindows()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

import cv2

import numpy as np

import time

from adafruit_servokit import ServoKit

print(cv2.__version__)

timeMark=time.time()

dtFIL=0

def nothing(x):

pass

cv2.namedWindow('TrackBars')

cv2.moveWindow('TrackBars',1320,0)

cv2.createTrackbar('hueLower', 'TrackBars',100,179,nothing)

cv2.createTrackbar('hueUpper', 'TrackBars',116,179,nothing)

cv2.createTrackbar('satLow', 'TrackBars',160,255,nothing)

cv2.createTrackbar('satHigh', 'TrackBars',255,255,nothing)

cv2.createTrackbar('valLow', 'TrackBars',150,255,nothing)

cv2.createTrackbar('valHigh', 'TrackBars',255,255,nothing)

kit=ServoKit(channels=16)

tilt1=90

pan1=90

tilt2=90

pan2=90

kit.servo[0].angle=pan1

kit.servo[1].angle=tilt1

kit.servo[2].angle=pan2

kit.servo[3].angle=tilt2

width=720

height=480

flip=2

font=cv2.FONT_HERSHEY_SIMPLEX

#camSet ='v4l2src device=/dev/video1 ! video/x-raw,width='+str(width)+',height='+str(height)+',framerate=20/1 ! videoconvert ! appsink'

cam1=cv2.VideoCapture(camSet1)

cam2=cv2.VideoCapture(camSet2)

while True:

_, frame1 = cam1.read()

_, frame2 = cam2.read()

hsv1=cv2.cvtColor(frame1,cv2.COLOR_BGR2HSV)

hsv2=cv2.cvtColor(frame2,cv2.COLOR_BGR2HSV)

hueLow=cv2.getTrackbarPos('hueLower', 'TrackBars')

hueUp=cv2.getTrackbarPos('hueUpper', 'TrackBars')

Ls=cv2.getTrackbarPos('satLow', 'TrackBars')

Us=cv2.getTrackbarPos('satHigh', 'TrackBars')

Lv=cv2.getTrackbarPos('valLow', 'TrackBars')

Uv=cv2.getTrackbarPos('valHigh', 'TrackBars')

l_b=np.array([hueLow,Ls,Lv])

u_b=np.array([hueUp,Us,Uv])

FGmask1=cv2.inRange(hsv1,l_b,u_b)

FGmask2=cv2.inRange(hsv2,l_b,u_b)

cv2.imshow('FGmask1',FGmask1)

cv2.moveWindow('FGmask1',0,0)

contours1,_ = cv2.findContours(FGmask1,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)

contours1=sorted(contours1,key=lambda x:cv2.contourArea(x),reverse=True)

for cnt in contours1:

area=cv2.contourArea(cnt)

(x,y,w,h)=cv2.boundingRect(cnt)

if area>=100:

cv2.rectangle(frame1,(x,y),(x+w,y+h),(0,255,255),3)

objX=x+w/2

objY=y+h/2

errorPan1=objX-width/2

errorTilt1=objY-height/2

if abs(errorPan1)>15:

pan1=pan1+errorPan1/40

if abs(errorTilt1)>15:

tilt1=tilt1-errorTilt1/40

if pan1>180:

pan1=180

print('Pan Out of Range')

if pan1<0:

pan1=0

print('Pan Out of Range')

if tilt1>180:

tilt1=180

print('Tilt Out of Range')

if tilt1<0:

tilt1=0

kit.servo[2].angle=pan1

kit.servo[3].angle=tilt1

break

contours2,_ = cv2.findContours(FGmask2,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)

contours2=sorted(contours2,key=lambda x:cv2.contourArea(x),reverse=True)

for cnt in contours2:

area=cv2.contourArea(cnt)

(x,y,w,h)=cv2.boundingRect(cnt)

if area>=100:

cv2.rectangle(frame2,(x,y),(x+w,y+h),(0,255,255),3)

objX=x+w/2

objY=y+h/2

errorPan2=objX-width/2

errorTilt2=objY-height/2

if abs(errorPan2)>15:

pan2=pan2+errorPan2/40

if abs(errorTilt2)>15:

tilt2=tilt2-errorTilt2/40

if pan2>180:

pan2=180

print('Pan Out of Range')

if pan2<0:

pan2=0

print('Pan Out of Range')

if tilt2>180:

tilt2=180

print('Tilt Out of Range')

if tilt2<0:

tilt2=0

kit.servo[0].angle=pan2

kit.servo[1].angle=tilt2

break

frame3=np.hstack((frame1,frame2))

dt=time.time()-timeMark

timeMark=time.time()

dtFIL=.9*dtFIL + .1*dt

fps=1/dtFIL

cv2.rectangle(frame3,(0,0),(150,40),(0,0,255),-1)

cv2.putText(frame3,'fps: '+str(round(fps,1)),(0,30),font,1,(0,255,255),2)

#cv2.imshow('myCam1',frame1)

#cv2.imshow('myCam2',frame2)

cv2.imshow('comboCam',frame3)

cv2.moveWindow('comboCam',0,450)

if cv2.waitKey(1)==ord('q'):

break

cam1.release()

cam2.release()

cv2.destroyAllWindows()

Jetson Xavier NX

Jetson Xavier NX Lesson 7: Connecting and Controlling Servos

July 1, 2020 admin

In this lesson we show you how to control a pan/tilt camera bracket with the NVIDIA Jetson Xavier NX. We go through the physical build of the bracket, how to connect the circuit, and then how to program the servos. We use the Adafruit circuitpython library, and show how to download and use the library. If you want to play along at home, you can pick the pan/tilt bracket and servos up HERE, and you can grab a couple of Raspberry Pi cameras HERE.

Below is the simple code for moving the servo using the Jetson Xavier NX:

import time
from adafruit_servokit import ServoKit
kit=ServoKit(channels=16)
tilt=125
pan=0
kit.servo[0].angle=pan
kit.servo[1].angle=tilt
for i in range(0,180):
    kit.servo[0].angle=i
    time.sleep(.05)
for i in range(180,0,-1):
    kit.servo[0].angle=i
    time.sleep(.05)
for i in range(45,180):
    kit.servo[1].angle=i
    time.sleep(.05)
for i in range(180,0,-1):
    kit.servo[1].angle=i
    time.sleep(.05)

import time

from adafruit_servokit import ServoKit

kit=ServoKit(channels=16)

tilt=125

pan=0

kit.servo[0].angle=pan

kit.servo[1].angle=tilt

for i in range(0,180):

kit.servo[0].angle=i

time.sleep(.05)

for i in range(180,0,-1):

kit.servo[0].angle=i

time.sleep(.05)

for i in range(45,180):

kit.servo[1].angle=i

time.sleep(.05)

for i in range(180,0,-1):

kit.servo[1].angle=i

time.sleep(.05)

Jetson Xavier NX

Jetson Xavier NX Lesson 6: Camera Gear and Setup for Future Lessons

June 24, 2020 admin

In this lesson we discuss the future direction of this series, and the gear needed to move forward.

Moving forward, I will be running two pan/tilt servos for the raspberry pi cameras. I suggest buying the same gear HERE. I suggest purchasing two units.

Then, we also need two Version two raspberry pi cameras. I like the following ones, because they include a neat little acrylic case, and the long cable, which makes it work much better on the pan/tilt bracket. You can get the cameras HERE.

Technology Tutorials

Tag Archives: Artificial intelligence

Jetson Xavier NX Lesson 11: Independently Tracking Different Objects in Different Cameras

AI on the Jetson Nano LESSON 52: Improving Picture Quality of the Raspberry Pi Camera with Gstreamer

Jetson Xavier NX Lesson 10: Tracking Multiple Objects of Interest with Servos in OpenCV

Jetson Xavier NX Lesson 7: Connecting and Controlling Servos

Jetson Xavier NX Lesson 6: Camera Gear and Setup for Future Lessons

Making The World a Better Place One High Tech Project at a Time. Enjoy!