Use mediapipe+opencv to make a simple virtual keyboard

Instructions ：

utilize opencv Draw keyboard and output bar ;

utilize mediapipe Identify the coordinate information of palm key points ;

The calculation determines the selected key according to the position relationship between the palm key coordinates and the keyboard key coordinates ;

Use the coordinate relationship between palm keys , Press the control key .

The implementation code is as follows ：

0. Environment configuration ：

pycharm、python=3.7、opencv=4.5、mediapipe=0.8.9、cvzone=1.4.1

1. Import required python library

import cv2 
from cvzone.HandTrackingModule import HandDetector 
import cvzone #  contain opencv And mediapipe function 
import time

2. Turn on the camera and capture

#  Turn on camera , Capture picture 
cap = cv2.VideoCapture(0)
#  Set window size 
cap.set(3, 2560)
#  Recognize gestures 
detector = HandDetector(detectionCon=1)

3. Define keys

#  Keyboard characters 
keys = [['Q', 'W', 'E', 'R', 'T', 'Y', 'U', 'I', 'O', 'P'],
        ['A', 'S', 'D', 'F', 'G', 'H', 'J', 'K', 'L', ';'],
        ['Z', 'X', 'C', 'V', 'B', 'N', 'M', ',', '.', '/']]
#  Define keys , Including the position of the key -> drawing , The value of the key -> Output , Key size -> Layout 
class Button():
    def __init__(self, pos, text, size=[50, 50]):
        self.pos = pos
        self.text = text
        self.size = size
#  Assign key positions 
buttonList = []
for i in range(len(keys)):
    for j, key in enumerate(keys[i]):
        buttonList.append(Button([80 * j + 20, 100 + i * 80], key))

4.  Draw buttons

#  Press the key to draw the function 
def draw_keyboard(img, buttonList):
    for button in buttonList:
        x, y = button.pos
        w, h = button.size
        cvzone.cornerRect(img, (x, y, w, h), 20, rt=0)
        cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 255), cv2.FILLED)
        cv2.putText(img, button.text, (x + 10, y + 40),
                    cv2.FONT_HERSHEY_PLAIN, 3, (255, 255, 255), 2)
    return img

5. Palm key diagram

 6. Virtual keyboard implementation

#  Define output text , When it reaches a certain length, it needs to be emptied , It can be modified to write 
finalText = ''
while True:
    #  Read the camera content 
    res, img = cap.read()
    while not res:
        break
    #  Image reversal , Remove mirror 
    img = cv2.flip(img, 1)
    cv2.rectangle(img, (20, 350), (800, 500), (0, 255, 0), cv2.FILLED)
    #  Check the palm 
    img = detector.findHands(img)
    lmList = detector.findPosition(img)[0]
    img = draw_keyboard(img, buttonList)
    #  Keyboard output 
    if lmList:
        for button in buttonList:
            x, y = button.pos
            w, h = button.size
            #  Determine the key area where the key point of the index finger is located 
            if x < lmList[8][0] < x + w and y < lmList[8][1] < y + h:
                cv2.rectangle(img, (x - 5, y - 5), (x + w + 5, y + h + 5), (175, 0, 175), cv2.FILLED)
                cv2.putText(img, button.text, (x + 10, y + 40),
                            cv2.FONT_HERSHEY_PLAIN, 3, (0, 0, 0), 2)
                #  When the length is greater than 10 when , Empty 
                if len(finalText) > 10:
                    finalText = ""
                #  Judge according to the distance between the tip of the index finger and the tip of the thumb , Whether to press the number 
                l, _, _ = detector.findDistance(8, 4, img, draw=False)
                #  Default does not judge , When judgment is needed , Need to 5000 Change a little 
                if l < 5000:
                    finalText += button.text
                    cv2.putText(img, finalText, (20, 465), cv2.FONT_HERSHEY_SIMPLEX, 3, (255, 255, 255), 4)
                    time.sleep(0.2)
    #  Output key and display 
    cv2.putText(img, finalText, (20, 465), cv2.FONT_HERSHEY_SIMPLEX, 3, (255, 255, 255), 4)
    cv2.imshow("Image", img)
    #  Press down q sign out 
    if cv2.waitKey(1) == ord('q'):
        break

7. Code link

liuweixue001/mediapipe (github.com)

8. Related links

1. Mediapipe Realize gesture tracking _Zcc The blog of -CSDN Blog _mediapipe gesture

2. Mediapipe Gesture recognition _Zcc The blog of -CSDN Blog _mediapipe Gesture recognition

3. Hands - mediapipe (google.github.io)