34、树莓派进行人体姿态检测并进行语音播报

基本思想：需要水一篇博客，套用了qiuqiu大佬的simplepose代码，使用ncnn框架，做了个动作检测，然后套用了树莓派4b和使用了语音模块播报

一、自己购买树莓派和喇叭，链接参考这篇博客26、友善NanoPi NEO进行目标检测和串口调试语音播报_sxj731533730的博客-CSDN博客_nanopi neo python

然后接线图上一下，串口工具和杜邦线是必不可少的，串口调试工具也是必不可少。

，

 第二步：写代码在树莓中，调用串口和摄像头（摄像头可以使用手机的摄像头或者笔记本的摄像头 直接用ffmpeg拉流推流给树莓派代码），我用的usb摄像头，推拉流摄像头使用手机摄像头当做实际树莓派的输入视频流 参考这篇38、静默活体检测测试及ncnn、mnn部署_sxj731533730的博客-CSDN博客_静默活体检测

#include "ncnn/net.h"

#include <algorithm>
#if defined(USE_NCNN_SIMPLEOCV)
#include "simpleocv.h"
#else
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#endif
#include <stdio.h>
#include <vector>
#include <iostream>

#include <fcntl.h>
#include <termios.h>
#include <unistd.h>
#include<unistd.h>
#include<stdio.h>
#include<sys/types.h>
#include<sys/stat.h>

struct KeyPoint
{
    cv::Point2f p;
    float prob;
};
ncnn::Net posenet;
int initial(){
    posenet.opt.use_vulkan_compute = true;

    // the simple baseline human pose estimation from gluon-cv
    // https://gluon-cv.mxnet.io/build/examples_pose/demo_simple_pose.html
    // mxnet model exported via
    //      pose_net.hybridize()
    //      pose_net.export('pose')
    // then mxnet2ncnn
    // the ncnn model https://github.com/nihui/ncnn-assets/tree/master/models
    if (posenet.load_param("../model/pose.param"))
        exit(-1);
    if (posenet.load_model("../model/pose.bin"))
        exit(-1);
}
static int detect_posenet(const cv::Mat& bgr, std::vector<KeyPoint>& keypoints)
{




    int w = bgr.cols;
    int h = bgr.rows;

    ncnn::Mat in = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR2RGB, w, h, 192, 256);

    // transforms.ToTensor(),
    // transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
    // R' = (R / 255 - 0.485) / 0.229 = (R - 0.485 * 255) / 0.229 / 255
    // G' = (G / 255 - 0.456) / 0.224 = (G - 0.456 * 255) / 0.224 / 255
    // B' = (B / 255 - 0.406) / 0.225 = (B - 0.406 * 255) / 0.225 / 255
    const float mean_vals[3] = {0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f};
    const float norm_vals[3] = {1 / 0.229f / 255.f, 1 / 0.224f / 255.f, 1 / 0.225f / 255.f};
    in.substract_mean_normalize(mean_vals, norm_vals);

    ncnn::Extractor ex = posenet.create_extractor();

    ex.input("data", in);

    ncnn::Mat out;
    ex.extract("conv3_fwd", out);

    // resolve point from heatmap
    keypoints.clear();
    for (int p = 0; p < out.c; p++)
    {
        const ncnn::Mat m = out.channel(p);

        float max_prob = 0.f;
        int max_x = 0;
        int max_y = 0;
        for (int y = 0; y < out.h; y++)
        {
            const float* ptr = m.row(y);
            for (int x = 0; x < out.w; x++)
            {
                float prob = ptr[x];
                if (prob > max_prob)
                {
                    max_prob = prob;
                    max_x = x;
                    max_y = y;
                }
            }
        }

        KeyPoint keypoint;
        keypoint.p = cv::Point2f(max_x * w / (float)out.w, max_y * h / (float)out.h);
        keypoint.prob = max_prob;

        keypoints.push_back(keypoint);
    }

    return 0;
}

int sendSerialPort(const char W_BUF[], int length) {

    int tty_fd = -1;

    int rv = -1;

    struct termios options;
    system("sudo chmod 777 /dev/ttyS0");
    tty_fd = open("/dev/ttyS0", O_RDWR | O_NOCTTY | O_NDELAY); //打开串口设备
    fcntl(tty_fd, F_SETFL, 0);
    if (tty_fd < 0) {
        printf("open tty failed:%s\n", strerror(errno));
        close(tty_fd);
        return -1;
    }

    printf("open devices sucessful!\n");


    memset(&options, 0, sizeof(options));

    rv = tcgetattr(tty_fd, &options); //获取原有的串口属性的配置
    if (rv != 0) {
        printf("tcgetattr() failed:%s\n", strerror(errno));
        close(tty_fd);
        return -1;

    }

    options.c_cflag |= (CLOCAL | CREAD); // CREAD 开启串行数据接收，CLOCAL并打开本地连接模式
    options.c_cflag &= ~CSIZE;// 先使用CSIZE做位屏蔽
    options.c_cflag |= CS8; //设置8位数据位
    options.c_cflag &= ~PARENB; //无校验位
    cfsetispeed(&options, B9600);
    cfsetospeed(&options, B9600);
    options.c_cflag &= ~CSTOPB;
    options.c_cc[VTIME] = 0;
    options.c_cc[VMIN] = 0;
    tcflush(tty_fd, TCIFLUSH);


    if ((tcsetattr(tty_fd, TCSANOW, &options)) != 0) {
        printf("tcsetattr failed:%s\n", strerror(errno));
        close(tty_fd);
        return -1;

    }


    std::cout << std::endl << "length= " << length << std::endl;
    rv = write(tty_fd, W_BUF, length);
    if(rv<=0) /* 出错了*/
    {
        if (errno == EINTR) /* 中断错误 我们继续写*/
        {
            close(tty_fd);
            printf("[SeanSend]error errno==EINTR continue\n");
        } else if (errno == EAGAIN) /* EAGAIN : Resource temporarily unavailable*/
        {
            sleep(1);//等待一秒，希望发送缓冲区能得到释放
            close(tty_fd);
            printf("[SeanSend]error errno==EAGAIN continue\n");
        } else /* 其他错误 没有办法,只好退了*/
        {
            printf("[SeanSend]ERROR: errno = %d, strerror = %s \n", errno, strerror(errno));
            return (-1);
        }
    }
    if (rv < 0) {
        printf("Write() error:%s\n", strerror(errno));
        close(tty_fd);
        return -1;
    }
    for (int i = 0; i < length; i++) {
        std::cout << std::hex << (int) W_BUF[i] << " ";

    }
    close(tty_fd);
    printf("\nWrite() successfully\n");
    return 0;

}

static void draw_pose(const cv::Mat& bgr, const std::vector<KeyPoint>& keypoints)
{
    cv::Mat image = bgr.clone();

    // draw bone
    static const int joint_pairs[16][2] = {
            {0, 1}, {1, 3}, {0, 2}, {2, 4}, {5, 6}, {5, 7}, {7, 9}, {6, 8}, {8, 10}, {5, 11}, {6, 12}, {11, 12}, {11, 13}, {12, 14}, {13, 15}, {14, 16}
    };

//    for (int i = 0; i < 16; i++)
//    {
//        const KeyPoint& p1 = keypoints[joint_pairs[i][0]];
//        const KeyPoint& p2 = keypoints[joint_pairs[i][1]];
//
//        if (p1.prob < 0.2f || p2.prob < 0.2f)
//            continue;
//
//        cv::line(image, p1.p, p2.p, cv::Scalar(255, 0, 0), 2);
//
//

//    }


 //   cv::line(image, keypoints[joint_pairs[4][0]].p, keypoints[joint_pairs[4][1]].p, cv::Scalar(255, 0, 255), 8);

  //  cv::line(image, keypoints[joint_pairs[5][0]].p, keypoints[joint_pairs[5][1]].p, cv::Scalar(0, 0, 255), 8);
    //cv::line(image, keypoints[joint_pairs[6][0]].p, keypoints[joint_pairs[6][1]].p, cv::Scalar(255, 255, 120), 8);

    //cv::line(image, keypoints[joint_pairs[7][0]].p, keypoints[joint_pairs[7][1]].p, cv::Scalar(0, 255, 255), 8);

    //cv::line(image, keypoints[joint_pairs[8][0]].p, keypoints[joint_pairs[8][1]].p, cv::Scalar(255, 255, 0), 8);

    // draw joint
//    for (size_t i = 0; i < keypoints.size(); i++)
//    {
//        const KeyPoint& keypoint = keypoints[i];
//
//        fprintf(stderr, "%.2f %.2f = %.5f\n", keypoint.p.x, keypoint.p.y, keypoint.prob);
//
//        if (keypoint.prob < 0.2f)
//            continue;
//
//        cv::circle(image, keypoint.p, 3, cv::Scalar(0, 255, 0), -1);
//    }
    const  char data[] = {0xFD,0x00 ,0x07 ,0x01 ,0x01 ,0xB4, 0xF2 ,0xC8, 0xAD ,0xD9 };


    cv::circle(image, keypoints[5].p, 3, cv::Scalar(255, 255, 255), -1);
    cv::putText(image, "5", keypoints[5].p, cv::FONT_HERSHEY_COMPLEX, 1, cv::Scalar(255, 255, 255), 1, 8, 0);
    cv::circle(image, keypoints[6].p, 3, cv::Scalar(0, 255, 0), -1);
    cv::putText(image, "6", keypoints[6].p, cv::FONT_HERSHEY_COMPLEX, 1, cv::Scalar(0, 255, 255), 1, 8, 0);


    cv::circle(image, keypoints[8].p, 3, cv::Scalar(0, 255, 0), -1);
    cv::putText(image, "8", keypoints[8].p, cv::FONT_HERSHEY_COMPLEX, 1, cv::Scalar(0, 255, 255), 1, 8, 0);
    cv::circle(image, keypoints[7].p, 3, cv::Scalar(0, 255, 0), -1);
    cv::putText(image, "7", keypoints[7].p, cv::FONT_HERSHEY_COMPLEX, 1, cv::Scalar(0, 255, 255), 1, 8, 0);

    cv::circle(image, keypoints[9].p, 3, cv::Scalar(0, 255, 0), -1);
    cv::putText(image, "9", keypoints[9].p, cv::FONT_HERSHEY_COMPLEX, 1, cv::Scalar(0, 255, 255), 1, 8, 0);
    cv::circle(image, keypoints[10].p, 3, cv::Scalar(0, 255, 0), -1);
    cv::putText(image, "10", keypoints[10].p, cv::FONT_HERSHEY_COMPLEX, 1, cv::Scalar(0, 255, 255), 1, 8, 0);

    if(keypoints[9].p.y<keypoints[7].p.y&&keypoints[10].p.y<keypoints[8].p.y){
        //printf("rise hand \n");
        int distance0=sqrt( pow(keypoints[9].p.y-keypoints[5].p.y,2)+pow(keypoints[9].p.x-keypoints[5].p.x,2));
        int distance1=sqrt( pow(keypoints[6].p.y-keypoints[10].p.y,2)+pow(keypoints[6].p.x-keypoints[10].p.x,2));
        //printf("disntance %d\n",distance);
        if(distance0>60&&distance1>60){
           printf("make pose\n");
            sendSerialPort(data, sizeof(data));//测试

        }
    }

    cv::imshow("image", image);
    cv::waitKey(1);
}

int main(int argc, char** argv)
{
    cv::VideoCapture capture(0);

    if (!capture.isOpened()) {
        std::cout << "open camera error!" << std::endl;
        return -1;
    }
    initial();
    cv::Mat frame;
    while (1) {
        capture >> frame;
        cv::resize(frame,frame,cv::Size(640,640));
        std::vector<KeyPoint> keypoints;
        detect_posenet(frame, keypoints);

        draw_pose(frame, keypoints);
        if (frame.empty()) {
            std::cout << "capture empty frame" << std::endl;
            continue;
        }

    }

    return 0;
}

 其中中文生成语音十六进制使用喇叭官方软件拷贝就行，只上个简单图

测试结果

[email protected]:~/Orienmask_project/build $ ./Orienmask_project 
hit box
open devices sucessful!

length= 10
fd 0 7 1 1 b4 f2 c8 ad d9 
Write() successfully
make pose
open devices sucessful!

length= a
fd 0 7 1 1 b4 f2 c8 ad d9  
Write() successfully
make pose
open devices sucessful!

length= a
fd 0 7 1 1 b4 f2 c8 ad d9   
Write() successfully
make pose
open devices sucessful!

length= a
fd 0 7 1 1 b4 f2 c8 ad d9 
Write() successfully
make pose
open devices sucessful!

length= a
fd 0 7 1 1 b4 f2 c8 ad d9 
Write() successfully
make pose
open devices sucessful!