一、大体流程图上图是rockxrv1126的大体流程首先要初始化模块包括VI模块、VENC模块、并启动VI模块采集视频流、rockx模块的初始化。初始化模块后就要分两个线程处理了。主线程是负责rockx对VI视频流的处理并用OPENCV对人脸进行画框最后把处理后的VI数据传输到VENC模块里面。第二个线程rockx_face_detect_venc_thread从VENC模块获取到H264的编码码流数据并把VENC码流数据保存。二代码#include assert.h #include cstddef #include fcntl.h #include getopt.h #include opencv2/imgproc.hpp #include opencv2/imgproc/imgproc_c.h #include pthread.h #include signal.h #include stdbool.h #include stdio.h #include stdlib.h #include time.h #include unistd.h // #include common/sample_common.h #include rkmedia_api.h #include rknn_rockx_include/rockx_type.h #include rockx.h #include opencv2/core.hpp // #include opencv2/imgoroc.hpp #include opencv2/highgui.hpp #include opencv2/opencv.hpp #define CAMERA_PATH rkispp_scale0 #define CAMERA_ID 0 #define CAMERA_CHN 0 #define VENC_CHN 0 #define WIDTH 1920 #define HEIGHT 1080 using namespace cv; void * rockx_face_detect_venc_thread(void * args) { pthread_detach(pthread_self()); FILE * face_detect_h264 fopen(face_detect_venc.h264, w); MEDIA_BUFFER mb NULL; while(1) { mb RK_MPI_SYS_GetMediaBuffer(RK_ID_VENC, VENC_CHN, -1); if(!mb) { printf(Get Rockx_Venc Data berek...\n); break; } printf(Get Rockx_Venc Data Success...\n); fwrite(RK_MPI_MB_GetPtr(mb), RK_MPI_MB_GetSize(mb), 1, face_detect_h264); RK_MPI_MB_ReleaseBuffer(mb); } return NULL; } int main(int argc, char **argv) { int ret; VI_CHN_ATTR_S vi_chn_attr; vi_chn_attr.pcVideoNode CAMERA_PATH; // Path vi_chn_attr.u32Width WIDTH; // Width vi_chn_attr.u32Height HEIGHT; // Height vi_chn_attr.enPixFmt IMAGE_TYPE_NV12; // ImageType vi_chn_attr.enBufType VI_CHN_BUF_TYPE_MMAP; // BufType vi_chn_attr.u32BufCnt 3; // Cnt vi_chn_attr.enWorkMode VI_WORK_MODE_NORMAL; // Mode ret RK_MPI_VI_SetChnAttr(CAMERA_ID, CAMERA_CHN, vi_chn_attr); if (ret) { printf(Vi Set Attr Failed.....\n); return 0; } else { printf(Vi Set Attr Success.....\n); } ret RK_MPI_VI_EnableChn(CAMERA_ID, CAMERA_CHN); if (ret) { printf(Vi Enable Attr Failed.....\n); return 0; } else { printf(Vi Enable Attr Success.....\n); } VENC_CHN_ATTR_S venc_chn_attr; memset(venc_chn_attr, 0, sizeof(VENC_CHN_ATTR_S)); venc_chn_attr.stVencAttr.u32PicWidth WIDTH; venc_chn_attr.stVencAttr.u32PicHeight HEIGHT; venc_chn_attr.stVencAttr.u32VirWidth WIDTH; venc_chn_attr.stVencAttr.u32VirHeight HEIGHT; venc_chn_attr.stVencAttr.imageType IMAGE_TYPE_NV12; venc_chn_attr.stVencAttr.enType RK_CODEC_TYPE_H264; venc_chn_attr.stVencAttr.u32Profile 66; venc_chn_attr.stRcAttr.enRcMode VENC_RC_MODE_H264CBR; venc_chn_attr.stRcAttr.stH264Cbr.u32Gop 25; venc_chn_attr.stRcAttr.stH264Cbr.u32BitRate WIDTH * HEIGHT * 3; venc_chn_attr.stRcAttr.stH264Cbr.fr32DstFrameRateDen 1; venc_chn_attr.stRcAttr.stH264Cbr.fr32DstFrameRateNum 25; venc_chn_attr.stRcAttr.stH264Cbr.u32SrcFrameRateDen 1; venc_chn_attr.stRcAttr.stH264Cbr.u32SrcFrameRateNum 25; ret RK_MPI_VENC_CreateChn(VENC_CHN, venc_chn_attr); if (ret) { printf(ERROR: Create venc failed!\n); exit(0); } ret RK_MPI_VI_StartStream(CAMERA_ID, CAMERA_CHN); if (ret) { printf(RK_MPI_VI_StartStream Failed.....\n); return 0; } else { printf(RK_MPI_VI_StartStream Success.....\n); } pthread_t pid; pthread_create(pid, NULL, rockx_face_detect_venc_thread, NULL); rockx_config_t * face_detect_config rockx_create_config(); rockx_add_config(face_detect_config, ROCKX_CONFIG_DATA_PATH, /userdata/rockx_data); rockx_handle_t face_detect_handle; rockx_ret_t face_detect_ret; rockx_module_t face_detect_module ROCKX_MODULE_FACE_DETECTION_V2; face_detect_ret rockx_create(face_detect_handle, face_detect_module, face_detect_config, 0); if(face_detect_ret ! ROCKX_RET_SUCCESS) { printf(rockx_creat face_detect failed...\n); return -1; } rockx_release_config(face_detect_config); rockx_image_t rv1126_rockx_image; memset(rv1126_rockx_image, 0 ,sizeof(rv1126_rockx_image)); rv1126_rockx_image.width WIDTH; rv1126_rockx_image.height HEIGHT; rv1126_rockx_image.pixel_format ROCKX_PIXEL_FORMAT_YUV420SP_NV12; MEDIA_BUFFER mb; while(1) { mb RK_MPI_SYS_GetMediaBuffer(RK_ID_VI, CAMERA_CHN, -1); if(!mb) { printf(Get Vi Stream break....\n); break; } rv1126_rockx_image.data (uint8_t *)RK_MPI_MB_GetPtr(mb); rv1126_rockx_image.size RK_MPI_MB_GetSize(mb); rockx_object_array_t face_detect_array; face_detect_ret rockx_face_detect(face_detect_handle, rv1126_rockx_image, face_detect_array, NULL); if(face_detect_ret ! ROCKX_RET_SUCCESS) { printf(face_detect failed....\n); } Mat rv1126_rockx_mat Mat(HEIGHT, WIDTH, CV_8UC1, rv1126_rockx_image.data); for(int i 0; i face_detect_array.count; i) { int left face_detect_array.object[i].box.left; int top face_detect_array.object[i].box.top; int w face_detect_array.object[i].box.right - face_detect_array.object[i].box.left; int h face_detect_array.object[i].box.bottom - face_detect_array.object[i].box.top; Rect boundingRect(left, top, w, h); rectangle(rv1126_rockx_mat, boundingRect, Scalar(255,255,0)); } RK_MPI_SYS_SendMediaBuffer(RK_ID_VENC, VENC_CHN, mb); RK_MPI_MB_ReleaseBuffer(mb); } RK_MPI_VENC_DestroyChn(VENC_CHN); RK_MPI_VI_DisableChn(CAMERA_ID, CAMERA_CHN); return 0; }2.1. RV1126模块初始化并启动VI工作RV1126模块的初始化包括VI模块、VENC模块的初始化初始化上述模块后则调用RK_MPI_VI_StartStream启动VI开始采集摄像头的视频流。关于VI模块、VENC模块的初始化参数这里就不阐述了因为之前的课程里面已经讲了很多次。2.2. rockx人脸检测模块的初始化rockx_config_t * face_detect_config rockx_create_config(); rockx_add_config(face_detect_config, ROCKX_CONFIG_DATA_PATH, /userdata/rockx_data); rockx_handle_t face_detect_handle; rockx_ret_t face_detect_ret; rockx_module_t face_detect_module ROCKX_MODULE_FACE_DETECTION_V2; face_detect_ret rockx_create(face_detect_handle, face_detect_module, face_detect_config, 0); if(face_detect_ret ! ROCKX_RET_SUCCESS) { printf(rockx_creat face_detect failed...\n); return -1; } rockx_release_config(face_detect_config);这段代码是初始化rockx的模块首先要使用rockx_create_config分配rockx_config_t结构体并使用rockx_add_config把对应的rockx路径配置进去在我们的板子里面在/userdata/rockx_data里面,并使用rockx_create创建rockx_handle_t句柄rockx_create的传参第一个参数rockx_handle_t结构体指针、 第二个参数rockx_module_t是ROCKX_MODULE_FACE_DETECTION_V2ROCKX_MODULE_FACE_DETECTION_V2是人脸检测的Version2模块、第三个参数是rockx_config_t结构体指针、第四个参数默认是0。2.3.使用rockx对VI模块的数据进行人脸检测处理rockx_image_t rv1126_rockx_image; memset(rv1126_rockx_image, 0 ,sizeof(rv1126_rockx_image)); rv1126_rockx_image.width WIDTH; rv1126_rockx_image.height HEIGHT; rv1126_rockx_image.pixel_format ROCKX_PIXEL_FORMAT_YUV420SP_NV12;MEDIA_BUFFER mb; while(1) { mb RK_MPI_SYS_GetMediaBuffer(RK_ID_VI, CAMERA_CHN, -1); if(!mb) { printf(Get Vi Stream break....\n); break; } rv1126_rockx_image.data (uint8_t *)RK_MPI_MB_GetPtr(mb); rv1126_rockx_image.size RK_MPI_MB_GetSize(mb); rockx_object_array_t face_detect_array; face_detect_ret rockx_face_detect(face_detect_handle, rv1126_rockx_image, face_detect_array, NULL); if(face_detect_ret ! ROCKX_RET_SUCCESS) { printf(face_detect failed....\n); }这部分代码是整个DEMO的核心也是ROCKX检测VI视频数据的核心。第一段是初始化rockx_image_t结构体初始化需要传三个值分别是width WIDTH(1920)、height HEIGHT(1080)、pixel_formatROCKX_PIXEL_FORMAT_YUV420SP_NV12。这三个值都需要和VI模块的配置是一样的。初始化rockx_image_t后则需要通过RK_MPI_SYS_GetMediaBuffer获取每一帧VI模块的数据并把每一帧VI模块的缓冲区和长度传输给rockx_image_t。具体的代码是rv1126_rockx_image.data (uint8_t *)RK_MPI_MB_GetPtr(mb)(把每一帧VI缓冲区数据赋值到rockx_image_t的data)、rv1126_rockx_image.size RK_MPI_MB_GetSize(mb)(把每一帧VI大小赋值到rockx_image_t的size)赋值到rockx_image_t后则调用rockx_face_detect对每一帧的rockx_image_t图像进行人脸检测并把人脸检测的结果输出到rockx_object_array_t。rockx_object_array_t的内容主要存储的是人脸检测数量和人脸检测区域信息(如left、top、right、bottom的坐标信息)2.4.使用opencv对人脸检测的结果进行画框Mat rv1126_rockx_mat Mat(HEIGHT, WIDTH, CV_8UC1, rv1126_rockx_image.data); for(int i 0; i face_detect_array.count; i) { int left face_detect_array.object[i].box.left; int top face_detect_array.object[i].box.top; int w face_detect_array.object[i].box.right - face_detect_array.object[i].box.left; int h face_detect_array.object[i].box.bottom - face_detect_array.object[i].box.top; Rect boundingRect(left, top, w, h); rectangle(rv1126_rockx_mat, boundingRect, Scalar(255,255,0)); }检测完每一帧人脸数据后就需要对每个人脸区域进行画框了这里画框是用opencv进行处理。首先要先创建OPENCV的Mat矩阵Mat rv1126_image_mat Mat(HEIGHT, WIDTH, CV_8UC1, rv1126_rockx_image.data)。创建完Mat之后则需要根据rockx_object_array_t的坐标信息进行画框先循环遍历人脸的数量(rockx_object_array_t.count)然后获取每一帧人脸的坐标信息,主要是left、top、right、bottom, 最后使用OPENCV的rectangle函数把坐标信息描绘出一个矩形表现出来。2.5.把处理后的数据发送到VENC模块RK_MPI_SYS_SendMediaBuffer(RK_ID_VENC, VENC_CHN, mb); RK_MPI_MB_ReleaseBuffer(mb);把上述的数据处理完成之后则把每一帧数据传输给VENC模块这里使用的API是RK_MPI_SYS_SendMediaBuffer。此时此刻VENC模块就有VENC码流数据了2.6.创建rockx_face_detect_venc _thread线程保存每一帧H264的编码码流数据pthread_t pid; pthread_create(pid, NULL, rockx_face_detect_venc_thread, NULL);void * rockx_face_detect_venc_thread(void * args) { pthread_detach(pthread_self()); FILE * face_detect_h264 fopen(face_detect_venc.h264, w); MEDIA_BUFFER mb NULL; while(1) { mb RK_MPI_SYS_GetMediaBuffer(RK_ID_VENC, VENC_CHN, -1); if(!mb) { printf(Get Rockx_Venc Data berek...\n); break; } printf(Get Rockx_Venc Data Success...\n); fwrite(RK_MPI_MB_GetPtr(mb), RK_MPI_MB_GetSize(mb), 1, face_detect_h264); RK_MPI_MB_ReleaseBuffer(mb); } return NULL; }通过pthread_create创建venc码流线程这个线程的名字是rockx_face_detect_venc thread。在这个线程里面通过RK_MPI_SYS_GetMediaBuffer获取每一帧通过rockx人脸检测处理后的VENC码流数据并用fwrite保存起来(fwrite(RK_MPI_MB_GetPtr(mb), RK_MPI_MB_GetSize(mb), 1, face_detect_h264))。最终输出的结果是在视频中检测出对应的人脸并用opencv画矩形出来。