KINECT+opencv基于骨骼信息对视频进行动作识别

时间：2016-05-18 19:30:55 阅读：294 评论：0 收藏：0 [点我收藏+]

标签：

KINECT+opencv基于骨骼信息对视频进行动作识别

环境：kinect1.7+opencv2.4+vc2015

使用kinect获取并按批处理三维空间内的骨骼信息
基于视频帧差计算各关节运动向量并与本地模板匹配

自前一篇过去一周了。这次开始对视频流进行识别匹配。大体的框架是：kinect按批读入视频帧，然后将这一批视频流与模板视频流进行匹配识别。即，kinect先读入NUM_OF_FRAME（120）帧的骨骼信息，然后每一帧减去后一帧计算出各关节点的位移向量，将这些位移向量与模板匹配。
同样又写了两个程序，一个对当前视频处理并匹配，另一个记录并保存模板到本地。

1.对当前帧处理并匹配

#include <afxsock.h>
#include<opencv.hpp>
#include<Windows.h>
#include<NuiApi.h>
using namespace std;
using namespace cv;

//---参数--------------------------------------------  
#define NUM_OF_MODEL 3              //本地模板的个数
#define NUM_OF_SKELETONS 10         //kinect使用的关节数目，在坐姿模式里只读取10个关节点，编号是从2-11号骨骼
#define NUM_OF_FRAME 120            //一批处理的视频帧的数量
#define SHAKE_THRESHOLD_VALUE 0.1   //关节点防抖动的阈值，只有关节位移大于0.1米才计算位移向量
#define INERRED_THRESHOLD_VALUE 0.3 //关节点丢失后kinect会根据内置算法推断该关节点位置，这个阈值是限制推断出来的关节三维坐标位移超过0.3米

Mat modelskeleton[NUM_OF_MODEL];    //模板放进这里

string int2type(int _intval)        //模板中各个动作的名字写进这里
{
    string time_str;
    switch (_intval)
    {
    case 1:
        time_str = "drinking";      //第一个动作喝水
        break;
    case 2:
        time_str = "writing";
        break;
    case 3:
        time_str = "on_phone";
        break;
    case 0:
    time_str = "no action";
    break;
    }
    return time_str;
}
string int2str(int _intval)
{
    string time_str;
    char c[10];
    sprintf_s(c, "%d", _intval);
    time_str = c;
    return c;
}
Mat setInput0(Mat now, int id)//从本地xml文件里读入模板动作后计算每两帧模板动作的位移向量，返回值存入modelskeleton[NUM_OF_MODEL]
{
    cout << "done" << endl;
    Mat temp(NUM_OF_FRAME, NUM_OF_SKELETONS, CV_32FC3, Scalar(0, 0, 0));
    for (int i = 0;i < NUM_OF_FRAME;i++)
    {
        for (int j = 0;j < NUM_OF_SKELETONS; j++)
        {
            temp.ptr<float>(i)[j * 3] = now.ptr<float>(i + 1)[j * 3] - now.ptr<float>(i)[j * 3];
            temp.ptr<float>(i)[j * 3 + 1] = now.ptr<float>(i + 1)[j * 3 + 1] - now.ptr<float>(i)[j * 3 + 1];
            temp.ptr<float>(i)[j * 3 + 2] = now.ptr<float>(i + 1)[j * 3 + 2] - now.ptr<float>(i)[j * 3 + 2];
        }
    }
    return temp;
}

void drawSkeleton(Mat &image, CvPoint pointSet[], NUI_SKELETON_FRAME now,int userid)//画出骨骼，推断出来的关节点用灰色表示
{
    CvScalar color;
    color = cvScalar(255);
    if ((pointSet[NUI_SKELETON_POSITION_HEAD].x != 0 || pointSet[NUI_SKELETON_POSITION_HEAD].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_HEAD], pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_SPINE].x != 0 || pointSet[NUI_SKELETON_POSITION_SPINE].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER], pointSet[NUI_SKELETON_POSITION_SPINE], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_SPINE].x != 0 || pointSet[NUI_SKELETON_POSITION_SPINE].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_HIP_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_CENTER].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_SPINE], pointSet[NUI_SKELETON_POSITION_HIP_CENTER], color, 2);

    //左上肢   
    if ((pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_SHOULDER_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER], pointSet[NUI_SKELETON_POSITION_SHOULDER_LEFT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_SHOULDER_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_LEFT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_ELBOW_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_ELBOW_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_SHOULDER_LEFT], pointSet[NUI_SKELETON_POSITION_ELBOW_LEFT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_ELBOW_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_ELBOW_LEFT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_WRIST_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_WRIST_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_ELBOW_LEFT], pointSet[NUI_SKELETON_POSITION_WRIST_LEFT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_WRIST_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_WRIST_LEFT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_HAND_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_HAND_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_WRIST_LEFT], pointSet[NUI_SKELETON_POSITION_HAND_LEFT], color, 2);

    //右上肢   
    if ((pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_SHOULDER_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER], pointSet[NUI_SKELETON_POSITION_SHOULDER_RIGHT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_SHOULDER_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_ELBOW_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_ELBOW_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_SHOULDER_RIGHT], pointSet[NUI_SKELETON_POSITION_ELBOW_RIGHT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_ELBOW_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_ELBOW_RIGHT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_WRIST_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_WRIST_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_ELBOW_RIGHT], pointSet[NUI_SKELETON_POSITION_WRIST_RIGHT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_WRIST_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_WRIST_RIGHT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_HAND_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_HAND_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_WRIST_RIGHT], pointSet[NUI_SKELETON_POSITION_HAND_RIGHT], color, 2);

    //左下肢   
    if ((pointSet[NUI_SKELETON_POSITION_HIP_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_CENTER].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_HIP_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_HIP_CENTER], pointSet[NUI_SKELETON_POSITION_HIP_LEFT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_HIP_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_LEFT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_KNEE_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_KNEE_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_HIP_LEFT], pointSet[NUI_SKELETON_POSITION_KNEE_LEFT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_KNEE_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_KNEE_LEFT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_ANKLE_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_ANKLE_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_KNEE_LEFT], pointSet[NUI_SKELETON_POSITION_ANKLE_LEFT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_ANKLE_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_ANKLE_LEFT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_FOOT_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_FOOT_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_ANKLE_LEFT], pointSet[NUI_SKELETON_POSITION_FOOT_LEFT], color, 2);

    //右下肢   
    if ((pointSet[NUI_SKELETON_POSITION_HIP_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_CENTER].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_HIP_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_HIP_CENTER], pointSet[NUI_SKELETON_POSITION_HIP_RIGHT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_HIP_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_RIGHT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_KNEE_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_KNEE_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_HIP_RIGHT], pointSet[NUI_SKELETON_POSITION_KNEE_RIGHT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_KNEE_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_KNEE_RIGHT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_ANKLE_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_ANKLE_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_KNEE_RIGHT], pointSet[NUI_SKELETON_POSITION_ANKLE_RIGHT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_ANKLE_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_ANKLE_RIGHT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_FOOT_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_FOOT_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_ANKLE_RIGHT], pointSet[NUI_SKELETON_POSITION_FOOT_RIGHT], color, 2);
    for (int i = 0;i < 20 ;i++)
    {if(now.SkeletonData[userid].SkeletonPositions[i].x)
        if (now.SkeletonData[userid].eSkeletonPositionTrackingState[i] == 1|| now.SkeletonData[userid].eSkeletonPositionTrackingState[i] == 0)
            circle(image, pointSet[i], 5, Scalar(128), 2);
        else
            circle(image, pointSet[i], 5, Scalar(255), 2);
    }
}

//用于匹配的类
class Process {
private:
    Mat framess;
    Vector4 mark;//追踪到当前最后一帧人物的中心坐标
    int matching;
    float disvalsModel[NUM_OF_MODEL][NUM_OF_FRAME] = { 0 };
public:
    Process() :matching(0)
    {
        framess.create(NUM_OF_FRAME, NUM_OF_SKELETONS, CV_32FC3);
        mark = { 0 };
    }
    void classify(Mat &outmark)//将匹配到的动作编号转化为文字保存进mark里
    {
        float fx, fy;
        NuiTransformSkeletonToDepthImage(mark, &fx, &fy);
        fx *= 2;
        fy *= 2;
        circle(outmark, cvPoint((int)fx,(int)fy),2,Scalar(64,255,255 ),3);
        const string str = "the pose is " + int2type(matching);
        putText(outmark, str, cvPoint((int)fx, (int)fy), CV_FONT_HERSHEY_COMPLEX,1,Scalar(64, 255, 255));
    }
    void shakeMask(NUI_SKELETON_FRAME past, NUI_SKELETON_FRAME &now, int userid)
    {
            for (int j = 3;j < NUM_OF_SKELETONS;j++)
            {
                if (now.SkeletonData[userid].eSkeletonPositionTrackingState[j] == 1 && (
                    (now.SkeletonData[userid].SkeletonPositions[j].x - past.SkeletonData[userid].SkeletonPositions[j].x)>INERRED_THRESHOLD_VALUE ||
                    (now.SkeletonData[userid].SkeletonPositions[j].y - past.SkeletonData[userid].SkeletonPositions[j].y) > INERRED_THRESHOLD_VALUE ||
                    (now.SkeletonData[userid].SkeletonPositions[j].z - past.SkeletonData[userid].SkeletonPositions[j].z) > INERRED_THRESHOLD_VALUE))
                {
                    now.SkeletonData[userid].SkeletonPositions[j].x = past.SkeletonData[userid].SkeletonPositions[j].x;
                    now.SkeletonData[userid].SkeletonPositions[j].y = past.SkeletonData[userid].SkeletonPositions[j].y;
                    now.SkeletonData[userid].SkeletonPositions[j].z = past.SkeletonData[userid].SkeletonPositions[j].z;
                }
                if (now.SkeletonData[userid].SkeletonPositions[j].x - past.SkeletonData[userid].SkeletonPositions[j].x < SHAKE_THRESHOLD_VALUE)
                    now.SkeletonData[userid].SkeletonPositions[j].x = past.SkeletonData[userid].SkeletonPositions[j].x;
                if (now.SkeletonData[userid].SkeletonPositions[j].y - past.SkeletonData[userid].SkeletonPositions[j].y < SHAKE_THRESHOLD_VALUE)
                    now.SkeletonData[userid].SkeletonPositions[j].y = past.SkeletonData[userid].SkeletonPositions[j].y;
                if (now.SkeletonData[userid].SkeletonPositions[j].z - past.SkeletonData[userid].SkeletonPositions[j].z < SHAKE_THRESHOLD_VALUE)
                    now.SkeletonData[userid].SkeletonPositions[j].z = past.SkeletonData[userid].SkeletonPositions[j].z;
            }
    }
    void setInput(NUI_SKELETON_FRAME past, NUI_SKELETON_FRAME now, int counter, int userid)//前后两帧做差求位移向量
    {
        if (counter == NUM_OF_FRAME-1)
        {
            mark.x = now.SkeletonData[userid].Position.x;
            mark.y = now.SkeletonData[userid].Position.y;
            mark.z = now.SkeletonData[userid].Position.z;
        }

        for (int j = 0;j < NUM_OF_SKELETONS;j++)
        {
            framess.ptr<float>(counter)[3 * j] = now.SkeletonData[userid].SkeletonPositions[j].x - past.SkeletonData[userid].SkeletonPositions[j].x;
            framess.ptr<float>(counter)[3 * j + 1] = now.SkeletonData[userid].SkeletonPositions[j].y - past.SkeletonData[userid].SkeletonPositions[j].y;
            framess.ptr<float>(counter)[3 * j + 2] = now.SkeletonData[userid].SkeletonPositions[j].z - past.SkeletonData[userid].SkeletonPositions[j].z;
        }

    }
    float rtwCal(int flag)
    {
        float rtwMat[3] = { 0 };
        Mat rtwmat(NUM_OF_FRAME, NUM_OF_FRAME, CV_32FC1, Scalar(0));
        for (int i = 0;i < NUM_OF_FRAME;i++)
        {
            for (int j = 0;j < NUM_OF_FRAME;j++)
            {
                for (int k = 0;k < (NUM_OF_SKELETONS * 3);k++)//多维向量余弦计算
                {
                    rtwMat[0] += framess.ptr<float>(i)[k] * modelskeleton[flag].ptr<float>(j)[k];
                    rtwMat[1] += framess.ptr<float>(i)[k] * framess.ptr<float>(i)[k];
                    rtwMat[2] += modelskeleton[flag].ptr<float>(j)[k] * modelskeleton[flag].ptr<float>(j)[k];
                }
                rtwmat.ptr<float>(i)[j] = rtwMat[0] / (sqrt(rtwMat[1])*sqrt(rtwMat[2]));            }
        }

        float routeVal[NUM_OF_FRAME] = { 0 };
        for (int i = 0;i < NUM_OF_FRAME;i++)
        {
            for (int j = 0;j < NUM_OF_FRAME;j++)
            {
                routeVal[i] += rtwmat.ptr<float>(j)[(j + i) % NUM_OF_FRAME];
            }
        }
        float result = routeVal[0];
        for (int i = 0;i < NUM_OF_FRAME-1;i++)
        {
            if (result<routeVal[i+1])
                result = routeVal[i + 1];
        }
        return result;
    }
    void processBegin()
    {
        int result = 0;
        float rtwDistance[NUM_OF_MODEL] = { 0 };
        for (int i = 0;i < NUM_OF_MODEL;i++)
        {
            rtwDistance[i] = rtwCal(i);
            cout << "the distance between " + int2str(i+1) << "is" << rtwDistance[i] << endl;
        }
        float temp = rtwDistance[0];
        for (int i = 0;i < NUM_OF_MODEL - 1;i++)
        {
            if (temp < rtwDistance[i + 1])
            {
                temp = rtwDistance[i + 1];
                result = i + 1;
            }
        }
        matching = result+1;
        if (temp < 0.3)
            matching = 0;
        cout << "this time is type" << int2str(matching) << endl << endl;
    }
};

int main()
{
    Mat image(480, 640, CV_8UC3, Scalar(0, 0, 0));
    //
    HRESULT hr = NuiInitialize(NUI_INITIALIZE_FLAG_USES_SKELETON | NUI_INITIALIZE_FLAG_USES_COLOR);
    HANDLE skeletonEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
    hr = NuiSkeletonTrackingEnable(skeletonEvent, NUI_SKELETON_TRACKING_FLAG_ENABLE_SEATED_SUPPORT);
    //
    HANDLE ColorFrameStream = NULL;
    HANDLE NextFrameEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
    hr = NuiImageStreamOpen(NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, 0, 2, NextFrameEvent, &ColorFrameStream);
    if (FAILED(hr))
    {
        cout << "Could not open image stream video" << endl;
        return hr;
    }
    //
    NUI_SKELETON_FRAME pastFrame = { 0 };
    NUI_SKELETON_FRAME nowFrame = { 0 };
    Mat mark(image.size(), CV_8UC3, Scalar(0, 0, 0));//整个匹配运算的结果保存在mark里，和颜色通道一起叠加输出到屏幕
    int userid = 0;
    bool bFoundSkeleton = FALSE;
    //启动kinect
    while (!bFoundSkeleton)
    {
        if (WaitForSingleObject(skeletonEvent, INFINITE) == 0)
        {
            hr = NuiSkeletonGetNextFrame(0, &nowFrame);
            for (int i = 0;i < NUI_SKELETON_COUNT;i++)
            {
                NUI_SKELETON_TRACKING_STATE trackingState = nowFrame.SkeletonData[i].eTrackingState;
                if (trackingState == NUI_SKELETON_TRACKED)
                {
                    bFoundSkeleton = TRUE;
                    userid = i;
                }
            }
        }
    }

    int idcounter = 0;
    //读入模板初始化
    FileStorage fds("monitor_file.xml", FileStorage::READ);
    for (int i = 0;i < NUM_OF_MODEL;i++)
    {
        modelskeleton[i].create(NUM_OF_FRAME, NUM_OF_SKELETONS, CV_32FC3);
        Mat temp1(NUM_OF_FRAME + 1, NUM_OF_SKELETONS, CV_32FC3);
        fds["mat_id" + int2str(idcounter)] >> temp1;
        modelskeleton[i] = setInput0(temp1, idcounter);
        idcounter++;
    }
    fds.release();
    while (1)
    {
        //
        int frameCounter = 0;
        Process process=Process();
        for (;userid < NUI_SKELETON_COUNT;userid++)
        {
            if (nowFrame.SkeletonData[userid].eTrackingState == NUI_SKELETON_TRACKED &&
                nowFrame.SkeletonData[userid].eSkeletonPositionTrackingState[NUI_SKELETON_POSITION_SHOULDER_CENTER] != NUI_SKELETON_POSITION_NOT_TRACKED)
            {
                while (frameCounter < NUM_OF_FRAME)
                {
                    //显示颜色通道图像
                        const NUI_IMAGE_FRAME * pImageFrame = NULL;
                        if (WaitForSingleObject(skeletonEvent, INFINITE) == 0)
                        {
                            NuiImageStreamGetNextFrame(ColorFrameStream, 0, &pImageFrame);
                            INuiFrameTexture * pTexture = pImageFrame->pFrameTexture;
                            NUI_LOCKED_RECT LockedRect;
                            pTexture->LockRect(0, &LockedRect, NULL, 0);
                            for (int i = 0; i < image.rows; i++)
                            {
                                uchar *ptr = image.ptr<uchar>(i);
                                uchar *pBuffer = (uchar*)(LockedRect.pBits) + i * LockedRect.Pitch;
                                for (int j = 0; j < image.cols; j++)
                                {
                                    ptr[3 * j] = pBuffer[4 * j];
                                    ptr[3 * j + 1] = pBuffer[4 * j + 1];
                                    ptr[3 * j + 2] = pBuffer[4 * j + 2];
                                }
                            }
                            imshow("colorImage", image + mark);
                            pTexture->UnlockRect(0);
                            cvWaitKey(33);
                            NuiImageStreamReleaseFrame(ColorFrameStream, pImageFrame);
                        //
                        pastFrame = nowFrame;
                        hr = NuiSkeletonGetNextFrame(0, &nowFrame);
                        NuiTransformSmooth(&nowFrame, NULL);

                        Mat skeletonimg(240, 320, CV_8UC1, Scalar(0));
                        Vector4 cpoint[20] = { 0 };
                        CvPoint cvp[20];
                        for (int j = 0;j < 20;j++)
                        {
                            cpoint[j].x = nowFrame.SkeletonData[userid].SkeletonPositions[j].x;
                            cpoint[j].y = nowFrame.SkeletonData[userid].SkeletonPositions[j].y;
                            cpoint[j].z = nowFrame.SkeletonData[userid].SkeletonPositions[j].z;
                        }
                        float fx, fy;
                        for (int i = 0;i < 20;i++)
                        {
                            NuiTransformSkeletonToDepthImage(cpoint[i], &fx, &fy);
                            cvp[i].x = (int)fx;
                            cvp[i].y = (int)fy;
                        }
                        drawSkeleton(skeletonimg, cvp,nowFrame,userid);
                        imshow("skeleton", skeletonimg);
                        process.setInput(pastFrame, nowFrame, frameCounter, userid);
                        frameCounter++;
                    }
                }
                process.processBegin();
                Mat mark1(image.size(), CV_8UC3, Scalar(0, 0, 0));
                process.classify(mark1);
                mark = mark1;
                imshow("colorImage", mark1);
                waitKey(33);
                break;
            }
        }
    }
    NuiShutdown();
    return 0;
}

kinect对帧的处理

kinect每读入一帧骨骼数据，消除抖动后，与上一帧做差，得到这一帧里发生的位移向量，存入Mat framess(NUM_OF_FRAME,NUM_OF_SKELETONS,CV_32FC3)矩阵中。

与模板的向量余弦计算

获得了NUM_OF_FRAME帧数据，即framess写满了后，计算framess中每一帧的位移向量分别与模板的各帧位移向量的夹角余弦。当前程序里一批读入120帧，framess是120*10*3的结构，匹配用的模板计算后也是120*10*3的结构，两个结构按行求夹角余弦，即30维向量与30维向量的运算。得到的余弦值放入用于动态时间规划的120*120的矩阵中。

根据动态时间规划法匹配

因为对于两个相同的动作，如果他们存在相位差，且无法判断用以匹配的动作的起始点在哪里时，对动态时间规划的矩阵求其各条斜率为1的直线上的矩阵元素的和。要注意这个方法只适合用一对一对应的动态时间规划。

2. 记录并保存模板到本地

//@这个程序就是随便写的专门用来采集模板动作的程序
#include <afxsock.h>
#include<opencv.hpp>
#include<Windows.h>
#include<NuiApi.h>
using namespace std;
using namespace cv;

#define NUM_OF_SKELETONS 10
#define NUM_OF_FRAMES 121 //文件操作里的帧数必须比识别中的多一
void drawSkeleton(Mat &image, CvPoint pointSet[])
{
    CvScalar color;
    color = cvScalar(255);
    if ((pointSet[NUI_SKELETON_POSITION_HEAD].x != 0 || pointSet[NUI_SKELETON_POSITION_HEAD].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_HEAD], pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_SPINE].x != 0 || pointSet[NUI_SKELETON_POSITION_SPINE].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER], pointSet[NUI_SKELETON_POSITION_SPINE], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_SPINE].x != 0 || pointSet[NUI_SKELETON_POSITION_SPINE].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_HIP_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_CENTER].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_SPINE], pointSet[NUI_SKELETON_POSITION_HIP_CENTER], color, 2);

    //左上肢   
    if ((pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_SHOULDER_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER], pointSet[NUI_SKELETON_POSITION_SHOULDER_LEFT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_SHOULDER_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_LEFT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_ELBOW_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_ELBOW_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_SHOULDER_LEFT], pointSet[NUI_SKELETON_POSITION_ELBOW_LEFT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_ELBOW_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_ELBOW_LEFT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_WRIST_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_WRIST_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_ELBOW_LEFT], pointSet[NUI_SKELETON_POSITION_WRIST_LEFT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_WRIST_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_WRIST_LEFT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_HAND_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_HAND_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_WRIST_LEFT], pointSet[NUI_SKELETON_POSITION_HAND_LEFT], color, 2);

    //右上肢   
    if ((pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_SHOULDER_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_SHOULDER_CENTER], pointSet[NUI_SKELETON_POSITION_SHOULDER_RIGHT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_SHOULDER_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_SHOULDER_RIGHT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_ELBOW_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_ELBOW_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_SHOULDER_RIGHT], pointSet[NUI_SKELETON_POSITION_ELBOW_RIGHT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_ELBOW_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_ELBOW_RIGHT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_WRIST_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_WRIST_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_ELBOW_RIGHT], pointSet[NUI_SKELETON_POSITION_WRIST_RIGHT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_WRIST_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_WRIST_RIGHT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_HAND_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_HAND_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_WRIST_RIGHT], pointSet[NUI_SKELETON_POSITION_HAND_RIGHT], color, 2);

    //左下肢   
    if ((pointSet[NUI_SKELETON_POSITION_HIP_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_CENTER].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_HIP_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_HIP_CENTER], pointSet[NUI_SKELETON_POSITION_HIP_LEFT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_HIP_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_LEFT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_KNEE_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_KNEE_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_HIP_LEFT], pointSet[NUI_SKELETON_POSITION_KNEE_LEFT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_KNEE_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_KNEE_LEFT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_ANKLE_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_ANKLE_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_KNEE_LEFT], pointSet[NUI_SKELETON_POSITION_ANKLE_LEFT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_ANKLE_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_ANKLE_LEFT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_FOOT_LEFT].x != 0 || pointSet[NUI_SKELETON_POSITION_FOOT_LEFT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_ANKLE_LEFT], pointSet[NUI_SKELETON_POSITION_FOOT_LEFT], color, 2);

    //右下肢   
    if ((pointSet[NUI_SKELETON_POSITION_HIP_CENTER].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_CENTER].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_HIP_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_HIP_CENTER], pointSet[NUI_SKELETON_POSITION_HIP_RIGHT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_HIP_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_HIP_RIGHT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_KNEE_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_KNEE_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_HIP_RIGHT], pointSet[NUI_SKELETON_POSITION_KNEE_RIGHT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_KNEE_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_KNEE_RIGHT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_ANKLE_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_ANKLE_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_KNEE_RIGHT], pointSet[NUI_SKELETON_POSITION_ANKLE_RIGHT], color, 2);
    if ((pointSet[NUI_SKELETON_POSITION_ANKLE_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_ANKLE_RIGHT].y != 0) &&
        (pointSet[NUI_SKELETON_POSITION_FOOT_RIGHT].x != 0 || pointSet[NUI_SKELETON_POSITION_FOOT_RIGHT].y != 0))
        line(image, pointSet[NUI_SKELETON_POSITION_ANKLE_RIGHT], pointSet[NUI_SKELETON_POSITION_FOOT_RIGHT], color, 2);
}
string int2str(int _intval)
{
    string time_str;
    char c[10];
    sprintf_s(c, "%d", _intval);             
    time_str = c;                                                 
    return c;
}

int main()
{
    Mat image(480, 640, CV_8UC3, Scalar(0, 0, 0));
    //
    HRESULT hr = NuiInitialize(NUI_INITIALIZE_FLAG_USES_SKELETON | NUI_INITIALIZE_FLAG_USES_COLOR);
    HANDLE skeletonEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
    hr = NuiSkeletonTrackingEnable(skeletonEvent, NUI_SKELETON_TRACKING_FLAG_ENABLE_SEATED_SUPPORT);
    int userid = 0;
    if (FAILED(hr))
    {
        cout << "Could not open skeleton stream video" << endl;
        NuiShutdown();
        return hr;
    }
    //
    HANDLE ColorFrameStream = NULL;
    HANDLE NextFrameEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
    hr = NuiImageStreamOpen(NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, 0, 2, NextFrameEvent, &ColorFrameStream);
    if (FAILED(hr))
    {
        cout << "Could not open image stream video" << endl;
        return hr;
    }
    //
    NUI_SKELETON_FRAME nowFrame = { 0 };
    bool bFoundSkeleton = FALSE;
    bool getdata = FALSE;
    int counter = 0;
    Mat temp(NUM_OF_FRAMES, NUM_OF_SKELETONS, CV_32FC3,Scalar(0,0,0));
    //
    FileStorage fs("monitor_file.xml", FileStorage::WRITE);
    fs.release();
    //
    while (!bFoundSkeleton)
    {
        if (WaitForSingleObject(skeletonEvent, INFINITE) == 0)
        {
            hr = NuiSkeletonGetNextFrame(0, &nowFrame);
            for (int i=0;i < NUI_SKELETON_COUNT;i++)
            {
                NUI_SKELETON_TRACKING_STATE trackingState = nowFrame.SkeletonData[i].eTrackingState;
                if (trackingState == NUI_SKELETON_TRACKED)
                {
                    bFoundSkeleton = TRUE;
                    userid = i;
                }
            }
        }
    }
    imshow("colorImage", Mat(480, 640, CV_8UC3, Scalar(64, 255, 255)));
    waitKey(1000);
    while (!getdata)
    { 
        int frameCounter = 0;
        for (;userid < NUI_SKELETON_COUNT;userid++)
        {
            if (nowFrame.SkeletonData[userid].eTrackingState == NUI_SKELETON_TRACKED &&
                nowFrame.SkeletonData[userid].eSkeletonPositionTrackingState[NUI_SKELETON_POSITION_SHOULDER_CENTER] != NUI_SKELETON_POSITION_NOT_TRACKED)
            {
                while (frameCounter < NUM_OF_FRAMES)
                {
                    //
                    const NUI_IMAGE_FRAME * pImageFrame = NULL;
                    if (WaitForSingleObject(skeletonEvent, INFINITE) == 0)
                    {
                        NuiImageStreamGetNextFrame(ColorFrameStream, 0, &pImageFrame);
                        INuiFrameTexture * pTexture = pImageFrame->pFrameTexture;
                        NUI_LOCKED_RECT LockedRect;
                        pTexture->LockRect(0, &LockedRect, NULL, 0);
                        for (int i = 0; i < image.rows; i++)
                        {
                            uchar *ptr = image.ptr<uchar>(i);
                            uchar *pBuffer = (uchar*)(LockedRect.pBits) + i * LockedRect.Pitch;
                            for (int j = 0; j < image.cols; j++)
                            {
                                ptr[3 * j] = pBuffer[4 * j];
                                ptr[3 * j + 1] = pBuffer[4 * j + 1];
                                ptr[3 * j + 2] = pBuffer[4 * j + 2];
                            }
                        }
                        imshow("colorImage", image);
                        pTexture->UnlockRect(0);
                        NuiImageStreamReleaseFrame(ColorFrameStream, pImageFrame);
                        cvWaitKey(33);

                        //
                        hr = NuiSkeletonGetNextFrame(0, &nowFrame);
                        NuiTransformSmooth(&nowFrame, 0);
                        Mat skeletonimg(240, 320, CV_8UC1, Scalar(0));//接下来kinect读取到的骨骼数据转换为深度空间图像后保存在image中
                        Vector4 cpoint[20] = { 0 };
                        CvPoint cvp[20];
                        for (int j = 0;j < 20;j++)//kinect一共获取20个关节的空间坐标
                        {
                            cpoint[j].x = nowFrame.SkeletonData[userid].SkeletonPositions[j].x;
                            cpoint[j].y = nowFrame.SkeletonData[userid].SkeletonPositions[j].y;
                            cpoint[j].z = nowFrame.SkeletonData[userid].SkeletonPositions[j].z;
                        }
                        float fx, fy;
                        for (int i = 0;i < 20;i++)
                        {
                            NuiTransformSkeletonToDepthImage(cpoint[i], &fx, &fy);
                            cvp[i].x = (int)fx;
                            cvp[i].y = (int)fy;
                        }
                        drawSkeleton(skeletonimg, cvp);//使用函数drawSkeleton将关节的点连成线
                        imshow("skeleton", skeletonimg);
                        if (FAILED(hr))
                        {
                            cout << "Could not get next skeleton stream " << endl;
                            return hr;
                        }
                        for (int j = 0;j < NUM_OF_SKELETONS;j++)
                        {
                            temp.ptr<float>(frameCounter)[3*j] = nowFrame.SkeletonData[userid].SkeletonPositions[j].x;
                            temp.ptr<float>(frameCounter)[3 * j+1] = nowFrame.SkeletonData[userid].SkeletonPositions[j].y;
                            temp.ptr<float>(frameCounter)[3 * j+2] = nowFrame.SkeletonData[userid].SkeletonPositions[j].z;
                            //cout << "model" + int2str(counter) << " frame" + int2str(frameCounter) << "data is: " << nowFrame.SkeletonData[userid].SkeletonPositions[j].x << nowFrame.SkeletonData[userid].SkeletonPositions[j].y << nowFrame.SkeletonData[userid].SkeletonPositions[j].z << endl;
                        }
                    }
                    frameCounter++;
                }
                FileStorage fs("monitor_file.xml", FileStorage::APPEND);
                fs << "mat_id" + int2str(counter) << temp;
                fs.release();
                counter++;
                break;
            }
        }
        if (counter == 3)
            getdata = TRUE;
        imshow("colorImage", Mat(480, 640, CV_8UC3, Scalar(255, 64, 255)));
        waitKey(1000);
    }
    NuiShutdown();
    return 0;
}

使用opencv的FileStorage类生成xml文件

值得一提的是如果直接用WRITE方式连续写三个数据进去，比如：
fs<<”mat_”<< image1<< image2<< image3;
那么读取的时候出现打开了xml文件却读不到任何数据的情况，所以我在外部先把xml用write方式打开，初始化掉，在循环里用append的方式追加写入。

KINECT+opencv基于骨骼信息对视频进行动作识别

标签：

原文地址：http://blog.csdn.net/daky_u/article/details/51405913

踩

(0)

评论一句话评论（0）

分享档案

更多>

2021年07月29日 (22)
2021年07月28日 (40)
2021年07月27日 (32)
2021年07月26日 (79)
2021年07月23日 (29)
2021年07月22日 (30)
2021年07月21日 (42)
2021年07月20日 (16)
2021年07月19日 (90)
2021年07月16日 (35)

周排行

KINECT+opencv基于骨骼信息对视频进行动作识别

KINECT+opencv基于骨骼信息对视频进行动作识别

目录

写在前面

1.对当前帧处理并匹配

kinect对帧的处理

与模板的向量余弦计算

根据动态时间规划法匹配

2. 记录并保存模板到本地

使用opencv的FileStorage类生成xml文件