针对于tinynet进行了三种数据传输方式的测试,包括最初的byte stream,protobuf,以及比较流行json方式。跟之前的几次测试一样,模型为echo模型,都是以epoll为例,每个连接每秒发送10个包,每个数据包约100bytes,数据包括包头以及数据,包头12bytes,包括长度以及其他8bytes信息,客户端连接为每秒5个,此次测试总计连接10000个.
之前的测试报告:
工程源代码:
服务器是多线程模型,主线程负责接收处理网络数据,并添加的输入缓冲区中;读线程从输入缓冲中读取数据并进行处理,将处理的数据添加的输出缓冲区中;写线程将输出缓冲区的数据发送到指定的客户端连接。由于数据处理只是在读线程处理,所以只需要变更读线程的处理代码就可以了。( 默认读写线程的sleep时间为500ms)
下面分别为服务器端读线程byte stream,protobuf,json三种方式的部分代码:
- // byte stream
- void Server_Impl::_read_thread()
- {
- static const int __head_size = 12;
- char __read_buf[max_buffer_size_] = {};
- while (true)
- {
- lock_.acquire_lock();
- #ifdef __DEBUG
- struct timeval __start_timeval;
- gettimeofday(&__start_timeval, NULL);
- long __start_time = __start_timeval.tv_usec;
- #endif //__DEBUG
- for (std::vector<Buffer*>::iterator __it = connects_copy.begin(); __it != connects_copy.end(); ++__it)
- {
- if(*__it)
- {
- Buffer::ring_buffer* __input = (*__it)->input_;
- Buffer::ring_buffer* __output = (*__it)->output_;
- if (!__input || !__output)
- {
- continue;
- }
- #ifdef __DEBUG
- if(0 == (*__it)->fd_ % 1000)
- {
- printf("fd =%d,rpos = %d, wpos = %d\n",(*__it)->fd_,__input->rpos(),__input->wpos());
- }
- #endif //__DEBUG
- while (!__input->read_finish())
- {
- int __packet_length = 0;
- int __log_level = 0;
- int __frame_number = 0;
- unsigned char __packet_head[__head_size] = {};
- int __head = 0;
- unsigned int __guid = 0;
- if(!__input->pre_read((unsigned char*)&__packet_head,__head_size))
- {
- // not enough data for read
- break;
- }
- memcpy(&__packet_length,__packet_head,4);
- #if 0
- memcpy(&__head,__packet_head + 4,4);
- memcpy(&__guid,__packet_head + 8,4);
- #endif
- if(!__packet_length)
- {
- printf("__packet_length error\n");
- break;
- }
- #if 0
- __log_level = (__head) & 0x000000ff;
- __frame_number = (__head >> 8);
- #endif
- memset(__read_buf,0,max_buffer_size_);
- if(__input->read((unsigned char*)__read_buf,__packet_length + __head_size))
- {
- __output->append((unsigned char*)__read_buf,__packet_length + __head_size);
- }
- else
- {
- break;
- }
- }
- }
- }
- #ifdef __DEBUG
- struct timeval __end_timeval;
- gettimeofday(&__end_timeval, NULL);
- long __end_time = __end_timeval.tv_usec;
- long __time_read = __end_time - __start_time;
- printf("start time = %ld, end time = %ld,server impl time read = %ld\n",__start_time,__end_time,__time_read);
- #endif //__DEBUG
- lock_.release_lock();
- #ifdef __LINUX
- usleep(max_sleep_time_);
- #endif // __LINUX
- }
- }
- // protobuf
- void Server_Impl::_read_thread()
- {
- transfer::Packet __packet_protobuf;
- std::string __string_packet;
- static const int __head_size = 4;
- while (true)
- {
- lock_.acquire_lock();
- #ifdef __DEBUG
- struct timeval __start_timeval;
- gettimeofday(&__start_timeval, NULL);
- long __start_time = __start_timeval.tv_usec;
- #endif //__DEBUG
- for (std::vector<Buffer*>::iterator __it = connects_copy.begin(); __it != connects_copy.end(); ++__it)
- {
- if(*__it)
- {
- Buffer::ring_buffer* __input = (*__it)->input_;
- Buffer::ring_buffer* __output = (*__it)->output_;
- if (!__input || !__output)
- {
- continue;
- }
- #ifdef __DEBUG
- if(0 == (*__it)->fd_ % 1000)
- {
- printf("fd =%d,rpos = %d, wpos = %d\n",(*__it)->fd_,__input->rpos(),__input->wpos());
- }
- #endif //__DEBUG
- while (!__input->read_finish())
- {
- int __packet_length = 0;
- int __log_level = 0;
- int __frame_number = 0;
- unsigned char __packet_head[__head_size] = {};
- int __head = 0;
- unsigned int __guid = 0;
- if(!__input->pre_read(__packet_head,__head_size))
- {
- // not enough data for read
- break;
- }
- __packet_length = (int)*__packet_head;
- if(!__packet_length)
- {
- printf("__packet_length error\n");
- break;
- }
- #if 0
- // easy_bool read(easy_uint8* des,size_t len)
- char __read_buf[max_buffer_size_] = {};
- if(__input->read((unsigned char*)__read_buf,__packet_length + __head_size))
- {
- __string_packet = __read_buf + __head_size;
- __packet_protobuf.ParseFromString(__string_packet);
- __output->append((unsigned char*)__read_buf,__packet_length + __head_size);
- #else
- // easy_bool read(std::string& des,size_t len)
- __packet_protobuf.Clear();
- __string_packet.clear();
- if(__input->read(__string_packet,__packet_length + __head_size))
- {
- __packet_protobuf.ParseFromString(__string_packet.c_str() + __head_size);
- __output->append((unsigned char*)__string_packet.c_str(),__packet_length + __head_size);
- #endif
- #ifdef __DEBUG
- printf("__packet_protobuf.head = %d\n",__packet_protobuf.head());
- printf("__packet_protobuf.guid = %d\n",__packet_protobuf.guid());
- printf("__packet_protobuf.content = %s\n",__packet_protobuf.content().c_str());
- #endif //__DEBUG
- }
- else
- {
- break;
- }
- }
- }
- }
- #ifdef __DEBUG
- struct timeval __end_timeval;
- gettimeofday(&__end_timeval, NULL);
- long __end_time = __end_timeval.tv_usec;
- long __time_read = __end_time - __start_time;
- printf("start time = %ld, end time = %ld,server protobuf impl time read = %ld\n",__start_time,__end_time,__time_read);
- #endif //__DEBUG
- lock_.release_lock();
- #ifdef __LINUX
- usleep(max_sleep_time_);
- #endif // __LINUX
- }
- }
- // json
- void Server_Impl::_read_thread()
- {
- static const int __head_size = 4;
- while (true)
- {
- lock_.acquire_lock();
- #ifdef __DEBUG
- struct timeval __start_timeval;
- gettimeofday(&__start_timeval, NULL);
- long __start_time = __start_timeval.tv_usec;
- #endif //__DEBUG
- for (std::vector<Buffer*>::iterator __it = connects_copy.begin(); __it != connects_copy.end(); ++__it)
- {
- if(*__it)
- {
- Buffer::ring_buffer* __input = (*__it)->input_;
- Buffer::ring_buffer* __output = (*__it)->output_;
- if (!__input || !__output)
- {
- continue;
- }
- #ifdef __DEBUG
- if(0 == (*__it)->fd_ % 1000)
- {
- printf("fd =%d,rpos = %d, wpos = %d\n",(*__it)->fd_,__input->rpos(),__input->wpos());
- }
- #endif //__DEBUG
- while (!__input->read_finish())
- {
- int __packet_length = 0;
- int __log_level = 0;
- int __frame_number = 0;
- unsigned char __packet_head[__head_size] = {};
- int __head = 0;
- unsigned int __guid = 0;
- if(!__input->pre_read(__packet_head,__head_size))
- {
- // not enough data for read
- break;
- }
- __packet_length = (int)*__packet_head;
- if(!__packet_length)
- {
- printf("__packet_length error\n");
- break;
- }
- char __read_buf[max_buffer_size_] = {};
- if(__input->read((unsigned char*)__read_buf,__packet_length + __head_size))
- {
- json_error_t* __json_error = NULL;
- json_t* __json_loads = json_loads(__read_buf + __head_size,JSON_DECODE_ANY,__json_error);
- __output->append((unsigned char*)__read_buf,__packet_length + __head_size);
- #ifdef __DEBUG
- json_t* __json_loads_head = json_object_get(__json_loads,"head");
- json_t* __json_loads_guid = json_object_get(__json_loads,"guid");
- json_t* __json_loads_content = json_object_get(__json_loads,"content");
- printf("json.head = %d\n",json_integer_value(__json_loads_head));
- printf("json.guid = %d\n",json_integer_value(__json_loads_guid));
- printf("json.content = %s\n",json_string_value(__json_loads_content));
- #endif //__DEBUG
- json_decref(__json_loads);
- }
- else
- {
- break;
- }
- }
- }
- }
- #ifdef __DEBUG
- struct timeval __end_timeval;
- gettimeofday(&__end_timeval, NULL);
- long __end_time = __end_timeval.tv_usec;
- long __time_read = __end_time - __start_time;
- printf("start time = %ld, end time = %ld,server json impl time read = %ld\n",__start_time,__end_time,__time_read);
- #endif //__DEBUG
- lock_.release_lock();
- #ifdef __LINUX
- usleep(max_sleep_time_);
- #endif // __LINUX
- }
- }
nmon监控部分结果:
sys summ
byte stream
protobuf
json
cpu summ
byte stream
protobuf
json
memory
net
netpackets
byte stream
protobuf
json
我们主要通过cpu,内存,网络流量,网络发包数量来分析:
(1) 就cpu消耗来说,使用byte stream所消耗的cpu要比使用protobuf,json要小的多,主要是因为后者需要额外的创建对象,序列化,还原的开销;
(2) 内存变化都不太大,基本保持稳定;
(3) 就网络流量来说,byte stream使用流量比较小,也比较恒定,平均流量在5M左右,而protobuf,json方式平均值都在20M左右,而且前者每秒处理的数据包约为30000个,而后两者在150000~200000之间。这看起来比较奇怪,客户端都是每秒发送10个包,阻塞,服务器读写线程sleep的时间都是500ms,那么正常来说,如果按服务器的处理来看,每秒处理约2个数据包,客户端时间间隔再小,也是如此,这还是都能完全处理的情况下,那么,byte stream的结果应该更加接近事实。但是后两者与前者为何相差这么大呢?
带着疑问,进行了下面的测试,服务器相关代码只是更改了读线程部分,所以我在线程的每次tick前后加上时间戳,记录一次tick所花费的时间,并记录输入缓冲区读写的位置,因为连接数比较多,日志太多,而且刷屏太快,容易影响测试结果,仅仅采样fd为1000的倍数,下面是结果的截图:
由上述结果可知:byte stream每次tick时间为10+毫秒,而且输入缓冲区的r/w pos基本是一致的,也就是说数据每次都处理完毕;而protobuf每次tick时间为100+毫秒,r/w pos的位置差在600~700bytes左右; 使用json方式,每次tick时间多达数百毫秒,且r/w pos的位置差在2000~3000bytes之间。从这组数据基本可以确定,在传输效率上来说:byte stream > protobuf > json,其差距差不多在几倍到一个数量级之间。
为了进一步验证,又做了一组测试,对一个模拟的数据包分别按三种方式进行打包,解包操作,执行10000次,查看执行的总时间:
- /****************************************************************************
- Copyright (c) 2013-2014 King Lee
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
- general:
- $export LD_LIBRARY_PATH=$LD_LIBRARY_PATH../easy/dep/protobuf/src/.libs:../easy/dep/jansson/src/.libs
- $../easy/dep/protobuf/src/.libs/protoc -I./ --cpp_out=. transfer.proto
-
- compiler:
- $g++ -g -o byte_stream_vs_protobuf_vs_json transfer.pb.h transfer.pb.cc byte_stream_vs_protobuf_vs_json.cc -I../easy/dep/protobuf/src/ -I../easy/dep/jansson/src/ -L../easy/dep/protobuf/src/.libs -L../easy/dep/jansson/src/.libs -lprotobuf -ljansson
- ****************************************************************************/
- #include <stdlib.h> // exit
- #include <stdio.h>
- #include <stdarg.h>
- #include <string>
- #include <sys/time.h>
- #include <string.h>
- #include "transfer.pb.h"
- #include <jansson.h>
- static std::string __random_string[] =
- {
- "[0x000085e4][T]AdvertisingIndentitifer: ‘‘, IdentifierForVendor: ‘‘, DeviceName: ‘PC‘, ModelName: ‘x86‘, SystemName: ‘‘, SystemVersion: ‘‘, HardwareID: ‘74d435046509‘",
- "nice to meet you!",
- "It is the tears of the earth that keep here smiles in bloom.",
- "The mighty desert is burning for the love of a blade of grass who shakes her head and laughs and flies away.",
- "If you shed tears when you miss the sun, you also miss the stars.",
- "Her wishful face haunts my dreams like the rain at night.",
- "Once we dreamt that we were strangers.We wake up to find that we were dear to each other.",
- "Sorrow is hushed into peace in my heart like the evening among the silent trees.",
- "Some unseen fingers, like an idle breeze, are playing upon my heart the music of the ripples.",
- "Listen, my heart, to the whispers of the world with which it makes love to you.",
- "Do not seat your love upon a precipice because it is high.",
- "I sit at my window this morning where the world like a passer-by stops for a moment, nods to me and goes.",
- "There little thoughts are the rustle of leaves; they have their whisper of joy in my mind.",
- "What you are you do not see, what you see is your shadow.",
- "My wishes are fools, they shout across thy song, my Master.Let me but listen.",
- "They throw their shadows before them who carry their lantern on their back.",
- "That I exist is a perpetual surprise which is life.",
- "We, the rustling leaves, have a voice that answers the storms,but who are you so silent?I am a mere flower.",
- "Do not blame your food because you have no appetite.",
- "Success is not final, failure is not fatal: it is the courage to continue that counts.",
- "I cannot tell why this heart languishes in silence.It is for small needs it never asks, or knows or remembers.",
- "The bird wishes it were a cloud.The cloud wishes it were a bird."
- };
- static const int __loop_count = 10000;
- static int __buf_size = 256;
- void byte_stream_test()
- {
- int __guid = 15;
- int __head = 567;
- int __length2 = 0;
- int __head2 = 0;
- int __guid2 = 0;
- char __buf[__buf_size];
- int __packet_head_size = 12;
- int __random_index = 0;
- unsigned char __packet_head[__packet_head_size];
- int __length = __random_string[__random_index].size();
-
- struct timeval __start_timeval;
- gettimeofday(&__start_timeval, NULL);
- long __start_time = __start_timeval.tv_usec;
- for(int __i = 0; __i < __loop_count; ++__i)
- {
- memset(__buf,0,__buf_size);
- memset(__packet_head,0,__packet_head_size);
- // pack
- memcpy(__buf,(void*)&__length,4);
- memcpy(__buf + 4,(void*)&__head,4);
- memcpy(__buf + 8,(void*)&__guid,4);
- strcpy(__buf + __packet_head_size,__random_string[__random_index].c_str());
- // unpack
- memcpy(&__length2,(void*)__packet_head,4);
- memcpy(&__head2,(void*)(__packet_head + 4),4);
- memcpy(&__guid2,(void*)(__packet_head + 8),4);
- }
- struct timeval __end_timeval;
- gettimeofday(&__end_timeval, NULL);
- long __end_time = __end_timeval.tv_usec;
- long __time_total = __end_time - __start_time;
- printf("start time = %ld, end time = %ld,byte stream total time = %ld\n",__start_time,__end_time,__time_total);
- }
- void protobuf_test()
- {
- transfer::Packet __packet_protobuf;
- std::string __string_packet;
- int __guid = 15;
- int __head = 567;
- int __length2 = 0;
- int __head2 = 0;
- int __guid2 = 0;
- char __buf[__buf_size];
- int __packet_head_size = 4;
- int __random_index = 0;
- unsigned char __packet_head[__packet_head_size];
-
- struct timeval __start_timeval;
- gettimeofday(&__start_timeval, NULL);
- long __start_time = __start_timeval.tv_usec;
- for(int __i = 0; __i < __loop_count; ++__i)
- {
- memset(__buf,0,__buf_size);
- // pack
- __packet_protobuf.Clear();
- __string_packet.clear();
- __packet_protobuf.set_head(__head);
- __packet_protobuf.set_guid(__guid);
- __packet_protobuf.set_content(__random_string[__random_index]);
- __packet_protobuf.SerializeToString(&__string_packet);
- int __length = __string_packet.length();
- memcpy(__buf,(void*)&__length,4);
- // unpack
- __packet_protobuf.Clear();
- __packet_protobuf.ParseFromString(__string_packet);
- __head2 = __packet_protobuf.head();
- __guid2 = __packet_protobuf.guid();
- }
- struct timeval __end_timeval;
- gettimeofday(&__end_timeval, NULL);
- long __end_time = __end_timeval.tv_usec;
- long __time_total = __end_time - __start_time;
- printf("start time = %ld, end time = %ld,protobuf total time = %ld\n",__start_time,__end_time,__time_total);
- }
- void json_test()
- {
- int __guid = 15;
- int __head = 567;
- int __length2 = 0;
- int __head2 = 0;
- int __guid2 = 0;
- char __buf[__buf_size];
- int __random_index = 0;
-
- struct timeval __start_timeval;
- gettimeofday(&__start_timeval, NULL);
- long __start_time = __start_timeval.tv_usec;
- for(int __i = 0; __i < __loop_count; ++__i)
- {
- // pack
- json_t* __json_msg = json_object();
- json_t* __json_head = json_integer(__head);
- json_t* __json_guid = json_integer(__guid);
- json_t* __json_content = json_string(__random_string[__random_index].c_str());
- json_object_set(__json_msg, "head", __json_head);
- json_object_set(__json_msg, "guid", __json_guid);
- json_object_set(__json_msg, "content", __json_content);
- char* __json_dumps_string = json_dumps(__json_msg,0);
- int __length = strlen(__json_dumps_string);
- json_decref(__json_head);
- json_decref(__json_guid);
- json_decref(__json_content);
- json_decref(__json_msg);
-
- // unpack
- json_error_t* __json_error = NULL;
- json_t* __json_loads = json_loads(__json_dumps_string,JSON_DECODE_ANY,__json_error);
- json_t* __json_loads_head = json_object_get(__json_loads,"head");
- json_t* __json_loads_guid = json_object_get(__json_loads,"guid");
- __head2 = json_integer_value(__json_loads_head);
- __guid2 = json_integer_value(__json_loads_guid);
- json_decref(__json_loads);
- free(__json_dumps_string);
- }
- struct timeval __end_timeval;
- gettimeofday(&__end_timeval, NULL);
- long __end_time = __end_timeval.tv_usec;
- long __time_total = __end_time - __start_time;
- printf("start time = %ld, end time = %ld,json total time = %ld\n",__start_time,__end_time,__time_total);
- }
- int main(int __arg_num, char** __args)
- {
- byte_stream_test();
- protobuf_test();
- json_test();
- }
其结果与上面的推断基本一致.
其他疑问:使用protobuf,json的方式,读线程的每个tick时间较长,意味着处理的数据可能更少,但是为什么带宽和处理数据包更多了,且接近于满值。目前想到的可能性是使用byte stream的方式服务器基本能及时处理客户端的请求,大部分时间都在sleep了,因此cpu使用量也比较低。(也额外做过测试,将服务器线程的sleep时间更改为10毫秒,这样带宽,以及数据包数量也与后两者相近了,是不是原因就如此呢?)