使用locust对设备ID的生成逻辑的并发测试初步实践

标签：locust   压测   分布式测试   

1、同A，不管后面的B,C是否不一致
首先根据传入消息的A ，到ES中查询 如果存在相同的则用原A的设备ID(DeviceID)
2、不同A
1）B一样，C不一样，则用原B的设备ID(DeviceID)
2）C一样,B不一样，则用原C的设备ID(DeviceID)
3）B不一样，C不一样，则生成新设备ID(DeviceID)，生成新的设备ID的逻是原来的最大的设备ID号，加1最为新的设备ID，例如：原数据中最大设备ID是D000000009，则新的设备ID是D000000010.

设备ID的生成在数据量大并发的情况下是非常有可能是同一时间过来的，所以，当开启脚本后，即使是A、B、C因子都不一样的时候，因为生成时间是相同的（精确到毫秒级），所以生成的设备ID还是相同的（实际情况下是不可能生成相同设备的），此问题的解决方案有很多，可以搜索，分布式ID的生成，我们开发采取的方案是使用redis的原子性，ID生成器解决的。主要的locust脚本如下：

#!/user/bin/env python
#coding=utf-8
author = ‘zengweifang‘

#风控系统的压测脚本
#第一步：模拟设备指纹的100万数据的生成

from locust import HttpLocust,TaskSet,task
import queue
import json
import requests
from random import Random
import random
import time
import string
import socket
import struct

RockMQ = "http://192.168.46.154:8765/topic/sendTopicMessage.do"

def AutoGeneratedString(number):
‘‘‘随机生成字符串方法，主要用于输入框不能超过多少字符串的场景,此一次性产生的最大的字符串是62个‘‘‘
return ‘‘.join(random.sample(string.ascii_letters + string.digits, number))

def getRandomBool():
‘‘‘随机获取是否是代理‘‘‘
isProxy = ["true","false"]
return random.sample(isProxy, 1)[0]

def get_random_ip():
‘‘‘随机生成合法的IP‘‘‘
RANDOM_IP_POOL=[‘192.168.10.222/0‘]
str_ip = RANDOM_IP_POOL[random.randint(0,len(RANDOM_IP_POOL) - 1)]
str_ip_addr = str_ip.split(‘/‘)[0]
str_ip_mask = str_ip.split(‘/‘)[1]
ip_addr = struct.unpack(‘>I‘,socket.inet_aton(str_ip_addr))[0]
mask = 0x0
for i in range(31, 31 - int(str_ip_mask), -1):
mask = mask | ( 1 << i)
ip_addr_min = ip_addr & (mask & 0xffffffff)
ip_addr_max = ip_addr | (~mask & 0xffffffff)
return socket.inet_ntoa(struct.pack(‘>I‘, random.randint(ip_addr_min, ip_addr_max)))

def getUnixTime():
‘‘‘获得13位的unix时间戳:1519800472673‘‘‘
return int(round(time.time() * 1000))

def getPCUserAgent():
‘‘‘获取PC端的UserAgent‘‘‘
pc_user_agent = [
‘Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50‘,
‘Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50‘,
‘Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0‘,
‘Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0‘,
‘Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0‘,
‘Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1‘,
‘Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)‘,
‘Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko‘,
‘Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv,2.0.1) Gecko/20100101 Firefox/4.0.1‘,
‘Mozilla/5.0 (Windows NT 6.1; rv,2.0.1) Gecko/20100101 Firefox/4.0.1‘,
‘Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11‘,
‘Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11‘,
‘Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11‘,
‘Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)‘,
‘Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)‘,
‘Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)‘,
‘Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)‘,
‘Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)‘,
‘Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)‘,
‘Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)‘,
‘Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)‘]
list(set(pc_user_agent))
return random.sample(pc_user_agent, 1)[0]

def messageDict(fingerId, sessionId):

topic = {"topic": "meta-fingerprint"}
tag = {"tag": ""}
key = {"key": ""}

appId = {"appId": ‘A10007‘}
clientType = {"clientType": "1"}
fingerId = {"fingerId": fingerId}
ip = {"ip": get_random_ip()}
proxyFlag = {"proxyFlag": getRandomBool()}
sessionId = {"sessionId": sessionId}
systemId = {"systemId": ‘7b6a99f3bce14915863cde5104bdf2c3‘}
timestamp = {"timestamp": str(getUnixTime())}

user_agent = ‘{\"key\":\"user_agent\",\"value\":\"%s\"}‘ % getPCUserAgent()
language = ‘{\"key\":\"language\",\"value\":\"zh-cn\"}‘
color_depth = ‘{\"key\":\"color_depth\",\"value\":32}‘
device_memory = ‘{\"key\":\"device_memory\",\"value\":-1}‘
pixel_ratio = ‘{\"key\":\"pixel_ratio\",\"value\":\"\"}‘
hardware_concurrency = ‘{\"key\":\"hardware_concurrency\",\"value\":\"unknown\"}‘
resolution = ‘{\"key\":\"resolution\",\"value\":[1920,1080]}‘
available_resolution = ‘{\"key\":\"available_resolution\",\"value\":[1920,1040]}‘
timezone_offset = ‘{\"key\":\"timezone_offset\",\"value\":-480}‘
session_storage = ‘{\"key\":\"session_storage\",\"value\":1}‘
local_storage = ‘{\"key\":\"local_storage\",\"value\":1}‘
add_behavior = ‘{\"key\":\"add_behavior\",\"value\":1}‘
cpu_class = ‘{\"key\":\"cpu_class\",\"value\":\"x86\"}‘
navigator_platform = ‘{\"key\":\"navigator_platform\",\"value\":\"Win32\"}‘
do_not_track = ‘{\"key\":\"do_not_track\",\"value\":\"unknown\"}‘
ie_plugins = ‘{\"key\":\"ie_plugins\",\"value\":[\"AcroPDF.PDF\",null,null,null,\"MacromediaFlashPaper.MacromediaFlashPaper\",\"Msxml2.DOMDocument\",\"Msxml2.XMLHTTP\",null,null,null,null,null,null,\"Scripting.Dictionary\",null,\"Shell.UIHelper\",\"ShockwaveFlash.ShockwaveFlash\",null,\"TDCCtl.TDCCtl\",\"WMPlayer.OCX\",null,null]}‘
adblock = ‘{\"key\":\"adblock\",\"value\":false}‘
has_lied_languages = ‘{\"key\":\"has_lied_languages\",\"value\":false}‘
has_lied_resolution = ‘{\"key\":\"has_lied_resolution\",\"value\":false}‘
has_lied_os = ‘{\"key\":\"has_lied_os\",\"value\":false}‘
has_lied_browser = ‘{\"key\":\"has_lied_browser\",\"value\":false}‘
touch_support = ‘{\"key\":\"touch_support\",\"value\":[0,false,false]}‘
js_fonts = ‘{\"key\":\"js_fonts\",\"value\":[\"Arial\",\"Arial Black\",\"Calibri\",\"Cambria\",\"Cambria Math\",\"Comic Sans MS\",\"Consolas\",\"Courier\",\"Courier New\",\"Georgia\",\"Helvetica\",\"Impact\",\"Lucida Console\",\"Lucida Sans Unicode\",\"Microsoft Sans Serif\",\"MS Gothic\",\"MS PGothic\",\"MS Sans Serif\",\"MS Serif\",\"Palatino Linotype\",\"Segoe Print\",\"Segoe Script\",\"Segoe UI\",\"Segoe UI Light\",\"Segoe UI Semibold\",\"Segoe UI Symbol\",\"Tahoma\",\"Times\",\"Times New Roman\",\"Trebuchet MS\",\"Verdana\",\"Wingdings\"]}‘

fingerParametersList = [user_agent, language, color_depth, device_memory, pixel_ratio, hardware_concurrency,
                            resolution, available_resolution, timezone_offset,
                            session_storage, local_storage, add_behavior, cpu_class, navigator_platform,
                            do_not_track, ie_plugins, adblock, has_lied_languages,
                            has_lied_resolution, has_lied_os, has_lied_browser, touch_support, js_fonts]

fingerParameters = {"fingerParameters": str(fingerParametersList)}

{"messageBody":fingerParameters}
messageBodyDict = fingerParameters.copy()
messageBodyDict.update(clientType)
messageBodyDict.update(fingerId)
messageBodyDict.update(ip)
messageBodyDict.update(proxyFlag)
messageBodyDict.update(sessionId)
messageBodyDict.update(systemId)
messageBodyDict.update(timestamp)
messageBodyDict.update(appId)

messageBody = {"messageBody": str(messageBodyDict)}
    # print(messageBody)

NewmessageBody = messageBody.copy()
NewmessageBody.update(topic)
NewmessageBody.update(tag)
NewmessageBody.update(key)

return NewmessageBody

def toJson(message):
return json.dumps(message, ensure_ascii=False)

def send(url, jsonbody, header):
print(‘发送的消息体为:%s‘ % jsonbody)
re = requests.post(url=url, data=jsonbody, headers=header).json()
return re

def getHeader():
‘‘‘获取请求头‘‘‘
header = {"Content-Type": "application/json"}
return header

class rcpTaskSet(TaskSet):

@task(1)
def fingerprint(self):
    ‘‘‘生产设备指纹‘‘‘
    try:
        sessionId_data = self.locust.sessionId_queue.get()
        print(sessionId_data)
    except queue.Empty:
        print(‘sessionId data run out, test ended.‘)
        exit(0)

    try:
        fingerId_data = self.locust.fingerId_queue.get()
        print(fingerId_data)
    except queue.Empty:
        print(‘fingerId data run out, test ended.‘)
        exit(0)

    #self.messageDict(fingerId_data,sessionId_data)
    json1 = toJson(messageDict(fingerId_data,sessionId_data))
    send(RockMQ, json1, getHeader())

    #保证并发测试数据唯一性，循环取数据
    self.locust.sessionId_queue.put_nowait(sessionId_data)
    self.locust.fingerId_queue.put_nowait(fingerId_data)

    # payload = FG.messageDict( sessionId)
    # json1 = FG.toJson(payload)
    # self.send(self.RockMQ, json1, self.getHeader())

# @task
# def sendMQ(self):
#     ‘‘‘发送消息‘‘‘
#     pass

class rcpLocust(HttpLocust):

task_set = rcpTaskSet

sessionId_queue = queue.Queue()

for a in range(1000000):
    sessionId = AutoGeneratedString(32)
    sessionId_queue.put_nowait(sessionId)

fingerId_queue = queue.Queue()
for b in range(1000000):
    fingerId = AutoGeneratedString(32)
    fingerId_queue.put_nowait(fingerId)

min_wait = 1000
max_wait = 3000

使用locust对设备ID的生成逻辑的并发测试初步实践

标签：locust   压测   分布式测试   

原文地址：http://blog.51cto.com/11565901/2107163