码迷,mamicode.com
首页 > 其他好文 > 详细

iptv

时间:2019-01-03 13:58:25      阅读:152      评论:0      收藏:0      [点我收藏+]

标签:import   gid   track   json   from   file path   btv   serve   orm   

# -*- coding: utf-8 -*-
import datetime, time, json, re, os
#from pwd import getpwnam
#quality
str_quality = ‘‘.join([{StartTime}|||{SubscriberID},     |*11, {Bitrate}|{MOSAvg}|{DF}|{MLR}||||, {CPUUsageHistogram}|,     {ProfilePlayoutSecondsHistogram_M}, |*8, {ProfilePlayoutSecondsHistogram_m},     |*8, {PlayDuration_M}|{BadDuration_M}|{PlayDuration_S}|{BadDuration_S}||||,     {CurrentPlaySuccNum_CurrentPlayErrNum}|{CurrentPlaySuccNum}, |*11,     {TVGuideReqNum}|{TVGuideSuccessNum}|{TVGuideDelayAvg}|||||,     {StallingCount_S}|{StallingDuration_S}||,     {BufferTrack_A}|{BufferTrack_M}|{BufferTrack_m}|||,     {RTTTrack_A}|{RTTTrack_M}|{RTTTrack_m}|{LostRatioTrack_A}|{LostRatioTrack_M}|{LostRatioTrack_m},     |*7, {RAMUsageHistogram_A}|{RAMUsageHistogram_M}|{RAMUsageHistogram_m}|{DiskUsage},     |*6, {DownloadSpeed_A}|{DownloadSpeed_M}|{DownloadSpeed_m}||||,     {UpBandWidthHistogram_A}|{UpBandWidthHistogram_M}|{UpBandWidthHistogram_m}|,     {DownBandWidthHistogram_A}|{DownBandWidthHistogram_M}|{DownBandWidthHistogram_m}||,     {URL}|{ServiceType}|{URL_IP}||||, {DeviceSupplier}||||\n])
#behavior
str_behavior = ‘‘.join([{StartTime}|{SubscriberID},     |*7, {ServiceType}, |*9, {PlayDuration_M},     |*19, {URL}|{URL_IP}||||\n])
#CPU, RAM
c_r = [0.1, 0.35, 0.6, 0.75, 0.85, 0.95]
#up
up = [125, 375, 750, 1500, 3000, 5000]
#down, ProfilePlayoutSecondsHistogram
down = [1000, 3500, 7500, 15000, 35000, 60000]
pattern = re.compile(rhttp://(\d+.\d+.\d+.\d+)/.*)
#get the log file path
def get_log_path(path):
    for fpathe,dirs,fs in os.walk(path):
        for f in fs:
            if f.endswith(.log):
                return os.path.join(fpathe, f)
def get_his_MIN(lis):
    for x in lis:
        if x != 0:
            return lis.index(x)
    return 0
def get_his_MAX(lis):
    for x in lis[::-1]:
        if x != 0:
            return lis.index(x)
    return 0
#CPU, RAM, up down
def get_his_AVE_R_C(lis):
    s = int(100*(0.1*lis[0]+0.35*lis[1]+0.6*lis[2]+0.75*lis[3]+0.85*lis[4]+0.95*lis[5]))
    if sum(lis) != 0:
        return int (s/ sum(lis))
    else:
        return 0
def get_up_AVE(lis):
    s = int(125*lis[0]+375*lis[1]+750*lis[2]+1500*lis[3]+3000*lis[4]+5000*lis[5])
    if sum(lis) != 0:
        return int(s / sum(lis))
    else:
        return 0
def get_down_AVE(lis):
    s = int(1000*lis[0]+3500*lis[1]+7500*lis[2]+15000*lis[3]+35000*lis[4]+60000*lis[5])
    if sum(lis) != 0:
        return int(s / sum(lis))
    else:
        return 0

#get max PlayDuration time < 300
def get_max_PlayDuration(lis_b_p):
    lis_b_p_ = [x for x in lis_b_p if x[0] < x[1] and x[1] <= 300]
    if not lis_b_p_:
        return 300
    else:
        return max(lis_b_p_, key=lambda x:x[0])[1]

def get_sum_PlayDuration(lis_p):
    lis_p_ = [x for x in lis_p if x <= 300]
    if sum(lis_p_) <= 300:
        return sum(lis_p_)
    return 300
#get the max badduration value,make sure the value is smaller than playduration
def get_max_BadDuration(lis_b_p):
    lis_b_p_ = [x for x in lis_b_p if x[0] < x[1] and x[1] <= 300]
    if not lis_b_p_:
        return 0
    else:
        return max(lis_b_p_, key=lambda x:x[0])[0]

def get_sum_BadDuration(lis_b, sum_p):
    lis_b_ = [x for x in lis_b if x <= 300]
    if sum(lis_b_) < sum_p:
        return sum(lis_b_)
    else:
        return sum_p

#recursively get all the dict of a json line
def get_outcome_iter(dict_):
    for key, value in dict_.items():
        if isinstance(value, dict):
            yield from get_outcome_iter(value)
        elif isinstance(value, list):
            for v in value:
                yield from get_outcome_iter(v)
        else:
            yield key, value

def get_dict_str(json_2_dict):
    str_quality_tmp, str_behavior_tmp = str_quality, str_behavior
    str_quality_dict = dict(StartTime=‘‘, SubscriberID=‘‘, 
        Bitrate=‘‘, MOSAvg=‘‘, DF=‘‘, MLR=‘‘, CPUUsageHistogram=‘‘, 
        ProfilePlayoutSecondsHistogram_M=‘‘, ProfilePlayoutSecondsHistogram_m=‘‘, 
        PlayDuration_M=‘‘, BadDuration_M=‘‘, PlayDuration_S=‘‘, BadDuration_S=‘‘, 
        CurrentPlaySuccNum_CurrentPlayErrNum=‘‘, CurrentPlaySuccNum=‘‘, 
        TVGuideReqNum=‘‘, TVGuideSuccessNum=‘‘, TVGuideDelayAvg=‘‘, 
        StallingCount_S=‘‘, StallingDuration_S=‘‘, 
        BufferTrack_A=‘‘, BufferTrack_M=‘‘, BufferTrack_m=‘‘, 
        RTTTrack_A=‘‘, RTTTrack_M=‘‘, RTTTrack_m=‘‘, 
        LostRatioTrack_A=‘‘, LostRatioTrack_M=‘‘, LostRatioTrack_m=‘‘, 
        RAMUsageHistogram_A=‘‘, RAMUsageHistogram_M=‘‘, RAMUsageHistogram_m=‘‘, DiskUsage=‘‘, 
        DownloadSpeed_A=‘‘, DownloadSpeed_M=‘‘, DownloadSpeed_m=‘‘, 
        UpBandWidthHistogram_A=‘‘, UpBandWidthHistogram_M=‘‘, UpBandWidthHistogram_m=‘‘, 
        DownBandWidthHistogram_A=‘‘, DownBandWidthHistogram_M=‘‘, DownBandWidthHistogram_m=‘‘, 
        URL=‘‘, ServiceType=‘‘, URL_IP=‘‘, DeviceSupplier=‘‘)

    str_behavior_dict = dict(StartTime=‘‘, SubscriberID=‘‘,ServiceType=‘‘, 
        PlayDuration_M=‘‘, URL=‘‘, URL_IP=‘‘)

    str_dict_tmp = dict(StartTime=[], SubscriberID=[], Bitrate=[], 
        MOSAvg=[], DF=[], MLR=[], CPUUsageHistogram=[], 
        ProfilePlayoutSecondsHistogram=[], PlayDuration=[], BadDuration=[], 
        CurrentPlaySuccNum=[], CurrentPlayErrNum=[], 
        TVGuideReqNum=[], TVGuideSuccessNum=[], TVGuideDelayAvg=[], 
        StallingCount=[], StallingDuration=[], 
        BufferTrack=[], RTTTrack=[], LostRatioTrack=[], RAMUsageHistogram=[], 
        DiskUsage=[], DownloadSpeed=[], UpBandWidthHistogram=[], DownBandWidthHistogram=[], 
        URL=[], ServiceType=[], DeviceSupplier=[])

    Histogram_list = [CPUUsageHistogram, RAMUsageHistogram, UpBandWidthHistogram,
        ProfilePlayoutSecondsHistogram, DownBandWidthHistogram, 
        RTTTrack, LostRatioTrack, BufferTrack]

    line_dict = get_outcome_iter(json_2_dict)
    for key, value in line_dict:
        if value == None or value == ‘‘:
            continue
        if key in Histogram_list:
            value = [int(x) for x in value.split(,)]
        if key in str_dict_tmp:
            str_dict_tmp[key].append(value)

    if str_dict_tmp[StartTime]:
        str_quality_dict[StartTime] = str_dict_tmp[StartTime][0]
        str_behavior_dict[StartTime] = str_dict_tmp[StartTime][0]
    if str_dict_tmp[SubscriberID]:
        str_quality_dict[SubscriberID] = str_dict_tmp[SubscriberID][0]
        str_behavior_dict[SubscriberID] = str_dict_tmp[SubscriberID][0]
    if str_dict_tmp[Bitrate]:
        str_quality_dict[Bitrate] = round(max(str_dict_tmp[Bitrate]) / 1024, 2)
    if str_dict_tmp[MOSAvg]:
        str_quality_dict[MOSAvg] = round(str_dict_tmp[MOSAvg][0] / 10, 1)
    if str_dict_tmp[DF]:
        str_quality_dict[DF] = int(str_dict_tmp[DF][0] / 1000)
    if str_dict_tmp[MLR]:
        str_quality_dict[MLR] = round(str_dict_tmp[MLR][0] / 1000000, 1)
    if str_dict_tmp[CPUUsageHistogram]:
        str_quality_dict[CPUUsageHistogram] = get_his_AVE_R_C(str_dict_tmp[CPUUsageHistogram][0])
    if str_dict_tmp[ProfilePlayoutSecondsHistogram]:
        str_quality_dict[ProfilePlayoutSecondsHistogram_M] =             down[get_his_MAX(str_dict_tmp[ProfilePlayoutSecondsHistogram][0])]
        str_quality_dict[ProfilePlayoutSecondsHistogram_m] =             down[get_his_MIN(str_dict_tmp[ProfilePlayoutSecondsHistogram][0])]

    if str_dict_tmp[PlayDuration] and str_dict_tmp[BadDuration]:
        lis_b_p = list(zip(str_dict_tmp[BadDuration], str_dict_tmp[PlayDuration]))
        str_quality_dict[PlayDuration_M] = get_max_PlayDuration(lis_b_p)
        str_quality_dict[PlayDuration_S] = get_sum_PlayDuration(str_dict_tmp[PlayDuration])
        str_behavior_dict[PlayDuration_M] = str_quality_dict[PlayDuration_M]
        str_quality_dict[BadDuration_M] = get_max_BadDuration(lis_b_p)
        str_quality_dict[BadDuration_S] =             get_sum_BadDuration(str_dict_tmp[BadDuration], str_quality_dict[PlayDuration_S])

    if str_dict_tmp[CurrentPlaySuccNum] and str_dict_tmp[CurrentPlayErrNum]:
        str_quality_dict[CurrentPlaySuccNum_CurrentPlayErrNum] =             str_dict_tmp[CurrentPlaySuccNum][0] + str_dict_tmp[CurrentPlayErrNum][0]
        str_quality_dict[CurrentPlaySuccNum] = str_dict_tmp[CurrentPlaySuccNum][0]
    if str_dict_tmp[TVGuideReqNum]:
        str_quality_dict[TVGuideReqNum] = str_dict_tmp[TVGuideReqNum][0]
    if str_dict_tmp[TVGuideSuccessNum]:
        str_quality_dict[TVGuideSuccessNum] = str_dict_tmp[TVGuideSuccessNum][0]
    if str_dict_tmp[TVGuideDelayAvg]:
        str_quality_dict[TVGuideDelayAvg] = str_dict_tmp[TVGuideDelayAvg][0]
    if str_dict_tmp[StallingDuration]:
        str_quality_dict[StallingDuration_S] = sum(str_dict_tmp[StallingDuration])
    if str_dict_tmp[StallingCount]:
        str_quality_dict[StallingCount_S] = sum(str_dict_tmp[StallingCount])
    if str_dict_tmp[BufferTrack]:
        str_quality_dict[BufferTrack_A] = sum(str_dict_tmp[BufferTrack][0])
        str_quality_dict[BufferTrack_M] = max(str_dict_tmp[BufferTrack][0])
        str_quality_dict[BufferTrack_m] = min(str_dict_tmp[BufferTrack][0])
    if str_dict_tmp[RTTTrack]:
        str_quality_dict[RTTTrack_A] = sum(str_dict_tmp[RTTTrack][0])
        str_quality_dict[RTTTrack_M] = max(str_dict_tmp[RTTTrack][0])
        str_quality_dict[RTTTrack_m] = min(str_dict_tmp[RTTTrack][0])
    if str_dict_tmp[LostRatioTrack]:
        str_quality_dict[LostRatioTrack_A] =             sum(str_dict_tmp[LostRatioTrack][0]) / (1000000 * len(str_dict_tmp[LostRatioTrack][0]))
        str_quality_dict[LostRatioTrack_M] = max(str_dict_tmp[LostRatioTrack][0]) / 1000000
        str_quality_dict[LostRatioTrack_m] = min(str_dict_tmp[LostRatioTrack][0]) / 1000000
    if str_dict_tmp[RAMUsageHistogram]:
        str_quality_dict[RAMUsageHistogram_A] = get_his_AVE_R_C(str_dict_tmp[RAMUsageHistogram][0])
        str_quality_dict[RAMUsageHistogram_M] = int(c_r[get_his_MAX(str_dict_tmp[RAMUsageHistogram][0])] * 100)
        str_quality_dict[RAMUsageHistogram_m] = int(c_r[get_his_MIN(str_dict_tmp[RAMUsageHistogram][0])] * 100)
    if str_dict_tmp[DiskUsage]:
        str_quality_dict[DiskUsage] = str_dict_tmp[DiskUsage][0]
    if str_dict_tmp[DownloadSpeed]:
        str_quality_dict[DownloadSpeed_A] = sum(str_dict_tmp[DownloadSpeed]) / len(str_dict_tmp[DownloadSpeed])
        str_quality_dict[DownloadSpeed_M] = max(str_dict_tmp[DownloadSpeed])
        str_quality_dict[DownloadSpeed_m] = min(str_dict_tmp[DownloadSpeed])
    if str_dict_tmp[UpBandWidthHistogram]:
        str_quality_dict[UpBandWidthHistogram_A] = get_up_AVE(str_dict_tmp[UpBandWidthHistogram][0])
        str_quality_dict[UpBandWidthHistogram_M] = up[get_his_MAX(str_dict_tmp[UpBandWidthHistogram][0])]
        str_quality_dict[UpBandWidthHistogram_m] = up[get_his_MIN(str_dict_tmp[UpBandWidthHistogram][0])]
    if str_dict_tmp[DownBandWidthHistogram]:
        str_quality_dict[DownBandWidthHistogram_A] = get_down_AVE(str_dict_tmp[DownBandWidthHistogram][0])
        str_quality_dict[DownBandWidthHistogram_M] = down[get_his_MAX(str_dict_tmp[DownBandWidthHistogram][0])]
        str_quality_dict[DownBandWidthHistogram_m] = down[get_his_MIN(str_dict_tmp[DownBandWidthHistogram][0])]
    if str_dict_tmp[URL]:
        URL_index = str_dict_tmp[PlayDuration].index(str_quality_dict[PlayDuration_M])             if str_quality_dict[PlayDuration_M] != 300 else 0
        str_quality_dict[URL] = str_dict_tmp[URL][URL_index]
        str_behavior_dict[URL] = str_quality_dict[URL]
        str_quality_dict[URL_IP] = pattern.search(str_quality_dict[URL]).group(1)
        str_behavior_dict[URL_IP] = str_quality_dict[URL_IP]
    if str_dict_tmp[ServiceType]:
        if str_dict_tmp[ServiceType][0] == LiveTV:
            str_dict_tmp[ServiceType][0] = BTV
        if str_dict_tmp[ServiceType][0] != BTV and str_dict_tmp[ServiceType][0] != VOD:
            str_dict_tmp[ServiceType][0] = PLAYBACK
        str_quality_dict[ServiceType] = str_dict_tmp[ServiceType][0]
        str_behavior_dict[ServiceType] = str_quality_dict[ServiceType]
    if str_dict_tmp[DeviceSupplier]:
        str_quality_dict[DeviceSupplier] = str_dict_tmp[DeviceSupplier][0]

    str_quality_tmp = str_quality_tmp.format(**str_quality_dict)
    str_behavior_tmp = str_behavior_tmp.format(**str_behavior_dict)
    return str_quality_tmp, str_behavior_tmp

def get_outcome_files(path_file_source, path_outcome):
    if path_file_source != None:
        ftime = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
        path_file_outcome = path_outcome + isa_stb_quality_ + ftime + .dat
        with open(path_file_outcome, a) as f_quality,             open(path_file_source, r) as f:
                for line in f:
                    if line == <==SPLIT==>\n:
                        continue
                    json_2_dict = json.loads(line)
                    outcome = get_dict_str(json_2_dict)
                    f_quality.write(outcome[0])
        #appserver_uid = getpwnam(‘appserver‘)[2]
        #appserver_gid = getpwnam(‘appserver‘)[3]
        #os.chown(path_file_outcome, appserver_uid, appserver_gid)
        #os.remove(path_file_source)#delete log files

if __name__ == __main__:
    #path_source = ‘/home/omc/‘#folder contains log file 
    #path_outcome = ‘/srv/smartcare/share/data/fbb/src/vcem/xDR‘#folder contains dat file 
    path_source = C:\\Users\\l00429182\\Desktop\\
    path_outcome = C:\\Users\\l00429182\\Desktop\\
    path_file_source = get_log_path(path_source)#log files that found in 5 mins
    print(path_file_source)
    get_outcome_files(path_file_source, path_outcome)

 

iptv

标签:import   gid   track   json   from   file path   btv   serve   orm   

原文地址:https://www.cnblogs.com/kHrystal/p/10213601.html

(0)
(0)
   
举报
评论 一句话评论(0
登录后才能评论!
© 2014 mamicode.com 版权所有  联系我们:gaon5@hotmail.com
迷上了代码!