iptv

# -*- coding: utf-8 -*-
import datetime, time, json, re, os
#from pwd import getpwnam
#quality
str_quality = ‘‘.join([‘{StartTime}|||{SubscriberID}‘,     ‘|‘*11, ‘{Bitrate}|{MOSAvg}|{DF}|{MLR}||||‘, ‘{CPUUsageHistogram}|‘,     ‘{ProfilePlayoutSecondsHistogram_M}‘, ‘|‘*8, ‘{ProfilePlayoutSecondsHistogram_m}‘,     ‘|‘*8, ‘{PlayDuration_M}|{BadDuration_M}|{PlayDuration_S}|{BadDuration_S}||||‘,     ‘{CurrentPlaySuccNum_CurrentPlayErrNum}|{CurrentPlaySuccNum}‘, ‘|‘*11,     ‘{TVGuideReqNum}|{TVGuideSuccessNum}|{TVGuideDelayAvg}|||||‘,     ‘{StallingCount_S}|{StallingDuration_S}||‘,     ‘{BufferTrack_A}|{BufferTrack_M}|{BufferTrack_m}|||‘,     ‘{RTTTrack_A}|{RTTTrack_M}|{RTTTrack_m}|{LostRatioTrack_A}|{LostRatioTrack_M}|{LostRatioTrack_m}‘,     ‘|‘*7, ‘{RAMUsageHistogram_A}|{RAMUsageHistogram_M}|{RAMUsageHistogram_m}|{DiskUsage}‘,     ‘|‘*6, ‘{DownloadSpeed_A}|{DownloadSpeed_M}|{DownloadSpeed_m}||||‘,     ‘{UpBandWidthHistogram_A}|{UpBandWidthHistogram_M}|{UpBandWidthHistogram_m}|‘,     ‘{DownBandWidthHistogram_A}|{DownBandWidthHistogram_M}|{DownBandWidthHistogram_m}||‘,     ‘{URL}|{ServiceType}|{URL_IP}||||‘, ‘{DeviceSupplier}||||\n‘])
#behavior
str_behavior = ‘‘.join([‘{StartTime}|{SubscriberID}‘,     ‘|‘*7, ‘{ServiceType}‘, ‘|‘*9, ‘{PlayDuration_M}‘,     ‘|‘*19, ‘{URL}|{URL_IP}||||\n‘])
#CPU, RAM
c_r = [0.1, 0.35, 0.6, 0.75, 0.85, 0.95]
#up
up = [125, 375, 750, 1500, 3000, 5000]
#down, ProfilePlayoutSecondsHistogram
down = [1000, 3500, 7500, 15000, 35000, 60000]
pattern = re.compile(r‘http://(\d+.\d+.\d+.\d+)/.*‘)
#get the log file path
def get_log_path(path):
    for fpathe,dirs,fs in os.walk(path):
        for f in fs:
            if f.endswith(‘.log‘):
                return os.path.join(fpathe, f)
def get_his_MIN(lis):
    for x in lis:
        if x != 0:
            return lis.index(x)
    return 0
def get_his_MAX(lis):
    for x in lis[::-1]:
        if x != 0:
            return lis.index(x)
    return 0
#CPU, RAM, up down
def get_his_AVE_R_C(lis):
    s = int(100*(0.1*lis[0]+0.35*lis[1]+0.6*lis[2]+0.75*lis[3]+0.85*lis[4]+0.95*lis[5]))
    if sum(lis) != 0:
        return int (s/ sum(lis))
    else:
        return 0
def get_up_AVE(lis):
    s = int(125*lis[0]+375*lis[1]+750*lis[2]+1500*lis[3]+3000*lis[4]+5000*lis[5])
    if sum(lis) != 0:
        return int(s / sum(lis))
    else:
        return 0
def get_down_AVE(lis):
    s = int(1000*lis[0]+3500*lis[1]+7500*lis[2]+15000*lis[3]+35000*lis[4]+60000*lis[5])
    if sum(lis) != 0:
        return int(s / sum(lis))
    else:
        return 0

#get max PlayDuration time < 300
def get_max_PlayDuration(lis_b_p):
    lis_b_p_ = [x for x in lis_b_p if x[0] < x[1] and x[1] <= 300]
    if not lis_b_p_:
        return 300
    else:
        return max(lis_b_p_, key=lambda x:x[0])[1]

def get_sum_PlayDuration(lis_p):
    lis_p_ = [x for x in lis_p if x <= 300]
    if sum(lis_p_) <= 300:
        return sum(lis_p_)
    return 300
#get the max badduration value，make sure the value is smaller than playduration
def get_max_BadDuration(lis_b_p):
    lis_b_p_ = [x for x in lis_b_p if x[0] < x[1] and x[1] <= 300]
    if not lis_b_p_:
        return 0
    else:
        return max(lis_b_p_, key=lambda x:x[0])[0]

def get_sum_BadDuration(lis_b, sum_p):
    lis_b_ = [x for x in lis_b if x <= 300]
    if sum(lis_b_) < sum_p:
        return sum(lis_b_)
    else:
        return sum_p

#recursively get all the dict of a json line
def get_outcome_iter(dict_):
    for key, value in dict_.items():
        if isinstance(value, dict):
            yield from get_outcome_iter(value)
        elif isinstance(value, list):
            for v in value:
                yield from get_outcome_iter(v)
        else:
            yield key, value

def get_dict_str(json_2_dict):
    str_quality_tmp, str_behavior_tmp = str_quality, str_behavior
    str_quality_dict = dict(StartTime=‘‘, SubscriberID=‘‘, 
        Bitrate=‘‘, MOSAvg=‘‘, DF=‘‘, MLR=‘‘, CPUUsageHistogram=‘‘, 
        ProfilePlayoutSecondsHistogram_M=‘‘, ProfilePlayoutSecondsHistogram_m=‘‘, 
        PlayDuration_M=‘‘, BadDuration_M=‘‘, PlayDuration_S=‘‘, BadDuration_S=‘‘, 
        CurrentPlaySuccNum_CurrentPlayErrNum=‘‘, CurrentPlaySuccNum=‘‘, 
        TVGuideReqNum=‘‘, TVGuideSuccessNum=‘‘, TVGuideDelayAvg=‘‘, 
        StallingCount_S=‘‘, StallingDuration_S=‘‘, 
        BufferTrack_A=‘‘, BufferTrack_M=‘‘, BufferTrack_m=‘‘, 
        RTTTrack_A=‘‘, RTTTrack_M=‘‘, RTTTrack_m=‘‘, 
        LostRatioTrack_A=‘‘, LostRatioTrack_M=‘‘, LostRatioTrack_m=‘‘, 
        RAMUsageHistogram_A=‘‘, RAMUsageHistogram_M=‘‘, RAMUsageHistogram_m=‘‘, DiskUsage=‘‘, 
        DownloadSpeed_A=‘‘, DownloadSpeed_M=‘‘, DownloadSpeed_m=‘‘, 
        UpBandWidthHistogram_A=‘‘, UpBandWidthHistogram_M=‘‘, UpBandWidthHistogram_m=‘‘, 
        DownBandWidthHistogram_A=‘‘, DownBandWidthHistogram_M=‘‘, DownBandWidthHistogram_m=‘‘, 
        URL=‘‘, ServiceType=‘‘, URL_IP=‘‘, DeviceSupplier=‘‘)

    str_behavior_dict = dict(StartTime=‘‘, SubscriberID=‘‘,ServiceType=‘‘, 
        PlayDuration_M=‘‘, URL=‘‘, URL_IP=‘‘)

    str_dict_tmp = dict(StartTime=[], SubscriberID=[], Bitrate=[], 
        MOSAvg=[], DF=[], MLR=[], CPUUsageHistogram=[], 
        ProfilePlayoutSecondsHistogram=[], PlayDuration=[], BadDuration=[], 
        CurrentPlaySuccNum=[], CurrentPlayErrNum=[], 
        TVGuideReqNum=[], TVGuideSuccessNum=[], TVGuideDelayAvg=[], 
        StallingCount=[], StallingDuration=[], 
        BufferTrack=[], RTTTrack=[], LostRatioTrack=[], RAMUsageHistogram=[], 
        DiskUsage=[], DownloadSpeed=[], UpBandWidthHistogram=[], DownBandWidthHistogram=[], 
        URL=[], ServiceType=[], DeviceSupplier=[])

    Histogram_list = [‘CPUUsageHistogram‘, ‘RAMUsageHistogram‘, ‘UpBandWidthHistogram‘,
        ‘ProfilePlayoutSecondsHistogram‘, ‘DownBandWidthHistogram‘, 
        ‘RTTTrack‘, ‘LostRatioTrack‘, ‘BufferTrack‘]

    line_dict = get_outcome_iter(json_2_dict)
    for key, value in line_dict:
        if value == None or value == ‘‘:
            continue
        if key in Histogram_list:
            value = [int(x) for x in value.split(‘,‘)]
        if key in str_dict_tmp:
            str_dict_tmp[key].append(value)

    if str_dict_tmp[‘StartTime‘]:
        str_quality_dict[‘StartTime‘] = str_dict_tmp[‘StartTime‘][0]
        str_behavior_dict[‘StartTime‘] = str_dict_tmp[‘StartTime‘][0]
    if str_dict_tmp[‘SubscriberID‘]:
        str_quality_dict[‘SubscriberID‘] = str_dict_tmp[‘SubscriberID‘][0]
        str_behavior_dict[‘SubscriberID‘] = str_dict_tmp[‘SubscriberID‘][0]
    if str_dict_tmp[‘Bitrate‘]:
        str_quality_dict[‘Bitrate‘] = round(max(str_dict_tmp[‘Bitrate‘]) / 1024, 2)
    if str_dict_tmp[‘MOSAvg‘]:
        str_quality_dict[‘MOSAvg‘] = round(str_dict_tmp[‘MOSAvg‘][0] / 10, 1)
    if str_dict_tmp[‘DF‘]:
        str_quality_dict[‘DF‘] = int(str_dict_tmp[‘DF‘][0] / 1000)
    if str_dict_tmp[‘MLR‘]:
        str_quality_dict[‘MLR‘] = round(str_dict_tmp[‘MLR‘][0] / 1000000, 1)
    if str_dict_tmp[‘CPUUsageHistogram‘]:
        str_quality_dict[‘CPUUsageHistogram‘] = get_his_AVE_R_C(str_dict_tmp[‘CPUUsageHistogram‘][0])
    if str_dict_tmp[‘ProfilePlayoutSecondsHistogram‘]:
        str_quality_dict[‘ProfilePlayoutSecondsHistogram_M‘] =             down[get_his_MAX(str_dict_tmp[‘ProfilePlayoutSecondsHistogram‘][0])]
        str_quality_dict[‘ProfilePlayoutSecondsHistogram_m‘] =             down[get_his_MIN(str_dict_tmp[‘ProfilePlayoutSecondsHistogram‘][0])]

    if str_dict_tmp[‘PlayDuration‘] and str_dict_tmp[‘BadDuration‘]:
        lis_b_p = list(zip(str_dict_tmp[‘BadDuration‘], str_dict_tmp[‘PlayDuration‘]))
        str_quality_dict[‘PlayDuration_M‘] = get_max_PlayDuration(lis_b_p)
        str_quality_dict[‘PlayDuration_S‘] = get_sum_PlayDuration(str_dict_tmp[‘PlayDuration‘])
        str_behavior_dict[‘PlayDuration_M‘] = str_quality_dict[‘PlayDuration_M‘]
        str_quality_dict[‘BadDuration_M‘] = get_max_BadDuration(lis_b_p)
        str_quality_dict[‘BadDuration_S‘] =             get_sum_BadDuration(str_dict_tmp[‘BadDuration‘], str_quality_dict[‘PlayDuration_S‘])

    if str_dict_tmp[‘CurrentPlaySuccNum‘] and str_dict_tmp[‘CurrentPlayErrNum‘]:
        str_quality_dict[‘CurrentPlaySuccNum_CurrentPlayErrNum‘] =             str_dict_tmp[‘CurrentPlaySuccNum‘][0] + str_dict_tmp[‘CurrentPlayErrNum‘][0]
        str_quality_dict[‘CurrentPlaySuccNum‘] = str_dict_tmp[‘CurrentPlaySuccNum‘][0]
    if str_dict_tmp[‘TVGuideReqNum‘]:
        str_quality_dict[‘TVGuideReqNum‘] = str_dict_tmp[‘TVGuideReqNum‘][0]
    if str_dict_tmp[‘TVGuideSuccessNum‘]:
        str_quality_dict[‘TVGuideSuccessNum‘] = str_dict_tmp[‘TVGuideSuccessNum‘][0]
    if str_dict_tmp[‘TVGuideDelayAvg‘]:
        str_quality_dict[‘TVGuideDelayAvg‘] = str_dict_tmp[‘TVGuideDelayAvg‘][0]
    if str_dict_tmp[‘StallingDuration‘]:
        str_quality_dict[‘StallingDuration_S‘] = sum(str_dict_tmp[‘StallingDuration‘])
    if str_dict_tmp[‘StallingCount‘]:
        str_quality_dict[‘StallingCount_S‘] = sum(str_dict_tmp[‘StallingCount‘])
    if str_dict_tmp[‘BufferTrack‘]:
        str_quality_dict[‘BufferTrack_A‘] = sum(str_dict_tmp[‘BufferTrack‘][0])
        str_quality_dict[‘BufferTrack_M‘] = max(str_dict_tmp[‘BufferTrack‘][0])
        str_quality_dict[‘BufferTrack_m‘] = min(str_dict_tmp[‘BufferTrack‘][0])
    if str_dict_tmp[‘RTTTrack‘]:
        str_quality_dict[‘RTTTrack_A‘] = sum(str_dict_tmp[‘RTTTrack‘][0])
        str_quality_dict[‘RTTTrack_M‘] = max(str_dict_tmp[‘RTTTrack‘][0])
        str_quality_dict[‘RTTTrack_m‘] = min(str_dict_tmp[‘RTTTrack‘][0])
    if str_dict_tmp[‘LostRatioTrack‘]:
        str_quality_dict[‘LostRatioTrack_A‘] =             sum(str_dict_tmp[‘LostRatioTrack‘][0]) / (1000000 * len(str_dict_tmp[‘LostRatioTrack‘][0]))
        str_quality_dict[‘LostRatioTrack_M‘] = max(str_dict_tmp[‘LostRatioTrack‘][0]) / 1000000
        str_quality_dict[‘LostRatioTrack_m‘] = min(str_dict_tmp[‘LostRatioTrack‘][0]) / 1000000
    if str_dict_tmp[‘RAMUsageHistogram‘]:
        str_quality_dict[‘RAMUsageHistogram_A‘] = get_his_AVE_R_C(str_dict_tmp[‘RAMUsageHistogram‘][0])
        str_quality_dict[‘RAMUsageHistogram_M‘] = int(c_r[get_his_MAX(str_dict_tmp[‘RAMUsageHistogram‘][0])] * 100)
        str_quality_dict[‘RAMUsageHistogram_m‘] = int(c_r[get_his_MIN(str_dict_tmp[‘RAMUsageHistogram‘][0])] * 100)
    if str_dict_tmp[‘DiskUsage‘]:
        str_quality_dict[‘DiskUsage‘] = str_dict_tmp[‘DiskUsage‘][0]
    if str_dict_tmp[‘DownloadSpeed‘]:
        str_quality_dict[‘DownloadSpeed_A‘] = sum(str_dict_tmp[‘DownloadSpeed‘]) / len(str_dict_tmp[‘DownloadSpeed‘])
        str_quality_dict[‘DownloadSpeed_M‘] = max(str_dict_tmp[‘DownloadSpeed‘])
        str_quality_dict[‘DownloadSpeed_m‘] = min(str_dict_tmp[‘DownloadSpeed‘])
    if str_dict_tmp[‘UpBandWidthHistogram‘]:
        str_quality_dict[‘UpBandWidthHistogram_A‘] = get_up_AVE(str_dict_tmp[‘UpBandWidthHistogram‘][0])
        str_quality_dict[‘UpBandWidthHistogram_M‘] = up[get_his_MAX(str_dict_tmp[‘UpBandWidthHistogram‘][0])]
        str_quality_dict[‘UpBandWidthHistogram_m‘] = up[get_his_MIN(str_dict_tmp[‘UpBandWidthHistogram‘][0])]
    if str_dict_tmp[‘DownBandWidthHistogram‘]:
        str_quality_dict[‘DownBandWidthHistogram_A‘] = get_down_AVE(str_dict_tmp[‘DownBandWidthHistogram‘][0])
        str_quality_dict[‘DownBandWidthHistogram_M‘] = down[get_his_MAX(str_dict_tmp[‘DownBandWidthHistogram‘][0])]
        str_quality_dict[‘DownBandWidthHistogram_m‘] = down[get_his_MIN(str_dict_tmp[‘DownBandWidthHistogram‘][0])]
    if str_dict_tmp[‘URL‘]:
        URL_index = str_dict_tmp[‘PlayDuration‘].index(str_quality_dict[‘PlayDuration_M‘])             if str_quality_dict[‘PlayDuration_M‘] != 300 else 0
        str_quality_dict[‘URL‘] = str_dict_tmp[‘URL‘][URL_index]
        str_behavior_dict[‘URL‘] = str_quality_dict[‘URL‘]
        str_quality_dict[‘URL_IP‘] = pattern.search(str_quality_dict[‘URL‘]).group(1)
        str_behavior_dict[‘URL_IP‘] = str_quality_dict[‘URL_IP‘]
    if str_dict_tmp[‘ServiceType‘]:
        if str_dict_tmp[‘ServiceType‘][0] == ‘LiveTV‘:
            str_dict_tmp[‘ServiceType‘][0] = ‘BTV‘
        if str_dict_tmp[‘ServiceType‘][0] != ‘BTV‘ and str_dict_tmp[‘ServiceType‘][0] != ‘VOD‘:
            str_dict_tmp[‘ServiceType‘][0] = ‘PLAYBACK‘
        str_quality_dict[‘ServiceType‘] = str_dict_tmp[‘ServiceType‘][0]
        str_behavior_dict[‘ServiceType‘] = str_quality_dict[‘ServiceType‘]
    if str_dict_tmp[‘DeviceSupplier‘]:
        str_quality_dict[‘DeviceSupplier‘] = str_dict_tmp[‘DeviceSupplier‘][0]

    str_quality_tmp = str_quality_tmp.format(**str_quality_dict)
    str_behavior_tmp = str_behavior_tmp.format(**str_behavior_dict)
    return str_quality_tmp, str_behavior_tmp

def get_outcome_files(path_file_source, path_outcome):
    if path_file_source != None:
        ftime = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
        path_file_outcome = path_outcome + ‘isa_stb_quality_‘ + ftime + ‘.dat‘
        with open(path_file_outcome, ‘a‘) as f_quality,             open(path_file_source, ‘r‘) as f:
                for line in f:
                    if line == ‘<==SPLIT==>\n‘:
                        continue
                    json_2_dict = json.loads(line)
                    outcome = get_dict_str(json_2_dict)
                    f_quality.write(outcome[0])
        #appserver_uid = getpwnam(‘appserver‘)[2]
        #appserver_gid = getpwnam(‘appserver‘)[3]
        #os.chown(path_file_outcome, appserver_uid, appserver_gid)
        #os.remove(path_file_source)#delete log files

if __name__ == ‘__main__‘:
    #path_source = ‘/home/omc/‘#folder contains log file 
    #path_outcome = ‘/srv/smartcare/share/data/fbb/src/vcem/xDR‘#folder contains dat file 
    path_source = ‘C:\\Users\\l00429182\\Desktop\\‘
    path_outcome = ‘C:\\Users\\l00429182\\Desktop\\‘
    path_file_source = get_log_path(path_source)#log files that found in 5 mins
    print(path_file_source)
    get_outcome_files(path_file_source, path_outcome)