nginx日志每日定时写入Hdfs

#!/bin/bash
hadoop_home=/opt/hadoop-2.4.0
tw_nginx_log_file=/home/chiline.com.all/access_com_tw.log
cn_nginx_log_file=/home/chiline.com.all/access_com_cn.log
current_date=$(date +%Y%m%d)
hdfs_url=hdfs://xx.xx.xx.xx:9100
analyse_jar_path=$hadoop_home/ianc


echo "hadoop_home = $hadoop_home"
echo "tw_nginx_log_file = $tw_nginx_log_file"
echo "cn_nginx_log_file = $cn_nginx_log_file"
echo "hdfs_url = $hdfs_url" 
echo "analyse_jar_path = $hadoop_home/ianc"


function putTodayLogToHdfs(){
  
  $hadoop_home/bin/hdfs dfs -rm -r $hdfs_url/user/day-$(addDate $current_date 1)
  $hadoop_home/bin/hdfs dfs -mkdir $hdfs_url/user/day-$(addDate $current_date 1)
  $hadoop_home/bin/hdfs dfs -mkdir $hdfs_url/user/day-$(addDate $current_date 1)/tw-log
  $hadoop_home/bin/hdfs dfs -mkdir $hdfs_url/user/day-$(addDate $current_date 1)/cn-log
  $hadoop_home/bin/hdfs dfs -put $tw_nginx_log_file $hdfs_url/user/day-$(addDate $current_date 1)/tw-log
  $hadoop_home/bin/hdfs dfs -put $cn_nginx_log_file $hdfs_url/user/day-$(addDate $current_date 1)/cn-log
}


function addDate(){
  str=$1
  days=$2
  yy=`echo $str|cut -c 1-4`
  mm=`echo $str|cut -c 5-6`
  dd=`echo $str|cut -c 7-8`
  sav_dd=$days
  days=`expr $days - $dd`
  while [ $days -ge 0 ]
  do
        mm=`expr $mm - 1`
        [ $mm -eq 0 ] && mm=12 && yy=`expr $yy - 1`
        aaa=`cal $mm $yy`
        bbb=`echo $aaa|awk ‘{print $NF}‘`
        days=`expr $days - $bbb`
  done
  dd=`expr 0 - $days`
  expr $dd : "^.$" > /dev/null && dd=0$dd
  expr $mm : "^.$" > /dev/null && mm=0$mm
  echo $yy$mm$dd
  return $yy$mmSdd
}


function getLogFileName(){
   logFileName=$1
   IFS=‘/‘ arr=($logFileName)
   arr_length=${#arr[@]}
   echo ${arr[$arr_length-1]}
   return ${arr[$arr_length-1]}
}


function removeLastWeekLog(){
  remove_date=$(addDate $current_date 7)
  echo "start remove history log file,remove_date is $remove_date"
  $hadoop_home/bin/hdfs dfs -rm -r $hdfs_url/user/day-$remove_date
}


function analyseTodayLog(){


   #tw_log_file_name=getLogFileName $tw_nginx_log_file
   tw_log_file_name=`basename $tw_nginx_log_file`
   #cn_log_file_name=getLogFileName $cn_nginx_log_file
   cn_log_file_name=`basename $cn_nginx_log_file`
   tw_log_file=$hdfs_url/user/day-$(addDate $current_date 1)/tw-log/$tw_log_file_name
   cn_log_file=$hdfs_url/user/day-$(addDate $current_date 1)/cn-log/$cn_log_file_name
   analyse_path=$hdfs_url/user/day-$(addDate $current_date 1)/analyse
   
   #pv analyse
   $hadoop_home/bin/hadoop jar  $analyse_jar_path/pvanalyse-1.0.jar $tw_log_file $analyse_path/tw-pv
   $hadoop_home/bin/hadoop jar  $analyse_jar_path/pvanalyse-1.0.jar $cn_log_file $analyse_path/cn-pv
   #time analyse
   $hadoop_home/bin/hadoop jar  $analyse_jar_path/timeanalyse-1.0.jar $tw_log_file $analyse_path/tw-time
   $hadoop_home/bin/hadoop jar  $analyse_jar_path/timeanalyse-1.0.jar $cn_log_file $analyse_path/cn-time
   #area analyse
   $hadoop_home/bin/hadoop jar  $analyse_jar_path/locationanalyse-1.0.jar $tw_log_file $analyse_path/tw-location
   $hadoop_home/bin/hadoop jar  $analyse_jar_path/locationanalyse-1.0.jar $cn_log_file $analyse_path/cn-location
}


echo "start put local log to hdfs"
putTodayLogToHdfs;
echo "start analyse today log"
analyseTodayLog;
echo "remove last week log"
removeLastWeekLog;