标签：
功能：目录下面有 batch.total.1453804488.01.dat batch.total.1453804488.01.md5
batch.total.1453804488.02.dat batch.total.1453804488.02.md5
batch.total.1453804488.01.dat batch.total.1453804488.03.md5
...
index.list.1453804488.dat index.list.1453804488.md5
文件；
其中index.list.1453804488.dat 是索引文件，里面就是所有batch开头文件名字
batch.total.1453804488.01.dat 是数据文件
.md5 是校验文件，用文件大小校验
每五分钟定时产生 realtime.add.1453802333.dat ealtime.add.1453802333.01.md5 这样的文件，1453802333是时间戳
每天需要把realtime.add.1453802333.dat 写入 batch.total.1453804488.03.dat 文件，batch.total.1453804488.03.dat 达到一定大小就创建 batch.total.1453804488.04.dat文件
DIRLIST=($TOPIC_PATH_FROM $REPLY_PATH_FROM $USER_PATH_FROM)
for dir in ${DIRLIST[@]};do
rm $dir/$RELLY_INDEX
curfile=0
for i in $(seq 20);do
count=$i
if [ "$i" -lt "10" ];then
count=0$i
fi
if [ -f $dir/batch.total.$TIME.$count.dat ];then
curfile=$count
echo batch.total.$TIME.$curfile.dat >> $dir/index.list.$TIME.dat
else
curfile=$count
break
fi
done
ls -l $dir/index.list.$TIME.dat | cut -d ‘ ‘ -f 5 > $dir/index.list.$TIME.md5
nowfile=$[ $curfile - 1 ]
if [ "$curfile" -lt "10" ];then
nowfile=0$nowfile
fi
file_size=`ls -l $dir/batch.total.$TIME.$nowfile.dat | cut -d ‘ ‘ -f 5`
if [ "$file_size" -ge "20000" ];then #1202949134 - 1G
nowfile=$[ $nowfile + 1 ]
if [ "$curfile" -lt "10" ];then
nowfile=0$nowfile
fi
touch $dir/batch.total.$TIME.$nowfile.dat
echo batch.total.$TIME.$nowfile.dat >> $dir/index.list.$TIME.dat
ls -l $dir/index.list.$TIME.dat | cut -d ‘ ‘ -f 5 > $dir/index.list.$TIME.md5
fi
cat $dir/$RELLY_LOG >> $dir/batch.total.$TIME.$nowfile.dat
ls -l $dir/batch.total.$TIME.$nowfile.dat | cut -d ‘ ‘ -f 5 > $dir/batch.total.$TIME.$nowfile.md5
rm $dir/$RELLY_LOG
rm $dir/$RELLY_LOG_MD5
done
scp 自动输入密码 pcp.sh
./pcp.sh $HOST $PORT $USR $PASSWORD ${TOPIC_PATH_FROM}/$LOG $TOPIC_PATH
把上面这段写入脚本里面就可以了
#!/usr/bin/expect --
if { [llength $argv] < 6} {
puts "usage: $argv0 ip port user passwd filename destpath"
exit 1
}
set success 0
set maxRetry 1
for {set retryNum 0} {$retryNum<$maxRetry} {incr retryNum} {
spawn /usr/local/bin/scp -r -P[lindex $argv 1] [lindex $argv 4] [lindex $argv 2]@[lindex $argv 0]:[lindex $argv 5]
set timeout -1
expect {
"password:" {
send "[lindex $argv 3]\r"
set timeout -1
expect eof
set success 1
break
}
"yes/no)?" {
send "yes\r"
expect "password:" {
send "[lindex $argv 3]\r"
set timeout -1
expect eof
set success 1
break
}
}
timeout {continue}
eof {continue}
}
}
if { $success==0 } {
exit 1
}
标签：
原文地址：http://www.cnblogs.com/20120125llcai/p/5199250.html
