分布式存储之MogileFS基于Nginx实现负载均衡（Nginx+MogileFS）

[root@www ~]# vim /etc/hosts
     192.168.80.136 mog1.daixiang.com
     192.168.80.137 mog2.daixiang.com
     192.168.80.138 mog3.daixiang.com
                           

 1 [root@mog1 ~]# vim /etc/mogilefs/mogilefsd.conf
 2 
 3 # Enable daemon mode to work in background and use syslog
 4 daemonize = 1                                           #以守护进程的形式运行
 5 # Where to store the pid of the daemon (must be the same in the init script)
 6 pidfile = /var/run/mogilefsd/mogilefsd.pid
 7 # Database connection information
 8 db_dsn = DBI:mysql:mogilefs:host=192.168.80.135      #定义数据库名为mogilefs和数据库服务器地址
 9 db_user = moguser                                     #定义管理此数据库的用户名
10 db_pass = mogpass                                     #定义密码
11 # IP:PORT to listen on for mogilefs client requests
12 listen = 0.0.0.0:7001                                  #定义监听的地址和端口
13 # Optional, if you don‘t define the port above.
14 conf_port = 7001                                                            
15 # Number of query workers to start by default.
16 query_jobs = 10                                         #定义启动查询线程个数
17 # Number of delete workers to start by default.
18 delete_jobs = 1                                          #定义启动删除线程个数
19 # Number of replicate workers to start by default.
20 replicate_jobs = 5                                       #定义启动复制线程个数
21 # Number of reaper workers to start by default.
22 # (you don‘t usually need to increase this)
23 reaper_jobs = 1                            #响应客户端请求，在磁盘失败后将请求重新放到队列中
24 # Number of fsck workers to start by default.
25 # (these can cause a lot of load when fsck‘ing)
26 #fsck_jobs = 1                              #对磁盘进行检测，默认没有启动
27 # Minimum amount of space to reserve in megabytes
28 # default: 100                               #默认保留100M空间
29 # Consider setting this to be larger than the largest file you
30 # would normally be uploading.
31 #min_free_space = 200                #最小空闲空间为200M
32 # Number of seconds to wait for a storage node to respond.
33 # default: 2                                   #等待存储节点相应的时间，默认为2s
34 # Keep this low, so busy storage nodes are quickly ignored.
35 #node_timeout = 2                        #等待节点的超时时长
36 # Number of seconds to wait to connect to a storage node.
37 # default: 2                                     #连接存储节点的超时时长
38 # Keep this low so overloaded nodes get skipped.
39 #conn_timeout = 2                         
40 # Allow replication to use the secondary node get port,
41 # if you have apache or similar configured for GET‘s
42 #repl_use_get_port = 1

 1 MariaDB [(none)]> show databases;
 2 +--------------------+
 3 | Database           |
 4 +--------------------+
 5 | information_schema |
 6 | mogilefs           |
 7 | mysql              |
 8 | performance_schema |
 9 | test               |
10 +--------------------+
11 5 rows in set (0.00 sec)
12 
13 MariaDB [(none)]> use mogilefs;
14 Database changed
15 MariaDB [mogilefs]> show tables;
16 +----------------------+
17 | Tables_in_mogilefs   |
18 +----------------------+
19 | checksum             |
20 | class                |
21 | device               |
22 | domain               |
23 | file                 |
24 | file_on              |
25 | file_on_corrupt      |
26 | file_to_delete       |
27 | file_to_delete2      |
28 | file_to_delete_later |
29 | file_to_queue        |
30 | file_to_replicate    |
31 | fsck_log             |
32 | host                 |
33 | server_settings      |
34 | tempfile             |
35 | unreachable_fids     |
36 +----------------------+
37 17 rows in set (0.00 sec)

1 [root@mog1 ~]# ss -tnl | grep ‘7001‘
2 LISTEN     0      128                       *:7001                     *:*  

[root@mog1 ~]# vim /etc/mogilefs/mogstored.conf

maxconns = 10000
httplisten = 0.0.0.0:7500
mgmtlisten = 0.0.0.0:7501
docroot = /dfs/mogdata

[root@mog1 ~]# ss -tnlp | grep ‘mogstored‘
LISTEN     0      128                       *:7500                     *:*      users:(("mogstored",2288,4))
LISTEN     0      128                       *:7501                     *:*      users:(("mogstored",2288,9))

[root@mog2 ~]# ss -tnlp | grep ‘mogstored‘
LISTEN     0      128                       *:7500                     *:*      users:(("mogstored",2288,4))
LISTEN     0      128                       *:7501                     *:*      users:(("mogstored",2288,9))

[root@mog3 ~]# ss -tnlp | grep ‘mogstored‘
LISTEN     0      128                       *:7500                     *:*      users:(("mogstored",2288,4))
LISTEN     0      128                       *:7501                     *:*      users:(("mogstored",2288,9))

[root@mog1 ~]# mogadm check

Checking trackers...
  127.0.0.1:7001 ... OK

Checking hosts...
  [ 1] mog1 ... OK
  [ 2] mog2 ... OK
  [ 3] mog3 ... OK

Checking devices...
  host device         size(G)    used(G)    free(G)   use%   ob state   I/O%
  ---- ------------ ---------- ---------- ---------- ------ ---------- -----
  [ 1] dev1            16.509      3.901     12.608  23.63%  writeable   4.4
  [ 2] dev2            16.509      3.901     12.608  23.63%  writeable   0.3
  [ 3] dev3            16.509      3.897     12.612  23.60%  writeable   0.0
  ---- ------------ ---------- ---------- ---------- ------
             total:    49.527     11.699     37.828  23.62%



[root@mog1 ~]# mogadm domain list 
 domain               class                mindevcount   replpolicy   hashtype
-------------------- -------------------- ------------- ------------ -------
 linux1               class1                    3        MultipleHosts() NONE   
 linux1               class2                    2        MultipleHosts() NONE   
 linux1               default                   2        MultipleHosts() NONE   

 python1              default                   2        MultipleHosts() NONE   
 python1              dx1                       2        MultipleHosts() NONE   

[root@mog1 ~]# mogfileinfo --tracker=192.168.80.137:7001 --domain=linux1 --key=‘1.jpg‘
- file: 1.jpg
     class:              default
  devcount:                    2
    domain:               linux1
       fid:                    5
       key:                1.jpg
    length:               134783
 - http://192.168.80.137:7500/dev2/0/000/000/0000000005.fid
 - http://192.168.80.136:7500/dev1/0/000/000/0000000005.fid

[root@nginx ~]# vim /etc/rc.d/init.d/nginx

#!/bin/sh
#
# nginx - this script starts and stops the nginx daemon
#
# chkconfig:   - 85 15 
# description:  Nginx is an HTTP(S) server, HTTP(S) reverse #               proxy and IMAP/POP3 proxy server
# processname: nginx
# config:      /etc/nginx/nginx.conf
# config:      /etc/sysconfig/nginx
# pidfile:     /var/run/nginx.pid
 
# Source function library.
. /etc/rc.d/init.d/functions
 
# Source networking configuration.
. /etc/sysconfig/network
 
# Check that networking is up.
[ "$NETWORKING" = "no" ] && exit 0
 
nginx="/usr/sbin/nginx"
prog=$(basename $nginx)
 
NGINX_CONF_FILE="/etc/nginx/nginx.conf"
 
[ -f /etc/sysconfig/nginx ] && . /etc/sysconfig/nginx
 
lockfile=/var/lock/subsys/nginx
 
make_dirs() {
   # make required directories
   user=`nginx -V 2>&1 | grep "configure arguments:" | sed ‘s/[^*]*--user=\([^ ]*\).*/\1/g‘ -`
   options=`$nginx -V 2>&1 | grep ‘configure arguments:‘`
   for opt in $options; do
       if [ `echo $opt | grep ‘.*-temp-path‘` ]; then
           value=`echo $opt | cut -d "=" -f 2`
           if [ ! -d "$value" ]; then
               # echo "creating" $value
               mkdir -p $value && chown -R $user $value
           fi
       fi
   done
}
 
start() {
    [ -x $nginx ] || exit 5
    [ -f $NGINX_CONF_FILE ] || exit 6
    make_dirs
    echo -n $"Starting $prog: "
    daemon $nginx -c $NGINX_CONF_FILE
    retval=$?
    echo
    [ $retval -eq 0 ] && touch $lockfile
    return $retval
}
 
stop() {
    echo -n $"Stopping $prog: "
    killproc $prog -QUIT
    retval=$?
    echo
    [ $retval -eq 0 ] && rm -f $lockfile
    return $retval
}
 
restart() {
    configtest || return $?
    stop
    sleep 1
    start
}
 
reload() {
    configtest || return $?
    echo -n $"Reloading $prog: "
    killproc $nginx -HUP
    RETVAL=$?
    echo
}
 
force_reload() {
    restart
}
 
configtest() {
  $nginx -t -c $NGINX_CONF_FILE
}
 
rh_status() {
    status $prog
}
 
rh_status_q() {
    rh_status >/dev/null 2>&1
}
 
case "$1" in
    start)
        rh_status_q && exit 0
        $1
        ;;
    stop)
        rh_status_q || exit 0
        $1
        ;;
    restart|configtest)
        $1
        ;;
    reload)
        rh_status_q || exit 7
        $1
        ;;
    force-reload)
        force_reload
        ;;
    status)
        rh_status
        ;;
    condrestart|try-restart)
        rh_status_q || exit 0
            ;;
    *)
        echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}"
        exit 2
esac

http {

    upstream images {
        server 192.168.80.136:7001;
        server 192.168.80.137:7001;
        server 192.168.80.138:7001;
    }

    server {
        listen       80;
        server_name  localhost;
        location /image {
            mogilefs_tracker images;
            mogilefs_domain linux1;
            mogilefs_methods GET;
            mogilefs_noverify on;
            mogilefs_pass {
                proxy_pass $mogilefs_path;
                proxy_hide_header Content-Type;
                proxy_buffering off;
            }
        }

        location /files {
            mogilefs_tracker images;
            mogilefs_domain python1;
            mogilefs_methods GET;
            mogilefs_noverify on;
            mogilefs_pass {
                proxy_pass $mogilefs_path;
                proxy_hide_header Content-Type;
                proxy_buffering off;
            }
        }
    }
}