基于Consul的数据库高可用架构【转】

[root@db-server-yayun-01 consul]# cat consul.log 
==> WARNING: BootstrapExpect Mode is specified as 1; this is the same as Bootstrap mode.
==> WARNING: Bootstrap mode enabled! Do not enable unless necessary
==> Starting Consul agent...
==> Consul agent running!
           Version: ‘v0.9.2‘
           Node ID: ‘5e612623-ec5b-386c-19be-d38876a9a46f‘
         Node name: ‘db-server-yayun-01‘
        Datacenter: ‘dc1‘
            Server: true (bootstrap: true)
       Client Addr: 192.168.0.10 (HTTP: 8500, HTTPS: -1, DNS: 8600)
      Cluster Addr: 192.168.0.10 (LAN: 8301, WAN: 8302)
    Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false

==> Log data will now stream in as it occurs:

    2017/12/09 09:49:53 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:192.168.0.10:8300 Address:192.168.0.10:8300}]
    2017/12/09 09:49:53 [INFO] raft: Node at 192.168.0.10:8300 [Follower] entering Follower state (Leader: "")
    2017/12/09 09:49:53 [INFO] serf: EventMemberJoin: db-server-yayun-01.dc1 192.168.0.10
    2017/12/09 09:49:53 [INFO] serf: EventMemberJoin: db-server-yayun-01 192.168.0.10
    2017/12/09 09:49:53 [INFO] agent: Started DNS server 192.168.0.10:8600 (udp)
    2017/12/09 09:49:53 [INFO] consul: Adding LAN server db-server-yayun-01 (Addr: tcp/192.168.0.10:8300) (DC: dc1)
    2017/12/09 09:49:53 [INFO] consul: Handled member-join event for server "db-server-yayun-01.dc1" in area "wan"
    2017/12/09 09:49:53 [INFO] agent: Started DNS server 192.168.0.10:8600 (tcp)
    2017/12/09 09:49:53 [INFO] agent: Started HTTP server on 192.168.0.10:8500
    2017/12/09 09:50:00 [ERR] agent: failed to sync remote state: No cluster leader
    2017/12/09 09:50:00 [WARN] raft: Heartbeat timeout from "" reached, starting election
    2017/12/09 09:50:00 [INFO] raft: Node at 192.168.0.10:8300 [Candidate] entering Candidate state in term 2
    2017/12/09 09:50:00 [INFO] raft: Election won. Tally: 1
    2017/12/09 09:50:00 [INFO] raft: Node at 192.168.0.10:8300 [Leader] entering Leader state
    2017/12/09 09:50:00 [INFO] consul: cluster leadership acquired
    2017/12/09 09:50:00 [INFO] consul: New leader elected: db-server-yayun-01
    2017/12/09 09:50:00 [INFO] consul: member ‘db-server-yayun-01‘ joined, marking health alive
    2017/12/09 09:50:03 [INFO] agent: Synced node info

[root@db-server-yayun-02 consul]# cat /data/consul/consul.log 
==> Starting Consul agent...
==> Joining cluster...
    Join completed. Synced with 1 initial agents
==> Consul agent running!
           Version: ‘v0.9.2‘
           Node ID: ‘0ec901ab-6c66-2461-95e6-50a77a28ed72‘
         Node name: ‘db-server-yayun-02‘
        Datacenter: ‘dc1‘
            Server: false (bootstrap: false)
       Client Addr: 127.0.0.1 (HTTP: 8500, HTTPS: -1, DNS: 8600)
      Cluster Addr: 192.168.0.20 (LAN: 8301, WAN: 8302)
    Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false

==> Log data will now stream in as it occurs:

    2017/12/09 10:06:10 [INFO] serf: EventMemberJoin: db-server-yayun-02 192.168.0.20
    2017/12/09 10:06:10 [INFO] agent: Started DNS server 127.0.0.1:8600 (udp)
    2017/12/09 10:06:10 [INFO] agent: Started DNS server 127.0.0.1:8600 (tcp)
    2017/12/09 10:06:10 [INFO] agent: Started HTTP server on 127.0.0.1:8500
    2017/12/09 10:06:10 [INFO] agent: (LAN) joining: [192.168.0.10]
    2017/12/09 10:06:10 [INFO] agent: Retry join is supported for: aws azure gce softlayer
    2017/12/09 10:06:10 [INFO] agent: Joining cluster...
    2017/12/09 10:06:10 [INFO] agent: (LAN) joining: [192.168.0.10]
    2017/12/09 10:06:10 [INFO] serf: EventMemberJoin: db-server-yayun-01 192.168.0.10
    2017/12/09 10:06:10 [INFO] agent: (LAN) joined: 1 Err: <nil>
    2017/12/09 10:06:10 [INFO] consul: adding server db-server-yayun-01 (Addr: tcp/192.168.0.10:8300) (DC: dc1)
    2017/12/09 10:06:10 [INFO] agent: (LAN) joined: 1 Err: <nil>
    2017/12/09 10:06:10 [INFO] agent: Join completed. Synced with 1 initial agents
    2017/12/09 10:06:10 [INFO] agent: Synced node info

[root@db-server-yayun-02 consul.d]# cat w-6029-redis-test.json 
{
  "services": [
    {
      "name": "w-6029-redis-test",
      "tags": [
        "master-test-6029"
      ],
      "address": "192.168.0.20",
      "port": 6029,
      "checks": [
        {
          "script": "/data/consul/shell/check_redis_master.sh 6029 ",
          "interval": "15s"
        }
      ]
    }
  ]
}

[root@db-server-yayun-02 consul.d]# 

[root@db-server-yayun-02 consul.d]# cat r-6029-redis-test.json 
{
  "services": [
    {
      "name": "r-6029-redis-test",
      "tags": [
        "slave-test-6029"
      ],
      "address": "192.168.0.20",
      "port": 6029,
      "checks": [
        {
          "script": "/data/consul/shell/check_redis_slave.sh 6029 ",
          "interval": "15s"
        }
      ]
    }
  ]
}

[root@db-server-yayun-02 consul.d]# 

[root@db-server-yayun-03 shell]# cat check_redis_master.sh 
#!/bin/bash
myport=$1
auth=$2
if [ ! -n "$auth" ]
then
auth=‘\"\"‘
fi
comm="/usr/local/bin/redis-cli -p $myport -a $auth "
role=`echo ‘INFO Replication‘|$comm |grep -Ec ‘role:master‘`
echo ‘INFO Replication‘|$comm

if [ $role -ne 1 ]
then
    exit 2
fi

[root@db-server-yayun-03 shell]# 

[root@db-server-yayun-03 shell]# cat check_redis_slave.sh 
#!/bin/bash
myport=$1
auth=$2
if [ ! -n "$auth" ]
then
auth=‘\"\"‘
fi
comm="/usr/local/bin/redis-cli -p $myport -a $auth "
role=`echo ‘INFO Replication‘|$comm |grep -Ec ‘^role:slave|^master_link_status:up‘`
single=`echo ‘INFO Replication‘|$comm |grep -Ec ‘^role:master|^connected_slaves:0‘`
echo ‘INFO Replication‘|$comm

if [ $role -ne 2 -a $single -ne 2  ]
then
    exit 2
fi

[root@db-server-yayun-03 shell]# 

[root@db-server-yayun-02 shell]# cat check_mysql_master.sh 
#!/bin/bash
port=$1
user="root"
passwod="123"

comm="/usr/local/mysql/bin/mysql -u$user -h 127.0.0.1 -P $port -p$passwod"
slave_info=`$comm -e "show slave status" |wc -l`
value=`$comm -Nse "select 1"`

# 判断是不是从库
if [ $slave_info -ne 0 ]
then
   echo "MySQL $port  Instance is Slave........"
   $comm -e "show slave status\G" | egrep -w "Master_Host|Master_User|Master_Port|Master_Log_File|Read_Master_Log_Pos|Relay_Log_File|Relay_Log_Pos|Relay_Master_Log_File|Slave_IO_Running|Slave_SQL_Running|Exec_Master_Log_Pos|Relay_Log_Space|Seconds_Behind_Master"
   exit 2
fi


# 判断mysql是否存活
if [ -z $value ]
then
   exit 2
fi

echo "MySQL $port  Instance is Master........"
$comm -e "select * from information_schema.PROCESSLIST where user=‘repl‘ and COMMAND like ‘%Dump%‘"
[root@db-server-yayun-02 shell]# 

[root@db-server-yayun-02 shell]# cat check_mysql_slave.sh 
#!/bin/bash
port=$1
user="root"
passwod="123"
repl_check_user="root"
repl_check_pwd="123"

master_comm="/usr/local/mysql/bin/mysql -u$user -h 127.0.0.1 -P $port -p$passwod"
slave_comm="/usr/local/mysql/bin/mysql -u$repl_check_user  -P $port -p$repl_check_pwd"

# 判断mysql是否存活
value=`$master_comm -Nse "select 1"`
if [ -z $value ]
then
   echo "MySQL Server is Down....."
   exit 2
fi

get_slave_count=0
is_slave_role=0
slave_mode_repl_delay=0
master_mode_repl_delay=0
master_mode_repl_dead=0
slave_mode_repl_status=0
max_delay=120

get_slave_hosts=`$master_comm -Nse "select substring_index(HOST,‘:‘,1) from information_schema.PROCESSLIST where user=‘repl‘ and COMMAND like ‘%Binlog Dump%‘;" `
get_slave_count=`$master_comm -Nse "select count(1) from information_schema.PROCESSLIST where user=‘repl‘ and COMMAND like ‘%Binlog Dump%‘;" `
is_slave_role=`$master_comm -e "show slave status\G"|grep -Ewc "Slave_SQL_Running|Slave_IO_Running"`


### 单点模式(如果 get_slave_count=0 and is_slave_role=0)
function single_mode
{
if [ $get_slave_count -eq 0 -a $is_slave_role -eq 0 ]
then
    echo "MySQL $port  Instance is Single Master........"
    exit 0 
fi
}

### 从节点模式(如果 get_slave_count=0 and is_slave_role=2 ) 
function slave_mode
{
#如果是从节点,必须满足不延迟,
if [  $is_slave_role -ge 2 ]
then
        echo "MySQL $port  Instance is Slave........"
        $master_comm -e "show slave status\G" | egrep -w "Master_Host|Master_User|Master_Port|Master_Log_File|Read_Master_Log_Pos|Relay_Log_File|Relay_Log_Pos|Relay_Master_Log_File|Slave_IO_Running|Slave_SQL_Running|Exec_Master_Log_Pos|Relay_Log_Space|Seconds_Behind_Master"
        slave_mode_repl_delay=`$master_comm -e "show slave status\G" | grep -w "Seconds_Behind_Master" | awk ‘{print $NF}‘`
        slave_mode_repl_status=`$master_comm -e "show slave status\G"|grep -Ec "Slave_IO_Running: Yes|Slave_SQL_Running: Yes"`
if [ X"$slave_mode_repl_delay" == X"NULL" ]
then
        slave_mode_repl_delay=99999
fi
        if [ $slave_mode_repl_delay != "NULL" -a $slave_mode_repl_delay -lt $max_delay -a $slave_mode_repl_status -ge 2 ] 
        then
             exit 0
        fi
fi
} 

function master_mode
{
###如果是主节点，必须满足从节点为延迟或复制错误。才可读
if [ $get_slave_count -gt 0 -a $is_slave_role -eq  0 ]
then
    echo "MySQL $port  Instance is Master........"
    $master_comm -e "select * from information_schema.PROCESSLIST where user=‘repl‘ and COMMAND like ‘%Dump%‘"
    for my_slave in $get_slave_hosts
do 
master_mode_repl_delay=`$slave_comm -h $my_slave -e "show slave status\G" | grep -w "Seconds_Behind_Master" | awk ‘{print $NF}‘ `
master_mode_repl_thread=`$slave_comm -h $my_slave -e "show slave status\G"|grep -Ec "Slave_IO_Running: Yes|Slave_SQL_Running: Yes"`
if [ X"$master_mode_repl_delay" == X"NULL" ]
then
     master_mode_repl_delay=99999
fi

if [ $master_mode_repl_delay -lt $max_delay -a $master_mode_repl_thread -ge 2 ]
then
    exit 2
fi
done 
exit 0
fi
} 

single_mode
slave_mode
master_mode
exit 2
[root@db-server-yayun-02 shell]# 

[root@db-server-yayun-02 ~]# dig @192.168.0.10 -p 8600 r-6029-redis-test.service.consul

; <<>> DiG 9.8.2rc1-RedHat-9.8.2-0.17.rc1.el6_4.6 <<>> @192.168.0.10 -p 8600 r-6029-redis-test.service.consul
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 34508
;; flags: qr aa rd; QUERY: 1, ANSWER: 2, AUTHORITY: 0, ADDITIONAL: 0
;; WARNING: recursion requested but not available

;; QUESTION SECTION:
;r-6029-redis-test.service.consul. IN   A

;; ANSWER SECTION:
r-6029-redis-test.service.consul. 0 IN  A       192.168.0.30
r-6029-redis-test.service.consul. 0 IN  A       192.168.0.40

;; Query time: 1 msec
;; SERVER: 192.168.0.10#8600(192.168.0.10)
;; WHEN: Sat Dec  9 11:26:38 2017
;; MSG SIZE  rcvd: 82

[root@db-server-yayun-02 ~]# 

[root@db-server-yayun-02 ~]# dig @192.168.0.10 -p 8600 w-6029-redis-test.service.consul

; <<>> DiG 9.8.2rc1-RedHat-9.8.2-0.17.rc1.el6_4.6 <<>> @192.168.0.10 -p 8600 w-6029-redis-test.service.consul
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 7451
;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0
;; WARNING: recursion requested but not available

;; QUESTION SECTION:
;w-6029-redis-test.service.consul. IN   A

;; ANSWER SECTION:
w-6029-redis-test.service.consul. 0 IN  A       192.168.0.20

;; Query time: 1 msec
;; SERVER: 192.168.0.10#8600(192.168.0.10)
;; WHEN: Sat Dec  9 11:27:59 2017
;; MSG SIZE  rcvd: 66

[root@db-server-yayun-02 ~]# 

[root@db-server-yayun-02 ~]# dig @192.168.0.10 -p 8600 r-6029-redis-test.service.consul

; <<>> DiG 9.8.2rc1-RedHat-9.8.2-0.17.rc1.el6_4.6 <<>> @192.168.0.10 -p 8600 r-6029-redis-test.service.consul
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 41984
;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0
;; WARNING: recursion requested but not available

;; QUESTION SECTION:
;r-6029-redis-test.service.consul. IN   A

;; ANSWER SECTION:
r-6029-redis-test.service.consul. 0 IN  A       192.168.0.40

;; Query time: 8 msec
;; SERVER: 192.168.0.10#8600(192.168.0.10)
;; WHEN: Sat Dec  9 11:32:46 2017
;; MSG SIZE  rcvd: 66

[root@db-server-yayun-02 ~]# 

[root@db-server-yayun-02 ~]# dig @192.168.0.10 -p 8600 r-6029-redis-test.service.consul

; <<>> DiG 9.8.2rc1-RedHat-9.8.2-0.17.rc1.el6_4.6 <<>> @192.168.0.10 -p 8600 r-6029-redis-test.service.consul
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 58564
;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0
;; WARNING: recursion requested but not available

;; QUESTION SECTION:
;r-6029-redis-test.service.consul. IN   A

;; ANSWER SECTION:
r-6029-redis-test.service.consul. 0 IN  A       192.168.0.20

;; Query time: 4 msec
;; SERVER: 192.168.0.10#8600(192.168.0.10)
;; WHEN: Sat Dec  9 11:35:11 2017
;; MSG SIZE  rcvd: 66

[root@db-server-yayun-02 ~]# dig @192.168.0.10 -p 8600 w-6029-redis-test.service.consul

; <<>> DiG 9.8.2rc1-RedHat-9.8.2-0.17.rc1.el6_4.6 <<>> @192.168.0.10 -p 8600 w-6029-redis-test.service.consul
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 56965
;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0
;; WARNING: recursion requested but not available

;; QUESTION SECTION:
;w-6029-redis-test.service.consul. IN   A

;; ANSWER SECTION:
w-6029-redis-test.service.consul. 0 IN  A       192.168.0.20

;; Query time: 5 msec
;; SERVER: 192.168.0.10#8600(192.168.0.10)
;; WHEN: Sat Dec  9 11:35:16 2017
;; MSG SIZE  rcvd: 66

[root@db-server-yayun-02 ~]# 

[root@db-server-yayun-02 ~]# dig @192.168.0.10 -p 8600 w-6029-redis-test.service.consul

; <<>> DiG 9.8.2rc1-RedHat-9.8.2-0.17.rc1.el6_4.6 <<>> @192.168.0.10 -p 8600 w-6029-redis-test.service.consul
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 62669
;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0
;; WARNING: recursion requested but not available

;; QUESTION SECTION:
;w-6029-redis-test.service.consul. IN   A

;; ANSWER SECTION:
w-6029-redis-test.service.consul. 0 IN  A       192.168.0.20

;; Query time: 2 msec
;; SERVER: 192.168.0.10#8600(192.168.0.10)
;; WHEN: Sat Dec  9 11:43:04 2017
;; MSG SIZE  rcvd: 66

[root@db-server-yayun-02 ~]# dig @192.168.0.10 -p 8600 r-6029-redis-test.service.consul

; <<>> DiG 9.8.2rc1-RedHat-9.8.2-0.17.rc1.el6_4.6 <<>> @192.168.0.10 -p 8600 r-6029-redis-test.service.consul
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 41305
;; flags: qr aa rd; QUERY: 1, ANSWER: 2, AUTHORITY: 0, ADDITIONAL: 0
;; WARNING: recursion requested but not available

;; QUESTION SECTION:
;r-6029-redis-test.service.consul. IN   A

;; ANSWER SECTION:
r-6029-redis-test.service.consul. 0 IN  A       192.168.0.30
r-6029-redis-test.service.consul. 0 IN  A       192.168.0.40

;; Query time: 2 msec
;; SERVER: 192.168.0.10#8600(192.168.0.10)
;; WHEN: Sat Dec  9 11:43:08 2017
;; MSG SIZE  rcvd: 82

[root@db-server-yayun-02 ~]# 

[root@db-server-yayun-02 ~]# dig @192.168.0.10 -p 8600 w-6029-redis-test.service.consul

; <<>> DiG 9.8.2rc1-RedHat-9.8.2-0.17.rc1.el6_4.6 <<>> @192.168.0.10 -p 8600 w-6029-redis-test.service.consul
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 55527
;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0
;; WARNING: recursion requested but not available

;; QUESTION SECTION:
;w-6029-redis-test.service.consul. IN   A

;; ANSWER SECTION:
w-6029-redis-test.service.consul. 0 IN  A       192.168.0.30

;; Query time: 2 msec
;; SERVER: 192.168.0.10#8600(192.168.0.10)
;; WHEN: Sat Dec  9 11:45:46 2017
;; MSG SIZE  rcvd: 66

[root@db-server-yayun-02 ~]# dig @192.168.0.10 -p 8600 r-6029-redis-test.service.consul

; <<>> DiG 9.8.2rc1-RedHat-9.8.2-0.17.rc1.el6_4.6 <<>> @192.168.0.10 -p 8600 r-6029-redis-test.service.consul
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 11563
;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 0
;; WARNING: recursion requested but not available

;; QUESTION SECTION:
;r-6029-redis-test.service.consul. IN   A

;; ANSWER SECTION:
r-6029-redis-test.service.consul. 0 IN  A       192.168.0.40

;; Query time: 1 msec
;; SERVER: 192.168.0.10#8600(192.168.0.10)
;; WHEN: Sat Dec  9 11:45:50 2017
;; MSG SIZE  rcvd: 66

[root@db-server-yayun-02 ~]#