haproxy/nginx+keepalived负载均衡 双机热备 邮件报警 实战及常见问题

nameserver 10.1.1.10

export http_proxy=http://10.1.1.10:8000

proxy=http://10.1.1.10:8000

*          soft    nofile          1024
*          hard    nofile          65536
*          soft    nproc           2048
*          hard    nproc           16384
*          soft    stack           10240
*          hard    stack           32768

KEEPALIVED_OPTIONS="-d -D -S 0"

SYSLOGD_OPTIONS="-r -c 2"

local0.*    /var/log/keepalived.log
local2.*    /var/log/haproxy.log

# Firewall configuration written by system-config-firewall
# Manual customization of this file is not recommended.
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
-A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
-A INPUT -p icmp -j ACCEPT
-A INPUT -i lo -j ACCEPT
-A INPUT -m state --state NEW -m tcp -p tcp --dport 2888 -j ACCEPT
-A INPUT -m state --state NEW -m tcp -p tcp --dport 80 -j ACCEPT
-A INPUT -m state --state NEW -m tcp -p tcp --dport 8080 -j ACCEPT
-A INPUT -m state --state NEW -m tcp -p tcp --dport 443 -j ACCEPT
-A INPUT -m state --state NEW -s 10.1.1.0/24 -m tcp -p tcp --dport 8000 -j ACCEPT
-A INPUT -m state --state NEW -s 10.1.1.0/24 -m tcp -p tcp --dport 53 -j ACCEPT
-A INPUT -m state --state NEW -s 10.1.1.0/24 -m udp -p udp --dport 53 -j ACCEPT
-A INPUT -d 224.0.0.18 -j ACCEPT
-A INPUT -j REJECT --reject-with icmp-host-prohibited
-A FORWARD -j REJECT --reject-with icmp-host-prohibited
COMMIT

server {
    listen       8000;
    server_name  localhost;
    resolver     10.1.1.10;
    #charset koi8-r;
    access_log  /var/log/nginx/default.access.log  main;

    location / {
     proxy_pass http://$http_host$request_uri;
    }

! Configuration File for keepalived

global_defs {

   router_id HAP_KAD1

! 默认配置文件里本段还有notification_email设置，我不用，所以删掉了。
}


! vrrp_script 设置检查服务健康状况的脚本，脚本内容见下文。
! interval 1 每秒运行一次
! weight -10 如果脚本运行结果不为0（即失败或无结果），即说明主机haproxy服务故障，priority减10，此时备机priority比主机大，将自动选举为主机，原主机降为备机。
! priority 范围是1到254。为了避免priority不断降低最后两机都变为1，理论上应该再配置vrrp_script使恢复正常的服务器提升priority，提升脚本需要配 weight <正数>，但实际实验中发现基本没有这个必要。如果生产中两机不断的竞相自降 priority，说明有大麻烦发生了。

! 检查haproxy的脚本
vrrp_script check-haproxy {
    script "/usr/local/bin/chk-haproxy.sh"
    interval 1
    weight -10
}

! 检查nginx的脚本
vrrp_script check-nginx {
    script "/usr/local/bin/chk-nginx.sh"
    interval 1
    weight -10
}

vrrp_instance haproxy {

    ! 不设 MASTER，双机都是 BACKUP，只根据 priority选举主机地位。
    ! 备机 lbserver_02 的 priority 为 220

    state BACKUP
    interface eth0
    virtual_router_id 43
    priority 225

    advert_int 1
    authentication {
        auth_type PASS
        auth_pass PasswordForAuth
    }
    virtual_ipaddress {
    202.1.1.97/24
    202.1.1.98/24
    202.1.1.99/24
    }
    track_script {
        check-haproxy
    }

    ! 如果本机变为主机，则运行指定的脚本，脚本内容是使用 /etc/mail.rc 中配置的发信帐号向指定的接收邮箱发邮件。
    notify_master "/usr/local/bin/haproxy-master-change.sh"
}

! 为内部正向代理设置主备切换。其实没有必要，但是如果 nginx 要配置 http 反向代理，这部分还是必须的。
! 注意这里的 interface, virtual_router_id 跟上面vrrp_instance haproxy 的是不同的。
! virtual_ipaddress 10.1.1.10/24 ，还记得这个IP么？

vrrp_instance internalproxy {
    state BACKUP
    interface eth1
    virtual_router_id 80
    priority 225
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass PasswordForAuth
    }

    virtual_ipaddress {
        10.1.1.10/24
    }
    track_script {
        check-nginx
    }
    notify_master "/usr/local/bin/nginx-master-change.sh"
}

#---------------------------------------------------------------------
# Example configuration for a possible web application.  See the
# full configuration options online.
#
#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the ‘-r‘ option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log

    #这一条配合/etc/rsyslog.conf
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid

    #自定义最大连接数
    maxconn     10240

    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

    #默认值是1024，haproxy启动时会报错，但并不影响服务。改成2048后据说会有一点儿性能问题。
    tune.ssl.default-dh-param 2048

#---------------------------------------------------------------------
# common defaults that all the ‘listen‘ and ‘backend‘ sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 8192

#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------

# https网站发布，server.pem 是crt文件和key文件合并而成：cat server.cer server.key | tee server.pem
# http网站发布除了 bind *:80 和无需指定服务器证书之外，其他配置相同
# 另外，haproxy 跟 keepalived 配合时，bind 后面只能用 * 。如果指定了 IP，备机haproxy将无法启动，因为平时状态备机是不具备虚拟IP地址的。

frontend ssl
    mode http
    bind *:443  ssl crt /etc/haproxy/server.pem
    option httplog
    option httpclose

    # 这条可以使后端web服务器获得用户真实 IP，在httpd，tomcat 等日志格式配置里，用 %{X-Forwarded-For}i 记录用户真实 IP 。
    option forwardfor   except 127.0.0.0/8    

    stats  hide-version

    #根据域名指定后端服务器
    #    acl名称                     条件              域名（-i 忽略大小写）
    acl domain_starts_with_www      hdr_beg(host)    -i www.example.com
    acl domain_starts_with_auth     hdr_beg(host)    -i auth.example.com
    acl domain_starts_with_shops    hdr_beg(host)    -i shops.example.com

    #这里千万不能单独指定 www 的backend，那样会使所有的url规则失效。www 只能作为default后端
    use_backend auth    if domain_starts_with_auth  
    use_backend shops   if domain_starts_with_shops

    #根据域名后的路径指定后端服务器
    acl url_users       path_beg    -i /users
    acl url_topsales    path_beg    -i /topsales

    #if 后面多个 acl 时，and用空格代替， or 必须要写
    use_backend users    if domain_starts_with_www  url_users
    use_backend topsales if domain_starts_with_www  url_topsales

    #其他不符和acl条件的统统指定给后端 www
    default_backend        www

backend www
    #需要保持会话session的必须用source算法，即同一个外网IP的用户始终访问同一台后端。
    balance    source

    # haproxy 设置了服务器证书，后端真实web服务器用普通http就可以了
    server    httpd_tomcat1 10.1.1.20:80 check
    server    httpd_tomcat2 10.1.1.21:80 check
    server    httpd_tomcat1 10.1.1.22:80 check
    server    httpd_tomcat2 10.1.1.23:80 check

backend auth
    #auth 无需保持会话，所以用随机负载均衡算法 roundrobin 即可。
    balance    roundrobin
    server    httpd_tomcat_auth 10.1.1.30:80 check
    server    httpd_tomcat_auth 10.1.1.31:80 check

backend shops
    balance    source
    server    httpd_tomcat_auth 10.1.1.40:80 check
    server    httpd_tomcat_auth 10.1.1.41:80 check
    server    httpd_tomcat_auth 10.1.1.42:80 check
    server    httpd_tomcat_auth 10.1.1.43:80 check

backend users
    balance    source
    server    httpd_tomcat_auth 10.1.1.50:80 check
    server    httpd_tomcat_auth 10.1.1.51:80 check

backend topsales
    balance    source
    server    httpd_tomcat_auth 10.1.1.60:80 check
    server    httpd_tomcat_auth 10.1.1.61:80 check

#haproxy 还支持tcp反向代理
frontend ssh *:3306
    mode        tcp
    maxconn     128
    option      tcplog
    default_backend mysql

backend mysql
    mode        tcp
    balance     source
    server      mysql1    10.1.1.200:3306 check

#End

#!/bin/bash
ps aux|grep "/usr/sbin/haproxy"|grep -v grep

#!/bin/bash
ps aux|grep nginx|egrep ‘(master|worker)‘

echo "`uptime; ip addr show eth0; echo`" | mail -s "`hostname -s` to HAPROXY master."  -c supervisor@example.com receiver-01@example.com receiver-02@example.com

#!/bin/bash
echo "`uptime; ip addr show eth1; echo`" | mail -s "`hostname -s` to NGINX master." receiver-03@example.com receiver-04@example.com

#!/bin/bash
tcpdump -vvv -n -i eth0 dst 224.0.0.18

tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 65535 bytes
23:10:14.066857 IP (tos 0xc0, ttl 255, id 7007, offset 0, flags [none], proto VRRP (112), length 52)
    202.1.1.101 > 224.0.0.18: VRRPv2, Advertisement, vrid 43, prio 225, authtype simple, intvl 1s, length 32, addrs(3): 202.1.1.97,202.1.1.98,202.1.1.99 auth "PasswordForAuth"
23:10:15.068072 IP (tos 0xc0, ttl 255, id 7008, offset 0, flags [none], proto VRRP (112), length 52)
    202.1.1.101 > 224.0.0.18: VRRPv2, Advertisement, vrid 43, prio 225, authtype simple, intvl 1s, length 32, addrs(3): 202.1.1.97,202.1.1.98,202.1.1.99 auth "PasswordForAuth"
23:10:16.069224 IP (tos 0xc0, ttl 255, id 7009, offset 0, flags [none], proto VRRP (112), length 52)
    202.1.1.101 > 224.0.0.18: VRRPv2, Advertisement, vrid 43, prio 225, authtype simple, intvl 1s, length 32, addrs(3): 202.1.1.97,202.1.1.98,202.1.1.99 auth "PasswordForAuth"
......

#!/bin/bash
netstat -lntp

#!/bin/bash
netstat -antp

#!/bin/bash
netstat -antp|grep ESTABLISHED

[root@lbserver_01:/root]# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
4: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether fa:b1:3b:ec:b4:c2 brd ff:ff:ff:ff:ff:ff
    inet 202.1.1.101/24 brd 202.1.1.255 scope global eth0
    inet 202.1.1.97/24 scope global secondary eth0
    inet 202.1.1.98/24 scope global secondary eth0
    inet 202.1.1.99/24 scope global secondary eth0
    inet6 fe80::f0a1:8bfe:feda:c532/64 scope link
       valid_lft forever preferred_lft forever
5: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether fa:b1:3b:3c:01:0e brd ff:ff:ff:ff:ff:ff
    inet 10.1.1.11/24 brd 10.1.1.255 scope global eth1
    inet 10.1.1.10/24 scope global secondary eth1
    inet6 fe80::f001:aff:fb3c:12e/64 scope link
       valid_lft forever preferred_lft forever