haproxy+keepalived+nginx实现高并发负载均衡

高并发负责均衡(HA)主备切换,当LVS笨重,Nginx无法满足需求的时候,Haproxy 是最佳选择

安装环境 (Mysql主从省略,memcached配置省略,主要记录下haproxy+keepalived,后端nginx也省了,本站都有资料,可以搜索):直接用haproxy做负载+代理,因为haproxy的负载比nginx强,比nginx+haproxy强,so…you know.

OS: Centos6.4(64X)
MASTER(haproxy):  192.168.101.110
BACKUP(haproxy):  192.168.101.111
VIP:  192.168.101.100
web1: 192.168.101.112
web2: 192.168.101.113
web3: 192.168.101.114



Haproxy安装和配置 
yum install haproxy
[root@80uncle Desktop]# cat /etc/haproxy/haproxy.cfg
global
    log         127.0.0.1 local0
    pidfile     /etc/haproxy/haproxy.pid
    maxconn     65535
    user        nobody
    group       nobody
    nbproc      1
    daemon
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option                  httpclose
    option                  forwardfor
    option                  redispatch
    retries                 2
    option                  abortonclose
    stats                   refresh 10
    contimeout              5000
    clitimeout              50000
    srvtimeout              50000
listen web *:80
mode http
maxconn 2000
balance         roundrobin
capture request header COOKIE len 200
COOKIE ServerID insert nocache indirect
appsession JSESSIONID len 52 timeout 10800000
server web1  192.168.101.112:80 COOKIE web1 check inter 1500 rise 3 fall 3 weight 3
server web2  192.168.101.113:80 COOKIE web2 check inter 1500 rise 3 fall 3 weight 3
server web3  192.168.101.114:80 COOKIE web3 check inter 1500 rise 3 fall 3 weight 3
option httpchk HEAD /index.html
srvtimeout      20000
listen stats_auth 192.168.101.110:9999
stats enable
stats uri /haproxy-80
stats auth  80uncle:123456
stats admin if TRUE

yum install haproxy
[root@80uncle Desktop]# cat /etc/haproxy/haproxy.cfg
global
    log         127.0.0.1 local0                  
    pidfile     /etc/haproxy/haproxy.pid    
    maxconn     65535                  
    user        nobody                    
    group       nobody                    
    nbproc      1                      
    daemon                              
defaults
    mode                    http          
    log                     global
    option                  httplog    
    option                  dontlognull
    option                  httpclose  
    option                  forwardfor  
    option                  redispatch
    retries                 2            
    option                  abortonclose    
    stats                   refresh 10  
    contimeout              5000       
    clitimeout              50000        
    srvtimeout              50000      
listen web *:80
mode http  
maxconn 2000    
balance        roundrobin  
capture request header COOKIE len 200  
COOKIE ServerID insert nocache indirect
appsession JSESSIONID len 52 timeout 10800000
server web1  192.168.101.112:80 COOKIE web1 check inter 1500 rise 3 fall 3 weight 3
server web2  192.168.101.113:80 COOKIE web2 check inter 1500 rise 3 fall 3 weight 3
server web3  192.168.101.114:80 COOKIE web3 check inter 1500 rise 3 fall 3 weight 3
option httpchk HEAD /index.html
srvtimeout      20000
listen stats_auth 192.168.101.110:9999  
stats enable
stats uri /haproxy-80
stats auth  80uncle:123456  
stats admin if TRUE
 
注:server 里面的80端口也可以是其他端口,比如8090端口反代理和nginx一样,备用机配置差不多只要把管理地址改为备用机真实IP就好,其他不变

配置主备机日志  vim /etc/rsyslog.conf
添加 以下两行
local3.*        /var/log/haproxy.log
local0.*        /var/log/haproxy.log

安装和配置 keepalived
[root@80uncle Desktop]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
   router_id LVS_DEVEL
}
vrrp_sync_group VGM {
group {
VI_1
}
}
vrrp_instance VI_1 {
    state MASTER
    interface eth1
    virtual_router_id 51
    priority 101
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.101.100
        }
}

[root@80uncle Desktop]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
   router_id LVS_DEVEL
}
vrrp_sync_group VGM {
group {
VI_1
}
}
vrrp_instance VI_1 {
    state MASTER
    interface eth1
    virtual_router_id 51
    priority 101
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.101.100  
    }
}

主备机器一样,只需修改权重和主备标示即可,还有网卡看下当前的是否一致
如果你前端开启了防火墙或者selinux,请关闭防火墙测试或者selinux测试,否则你会出现两个master,哈哈,两个vip地址同时出现,或者起不来服务,如果不关闭iptables,请加入以下策略

### 允许80端口对外提供服务
-A INPUT -m state --state NEW -m tcp -p tcp --dport 80 -j ACCEPT
### 只前端响应客户端的ARP广播包，允许vrrp虚拟路由器冗余协议
-A   INPUT   -d   224.0.0.0/8   -j   ACCEPT
-A   INPUT    -p   vrrp   -j   ACCEPT
有人说只需允许vrrp协议通过就好,个人建议两条都写上

### 允许80端口对外提供服务
-A INPUT -m state --state NEW -m tcp -p tcp --dport 80 -j ACCEPT  
### 只前端响应客户端的ARP广播包，允许vrrp虚拟路由器冗余协议    
-A   INPUT   -d   224.0.0.0/8   -j   ACCEPT  
-A   INPUT    -p   vrrp   -j   ACCEPT  
有人说只需允许vrrp协议通过就好,个人建议两条都写上

 后端Nginx服务器分别绑定VIP地址,执行脚本 {三台Nginx都执此脚本}
#!/bin/bash
### 后端 webserver 配置
SNS_VIP=192.168.101.100
. /etc/rc.d/init.d/functions
case "$1" in
start)
ifconfig lo:0 $SNS_VIP netmask 255.255.255.255 broadcast $SNS_VIP
/sbin/route add -host $SNS_VIP dev lo:0
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
sysctl -p >/dev/null 2>&1
echo "RealServer Start OK"
;;
stop)
ifconfig lo:0 down
route del $SNS_VIP >/dev/null 2>&1
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
echo "RealServer Stoped"
;;
*)
echo "Usage: $0 {start|stop}"
exit 1
esac
exit 0

#!/bin/bash
### 后端 webserver 配置
SNS_VIP=192.168.101.100
. /etc/rc.d/init.d/functions
case "$1" in
start)
ifconfig lo:0 $SNS_VIP netmask 255.255.255.255 broadcast $SNS_VIP
/sbin/route add -host $SNS_VIP dev lo:0
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
sysctl -p >/dev/null 2>&1
echo "RealServer Start OK"
;;
stop)
ifconfig lo:0 down
route del $SNS_VIP >/dev/null 2>&1
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
echo "RealServer Stoped"
;;
*)
echo "Usage: $0 {start|stop}"
exit 1
esac
exit 0