使用Varnish加速图片文件访问

varnish是和squid类似的高性能开源HTTP加速器,我这里用来缓存图片,js,css等小文件

varnish cache 192.168.0.15 centos6.0
nagios www后端 192.168.0.11 centos5.3

1.安装varnish

wget http://repo.varnish-cache.org/source/varnish-3.0.0.tar.gz

tar zxvf varnish-3.0.0.tar.gz

cd varnish-3.0.0

./configure --prefix=/opt/varnish-3.0.0

make

make install

ln -s /opt/varnish-3.0.0 /opt/varnish

2.设置权限

cd /opt/varnish

#varnish以www:website来运行

chown -R www:website /opt/varnish/var/varnish/

mkdir /var/log/varnish

chown -R www:website /var/log/varnish

chown -R www:website /opt/varnish/var/varnish/`hostname`

mkdir /opt/varnish/var/varnish/`hostname`

3.配置文件

#查看默认配置文件
cat etc/varnish/default.vcl

#编辑新配置文件
vi etc/vcl.conf

4.启动varnish

/opt/varnish/sbin/varnishd -n /opt/varnish/var/varnish -f /opt/varnish/etc/vcl.conf -a 0.0.0.0:80 -s malloc,1G -g website -u www  -T 127.0.0.1:3200 -p sess_workspace=64768 -p thread_pools=2 -p listen_depth=4096 -p first_byte_timeout=10 -p sess_timeout=15 -w 200,5000,10

#参数说明

-n vcache /              #临时文件实例名.如果以"/"开头,就必须是一个可用的路径.

        -a :80 /       #服务所在端口.":80"是默认所有网络都建立80端口,":"前面是服务器IP.

        -T :5000 /          #管理端口.

        -s file,/data1/vcache,80g /                        #虚拟内存文件映射类型,路径以及容量. 包括两种类型"malloc"和"file"

        -s file,/data2/vcache,80g /                        #malloc是内存+swap交换模式.很简单.没得说.

        -s file,/data3/vcache,80g /                        #file是mmap的文件内存映射机制.(具体情况,参阅"mmap"函数说明)

        -s file,/data4/vcache,80g /

        -f /usr/local/varnish/etc/varnish.vcl /           #VCL文件路径.

        -P /var/run/varnish.pid /          #PID文件地址.

        -w 200,5000,10 /     #工作进程数.三个参数分别是:,,

        -h classic,16383 /    #hash列表类型,以及长度.默认长度是16383.具体用处和调整实际效果要等我看完源代码才知道.

        -p user=www /    #"-p"是变量配置参数

        -p group=website/                  #服务运行用户和用户组配置.

        -p thread_pools=4 /                #进程connections pools的个数,数量越多,越耗用cpu和mem,但是处理并发能力越强.

                 #系统手册上说,一个cpu用一个.

        -p listen_depth=4096 /            #TCP队列长度.默认是1024.

        -p first_byte_timeout=10         #从后端接受第一个字节的超时时间。默认60秒

        -p between_bytes_timeout=60                    #从后端接收数据后,连接空闲时间,默认60秒

        -p sess_timeout=15    #客户端和varnish连接超时时间,默认5秒

5.记录日志
/opt/varnish/bin/varnishncsa -n /opt/varnish/var/varnish -w /var/log/varnish/varnish.log &

#定时切割日志
vi /opt/shell/cutvarnishlog.sh

#!/bin/sh

# 0 0 * * * /bin/sh /opt/shell/cutvarnishlog.sh  > /dev/null 2>&1

date=$(date -d "yesterday" +"%Y%m%d")

pkill -9 varnishncsa

mv /var/log/varnish/varnish.log /var/log/varnish/varnish.${date}.log

/opt/varnish/bin/varnishncsa -n /opt/varnish/var/varnish -w /var/log/varnish/varnish.log &

mkdir -p /var/log/varnish/old

gzip -c /var/log/varnish/varnish.${date}.log > /var/log/varnish/old/varnish.${date}.log.gz

rm -f /var/log/varnish/varnish.${date}.log

rm -f /var/log/varnish/old/varnish$(date -d "-1 month" +"%Y%m*").log.gz

crontab -e

0 0 * * * /bin/sh /opt/shell/cutvarnishlog.sh  > /dev/null 2>&1

6.查看运行统计
/opt/varnish/bin/varnishstat -n /opt/varnish/var/varnish

1+01:13:37       /opt/varnish/var/varnish

Hitrate ratio:       10      100      288

Hitrate avg:     0.9987   0.9981   0.9978

    22251295       371.40       245.01 client_conn - Client connections accepted

    22250487       371.40       245.00 client_req - Client requests received

    22185321       371.40       244.29 cache_hit - Cache hits

       62904         0.00         0.69 cache_miss - Cache misses

        4615         0.00         0.05 backend_conn - Backend conn. success

          22         0.00         0.00 backend_fail - Backend conn. failures

       59164         0.00         0.65 backend_reuse - Backend conn. reuses

         456         0.00         0.01 backend_toolate - Backend conn. was closed

       59622         0.00         0.66 backend_recycle - Backend conn. recycles

       47470         0.00         0.52 fetch_length - Fetch with Length

       16307         0.00         0.18 fetch_chunked - Fetch chunked

           2         0.00         0.00 fetch_close - Fetch wanted close

        1873          .            .   n_sess_mem - N struct sess_mem

        1834          .            .   n_sess - N struct sess

         655          .            .   n_object - N struct object

         685          .            .   n_objectcore - N struct objectcore

         784          .            .   n_objecthead - N struct objecthead

         405          .            .   n_waitinglist - N struct waitinglist

           2          .            .   n_vbc - N struct vbc

          31          .            .   n_wrk - N worker threads

         381         0.00         0.00 n_wrk_create - N worker threads created

        2584         0.00         0.03 n_wrk_queued - N queued work requests

           2          .            .   n_backend - N backends

       62227          .            .   n_expired - N expired objects

     5365503          .            .   n_lru_moved - N LRU moved objects

        1362         0.00         0.01 losthdr - HTTP header overflows

    18551363       326.47       204.27 n_objwrite - Objects sent with write

    22251295       371.40       245.01 s_sess - Total Sessions

    22250487       371.40       245.00 s_req - Total Requests

         898         0.00         0.01 s_pass - Total pass

       63779         0.00         0.70 s_fetch - Total fetch

  7539848276    127352.96     83022.43 s_hdrbytes - Total header bytes 

141933911830   2248780.45   1562856.20 s_bodybytes - Total body bytes   

    22251292       371.40       245.01 sess_closed - Session Closed

           1         0.00         0.00 sess_herd - Session herd

   998035729     16610.26     10989.53 shm_records - SHM records

    89193699      1488.60       982.13 shm_writes - SHM writes

      328009         8.99         3.61 shm_cont - SHM MTX contention

         385         0.00         0.00 shm_cycles - SHM cycles through buffer

        1387         0.00         0.02 sms_nreq - SMS allocator requests

7.管理清除缓存
7.1通过Varnish管理端口进行管理
/opt/varnish/bin/varnishadm -T 127.0.0.1:3200 help

CLI connected to 127.0.0.1:3200

help [command]

ping [timestamp]

auth response

quit

banner

status

start

stop

vcl.load

vcl.inline

vcl.use

vcl.discard

vcl.list

vcl.show

param.show [-l] []

param.set

panic.show

panic.clear

storage.list

ban.url

ban [&& ]...

ban.list

通过Varnish管理端口清除缓存,支持正则表达式,1.0时为url.purge参数：
/opt/varnish/bin/varnishadm -T 127.0.0.1:3200 ban.url /shanghai-4.html

例：清除所有缓存：
/opt/varnish/bin/varnishadm -T 127.0.0.1:3200 ban.url *$

7.2通过telnet方式清除

telnet 127.0.0.1 3200

Trying 127.0.0.1 ...

Connected to 127.0.0.1.

Escape character is '^]'.

200 205     

-----------------------------

Varnish Cache CLI 1.0

-----------------------------

Linux,2.6.32-71.el6.i686,i686,-smalloc,-smalloc,-hcritbit

Type 'help' for command list.

Type 'quit' to close CLI session.

help

200 401     

help [command]

ping [timestamp]

auth response

quit

banner

status

start

stop

vcl.load

vcl.inline

vcl.use

vcl.discard

vcl.list

vcl.show

param.show [-l] []

param.set

panic.show

panic.clear

storage.list

ban.url

ban [&& ]...

ban.list

#1.0时的方法现在不支持

purge.url /shanghai-4.html

200 0 101 44     

Unknown request.

Type 'help' for more info.

#正确方法

ban.url /shanghai-4.html

200 0

7.3通过php等其它web请求清除缓存

function purge($ip,$port=80,$domain, $url) 

{ 

    $errstr = ''; 

    $errno = ''; 

    $fp = fsockopen ($ip, $port, $errno, $errstr, 2); 

    if (!$fp) 

    { 

         return false; 

    } 

    else 

    { 

        $out = "PURGE $url HTTP/1.1\r\n"; 

        $out .= "Host:$domain\r\n"; 

        $out .= "Connection: close\r\n\r\n"; 

        fputs ($fp, $out); 

        $out = fgets($fp , 4096); 

        fclose ($fp); 

        return true; 

    } 

} 

purge('192.168.0.15','80','blog.c1gstudio.com','/shanghai-4.html');

8.varnish的nginx前端
测试下来nginx和varnish在同一机器上会产生大量time_wait,单独使用没有问题.

upstream mysvr {

server 127.0.0.1:82;

     }

     server

     {

             listen       80;

             server_name  static.c1gstudio.net;

             index index.html index.htm index.php;

             root  /opt/lampp/htdocs/web;

       location ~/\.ht {

         deny all;

     }

       location ~(favicon.ico) {

                 log_not_found off;

expires 99d;

break;

     }

             location ~ .*\.(php|html|htm)?$

             {

return 403;      

             }

     location / {

valid_referers none blocked *.c1gstudio.com *.c1gstudio.net ;

if ($invalid_referer) {

    rewrite ^/ http://leech.c1gstudio.com/leech.gif;

    return 412;

    break;

}

  proxy_pass http://mysvr;

  proxy_set_header   Host             $host;

  proxy_set_header   X-Real-IP        $remote_addr;

  proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;

     }           

             access_log  /var/log/nginx/static.c1gstudio.net.log  access;

     }

9.内核优化
vi /etc/sysctl.conf

net.ipv4.tcp_fin_timeout = 30

net.ipv4.tcp_keepalive_time = 300

net.ipv4.tcp_synCOOKIEs = 1

net.ipv4.tcp_tw_reuse = 1

net.ipv4.tcp_tw_recycle = 1

net.ipv4.ip_local_port_range = 5000    65000

sysctl -p

varnish服务器运行基本没有负载

top - 15:54:34 up 34 days, 23:49,  1 user,  load average: 0.00, 0.01, 0.00

Tasks: 125 total,   1 running, 124 sleeping,   0 stopped,   0 zombie

Cpu(s):  1.8%us,  1.3%sy,  0.0%ni, 95.0%id,  0.4%wa,  0.0%hi,  1.5%si,  0.0%st

Mem:   2070548k total,  2017996k used,    52552k free,    83556k buffers

Swap:  2097144k total,        0k used,  2097144k free,  1612756k cached

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND   

26631 www       20   0  228m 134m  81m S  7.6  6.7  74:46.86 varnishd                   

 6070 www       20   0 31852  25m 1000 S  3.3  1.3   7:28.79 nginx                     

 6071 www       20   0 31076  24m 1000 S  2.0  1.2   7:22.34 nginx                     

 6068 www       20   0 31356  25m  976 S  1.7  1.3   7:21.36 nginx

tcp状态
netstat -n | awk ‘/^tcp/ {++S[$NF]} END {for(a in S) print a, S[a]}’

LAST_ACK 9

SYN_RECV 5

CLOSE_WAIT 3

ESTABLISHED 2083

FIN_WAIT1 95

FIN_WAIT2 247

TIME_WAIT 14412