OpenVZ基础使用方法

准备使用KVM+OpenVZ混合来满足部分的虚拟化需求

系统是CentOS 5.5 x64, 虚拟一样的系统
1. 安装
cd /etc/yum.repos.d
wget http://download.openvz.org/openvz.repo
rpm --import http://download.openvz.org/RPM-GPG-Key-OpenVZ

如果是RHEL6则需要修改，把rhel5注释
vi openvz.repo

[openvz-kernel-rhel5]
name=OpenVZ RHEL5-based kernel
#baseurl=http://download.openvz.org/kernel/branches/rhel5-2.6.18/current/
mirrorlist=http://download.openvz.org/kernel/mirrors-rhel5-2.6.18
enabled=0
gpgcheck=1
gpgkey=http://download.openvz.org/RPM-GPG-Key-OpenVZ
[openvz-kernel-rhel6]
name=OpenVZ RHEL6-based kernel
#baseurl=http://download.openvz.org/kernel/branches/rhel6-2.6.32/current/
mirrorlist=http://download.openvz.org/kernel/mirrors-rhel6-2.6.32
enabled=1
gpgcheck=1
gpgkey=http://download.openvz.org/RPM-GPG-Key-OpenVZ

yum install vzkernel

cat /boot/grub/menu.lst
default=0
timeout=5
serial --unit=0 --speed=115200
terminal --timeout=5 serial console
title CentOS (2.6.18-194.26.1.el5.028stab079.2)
    root (hd0,0)
    kernel /boot/vmlinuz-2.6.18-194.26.1.el5.028stab079.2 ro root=LABEL=/ cOnsole=ttyS0,115200n8 rhgb quiet
    initrd /boot/initrd-2.6.18-194.26.1.el5.028stab079.2.img
title CentOS (2.6.18-194.el5)
    root (hd0,0)
    kernel /boot/vmlinuz-2.6.18-194.el5 ro root=LABEL=/ cOnsole=ttyS0,115200n8 rhgb quiet
    initrd /boot/initrd-2.6.18-194.el5.img

安装OpenVZ工具
yum install vzctl vzquota

修改系统设定

vi /etc/sysctl.conf

net.ipv4.ip_forward = 1
net.ipv4.conf.default.proxy_arp = 0
net.ipv4.conf.all.rp_filter = 1
kernel.sysrq = 1
net.ipv4.conf.default.send_redirects = 1
net.ipv4.conf.all.send_redirects = 0
net.ipv4.icmp_echo_ignore_broadcasts=1
net.ipv4.conf.default.forwarding=1
#sysctl -p

设定网络
vi /etc/vz/vz.conf

NEIGHBOUR_DEVS=all
关闭selinux
vi /etc/sysconfig/selinux
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
#       enforcing - SELinux security policy is enforced.
#       permissive - SELinux prints warnings instead of enforcing.
#       disabled - SELinux is fully disabled.
SELINUX=disabled
# SELINUXTYPE= type of policy in use. Possible values are:
#       targeted - Only targeted network daemons are protected.
#       strict - Full SELinux protection.
SELINUXTYPE=targeted
reboot系统

# uname -a
Linux OpenVZ-CentOS5 2.6.18-194.26.1.el5.028stab079.2 #1 SMP Fri Dec 17 19:25:15 MSK 2010 x86_64 x86_64 x86_64 GNU/Linux

cd /vz/template/cache
wget http://download.openvz.org/template/precreated/centos-5-x86_64-devel.tar.gz

创建虚拟机
vzctl create 101 --ostemplate centos-5-x86_64-devel --config basic

主机重启后，虚拟机自动引导
vzctl set 101 --onboot yes --save

设定hostname
vzctl set 101 --hostname openvz-101 --save

设定ip
vzctl set 101 --ipadd 192.168.0.101 --save

设定nameserver
vzctl set 101 --nameserver 8.8.8.8 --nameserver 8.8.4.4 --nameserver 145.253.2.75 --save

设定netmask
#  grep TEMPL /etc/vz/conf/101.conf
OSTEMPLATE="centos-5-x86_64-devel"

# grep =  /etc/vz/dists/centos.conf
ADD_IP=redhat-add_ip.sh
DEL_IP=redhat-del_ip.sh
SET_HOSTNAME=redhat-set_hostname.sh
SET_DNS=set_dns.sh
SET_USERPASS=set_userpass.sh
SET_UGID_QUOTA=set_ugid_quota.sh
POST_CREATE=postcreate.sh

修改成
# grep NETMASK /etc/vz/dists/scripts/redhat-add_ip.sh
NETMASK=255.255.254.0
NETMASK=255.255.254.0" > $file ||

#vzctl start 101
# ifconfig

venet0:0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00
 ........ Mask:255.255.254.0
UP BROADCAST POINTOPOINT RUNNING NOARP MTU:1500 Metric:1

101虚拟机配置文件
/etc/vz/conf/

启动虚拟机
vzctl start 101

主机执行虚拟机命令
#vzctl exec 101 ls -l

直接进入虚拟环境
vzctl enter 101

停止虚拟机
vzctl stop 101

重启
vzctl restart 101

删除虚拟机，建议停止后在进行删除
vzctl destroy 101

显示虚拟机状态
# vzlist -a
CTID      NPROC STATUS    IP_ADDR         HOSTNAME
101         12 running   172.30.0.10     OpenVZ-101

磁盘管理
磁盘管理建议关闭虚拟机设定
quota信息/var/vzquota/quota.vpsid

默认是打开的
# grep 'DISK_QUOTA' /etc/vz/vz.conf
DISK_QUOTA=yes

# vzctl set 115 --diskspace 5048576:6153434 --save
# vzctl set 115 --diskinodes 400000:420000 --save
# vzctl set 115 --quotatime 6000 --save

配置文件
vi /etc/vz/conf/101.conf
# Disk quota parameters (in form of softlimit:hardlimit)
DISKSPACE="5048576:6153434"  #修改为5GB
DISKINODES="400000:420000" #inode设定
QUOTATIME="6000" 超过软quota多少秒之后，软quota转换成硬quota

这一步一定要关机操作，上面可以在线。
设定user，group数量限制，0为关闭该设定
vzctl set 115 --quotaugidlimit 200 --save

#vzctl enter 101  发现根多了两个链接文件
lrwxrwxrwx  1 root root   39 Jan  3 11:43 aquota.group -> /proc/vz/vzaquota/00000015/aquota.group
lrwxrwxrwx  1 root root   38 Jan  3 11:43 aquota.user -> /proc/vz/vzaquota/00000015/aquota.user

# cat /proc/mounts
/dev/simfs / simfs rw,usrquota,grpquota 0 0

#edquota root
Disk quotas for user root (uid 0):
Filesystem                   blocks       soft       hard     inodes     soft     hard
/dev/simfs                   616652          0          0      29296        0        0

# repquota -a
*** Report for user quotas on device /dev/simfs
Block grace time: 00:00; Inode grace time: 00:00
Block limits                File limits
User            used    soft    hard  grace    used  soft  hard  grace
----------------------------------------------------------------------
root      --  616652       0       0          29295     0     0      
named     --      24       0       0              6     0     0      
smmsp     --       8       0       0              2     0     0      
rpc       --       0       0       0              1     0     0      
apache    --       8       0       0              2     0     0   

主机上检查
# vzquota stat 101 -t
resource          usage       softlimit      hardlimit    grace
  1k-blocks         616692         5048576        6153434        
inodes          29306          400000         420000        
User/group quota: on,active
Ugids: loaded 20, total 20, limit 200
Ugid limit was exceeded: no

User/group grace times and quotafile flags:
 type block_exp_time inode_exp_time  dqi_flags
 user                                       0h
group                                       0h

User/group objects:
ID           type  resource       usage   softlimit   hardlimit    grace status
0            user 1k-blocks      616652           0           0          loaded
0            user    inodes       29295           0           0          loaded
0           group 1k-blocks      614872           0           0          loaded
0           group    inodes       29113           0           0          loaded
5           group 1k-blocks          24           0           0          loaded
5           group    inodes          98           0           0          loaded
6           group 1k-blocks           0           0           0          loaded
6           group    inodes           4           0           0          loaded
9           group 1k-blocks           0           0           0          loaded
9           group    inodes           6           0           0          loaded
12          group 1k-blocks         204           0           0          loaded
12          group    inodes           6           0           0          loaded
15          group 1k-blocks         128           0           0          loaded
15          group    inodes          32           0           0          loaded
21          group 1k-blocks          28           0           0          loaded
21          group    inodes           2           0           0          loaded
22          group 1k-blocks          24           0           0          loaded
22          group    inodes           4           0           0          loaded
25           user 1k-blocks          24           0           0          loaded
25           user    inodes           6           0           0          loaded
25          group 1k-blocks         100           0           0          loaded
25          group    inodes          28           0           0          loaded
32           user 1k-blocks           0           0           0          loaded
32           user    inodes           1           0           0          loaded
48           user 1k-blocks           8           0           0          loaded
48           user    inodes           2           0           0          loaded
48          group 1k-blocks          24           0           0          loaded
48          group    inodes           3           0           0          loaded
51           user 1k-blocks           8           0           0          loaded
51           user    inodes           2           0           0          loaded
51          group 1k-blocks         788           0           0          loaded
51          group    inodes           5           0           0          loaded
54          group 1k-blocks           4           0           0          loaded
54          group    inodes           1           0           0          loaded
81          group 1k-blocks          44           0           0          loaded
81          group    inodes           1           0           0          loaded
84          group 1k-blocks         360           0           0          loaded
84          group    inodes           2           0           0          loaded
99          group 1k-blocks          92           0           0          loaded
99          group    inodes           1           0           0          loaded

cpu设定
#  vzcpucheck
Current CPU utilization: 101000
Power of the node: 205921

VE0使用的CPU，主机
# grep VE0CPUUNITS=1000 /etc/vz/vz.conf
VE0CPUUNITS=1000

#vzctl set 115 --cpuunits 100000 --cpulimit 50 --save

 vi /etc/vz/conf/101.conf
# CPU fair sheduler parameter
CPUUNITS="100000"   #范围8-500000，值越大，则占cpu使用时间比重越多。101000/205921  除以上面Power of the node选项，则是比重
CPULIMIT="50" #0 是没有限制，cpu使用limit值，如果是两cpu，则是200%为最大值，我设定50%，如果是双cpu，没搞清楚是双核还是双路

cpuunits
设定VE0
vzctl set 0 --cpuunits num

内存设定

kmemsize：内核占用内存资源，不能被swap，2768679 bytes；

privmmpages：在内存不紧张的情况下生效，Private virtual memory，主机分配内存(ram+swap)，但包括部分可能已经释放，没有使用的，262144x4/1024=1024MB，这个选项也是free命令看到的值。可分配到的最大内存页面受privmmpages的barrier（49152 pages）限制，而对于拥有足够高的优先权的VE system进程，其还可以获得最大privmmpages的limit（53575 pages）的内存资源，但再多就没有了；

physpages：Physical pages，在上面privmmpages中真正在用的内存资源，3914 pages；

vmguarpages：用于保证在正常情况下，VE可分配到的最小内存页面; 但是有一种情况例外，当主机内存不够的情况下，这个选项的barrier会变成oomguarpages的值。我个人理解则是这个选项被oomguarpages接管。

oomguarpages：这个值主要在内存不够的情况下生效，其“current usage”为当前VE使用的RAM+SWAP，其barrier值（6144 pages）是OOM（内存溢出）的条件，一旦VE使用的内存资源超过该值，就会触发Linux OOM的机制。

工作过程：

oomguarpages current value (MEM+SWAP actual usage) = 59239 pages = 242642944 bytes

socket buffers current value: 79920 + 2220 + 19552 + 0 bytes

kmemsize current value: 5125208 bytes

the sum is 247869844 bytes. If this sum is more than oomguarpages barrier = 125000 pages = 512000000 bytes (it is not), some of the processes on the container can be killed in case of OOM situation.

个人理解是正常情况privmmpages vmguarpages作用，极端情况下oomguarpages接管。

设定
# vzctl set 101 --privvmpages 512m:1024m --vmguarpages 384m:512m --oomguarpages 384m:512m --save
UB limits were set successfully
Saved parameters for CT 101

# vzctl exec 101 cat /proc/user_beancounters
Version: 2.5
       uid  resource                     held              maxheld              barrier                limit              failcnt
      101:  kmemsize                  2791370              4230912             14372700             14790164                    0
lockedpages                     0                    0                 2048                 2048                    0
privvmpages                  3918                17953               131072               262144                    0
shmpages                       31                  687                21504                21504                    0
dummy                           0                    0                    0                    0                    0
numproc                        13                   25                  240                  240                    0
physpages                    2385                 5032                    0  9223372036854775807                    0
vmguarpages                     0                    0                98304               131072                    0
oomguarpages                 2385                 5032                98304               131072                    0
numtcpsock                      4                    5                  360                  360                    0
numflock                        5                    7                  188                  206                    0
numpty                          0                    3                   16                   16                    0
numsiginfo                      0                    4                  256                  256                    0
tcpsndbuf                   70016                94224              1720320              2703360                    0
tcprcvbuf                   65536               173816              1720320              2703360                    0
othersockbuf                 6984                60328              1126080              2097152                    0
dgramrcvbuf                     0                 8472               262144               262144                    0
numothersock                    9                   16                  360                  360                    0
dcachesize                      0                    0              3409920              3624960                    0
numfile                       381                  686                 9312                 9312                    0
dummy                           0                    0                    0                    0                    0
dummy                           0                    0                    0                    0                    0
dummy                           0                    0                    0                    0                    0
numiptent                      14                   14                  128                  128                    0

# vzcalc 101
Resource     Current(%)  Promised(%)  Max(%)
Memory           0.45      10.27      26.49

# vzcalc -v 101
Resource     Current(%)  Promised(%)  Max(%)
Low Mem          0.17       1.36       1.36
Total RAM        0.30        n/a        n/a
Mem + Swap       0.30      10.27        n/a
Alloc. Mem       0.45      10.27      26.49
Num. Proc        0.02        n/a       0.33
--------------------------------------------
Memory           0.45      10.27      26.49

# vzctl exec 101 free -m
total       used       free     shared    buffers     cached
Mem:           512         15        496          0          0          0
-/+ buffers/cache:         15        496
Swap:            0          0          0