see https://access.redhat.com/solutions/90883
check
sysctl vm.min_free_kbytes
sysctl vm.zone_reclaim_mode
gvim /etc/sysctl.conf
add:
vm.min_free_kbytes = 131072
vm.zone_reclaim_mode = 1
then
[root@titanic 2]# sysctl -p /etc/sysctl.conf
net.ipv4.ip_forward = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
vm.min_free_kbytes = 131072
vm.zone_reclaim_mode = 1
--------------------------------
The problems seems to have something witn VirtualBox: different programs complain about page allocation failure, but notice of VBoxNetFlt:is common to all errors.
cause and symptoms:
tail /var/log/messages
Feb 26 12:21:23 titanic kernel: firefox: page allocation failure. order:3, mode:0x20
Feb 26 12:21:23 titanic kernel: Pid: 3090, comm: firefox Not tainted 2.6.32-573.18.1.el6.x86_64 #1
Feb 26 12:21:23 titanic kernel: Call Trace:
Feb 26 12:21:23 titanic kernel: <IRQ> [<ffffffff811376ac>] ? __alloc_pages_nodemask+0x7dc/0x950
Feb 26 12:21:23 titanic kernel: [<ffffffff81014a19>] ? read_tsc+0x9/0x10
Feb 26 12:21:23 titanic kernel: [<ffffffff811771c2>] ? kmem_getpages+0x62/0x170
Feb 26 12:21:23 titanic kernel: [<ffffffff81177dda>] ? fallback_alloc+0x1ba/0x270
Feb 26 12:21:23 titanic kernel: [<ffffffff8117782f>] ? cache_grow+0x2cf/0x320
Feb 26 12:21:23 titanic kernel: [<ffffffff81177b59>] ? ____cache_alloc_node+0x99/0x160
Feb 26 12:21:23 titanic kernel: [<ffffffff81178dd0>] ? kmem_cache_alloc_node_trace+0x90/0x200
Feb 26 12:21:23 titanic kernel: [<ffffffff81178fed>] ? __kmalloc_node+0x4d/0x60
Feb 26 12:21:23 titanic kernel: [<ffffffff8145eaea>] ? __alloc_skb+0x7a/0x180
Feb 26 12:21:23 titanic kernel: [<ffffffff8145faf0>] ? skb_copy+0x40/0xb0
Feb 26 12:21:23 titanic kernel: [<ffffffffa03c5b75>] ? vboxNetFltLinuxPacketHandler+0xb5/0x600 [vboxnetflt]
Feb 26 12:21:23 titanic kernel: [<ffffffff81014a19>] ? read_tsc+0x9/0x10
Feb 26 12:21:23 titanic kernel: [<ffffffff810ac488>] ? getnstimeofday+0x58/0xf0
Feb 26 12:21:23 titanic kernel: [<ffffffff8146b479>] ? __netif_receive_skb+0x4b9/0x570
Feb 26 12:21:23 titanic kernel: [<ffffffff8146eaa8>] ? netif_receive_skb+0x58/0x60
Feb 26 12:21:23 titanic kernel: [<ffffffff8146ec98>] ? napi_gro_complete+0xc8/0x150
Feb 26 12:21:23 titanic kernel: [<ffffffff8146f20f>] ? napi_gro_flush+0x2f/0x50
Feb 26 12:21:23 titanic kernel: [<ffffffff8146f254>] ? napi_complete+0x24/0x60
Feb 26 12:21:23 titanic kernel: [<ffffffffa02477d2>] ? e1000e_poll+0x122/0x280 [e1000e]
Feb 26 12:21:23 titanic kernel: [<ffffffff81470b33>] ? net_rx_action+0x103/0x2f0
Feb 26 12:21:23 titanic kernel: [<ffffffff8107ffa1>] ? __do_softirq+0xc1/0x1e0
Feb 26 12:21:23 titanic kernel: [<ffffffff810ed920>] ? handle_IRQ_event+0x60/0x170
Feb 26 12:21:23 titanic kernel: [<ffffffff8107ffff>] ? __do_softirq+0x11f/0x1e0
Feb 26 12:21:23 titanic kernel: [<ffffffff8100c38c>] ? call_softirq+0x1c/0x30
Feb 26 12:21:23 titanic kernel: [<ffffffff8100fbd5>] ? do_softirq+0x65/0xa0
Feb 26 12:21:23 titanic kernel: [<ffffffff8107fe55>] ? irq_exit+0x85/0x90
Feb 26 12:21:23 titanic kernel: [<ffffffff81543015>] ? do_IRQ+0x75/0xf0
Feb 26 12:21:23 titanic kernel: [<ffffffff8100ba53>] ? ret_from_intr+0x0/0x11
Feb 26 12:21:23 titanic kernel: <EOI> VBoxNetFlt: Failed to allocate packet buffer, dropping the packet.
