apache无法启动报错No space left on device
故障现象:apache无法启动ipcs信号量很多
# service httpd start
Starting httpd : [Failed]
当达到极限信号量：
# ipcs -s | wc -l
32004
而：
# cat /proc/sys/kernel/msgmni
32000
# cat /proc/sys/kernel/sem
250 24000 32 1024
原因
这些错误意味着有缺乏在系统内处理的通信资源，例如信号量或共享存储器段。
解析度
登录到通过服务器的SSH。
增加的限制sysctl 配置：
# mkdir -p /etc/sysctl.d/
# touch /etc/sysctl.d/99-zz_plesk_semaphores.conf
# chmod 755 /etc/sysctl.d/ /etc/sysctl.d/99-zz_plesk_semaphores.conf
# chown root:root /etc/sysctl.d/ /etc/sysctl.d/99-zz_plesk_semaphores.conf
这样的 ?/etc/sysctl.d/99-zz_plesk_semaphores.conf配置文件必须包含：
kernel.msgmni = 64000
kernel.sem = 250 256000 32 1024
负荷了新的sysctl设置 sysctl
# sysctl -p
检查哪些用户用尽信号灯：
# ipcs -s | awk ‘{print $3}‘ | uniq -c
4 root
33 httpd
234 somesoftware
阿帕奇保持清洁信号灯：
# for i in `ipcs -s | awk ‘/httpd/ {print $2}‘`; do (ipcrm -s $i); done
在某些情况下，其他软件滥用严重限制信号，禁用该软件，清理作进一步调查其信号量和接触软件供应商：
# for i in `ipcs -s | awk ‘/httpd/ {print $2}‘`; do (ipcrm -s $i); done
for i in `ipcs -s | awk ‘/httpd/{ next; } {print $2}‘`; do (ipcrm -s $i); done
执行清理后问题依旧，反复执行也无效
[root@server:~]# ipcs -s
------ Semaphore Arrays --------
key ???????semid ?????owner ?????perms ?????nsems ????
0x00000000 0 ?????????root ??????600 ???????1 ????????
0x00000000 65537 ?????root ??????600 ???????1 ????????
0x00000000 131074 ????apache ????600 ???????1 ????????
0x00000000 163843 ????apache ????600 ???????1 ????????
0x7a004a4e 196612 ????zabbix ????600 ???????13 ???????
0x00000000 229381 ????apache ????600 ???????1 ????????
0x00000000 262150 ????apache ????600 ???????1 ????????
0x00000000 294919 ????apache ????600 ???????1 ????????
0x00000000 327688 ????apache ????600 ???????1 ????????
[root@server:~]# ipcs -s | grep apache | perl -e ‘while (<STDIN>) { @a=split(/\s+/); print `ipcrm sem $a[1]`}‘
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
for i in `ipcs -s | awk ‘/httpd/{ next; } {print $2}‘`; do (ipcrm -s $i); done
[root@server:~]# ipcs -s
------ Semaphore Arrays --------
key ???????semid ?????owner ?????perms ?????nsems ????
0x00000000 0 ?????????root ??????600 ???????1 ????????
0x00000000 65537 ?????root ??????600 ???????1 ????????
0x7a004a4e 196612 ????zabbix ????600 ???????13 ????
日志：
[root@server:~]# tail -f /var/log/messages
Dec ?4 04:47:36 server nrpe[36326]: Host 192.168.254.63 is not allowed to talk to us!
Dec ?4 04:47:36 server nrpe[36328]: Host 192.168.254.63 is not allowed to talk to us!
Dec ?4 04:47:36 server nrpe[36330]: Host 192.168.254.63 is not allowed to talk to us!
Dec ?4 04:47:36 server nrpe[36332]: Host 192.168.254.63 is not allowed to talk to us!
Dec ?4 04:47:36 server nrpe[36334]: Host 192.168.254.63 is not allowed to talk to us!
Dec ?4 04:48:07 server nrpe[36711]: Host 192.168.254.63 is not allowed to talk to us!
Dec ?4 04:48:07 server nrpe[36713]: Host 192.168.254.63 is not allowed to talk to us!
Dec ?4 04:48:07 server nrpe[36715]: Host 192.168.254.63 is not allowed to talk to us!
Dec ?4 04:49:04 server nrpe[37324]: Host 192.168.254.63 is not allowed to talk to us!
Dec ?4 04:49:04 server nrpe[37333]: Host 192.168.254.63 is not allowed to talk to us!
# 原因是因为切分、监控等程序造成的，全部kill就可以了
[root@server:~]# ps -ef|grep apache|grep -v grep
root ?????18475 ?18474 ?0 16:50 ? ???????00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root ?????18477 ?18475 ?0 16:50 ? ???????00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root ?????26621 ?26618 ?0 17:00 ? ???????00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root ?????26623 ?26621 ?0 17:00 ? ???????00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root ?????34354 ?34352 ?0 17:10 ? ???????00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root ?????34355 ?34354 ?0 17:10 ? ???????00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root ?????41177 ?41175 ?0 17:20 ? ???????00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root ?????41178 ?41177 ?0 17:20 ? ???????00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root ?????47964 ?47962 ?0 17:30 ? ???????00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root ?????47965 ?47964 ?0 17:30 ? ???????00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root ?????49905 ?49904 ?0 14:20 ? ???????00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root ?????49907 ?49905 ?0 14:20 ? ???????00:00:03 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root ?????54555 ?54554 ?0 17:40 ? ???????00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root ?????54557 ?54555 ?0 17:40 ? ???????00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root ?????56774 ?56773 ?0 14:30 ? ???????00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root ?????56775 ?56774 ?0 14:30 ? ???????00:00:03 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root ?????61962 ?61960 ?0 17:50 ? ???????00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root ?????61963 ?61962 ?0 17:50 ? ???????00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root ?????70159 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/error.log.%Y-%m-%d 86400
root ?????70160 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/i-job.chinasoft.com_error.log.%Y-%m-%d 86400
root ?????70161 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/support.chinasoft.com.old_error.log.%Y-%m-%d 86400
root ?????70162 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/dlcbs.chinasoft.com_error.log.%Y-%m-%d 86400
root ?????70163 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/access.log.%Y-%m-%d 86400
root ?????70164 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/i-job.chinasoft.com_access.log.%Y-%m-%d 86400
root ?????70165 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/dlcbs.chinasoft.com_access.log.%Y-%m-%d 86400
apache ???70166 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache ???70167 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache ???70168 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache ???70169 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache ???70170 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache ???70171 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache ???70172 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache ???70173 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache ???70174 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache ???70175 ?70158 ?0 17:57 ? ???????00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
root ?????84551 ?84548 ?0 06:00 ? ???????00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root ?????84553 ?84551 ?0 06:00 ? ???????00:00:23 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root ?????97069 ?97068 ?0 15:30 ? ???????00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root ?????97071 ?97069 ?0 15:30 ? ???????00:00:01 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root ????109607 109605 ?0 15:50 ? ???????00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root ????109608 109607 ?0 15:50 ? ???????00:00:01 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root ????115643 115642 ?0 16:00 ? ???????00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root ????115645 115643 ?0 16:00 ? ???????00:00:01 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
[root@server:~]# ps -ef|grep apache|grep -v grep|awk ‘{print $2}‘|xargs kill -9
[root:~]# ps -ef|grep apache
root ?????70590 ?70566 ?0 17:58 ? ???????00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/access.log.%Y-%m-%d 86400
root ?????70591 ?70566 ?0 17:58 ? ???????00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/i-job.chinasoft.com_access.log.%Y-%m-%d 86400
root ?????70592 ?70566 ?0 17:58 ? ???????00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/dlcbs.chinasoft.com_access.log.%Y-%m-%d 86400
apache ???70608 ?70566 ?0 17:58 ? ???????00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache ???70609 ?70566 ?0 17:58 ? ???????00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache ???70610 ?70566 ?0 17:58 ? ???????00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
root ?????70612 ?35669 ?0 17:58 pts/0 ???00:00:00 grep apache
[root:~]# killall httpd
[root:~]# ipcs -s
------ Semaphore Arrays --------
key ???????semid ?????owner ?????perms ?????nsems ?
# 删除信号
for i in `ipcs -s | awk ‘/httpd/{ next; } {print $2}‘`; do (ipcrm -s $i); done
apache无法启动报错No space left on device
原文地址：https://www.cnblogs.com/reblue520/p/10337890.html