hadoop3 无法启动 查找内存消耗原因

[root@hadoop3 hadoop]# xlo
Starting namenodes on [hadoop3]
上一次登录:三 12月 27 15:07:11 CST 2017pts/24 上
hadoop3: ERROR: Cannot set priority of namenode process 9025
Starting datanodes
上一次登录:三 12月 27 15:16:59 CST 2017pts/24 上
hadoop3: ERROR: Cannot set priority of datanode process 9236
hadoop1: ERROR: Cannot set priority of datanode process 22612
Starting secondary namenodes [hadoop3]
上一次登录:三 12月 27 15:17:01 CST 2017pts/24 上
Starting resourcemanager
上一次登录:三 12月 27 15:17:04 CST 2017pts/24 上
Starting nodemanagers
上一次登录:三 12月 27 15:17:08 CST 2017pts/24 上
[root@hadoop3 hadoop]#

[root@hadoop3 hadoop]# cat ~/.bashrc
# .bashrc

# User specific aliases and functions

alias rm='rm -i'
alias cp='cp -i'
alias mv='mv -i'
alias xlpy='ps -aux|grep python|grep -v grep'
alias xlfg='free -g'
alias xlfm='free -m'
alias xlce='crontab -e'
alias xlcl='crontab -l'
alias xlrp='python '
alias xlhdfsf='/usr/local/hadoop/bin/hdfs namenode -format'
alias xlo='/usr/local/hadoop/sbin/start-all.sh'
alias xlc='/usr/local/hadoop/sbin/stop-all.sh'

# Source global definitions
if [ -f /etc/bashrc ]; then
. /etc/bashrc
fi

[root@hadoop3 hadoop]#

[root@hadoop3 ~]# xlfg
              total        used        free      shared  buff/cache   available
Mem:             15           2           0           0          12          12
Swap:             7           0           7
[root@hadoop3 ~]# 
[root@hadoop3 ~]# ps -eo rss,pmem,vsize,args | sort -k 1 -r -n 
484708  2.9 6007736 /usr/local/jdk/bin/java -Dproc_namenode -Djava.net.preferIPv4Stack=true -Dhdfs.audit.logger=INFO,NullAppender -Dhadoop.security.logger=INFO,RFAS -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-namenode-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-namenode-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml org.apache.hadoop.hdfs.server.namenode.NameNode
383044  2.3 6177364 /usr/local/jdk/bin/java -Dproc_resourcemanager -Djava.net.preferIPv4Stack=true -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-resourcemanager-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-resourcemanager-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.security.logger=INFO,NullAppender org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
381180  2.3 6043360 /usr/local/jdk/bin/java -Dproc_nodemanager -Djava.net.preferIPv4Stack=true -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-nodemanager-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-nodemanager-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.security.logger=INFO,NullAppender org.apache.hadoop.yarn.server.nodemanager.NodeManager
375868  2.3 5962452 /usr/local/jdk/bin/java -Dproc_secondarynamenode -Djava.net.preferIPv4Stack=true -Dhdfs.audit.logger=INFO,NullAppender -Dhadoop.security.logger=INFO,RFAS -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-secondarynamenode-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-secondarynamenode-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode
335036  2.0 5998380 /usr/local/jdk/bin/java -Dproc_datanode -Djava.net.preferIPv4Stack=true -Dhadoop.security.logger=ERROR,RFAS -Dyarn.log.dir=/usr/local/hadoop/logs -Dyarn.log.file=hadoop-root-datanode-hadoop3.log -Dyarn.home.dir=/usr/local/hadoop -Dyarn.root.logger=INFO,console -Djava.library.path=/usr/local/hadoop/lib/native -Dhadoop.log.dir=/usr/local/hadoop/logs -Dhadoop.log.file=hadoop-root-datanode-hadoop3.log -Dhadoop.home.dir=/usr/local/hadoop -Dhadoop.id.str=root -Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml org.apache.hadoop.hdfs.server.datanode.DataNode
68436  0.4 110564 /usr/lib/systemd/systemd-journald
59936  0.3 3077632 /usr/local/phantomjs/bin/phantomjs --cookies-file=/tmp/tmprwyetcbf --webdriver=49908
40744  0.2 552392 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
39820  0.2 352972 /usr/sbin/rsyslogd -n
38052  0.2 478916 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
36612  0.2 478660 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
36112  0.2 2838472 /usr/local/phantomjs/bin/phantomjs --cookies-file=/tmp/tmp8_stcm62 --webdriver=33070
35688  0.2 478660 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
34256  0.2 552392 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
33968  0.2 404672 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
33572  0.2 404672 python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py
33220  0.2 2839092 /usr/local/phantomjs/bin/phantomjs --cookies-file=/tmp/tmpq9k5ga_d --webdriver=37099
12152  0.0 529480 /usr/lib/polkit-1/polkitd --no-debug
 8132  0.0 439592 /usr/sbin/NetworkManager --no-daemon
 6424  0.0 144460 sshd: root@notty
 5724  0.0 143932 sshd: root@pts/28,pts/29,pts/30,pts/31,pts/32
 5712  0.0 143932 sshd: root@pts/26,pts/21,pts/22
 5708  0.0 144064 sshd: root@pts/24,pts/6,pts/7
 5704  0.0 144092 sshd: root@notty
 5700  0.0 143932 sshd: root@pts/1
 5684  0.0 144068 sshd: root@pts/0
 5676  0.0 143932 sshd: root@pts/2
 5668  0.0 143932 sshd: root@pts/8
 5668  0.0 143932 sshd: root@pts/27
 5668  0.0 143932 sshd: root@pts/23
 4024  0.0  49748 /usr/lib/systemd/systemd --switched-root --system --deserialize 21
 3800  0.0  83840 /usr/sbin/sendmail -FCronDaemon -i -odi -oem -oi -t -f root
 3792  0.0  83840 /usr/sbin/sendmail -s zh_cn_xiaomi_buqiang_jiankong g1e2t3@qq.com
 3664  0.0  80936 /usr/sbin/sshd -D
 2700  0.0  51404 /usr/sbin/wpa_supplicant -u -f /var/log/wpa_supplicant.log -c /etc/wpa_supplicant/wpa_supplicant.conf -u -f /var/log/wpa_supplicant.log -P /var/run/wpa_supplicant.pid
 2532  0.0 176076 /usr/sbin/CROND -n
 2488  0.0 176076 /usr/sbin/CROND -n
 2488  0.0 176076 /usr/sbin/CROND -n
 2488  0.0 176076 /usr/sbin/CROND -n
 2488  0.0 176076 /usr/sbin/CROND -n
 2488  0.0 176076 /usr/sbin/CROND -n
 2488  0.0 176076 /usr/sbin/CROND -n
 2488  0.0 176076 /usr/sbin/CROND -n
 2488  0.0 176076 /usr/sbin/CROND -n
 2356  0.0  43704 /usr/lib/systemd/systemd-udevd
 2264  0.0 115636 -bash
 2252  0.0 146328 top
 2224  0.0 115528 -bash
 2212  0.0  22008 nload
 2204  0.0 146316 top
 2204  0.0 115608 -bash
 2192  0.0 115652 -bash
 2188  0.0 115608 -bash
 2184  0.0 115528 -bash
 2184  0.0 115480 -bash
 2144  0.0 115528 -bash
 2136  0.0  53248 /usr/libexec/openssh/sftp-server
 2136  0.0 115516 -bash
 2116  0.0 115480 -bash
 2116  0.0 115480 -bash
 2112  0.0 115480 -bash
 2112  0.0 115400 -bash
 2112  0.0 115400 -bash
 2108  0.0 115400 -bash
 2108  0.0 115400 -bash
 2104  0.0 115400 -bash
 1792  0.0  24724 /bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation
 1724  0.0  24208 /usr/lib/systemd/systemd-logind
 1632  0.0 124192 /usr/sbin/crond -n
 1584  0.0  49020 /sbin/auditd -n
 1400  0.0 192564 /usr/sbin/lvmetad -f
 1396  0.0 113132 /bin/sh -c cd /home/data/xl_project;cat r1 > rtmp;date >r1; free -g >>r1;cat rtmp>>r1;/usr/sbin/sendmail -s 'zh_cn_xiaomi_buqiang_jiankong'  g1e2t3@qq.com < r1;
 1276  0.0 137392 ps -eo rss,pmem,vsize,args
 1228  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
 1228  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
 1228  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
 1224  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
 1224  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
 1224  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
 1220  0.0 113132 /bin/sh -c python /home/data/crontab_chk_url/personas/trunk/plugins/spider/chk_url_status_insert_tmp_error_30min.py &> /dev/null;
 1192  0.0  21480 /usr/sbin/irqbalance --foreground
  920  0.0 126696 sort -k 1 -r -n
  824  0.0 110048 /sbin/agetty --noclear tty1 linux
  RSS %MEM    VSZ COMMAND
    0  0.0      0 [xfs_mru_cache]
    0  0.0      0 [xfs-data/sda1]
    0  0.0      0 [xfs-data/dm-2]
    0  0.0      0 [xfs-data/dm-0]
    0  0.0      0 [xfs-conv/sda1]
    0  0.0      0 [xfs-conv/dm-2]
    0  0.0      0 [xfs-conv/dm-0]
    0  0.0      0 [xfs-cil/sda1]
    0  0.0      0 [xfs-cil/dm-2]
    0  0.0      0 [xfs-cil/dm-0]
    0  0.0      0 [xfs-buf/sda1]
    0  0.0      0 [xfs-buf/dm-2]
    0  0.0      0 [xfs-buf/dm-0]
    0  0.0      0 [xfsalloc]
    0  0.0      0 [xfsaild/sda1]
    0  0.0      0 [xfsaild/dm-2]
    0  0.0      0 [xfsaild/dm-0]
    0  0.0      0 [writeback]
    0  0.0      0 [watchdog/7]
    0  0.0      0 [watchdog/6]
    0  0.0      0 [watchdog/5]
    0  0.0      0 [watchdog/4]
    0  0.0      0 [watchdog/3]
    0  0.0      0 [watchdog/2]
    0  0.0      0 [watchdog/1]
    0  0.0      0 [watchdog/0]
    0  0.0      0 [ttm_swap]
    0  0.0      0 [scsi_tmf_5]
    0  0.0      0 [scsi_tmf_4]
    0  0.0      0 [scsi_tmf_3]
    0  0.0      0 [scsi_tmf_2]
    0  0.0      0 [scsi_tmf_1]
    0  0.0      0 [scsi_tmf_0]
    0  0.0      0 [scsi_eh_5]
    0  0.0      0 [scsi_eh_4]
    0  0.0      0 [scsi_eh_3]
    0  0.0      0 [scsi_eh_2]
    0  0.0      0 [scsi_eh_1]
    0  0.0      0 [scsi_eh_0]
    0  0.0      0 [rcu_sched]
    0  0.0      0 [rcuos/9]
    0  0.0      0 [rcuos/8]
    0  0.0      0 [rcuos/7]
    0  0.0      0 [rcuos/6]
    0  0.0      0 [rcuos/5]
    0  0.0      0 [rcuos/4]
    0  0.0      0 [rcuos/3]
    0  0.0      0 [rcuos/2]
    0  0.0      0 [rcuos/15]
    0  0.0      0 [rcuos/14]
    0  0.0      0 [rcuos/13]
    0  0.0      0 [rcuos/12]
    0  0.0      0 [rcuos/11]
    0  0.0      0 [rcuos/10]
    0  0.0      0 [rcuos/1]
    0  0.0      0 [rcuos/0]
    0  0.0      0 [rcuob/9]
    0  0.0      0 [rcuob/8]
    0  0.0      0 [rcuob/7]
    0  0.0      0 [rcuob/6]
    0  0.0      0 [rcuob/5]
    0  0.0      0 [rcuob/4]
    0  0.0      0 [rcuob/3]
    0  0.0      0 [rcuob/2]
    0  0.0      0 [rcuob/15]
    0  0.0      0 [rcuob/14]
    0  0.0      0 [rcuob/13]
    0  0.0      0 [rcuob/12]
    0  0.0      0 [rcuob/11]
    0  0.0      0 [rcuob/10]
    0  0.0      0 [rcuob/1]
    0  0.0      0 [rcuob/0]
    0  0.0      0 [rcu_bh]
    0  0.0      0 [phantomjs] <defunct>
    0  0.0      0 [phantomjs] <defunct>
    0  0.0      0 [phantomjs] <defunct>
    0  0.0      0 [phantomjs] <defunct>
    0  0.0      0 [perf]
    0  0.0      0 [netns]
    0  0.0      0 [migration/7]
    0  0.0      0 [migration/6]
    0  0.0      0 [migration/5]
    0  0.0      0 [migration/4]
    0  0.0      0 [migration/3]
    0  0.0      0 [migration/2]
    0  0.0      0 [migration/1]
    0  0.0      0 [migration/0]
    0  0.0      0 [md]
    0  0.0      0 [kworker/u32:2]
    0  0.0      0 [kworker/u32:1]
    0  0.0      0 [kworker/7:2]
    0  0.0      0 [kworker/7:1H]
    0  0.0      0 [kworker/7:0H]
    0  0.0      0 [kworker/7:0]
    0  0.0      0 [kworker/6:1H]
    0  0.0      0 [kworker/6:1]
    0  0.0      0 [kworker/6:0H]
    0  0.0      0 [kworker/6:0]
    0  0.0      0 [kworker/5:1H]
    0  0.0      0 [kworker/5:1]
    0  0.0      0 [kworker/5:0H]
    0  0.0      0 [kworker/5:0]
    0  0.0      0 [kworker/4:2H]
    0  0.0      0 [kworker/4:2]
    0  0.0      0 [kworker/4:1]
    0  0.0      0 [kworker/4:0H]
    0  0.0      0 [kworker/3:2H]
    0  0.0      0 [kworker/3:1H]
    0  0.0      0 [kworker/3:1]
    0  0.0      0 [kworker/3:0]
    0  0.0      0 [kworker/2:2H]
    0  0.0      0 [kworker/2:1H]
    0  0.0      0 [kworker/2:1]
    0  0.0      0 [kworker/2:0]
    0  0.0      0 [kworker/1:2]
    0  0.0      0 [kworker/1:1H]
    0  0.0      0 [kworker/1:0H]
    0  0.0      0 [kworker/1:0]
    0  0.0      0 [kworker/0:2]
    0  0.0      0 [kworker/0:1H]
    0  0.0      0 [kworker/0:0H]
    0  0.0      0 [kworker/0:0]
    0  0.0      0 [kthrotld]
    0  0.0      0 [kthreadd]
    0  0.0      0 [kswapd0]
    0  0.0      0 [ksoftirqd/7]
    0  0.0      0 [ksoftirqd/6]
    0  0.0      0 [ksoftirqd/5]
    0  0.0      0 [ksoftirqd/4]
    0  0.0      0 [ksoftirqd/3]
    0  0.0      0 [ksoftirqd/2]
    0  0.0      0 [ksoftirqd/1]
    0  0.0      0 [ksoftirqd/0]
    0  0.0      0 [ksmd]
    0  0.0      0 [kpsmoused]
    0  0.0      0 [kmpath_rdacd]
    0  0.0      0 [kipmi0]
    0  0.0      0 [kintegrityd]
    0  0.0      0 [khungtaskd]
    0  0.0      0 [khugepaged]
    0  0.0      0 [khelper]
    0  0.0      0 [kdmflush]
    0  0.0      0 [kdmflush]
    0  0.0      0 [kdmflush]
    0  0.0      0 [kdevtmpfs]
    0  0.0      0 [kblockd]
    0  0.0      0 [kauditd]
    0  0.0      0 [ipv6_addrconf]
    0  0.0      0 [fsnotify_mark]
    0  0.0      0 [events_power_ef]
    0  0.0      0 [edac-poller]
    0  0.0      0 [deferwq]
    0  0.0      0 [crypto]
    0  0.0      0 [bioset]
    0  0.0      0 [bioset]
    0  0.0      0 [bioset]
    0  0.0      0 [bioset]
    0  0.0      0 [ata_sff]
[root@hadoop3 ~]# 

  

http://www.cnblogs.com/JemBai/archive/2011/06/21/2086184.html

ps - 按进程消耗内存多少排序

ps -eo rss,pmem,pcpu,vsize,args |  sort -k 1 -r -n | less

解析一下:

ps 都知道,是linux,unix显示进程信息的, -e 是显示所有进程, -o是定制显示信息的格式

rss: resident set size, 表示进程占用RAM(内存)的大小,单位是KB
pmem: %M, 占用内存的百分比

pcpu:%C,占用cpu的百分比

vsize: 表示进程占用的虚拟内存的大小,KB

args:进程名(command)

sort命令对ps结果进行排序

-k 1 :按第一个参数 rss进行排序

-r:逆序

-n:numeric,按数字来排序

 
 
 
 

1:在命令行提示符执行top命令

2:输入大写P,则结果按CPU占用降序排序。输入大写M,结果按内存占用降序排序。(注:大写P可以在capslock状态输入p,或者按Shift+p)

ps -aux | grep phantomjs | cut -c 10-15 | xargs kill -9   

手动杀死进程

free内存变为2G,namenode可以启动,但datanode仍然不可以启动,此处说明“腾出内存”方向正确,进一步查内存消耗:已经不能通过手动杀死进程来实现腾出内存,故因为缓存占用大量内存考虑释放缓存。

原文地址:https://www.cnblogs.com/rsapaper/p/8126067.html