系统过程分析

一。先列出系统中进程的pid，用ps或pstree都可以，哪个方便用哪个
[root@250-shiyan ~]# pstree -p
init(1)─┬─auditd(934)───{auditd}(935)
        ├─crond(1130)
        ├─master(1120)─┬─pickup(9284)
        │              └─qmgr(1129)
        ├─mfsmount(5061)─┬─{mfsmount}(5062)
        │                ├─{mfsmount}(5063)
        │                ├─{mfsmount}(5064)
        │                ├─{mfsmount}(5065)
        │                ├─{mfsmount}(5066)
        │                ├─{mfsmount}(5067)
        │                ├─{mfsmount}(5068)
        │                ├─{mfsmount}(5069)
        │                ├─{mfsmount}(5071)
        │                ├─{mfsmount}(5072)
        │                └─{mfsmount}(5089)
        ├─mingetty(1143)
        ├─mingetty(1145)
        ├─mingetty(1147)
        ├─mingetty(1149)
        ├─mingetty(1151)
        ├─mingetty(1153)
        ├─rpc.idmapd(14858)
        ├─rpc.mountd(14820)
        ├─rpc.statd(991)
        ├─rpcbind(973)
        ├─rsyslogd(2453)─┬─{rsyslogd}(2454)
        │                ├─{rsyslogd}(2456)
        │                └─{rsyslogd}(2457)
        ├─sshd(12432)─┬─sshd(3634)───bash(3636)
        │             └─sshd(7655)───bash(7657)───pstree(9361)
        └─udevd(379)─┬─udevd(1159)
                     └─udevd(1160)
[root@250-shiyan ~]# ps -C rsyslogd
  PID TTY          TIME CMD
 2453 ?        00:00:00 rsyslogd

二。再查看相关线程信息
[root@250-shiyan ~]# pstack 2453
Thread 4 (Thread 0x7f59c23ac700 (LWP 2454)):
#0  0x00007f59c3a005bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007f59c4078184 in wtiWorker ()
#2  0x00007f59c4077c1a in ?? ()
#3  0x00007f59c39fc9d1 in start_thread () from /lib64/libpthread.so.0
#4  0x00007f59c3127b6d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f59c19ab700 (LWP 2456)):
#0  0x00007f59c31205e3 in select () from /lib64/libc.so.6
#1  0x00007f59c25c4d51 in ?? () from /lib64/rsyslog/imuxsock.so
#2  0x00007f59c4086b6a in ?? ()
#3  0x00007f59c39fc9d1 in start_thread () from /lib64/libpthread.so.0
#4  0x00007f59c3127b6d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f59c0faa700 (LWP 2457)):
#0  0x00007f59c3a0375d in read () from /lib64/libpthread.so.0
#1  0x00007f59c23afd04 in klogLogKMsg () from /lib64/rsyslog/imklog.so
#2  0x00007f59c23af16c in ?? () from /lib64/rsyslog/imklog.so
#3  0x00007f59c4086b6a in ?? ()
#4  0x00007f59c39fc9d1 in start_thread () from /lib64/libpthread.so.0
#5  0x00007f59c3127b6d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f59c403c700 (LWP 2453)):
#0  0x00007f59c31205e3 in select () from /lib64/libc.so.6
#1  0x00007f59c40592f5 in ?? ()
#2  0x00007f59c405a9fa in realMain ()
#3  0x00007f59c305dd1d in __libc_start_main () from /lib64/libc.so.6
#4  0x00007f59c4056629 in _start ()

对待每一个守护进程都是这个过程。
一。先查看系统中都有哪些进程在运行
[root@84-monitor logs]# pstree
init─┬─auditd───{auditd}
     ├─crond───4*[crond─┬─sendmail───postdrop]
     │                  └─sh───sh───sh───sh───mail───mail]
     ├─httpd───8*[httpd]
     ├─java───23*[{java}]
     ├─master─┬─cleanup
     │        ├─local
     │        ├─pickup
     │        └─qmgr
     ├─6*[mingetty]
     ├─mysqld_safe───mysqld───9*[{mysqld}]
     ├─rpc.statd
     ├─rpcbind
     ├─rsyslogd───3*[{rsyslogd}]
     ├─sshd─┬─sshd───bash───pstree
     │      └─3*[sshd───bash───bash───ssh]
     └─udevd───2*[udevd]

二。其次列出以rsys开头的进程打开的所有文件
[root@84-monitor 972]# lsof -c rsys
COMMAND  PID USER   FD   TYPE             DEVICE SIZE/OFF       NODE NAME
rsyslogd 972 root  cwd    DIR              253,0     4096          2 /
rsyslogd 972 root  rtd    DIR              253,0     4096          2 /
rsyslogd 972 root  txt    REG              253,0   396064     521732 /sbin/rsyslogd
rsyslogd 972 root  mem    REG              253,0    27232     521711 /lib64/rsyslog/imklog.so
rsyslogd 972 root  mem    REG              253,0   340568     521717 /lib64/rsyslog/imuxsock.so
rsyslogd 972 root  mem    REG              253,0   110960     521867 /lib64/libresolv-2.12.so
rsyslogd 972 root  mem    REG              253,0    27424     521245 /lib64/libnss_dns-2.12.so
rsyslogd 972 root  mem    REG              253,0    65928     521865 /lib64/libnss_files-2.12.so
rsyslogd 972 root  mem    REG              253,0    26984     521718 /lib64/rsyslog/lmnet.so
rsyslogd 972 root  mem    REG              253,0  1921176     521231 /lib64/libc-2.12.so
rsyslogd 972 root  mem    REG              253,0    90880     521844 /lib64/libgcc_s-4.4.7-20120601.so.1
rsyslogd 972 root  mem    REG              253,0    43880     521868 /lib64/librt-2.12.so
rsyslogd 972 root  mem    REG              253,0    19536     521861 /lib64/libdl-2.12.so
rsyslogd 972 root  mem    REG              253,0   142640     521255 /lib64/libpthread-2.12.so
rsyslogd 972 root  mem    REG              253,0    88600     521285 /lib64/libz.so.1.2.3
rsyslogd 972 root  mem    REG              253,0   154624     521489 /lib64/ld-2.12.so
rsyslogd 972 root    0u  unix 0xffff88001fbd06c0      0t0      10252 /dev/log
rsyslogd 972 root    1w   REG              253,0      292     786284 /var/log/messages
rsyslogd 972 root    2w   REG              253,0  1191255     785232 /var/log/cron
rsyslogd 972 root    3r   REG                0,3        0 4026532040 /proc/kmsg
rsyslogd 972 root    4w   REG              253,0   564219     785245 /var/log/maillog
rsyslogd 972 root    5w   REG              253,0     1004     786285 /var/log/secure

三。随后进入到972的fd目录，列出文件列表，打开了5个文件
[root@84-monitor 972]# cd /proc/972/fd
[root@84-monitor fd]# ll
total 0
lrwx------. 1 root root 64 Mar 18 09:39 0 -> socket:[10252]
l-wx------. 1 root root 64 Mar 18 09:39 1 -> /var/log/messages
l-wx------. 1 root root 64 Mar 18 09:39 2 -> /var/log/cron
lr-x------. 1 root root 64 Mar 18 09:39 3 -> /proc/kmsg
l-wx------. 1 root root 64 Mar 18 09:39 4 -> /var/log/maillog
l-wx------. 1 root root 64 Mar 18 09:39 5 -> /var/log/secure

四。查漏补缺
FD列
txt  program text (code and data);
rtd  root directory;
cwd  current working directory;
cwd,rtd这两个经常是一样的，因为如果没有具体的目录的话，默认全放到根下。
mem  memory-mapped file;
u    for read and write access;
TYPE列
unix     for a UNIX domain socket;
REG    for a regular file;
DIR    for a directory;

一。
[root@84-monitor fd]# lsof -c rpcbind
COMMAND   PID USER   FD   TYPE             DEVICE SIZE/OFF    NODE NAME
rpcbind 32580  rpc  cwd    DIR              253,0     4096       2 /
rpcbind 32580  rpc  rtd    DIR              253,0     4096       2 /
rpcbind 32580  rpc  txt    REG              253,0    54408  521226 /sbin/rpcbind
rpcbind 32580  rpc  mem    REG              253,0    65928  521865 /lib64/libnss_files-2.12.so
rpcbind 32580  rpc  mem    REG              253,0  1921176  521231 /lib64/libc-2.12.so
rpcbind 32580  rpc  mem    REG              253,0   142640  521255 /lib64/libpthread-2.12.so
rpcbind 32580  rpc  mem    REG              253,0    19536  521861 /lib64/libdl-2.12.so
rpcbind 32580  rpc  mem    REG              253,0    36584  521220 /lib64/libgssglue.so.1.0.0
rpcbind 32580  rpc  mem    REG              253,0   113432  521863 /lib64/libnsl-2.12.so
rpcbind 32580  rpc  mem    REG              253,0   162016  521225 /lib64/libtirpc.so.1.0.10
rpcbind 32580  rpc  mem    REG              253,0    40792  521329 /lib64/libwrap.so.0.7.6
rpcbind 32580  rpc  mem    REG              253,0   154624  521489 /lib64/ld-2.12.so
rpcbind 32580  rpc    0u   CHR                1,3      0t0    3782 /dev/null
rpcbind 32580  rpc    1u   CHR                1,3      0t0    3782 /dev/null
rpcbind 32580  rpc    2u   CHR                1,3      0t0    3782 /dev/null
rpcbind 32580  rpc    3r   REG              253,0        0  786245 /var/run/rpcbind.lock
rpcbind 32580  rpc    4u  sock                0,6      0t0 3617563 can‘t identify protocol
rpcbind 32580  rpc    5u  unix 0xffff88001dfc3080      0t0 3617538 /var/run/rpcbind.sock
rpcbind 32580  rpc    6u  IPv4            3617540      0t0     UDP *:sunrpc
rpcbind 32580  rpc    7u  IPv4            3617542      0t0     UDP *:955
rpcbind 32580  rpc    8u  IPv4            3617543      0t0     TCP *:sunrpc (LISTEN)
rpcbind 32580  rpc    9u  IPv6            3617545      0t0     UDP *:sunrpc
rpcbind 32580  rpc   10u  IPv6            3617547      0t0     UDP *:955
rpcbind 32580  rpc   11u  IPv6            3617548      0t0     TCP *:sunrpc (LISTEN)

二。查漏补缺
TYPE列
sock    for a socket of unknown domain;
IPv4    for an IPv4 socket;
IPv6    for an open IPv6 network file - even if its address is IPv4, mapped in an IPv6 address;

程序占用内存分析
一。
  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 5061 root       1 -19  649m  17m  908 S  0.0  3.6   1:17.03 mfsmount
二。
[root@250-shiyan ~]# lsof -c mfsmount
COMMAND   PID USER   FD   TYPE             DEVICE SIZE/OFF    NODE NAME
mfsmount 5061 root  cwd    DIR              253,0     4096  781826 /root
mfsmount 5061 root  rtd    DIR              253,0     4096       2 /
mfsmount 5061 root  txt    REG              253,0   236648  403887 /usr/bin/mfsmount
mfsmount 5061 root  mem    REG              253,0    65928  260640 /lib64/libnss_files-2.12.so
mfsmount 5061 root  mem    REG              253,0  1921216  260624 /lib64/libc-2.12.so
mfsmount 5061 root  mem    REG              253,0   142640  260648 /lib64/libpthread-2.12.so
mfsmount 5061 root  mem    REG              253,0   596264  260632 /lib64/libm-2.12.so
mfsmount 5061 root  mem    REG              253,0    43832  260652 /lib64/librt-2.12.so
mfsmount 5061 root  mem    REG              253,0   258504  402028 /usr/lib64/libpcap.so.1.4.0
mfsmount 5061 root  mem    REG              253,0    19536  260630 /lib64/libdl-2.12.so
mfsmount 5061 root  mem    REG              253,0   221728  261115 /lib64/libfuse.so.2.8.3
mfsmount 5061 root  mem    REG              253,0   154520  260617 /lib64/ld-2.12.so
mfsmount 5061 root    0u   CHR                1,3      0t0    3782 /dev/null
mfsmount 5061 root    1u   CHR                1,3      0t0    3782 /dev/null
mfsmount 5061 root    2u   CHR                1,3      0t0    3782 /dev/null
mfsmount 5061 root    3r  FIFO                0,8      0t0 1586590 pipe
mfsmount 5061 root    4u  IPv4            1892119      0t0     TCP 192.168.2.250:44567->mfsmaster:9421 (ESTABLISHED)
mfsmount 5061 root    5u  unix 0xffff88001fb876c0      0t0 1616111 socket
mfsmount 5061 root    6u  IPv4            1616113      0t0     TCP localhost:44911 (LISTEN)
mfsmount 5061 root    8u   CHR             10,229      0t0    6954 /dev/fuse
[root@250-shiyan ~]# bc
bc 1.06.95
Copyright 1991-1994, 1997, 1998, 2000, 2004, 2006 Free Software Foundation, Inc.
This is free software with ABSOLUTELY NO WARRANTY.
For details type `warranty‘.
65928+1921216+142640+596264+43832+258504+19536+221728+154520
3424168
[root@250-shiyan ~]# ll /usr/bin/mfsmount
-rwxr-xr-x 1 root root 236648 Feb 10 19:27 /usr/bin/mfsmount
[root@250-shiyan ~]# size /usr/bin/mfsmount
   text    data     bss     dec     hex filename
 229679    4352 16923472        17157503        105cd7f /usr/bin/mfsmount
229679+4352+16923472=17157503
[root@250-shiyan ~]# pmap -x 5061
5061:   mfsmount /mnt/mfs1
Address           Kbytes     RSS   Dirty Mode   Mapping
----------------  ------  ------  ------
total kB          664836   17980   17072
三。分析
top中显示的某一个进程的RES列大小，与size某个文件显示的dec列是一样的，lsof中的SIZE列只是size命令中所显示的text列
即：
top-RES=size-dec
lsof-SIZE=size-text