Originally posted by: yangwtao
I have a AIX5300-12 sp6 syslog server is sufferring from CPU kernel usage is great than 70%.
Application is syslog and some shell script. No DB,no WAS etc...
I am a new comer for performance. Anybody can teach me how to find out the reason. Server setting as below listed. Thanks!
Topas Monitor for host: xxxxxxxxxxxxxx EVENTS/QUEUES FILE/TTY
Thu Jul 4 04:39:54 2013 Interval: 2 Cswitch 1815 Readch 4162.8K
Syscall 34609 Writech 15620
CPU User% Kern% Wait% Idle% Reads 2616 Rawin 0
ALL 16.0 84.0 0.0 0.0 Writes 101 Ttyout 323
Forks 285 Igets 0
Network KBPS I-Pack O-Pack KB-In KB-Out Execs 247 Namei 3665
en0 13.1 59.0 16.0 11.4 1.6 Runqueue 3.5 Dirblk 0
lo0 0.1 0.5 0.5 0.0 0.0 Waitqueue 0.0
Disk Busy% KBPS TPS KB-Read KB-Writ PAGING MEMORY
hdisk1 0.0 0.0 0.0 0.0 0.0 Faults 29890 Real,MB 1024
hdisk0 0.0 0.0 0.0 0.0 0.0 Steals 0 % Comp 39
cd0 0.0 0.0 0.0 0.0 0.0 PgspIn 0 % Noncomp 45
PgspOut 0 % Client 2
Name PID CPU% PgSp Owner PageIn 0
sh 25996 6.1 0.5 syslog PageOut 0 PAGING SPACE
syslogd 11612 0.8 0.4 root Sios 0 Size,MB 2560
kulagent 14020 0.4 6.0 root % Used 5
random 6098 0.3 0.0 root NFS (calls/sec) % Free 95
topas 28874 0.2 1.2 cn210903 ServerV2 0
$ vmstat -v
262144 memory pages
222549 lruable pages
39268 free pages
1 memory pools
77283 pinned pages
80.0 maxpin percentage
20.0 minperm percentage
80.0 maxperm percentage
53.2 numperm percentage
118495 file pages
0.0 compressed percentage
0 compressed pages
3.0 numclient percentage
80.0 maxclient percentage
6762 client pages
0 remote pageouts scheduled
145 pending disk I/Os blocked with no pbuf
104 paging space I/Os blocked with no psbuf
95545 filesystem I/Os blocked with no fsbuf
0 client filesystem I/Os blocked with no fsbuf
29 external pager filesystem I/Os blocked with no fsbuf
0 Virtualized Partition Memory Page Faults
0.00 Time resolving virtualized partition memory page faults
$ no -a
arpqsize = 12
arpt_killc = 20
arptab_bsiz = 7
arptab_nb = 149
bcastping = 0
clean_partial_conns = 0
delayack = 0
delayackports = {}
dgd_packets_lost = 3
dgd_ping_time = 5
dgd_retry_time = 5
directed_broadcast = 0
extendednetstats = 1
fasttimo = 200
icmp6_errmsg_rate = 10
icmpaddressmask = 0
ie5_old_multicast_mapping = 0
ifsize = 256
inet_stack_size = 16
ip6_defttl = 64
ip6_prune = 1
ip6forwarding = 0
ip6srcrouteforward = 1
ip_ifdelete_notify = 0
ip_nfrag = 200
ipforwarding = 0
ipfragttl = 2
ipignoreredirects = 0
ipqmaxlen = 100
ipsendredirects = 1
ipsrcrouteforward = 1
ipsrcrouterecv = 0
ipsrcroutesend = 1
llsleep_timeout = 3
lo_perf = 1
lowthresh = 90
main_if6 = 0
main_site6 = 0
maxnip6q = 20
maxttl = 255
medthresh = 95
mpr_policy = 1
multi_homed = 1
nbc_limit = 131072
nbc_max_cache = 131072
nbc_min_cache = 1
nbc_ofile_hashsz = 12841
nbc_pseg = 0
nbc_pseg_limit = 262144
ndd_event_name = {all}
ndd_event_tracing = 0
ndp_mmaxtries = 3
ndp_umaxtries = 3
ndpqsize = 50
ndpt_down = 3
ndpt_keep = 120
ndpt_probe = 5
ndpt_reachable = 30
ndpt_retrans = 1
net_buf_size = {all}
net_buf_type = {all}
net_malloc_police = 0
nonlocsrcroute = 0
nstrpush = 8
passive_dgd = 0
pmtu_default_age = 10
pmtu_expire = 10
pmtu_rediscover_interval = 30
psebufcalls = 20
psecache = 1
pseintrstack = 12288
psetimers = 20
rfc1122addrchk = 0
rfc1323 = 0
rfc2414 = 1
route_expire = 1
routerevalidate = 0
rto_high = 64
rto_length = 13
rto_limit = 7
rto_low = 1
sack = 0
sb_max = 1048576
send_file_duration = 300
site6_index = 0
sockthresh = 85
sodebug = 0
sodebug_env = 0
somaxconn = 1024
strctlsz = 1024
strmsgsz = 0
strthresh = 85
strturncnt = 15
subnetsarelocal = 1
tcp_bad_port_limit = 0
tcp_ecn = 0
tcp_ephemeral_high = 65535
tcp_ephemeral_low = 32768
tcp_finwait2 = 1200
tcp_icmpsecure = 0
tcp_init_window = 0
tcp_inpcb_hashtab_siz = 24499
tcp_keepcnt = 8
tcp_keepidle = 14400
tcp_keepinit = 150
tcp_keepintvl = 150
tcp_limited_transmit = 1
tcp_low_rto = 0
tcp_maxburst = 0
tcp_mssdflt = 1460
tcp_nagle_limit = 65535
tcp_nagleoverride = 0
tcp_ndebug = 100
tcp_newreno = 1
tcp_nodelayack = 0
tcp_pmtu_discover = 1
tcp_recvspace = 16384
tcp_sendspace = 16384
tcp_tcpsecure = 0
tcp_timewait = 1
tcp_ttl = 60
tcprexmtthresh = 3
tcptr_enable = 0
thewall = 524288
timer_wheel_tick = 0
udp_bad_port_limit = 0
udp_ephemeral_high = 65535
udp_ephemeral_low = 32768
udp_inpcb_hashtab_siz = 24499
udp_pmtu_discover = 1
udp_recvspace = 42080
udp_sendspace = 9216
udp_ttl = 30
udpcksum = 1
use_isno = 1
use_sndbufpool = 1
$
$ vmstat 1 10
System configuration: lcpu=1 mem=1024MB
kthr memory page faults cpu
----- ----------- ------------------------ ------------ -----------
r b avm fre re pi po fr sr cy in sy cs us sy id wa
5 0 132019 38983 0 0 0 0 0 0 101 33461 1774 9 91 0 0
4 0 131937 39065 0 0 0 0 0 0 62 34553 1818 9 91 0 0
2 0 132022 38980 0 0 0 0 0 0 107 35060 1856 23 77 0 0
5 0 131987 39015 0 0 0 0 0 0 67 35015 1837 14 86 0 0
5 0 132003 38999 0 0 0 0 0 0 80 34604 1816 23 77 0 0
4 0 132003 38999 0 0 0 0 0 0 59 34630 1828 16 84 0 0
4 0 131986 39015 0 0 0 0 0 0 64 34603 1837 13 87 0 0
4 0 131966 39034 0 0 0 0 0 0 75 34464 1848 16 84 0 0
5 0 131981 39018 0 0 0 0 0 0 82 34496 1831 10 90 0 0
5 0 132025 38973 0 0 0 0 0 0 82 34568 1840 10 90 0 0
$
$ oslevel -s
5300-12-06-1216
$
XXXXXXXXXXX:/ # vmo -a
cpu_scale_memp = 8
data_stagger_interval = 161
defps = 1
force_relalias_lite = 0
framesets = 2
htabscale = -1
kernel_heap_psize = n/a
kernel_psize = n/a
large_page_heap_size = n/a
lgpg_regions = 0
lgpg_size = 0
low_ps_handling = 1
lru_file_repage = 1
lru_poll_interval = 10
lrubucket = 131072
maxclient% = 80
maxfree = 1088
maxperm = 178039
maxperm% = 80
maxpin = 216811
maxpin% = 80
mbuf_heap_psize = n/a
memory_affinity = 1
memory_frames = 262144
memplace_data = 2
memplace_mapped_file = 2
memplace_shm_anonymous = 2
memplace_shm_named = 2
memplace_stack = 2
memplace_text = 2
memplace_unmapped_file = 2
mempools = 1
minfree = 960
minperm = 44509
minperm% = 20
nokilluid = 0
npskill = 5120
npsrpgmax = 40960
npsrpgmin = 30720
npsscrubmax = 40960
npsscrubmin = 30720
npswarn = 20480
num_spec_dataseg = n/a
numpsblks = 655360
page_steal_method = 0
pagecoloring = 0
pinnable_frames = 188697
psm_timeout_interval = n/a
pta_balance_threshold = 50
relalias_percentage = 0
rpgclean = 0
rpgcontrol = 2
scrub = 0
scrubclean = 0
soft_min_lgpgs_vmpool = 0
spec_dataseg_int = n/a
strict_maxclient = 1
strict_maxperm = 0
v_pinshm = 0
vm_modlist_threshold = -1
vmm_fork_policy = 1
vmm_mpsize_support = n/a
wlm_memlimit_nonpg = 1
sgsnidcwbzsyslog:/ # ioo -a
j2_atimeUpdateSymlink = 0
j2_dynamicBufferPreallocation = 16
j2_inodeCacheSize = 400
j2_maxPageReadAhead = 128
j2_maxRandomWrite = 0
j2_maxUsableMaxTransfer = 512
j2_metadataCacheSize = 400
j2_minPageReadAhead = 2
j2_nBufferPerPagerDevice = 512
j2_nPagesPerWriteBehindCluster = 32
j2_nRandomCluster = 0
j2_nonFatalCrashesSystem = 0
j2_syncModifiedMapped = 1
j2_syncPageCount = 0
j2_syncPageLimit = 256
j2_syncdLogSyncInterval = 1
j2_unmarkComp = 0
jfs_clread_enabled = 0
jfs_use_read_lock = 1
lvm_bufcnt = 9
maxpgahead = 8
maxrandwrt = 0
memory_frames = 262144
minpgahead = 2
numclust = 1
numfsbufs = 186
pd_npages = 65536
pgahd_scale_thresh = 0
pv_min_pbuf = 256
sync_release_ilock = 0
XXXXXXXXXXXXX:/ # schedo -a
%usDelta = 100
affinity_lim = 7
allowMCMmigrate = 0
allow_vmx = 1
big_tick_size = 1
ded_cpu_donate_thresh = 80
fast_locks = n/a
fixed_pri_global = 0
force_grq = 0
hotlocks_enable = n/a
idle_migration_barrier = 4
krlock_confer2self = n/a
krlock_conferb4alloc = n/a
krlock_enable = n/a
krlock_spinb4alloc = n/a
krlock_spinb4confer = n/a
maxspin = 16384
n_idle_loop_vlopri = n/a
pacefork = 10
proc_disk_stats = 1
sched_D = 16
sched_R = 16
search_globalrq_mload = n/a
search_smtrunq_mload = n/a
setnewrq_sidle_mload = n/a
shed_primrunq_mload = n/a
sidle_S1runq_mload = n/a
sidle_S2runq_mload = n/a
sidle_S3runq_mload = n/a
sidle_S4runq_mload = n/a
slock_spinb4confer = n/a
smt_snooze_delay = n/a
smtrunq_load_diff = n/a
tb_balance_S0 = 0
tb_balance_S1 = 2
tb_threshold = 100
timeslice = 1
unboost_inflih = n/a
v_exempt_secs = 2
v_min_process = 2
v_repage_hi = 0
v_repage_proc = 4
v_sec_wait = 1
vpm_fold_policy = 1
vpm_xvcpus = 0
XXXXXXXXXXXX:/ #