记一次线程爆满导致服务器崩溃的问题排查及解决

fork faild:Cannot allocate memory

echo 1000000 > /proc/sys/kernel/pid_max

echo "kernel.pid_max=1000000 " >> /etc/sysctl.conf
sysctl -p

ps -Lef |awk ‘{sum[$2]++}END{for(pid in sum) print pid, sum[pid]}'|sort -nr -k 2|head -n 20

[root@se-test-lky01 ~]# ps -Lef |awk '{sum[$2]++}END{for(pid in sum) print pid, sum[pid]}'|sort -nr -k 2|head -n 20
16074 3100
31386 1226
20120 1072
19548 985
9697 829
3005 796
641 344
19016 324
16924 315
17870 300
6417 293
8351 171
7332 168
18259 167
19821 161
16311 157
18433 151
18048 136
14347 104
2559 100

[root@se-test-lky01 ~]#jstack 16074 >thread_dump.log

"Thread-4655" #4774 prio=5 os_prio=0 tid=0x00007f84aa2fe000 nid=0xd408b waiting for monitor entry [0x00007f802b704000]
   java.lang.Thread.State: BLOCKED (on object monitor)
	at cn.cloudwalk.bat.util.http.FtpUtil.connect(FtpUtil.java:246)
	- waiting to lock <0x00000006c09c1888> (a java.lang.Class for cn.cloudwalk.bat.util.http.FtpUtil)
	at cn.cloudwalk.bat.schedule.ftp.process.FtpMonitorProcess$1.run(FtpMonitorProcess.java:85)
	at java.lang.Thread.run(Thread.java:748)

"Thread-4654" #4773 prio=5 os_prio=0 tid=0x00007f84aa2fc000 nid=0xd408a waiting for monitor entry [0x00007f802b805000]
   java.lang.Thread.State: BLOCKED (on object monitor)
	at cn.cloudwalk.bat.util.http.FtpUtil.connect(FtpUtil.java:246)
	- waiting to lock <0x00000006c09c1888> (a java.lang.Class for cn.cloudwalk.bat.util.http.FtpUtil)
	at cn.cloudwalk.bat.schedule.ftp.process.FtpMonitorProcess$2.run(FtpMonitorProcess.java:114)
at java.lang.Thread.run(Thread.java:748)

	private void listDeviceFiles() {
		
		new Thread(new Runnable() {
			@Override
			public void run() {
				logger.debug("开始获取[ftp-设备]文件...");
				try {
					String workDir = ftpConfig.getWorkdir();
					// 连接
					FTPClient ftpClient = FtpUtil.connect(ftpConfig);
					ftpClient.changeWorkingDirectory(workDir);
					ftpClient.changeWorkingDirectory(SubscribeDataTypeEnum.DEVICE_INFO.getKey().toString());
					FTPFile[] files = ftpClient.listFiles();
					for(FTPFile file : files) {
						decomposeFile(file,ftpClient);
					}
					ftpClient.logout();
				} catch (Exception e) {
					logger.error("ftp获取文件名出错：" + e.getMessage());
				}
			}
		}).start();
	}