package com.xxl.job.admin.core.thread; import com.xxl.job.admin.core.conf.XxlJobAdminConfig; import com.xxl.job.admin.core.model.XxlJobGroup; import com.xxl.job.admin.core.model.XxlJobInfo; import com.xxl.job.admin.core.model.XxlJobLog; import com.xxl.job.admin.core.trigger.TriggerTypeEnum; import com.xxl.job.admin.core.util.I18nUtil; import com.xxl.job.core.biz.model.ReturnT; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.mail.javamail.MimeMessageHelper; import javax.mail.internet.MimeMessage; import java.text.MessageFormat; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; /** * job monitor instance * * @author xuxueli 2015-9-1 18:05:56 */ public class JobFailMonitorHelper { private static Logger logger = LoggerFactory.getLogger(JobFailMonitorHelper.class); private static JobFailMonitorHelper instance = new JobFailMonitorHelper(); public static JobFailMonitorHelper getInstance(){ return instance; } // ---------------------- monitor ---------------------- private Thread monitorThread; private volatile boolean toStop = false; public void start(){ monitorThread = new Thread(new Runnable() { @Override public void run() { // monitor while (!toStop) { try { List failLogIds = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().findFailJobLogIds(1000); if (failLogIds!=null && !failLogIds.isEmpty()) { for (int failLogId: failLogIds) { // lock log int lockRet = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().updateAlarmStatus(failLogId, 0, -1); if (lockRet < 1) { continue; } XxlJobLog log = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().load(failLogId); XxlJobInfo info = XxlJobAdminConfig.getAdminConfig().getXxlJobInfoDao().loadById(log.getJobId()); // 1、fail retry monitor if (log.getExecutorFailRetryCount() > 0) { JobTriggerPoolHelper.trigger(log.getJobId(), TriggerTypeEnum.RETRY, (log.getExecutorFailRetryCount()-1), log.getExecutorShardingParam(), null); String retryMsg = "

>>>>>>>>>>>"+ I18nUtil.getString("jobconf_trigger_type_retry") +"<<<<<<<<<<<
"; log.setTriggerMsg(log.getTriggerMsg() + retryMsg); XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().updateTriggerInfo(log); } // 2、fail alarm monitor int newAlarmStatus = 0; // 告警状态:0-默认、-1=锁定状态、1-无需告警、2-告警成功、3-告警失败 if (info!=null && info.getAlarmEmail()!=null && info.getAlarmEmail().trim().length()>0) { boolean alarmResult = true; try { alarmResult = failAlarm(info, log); } catch (Exception e) { alarmResult = false; logger.error(e.getMessage(), e); } newAlarmStatus = alarmResult?2:3; } else { newAlarmStatus = 1; } XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().updateAlarmStatus(failLogId, -1, newAlarmStatus); } } TimeUnit.SECONDS.sleep(10); } catch (Exception e) { if (!toStop) { logger.error(">>>>>>>>>>> xxl-job, job fail monitor thread error:{}", e); } } } logger.info(">>>>>>>>>>> xxl-job, job fail monitor thread stop"); } }); monitorThread.setDaemon(true); monitorThread.start(); } public void toStop(){ toStop = true; // interrupt and wait monitorThread.interrupt(); try { monitorThread.join(); } catch (InterruptedException e) { logger.error(e.getMessage(), e); } } // ---------------------- alarm ---------------------- // email alarm template private static final String mailBodyTemplate = "
" + I18nUtil.getString("jobconf_monitor_detail") + ":" + "\n" + " " + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "
"+ I18nUtil.getString("jobinfo_field_jobgroup") +""+ I18nUtil.getString("jobinfo_field_id") +""+ I18nUtil.getString("jobinfo_field_jobdesc") +""+ I18nUtil.getString("jobconf_monitor_alarm_title") +""+ I18nUtil.getString("jobconf_monitor_alarm_content") +"
{0}{1}{2}"+ I18nUtil.getString("jobconf_monitor_alarm_type") +"{3}
"; /** * fail alarm * * @param jobLog */ private boolean failAlarm(XxlJobInfo info, XxlJobLog jobLog){ boolean alarmResult = true; // send monitor email if (info!=null && info.getAlarmEmail()!=null && info.getAlarmEmail().trim().length()>0) { String alarmContent = "Alarm Job LogId=" + jobLog.getId(); if (jobLog.getTriggerCode() != ReturnT.SUCCESS_CODE) { alarmContent += "
TriggerMsg=
" + jobLog.getTriggerMsg(); } if (jobLog.getHandleCode()>0 && jobLog.getHandleCode() != ReturnT.SUCCESS_CODE) { alarmContent += "
HandleCode=" + jobLog.getHandleMsg(); } Set emailSet = new HashSet(Arrays.asList(info.getAlarmEmail().split(","))); for (String email: emailSet) { XxlJobGroup group = XxlJobAdminConfig.getAdminConfig().getXxlJobGroupDao().load(Integer.valueOf(info.getJobGroup())); String personal = I18nUtil.getString("admin_name_full"); String title = I18nUtil.getString("jobconf_monitor"); String content = MessageFormat.format(mailBodyTemplate, group!=null?group.getTitle():"null", info.getId(), info.getJobDesc(), alarmContent); // make mail try { MimeMessage mimeMessage = XxlJobAdminConfig.getAdminConfig().getMailSender().createMimeMessage(); MimeMessageHelper helper = new MimeMessageHelper(mimeMessage, true); helper.setFrom(XxlJobAdminConfig.getAdminConfig().getEmailUserName(), personal); helper.setTo(email); helper.setSubject(title); helper.setText(content, true); XxlJobAdminConfig.getAdminConfig().getMailSender().send(mimeMessage); } catch (Exception e) { logger.error(">>>>>>>>>>> xxl-job, job fail alarm email send error, JobLogId:{}", jobLog.getId(), e); alarmResult = false; } } } // TODO, custom alarm strategy, such as sms return alarmResult; } }