问题背景：消息队列数据积压，经过排查发现小票在回传查询时耗时较长，最长可达42s。经排查发现是数据库数据已经上千万级别，并发较多，导致查询变慢。

解决方案：备份线上数据，删除线上数据，重新启动程序。

问题背景：积压问题得到解决后，查看日志发现，高并发状态下，数据库连接池不稳定，容易连接超时。

解决方案：修改数据库连接池大小，修改数据库连接超时时间，检查回滚事务是否是最小粒度，确保数据库连接占用时间最短，保证数据库并发。

src/main/java/com/pavis/ai/kwp/ioc/common/config/AsyncConfig.java

@@ -5,6 +5,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 
 import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadPoolExecutor;
 
 @Configuration
 public class AsyncConfig {
@@ -13,16 +14,36 @@ public class AsyncConfig {
     @Bean("irRequest")
     public Executor recAndSaveExecutor() {
         ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
-        //核心线程数2：线程池创建时候初始化的线程数
-        executor.setCorePoolSize(2);
-        //最大线程数3：线程池最大的线程数，只有在缓冲队列满了之后才会申请超过核心线程数的线程
-        executor.setMaxPoolSize(3);
-        //缓冲队列5000：用来缓冲执行任务的队列
-        executor.setQueueCapacity(10000);
+        //核心线程数1：线程池创建时候初始化的线程数
+        executor.setCorePoolSize(1);
+        //最大线程数2：线程池最大的线程数，只有在缓冲队列满了之后才会申请超过核心线程数的线程
+        executor.setMaxPoolSize(2);
+        //缓冲队列100：用来缓冲执行任务的队列
+        executor.setQueueCapacity(100);
         //允许线程的空闲时间180秒：当超过了核心线程出之外的线程在空闲时间到达之后会被销毁
         executor.setKeepAliveSeconds(180);
         //线程池名的前缀：设置好了之后可以方便我们定位处理任务所在的线程池
         executor.setThreadNamePrefix("OcrGoodsAsync-");
+        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
+        executor.initialize();
+        return executor;
+    }
+
+    // 声明一个线程池(并指定线程池的名字)
+    @Bean("ocrSave")
+    public Executor saveOcrRetExecutor() {
+        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+        //核心线程数5：线程池创建时候初始化的线程数
+        executor.setCorePoolSize(5);
+        //最大线程数8：线程池最大的线程数，只有在缓冲队列满了之后才会申请超过核心线程数的线程
+        executor.setMaxPoolSize(8);
+        //缓冲队列200：用来缓冲执行任务的队列
+        executor.setQueueCapacity(200);
+        //允许线程的空闲时间180秒：当超过了核心线程出之外的线程在空闲时间到达之后会被销毁
+        executor.setKeepAliveSeconds(180);
+        //线程池名的前缀：设置好了之后可以方便我们定位处理任务所在的线程池
+        executor.setThreadNamePrefix("OcrTripAsync-");
+        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
         executor.initialize();
         return executor;
     }

src/main/java/com/pavis/ai/kwp/ioc/common/jobs/ReRecJobConfig.java

@@ -22,8 +22,8 @@ public class ReRecJobConfig {
                 .newTrigger()
                 .forJob(reRecJobDetail())
                 .withIdentity("reRecJobTrigger")
-                // 每1分钟执行一次 0 */1 * * * ?
-                .withSchedule(CronScheduleBuilder.cronSchedule("0 */1 * * * ?"))
+                // 每1分钟执行一次 0 */1 * * * ? 每天凌晨3点执行 0 0 3 * * ?
+                .withSchedule(CronScheduleBuilder.cronSchedule("0 0 3 * * ?"))
                 .build();
     }
 }

src/main/java/com/pavis/ai/kwp/ioc/service/impl/CallbackServiceImpl.java

@@ -72,8 +72,11 @@ public class CallbackServiceImpl implements CallbackService {
             CbLog cbLog = initCbLog(recLog);
             cbLog.setCStartTime(DateTimeUtils.dateTimeNow());
             try {
+                log.info("upload_id:{},query callback start time:{}", uploadId, DateTimeUtils.dateTimeNow());
                 Map<String, Object> cbDet = adapter(cbLog);
+                log.info("upload_id:{},query callback finish time:{}", uploadId, DateTimeUtils.dateTimeNow());
                 ctrService.startCallback(cbDet);
+                log.info("upload_id:{},callback finish time:{}", uploadId, DateTimeUtils.dateTimeNow());
                 cbLog.setCDone(true);
             } catch (Exception e) {
                 log.error("upload_id:{},回传失败", cbLog.getUploadId(), e);

src/main/java/com/pavis/ai/kwp/ioc/service/impl/OcrGoodsServiceImpl.java

@@ -17,6 +17,7 @@ import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationEventPublisher;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
@@ -44,7 +45,7 @@ public class OcrGoodsServiceImpl implements OcrGoodsService {
     private SubImgRetMapper subImgRetMapper;
 
 
-    // @Async("irRequest")
+    @Async("irRequest")
     @Override
     public void recAndSave(OcrGoods ocrGoods) {
         log.info("start rec and save");

src/main/java/com/pavis/ai/kwp/ioc/service/impl/OcrTripServiceImpl.java

@@ -6,14 +6,21 @@ import com.pavis.ai.kwp.ioc.common.event.OcrEvent;
 import com.pavis.ai.kwp.ioc.common.utils.DateTimeUtils;
 import com.pavis.ai.kwp.ioc.dto.meta.OcrRetDto;
 import com.pavis.ai.kwp.ioc.dto.meta.OcrTripDto;
-import com.pavis.ai.kwp.ioc.mapper.*;
-import com.pavis.ai.kwp.ioc.model.*;
+import com.pavis.ai.kwp.ioc.mapper.OcrTripDetMapper;
+import com.pavis.ai.kwp.ioc.mapper.OcrTripMapper;
+import com.pavis.ai.kwp.ioc.mapper.OcrTripProbMapper;
+import com.pavis.ai.kwp.ioc.mapper.TripLogMapper;
+import com.pavis.ai.kwp.ioc.model.OcrTrip;
+import com.pavis.ai.kwp.ioc.model.OcrTripDet;
+import com.pavis.ai.kwp.ioc.model.OcrTripProb;
+import com.pavis.ai.kwp.ioc.model.TripLog;
 import com.pavis.ai.kwp.ioc.service.OcrTripService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationEventPublisher;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -35,6 +42,7 @@ public class OcrTripServiceImpl extends ServiceImpl<OcrTripMapper, OcrTrip> impl
     // @Autowired
     // private UpMainMapper upMainMapper;
 
+    @Async("ocrSave")
     @Override
     public void saveOcrRet(OcrRetDto ret) {
         // 获取OCR结果

src/main/resources/application-prod.yml

@@ -10,6 +10,9 @@ spring:
     driver-class-name: com.mysql.jdbc.Driver
     username: root
     password: Semi.1001
+    hikari:
+      maximum-pool-size: 100
+      connection-timeout: 40000
   rabbitmq:
     host: 180.76.139.81
     port: 5672

src/main/resources/application-test.yml

@@ -10,6 +10,9 @@ spring:
     driver-class-name: com.mysql.jdbc.Driver
     username: root
     password: iocPue0934r
+    hikari:
+      maximum-pool-size: 100
+      connection-timeout: 40000
   rabbitmq:
     host: 172.19.0.7
     port: 5672

src/test/java/com/pavis/ai/kwp/ioc/KwpIocApplicationTests.java

@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON;
 import com.pavis.ai.kwp.ioc.common.utils.DateTimeUtils;
 import com.pavis.ai.kwp.ioc.common.utils.ImageUtils;
 import com.pavis.ai.kwp.ioc.dto.meta.IrRes;
+import com.pavis.ai.kwp.ioc.service.CallbackAdapter;
 import com.pavis.ai.kwp.ioc.service.CtrService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
@@ -21,6 +22,9 @@ public class KwpIocApplicationTests {
     @Autowired
     private CtrService ctrService;
 
+    @Autowired
+    private CallbackAdapter callbackAdapter;
+
     @Test
     public void testCtrService() {
         try {
@@ -51,4 +55,12 @@ public class KwpIocApplicationTests {
         log.info(DateTimeUtils.nonEtDatetime("2020-11-20 10:23:34"));
     }
 
+    @Test
+    public void testCbDet() {
+        String uploadId = "8078278225779630080";
+        log.info("upload_id:{},query callback start time:{}", uploadId, DateTimeUtils.dateTimeNow());
+        callbackAdapter.det(uploadId, true);
+        log.info("upload_id:{},query callback finish time:{}", uploadId, DateTimeUtils.dateTimeNow());
+    }
+
 }