From 62654cb7d3e23074278c7c061bef8f6cbee90d73 Mon Sep 17 00:00:00 2001
From: ludc
Date: 星期五, 10 十一月 2023 09:48:21 +0800
Subject: [PATCH] 修改历史数据导入时限制单次导入通过配置做限制，开启多线程分批并行执行insert，注解开启事务回滚失效，对象方式开启事务实现手动提交事务，异常回滚事务
---
Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmIOServiceImpl.java | 11 ++++++++++-
Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmEngineServiceImpl.java | 10 +++++++---
Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmProductCodeServiceImpl.java | 22 +++++++++++++++++++++-
3 files changed, 38 insertions(+), 5 deletions(-)
diff --git a/Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmEngineServiceImpl.java b/Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmEngineServiceImpl.java
index e594481..886df4a 100644
--- a/Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmEngineServiceImpl.java
+++ b/Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmEngineServiceImpl.java
@@ -69,7 +69,11 @@
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.Cache;
import org.springframework.stereotype.Service;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.annotation.Transactional;
+import org.springframework.transaction.support.DefaultTransactionDefinition;
+import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.util.CollectionUtils;
import javax.annotation.Resource;
@@ -101,7 +105,7 @@
/**
* 鍗曟sql鐨勬渶澶氬鍏ユ暟閲�
*/
- @Value("${import.maxNum}")
+ @Value("${batchadd.single_maxnum:500}")
private Integer MAX_IMPORT_NUM;
/**
@@ -153,13 +157,11 @@
@Autowired
private FormulaServiceImpl formulaService;
-
/**
* 瀵筼md涓彁渚涚殑feign鎺ュ彛杩涜璋冪敤锛屼互鍙婂鐞嗙浉鍏抽�昏緫
*/
@Autowired
ICodeReferBtmTypeService codeReferBtmTypeService;
-
/**
* 鐩镐技椤规煡璇㈣鍒�
@@ -3573,6 +3575,7 @@
void bactchExecuteInsert(String tableName, List<Map<String, String>> maps) throws ServiceException{
ExecutorService executor = Executors.newFixedThreadPool(10); // 鍒涘缓涓�涓浐瀹氬ぇ灏忕殑绾跨▼姹�
List<Map<String, String>> threadSafeMaps = new CopyOnWriteArrayList<>(maps);
+
for (int i = 0; i < threadSafeMaps.size(); i += MAX_IMPORT_NUM) {
final int startIndex = i;
final int endIndex = Math.min(i + MAX_IMPORT_NUM, maps.size());
@@ -3583,6 +3586,7 @@
commonsMapper.insertByBaseModel(tableName, threadSafeMaps.get(0), subList);
});
}
+
// 鍏抽棴绾跨▼姹�
executor.shutdown();
try {
diff --git a/Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmIOServiceImpl.java b/Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmIOServiceImpl.java
index 588fe6f..dc8f6e8 100644
--- a/Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmIOServiceImpl.java
+++ b/Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmIOServiceImpl.java
@@ -3,6 +3,7 @@
import com.alibaba.fastjson.JSONObject;
import com.alibaba.nacos.common.utils.StringUtils;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
+import com.google.protobuf.ServiceException;
import com.vci.ubcs.code.applyjtcodeservice.feign.IMdmInterJtClient;
import com.vci.ubcs.code.bo.AttributeValue;
import com.vci.ubcs.code.bo.CodeClassifyFullInfoBO;
@@ -96,8 +97,12 @@
@Value("${batchadd.exportattr.type:鍩烘湰淇℃伅}")
public String BATCHADD_EXCEPORT_ATTR_TYPE;
- @Value("${batchadd.redis.time:6000000}")
+
public int BATCHADD_REDIS_TIME;
+
+ @Value("${batchadd.import_data_limit:5001}")
+ private Integer IMPORT_DATA_LIMIT;
+
/**
* 涓婚搴撳垎绫荤殑鏈嶅姟
*/
@@ -794,6 +799,10 @@
|| sheetDataSetList.get(i).getRowData().size() < 1) {
continue;
}
+ // 鍗曟瀵煎叆鏁伴噺闄愬埗
+ if(sheetDataSetList.get(i).getRowData().size() > IMPORT_DATA_LIMIT){
+ throw new ServiceException("涓轰簡淇濊瘉绯荤粺鐨勭ǔ瀹氭�э紝璇蜂竴娆′笉瑕佸鍏ヨ秴杩�"+IMPORT_DATA_LIMIT+"鏉＄殑鏁版嵁");
+ }
//鍘嗗彶瀵煎叆鐨勬椂鍊欎笉澶勭悊缂栫爜
//----閫昏緫鍐呭----
//1. 鍒嗙被鐨勮矾寰勫彲浠ュ湪椤甸潰涓婇�夋嫨鏄垎绫荤紪鍙疯繕鏄垎绫荤殑鍚嶇О
diff --git a/Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmProductCodeServiceImpl.java b/Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmProductCodeServiceImpl.java
index f58d927..5c2fa06 100644
--- a/Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmProductCodeServiceImpl.java
+++ b/Source/UBCS/ubcs-service/ubcs-code/src/main/java/com/vci/ubcs/code/service/impl/MdmProductCodeServiceImpl.java
@@ -36,7 +36,10 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
+import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.annotation.Transactional;
+import org.springframework.transaction.support.DefaultTransactionDefinition;
+import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.util.CollectionUtils;
import javax.annotation.Resource;
@@ -80,6 +83,10 @@
*/
@Autowired
private FormulaServiceImpl formulaService;
+
+ // 娉ㄥ叆浜嬪姟绠＄悊鍣�
+ @Autowired
+ private TransactionTemplate transactionTemplate;
@Override
@Transactional(rollbackFor = Exception.class)
@@ -265,6 +272,10 @@
if(!CollectionUtils.isEmpty(editCodeDOs)){
codeAllCodeService.updateBatchById(editCodeDOs);
}
+ // 鑾峰彇浜嬪姟瀹氫箟
+ DefaultTransactionDefinition def = new DefaultTransactionDefinition();
+ // 寮�濮嬩簨鍔�
+ TransactionStatus status = transactionTemplate.getTransactionManager().getTransaction(def);
if(!CollectionUtils.isEmpty(addCodeDOs)){
Map<String, String> statusMap = addCodeDOs.stream().collect(Collectors.toMap(s -> s.getOid(), s -> s.getLcStatus()));
addCodeDOs.stream().filter(s -> StringUtils.equalsIgnoreCase("codeallcode",s.getBtmname())).forEach(s -> {
@@ -303,10 +314,19 @@
if(takeBack.size()>0){
codeAllCodeService.updateBatchById(takeBack);
}
+
codeAllCodeService.saveBatch(addCodeDOs);
}
- mdmEngineService.insertBatchByType(dataCBOList.get(0).getBtmname(),dataCBOList);
+ try {
+ mdmEngineService.insertBatchByType(dataCBOList.get(0).getBtmname(),dataCBOList);
+ // 鎻愪氦浜嬪姟
+ transactionTemplate.getTransactionManager().commit(status);
+ }catch (Exception e){
+ // 鍑虹幇寮傚父鏃跺洖婊氫簨鍔�
+ transactionTemplate.getTransactionManager().rollback(status);
+ }
+
}
return codeList;
}else {
--
Gitblit v1.9.3