当前位置：网站首页>Multi thread import data and generate error files for redis storage
Multi thread import data and generate error files for redis storage
2022-07-27 18:09:00 【@Calm down】
Import excel file :
@Override
@Transactional(rollbackFor = Exception.class)
public Object importShopBlack(MultipartFile file, HttpServletResponse response, long userId) {
try {
// The total number of definitions is 0;
int coun = 0;
int fai = 0;
Map<Object, Object> map1 = new HashMap<>();
ArrayList<FwdcBlacklist> succlist = new ArrayList<>();
List<Map<String,String>> errorList = new ArrayList<>();
List<String> shopIdUpdate = new ArrayList<>();
String fileName = file.getOriginalFilename();
if (!fileName.matches("^.+\\.(?i)(xls)$") && !fileName.matches("^.+\\.(?i)(xlsx)$")) {
return ResultDataUtils.failResult(" The format of the uploaded file is incorrect ");
}
InputStream is = file.getInputStream();
Workbook wb = null;
wb = new HSSFWorkbook(is);
Sheet sheet = wb.getSheetAt(0);
if (sheet == null) {
throw new RuntimeException(" File data error ");
}
coun=sheet.getLastRowNum();
ExecutorService pool=new ThreadPoolExecutor(5,10,
1L, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(3),
Executors.defaultThreadFactory(),
new ThreadPoolExecutor.AbortPolicy());
final CountDownLatch endGate = new CountDownLatch(coun);
ArrayList<Map<String, String>> maps = new ArrayList<>();
for (int r = 1; r <= sheet.getLastRowNum(); r++) {
Row row = sheet.getRow(r);
if (isRowEmpty(row)) {
continue;
}
// obtain excel Data in the
Map<String,String> map = new HashMap<>();
map.put("sdmerid",getRowValue(row,0));
map.put("shopName",getRowValue(row,1));
map.put("creditCode",getRowValue(row,2));
map.put("idcard",getRowValue(row,3));
map.put("isOffline",getRowValue(row,4));
map.put("isDraw",getRowValue(row,5));
map.put("isNet",getRowValue(row,6));
maps.add(map);
}
pool.submit(()->{
for (Map<String, String> map : maps) {
try {
// Start to judge whether the data is correct
// The first step is to determine whether to fill in the business license number or ID number , One of the two is required
if (StrUtil.isEmpty(map.get("creditCode")) && StrUtil.isEmpty(map.get("idcard"))){
map.put("error"," Business license number or ID number is required ");
errorList.add(map);
continue;
}
// The second step is to determine whether to go offline , Whether to shield finance , Whether it is allowed to access the network again, and whether the three items are filled
if (StrUtil.isEmpty(map.get("isOffline")) || StrUtil.isEmpty(map.get("isDraw")) || StrUtil.isEmpty(map.get("isNet"))){
map.put("error"," Are you offline , Whether to shield finance , Three items must be filled in whether to allow re access ");
errorList.add(map);
continue;
}
// Three, four, five step query sql, Judge whether the commercial code is filled in , Do not enter the following judgment without filling in the commercial code
List<Map<String,Object>> shopDate=new ArrayList<>();
shopDate=fwdcBlacklistMapper.getShopDate(map.get("creditCode"),map.get("idcard"),null);
if (StrUtil.isNotEmpty(map.get("sdmerid"))){
shopDate=fwdcBlacklistMapper.getShopDate(null,null,map.get("sdmerid"));
if (CollUtil.isNotEmpty(shopDate)){
// Step 4: if you fill in the ID card, judge whether the ID card matches the commercial code
if (StrUtil.isNotEmpty(map.get("idcard")) && StrUtil.isNotEmpty(map.get("sdmerid")) && !(map.get("idcard").equals(shopDate.get(0).get("cardid")))){
map.put("error"," The ID card does not match the commercial code ");
errorList.add(map);
continue;
}
// Step 3: if you fill in the business license, judge whether the business license matches the commercial code
if (StrUtil.isNotEmpty(map.get("creditCode")) && StrUtil.isNotEmpty(map.get("sdmerid")) && !(map.get("creditCode").equals(shopDate.get(0).get("creditcode")))){
map.put("error"," The business license does not match the commercial code ");
errorList.add(map);
continue;
}
// Step 5 match the commercial code with the merchant name
if (StrUtil.isNotEmpty(map.get("shopName")) && StrUtil.isNotEmpty(map.get("sdmerid")) && !(map.get("shopName").equals(shopDate.get(0).get("shopName")))){
map.put("error"," The commercial code does not match the merchant name ");
errorList.add(map);
continue;
}
}else {
map.put("error"," Commercial editing does not exist ");
errorList.add(map);
continue;
}
}
// errorList.add(map);
// Assemble shielding content
ArrayList<String> arr = new ArrayList<>();
if (StrUtil.isNotEmpty(map.get("isOffline")) && map.get("isOffline").toString().equals(" yes ")){
arr.add("1");
}
if (StrUtil.isNotEmpty(map.get("isDraw"))&& map.get("isDraw").toString().equals(" yes ")){
arr.add("2");
}
if (StrUtil.isNotEmpty(map.get("isNet"))&& map.get("isNet").toString().equals(" no ")){
arr.add("3");
}
if(arr == null || arr.size()<= 0){
map.put("error"," Shielding functions cannot all be invalid ");
errorList.add(map);
continue;
}
// insert data
// adopt shopid Go to query the associated merchant information , At the same time, the merchant is not in the blacklist
if (CollUtil.isNotEmpty(shopDate)){
for (Map<String, Object> selectByShopId : shopDate) {
FwdcBlacklist fwdcBlacklist = new FwdcBlacklist();
fwdcBlacklist.setShopId(Long.valueOf(selectByShopId.get("shopId").toString()));
fwdcBlacklist.setId(IdUtil.getSnowflake().nextIdStr());
fwdcBlacklist.setShieldFunction(String.join(",",arr));
fwdcBlacklist.setRealShieldFunction(String.join(",",arr));
fwdcBlacklist.setAddTime(new Date());
fwdcBlacklist.setAddUserId(userId);
fwdcBlacklist.setCreditCode(selectByShopId.get("creditcode")==null?"":selectByShopId.get("creditcode").toString());
fwdcBlacklist.setIdcard(selectByShopId.get("cardid")==null?"":selectByShopId.get("cardid").toString());
fwdcBlacklist.setSdmerid(selectByShopId.get("sdmerid")==null?"":selectByShopId.get("sdmerid").toString());
fwdcBlacklist.setShopName(selectByShopId.get("shopName")==null?"":selectByShopId.get("shopName").toString());
fwdcBlacklist.setUpdateTime(new Date());
fwdcBlacklist.setUpdateUserId(userId);
succlist.add(fwdcBlacklist);
// Add merchants to the offline array
if(arr.contains("1")){
shopIdUpdate.add(fwdcBlacklist.getShopId().toString());
}
}
}else if (StrUtil.isNotBlank(map.get("idcard"))){
FwdcBlacklist fwdcBlacklist = new FwdcBlacklist();
fwdcBlacklist.setId(IdUtil.getSnowflake().nextIdStr());
fwdcBlacklist.setShieldFunction(String.join(",",arr));
fwdcBlacklist.setRealShieldFunction(String.join(",",arr));
fwdcBlacklist.setAddTime(new Date());
fwdcBlacklist.setAddUserId(userId);
fwdcBlacklist.setIdcard(map.get("idcard"));
fwdcBlacklist.setUpdateTime(new Date());
fwdcBlacklist.setUpdateUserId(userId);
succlist.add(fwdcBlacklist);
}
} catch (Exception e) {
e.printStackTrace();
}finally {
endGate.countDown();
}
}
return errorList;
});
try {
endGate.await();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
pool.shutdown();
String error = JSON.toJSONString(errorList);
// The number of judgment failures is greater than 0, Generate the failure table , Do not import data
if (coun-succlist.size()>0){
// use redis Cache failed data
String shopBlackErrorFile = "shopBlack"+userId;
redisUtil.set(shopBlackErrorFile, error ); // No new return true, There is returned false
}else {
// If there is no failed data, insert the data ,
for (FwdcBlacklist fwdcBlacklist : succlist) {
baseMapper.insertFwdcBlacklist(fwdcBlacklist);
}
}
// Insert suc, Number of failures fai
map1.put("suc", coun);
map1.put("fai", coun-succlist.size());
map1.put("addUserId", userId);
return ResultDataUtils.successResult(map1);
} catch (Exception e) {
logger.info("【{} Failed to batch import blacklist merchant information !】", "");
return ResultDataUtils.failResult(e.getMessage());
}
}
/** * Get row and column values **/
public static String getRowValue(Row row, int i) {
if (row.getCell(i) == null) {
return null;
}
row.getCell(i).setCellType(Cell.CELL_TYPE_STRING);
return row.getCell(i) == null || row.getCell(i).toString().isEmpty() ? null
: row.getCell(i).getStringCellValue();
}
Export error file :
/** * Blacklist error information table Export * * @param request * @return */
@RequestMapping("/errorFile")
public void errMsgExportExcel(HttpServletRequest request, HttpServletResponse response) {
logger.info("【 Blacklist error information table import 】 Export error message starts ......");
try {
Map<String, Object> map = new HashMap<>();
long startTime = System.currentTimeMillis();
// assemble redis key
String addUserId = request.getParameter("addUserId") == null ? null : request.getParameter("addUserId");
String keyRedis="shopBlack"+addUserId;
List<Map> list =null;
if (redisUtil.hasKey(keyRedis)){
list = JSONArray.parseArray(redisUtil.get(keyRedis).toString(), Map.class);
}
HSSFWorkbook wb = new HSSFWorkbook();
HSSFCellStyle cellStyle = wb.createCellStyle();
cellStyle.setVerticalAlignment(HSSFCellStyle.VERTICAL_CENTER);
HSSFSheet sheet = wb.createSheet(" Error message table ");
for (int i = 0; i <= 22; i++) {
sheet.setColumnWidth(i, 20 * 256);
}
HSSFRow rowNext = null;
HSSFRow row = sheet.createRow(0);
row.setHeight((short) (26.25 * 20));
// Set column name
row.createCell(0).setCellValue(" Yisheng business editor ");
row.createCell(1).setCellValue(" Merchant name ");
row.createCell(2).setCellValue(" Business license number ");
row.createCell(3).setCellValue(" ID number ");
row.createCell(4).setCellValue(" Are you offline ");
row.createCell(5).setCellValue(" Whether to shield finance ");
row.createCell(6).setCellValue(" Is it allowed to access the network again ");
row.createCell(7).setCellValue(" error message ");
if (CollUtil.isNotEmpty(list) && list.size() > 0) {
for (int i = 0; i < list.size(); i++) {
rowNext = sheet.createRow(i + 1);
rowNext.setHeight((short) (26.25 * 15));
rowNext.createCell(0).setCellValue(
list.get(i).get("sdmerid") == null || list.get(i).get("sdmerid") == "" ? null
: list.get(i).get("sdmerid").toString());
rowNext.createCell(1).setCellValue(
list.get(i).get("shopName") == null || list.get(i).get("shopName") == "" ? null
: list.get(i).get("shopName").toString());
rowNext.createCell(2).setCellValue(
list.get(i).get("creditCode") == null || list.get(i).get("creditCode") == "" ? null
: list.get(i).get("creditCode").toString());
rowNext.createCell(3).setCellValue(
list.get(i).get("idcard") == null || list.get(i).get("idcard") == "" ? null
: list.get(i).get("idcard").toString());
rowNext.createCell(4).setCellValue(
list.get(i).get("isOffline") == null || list.get(i).get("isOffline") == "" ? null
: list.get(i).get("isOffline").toString());
rowNext.createCell(5).setCellValue(
list.get(i).get("isDraw") == null || list.get(i).get("isDraw") == "" ? null
: list.get(i).get("isDraw").toString());
rowNext.createCell(6).setCellValue(
list.get(i).get("isNet") == null || list.get(i).get("isNet") == "" ? null
: list.get(i).get("isNet").toString());
rowNext.createCell(7).setCellValue(
list.get(i).get("error") == null || list.get(i).get("error") == "" ? null
: list.get(i).get("error").toString());
}
}
redisUtil.del(keyRedis);
response.setContentType("application/vnd.ms-excel;charset=utf-8");
OutputStream os = response.getOutputStream();
response.setHeader("Content-disposition",
"attachment;filename=" + DownChineseEncode.setFileDownHeader(request, response, " Error message table .xlsx"));
wb.write(os);
os.flush();
os.close();
long endTime = System.currentTimeMillis();
// Delete redis cache
logger.info("【 Blacklist error information table import and distribution 】 End of export error message , Time consuming ：【{}】", (endTime - startTime));
} catch (Exception e) {
logger.error("【 Blacklist error information table import and distribution 】 Export error message exception ：【{}】", e);
}
}
边栏推荐
- Salesforce certified sharing and visibility Designer (su20) certification examination summary
- 【Codeforces】 B. Make it Divisible by 25
- 知物由学 | 小游戏的安全风险在哪里？
- MySQL creates a student table and queries grades
- 2022 safety officer-c certificate special operation certificate examination question bank and answers
- golang 并发缓存击穿或合并请求
- [introduction to database system (Wang Shan)] Chapter 4 - Database Security
- Fast analysis combined with Haidian medicine
- [Southwest University] information sharing of postgraduate entrance examination and re examination
- The global cloud market is growing rapidly, and data security has entered a strong regulatory era of rule of law
猜你喜欢
快解析结合华途文档加密软件
WPF makes login interface
2022 safety officer-a certificate examination questions and online simulation examination
Convolutional neural network -- from r-cnn, fast r-cnn to fast r-cnn, mask r-cnn
微信小程序 实现拨打电话
微信小程序 实现位置地图显示，引入map地图，不含导航
JS to realize the right-click menu bar function
卷积神经网络——从R-CNN，Fast R-CNN到Faster R-CNN，Mask R-CNN
Fast parsing combined with Huatu document encryption software
js实现右键菜单栏功能
随机推荐
Does PostgreSQL 14 support winserver2022?
施耐德电气、欧莱雅等企业巨头如何开放式创新？DEMO WORLD世界创新峰会揭秘
How difficult the interview is! I was forced to survive the six rounds of interview of ant financial! Almost out (interview resumption)
IDEA打包war包与war包位置
【cf】＃681 A. Kids Seating (Div. 2, based on VK Cup 2019-2020 - Final)
MySQL creates a student table and queries grades
Understand JVM language
Likeshop takeout ordering system "100% open source without encryption"
机器学习之评价指标(一)——回归评价指标
VSS tip: search all checked out files (search checked out files according to users)
Application of knowing things and learning | correlation graph analysis in anti cheating business
fragmentTransaction.replace第二个参数报错
类的六大关系——依赖和关联的区别
最新大厂高级面试题 必备
Kubernetes 1.24 high availability cluster binary deployment
WebDriverException( selenium.common.exceptions.WebDriverException: Message: ‘chromedriver‘ executabl
X-sheet development tutorial: initialization configuration custom layout
SQL Server连接到服务器无效的解决办法
hutool- 数字计算
卷积神经网络之卷积计算过程个人理解