easyExcel+validation+正则实现excel导入校验
背景:导入excel的时候,除了数据库校验数据以外,还会碰到数据格式校验,如果直接报错,显得客户体验很不好,我想到的做法是,导入正确的数据,错误的数据重新导出,并在后面加上错误信息。
环境准备:
spring:springboot2.X
easyExcel:2.1.4
lombok(非必须)
easyExcel官网:https://alibaba-easyexcel.github.io/index.html
1、pom文件引入easyExcel
com.alibaba easyexcel 2.1.4
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>3.17</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>3.17</version>
</dependency>
<dependency>
<groupId>cglib</groupId>
<artifactId>cglib</artifactId>
<version>3.1</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>ooxml-schemas</artifactId>
<version>1.1</version>
</dependency>
2、通过easyExcel官方文档我们知道,easyExcel有一个监听器,用来读取excel数据,并且监听器不能被spring管理,要每次读取excel都要new,然后里面用到spring可以构造方法传进去
监听器实现如下:
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.excel.exception.ExcelAnalysisException;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import java.lang.reflect.Field;
import java.util.*;
/**
* @author zhy
* @title: EasyExcelListener
* @projectName cec-moutai-bd-display
* @description: easyExcel监听器
* @date 2019/12/2318:28
*/
@Data
public class EasyExcelListener <T> extends AnalysisEventListener<T> {
//成功结果集
private List<ExcelImportSucObjectDto> successList = new ArrayList<>();
//失败结果集
private List<ExcelImportErrObjectDto> errList = new ArrayList<>();
//处理逻辑service
private ExcelCheckManager excelCheckManager;
private List<T> list = new ArrayList<>();
//excel对象的反射类
private Class<T> clazz;
public EasyExcelListener(ExcelCheckManager excelCheckManager){
this.excelCheckManager = excelCheckManager;
}
public EasyExcelListener(ExcelCheckManager excelCheckManager,Class<T> clazz){
this.excelCheckManager = excelCheckManager;
this.clazz = clazz;
}
@Override
public void invoke(T t, AnalysisContext analysisContext) {
String errMsg;
try {
//根据excel数据实体中的javax.validation + 正则表达式来校验excel数据
errMsg = EasyExcelValiHelper.validateEntity(t);
} catch (NoSuchFieldException e) {
errMsg = "解析数据出错";
e.printStackTrace();
}
if (!StringUtils.isBlank(errMsg)){
ExcelImportErrObjectDto excelImportErrObjectDto = new ExcelImportErrObjectDto(t, errMsg);
errList.add(excelImportErrObjectDto);
}else{
list.add(t);
}
//每1000条处理一次
if (list.size() > 1000){
//校验
ExcelImportResult result = excelCheckManager.checkImportExcel(list);
successList.addAll(result.getSuccessDtos());
errList.addAll(result.getErrDtos());
list.clear();
}
// 达到BATCH_COUNT了,需要去存储一次数据库,防止数据几万条数据在内存,容易OOM
if (list.size() >= BATCH_COUNT) {
saveData();
// 存储完成清理 list
list.clear();
}
}
//所有数据解析完成了 都会来调用
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
ExcelImportResult result = excelCheckManager.checkImportExcel(list);
successList.addAll(result.getSuccessDtos());
errList.addAll(result.getErrDtos());
list.clear();
}
/**
* @description: 校验excel头部格式,必须完全匹配
* @param headMap 传入excel的头部(第一行数据)数据的index,name
* @param context
* @throws
* @return void
* @author zhy
* @date 2019/12/24 19:27
*/
@Override
public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
super.invokeHeadMap(headMap, context);
if (clazz != null){
try {
Map<Integer, String> indexNameMap = getIndexNameMap(clazz);
Set<Integer> keySet = indexNameMap.keySet();
for (Integer key : keySet) {
if (StringUtils.isBlank(headMap.get(key))){
throw new ExcelAnalysisException("解析excel出错,请传入正确格式的excel");
}
if (!headMap.get(key).equals(indexNameMap.get(key))){
throw new ExcelAnalysisException("解析excel出错,请传入正确格式的excel");
}
}
} catch (NoSuchFieldException e) {
e.printStackTrace();
}
}
}
/**
* @description: 获取注解里ExcelProperty的value,用作校验excel
* @param clazz
* @throws
* @return java.util.Map<java.lang.Integer,java.lang.String>
* @author zhy
* @date 2019/12/24 19:21
*/
public Map<Integer,String> getIndexNameMap(Class clazz) throws NoSuchFieldException {
Map<Integer,String> result = new HashMap<>();
Field field;
Field[] fields=clazz.getDeclaredFields();
for (int i = 0; i <fields.length ; i++) {
field=clazz.getDeclaredField(fields[i].getName());
field.setAccessible(true);
ExcelProperty excelProperty=field.getAnnotation(ExcelProperty.class);
if(excelProperty!=null){
int index = excelProperty.index();
String[] values = excelProperty.value();
StringBuilder value = new StringBuilder();
for (String v : values) {
value.append(v);
}
result.put(index,value.toString());
}
}
return result;
}
}
3、成功结果集对象:ExcelImportSucObjectDto
import lombok.Data;
/**
* @author zhy
* @title: ExcelImportErrObjectDto
* @projectName cec-moutai-bd-display
* @description: excel单条数据导入结果
* @date 2019/12/2318:23
*/
@Data
public class ExcelImportSucObjectDto {
private Object object;
}
4、失败结果集对象:ExcelImportErrObjectDto
import lombok.Data;
/**
* @author zhy
* @title: ExcelImportErrObjectDto
* @projectName cec-moutai-bd-display
* @description: excel单条数据导入结果
* @date 2019/12/2318:23
*/
@Data
public class ExcelImportErrObjectDto {
private Object object;
private String errMsg;
public ExcelImportErrObjectDto(){}
public ExcelImportErrObjectDto(Object object,String errMsg){
this.object = object;
this.errMsg = errMsg;
}
}
5、业务校验接口:ExcelCheckManager。需要校验excel业务的service接口可以继承这个接口,并在实现类中实现自己的方法,返回的是成功失败的结果集 ExcelImportResult
import java.util.List;
/**
* @author zhy
* @title: ExcelCheckManager
* @projectName cec-moutai-bd-display
* @description: excel校验接口
* @date 2019/12/2317:44
*/
public interface ExcelCheckManager<T> {
/**
* @description: 校验方法
* @param objects
* @throws
* @return com.cec.moutai.common.easyexcel.ExcelImportResult
* @author zhy
* @date 2019/12/24 14:57
*/
<T> ExcelImportResult checkImportExcel(List<T> objects);
}
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
/**
* @author zhy
* @title: ExcelImportErrObjectDto
* @projectName cec-moutai-bd-display
* @description: excel数据导入结果
* @date 2019/12/2318:23
*/
@Data
public class ExcelImportResult {
private List<ExcelImportSucObjectDto> successDtos;
private List<ExcelImportErrObjectDto> errDtos;
public ExcelImportResult(List<ExcelImportSucObjectDto> successDtos,List<ExcelImportErrObjectDto> errDtos){
this.successDtos =successDtos;
this.errDtos = errDtos;
}
public ExcelImportResult(List<ExcelImportErrObjectDto> errDtos){
this.successDtos =new ArrayList<>();
this.errDtos = errDtos;
}
}
6、数据校验逻辑实现:通过正则校验excel数据的功能,后续也可以在此扩展加入其他校验接口。
@ExcelProperty是easyExcel自带的注解
@ColumnWidth也是easyExcel的注解代表单元格宽度
@Length代表的是字符串长度,max代表的是最长允许多长
@Pattern就是正则表达式注解了,regexp代表的是正则表达式,message代表是,没有匹配成功返回的错误信息
关于validation的注解可以参考此篇大佬的博客:https://blog.csdn.net/weixin_42546729/article/details/89364431
可以发现,我这里的接收对象都是String类型的,也正因为是这样,才能通过正则表达式去校验各种格式。
所以在excel类和真正的实体类之间的转换,我是用fastjson的JSON.parseObject来进行转换的。所以需要保证excel类和数据库实体的字段名要保持一致,数据格式,尤其是日期格式,要保持一致。
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.excel.annotation.write.style.ColumnWidth;
import com.cec.moutai.common.easyexcel.ExcelPatternMsg;
import lombok.Data;
import org.hibernate.validator.constraints.Length;
import javax.validation.constraints.DecimalMin;
import javax.validation.constraints.Pattern;
/**
* @author zhy
* @title: CustomerExcelDto
* @projectName cec-moutai-bd-display
* @description: 客户excel对象
* @date 2019/12/2410:35
*/
@Data
public class CustomerExcelDto {
@ExcelProperty(index = 0,value = "客户全称")
@ColumnWidth(30)
private String name;
//客户简称
@ExcelProperty(index = 1,value = "客户简称")
@ColumnWidth(30)
@Length(max = 100)
private String shortName;
// 客户编码
@ExcelProperty(index = 2,value = "客户编码(导入必填)")
@ColumnWidth(30)
private String code;
@ExcelProperty(index = 3,value = "客户分类")
private String custclassName;
// 法人
@ExcelProperty(index = 6,value = "法定代表人")
@Length(max = 100)
private String legalbody;
// 营业期限
@ExcelProperty(index = 24,value = "营业期限(导入格式:yyyy-MM-dd)")
@Pattern(regexp = ExcelPatternMsg.DATE2,message = ExcelPatternMsg.DATE2_MSG)
private String businessEndDate;
}
7、正则表达式实现类:ExcelPatternMsg
import java.util.regex.Pattern;
/**
* @author zhy
* @title: ExcelPatternMsg
* @projectName cec-moutai-bd-display
* @description: excel正则表达式,以及错误信息
* @date 2019/12/2614:22
*/
public class ExcelPatternMsg {
//只能输入整数或者小数
public static final String DECIMAL = "^[0-9]+\\.{0,1}[0-9]{0,2}$";
public static final String DECIMAL_MSG = "只能输入整数或者小数";
//日期格式 yyyy/MM/dd
public static final String DATE1 = "(([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]{1}|[0-9]{1}[1-9][0-9]{2}|[1-9][0-9]{3})\\/(((0[13578]|1[02])\\/(0[1-9]|[12][0-9]|3[01]))|"+
"((0[469]|11)\\/(0[1-9]|[12][0-9]|30))|(02\\/(0[1-9]|[1][0-9]|2[0-8]))))|((([0-9]{2})(0[48]|[2468][048]|[13579][26])|"+
"((0[48]|[2468][048]|[3579][26])00))\\/02\\/29)$";
public static final String DATE1_MSG = "输入正确的日期格式:yyyy/MM/dd";
//日期格式 yyyy-MM-dd
public static final String DATE2 = "(([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]{1}|[0-9]{1}[1-9][0-9]{2}|[1-9][0-9]{3})-(((0[13578]|1[02])-(0[1-9]|[12][0-9]|3[01]))|"+
"((0[469]|11)-(0[1-9]|[12][0-9]|30))|(02-(0[1-9]|[1][0-9]|2[0-8]))))|((([0-9]{2})(0[48]|[2468][048]|[13579][26])|"+
"((0[48]|[2468][048]|[3579][26])00))-02-29)$";
public static final String DATE2_MSG = "输入正确的日期格式:yyyy-MM-dd";
//日期格式 yyyyMMdd
public static final String DATE3 = "(([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]{1}|[0-9]{1}[1-9][0-9]{2}|[1-9][0-9]{3})(((0[13578]|1[02])(0[1-9]|[12][0-9]|3[01]))|"+
"((0[469]|11)(0[1-9]|[12][0-9]|30))|(02(0[1-9]|[1][0-9]|2[0-8]))))|((([0-9]{2})(0[48]|[2468][048]|[13579][26])|"+
"((0[48]|[2468][048]|[3579][26])00))0229)$";
public static final String DATE3_MSG = "输入正确的日期格式:yyyyMMdd";
//日期格式 yyyy-MM-dd HH:mm:ss
public static final String DATE_TIME1 = "^((([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]{1}|[0-9]{1}[1-9][0-9]{2}|[1-9][0-9]{3})-(((0[13578]|1[02])-(0[1-9]|[12][0-9]|3[01]))|" +
"((0[469]|11)-(0[1-9]|[12][0-9]|30))|(02-(0[1-9]|[1][0-9]|2[0-8]))))|((([0-9]{2})(0[48]|[2468][048]|[13579][26])|((0[48]|[2468][048]|[3579][26])00))-02-29))\\s+([0-1]?[0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])$";
public static final String DATE_TIME1_MSG = "输入正确的日期格式:yyyy-MM-dd HH:mm:ss";
//日期格式 yyyy/MM/dd HH:mm:ss
public static final String DATE_TIME2 = "((([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]{1}|[0-9]{1}[1-9][0-9]{2}|[1-9][0-9]{3})\\/(((0[13578]|1[02])\\/(0[1-9]|[12][0-9]|3[01]))|"+
"((0[469]|11)\\/(0[1-9]|[12][0-9]|30))|(02\\/(0[1-9]|[1][0-9]|2[0-8]))))|((([0-9]{2})(0[48]|[2468][048]|[13579][26])|"+
"((0[48]|[2468][048]|[3579][26])00))\\/02\\/29))\\s([0-1][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])$";
public static final String DATE_TIME2_MSG = "输入正确的日期格式:yyyy/MM/dd HH:mm:ss";
//日期格式 yyyyMMddHHmmss
public static final String DATE_TIME3 = "((([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]{1}|[0-9]{1}[1-9][0-9]{2}|[1-9][0-9]{3})(((0[13578]|1[02])(0[1-9]|[12][0-9]|3[01]))|"+
"((0[469]|11)(0[1-9]|[12][0-9]|30))|(02(0[1-9]|[1][0-9]|2[0-8]))))|((([0-9]{2})(0[48]|[2468][048]|[13579][26])|((0[48]|[2468][048]|[3579][26])00))0229))"+
"([0-1][0-9]|2[0-3])([0-5][0-9])([0-5][0-9])$";
public static final String DATE_TIME3_MSG = "输入正确的日期格式:yyyyMMddHHmmss";
//日期格式 yyyyMMddHHmmssSSS
public static final String DATE_TIME4 = "((([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]{1}|[0-9]{1}[1-9][0-9]{2}|[1-9][0-9]{3})(((0[13578]|1[02])(0[1-9]|[12][0-9]|3[01]))|"+
"((0[469]|11)(0[1-9]|[12][0-9]|30))|(02(0[1-9]|[1][0-9]|2[0-8]))))|((([0-9]{2})(0[48]|[2468][048]|[13579][26])|"+
"((0[48]|[2468][048]|[3579][26])00))0229))([0-1][0-9]|2[0-3])([0-5][0-9])([0-5][0-9])([0-9]{3})$";
public static final String DATE_TIME4_MSG = "输入正确的日期格式:yyyyMMddHHmmssSSS";
//日期格式 yyyyMMdd HH:mm:ss
public static final String DATE_TIME5 = "((([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]{1}|[0-9]{1}[1-9][0-9]{2}|[1-9][0-9]{3})(((0[13578]|1[02])(0[1-9]|[12][0-9]|3[01]))|"+
"((0[469]|11)(0[1-9]|[12][0-9]|30))|(02(0[1-9]|[1][0-9]|2[0-8]))))|((([0-9]{2})(0[48]|[2468][048]|[13579][26])|"+
"((0[48]|[2468][048]|[3579][26])00))0229))\\s([0-1][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])$";
public static final String DATE_TIME5_MSG = "输入正确的日期格式:yyyyMMdd HH:mm:ss";
//数字和字母
public static final String NUMBER_LETTER = "^[a-z0-9A-Z]+$";
public static final String NUMBER_LETTER_MSG = "只能输入数字和字母";
//数字
public static final String NUMBER = "^[0-9]*$";
public static final String NUMBER_MSG = "只能输入数字";
public static void main(String[] args) {
System.out.println(Pattern.matches(DATE1,"2020/02/30"));
}
}
EasyExcelValiHelper.validateEntity(t)的写法
import com.alibaba.excel.annotation.ExcelProperty;
import javax.validation.ConstraintViolation;
import javax.validation.Validation;
import javax.validation.Validator;
import javax.validation.groups.Default;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
public class EasyExcelValiHelper {
private static Validator validator = Validation.buildDefaultValidatorFactory().getValidator();
public static <T> String validateEntity(T obj) throws NoSuchFieldException, SecurityException {
StringBuilder result = new StringBuilder();
Set<ConstraintViolation<T>> set = validator.validate(obj, Default.class);
if (set != null && set.size() != 0) {
for (ConstraintViolation<T> cv : set) {
Field declaredField = obj.getClass().getDeclaredField(cv.getPropertyPath().toString());
ExcelProperty annotation = declaredField.getAnnotation(ExcelProperty.class);
result.append(annotation.value()[0]+cv.getMessage()).append(";");
}
}
return result.toString();
}
}
8、控制层入口写法:
@PostMapping(value = "importExcel")
@ApiOperation(value = "客户信息导入")
@SysServiceLog(operateContent = "客户信息导入", operateType = Globals.Log_Type_UPLOAD)
public Result importExcel(HttpServletResponse response,@RequestParam MultipartFile file) throws IOException {
EasyExcelListener easyExcelListener = new EasyExcelListener(customerManager,CustomerExcelDto.class);
EasyExcel.read(file.getInputStream(),CustomerExcelDto.class,easyExcelListener).sheet().doRead();
//错误结果集
List<ExcelImportErrObjectDto> errList = easyExcelListener.getErrList();
if (errList.size() > 0){
List<CustomerCompleteDtoImprotResult> completeDtoImprotResults = errList.stream().map(excelImportErrObjectDto -> {
CustomerCompleteDtoImprotResult customerCompleteDtoImprotResult = BeanUtils.convert(excelImportErrObjectDto.getObject(), CustomerCompleteDtoImprotResult.class);
customerCompleteDtoImprotResult.setErrMsg(excelImportErrObjectDto.getErrMsg());
return customerCompleteDtoImprotResult;
}).collect(Collectors.toList());
//导出excel
EasyExcelUtils.webWriteExcel(response,completeDtoImprotResults,CustomerCompleteDtoImprotResult.class,"客户信息");
}
return addSucResult();
}
9、导出工具封装:EasyExcelUtils
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.write.metadata.style.WriteCellStyle;
import com.alibaba.excel.write.style.HorizontalCellStyleStrategy;
import org.apache.poi.ss.formula.functions.T;
import org.apache.poi.ss.usermodel.IndexedColors;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.List;
/**
* @author zhy
* @title: EasyExcelUtils
* @projectName cec-moutai-bd-display
* @description: easyExcel工具类
* @date 2019/12/2411:35
*/
public class EasyExcelUtils {
public static void webWriteExcel(HttpServletResponse response, List objects, Class clazz, String fileName) throws IOException {
String sheetName = fileName;
webWriteExcel(response,objects,clazz,fileName,sheetName);
}
public static void webWriteExcel(HttpServletResponse response, List objects, Class clazz, String fileName, String sheetName) throws IOException {
response.setContentType("application/vnd.ms-excel");
response.setCharacterEncoding("utf-8");
response.setHeader("Content-disposition", "attachment;filename=" + fileName + ".xlsx");
// 头的策略
WriteCellStyle headWriteCellStyle = new WriteCellStyle();
// 背景设置为白
headWriteCellStyle.setFillForegroundColor(IndexedColors.WHITE.getIndex());
// 内容的策略
WriteCellStyle contentWriteCellStyle = new WriteCellStyle();
HorizontalCellStyleStrategy horizontalCellStyleStrategy =
new HorizontalCellStyleStrategy(headWriteCellStyle, contentWriteCellStyle);
ServletOutputStream outputStream = response.getOutputStream();
try {
EasyExcel.write(outputStream, clazz).registerWriteHandler(horizontalCellStyleStrategy).sheet(sheetName).doWrite(objects);
}catch (Exception e){
e.printStackTrace();
}finally {
outputStream.close();
}
}
}