diff --git a/src/main/java/com/alibaba/excel/analysis/v07/XlsxSaxAnalyser.java b/src/main/java/com/alibaba/excel/analysis/v07/XlsxSaxAnalyser.java index 120486e..b7e58d6 100644 --- a/src/main/java/com/alibaba/excel/analysis/v07/XlsxSaxAnalyser.java +++ b/src/main/java/com/alibaba/excel/analysis/v07/XlsxSaxAnalyser.java @@ -103,7 +103,7 @@ public class XlsxSaxAnalyser implements ExcelExecutor { if (size < 0) { size = sharedStringsTablePackagePart.getInputStream().available(); } - if (size < USE_MAP_CACHE_SIZE) { + if (size < readWorkbookHolder.getSharedStringsUseEhcacheSize()) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Use map cache.size:{}", size); } diff --git a/src/main/java/com/alibaba/excel/cache/Ehcache.java b/src/main/java/com/alibaba/excel/cache/Ehcache.java index abf0169..3ec7927 100644 --- a/src/main/java/com/alibaba/excel/cache/Ehcache.java +++ b/src/main/java/com/alibaba/excel/cache/Ehcache.java @@ -32,9 +32,7 @@ public class Ehcache implements ReadCache { private static final Logger LOGGER = LoggerFactory.getLogger(Ehcache.class); private static final int BATCH_COUNT = 1000; - private static final int CHECK_INTERVAL = 500; - private static final int MAX_CACHE_ACTIVATE = 10; - + private static final int CHECK_INTERVAL = 1000; private static final String CACHE = "cache"; private static final String DATA_SEPARATOR = "@"; private static final String KEY_VALUE_SEPARATOR = "!"; @@ -50,6 +48,10 @@ public class Ehcache implements ReadCache { * Key index */ private int index = 0; + /** + * Maximum number of cache activation + */ + private int maxCacheActivate = 100; private StringBuilder data = new StringBuilder(); private CacheManager cacheManager; /** @@ -80,6 +82,12 @@ public class Ehcache implements ReadCache { */ private int cacheMiss = 0; + public Ehcache() {} + + public Ehcache(int maxCacheActivate) { + this.maxCacheActivate = maxCacheActivate; + } + @Override public void init(AnalysisContext analysisContext) { File readTempFile = analysisContext.readWorkbookHolder().getTempFile(); @@ -166,7 +174,7 @@ public class Ehcache implements ReadCache { } private void checkClear() { - if (countList.size() > MAX_CACHE_ACTIVATE) { + if (countList.size() > maxCacheActivate) { Integer route = countList.getFirst(); countList.removeFirst(); cacheMap.remove(route); diff --git a/src/main/java/com/alibaba/excel/write/metadata/holder/WriteWorkbookHolder.java b/src/main/java/com/alibaba/excel/write/metadata/holder/WriteWorkbookHolder.java index fe23b2a..fe5c395 100644 --- a/src/main/java/com/alibaba/excel/write/metadata/holder/WriteWorkbookHolder.java +++ b/src/main/java/com/alibaba/excel/write/metadata/holder/WriteWorkbookHolder.java @@ -10,8 +10,6 @@ import java.util.HashMap; import java.util.Map; import org.apache.poi.ss.usermodel.Workbook; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.alibaba.excel.enums.HolderEnum; import com.alibaba.excel.exception.ExcelGenerateException; diff --git a/src/test/java/com/alibaba/easyexcel/test/temp/large/LargeData.java b/src/test/java/com/alibaba/easyexcel/test/temp/large/LargeData.java new file mode 100644 index 0000000..1424c42 --- /dev/null +++ b/src/test/java/com/alibaba/easyexcel/test/temp/large/LargeData.java @@ -0,0 +1,60 @@ +package com.alibaba.easyexcel.test.temp.large; + +import lombok.Data; + +/** + * @author Jiaju Zhuang + */ +@Data +public class LargeData { + + private String str1; + + private String str2; + + private String str3; + + private String str4; + + private String str5; + + private String str6; + + private String str7; + + private String str8; + + private String str9; + + private String str10; + + private String str11; + + private String str12; + + private String str13; + + private String str14; + + private String str15; + + private String str16; + + private String str17; + + private String str18; + + private String str19; + + private String str20; + + private String str21; + + private String str22; + + private String str23; + + private String str24; + + private String str25; +} diff --git a/src/test/java/com/alibaba/easyexcel/test/temp/large/LargeDataListener.java b/src/test/java/com/alibaba/easyexcel/test/temp/large/LargeDataListener.java new file mode 100644 index 0000000..a5b6556 --- /dev/null +++ b/src/test/java/com/alibaba/easyexcel/test/temp/large/LargeDataListener.java @@ -0,0 +1,33 @@ +package com.alibaba.easyexcel.test.temp.large; + +import org.junit.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; +import com.alibaba.fastjson.JSON; + +/** + * @author Jiaju Zhuang + */ +public class LargeDataListener extends AnalysisEventListener { + private static final Logger LOGGER = LoggerFactory.getLogger(LargeDataListener.class); + private int count = 0; + + @Override + public void invoke(LargeData data, AnalysisContext context) { + if (count == 0) { + LOGGER.info("First row:{}", JSON.toJSONString(data)); + } + count++; + if (count % 100000 == 0) { + LOGGER.info("Already read:{}", count); + } + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + LOGGER.info("Large row count:{}", count); + } +} diff --git a/src/test/java/com/alibaba/easyexcel/test/temp/large/LargeDataTest.java b/src/test/java/com/alibaba/easyexcel/test/temp/large/LargeDataTest.java new file mode 100644 index 0000000..bf65673 --- /dev/null +++ b/src/test/java/com/alibaba/easyexcel/test/temp/large/LargeDataTest.java @@ -0,0 +1,26 @@ +package com.alibaba.easyexcel.test.temp.large; + +import java.io.FileInputStream; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.cache.MapCache; + +/** + * + * @author Jiaju Zhuang + */ +public class LargeDataTest { + private static final Logger LOGGER = LoggerFactory.getLogger(LargeDataTest.class); + + @Test + public void read() throws Exception { + long start = System.currentTimeMillis(); + EasyExcel.read(new FileInputStream("D:\\test\\MRP生产视图(1).xlsx"), LargeData.class, new LargeDataListener()).readCache(new eh) + .headRowNumber(2).sheet().doRead(); + LOGGER.info("Large data total time spent:{}", System.currentTimeMillis() - start); + } +} diff --git a/update.md b/update.md index 9d91b9f..19f0b31 100644 --- a/update.md +++ b/update.md @@ -1,3 +1,7 @@ +# 2.0.5 +* 优化07版超大文件读取参数配置 +* 支持自己设置超大文件读取参数 + # 2.0.4 * 修复07版整个excel仅存在数字时会出现的NPE * 修复03版 用String接收电话会出现科学计数法的问题