countIterator = countList.iterator();
- while (countIterator.hasNext()) {
- Integer route = countIterator.next();
- if (route.equals(entry.getKey())) {
- countIterator.remove();
- break;
+ if (cacheMiss++ % DEBUG_CACHE_MISS_SIZE == 0) {
+ LOGGER.debug("Cache misses count:{}", cacheMiss);
}
}
}
- lastCheckIntervalUsedSet.clear();
+ return dataMap.get(key);
}
@Override
public void putFinished() {
- if (StringUtils.isEmpty(data.toString())) {
+ if (CollectionUtils.isEmpty(dataMap)) {
return;
}
- cache.put(index / BATCH_COUNT, data.toString());
+ fileCache.put(index / BATCH_COUNT, dataMap);
}
@Override
public void destroy() {
- cacheManager.close();
+ fileCacheManager.removeCache(cacheAlias);
+ activeCacheManager.removeCache(cacheAlias);
}
}
diff --git a/src/main/java/com/alibaba/excel/cache/selector/EternalReadCacheSelector.java b/src/main/java/com/alibaba/excel/cache/selector/EternalReadCacheSelector.java
new file mode 100644
index 00000000..9730dc08
--- /dev/null
+++ b/src/main/java/com/alibaba/excel/cache/selector/EternalReadCacheSelector.java
@@ -0,0 +1,23 @@
+package com.alibaba.excel.cache.selector;
+
+import org.apache.poi.openxml4j.opc.PackagePart;
+
+import com.alibaba.excel.cache.ReadCache;
+
+/**
+ * Choose a eternal cache
+ *
+ * @author Jiaju Zhuang
+ **/
+public class EternalReadCacheSelector implements ReadCacheSelector {
+ private ReadCache readCache;
+
+ public EternalReadCacheSelector(ReadCache readCache) {
+ this.readCache = readCache;
+ }
+
+ @Override
+ public ReadCache readCache(PackagePart sharedStringsTablePackagePart) {
+ return readCache;
+ }
+}
diff --git a/src/main/java/com/alibaba/excel/cache/selector/ReadCacheSelector.java b/src/main/java/com/alibaba/excel/cache/selector/ReadCacheSelector.java
new file mode 100644
index 00000000..3a2e5024
--- /dev/null
+++ b/src/main/java/com/alibaba/excel/cache/selector/ReadCacheSelector.java
@@ -0,0 +1,21 @@
+package com.alibaba.excel.cache.selector;
+
+import org.apache.poi.openxml4j.opc.PackagePart;
+
+import com.alibaba.excel.cache.ReadCache;
+
+/**
+ * Select the cache
+ *
+ * @author Jiaju Zhuang
+ **/
+public interface ReadCacheSelector {
+
+ /**
+ * Select a cache
+ *
+ * @param sharedStringsTablePackagePart
+ * @return
+ */
+ ReadCache readCache(PackagePart sharedStringsTablePackagePart);
+}
diff --git a/src/main/java/com/alibaba/excel/cache/selector/SimpleReadCacheSelector.java b/src/main/java/com/alibaba/excel/cache/selector/SimpleReadCacheSelector.java
new file mode 100644
index 00000000..d7cedd07
--- /dev/null
+++ b/src/main/java/com/alibaba/excel/cache/selector/SimpleReadCacheSelector.java
@@ -0,0 +1,82 @@
+package com.alibaba.excel.cache.selector;
+
+import java.io.IOException;
+
+import org.apache.poi.openxml4j.opc.PackagePart;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.excel.cache.Ehcache;
+import com.alibaba.excel.cache.MapCache;
+import com.alibaba.excel.cache.ReadCache;
+
+/**
+ * Simple cache selector
+ *
+ * @author Jiaju Zhuang
+ **/
+public class SimpleReadCacheSelector implements ReadCacheSelector {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SimpleReadCacheSelector.class);
+ /**
+ * Convert bytes to megabytes
+ */
+ private static final long B2M = 1000 * 1000L;
+ /**
+ * If it's less than 5M, use map cache, or use ehcache.unit MB.
+ */
+ private static final int DEFAULT_MAX_USE_MAP_CACHE_SIZE = 5;
+ /**
+ * Maximum size of cache activation.unit MB.
+ */
+ private static final int DEFAULT_MAX_EHCACHE_ACTIVATE_SIZE = 20;
+
+ /**
+ * Shared strings exceeding this value will use {@link Ehcache},or use {@link MapCache}.unit MB.
+ */
+ private long maxUseMapCacheSize;
+
+ /**
+ * Maximum size of cache activation.unit MB.
+ */
+ private int maxCacheActivateSize;
+
+ public SimpleReadCacheSelector() {
+ this(DEFAULT_MAX_USE_MAP_CACHE_SIZE, DEFAULT_MAX_EHCACHE_ACTIVATE_SIZE);
+ }
+
+ public SimpleReadCacheSelector(long maxUseMapCacheSize, int maxCacheActivateSize) {
+ if (maxUseMapCacheSize <= 0) {
+ this.maxUseMapCacheSize = DEFAULT_MAX_USE_MAP_CACHE_SIZE;
+ } else {
+ this.maxUseMapCacheSize = maxUseMapCacheSize;
+ }
+ if (maxCacheActivateSize <= 0) {
+ this.maxCacheActivateSize = DEFAULT_MAX_EHCACHE_ACTIVATE_SIZE;
+ } else {
+ this.maxCacheActivateSize = maxCacheActivateSize;
+ }
+ }
+
+ @Override
+ public ReadCache readCache(PackagePart sharedStringsTablePackagePart) {
+ long size = sharedStringsTablePackagePart.getSize();
+ if (size < 0) {
+ try {
+ size = sharedStringsTablePackagePart.getInputStream().available();
+ } catch (IOException e) {
+ LOGGER.warn("Unable to get file size, default used MapCache");
+ return new MapCache();
+ }
+ }
+ if (size < maxUseMapCacheSize * B2M) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Use map cache.size:{}", size);
+ }
+ return new MapCache();
+ }
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Use ehcache.size:{}", size);
+ }
+ return new Ehcache(maxCacheActivateSize);
+ }
+}
diff --git a/src/main/java/com/alibaba/excel/read/builder/ExcelReaderBuilder.java b/src/main/java/com/alibaba/excel/read/builder/ExcelReaderBuilder.java
index 15d4a02c..2b5bcd6e 100644
--- a/src/main/java/com/alibaba/excel/read/builder/ExcelReaderBuilder.java
+++ b/src/main/java/com/alibaba/excel/read/builder/ExcelReaderBuilder.java
@@ -7,6 +7,7 @@ import java.util.List;
import com.alibaba.excel.ExcelReader;
import com.alibaba.excel.cache.ReadCache;
+import com.alibaba.excel.cache.selector.ReadCacheSelector;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.converters.Converter;
import com.alibaba.excel.event.AnalysisEventListener;
@@ -84,6 +85,17 @@ public class ExcelReaderBuilder {
return this;
}
+ /**
+ * Ignore empty rows.Default is true.
+ *
+ * @param ignoreEmptyRow
+ * @return
+ */
+ public ExcelReaderBuilder ignoreEmptyRow(Boolean ignoreEmptyRow) {
+ readWorkbook.setIgnoreEmptyRow(ignoreEmptyRow);
+ return this;
+ }
+
/**
* This object can be read in the Listener {@link AnalysisEventListener#invoke(Object, AnalysisContext)}
* {@link AnalysisContext#getCustom()}
@@ -97,7 +109,7 @@ public class ExcelReaderBuilder {
}
/**
- * A cache that stores temp data to save memory.Default use {@link com.alibaba.excel.cache.Ehcache}
+ * A cache that stores temp data to save memory.
*
* @param readCache
* @return
@@ -107,6 +119,17 @@ public class ExcelReaderBuilder {
return this;
}
+ /**
+ * Select the cache.Default use {@link com.alibaba.excel.cache.selector.SimpleReadCacheSelector}
+ *
+ * @param readCacheSelector
+ * @return
+ */
+ public ExcelReaderBuilder readCacheSelector(ReadCacheSelector readCacheSelector) {
+ readWorkbook.setReadCacheSelector(readCacheSelector);
+ return this;
+ }
+
/**
* Count the number of added heads when read sheet.
*
diff --git a/src/main/java/com/alibaba/excel/read/metadata/ReadWorkbook.java b/src/main/java/com/alibaba/excel/read/metadata/ReadWorkbook.java
index 360f941f..b21cdb80 100644
--- a/src/main/java/com/alibaba/excel/read/metadata/ReadWorkbook.java
+++ b/src/main/java/com/alibaba/excel/read/metadata/ReadWorkbook.java
@@ -4,6 +4,7 @@ import java.io.File;
import java.io.InputStream;
import com.alibaba.excel.cache.ReadCache;
+import com.alibaba.excel.cache.selector.ReadCacheSelector;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.excel.support.ExcelTypeEnum;
@@ -47,9 +48,17 @@ public class ReadWorkbook extends ReadBasicParameter {
*/
private Object customObject;
/**
- * A cache that stores temp data to save memory.Default use {@link com.alibaba.excel.cache.Ehcache}
+ * A cache that stores temp data to save memory.
*/
private ReadCache readCache;
+ /**
+ * Ignore empty rows.Default is true.
+ */
+ private Boolean ignoreEmptyRow;
+ /**
+ * Select the cache.Default use {@link com.alibaba.excel.cache.selector.SimpleReadCacheSelector}
+ */
+ private ReadCacheSelector readCacheSelector;
/**
* The default is all excel objects.Default is true.
*
@@ -139,4 +148,20 @@ public class ReadWorkbook extends ReadBasicParameter {
public void setDefaultReturnMap(Boolean defaultReturnMap) {
this.defaultReturnMap = defaultReturnMap;
}
+
+ public Boolean getIgnoreEmptyRow() {
+ return ignoreEmptyRow;
+ }
+
+ public void setIgnoreEmptyRow(Boolean ignoreEmptyRow) {
+ this.ignoreEmptyRow = ignoreEmptyRow;
+ }
+
+ public ReadCacheSelector getReadCacheSelector() {
+ return readCacheSelector;
+ }
+
+ public void setReadCacheSelector(ReadCacheSelector readCacheSelector) {
+ this.readCacheSelector = readCacheSelector;
+ }
}
diff --git a/src/main/java/com/alibaba/excel/read/metadata/holder/AbstractReadHolder.java b/src/main/java/com/alibaba/excel/read/metadata/holder/AbstractReadHolder.java
index f9dd0d66..4ca8d63c 100644
--- a/src/main/java/com/alibaba/excel/read/metadata/holder/AbstractReadHolder.java
+++ b/src/main/java/com/alibaba/excel/read/metadata/holder/AbstractReadHolder.java
@@ -5,17 +5,18 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.converters.Converter;
import com.alibaba.excel.converters.ConverterKeyBuild;
import com.alibaba.excel.converters.DefaultConverterLoader;
-import com.alibaba.excel.enums.CellDataTypeEnum;
import com.alibaba.excel.enums.HeadKindEnum;
import com.alibaba.excel.enums.HolderEnum;
import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.excel.exception.ExcelAnalysisException;
import com.alibaba.excel.exception.ExcelAnalysisStopException;
-import com.alibaba.excel.exception.ExcelDataConvertException;
import com.alibaba.excel.metadata.AbstractHolder;
import com.alibaba.excel.metadata.CellData;
import com.alibaba.excel.metadata.Head;
@@ -26,6 +27,7 @@ import com.alibaba.excel.read.listener.ReadListenerRegistryCenter;
import com.alibaba.excel.read.listener.event.AnalysisFinishEvent;
import com.alibaba.excel.read.metadata.ReadBasicParameter;
import com.alibaba.excel.read.metadata.property.ExcelReadHeadProperty;
+import com.alibaba.excel.util.CollectionUtils;
import com.alibaba.excel.util.ConverterUtils;
import com.alibaba.excel.util.StringUtils;
@@ -35,6 +37,8 @@ import com.alibaba.excel.util.StringUtils;
* @author Jiaju Zhuang
*/
public abstract class AbstractReadHolder extends AbstractHolder implements ReadHolder, ReadListenerRegistryCenter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(AbstractReadHolder.class);
+
/**
* Count the number of added heads when read sheet.
*
@@ -117,6 +121,14 @@ public abstract class AbstractReadHolder extends AbstractHolder implements ReadH
@Override
public void notifyEndOneRow(AnalysisFinishEvent event, AnalysisContext analysisContext) {
Map cellDataMap = event.getAnalysisResult();
+ if (CollectionUtils.isEmpty(cellDataMap)) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.warn("Empty row!");
+ }
+ if (analysisContext.readWorkbookHolder().getIgnoreEmptyRow()) {
+ return;
+ }
+ }
ReadRowHolder readRowHolder = analysisContext.readRowHolder();
readRowHolder.setCurrentRowAnalysisResult(cellDataMap);
int rowIndex = readRowHolder.getRowIndex();
diff --git a/src/main/java/com/alibaba/excel/read/metadata/holder/ReadWorkbookHolder.java b/src/main/java/com/alibaba/excel/read/metadata/holder/ReadWorkbookHolder.java
index 815ef06e..3fd88395 100644
--- a/src/main/java/com/alibaba/excel/read/metadata/holder/ReadWorkbookHolder.java
+++ b/src/main/java/com/alibaba/excel/read/metadata/holder/ReadWorkbookHolder.java
@@ -12,6 +12,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.excel.cache.ReadCache;
+import com.alibaba.excel.cache.selector.EternalReadCacheSelector;
+import com.alibaba.excel.cache.selector.ReadCacheSelector;
+import com.alibaba.excel.cache.selector.SimpleReadCacheSelector;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.enums.HolderEnum;
import com.alibaba.excel.event.AnalysisEventListener;
@@ -64,10 +67,17 @@ public class ReadWorkbookHolder extends AbstractReadHolder {
*/
private Object customObject;
/**
- * A cache that stores temp data to save memory.Default use {@link com.alibaba.excel.cache.Ehcache}
+ * Ignore empty rows.Default is true.
+ */
+ private Boolean ignoreEmptyRow;
+ /**
+ * A cache that stores temp data to save memory.
*/
private ReadCache readCache;
-
+ /**
+ * Select the cache.Default use {@link com.alibaba.excel.cache.selector.SimpleReadCacheSelector}
+ */
+ private ReadCacheSelector readCacheSelector;
/**
* Temporary files when reading excel
*/
@@ -140,9 +150,22 @@ public class ReadWorkbookHolder extends AbstractReadHolder {
getGlobalConfiguration().setUse1904windowing(Boolean.FALSE);
}
this.customObject = readWorkbook.getCustomObject();
- this.readCache = readWorkbook.getReadCache();
- if (readCache != null && ExcelTypeEnum.XLS == excelType) {
- LOGGER.warn("Xls not support 'readCache'!");
+ if (readWorkbook.getIgnoreEmptyRow() == null) {
+ this.ignoreEmptyRow = Boolean.TRUE;
+ } else {
+ this.ignoreEmptyRow = readWorkbook.getIgnoreEmptyRow();
+ }
+ if (readWorkbook.getReadCache() != null) {
+ if (readWorkbook.getReadCacheSelector() != null) {
+ throw new ExcelAnalysisException("'readCache' and 'readCacheSelector' only one choice.");
+ }
+ this.readCacheSelector = new EternalReadCacheSelector(readWorkbook.getReadCache());
+ } else {
+ if (readWorkbook.getReadCacheSelector() == null) {
+ this.readCacheSelector = new SimpleReadCacheSelector();
+ } else {
+ this.readCacheSelector = readWorkbook.getReadCacheSelector();
+ }
}
if (readWorkbook.getDefaultReturnMap() == null) {
this.defaultReturnMap = Boolean.TRUE;
@@ -201,6 +224,14 @@ public class ReadWorkbookHolder extends AbstractReadHolder {
this.customObject = customObject;
}
+ public Boolean getIgnoreEmptyRow() {
+ return ignoreEmptyRow;
+ }
+
+ public void setIgnoreEmptyRow(Boolean ignoreEmptyRow) {
+ this.ignoreEmptyRow = ignoreEmptyRow;
+ }
+
public ReadCache getReadCache() {
return readCache;
}
@@ -209,6 +240,14 @@ public class ReadWorkbookHolder extends AbstractReadHolder {
this.readCache = readCache;
}
+ public ReadCacheSelector getReadCacheSelector() {
+ return readCacheSelector;
+ }
+
+ public void setReadCacheSelector(ReadCacheSelector readCacheSelector) {
+ this.readCacheSelector = readCacheSelector;
+ }
+
public Boolean getMandatoryUseInputStream() {
return mandatoryUseInputStream;
}
diff --git a/src/test/java/com/alibaba/easyexcel/test/temp/cache/CacheTest.java b/src/test/java/com/alibaba/easyexcel/test/temp/cache/CacheTest.java
new file mode 100644
index 00000000..4ba065a3
--- /dev/null
+++ b/src/test/java/com/alibaba/easyexcel/test/temp/cache/CacheTest.java
@@ -0,0 +1,62 @@
+package com.alibaba.easyexcel.test.temp.cache;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.poi.xssf.streaming.SXSSFRow;
+import org.apache.poi.xssf.streaming.SXSSFSheet;
+import org.apache.poi.xssf.streaming.SXSSFWorkbook;
+import org.apache.poi.xssf.usermodel.XSSFRow;
+import org.apache.poi.xssf.usermodel.XSSFSheet;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.ehcache.Cache;
+import org.ehcache.PersistentCacheManager;
+import org.ehcache.config.builders.CacheConfigurationBuilder;
+import org.ehcache.config.builders.CacheManagerBuilder;
+import org.ehcache.config.builders.ResourcePoolsBuilder;
+import org.ehcache.config.units.MemoryUnit;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.easyexcel.test.temp.poi.Poi2Test;
+import com.alibaba.excel.util.FileUtils;
+import com.alibaba.fastjson.JSON;
+
+/**
+ *
+ * @author Jiaju Zhuang
+ **/
+@Ignore
+public class CacheTest {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Poi2Test.class);
+
+ @Test
+ public void cache() throws Exception {
+
+ File readTempFile = FileUtils.createCacheTmpFile();
+
+ File cacheFile = new File(readTempFile.getPath(), UUID.randomUUID().toString());
+ PersistentCacheManager persistentCacheManager =
+ CacheManagerBuilder.newCacheManagerBuilder().with(CacheManagerBuilder.persistence(cacheFile))
+ .withCache("cache", CacheConfigurationBuilder.newCacheConfigurationBuilder(Integer.class, HashMap.class,
+ ResourcePoolsBuilder.newResourcePoolsBuilder().disk(10, MemoryUnit.GB)))
+ .build(true);
+ Cache cache = persistentCacheManager.getCache("cache", Integer.class, HashMap.class);
+
+ HashMap map = new HashMap();
+ map.put(1, "test");
+
+ cache.put(1, map);
+ LOGGER.info("dd1:{}", JSON.toJSONString(cache.get(1)));
+
+ cache.clear();
+
+ LOGGER.info("dd2:{}", JSON.toJSONString(cache.get(1)));
+ }
+
+}
diff --git a/src/test/java/com/alibaba/easyexcel/test/temp/large/LargeDataTest.java b/src/test/java/com/alibaba/easyexcel/test/temp/large/TempLargeDataTest.java
similarity index 80%
rename from src/test/java/com/alibaba/easyexcel/test/temp/large/LargeDataTest.java
rename to src/test/java/com/alibaba/easyexcel/test/temp/large/TempLargeDataTest.java
index bf65673a..14a31c4d 100644
--- a/src/test/java/com/alibaba/easyexcel/test/temp/large/LargeDataTest.java
+++ b/src/test/java/com/alibaba/easyexcel/test/temp/large/TempLargeDataTest.java
@@ -6,20 +6,20 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.alibaba.easyexcel.test.core.large.LargeDataTest;
import com.alibaba.excel.EasyExcel;
-import com.alibaba.excel.cache.MapCache;
/**
*
* @author Jiaju Zhuang
*/
-public class LargeDataTest {
+public class TempLargeDataTest {
private static final Logger LOGGER = LoggerFactory.getLogger(LargeDataTest.class);
@Test
public void read() throws Exception {
long start = System.currentTimeMillis();
- EasyExcel.read(new FileInputStream("D:\\test\\MRP生产视图(1).xlsx"), LargeData.class, new LargeDataListener()).readCache(new eh)
+ EasyExcel.read(new FileInputStream("D:\\test\\MRP生产视图(1).xlsx"), LargeData.class, new LargeDataListener())
.headRowNumber(2).sheet().doRead();
LOGGER.info("Large data total time spent:{}", System.currentTimeMillis() - start);
}
diff --git a/src/test/java/com/alibaba/easyexcel/test/temp/simple/HgTest.java b/src/test/java/com/alibaba/easyexcel/test/temp/simple/HgTest.java
index eed6f92e..9b0d5402 100644
--- a/src/test/java/com/alibaba/easyexcel/test/temp/simple/HgTest.java
+++ b/src/test/java/com/alibaba/easyexcel/test/temp/simple/HgTest.java
@@ -24,7 +24,7 @@ public class HgTest {
@Test
public void hh() throws IOException {
List