Browse Source

* 优化大文件内存存储,减少内存占用 [Issue #2657](https://github.com/alibaba/easyexcel/issues/2657)

pull/2959/head
Jiaju Zhuang 2 years ago
parent
commit
58e5ca1739
  1. 29
      easyexcel-core/src/main/java/com/alibaba/excel/cache/Ehcache.java
  2. 62
      easyexcel-core/src/main/java/com/alibaba/excel/cache/selector/SimpleReadCacheSelector.java
  3. 4
      update.md

29
easyexcel-core/src/main/java/com/alibaba/excel/cache/Ehcache.java vendored

@ -15,6 +15,7 @@ import org.ehcache.config.CacheConfiguration;
import org.ehcache.config.builders.CacheConfigurationBuilder; import org.ehcache.config.builders.CacheConfigurationBuilder;
import org.ehcache.config.builders.CacheManagerBuilder; import org.ehcache.config.builders.CacheManagerBuilder;
import org.ehcache.config.builders.ResourcePoolsBuilder; import org.ehcache.config.builders.ResourcePoolsBuilder;
import org.ehcache.config.units.EntryUnit;
import org.ehcache.config.units.MemoryUnit; import org.ehcache.config.units.MemoryUnit;
/** /**
@ -50,13 +51,28 @@ public class Ehcache implements ReadCache {
*/ */
private int cacheMiss = 0; private int cacheMiss = 0;
public Ehcache(int maxCacheActivateSize) { @Deprecated
activeCacheConfiguration = CacheConfigurationBuilder public Ehcache(Integer maxCacheActivateSize) {
this(maxCacheActivateSize, null);
}
public Ehcache(Integer maxCacheActivateSize, Integer maxCacheActivateBatchCount) {
// In order to be compatible with the code
// If the user set up `maxCacheActivateSize`, then continue using it
if (maxCacheActivateSize != null) {
this.activeCacheConfiguration = CacheConfigurationBuilder
.newCacheConfigurationBuilder(Integer.class, ArrayList.class,
ResourcePoolsBuilder.newResourcePoolsBuilder()
.heap(maxCacheActivateSize, MemoryUnit.MB))
.build();
} else {
this.activeCacheConfiguration = CacheConfigurationBuilder
.newCacheConfigurationBuilder(Integer.class, ArrayList.class, .newCacheConfigurationBuilder(Integer.class, ArrayList.class,
ResourcePoolsBuilder.newResourcePoolsBuilder().heap(maxCacheActivateSize, MemoryUnit.MB)) ResourcePoolsBuilder.newResourcePoolsBuilder()
.withSizeOfMaxObjectGraph(1000 * 1000L).withSizeOfMaxObjectSize(maxCacheActivateSize, MemoryUnit.MB) .heap(maxCacheActivateBatchCount, EntryUnit.ENTRIES))
.build(); .build();
} }
}
static { static {
File cacheFile = FileUtils.createCacheTmpFile(); File cacheFile = FileUtils.createCacheTmpFile();
@ -64,9 +80,8 @@ public class Ehcache implements ReadCache {
CacheManagerBuilder.newCacheManagerBuilder().with(CacheManagerBuilder.persistence(cacheFile)).build(true); CacheManagerBuilder.newCacheManagerBuilder().with(CacheManagerBuilder.persistence(cacheFile)).build(true);
ACTIVE_CACHE_MANAGER = CacheManagerBuilder.newCacheManagerBuilder().build(true); ACTIVE_CACHE_MANAGER = CacheManagerBuilder.newCacheManagerBuilder().build(true);
FILE_CACHE_CONFIGURATION = CacheConfigurationBuilder FILE_CACHE_CONFIGURATION = CacheConfigurationBuilder
.newCacheConfigurationBuilder(Integer.class, ArrayList.class, .newCacheConfigurationBuilder(Integer.class, ArrayList.class, ResourcePoolsBuilder.newResourcePoolsBuilder()
ResourcePoolsBuilder.newResourcePoolsBuilder().disk(10, MemoryUnit.GB)) .disk(20, MemoryUnit.GB)).build();
.withSizeOfMaxObjectGraph(1000 * 1000L).withSizeOfMaxObjectSize(10, MemoryUnit.GB).build();
} }
@Override @Override

62
easyexcel-core/src/main/java/com/alibaba/excel/cache/selector/SimpleReadCacheSelector.java vendored

@ -2,6 +2,10 @@ package com.alibaba.excel.cache.selector;
import java.io.IOException; import java.io.IOException;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import org.apache.poi.openxml4j.opc.PackagePart; import org.apache.poi.openxml4j.opc.PackagePart;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -15,6 +19,9 @@ import com.alibaba.excel.cache.ReadCache;
* *
* @author Jiaju Zhuang * @author Jiaju Zhuang
**/ **/
@Getter
@Setter
@EqualsAndHashCode
public class SimpleReadCacheSelector implements ReadCacheSelector { public class SimpleReadCacheSelector implements ReadCacheSelector {
private static final Logger LOGGER = LoggerFactory.getLogger(SimpleReadCacheSelector.class); private static final Logger LOGGER = LoggerFactory.getLogger(SimpleReadCacheSelector.class);
/** /**
@ -24,38 +31,47 @@ public class SimpleReadCacheSelector implements ReadCacheSelector {
/** /**
* If it's less than 5M, use map cache, or use ehcache.unit MB. * If it's less than 5M, use map cache, or use ehcache.unit MB.
*/ */
private static final int DEFAULT_MAX_USE_MAP_CACHE_SIZE = 5; private static final long DEFAULT_MAX_USE_MAP_CACHE_SIZE = 5;
/** /**
* Maximum size of cache activation.unit MB. * Maximum batch of `SharedStrings` stored in memory.
* The batch size is 100.{@link Ehcache#BATCH_COUNT}
*/ */
private static final int DEFAULT_MAX_EHCACHE_ACTIVATE_SIZE = 20; private static final int DEFAULT_MAX_EHCACHE_ACTIVATE_BATCH_COUNT = 20;
/** /**
* Shared strings exceeding this value will use {@link Ehcache},or use {@link MapCache}.unit MB. * Shared strings exceeding this value will use {@link Ehcache},or use {@link MapCache}.unit MB.
*/ */
private final long maxUseMapCacheSize; private Long maxUseMapCacheSize;
/** /**
* Maximum size of cache activation.unit MB. * Maximum size of cache activation.unit MB.
*
* @deprecated Please use maxCacheActivateBatchCount to control the size of the occupied memory
*/
@Deprecated
private Integer maxCacheActivateSize;
/**
* Maximum batch of `SharedStrings` stored in memory.
* The batch size is 100.{@link Ehcache#BATCH_COUNT}
*/ */
private final int maxCacheActivateSize; private Integer maxCacheActivateBatchCount;
public SimpleReadCacheSelector() { public SimpleReadCacheSelector() {
this(DEFAULT_MAX_USE_MAP_CACHE_SIZE, DEFAULT_MAX_EHCACHE_ACTIVATE_SIZE);
} }
public SimpleReadCacheSelector(long maxUseMapCacheSize, int maxCacheActivateSize) { /**
if (maxUseMapCacheSize <= 0) { * Parameter maxCacheActivateSize has already been abandoned
this.maxUseMapCacheSize = DEFAULT_MAX_USE_MAP_CACHE_SIZE; *
} else { * @param maxUseMapCacheSize
* @param maxCacheActivateSize
*/
@Deprecated
public SimpleReadCacheSelector(Long maxUseMapCacheSize, Integer maxCacheActivateSize) {
this.maxUseMapCacheSize = maxUseMapCacheSize; this.maxUseMapCacheSize = maxUseMapCacheSize;
}
if (maxCacheActivateSize <= 0) {
this.maxCacheActivateSize = DEFAULT_MAX_EHCACHE_ACTIVATE_SIZE;
} else {
this.maxCacheActivateSize = maxCacheActivateSize; this.maxCacheActivateSize = maxCacheActivateSize;
} }
}
@Override @Override
public ReadCache readCache(PackagePart sharedStringsTablePackagePart) { public ReadCache readCache(PackagePart sharedStringsTablePackagePart) {
@ -68,6 +84,9 @@ public class SimpleReadCacheSelector implements ReadCacheSelector {
return new MapCache(); return new MapCache();
} }
} }
if (maxUseMapCacheSize == null) {
maxUseMapCacheSize = DEFAULT_MAX_USE_MAP_CACHE_SIZE;
}
if (size < maxUseMapCacheSize * B2M) { if (size < maxUseMapCacheSize * B2M) {
if (LOGGER.isDebugEnabled()) { if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Use map cache.size:{}", size); LOGGER.debug("Use map cache.size:{}", size);
@ -77,6 +96,17 @@ public class SimpleReadCacheSelector implements ReadCacheSelector {
if (LOGGER.isDebugEnabled()) { if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Use ehcache.size:{}", size); LOGGER.debug("Use ehcache.size:{}", size);
} }
return new Ehcache(maxCacheActivateSize);
// In order to be compatible with the code
// If the user set up `maxCacheActivateSize`, then continue using it
if (maxCacheActivateSize != null) {
return new Ehcache(maxCacheActivateSize, maxCacheActivateBatchCount);
} else {
if (maxCacheActivateBatchCount == null) {
maxCacheActivateBatchCount = DEFAULT_MAX_EHCACHE_ACTIVATE_BATCH_COUNT;
}
return new Ehcache(maxCacheActivateSize, maxCacheActivateBatchCount);
}
} }
} }

4
update.md

@ -1,6 +1,7 @@
# 3.2.1 # 3.2.1
* 兼容`LocalDate` [Issue #2908](https://github.com/alibaba/easyexcel/issues/2908) * 兼容`LocalDate` [Issue #2908](https://github.com/alibaba/easyexcel/issues/2908)
* 优化大文件内存存储,减少内存占用 [Issue #2657](https://github.com/alibaba/easyexcel/issues/2657)
# 3.2.0 # 3.2.0
@ -14,7 +15,8 @@
# 3.1.4 # 3.1.4
* 提高xlsx读取兼容性:在存在第一行很多空列的情况下,忽略空列 * 提高xlsx读取兼容性:在存在第一行很多空
* 列的情况下,忽略空列
# 3.1.3 # 3.1.3

Loading…
Cancel
Save