From 58e5ca173936fdf41069891b26604f94c62a6a16 Mon Sep 17 00:00:00 2001 From: Jiaju Zhuang Date: Fri, 10 Feb 2023 15:02:57 +0800 Subject: [PATCH] =?UTF-8?q?*=20=E4=BC=98=E5=8C=96=E5=A4=A7=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=86=85=E5=AD=98=E5=AD=98=E5=82=A8=EF=BC=8C=E5=87=8F?= =?UTF-8?q?=E5=B0=91=E5=86=85=E5=AD=98=E5=8D=A0=E7=94=A8=20[Issue=20#2657]?= =?UTF-8?q?(https://github.com/alibaba/easyexcel/issues/2657)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/alibaba/excel/cache/Ehcache.java | 33 +++++++--- .../selector/SimpleReadCacheSelector.java | 66 ++++++++++++++----- update.md | 4 +- 3 files changed, 75 insertions(+), 28 deletions(-) diff --git a/easyexcel-core/src/main/java/com/alibaba/excel/cache/Ehcache.java b/easyexcel-core/src/main/java/com/alibaba/excel/cache/Ehcache.java index b7b542ee..f5481b43 100644 --- a/easyexcel-core/src/main/java/com/alibaba/excel/cache/Ehcache.java +++ b/easyexcel-core/src/main/java/com/alibaba/excel/cache/Ehcache.java @@ -15,6 +15,7 @@ import org.ehcache.config.CacheConfiguration; import org.ehcache.config.builders.CacheConfigurationBuilder; import org.ehcache.config.builders.CacheManagerBuilder; import org.ehcache.config.builders.ResourcePoolsBuilder; +import org.ehcache.config.units.EntryUnit; import org.ehcache.config.units.MemoryUnit; /** @@ -50,12 +51,27 @@ public class Ehcache implements ReadCache { */ private int cacheMiss = 0; - public Ehcache(int maxCacheActivateSize) { - activeCacheConfiguration = CacheConfigurationBuilder - .newCacheConfigurationBuilder(Integer.class, ArrayList.class, - ResourcePoolsBuilder.newResourcePoolsBuilder().heap(maxCacheActivateSize, MemoryUnit.MB)) - .withSizeOfMaxObjectGraph(1000 * 1000L).withSizeOfMaxObjectSize(maxCacheActivateSize, MemoryUnit.MB) - .build(); + @Deprecated + public Ehcache(Integer maxCacheActivateSize) { + this(maxCacheActivateSize, null); + } + + public Ehcache(Integer maxCacheActivateSize, Integer maxCacheActivateBatchCount) { + // In order to be compatible with the code + // If the user set up `maxCacheActivateSize`, then continue using it + if (maxCacheActivateSize != null) { + this.activeCacheConfiguration = CacheConfigurationBuilder + .newCacheConfigurationBuilder(Integer.class, ArrayList.class, + ResourcePoolsBuilder.newResourcePoolsBuilder() + .heap(maxCacheActivateSize, MemoryUnit.MB)) + .build(); + } else { + this.activeCacheConfiguration = CacheConfigurationBuilder + .newCacheConfigurationBuilder(Integer.class, ArrayList.class, + ResourcePoolsBuilder.newResourcePoolsBuilder() + .heap(maxCacheActivateBatchCount, EntryUnit.ENTRIES)) + .build(); + } } static { @@ -64,9 +80,8 @@ public class Ehcache implements ReadCache { CacheManagerBuilder.newCacheManagerBuilder().with(CacheManagerBuilder.persistence(cacheFile)).build(true); ACTIVE_CACHE_MANAGER = CacheManagerBuilder.newCacheManagerBuilder().build(true); FILE_CACHE_CONFIGURATION = CacheConfigurationBuilder - .newCacheConfigurationBuilder(Integer.class, ArrayList.class, - ResourcePoolsBuilder.newResourcePoolsBuilder().disk(10, MemoryUnit.GB)) - .withSizeOfMaxObjectGraph(1000 * 1000L).withSizeOfMaxObjectSize(10, MemoryUnit.GB).build(); + .newCacheConfigurationBuilder(Integer.class, ArrayList.class, ResourcePoolsBuilder.newResourcePoolsBuilder() + .disk(20, MemoryUnit.GB)).build(); } @Override diff --git a/easyexcel-core/src/main/java/com/alibaba/excel/cache/selector/SimpleReadCacheSelector.java b/easyexcel-core/src/main/java/com/alibaba/excel/cache/selector/SimpleReadCacheSelector.java index 658806bd..278c63be 100644 --- a/easyexcel-core/src/main/java/com/alibaba/excel/cache/selector/SimpleReadCacheSelector.java +++ b/easyexcel-core/src/main/java/com/alibaba/excel/cache/selector/SimpleReadCacheSelector.java @@ -2,6 +2,10 @@ package com.alibaba.excel.cache.selector; import java.io.IOException; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import org.apache.poi.openxml4j.opc.PackagePart; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -15,6 +19,9 @@ import com.alibaba.excel.cache.ReadCache; * * @author Jiaju Zhuang **/ +@Getter +@Setter +@EqualsAndHashCode public class SimpleReadCacheSelector implements ReadCacheSelector { private static final Logger LOGGER = LoggerFactory.getLogger(SimpleReadCacheSelector.class); /** @@ -24,37 +31,46 @@ public class SimpleReadCacheSelector implements ReadCacheSelector { /** * If it's less than 5M, use map cache, or use ehcache.unit MB. */ - private static final int DEFAULT_MAX_USE_MAP_CACHE_SIZE = 5; + private static final long DEFAULT_MAX_USE_MAP_CACHE_SIZE = 5; + /** - * Maximum size of cache activation.unit MB. + * Maximum batch of `SharedStrings` stored in memory. + * The batch size is 100.{@link Ehcache#BATCH_COUNT} */ - private static final int DEFAULT_MAX_EHCACHE_ACTIVATE_SIZE = 20; + private static final int DEFAULT_MAX_EHCACHE_ACTIVATE_BATCH_COUNT = 20; /** * Shared strings exceeding this value will use {@link Ehcache},or use {@link MapCache}.unit MB. */ - private final long maxUseMapCacheSize; + private Long maxUseMapCacheSize; /** * Maximum size of cache activation.unit MB. + * + * @deprecated Please use maxCacheActivateBatchCount to control the size of the occupied memory + */ + @Deprecated + private Integer maxCacheActivateSize; + + /** + * Maximum batch of `SharedStrings` stored in memory. + * The batch size is 100.{@link Ehcache#BATCH_COUNT} */ - private final int maxCacheActivateSize; + private Integer maxCacheActivateBatchCount; public SimpleReadCacheSelector() { - this(DEFAULT_MAX_USE_MAP_CACHE_SIZE, DEFAULT_MAX_EHCACHE_ACTIVATE_SIZE); } - public SimpleReadCacheSelector(long maxUseMapCacheSize, int maxCacheActivateSize) { - if (maxUseMapCacheSize <= 0) { - this.maxUseMapCacheSize = DEFAULT_MAX_USE_MAP_CACHE_SIZE; - } else { - this.maxUseMapCacheSize = maxUseMapCacheSize; - } - if (maxCacheActivateSize <= 0) { - this.maxCacheActivateSize = DEFAULT_MAX_EHCACHE_ACTIVATE_SIZE; - } else { - this.maxCacheActivateSize = maxCacheActivateSize; - } + /** + * Parameter maxCacheActivateSize has already been abandoned + * + * @param maxUseMapCacheSize + * @param maxCacheActivateSize + */ + @Deprecated + public SimpleReadCacheSelector(Long maxUseMapCacheSize, Integer maxCacheActivateSize) { + this.maxUseMapCacheSize = maxUseMapCacheSize; + this.maxCacheActivateSize = maxCacheActivateSize; } @Override @@ -68,6 +84,9 @@ public class SimpleReadCacheSelector implements ReadCacheSelector { return new MapCache(); } } + if (maxUseMapCacheSize == null) { + maxUseMapCacheSize = DEFAULT_MAX_USE_MAP_CACHE_SIZE; + } if (size < maxUseMapCacheSize * B2M) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Use map cache.size:{}", size); @@ -77,6 +96,17 @@ public class SimpleReadCacheSelector implements ReadCacheSelector { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Use ehcache.size:{}", size); } - return new Ehcache(maxCacheActivateSize); + + // In order to be compatible with the code + // If the user set up `maxCacheActivateSize`, then continue using it + if (maxCacheActivateSize != null) { + return new Ehcache(maxCacheActivateSize, maxCacheActivateBatchCount); + } else { + if (maxCacheActivateBatchCount == null) { + maxCacheActivateBatchCount = DEFAULT_MAX_EHCACHE_ACTIVATE_BATCH_COUNT; + } + return new Ehcache(maxCacheActivateSize, maxCacheActivateBatchCount); + } + } } diff --git a/update.md b/update.md index 54c01471..3383244d 100644 --- a/update.md +++ b/update.md @@ -1,6 +1,7 @@ # 3.2.1 * 兼容`LocalDate` [Issue #2908](https://github.com/alibaba/easyexcel/issues/2908) +* 优化大文件内存存储，减少内存占用 [Issue #2657](https://github.com/alibaba/easyexcel/issues/2657) # 3.2.0 @@ -14,7 +15,8 @@ # 3.1.4 -* 提高xlsx读取兼容性：在存在第一行很多空列的情况下，忽略空列 +* 提高xlsx读取兼容性：在存在第一行很多空 +* 列的情况下，忽略空列 # 3.1.3