Browse Source

* 调整读写默认大小,防止大批量写的时候可能会full gc

pull/2177/head
Jiaju Zhuang 3 years ago
parent
commit
604ce6f0fa
  1. 2
      README.md
  2. 2
      pom.xml
  3. 2
      src/main/java/com/alibaba/excel/cache/Ehcache.java
  4. 2
      src/main/java/com/alibaba/excel/metadata/csv/CsvSheet.java
  5. 2
      src/main/java/com/alibaba/excel/read/listener/PageReadListener.java
  6. 8
      src/test/java/com/alibaba/easyexcel/test/core/large/LargeDataTest.java
  7. 10
      src/test/java/com/alibaba/easyexcel/test/demo/read/CellDataDemoHeadDataListener.java
  8. 8
      src/test/java/com/alibaba/easyexcel/test/demo/read/ConverterDataListener.java
  9. 4
      src/test/java/com/alibaba/easyexcel/test/demo/read/DemoDataListener.java
  10. 2
      src/test/java/com/alibaba/easyexcel/test/demo/read/DemoExceptionListener.java
  11. 9
      src/test/java/com/alibaba/easyexcel/test/demo/read/DemoHeadDataListener.java
  12. 8
      src/test/java/com/alibaba/easyexcel/test/demo/read/IndexOrNameDataListener.java
  13. 8
      src/test/java/com/alibaba/easyexcel/test/demo/read/NoModelDataListener.java
  14. 2
      src/test/java/com/alibaba/easyexcel/test/demo/read/ReadTest.java
  15. 2
      src/test/java/com/alibaba/easyexcel/test/demo/web/UploadDataListener.java
  16. 2
      src/test/java/com/alibaba/easyexcel/test/temp/LockDataListener.java
  17. 2
      src/test/java/com/alibaba/easyexcel/test/temp/read/HDListener.java
  18. 2
      src/test/java/com/alibaba/easyexcel/test/temp/read/HeadListener.java
  19. 8
      src/test/java/com/alibaba/easyexcel/test/temp/simple/HgListener.java
  20. 2
      src/test/java/com/alibaba/easyexcel/test/temp/simple/RepeatListener.java
  21. 3
      update.md

2
README.md

@ -41,7 +41,7 @@ Java解析、生成Excel比较有名的框架有Apache poi、jxl。但他们都
<dependency> <dependency>
<groupId>com.alibaba</groupId> <groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId> <artifactId>easyexcel</artifactId>
<version>3.0.3</version> <version>3.0.4</version>
</dependency> </dependency>
``` ```

2
pom.xml

@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba</groupId> <groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId> <artifactId>easyexcel</artifactId>
<version>3.0.3</version> <version>3.0.4</version>
<packaging>jar</packaging> <packaging>jar</packaging>
<name>easyexcel</name> <name>easyexcel</name>

2
src/main/java/com/alibaba/excel/cache/Ehcache.java vendored

@ -24,7 +24,7 @@ import org.ehcache.config.units.MemoryUnit;
*/ */
@Slf4j @Slf4j
public class Ehcache implements ReadCache { public class Ehcache implements ReadCache {
public static final int BATCH_COUNT = 1000; public static final int BATCH_COUNT = 100;
/** /**
* Key index * Key index
*/ */

2
src/main/java/com/alibaba/excel/metadata/csv/CsvSheet.java

@ -85,7 +85,7 @@ public class CsvSheet implements Sheet, Closeable {
public CsvSheet(CsvWorkbook csvWorkbook, Appendable out) { public CsvSheet(CsvWorkbook csvWorkbook, Appendable out) {
this.csvWorkbook = csvWorkbook; this.csvWorkbook = csvWorkbook;
this.out = out; this.out = out;
this.rowCacheCount = 3000; this.rowCacheCount = 100;
this.csvFormat = CSVFormat.DEFAULT; this.csvFormat = CSVFormat.DEFAULT;
this.lastRowIndex = -1; this.lastRowIndex = -1;
} }

2
src/main/java/com/alibaba/excel/read/listener/PageReadListener.java

@ -17,7 +17,7 @@ public class PageReadListener<T> implements ReadListener<T> {
/** /**
* Single handle the amount of data * Single handle the amount of data
*/ */
public static int BATCH_COUNT = 3000; public static int BATCH_COUNT = 100;
/** /**
* Temporary storage of data * Temporary storage of data
*/ */

8
src/test/java/com/alibaba/easyexcel/test/core/large/LargeDataTest.java

@ -59,7 +59,7 @@ public class LargeDataTest {
public void t02Fill() { public void t02Fill() {
ExcelWriter excelWriter = EasyExcel.write(fileFill07).withTemplate(template07).build(); ExcelWriter excelWriter = EasyExcel.write(fileFill07).withTemplate(template07).build();
WriteSheet writeSheet = EasyExcel.writerSheet().build(); WriteSheet writeSheet = EasyExcel.writerSheet().build();
for (int j = 0; j < 100; j++) { for (int j = 0; j < 5000; j++) {
excelWriter.fill(data(), writeSheet); excelWriter.fill(data(), writeSheet);
LOGGER.info("{} fill success.", j); LOGGER.info("{} fill success.", j);
} }
@ -72,7 +72,7 @@ public class LargeDataTest {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
ExcelWriter excelWriter = EasyExcel.write(fileCsv).build(); ExcelWriter excelWriter = EasyExcel.write(fileCsv).build();
WriteSheet writeSheet = EasyExcel.writerSheet().build(); WriteSheet writeSheet = EasyExcel.writerSheet().build();
for (int j = 0; j < 100; j++) { for (int j = 0; j < 5000; j++) {
excelWriter.write(data(), writeSheet); excelWriter.write(data(), writeSheet);
LOGGER.info("{} write success.", j); LOGGER.info("{} write success.", j);
} }
@ -97,7 +97,7 @@ public class LargeDataTest {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
excelWriter = EasyExcel.write(fileWrite07, LargeData.class).build(); excelWriter = EasyExcel.write(fileWrite07, LargeData.class).build();
writeSheet = EasyExcel.writerSheet().build(); writeSheet = EasyExcel.writerSheet().build();
for (int j = 0; j < 100; j++) { for (int j = 0; j < 5000; j++) {
excelWriter.write(data(), writeSheet); excelWriter.write(data(), writeSheet);
LOGGER.info("{} write success.", j); LOGGER.info("{} write success.", j);
} }
@ -130,7 +130,7 @@ public class LargeDataTest {
private List<LargeData> data() { private List<LargeData> data() {
List<LargeData> list = new ArrayList<>(); List<LargeData> list = new ArrayList<>();
int size = i + 5000; int size = i + 100;
for (; i < size; i++) { for (; i < size; i++) {
LargeData largeData = new LargeData(); LargeData largeData = new LargeData();
list.add(largeData); list.add(largeData);

10
src/test/java/com/alibaba/easyexcel/test/demo/read/CellDataDemoHeadDataListener.java

@ -3,13 +3,13 @@ package com.alibaba.easyexcel.test.demo.read;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* 读取头 * 读取头
* *
@ -18,9 +18,9 @@ import com.alibaba.fastjson.JSON;
public class CellDataDemoHeadDataListener extends AnalysisEventListener<CellDataReadDemoData> { public class CellDataDemoHeadDataListener extends AnalysisEventListener<CellDataReadDemoData> {
private static final Logger LOGGER = LoggerFactory.getLogger(CellDataDemoHeadDataListener.class); private static final Logger LOGGER = LoggerFactory.getLogger(CellDataDemoHeadDataListener.class);
/** /**
* 每隔5条存储数据库实际使用中可以3000条然后清理list 方便内存回收 * 每隔5条存储数据库实际使用中可以100条然后清理list 方便内存回收
*/ */
private static final int BATCH_COUNT = 5; private static final int BATCH_COUNT = 100;
List<CellDataReadDemoData> list = new ArrayList<CellDataReadDemoData>(); List<CellDataReadDemoData> list = new ArrayList<CellDataReadDemoData>();
@Override @Override

8
src/test/java/com/alibaba/easyexcel/test/demo/read/ConverterDataListener.java

@ -3,13 +3,13 @@ package com.alibaba.easyexcel.test.demo.read;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* 模板的读取类 * 模板的读取类
* *
@ -18,7 +18,7 @@ import com.alibaba.fastjson.JSON;
public class ConverterDataListener extends AnalysisEventListener<ConverterData> { public class ConverterDataListener extends AnalysisEventListener<ConverterData> {
private static final Logger LOGGER = LoggerFactory.getLogger(ConverterDataListener.class); private static final Logger LOGGER = LoggerFactory.getLogger(ConverterDataListener.class);
/** /**
* 每隔5条存储数据库实际使用中可以3000条然后清理list 方便内存回收 * 每隔5条存储数据库实际使用中可以100条然后清理list 方便内存回收
*/ */
private static final int BATCH_COUNT = 5; private static final int BATCH_COUNT = 5;
List<ConverterData> list = new ArrayList<ConverterData>(); List<ConverterData> list = new ArrayList<ConverterData>();

4
src/test/java/com/alibaba/easyexcel/test/demo/read/DemoDataListener.java

@ -19,9 +19,9 @@ import org.slf4j.LoggerFactory;
public class DemoDataListener extends AnalysisEventListener<DemoData> { public class DemoDataListener extends AnalysisEventListener<DemoData> {
private static final Logger LOGGER = LoggerFactory.getLogger(DemoDataListener.class); private static final Logger LOGGER = LoggerFactory.getLogger(DemoDataListener.class);
/** /**
* 每隔5条存储数据库实际使用中可以3000条然后清理list 方便内存回收 * 每隔5条存储数据库实际使用中可以100条然后清理list 方便内存回收
*/ */
private static final int BATCH_COUNT = 3000; private static final int BATCH_COUNT = 100;
/** /**
* 缓存的数据 * 缓存的数据
*/ */

2
src/test/java/com/alibaba/easyexcel/test/demo/read/DemoExceptionListener.java

@ -20,7 +20,7 @@ import com.alibaba.fastjson.JSON;
public class DemoExceptionListener extends AnalysisEventListener<ExceptionDemoData> { public class DemoExceptionListener extends AnalysisEventListener<ExceptionDemoData> {
private static final Logger LOGGER = LoggerFactory.getLogger(DemoExceptionListener.class); private static final Logger LOGGER = LoggerFactory.getLogger(DemoExceptionListener.class);
/** /**
* 每隔5条存储数据库实际使用中可以3000条然后清理list 方便内存回收 * 每隔5条存储数据库实际使用中可以100条然后清理list 方便内存回收
*/ */
private static final int BATCH_COUNT = 5; private static final int BATCH_COUNT = 5;
List<ExceptionDemoData> list = new ArrayList<ExceptionDemoData>(); List<ExceptionDemoData> list = new ArrayList<ExceptionDemoData>();

9
src/test/java/com/alibaba/easyexcel/test/demo/read/DemoHeadDataListener.java

@ -4,15 +4,14 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.excel.exception.ExcelDataConvertException; import com.alibaba.excel.exception.ExcelDataConvertException;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* 读取头 * 读取头
* *
@ -21,7 +20,7 @@ import com.alibaba.fastjson.JSON;
public class DemoHeadDataListener extends AnalysisEventListener<DemoData> { public class DemoHeadDataListener extends AnalysisEventListener<DemoData> {
private static final Logger LOGGER = LoggerFactory.getLogger(DemoHeadDataListener.class); private static final Logger LOGGER = LoggerFactory.getLogger(DemoHeadDataListener.class);
/** /**
* 每隔5条存储数据库实际使用中可以3000条然后清理list 方便内存回收 * 每隔5条存储数据库实际使用中可以100条然后清理list 方便内存回收
*/ */
private static final int BATCH_COUNT = 5; private static final int BATCH_COUNT = 5;
List<DemoData> list = new ArrayList<DemoData>(); List<DemoData> list = new ArrayList<DemoData>();

8
src/test/java/com/alibaba/easyexcel/test/demo/read/IndexOrNameDataListener.java

@ -3,13 +3,13 @@ package com.alibaba.easyexcel.test.demo.read;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* 模板的读取类 * 模板的读取类
* *
@ -18,7 +18,7 @@ import com.alibaba.fastjson.JSON;
public class IndexOrNameDataListener extends AnalysisEventListener<IndexOrNameData> { public class IndexOrNameDataListener extends AnalysisEventListener<IndexOrNameData> {
private static final Logger LOGGER = LoggerFactory.getLogger(IndexOrNameDataListener.class); private static final Logger LOGGER = LoggerFactory.getLogger(IndexOrNameDataListener.class);
/** /**
* 每隔5条存储数据库实际使用中可以3000条然后清理list 方便内存回收 * 每隔5条存储数据库实际使用中可以100条然后清理list 方便内存回收
*/ */
private static final int BATCH_COUNT = 5; private static final int BATCH_COUNT = 5;
List<IndexOrNameData> list = new ArrayList<IndexOrNameData>(); List<IndexOrNameData> list = new ArrayList<IndexOrNameData>();

8
src/test/java/com/alibaba/easyexcel/test/demo/read/NoModelDataListener.java

@ -4,13 +4,13 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* 直接用map接收数据 * 直接用map接收数据
* *
@ -19,7 +19,7 @@ import com.alibaba.fastjson.JSON;
public class NoModelDataListener extends AnalysisEventListener<Map<Integer, String>> { public class NoModelDataListener extends AnalysisEventListener<Map<Integer, String>> {
private static final Logger LOGGER = LoggerFactory.getLogger(NoModelDataListener.class); private static final Logger LOGGER = LoggerFactory.getLogger(NoModelDataListener.class);
/** /**
* 每隔5条存储数据库实际使用中可以3000条然后清理list 方便内存回收 * 每隔5条存储数据库实际使用中可以100条然后清理list 方便内存回收
*/ */
private static final int BATCH_COUNT = 5; private static final int BATCH_COUNT = 5;
List<Map<Integer, String>> list = new ArrayList<Map<Integer, String>>(); List<Map<Integer, String>> list = new ArrayList<Map<Integer, String>>();

2
src/test/java/com/alibaba/easyexcel/test/demo/read/ReadTest.java

@ -64,7 +64,7 @@ public class ReadTest {
/** /**
* 单次缓存的数据量 * 单次缓存的数据量
*/ */
public static final int BATCH_COUNT = 3000; public static final int BATCH_COUNT = 100;
/** /**
*临时存储 *临时存储
*/ */

2
src/test/java/com/alibaba/easyexcel/test/demo/web/UploadDataListener.java

@ -20,7 +20,7 @@ public class UploadDataListener extends AnalysisEventListener<UploadData> {
private static final Logger LOGGER = private static final Logger LOGGER =
LoggerFactory.getLogger(com.alibaba.easyexcel.test.demo.read.DemoDataListener.class); LoggerFactory.getLogger(com.alibaba.easyexcel.test.demo.read.DemoDataListener.class);
/** /**
* 每隔5条存储数据库实际使用中可以3000条然后清理list 方便内存回收 * 每隔5条存储数据库实际使用中可以100条然后清理list 方便内存回收
*/ */
private static final int BATCH_COUNT = 5; private static final int BATCH_COUNT = 5;
List<UploadData> list = new ArrayList<UploadData>(); List<UploadData> list = new ArrayList<UploadData>();

2
src/test/java/com/alibaba/easyexcel/test/temp/LockDataListener.java

@ -19,7 +19,7 @@ import com.alibaba.fastjson.JSON;
public class LockDataListener extends AnalysisEventListener<LockData> { public class LockDataListener extends AnalysisEventListener<LockData> {
private static final Logger LOGGER = LoggerFactory.getLogger(DemoDataListener.class); private static final Logger LOGGER = LoggerFactory.getLogger(DemoDataListener.class);
/** /**
* 每隔5条存储数据库实际使用中可以3000条然后清理list 方便内存回收 * 每隔5条存储数据库实际使用中可以100条然后清理list 方便内存回收
*/ */
private static final int BATCH_COUNT = 5; private static final int BATCH_COUNT = 5;
List<LockData> list = new ArrayList<LockData>(); List<LockData> list = new ArrayList<LockData>();

2
src/test/java/com/alibaba/easyexcel/test/temp/read/HDListener.java

@ -17,7 +17,7 @@ import com.alibaba.fastjson.JSON;
public class HDListener extends AnalysisEventListener<HeadReadData> { public class HDListener extends AnalysisEventListener<HeadReadData> {
private static final Logger LOGGER = LoggerFactory.getLogger(HDListener.class); private static final Logger LOGGER = LoggerFactory.getLogger(HDListener.class);
/** /**
* 每隔5条存储数据库实际使用中可以3000条然后清理list 方便内存回收 * 每隔5条存储数据库实际使用中可以100条然后清理list 方便内存回收
*/ */
private static final int BATCH_COUNT = 5; private static final int BATCH_COUNT = 5;

2
src/test/java/com/alibaba/easyexcel/test/temp/read/HeadListener.java

@ -17,7 +17,7 @@ import com.alibaba.fastjson.JSON;
public class HeadListener extends AnalysisEventListener<HeadReadData> { public class HeadListener extends AnalysisEventListener<HeadReadData> {
private static final Logger LOGGER = LoggerFactory.getLogger(HeadListener.class); private static final Logger LOGGER = LoggerFactory.getLogger(HeadListener.class);
/** /**
* 每隔5条存储数据库实际使用中可以3000条然后清理list 方便内存回收 * 每隔5条存储数据库实际使用中可以100条然后清理list 方便内存回收
*/ */
private static final int BATCH_COUNT = 5; private static final int BATCH_COUNT = 5;

8
src/test/java/com/alibaba/easyexcel/test/temp/simple/HgListener.java

@ -2,13 +2,13 @@ package com.alibaba.easyexcel.test.temp.simple;
import java.util.Map; import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* 模板的读取类 * 模板的读取类
* *
@ -17,7 +17,7 @@ import com.alibaba.fastjson.JSON;
public class HgListener extends AnalysisEventListener<Map<Integer, String>> { public class HgListener extends AnalysisEventListener<Map<Integer, String>> {
private static final Logger LOGGER = LoggerFactory.getLogger(HgListener.class); private static final Logger LOGGER = LoggerFactory.getLogger(HgListener.class);
/** /**
* 每隔5条存储数据库实际使用中可以3000条然后清理list 方便内存回收 * 每隔5条存储数据库实际使用中可以100条然后清理list 方便内存回收
*/ */
private static final int BATCH_COUNT = 5; private static final int BATCH_COUNT = 5;

2
src/test/java/com/alibaba/easyexcel/test/temp/simple/RepeatListener.java

@ -20,7 +20,7 @@ import com.alibaba.fastjson.JSON;
public class RepeatListener extends AnalysisEventListener<LockData> { public class RepeatListener extends AnalysisEventListener<LockData> {
private static final Logger LOGGER = LoggerFactory.getLogger(DemoDataListener.class); private static final Logger LOGGER = LoggerFactory.getLogger(DemoDataListener.class);
/** /**
* 每隔5条存储数据库实际使用中可以3000条然后清理list 方便内存回收 * 每隔5条存储数据库实际使用中可以100条然后清理list 方便内存回收
*/ */
private static final int BATCH_COUNT = 5; private static final int BATCH_COUNT = 5;
List<LockData> list = new ArrayList<LockData>(); List<LockData> list = new ArrayList<LockData>();

3
update.md

@ -1,3 +1,6 @@
# 3.0.4
* 调整读写默认大小,防止大批量写的时候可能会full gc
# 3.0.3 # 3.0.3
* 修复`HeadStyle`无效的bug * 修复`HeadStyle`无效的bug

Loading…
Cancel
Save