Java 使用POI将大型结果集写入Excel文件
这是一种内联文件，但所讨论的文件是Excel文件 我正在使用ApachePOI库编写一个Excel文件，其中包含从ResultSet对象检索到的大型数据集。数据范围从几千条记录到大约一百万条记录；不确定如何将其转换为Excel格式的文件系统字节 下面是我编写的测试代码，用于检查编写如此大的结果集所花费的时间，以及w.r.t CPU和内存的性能影响Java 使用POI将大型结果集写入Excel文件,java,jdbc,apache-poi,Java,Jdbc,Apache Poi,这是一种内联文件，但所讨论的文件是Excel文件 我正在使用ApachePOI库编写一个Excel文件，其中包含从ResultSet对象检索到的大型数据集。数据范围从几千条记录到大约一百万条记录；不确定如何将其转换为Excel格式的文件系统字节 下面是我编写的测试代码，用于检查编写如此大的结果集所花费的时间，以及w.r.t CPU和内存的性能影响 protected void writeResultsetToExcelFile(ResultSet rs, int numSheets, Strin
protected void writeResultsetToExcelFile(ResultSet rs, int numSheets, String fileNameAndPath) throws Exception {
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(fileNameAndPath));
int numColumns = rs.getMetaData().getColumnCount();
Workbook wb = ExcelFileUtil.createExcelWorkBook(true, numSheets);
Row heading = wb.getSheetAt(0).createRow(1);
ResultSetMetaData rsmd = rs.getMetaData();
for(int x = 0; x < numColumns; x++) {
Cell cell = heading.createCell(x+1);
cell.setCellValue(rsmd.getColumnLabel(x+1));
}
int rowNumber = 2;
int sheetNumber = 0;
while(rs.next()) {
if(rowNumber == 65001) {
log("Sheet " + sheetNumber + "written; moving onto to sheet " + (sheetNumber + 1));
sheetNumber++;
rowNumber = 2;
}
Row row = wb.getSheetAt(sheetNumber).createRow(rowNumber);
for(int y = 0; y < numColumns; y++) {
row.createCell(y+1).setCellValue(rs.getString(y+1));
wb.write(bos);
}
rowNumber++;
}
//wb.write(bos);
bos.close();
}
受保护的void writeResultsetToExcelFile（结果集rs、整数表单、字符串文件名和路径）引发异常{
BufferedOutputStream bos=新的BufferedOutputStream（新的FileOutputStream（FileName和Path））；
int numColumns=rs.getMetaData（）.getColumnCount（）；
工作簿wb=ExcelFileUtil.createExcelWorkBook（true，numSheets）；
行标题=wb.getSheetAt（0.createRow（1）；
ResultSetMetaData rsmd=rs.getMetaData（）；
对于（int x=0；x有什么想法吗？哦。我想你把练习册写了944000遍。您的wb.write（bos）调用位于内部循环中。我不确定这是否与工作簿类的语义完全一致？从该类的Javadocs中可以看出，该方法将整个工作簿写入指定的输出流。随着时间的推移，它会把你添加的每一行都写出来
这也解释了为什么你看到的也是一排。要写入文件的第一个工作簿（一行）是正在显示的所有文件，然后是7GB的垃圾。< P>除非您必须编写公式或格式，否则您应该考虑写出.CSV文件。无限简单、无限快速，Excel将根据定义自动正确地转换为.xls或.xlsx。目前，我接受了@Gian的建议，将每个工作簿的记录数限制为500k，并将剩余的记录滚动到下一个工作簿。看起来工作不错。对于上述配置，每个工作簿大约需要10分钟。我更新了BigGridDemo以支持多张工作表 BigExcelWriterImpl.java
package com.gdais.common.apache.poi.bigexcelwriter;
import static com.google.common.base.Preconditions.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.commons.io.FilenameUtils;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
public class BigExcelWriterImpl implements BigExcelWriter {
private static final String XML_ENCODING = "UTF-8";
@Nonnull
private final File outputFile;
@Nullable
private final File tempFileOutputDir;
@Nullable
private File templateFile = null;
@Nullable
private XSSFWorkbook workbook = null;
@Nonnull
private LinkedHashMap<String, XSSFSheet> addedSheets = new LinkedHashMap<String, XSSFSheet>();
@Nonnull
private Map<XSSFSheet, File> sheetTempFiles = new HashMap<XSSFSheet, File>();
BigExcelWriterImpl(@Nonnull File outputFile) {
this.outputFile = outputFile;
this.tempFileOutputDir = outputFile.getParentFile();
}
@Override
public BigExcelWriter createWorkbook() {
workbook = new XSSFWorkbook();
return this;
}
@Override
public BigExcelWriter addSheets(String... sheetNames) {
checkState(workbook != null, "workbook must be created before adding sheets");
for (String sheetName : sheetNames) {
XSSFSheet sheet = workbook.createSheet(sheetName);
addedSheets.put(sheetName, sheet);
}
return this;
}
@Override
public BigExcelWriter writeWorkbookTemplate() throws IOException {
checkState(workbook != null, "workbook must be created before writing template");
checkState(templateFile == null, "template file already written");
templateFile = File.createTempFile(FilenameUtils.removeExtension(outputFile.getName())
+ "-template", ".xlsx", tempFileOutputDir);
System.out.println(templateFile);
FileOutputStream os = new FileOutputStream(templateFile);
workbook.write(os);
os.close();
return this;
}
@Override
public SpreadsheetWriter createSpreadsheetWriter(String sheetName) throws IOException {
if (!addedSheets.containsKey(sheetName)) {
addSheets(sheetName);
}
return createSpreadsheetWriter(addedSheets.get(sheetName));
}
@Override
public SpreadsheetWriter createSpreadsheetWriter(XSSFSheet sheet) throws IOException {
checkState(!sheetTempFiles.containsKey(sheet), "writer already created for this sheet");
File tempSheetFile = File.createTempFile(
FilenameUtils.removeExtension(outputFile.getName())
+ "-sheet" + sheet.getSheetName(), ".xml", tempFileOutputDir);
Writer out = null;
try {
out = new OutputStreamWriter(new FileOutputStream(tempSheetFile), XML_ENCODING);
SpreadsheetWriter sw = new SpreadsheetWriterImpl(out);
sheetTempFiles.put(sheet, tempSheetFile);
return sw;
} catch (RuntimeException e) {
if (out != null) {
out.close();
}
throw e;
}
}
private static Function<XSSFSheet, String> getSheetName = new Function<XSSFSheet, String>() {
@Override
public String apply(XSSFSheet sheet) {
return sheet.getPackagePart().getPartName().getName().substring(1);
}
};
@Override
public File completeWorkbook() throws IOException {
FileOutputStream out = null;
try {
out = new FileOutputStream(outputFile);
ZipOutputStream zos = new ZipOutputStream(out);
Iterable<String> sheetEntries = Iterables.transform(sheetTempFiles.keySet(),
getSheetName);
System.out.println("Sheet Entries: " + sheetEntries);
copyTemplateMinusEntries(templateFile, zos, sheetEntries);
for (Map.Entry<XSSFSheet, File> entry : sheetTempFiles.entrySet()) {
XSSFSheet sheet = entry.getKey();
substituteSheet(entry.getValue(), getSheetName.apply(sheet), zos);
}
zos.close();
out.close();
return outputFile;
} finally {
if (out != null) {
out.close();
}
}
}
private static void copyTemplateMinusEntries(File templateFile,
ZipOutputStream zos, Iterable<String> entries) throws IOException {
ZipFile templateZip = new ZipFile(templateFile);
@SuppressWarnings("unchecked")
Enumeration<ZipEntry> en = (Enumeration<ZipEntry>) templateZip.entries();
while (en.hasMoreElements()) {
ZipEntry ze = en.nextElement();
if (!Iterables.contains(entries, ze.getName())) {
System.out.println("Adding template entry: " + ze.getName());
zos.putNextEntry(new ZipEntry(ze.getName()));
InputStream is = templateZip.getInputStream(ze);
copyStream(is, zos);
is.close();
}
}
}
private static void substituteSheet(File tmpfile, String entry,
ZipOutputStream zos)
throws IOException {
System.out.println("Adding sheet entry: " + entry);
zos.putNextEntry(new ZipEntry(entry));
InputStream is = new FileInputStream(tmpfile);
copyStream(is, zos);
is.close();
}
private static void copyStream(InputStream in, OutputStream out) throws IOException {
byte[] chunk = new byte[1024];
int count;
while ((count = in.read(chunk)) >= 0) {
out.write(chunk, 0, count);
}
}
@Override
public Workbook getWorkbook() {
return workbook;
}
@Override
public ImmutableList<XSSFSheet> getSheets() {
return ImmutableList.copyOf(addedSheets.values());
}
}
package com.gdais.common.apache.poi.bigexcelwriter;
import java.io.IOException;
import java.io.Writer;
import java.util.Calendar;
import org.apache.poi.ss.usermodel.DateUtil;
import org.apache.poi.ss.util.CellReference;
class SpreadsheetWriterImpl implements SpreadsheetWriter {
private static final String XML_ENCODING = "UTF-8";
private final Writer _out;
private int _rownum;
SpreadsheetWriterImpl(Writer out) {
_out = out;
}
@Override
public SpreadsheetWriter closeFile() throws IOException {
_out.close();
return this;
}
@Override
public SpreadsheetWriter beginSheet() throws IOException {
_out.write("<?xml version=\"1.0\" encoding=\""
+ XML_ENCODING
+ "\"?>"
+
"<worksheet xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\">");
_out.write("<sheetData>\n");
return this;
}
@Override
public SpreadsheetWriter endSheet() throws IOException {
_out.write("</sheetData>");
_out.write("</worksheet>");
closeFile();
return this;
}
/**
* Insert a new row
*
* @param rownum
* 0-based row number
*/
@Override
public SpreadsheetWriter insertRow(int rownum) throws IOException {
_out.write("<row r=\"" + (rownum + 1) + "\">\n");
this._rownum = rownum;
return this;
}
/**
* Insert row end marker
*/
@Override
public SpreadsheetWriter endRow() throws IOException {
_out.write("</row>\n");
return this;
}
@Override
public SpreadsheetWriter createCell(int columnIndex, String value, int styleIndex)
throws IOException {
String ref = new CellReference(_rownum, columnIndex).formatAsString();
_out.write("<c r=\"" + ref + "\" t=\"inlineStr\"");
if (styleIndex != -1) {
_out.write(" s=\"" + styleIndex + "\"");
}
_out.write(">");
_out.write("<is><t>" + value + "</t></is>");
_out.write("</c>");
return this;
}
@Override
public SpreadsheetWriter createCell(int columnIndex, String value) throws IOException {
createCell(columnIndex, value, -1);
return this;
}
@Override
public SpreadsheetWriter createCell(int columnIndex, double value, int styleIndex)
throws IOException {
String ref = new CellReference(_rownum, columnIndex).formatAsString();
_out.write("<c r=\"" + ref + "\" t=\"n\"");
if (styleIndex != -1) {
_out.write(" s=\"" + styleIndex + "\"");
}
_out.write(">");
_out.write("<v>" + value + "</v>");
_out.write("</c>");
return this;
}
@Override
public SpreadsheetWriter createCell(int columnIndex, double value) throws IOException {
createCell(columnIndex, value, -1);
return this;
}
@Override
public SpreadsheetWriter createCell(int columnIndex, Calendar value, int styleIndex)
throws IOException {
createCell(columnIndex, DateUtil.getExcelDate(value, false), styleIndex);
return this;
}
@Override
public SpreadsheetWriter createCell(int columnIndex, Calendar value)
throws IOException {
createCell(columnIndex, value, -1);
return this;
}
}
包com.gdais.common.apache.poi.bigexcelwriter；
导入静态com.google.common.base.Premissions.*；
导入java.io.File；
导入java.io.FileInputStream；
导入java.io.FileOutputStream；
导入java.io.IOException；
导入java.io.InputStream；
导入java.io.OutputStream；
导入java.io.OutputStreamWriter；
导入java.io.Writer；
导入java.util.Enumeration；
导入java.util.HashMap；
导入java.util.LinkedHashMap；
导入java.util.Map；
导入java.util.zip.ZipEntry；
导入java.util.zip.ZipFile；
导入java.util.zip.ZipoutStream；
导入javax.annotation.Nonnull；
导入javax.annotation.Nullable；
导入org.apache.commons.io.FilenameUtils；
导入org.apache.poi.ss.usermodel.工作簿；
导入org.apache.poi.xssf.usermodel.xssfheet；
导入org.apache.poi.xssf.usermodel.xssf工作簿；
导入com.google.common.base.Function；
导入com.google.common.collect.ImmutableList；
导入com.google.common.collect.Iterables；
公共类BigExcelWriterImpl实现BigExcelWriter{
私有静态最终字符串XML_ENCODING=“UTF-8”；
@非空
私有最终文件输出文件；
@可空
私有最终文件tempFileOutputDir；
@可空
私有文件templateFile=null；
@可空
私有XSSFWorkbook工作簿=空；
@非空
私有LinkedHashMap addedSheets=新LinkedHashMap（）；
@非空
私有映射sheetTempFiles=新HashMap（）；
BigExcelWriterImpl（@Nonnull File outputFile）{
this.outputFile=outputFile；
this.tempFileOutputDir=outputFile.getParentFile（）；
}
@凌驾
公共BigExcelWriter createWorkbook（）{
工作簿=新XSSFWorkbook（）；
归还这个；
}
@凌驾
公共BigExcelWriter添加工作表（字符串…工作表名称）{
checkState（工作簿！=null，“添加工作表之前必须创建工作簿”）；
用于（字符串sheetName:sheetNames）{
XSSFSheet sheet=workbook.createSheet（sheetName）；
添加表格。填写（表格名称、表格）；
}
归还这个；
}
@凌驾
public BigExcelWriter writeWorkbookTemplate（）引发IOException异常{
checkState（工作簿！=null，“必须在编写模板之前创建工作簿”）；
检查状态（templateFile==null，“模板文件已写入”）；
templateFile=File.createTempFile（FilenameUtils.removeExtension（outputFile.getName（））
+“-template”、“.xlsx”、tempFileOutputDir）；
System.out.println（模板文件）；
FileOutputStream os=新的FileOutputStream（templateFile）；
工作簿。编写（os）；
os.close（）；
归还这个；
}
@凌驾
公共电子表格编写器CreateSpeadsheetWriter（字符串sheetName）引发IOException{
如果（！addedSheets.containsKey（图纸名称））{
addSheets（sheetName）；
}
返回createSpreadsheetWriter（addedSheets.get（sheetName））；
}
@凌驾
公共电子表格书写器创建电子表格书写器（XSSFS
package example;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.util.CellReference;
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
public class SXSSFexample {
public static void main(String[] args) throws Throwable {
FileInputStream inputStream = new FileInputStream("mytemplate.xlsx");
XSSFWorkbook wb_template = new XSSFWorkbook(inputStream);
inputStream.close();
SXSSFWorkbook wb = new SXSSFWorkbook(wb_template);
wb.setCompressTempFiles(true);
SXSSFSheet sh = (SXSSFSheet) wb.getSheetAt(0);
sh.setRandomAccessWindowSize(100);// keep 100 rows in memory, exceeding rows will be flushed to disk
for(int rownum = 4; rownum < 100000; rownum++){
Row row = sh.createRow(rownum);
for(int cellnum = 0; cellnum < 10; cellnum++){
Cell cell = row.createCell(cellnum);
String address = new CellReference(cell).formatAsString();
cell.setCellValue(address);
}
}
FileOutputStream out = new FileOutputStream("tempsxssf.xlsx");
wb.write(out);
out.close();
}
}
List<Object[]> resultList =session.createSQLQuery("SELECT t1.employee_name, t1.employee_id ... from t_employee t1 ").list();
List<Employee> employeeList =session.createSQLQuery("SELECT t1.employee_name, t1.employee_id ... from t_employee t1 ").list();
int rowCount = 0;
int column = 0;
while(itr.hasNext()){
SXSSFRow row = xssfSheet.createRow(rowCount++);
Object[] object = (Object[]) itr.next();
//column 1
row.setCellValue(object[column++]); // write logic to create cell with required style in setCellValue method
//column 2
row.setCellValue(object[column++]);
itr.remove();
}