Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PRD-6159] - upgrading the poi, commons-io, commons-compiler jars and… #9442

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[PRD-6159] - upgrading the poi, commons-io, commons-compiler jars and… #9442

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion assemblies/client/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
<version>${commons-io.version}</version>
</dependency>
</dependencies>
</plugin>
Expand Down
2 changes: 1 addition & 1 deletion engine/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
<odfdom-java.version>0.8.6</odfdom-java.version>
<commons-collections4.version>4.4</commons-collections4.version>
<commons-math3.version>3.6.1</commons-math3.version>
<xmlbeans.version>3.1.0</xmlbeans.version>
<xmlbeans.version>5.2.0</xmlbeans.version>
<curvesapi.version>1.06</curvesapi.version>
<SassyReader.version>0.5</SassyReader.version>
<saxon.version>9.1.0.8</saxon.version>
Expand Down
22 changes: 20 additions & 2 deletions plugins/excel/core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
<name>PDI Excel Plugins Core</name>

<properties>
<poi.version>4.1.1</poi.version>
<poi.version>5.2.5</poi.version>
<xmlbeans.version>5.2.0</xmlbeans.version>
<platform.version>10.2.0.0-SNAPSHOT</platform.version>
</properties>

Expand Down Expand Up @@ -75,7 +76,7 @@
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml-schemas</artifactId>
<artifactId>poi-ooxml-full</artifactId>
<version>${poi.version}</version>
<exclusions>
<exclusion>
Expand All @@ -84,6 +85,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.xmlbeans</groupId>
<artifactId>xmlbeans</artifactId>
<version>${xmlbeans.version}</version>
</dependency>
<dependency>
<groupId>pentaho</groupId>
<artifactId>pentaho-platform-core</artifactId>
Expand Down Expand Up @@ -127,5 +133,17 @@
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>5.2.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>5.2.5</version>
<scope>compile</scope>
</dependency>
</dependencies>
</project>
6 changes: 3 additions & 3 deletions plugins/excel/core/src/main/java/org/pentaho/di/trans/steps/excelinput/poi/PoiCell.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import java.sql.Date;
import java.util.TimeZone;

import org.apache.poi.hssf.usermodel.HSSFDateUtil;
import org.apache.poi.ss.usermodel.DateUtil;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellType;
import org.apache.poi.ss.usermodel.Row;
Expand All @@ -45,7 +45,7 @@ public KCellType getType() {
if ( type == CellType.BOOLEAN ) {
return KCellType.BOOLEAN;
} else if ( type == CellType.NUMERIC ) {
if ( HSSFDateUtil.isCellDateFormatted( cell ) ) {
if ( DateUtil.isCellDateFormatted( cell ) ) {
return KCellType.DATE;
} else {
return KCellType.NUMBER;
Expand All @@ -63,7 +63,7 @@ public KCellType getType() {
} else if ( CellType.STRING.equals( cachedFormulaResultType ) ) {
return KCellType.STRING_FORMULA;
} else if ( CellType.NUMERIC.equals( cachedFormulaResultType ) ) {
if ( HSSFDateUtil.isCellDateFormatted( cell ) ) {
if ( DateUtil.isCellDateFormatted( cell ) ) {
return KCellType.DATE_FORMULA;
} else {
return KCellType.NUMBER_FORMULA;
Expand Down
3 changes: 2 additions & 1 deletion plugins/excel/core/src/main/java/org/pentaho/di/trans/steps/excelinput/poi/PoiWorkbook.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.pentaho.di.core.spreadsheet.KSheet;
import org.pentaho.di.core.spreadsheet.KWorkbook;
import org.pentaho.di.core.vfs.KettleVFS;
import org.apache.poi.xssf.usermodel.XSSFWorkbookFactory;

public class PoiWorkbook implements KWorkbook {

Expand Down Expand Up @@ -73,7 +74,7 @@ public PoiWorkbook( String filename, String encoding, String password ) throws K
} catch ( Exception ofe ) {
try {
opcpkg = OPCPackage.open( excelFile );
workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create( opcpkg );
workbook = XSSFWorkbookFactory.createWorkbook( opcpkg );
} catch ( Exception ex ) {
workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create( excelFile, password );
}
Expand Down
14 changes: 9 additions & 5 deletions .../excel/core/src/main/java/org/pentaho/di/trans/steps/excelinput/staxpoi/StaxPoiSheet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.ss.usermodel.DateUtil;
import org.apache.poi.xssf.eventusermodel.XSSFReader;
import org.apache.poi.xssf.model.SharedStringsTable;
import org.apache.poi.xssf.model.SharedStrings;
import org.apache.poi.xssf.model.StylesTable;
import org.apache.poi.xssf.usermodel.XSSFRichTextString;
import org.openxmlformats.schemas.spreadsheetml.x2006.main.CTXf;
Expand Down Expand Up @@ -87,7 +87,7 @@ public class StaxPoiSheet implements KSheet {
private KCell[] currentRowCells;

// full shared strings table
private SharedStringsTable sst;
private SharedStrings sst;
// custom styles
private StylesTable styles;

Expand Down Expand Up @@ -137,8 +137,10 @@ public StaxPoiSheet( XSSFReader reader, String sheetName, String sheetID )
event = sheetReader.next();
if ( event == XMLStreamConstants.START_ELEMENT && sheetReader.getLocalName().equals( TAG_V ) ) {
int idx = Integer.parseInt( sheetReader.getElementText() );
String content = new XSSFRichTextString( sst.getEntryAt( idx ) ).toString();
headerRow.add( content );
if (sst != null && sst.getCount() > 0) {
String content = new XSSFRichTextString(sst.getItemAt(idx).getString()).toString();
headerRow.add( content );
}
break;
}
}
Expand Down Expand Up @@ -287,7 +289,9 @@ private KCell[] parseRow() throws XMLStreamException {
// read content as string
if ( cellType != null && cellType.equals( "s" ) ) {
int idx = Integer.parseInt( sheetReader.getElementText() );
content = new XSSFRichTextString( sst.getEntryAt( idx ) ).toString();
if (sst != null && sst.getCount() > 0) {
content = new XSSFRichTextString(sst.getItemAt(idx).getString()).toString();
}
} else {
content = sheetReader.getElementText();
}
Expand Down
18 changes: 13 additions & 5 deletions plugins/excel/core/src/main/java/org/pentaho/di/trans/steps/excelwriter/ExcelWriterStep.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws

meta = (ExcelWriterStepMeta) smi;
data = (ExcelWriterStepData) sdi;
boolean createSXSSFWorkbook = true;

// get next row
Object[] r = getRow();
Expand All @@ -119,7 +120,9 @@ public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws
data.firstFileOpened = true;

try {
prepareNextOutputFile();
// As the file is being created and the rows are being initialized,
// so there will be no read and re-write operation for the first time, creating SXSSFWorkbook
prepareNextOutputFile(createSXSSFWorkbook);
} catch ( KettleException e ) {
logError( BaseMessages.getString( PKG, "ExcelWriterStep.Exception.CouldNotPrepareFile",
environmentSubstitute( meta.getFileName() ) ) );
Expand All @@ -133,7 +136,7 @@ public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws
// if we are supposed to init the file delayed, here we go
if ( meta.isDoNotOpenNewFileInit() ) {
data.firstFileOpened = true;
prepareNextOutputFile();
prepareNextOutputFile(false);
}

// Let's remember where the fields are in the input row
Expand Down Expand Up @@ -196,7 +199,7 @@ public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws
if ( !meta.isAppendLines() && meta.getSplitEvery() > 0 && data.datalines > 0
&& data.datalines % meta.getSplitEvery() == 0 ) {
closeOutputFile();
prepareNextOutputFile();
prepareNextOutputFile(false);
}

writeNextLine( r );
Expand Down Expand Up @@ -632,7 +635,7 @@ public static void copyFile( FileObject in, FileObject out ) throws KettleExcept
}
}

public void prepareNextOutputFile() throws KettleException {
public void prepareNextOutputFile(boolean createSXSSFWorkbook) throws KettleException {
try {
// sheet name shouldn't exceed 31 character
if ( data.realSheetname != null && data.realSheetname.length() > 31 ) {
Expand Down Expand Up @@ -691,6 +694,11 @@ public void prepareNextOutputFile() throws KettleException {
// In that case, one needs to initialize it later, after writing header/template, because
// SXSSFWorkbook can't read/rewrite existing data, only append.
data.wb = new XSSFWorkbook( inputStream );
// As the file is being created and the rows are being initialized,
// so there will be no read and re-write operation for the first time, creating SXSSFWorkbook
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There will be if you are writing over a template, hence the test failures.

if(createSXSSFWorkbook) {
data.wb = new SXSSFWorkbook( (XSSFWorkbook) data.wb, STREAMING_WINDOW_SIZE );
}
} else {
data.wb = new HSSFWorkbook( inputStream );
}
Expand Down Expand Up @@ -860,7 +868,7 @@ private void createFile() throws KettleException, IOException {
}
} else {
// handle fresh file case, just create a fresh workbook
try ( Workbook wb = XLSX.equalsIgnoreCase( meta.getExtension() ) ? new XSSFWorkbook() : new HSSFWorkbook();
try ( Workbook wb = XLSX.equalsIgnoreCase( meta.getExtension() ) ? new SXSSFWorkbook() : new HSSFWorkbook();
BufferedOutputStreamWithCloseDetection out =
new BufferedOutputStreamWithCloseDetection( KettleVFS.getOutputStream( data.file, false ) ) ) {
wb.createSheet( data.realSheetname );
Expand Down
11 changes: 6 additions & 5 deletions ...el/core/src/test/java/org/pentaho/di/trans/steps/excelinput/staxpoi/StaxPoiSheetTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@
import org.mockito.stubbing.Answer;
import org.openxmlformats.schemas.spreadsheetml.x2006.main.CTRst;
import org.openxmlformats.schemas.spreadsheetml.x2006.main.CTXf;
import org.apache.poi.xssf.model.SharedStrings;
import org.apache.poi.xssf.usermodel.XSSFRichTextString;
import org.apache.xmlbeans.XmlObject;
import org.pentaho.di.core.spreadsheet.KCell;
import org.pentaho.di.core.spreadsheet.KCellType;
import org.pentaho.di.core.spreadsheet.KSheet;
Expand Down Expand Up @@ -291,7 +294,7 @@ private StaxPoiSheet getSampleSheet() throws Exception {
}

private XSSFReader mockXSSFReader( final String sheetId,
final String sheetContent, final SharedStringsTable sst, final StylesTable styles )
final String sheetContent, final SharedStrings sst, final StylesTable styles )
throws Exception {
XSSFReader reader = mock( XSSFReader.class );
when( reader.getSharedStringsTable() ).thenReturn( sst );
Expand Down Expand Up @@ -330,10 +333,8 @@ public String answer( InvocationOnMock invocation ) throws Throwable {

private SharedStringsTable mockSharedStringsTable( String... strings ) {
SharedStringsTable sst = new SharedStringsTable();
for ( String str : strings ) {
CTRst st = CTRst.Factory.newInstance();
st.setT( str );
sst.addEntry( st );
for (String str : strings) {
sst.addSharedStringItem(new XSSFRichTextString(str));
}
return sst;
}
Expand Down
9 changes: 5 additions & 4 deletions .../excel/core/src/test/java/org/pentaho/di/trans/steps/excelwriter/ExcelWriterStepTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ public static URL getTemplateWithFormattingXlsx() {

private ExcelWriterStepMeta metaMock;
private ExcelWriterStepData dataMock;
boolean createSXSSFWorkbook = false;

@Before
public void setUp() throws Exception {
Expand Down Expand Up @@ -125,7 +126,7 @@ public void testMaxSheetNameLength() {
step.init( mockHelper.initStepMetaInterface, mockHelper.initStepDataInterface );

try {
step.prepareNextOutputFile();
step.prepareNextOutputFile(createSXSSFWorkbook);
// An exception should have been thrown!
fail();
} catch ( KettleException e ) {
Expand All @@ -149,7 +150,7 @@ public void testPrepareNextOutputFile() throws Exception {
dataMock.createNewFile = true;
dataMock.realTemplateFileName = getTemplateTestXlsx().getFile();
dataMock.realSheetname = SHEET_NAME;
step.prepareNextOutputFile();
step.prepareNextOutputFile(createSXSSFWorkbook);
}

@Test
Expand Down Expand Up @@ -181,7 +182,7 @@ public void testWriteUsingTemplateWithFormatting() throws Exception {
doReturn( 10 ).when( dataMock.inputRowMeta ).size();
doReturn( vmi ).when( dataMock.inputRowMeta ).getValueMeta( anyInt() );

step.prepareNextOutputFile();
step.prepareNextOutputFile(createSXSSFWorkbook);

assertTrue( "must use streaming", dataMock.sheet instanceof SXSSFSheet );
dataMock.posY = 1;
Expand Down Expand Up @@ -439,7 +440,7 @@ private void testBase( ValueMetaInterface vmi, Object vObj, String extension, St
doReturn( 1 ).when( dataMock.inputRowMeta ).size();
doReturn( vmi ).when( dataMock.inputRowMeta ).getValueMeta( anyInt() );

step.prepareNextOutputFile();
step.prepareNextOutputFile(createSXSSFWorkbook);

assertNull( dataMock.sheet.getRow( 1 ) );

Expand Down
3 changes: 2 additions & 1 deletion ...java/org/pentaho/di/trans/steps/excelwriter/ExcelWriterStep_FormulaRecalculationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ public void setUp() throws Exception {
mockHelper.stepMeta, mockHelper.stepDataInterface, 0, mockHelper.transMeta, mockHelper.trans );
step = spy( step );
// ignoring to avoid useless errors in log
doNothing().when( step ).prepareNextOutputFile();
boolean createSXSSFWorkbook = false;
doNothing().when( step ).prepareNextOutputFile(createSXSSFWorkbook);

data = new ExcelWriterStepData();

Expand Down
2 changes: 1 addition & 1 deletion plugins/pentaho-reporting/assemblies/plugin/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@
libxml,
poi,
poi-ooxml,
poi-ooxml-schemas,
poi-ooxml-full,
reactive-streams,
rsyntaxtextarea,
sac,
Expand Down
2 changes: 1 addition & 1 deletion plugins/xml/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
<pdi.version>10.2.0.0-SNAPSHOT</pdi.version>
<mockito.version>5.10.0</mockito.version>
<saxon.version>8.7</saxon.version>
<poi.version>4.1.1</poi.version>
<poi.version>5.2.5</poi.version>
<platform.version>10.2.0.0-SNAPSHOT</platform.version>
</properties>

Expand Down