Skip to content

Commit

Permalink
This improves performance for adding and removing pivot table
Browse files Browse the repository at this point in the history
  • Loading branch information
@xuri
xuri committed Oct 4, 2023
1 parent ecb4f62 commit df032fc
Show file tree
Hide file tree
Showing 4 changed files with 84 additions and 103 deletions.
139 changes: 67 additions & 72 deletions pivotTable.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ import (
type PivotTableOptions struct {
pivotTableXML string
pivotCacheXML string
pivotTableSheetName string
pivotSheetName string
pivotDataRange string
namedDataRange bool
DataRange string
PivotTableRange string
Name string
Expand Down Expand Up @@ -155,20 +157,20 @@ func (f *File) AddPivotTable(opts *PivotTableOptions) error {
pivotCacheID := f.countPivotCache() + 1

sheetRelationshipsPivotTableXML := "../pivotTables/pivotTable" + strconv.Itoa(pivotTableID) + ".xml"
pivotTableXML := strings.ReplaceAll(sheetRelationshipsPivotTableXML, "..", "xl")
pivotCacheXML := "xl/pivotCache/pivotCacheDefinition" + strconv.Itoa(pivotCacheID) + ".xml"
if err = f.addPivotCache(pivotCacheXML, opts); err != nil {
opts.pivotTableXML = strings.ReplaceAll(sheetRelationshipsPivotTableXML, "..", "xl")
opts.pivotCacheXML = "xl/pivotCache/pivotCacheDefinition" + strconv.Itoa(pivotCacheID) + ".xml"
if err = f.addPivotCache(opts); err != nil {
return err
}

// workbook pivot cache
workBookPivotCacheRID := f.addRels(f.getWorkbookRelsPath(), SourceRelationshipPivotCache, strings.TrimPrefix(pivotCacheXML, "xl/"), "")
workBookPivotCacheRID := f.addRels(f.getWorkbookRelsPath(), SourceRelationshipPivotCache, strings.TrimPrefix(opts.pivotCacheXML, "xl/"), "")
cacheID := f.addWorkbookPivotCache(workBookPivotCacheRID)

pivotCacheRels := "xl/pivotTables/_rels/pivotTable" + strconv.Itoa(pivotTableID) + ".xml.rels"
// rId not used
_ = f.addRels(pivotCacheRels, SourceRelationshipPivotCache, fmt.Sprintf("../pivotCache/pivotCacheDefinition%d.xml", pivotCacheID), "")
if err = f.addPivotTable(cacheID, pivotTableID, pivotTableXML, opts); err != nil {
if err = f.addPivotTable(cacheID, pivotTableID, opts); err != nil {
return err
}
pivotTableSheetRels := "xl/worksheets/_rels/" + strings.TrimPrefix(pivotTableSheetPath, "xl/worksheets/") + ".rels"
Expand All @@ -192,15 +194,11 @@ func (f *File) parseFormatPivotTableSet(opts *PivotTableOptions) (*xlsxWorksheet
if len(opts.Name) > MaxFieldLength {
return nil, "", ErrNameLength
}
opts.pivotTableSheetName = pivotTableSheetName
_, dataRangeRef, err := f.getPivotTableDataRange(pivotTableSheetName, opts.DataRange, opts.DataRange)
if err != nil {
opts.pivotSheetName = pivotTableSheetName
if err = f.getPivotTableDataRange(opts); err != nil {
return nil, "", err
}
if dataRangeRef == "" {
dataRangeRef = opts.DataRange
}
dataSheetName, _, err := f.adjustRange(dataRangeRef)
dataSheetName, _, err := f.adjustRange(opts.pivotDataRange)
if err != nil {
return nil, "", newPivotTableDataRangeError(err.Error())
}
Expand Down Expand Up @@ -247,19 +245,12 @@ func (f *File) adjustRange(rangeStr string) (string, []int, error) {

// getTableFieldsOrder provides a function to get order list of pivot table
// fields.
func (f *File) getTableFieldsOrder(sheetName, dataRange string) ([]string, error) {
func (f *File) getTableFieldsOrder(opts *PivotTableOptions) ([]string, error) {
var order []string
if dataRange == "" {
return order, newPivotTableDataRangeError(ErrParameterRequired.Error())
}
_, dataRangeRef, err := f.getPivotTableDataRange(sheetName, dataRange, dataRange)
if err != nil {
if err := f.getPivotTableDataRange(opts); err != nil {
return order, err
}
if dataRangeRef == "" {
dataRangeRef = dataRange
}
dataSheet, coordinates, err := f.adjustRange(dataRangeRef)
dataSheet, coordinates, err := f.adjustRange(opts.pivotDataRange)
if err != nil {
return order, newPivotTableDataRangeError(err.Error())
}
Expand All @@ -275,23 +266,14 @@ func (f *File) getTableFieldsOrder(sheetName, dataRange string) ([]string, error
}

// addPivotCache provides a function to create a pivot cache by given properties.
func (f *File) addPivotCache(pivotCacheXML string, opts *PivotTableOptions) error {
func (f *File) addPivotCache(opts *PivotTableOptions) error {
// validate data range
definedNameRef := true
_, dataRangeRef, err := f.getPivotTableDataRange(opts.pivotTableSheetName, opts.DataRange, opts.DataRange)
if err != nil {
return err
}
if dataRangeRef == "" {
definedNameRef = false
dataRangeRef = opts.DataRange
}
dataSheet, coordinates, err := f.adjustRange(dataRangeRef)
dataSheet, coordinates, err := f.adjustRange(opts.pivotDataRange)
if err != nil {
return newPivotTableDataRangeError(err.Error())
}
// data range has been checked
order, _ := f.getTableFieldsOrder(opts.pivotTableSheetName, dataRangeRef)
order, _ := f.getTableFieldsOrder(opts)
hCell, _ := CoordinatesToCellName(coordinates[0], coordinates[1])
vCell, _ := CoordinatesToCellName(coordinates[2], coordinates[3])
pc := xlsxPivotCacheDefinition{
Expand All @@ -309,7 +291,7 @@ func (f *File) addPivotCache(pivotCacheXML string, opts *PivotTableOptions) erro
},
CacheFields: &xlsxCacheFields{},
}
if definedNameRef {
if opts.namedDataRange {
pc.CacheSource.WorksheetSource = &xlsxWorksheetSource{Name: opts.DataRange}
}
for _, name := range order {
Expand All @@ -320,13 +302,13 @@ func (f *File) addPivotCache(pivotCacheXML string, opts *PivotTableOptions) erro
}
pc.CacheFields.Count = len(pc.CacheFields.CacheField)
pivotCache, err := xml.Marshal(pc)
f.saveFileList(pivotCacheXML, pivotCache)
f.saveFileList(opts.pivotCacheXML, pivotCache)
return err
}

// addPivotTable provides a function to create a pivot table by given pivot
// table ID and properties.
func (f *File) addPivotTable(cacheID, pivotTableID int, pivotTableXML string, opts *PivotTableOptions) error {
func (f *File) addPivotTable(cacheID, pivotTableID int, opts *PivotTableOptions) error {
// validate pivot table range
_, coordinates, err := f.adjustRange(opts.PivotTableRange)
if err != nil {
Expand Down Expand Up @@ -401,7 +383,7 @@ func (f *File) addPivotTable(cacheID, pivotTableID int, pivotTableXML string, op
_ = f.addPivotDataFields(&pt, opts)

pivotTable, err := xml.Marshal(pt)
f.saveFileList(pivotTableXML, pivotTable)
f.saveFileList(opts.pivotTableXML, pivotTable)
return err
}

Expand Down Expand Up @@ -539,7 +521,7 @@ func (f *File) addPivotColFields(pt *xlsxPivotTableDefinition, opts *PivotTableO
// addPivotFields create pivot fields based on the column order of the first
// row in the data region by given pivot table definition and option.
func (f *File) addPivotFields(pt *xlsxPivotTableDefinition, opts *PivotTableOptions) error {
order, err := f.getTableFieldsOrder(opts.pivotTableSheetName, opts.DataRange)
order, err := f.getTableFieldsOrder(opts)
if err != nil {
return err
}
Expand Down Expand Up @@ -645,7 +627,7 @@ func (f *File) countPivotCache() int {
// to a sequential index by given fields and pivot option.
func (f *File) getPivotFieldsIndex(fields []PivotTableField, opts *PivotTableOptions) ([]int, error) {
var pivotFieldsIndex []int
orders, err := f.getTableFieldsOrder(opts.pivotTableSheetName, opts.DataRange)
orders, err := f.getTableFieldsOrder(opts)
if err != nil {
return pivotFieldsIndex, err
}
Expand Down Expand Up @@ -761,30 +743,40 @@ func (f *File) GetPivotTables(sheet string) ([]PivotTableOptions, error) {
return pivotTables, nil
}

// getPivotTableDataRange returns pivot table data range name and reference from
// cell reference, table name or defined name.
func (f *File) getPivotTableDataRange(sheet, ref, name string) (string, string, error) {
dataRange := fmt.Sprintf("%s!%s", sheet, ref)
dataRangeRef, isTable := dataRange, false
if name != "" {
dataRange = name
for _, sheetName := range f.GetSheetList() {
tables, err := f.GetTables(sheetName)
e := ErrSheetNotExist{sheetName}
if err != nil && err.Error() != newNotWorksheetError(sheetName).Error() && err.Error() != e.Error() {
return dataRange, dataRangeRef, err
}
for _, table := range tables {
if table.Name == name {
dataRangeRef, isTable = fmt.Sprintf("%s!%s", sheetName, table.Range), true
}
// getPivotTableDataRange checking given if data range is a cell reference or
// named reference (defined name or table name), and set pivot table data range.
func (f *File) getPivotTableDataRange(opts *PivotTableOptions) error {
if opts.DataRange == "" {
return newPivotTableDataRangeError(ErrParameterRequired.Error())
}
if opts.pivotDataRange != "" {
return nil
}
if strings.Contains(opts.DataRange, "!") {
opts.pivotDataRange = opts.DataRange
return nil
}
for _, sheetName := range f.GetSheetList() {
tables, err := f.GetTables(sheetName)
e := ErrSheetNotExist{sheetName}
if err != nil && err.Error() != newNotWorksheetError(sheetName).Error() && err.Error() != e.Error() {
return err
}
for _, table := range tables {
if table.Name == opts.DataRange {
opts.pivotDataRange, opts.namedDataRange = fmt.Sprintf("%s!%s", sheetName, table.Range), true
return err
}
}
if !isTable {
dataRangeRef = f.getDefinedNameRefTo(name, sheet)
}
if !opts.namedDataRange {
opts.pivotDataRange = f.getDefinedNameRefTo(opts.DataRange, opts.pivotSheetName)
if opts.pivotDataRange != "" {
opts.namedDataRange = true
return nil
}
}
return dataRange, dataRangeRef, nil
return newPivotTableDataRangeError(ErrParameterInvalid.Error())
}

// getPivotTable provides a function to get a pivot table definition by given
Expand All @@ -810,17 +802,17 @@ func (f *File) getPivotTable(sheet, pivotTableXML, pivotCacheRels string) (Pivot
if err != nil {
return opts, err
}
dataRange, dataRangeRef, err := f.getPivotTableDataRange(sheet, pc.CacheSource.WorksheetSource.Ref, pc.CacheSource.WorksheetSource.Name)
if err != nil {
return opts, err
}
opts = PivotTableOptions{
pivotTableXML: pivotTableXML,
pivotCacheXML: pivotCacheXML,
pivotTableSheetName: sheet,
DataRange: dataRange,
PivotTableRange: fmt.Sprintf("%s!%s", sheet, pt.Location.Ref),
Name: pt.Name,
pivotTableXML: pivotTableXML,
pivotCacheXML: pivotCacheXML,
pivotSheetName: sheet,
DataRange: fmt.Sprintf("%s!%s", sheet, pc.CacheSource.WorksheetSource.Ref),
PivotTableRange: fmt.Sprintf("%s!%s", sheet, pt.Location.Ref),
Name: pt.Name,
}
if pc.CacheSource.WorksheetSource.Name != "" {
opts.DataRange = pc.CacheSource.WorksheetSource.Name
_ = f.getPivotTableDataRange(&opts)
}
fields := []string{"RowGrandTotals", "ColGrandTotals", "ShowDrill", "UseAutoFormatting", "PageOverThenDown", "MergeItem", "CompactData", "ShowError"}
immutable, mutable := reflect.ValueOf(*pt), reflect.ValueOf(&opts).Elem()
Expand All @@ -838,7 +830,10 @@ func (f *File) getPivotTable(sheet, pivotTableXML, pivotCacheRels string) (Pivot
opts.ShowLastColumn = si.ShowLastColumn
opts.PivotTableStyleName = si.Name
}
order, err := f.getTableFieldsOrder(pt.Name, dataRangeRef)
order, err := f.getTableFieldsOrder(&opts)
if err != nil {
return opts, err
}
f.extractPivotTableFields(order, pt, &opts)
return opts, err
}
Expand Down
36 changes: 6 additions & 30 deletions pivotTable_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,22 +272,14 @@ func TestPivotTable(t *testing.T) {
_, _, err = f.adjustRange("sheet1!")
assert.EqualError(t, err, "parameter is invalid")
// Test get table fields order with empty data range
_, err = f.getTableFieldsOrder("", "")
_, err = f.getTableFieldsOrder(&PivotTableOptions{})
assert.EqualError(t, err, `parameter 'DataRange' parsing error: parameter is required`)
// Test add pivot cache with empty data range
assert.EqualError(t, f.addPivotCache("", &PivotTableOptions{}), "parameter 'DataRange' parsing error: parameter is invalid")
// Test add pivot cache with invalid data range
assert.EqualError(t, f.addPivotCache("", &PivotTableOptions{
DataRange: "A1:E31",
PivotTableRange: "Sheet1!U34:O2",
Rows: []PivotTableField{{Data: "Month", DefaultSubtotal: true}, {Data: "Year"}},
Columns: []PivotTableField{{Data: "Type", DefaultSubtotal: true}},
Data: []PivotTableField{{Data: "Sales"}},
}), "parameter 'DataRange' parsing error: parameter is invalid")
assert.EqualError(t, f.addPivotCache(&PivotTableOptions{}), "parameter 'DataRange' parsing error: parameter is required")
// Test add pivot table with empty options
assert.EqualError(t, f.addPivotTable(0, 0, "", &PivotTableOptions{}), "parameter 'PivotTableRange' parsing error: parameter is required")
assert.EqualError(t, f.addPivotTable(0, 0, &PivotTableOptions{}), "parameter 'PivotTableRange' parsing error: parameter is required")
// Test add pivot table with invalid data range
assert.EqualError(t, f.addPivotTable(0, 0, "", &PivotTableOptions{}), "parameter 'PivotTableRange' parsing error: parameter is required")
assert.EqualError(t, f.addPivotTable(0, 0, &PivotTableOptions{}), "parameter 'PivotTableRange' parsing error: parameter is required")
// Test add pivot fields with empty data range
assert.EqualError(t, f.addPivotFields(nil, &PivotTableOptions{
DataRange: "A1:E31",
Expand Down Expand Up @@ -413,22 +405,6 @@ func TestParseFormatPivotTableSet(t *testing.T) {
assert.EqualError(t, err, "XML syntax error on line 1: invalid UTF-8")
}

func TestAddPivotCache(t *testing.T) {
f := NewFile()
// Create table in a worksheet
assert.NoError(t, f.AddTable("Sheet1", &Table{
Name: "Table1",
Range: "A1:D5",
}))
// Test add pivot table cache with unsupported table relationships charset
f.Pkg.Store("xl/tables/table1.xml", MacintoshCyrillicCharset)
assert.EqualError(t, f.addPivotCache("xl/pivotCache/pivotCacheDefinition1.xml", &PivotTableOptions{
DataRange: "Table1",
PivotTableRange: "Sheet1!G2:K7",
Rows: []PivotTableField{{Data: "Column1"}},
}), "XML syntax error on line 1: invalid UTF-8")
}

func TestAddPivotRowFields(t *testing.T) {
f := NewFile()
// Test invalid data range
Expand Down Expand Up @@ -465,7 +441,7 @@ func TestAddPivotColFields(t *testing.T) {
func TestGetPivotFieldsOrder(t *testing.T) {
f := NewFile()
// Test get table fields order with not exist worksheet
_, err := f.getTableFieldsOrder("", "SheetN!A1:E31")
_, err := f.getTableFieldsOrder(&PivotTableOptions{DataRange: "SheetN!A1:E31"})
assert.EqualError(t, err, "sheet SheetN does not exist")
// Create table in a worksheet
assert.NoError(t, f.AddTable("Sheet1", &Table{
Expand All @@ -474,7 +450,7 @@ func TestGetPivotFieldsOrder(t *testing.T) {
}))
// Test get table fields order with unsupported table relationships charset
f.Pkg.Store("xl/tables/table1.xml", MacintoshCyrillicCharset)
_, err = f.getTableFieldsOrder("Sheet1", "Table")
_, err = f.getTableFieldsOrder(&PivotTableOptions{DataRange: "Table"})
assert.EqualError(t, err, "XML syntax error on line 1: invalid UTF-8")
}

Expand Down
2 changes: 1 addition & 1 deletion slicer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ func (f *File) getSlicerSource(opts *SlicerOptions) (*Table, *PivotTableOptions,
return table, pivotTable, colIdx, newNoExistTableError(opts.TableName)
}
}
order, _ := f.getTableFieldsOrder(opts.TableSheet, dataRange)
order, _ := f.getTableFieldsOrder(&PivotTableOptions{DataRange: dataRange})
if colIdx = inStrSlice(order, opts.Name, true); colIdx == -1 {
return table, pivotTable, colIdx, newInvalidSlicerNameError(opts.Name)
}
Expand Down
10 changes: 10 additions & 0 deletions workbook_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,13 @@ func TestWorkbookProps(t *testing.T) {
_, err = f.GetWorkbookProps()
assert.EqualError(t, err, "XML syntax error on line 1: invalid UTF-8")
}

func TestDeleteWorkbookRels(t *testing.T) {
f := NewFile()
// Test delete pivot table without worksheet relationships
f.Relationships.Delete("xl/_rels/workbook.xml.rels")
f.Pkg.Delete("xl/_rels/workbook.xml.rels")
rID, err := f.deleteWorkbookRels("", "")
assert.Empty(t, rID)
assert.NoError(t, err)
}

0 comments on commit df032fc

Please sign in to comment.