Merge pull request #24 from qonto/add-quarter-partitioning

Add quarter partitioning

README.md

@@ -22,16 +22,17 @@ PPM will process all referenced partitions and exit with a non-zero code if it d
 - Support of PostgreSQL 14+
 - Only supports [`RANGE` partition strategy](https://www.postgresql.org/docs/current/ddl-partitioning.html#DDL-PARTITIONING-OVERVIEW-RANGE)
 - The partition key must be a column of `date`, `timestamp`, or `uuid` type
-- Support `daily`, `weekly`, `monthly`, and `yearly` partitioning
+- Support `daily`, `weekly`, `monthly`, `quarterly`, and `yearly` partitioning
 - Dates are implemented through UTC timezone
 - Partition names are enforced and not configurable
 
-  | Partition interval | Pattern                           | Example           |
-  | ------------------ | --------------------------------- | ----------------- |
-  | daily              | `<parent_table>_<YYYY>_<DD>_<MM>` | `logs_2024_06_25` |
-  | weekly             | `<parent_table>_w<week number>`   | `logs_2024_w26`   |
-  | monthly            | `<parent_table>_<YYYY>_<MM>`      | `logs_2024_06`    |
-  | yearly             | `<parent_table>_<YYYY>`           | `logs_2024`       |
+  | Partition interval | Pattern                                   | Example           |
+  | ------------------ | ----------------------------------------- | ----------------- |
+  | daily              | `<parent_table>_<YYYY>_<DD>_<MM>`         | `logs_2024_06_25` |
+  | weekly             | `<parent_table>_w<week number>`           | `logs_2024_w26`   |
+  | quarterly          | `<parent_table>_<YYYY>_q<quarter number>` | `logs_2024_q1`   |
+  | monthly            | `<parent_table>_<YYYY>_<MM>`              | `logs_2024_06`    |
+  | yearly             | `<parent_table>_<YYYY>`                   | `logs_2024`       |
 
 ## Installation
 
@@ -241,7 +242,7 @@ Partition object:
 | Parameter      | Description                                          | Default |
 | -------------- | ---------------------------------------------------- | ------- |
 | column         | Column used for partitioning                         |         |
-| interval       | Partitioning interval (`daily`, `weekly`, `monthly` or `yearly`) |         |
+| interval       | Partitioning interval (`daily`, `weekly`, `monthly`, `quarterly` or `yearly`) |         |
 | preProvisioned | Number of partitions to create in advance            |         |
 | retention      | Number of partitions to retain                       |         |
 | schema         | PostgreSQL schema                                    |         |

internal/infra/partition/bounds.go

@@ -8,8 +8,9 @@ import (
 )
 
 const (
-	UUIDv7Version uuid.Version = 7
-	nbDaysInAWeek int          = 7
+	UUIDv7Version      uuid.Version = 7
+	nbDaysInAWeek      int          = 7
+	nbMonthsInAQuarter int          = 3
 )
 
 var (
@@ -42,6 +43,18 @@ func getMonthlyBounds(date time.Time) (lowerBound, upperBound time.Time) {
 	return
 }
 
+func getQuarterlyBounds(date time.Time) (lowerBound, upperBound time.Time) {
+	year, _, _ := date.Date()
+
+	quarter := (int(date.Month()) - 1) / nbMonthsInAQuarter
+	firstMonthOfTheQuarter := time.Month(quarter*nbMonthsInAQuarter + 1)
+
+	lowerBound = time.Date(year, firstMonthOfTheQuarter, 1, 0, 0, 0, 0, date.UTC().Location())
+	upperBound = lowerBound.AddDate(0, nbMonthsInAQuarter, 0)
+
+	return
+}
+
 func getYearlyBounds(date time.Time) (lowerBound, upperBound time.Time) {
 	lowerBound = time.Date(date.Year(), 1, 1, 0, 0, 0, 0, date.UTC().Location())
 	upperBound = lowerBound.AddDate(1, 0, 0)

internal/infra/partition/configuration.go

@@ -18,7 +18,7 @@ type Configuration struct {
 	Schema         string        `mapstructure:"schema" validate:"required"`
 	Table          string        `mapstructure:"table" validate:"required"`
 	PartitionKey   string        `mapstructure:"partitionKey" validate:"required"`
-	Interval       Interval      `mapstructure:"interval" validate:"required,oneof=daily weekly monthly yearly"`
+	Interval       Interval      `mapstructure:"interval" validate:"required,oneof=daily weekly monthly quarterly yearly"`
 	Retention      int           `mapstructure:"retention" validate:"required,gt=0"`
 	PreProvisioned int           `mapstructure:"preProvisioned" validate:"required,gt=0"`
 	CleanupPolicy  CleanupPolicy `mapstructure:"cleanupPolicy" validate:"required,oneof=drop detach"`
@@ -40,6 +40,24 @@ func (p Configuration) GeneratePartition(forDate time.Time) (Partition, error) {
 	case Monthly:
 		suffix = forDate.Format("2006_01")
 		lowerBound, upperBound = getMonthlyBounds(forDate)
+	case Quarterly:
+		year, month, _ := forDate.Date()
+
+		var quarter int
+
+		switch {
+		case month >= 1 && month <= 3:
+			quarter = 1
+		case month >= 4 && month <= 6:
+			quarter = 2
+		case month >= 7 && month <= 9:
+			quarter = 3
+		case month >= 10 && month <= 12:
+			quarter = 4
+		}
+
+		suffix = fmt.Sprintf("%d_q%d", year, quarter)
+		lowerBound, upperBound = getQuarterlyBounds(forDate)
 	case Yearly:
 		suffix = forDate.Format("2006")
 		lowerBound, upperBound = getYearlyBounds(forDate)
@@ -108,6 +126,8 @@ func (p Configuration) getPrevDate(forDate time.Time, i int) (t time.Time, err e
 		year, month, _ := forDate.Date()
 
 		t = time.Date(year, month-time.Month(i), 1, 0, 0, 0, 0, forDate.Location())
+	case Quarterly:
+		t = forDate.AddDate(0, -i*nbMonthsInAQuarter, 0)
 	case Yearly:
 		year, _, _ := forDate.Date()
 
@@ -129,6 +149,8 @@ func (p Configuration) getNextDate(forDate time.Time, i int) (t time.Time, err e
 		year, month, _ := forDate.Date()
 
 		t = time.Date(year, month+time.Month(i), 1, 0, 0, 0, 0, forDate.Location())
+	case Quarterly:
+		t = forDate.AddDate(0, i*nbMonthsInAQuarter, 0)
 	case Yearly:
 		year, _, _ := forDate.Date()
 

internal/infra/partition/interval.go

@@ -5,8 +5,9 @@ type (
 )
 
 const (
-	Daily   Interval = "daily"
-	Weekly  Interval = "weekly"
-	Monthly Interval = "monthly"
-	Yearly  Interval = "yearly"
+	Daily     Interval = "daily"
+	Weekly    Interval = "weekly"
+	Monthly   Interval = "monthly"
+	Quarterly Interval = "quarterly"
+	Yearly    Interval = "yearly"
 )

pkg/ppm/checkpartition_test.go

@@ -45,6 +45,7 @@ func TestCheckPartitions(t *testing.T) {
 	partitions["daily partition without retention"] = partition.Configuration{Schema: "public", Table: "daily_table2", PartitionKey: "created_at", Interval: partition.Daily, Retention: 0, PreProvisioned: 1}
 	partitions["daily partition without preprovisioned"] = partition.Configuration{Schema: "public", Table: "daily_table3", PartitionKey: "column", Interval: partition.Daily, Retention: 4, PreProvisioned: 0}
 	partitions["weekly partition"] = partition.Configuration{Schema: "public", Table: "weekly_table", PartitionKey: "weekly", Interval: partition.Weekly, Retention: 2, PreProvisioned: 2}
+	partitions["quarterly partition"] = partition.Configuration{Schema: "public", Table: "quarterly_table", PartitionKey: "quarterly", Interval: partition.Quarterly, Retention: 2, PreProvisioned: 2}
 	partitions["monthly partition"] = partition.Configuration{Schema: "public", Table: "monthly_table", PartitionKey: "month", Interval: partition.Monthly, Retention: 2, PreProvisioned: 2}
 	partitions["yearly partition"] = partition.Configuration{Schema: "public", Table: "yearly_table", PartitionKey: "year", Interval: partition.Yearly, Retention: 4, PreProvisioned: 4}
 
@@ -60,6 +61,8 @@ func TestCheckPartitions(t *testing.T) {
 				table, _ = p.GeneratePartition(time.Now().AddDate(0, 0, -i))
 			case partition.Weekly:
 				table, _ = p.GeneratePartition(time.Now().AddDate(0, 0, -i*7))
+			case partition.Quarterly:
+				table, _ = p.GeneratePartition(time.Now().AddDate(0, i*-3, 0))
 			case partition.Monthly:
 				table, _ = p.GeneratePartition(time.Now().AddDate(0, -i, 0))
 			case partition.Yearly:
@@ -80,6 +83,8 @@ func TestCheckPartitions(t *testing.T) {
 				table, _ = p.GeneratePartition(time.Now().AddDate(0, 0, i*7))
 			case partition.Monthly:
 				table, _ = p.GeneratePartition(time.Now().AddDate(0, i, 0))
+			case partition.Quarterly:
+				table, _ = p.GeneratePartition(time.Now().AddDate(0, i*3, 0))
 			case partition.Yearly:
 				table, _ = p.GeneratePartition(time.Now().AddDate(i, 0, 0))
 			default:

pkg/ppm/ppm_test.go

@@ -7,38 +7,33 @@ import (
 	"github.com/qonto/postgresql-partition-manager/internal/infra/postgresql"
 )
 
-func formatLowerBound(t *testing.T, p partition.Partition, config partition.Configuration) (output string) {
-	t.Helper()
-
+func getBoundFormat(config partition.Configuration) string {
 	var dateFormat string
 
 	switch config.Interval {
 	case partition.Daily, partition.Weekly:
 		dateFormat = "2006-01-02"
 	case partition.Monthly:
 		dateFormat = "2006-01"
+	case partition.Quarterly:
+		dateFormat = "2006-01"
 	case partition.Yearly:
 		dateFormat = "2006"
 	}
 
-	return p.LowerBound.Format(dateFormat)
+	return dateFormat
 }
 
-func formatUpperBound(t *testing.T, p partition.Partition, config partition.Configuration) (output string) {
+func formatLowerBound(t *testing.T, p partition.Partition, config partition.Configuration) (output string) {
 	t.Helper()
 
-	var dateFormat string
+	return p.LowerBound.Format(getBoundFormat(config))
+}
 
-	switch config.Interval {
-	case partition.Daily, partition.Weekly:
-		dateFormat = "2006-01-02"
-	case partition.Monthly:
-		dateFormat = "2006-01"
-	case partition.Yearly:
-		dateFormat = "2006"
-	}
+func formatUpperBound(t *testing.T, p partition.Partition, config partition.Configuration) (output string) {
+	t.Helper()
 
-	return p.UpperBound.Format(dateFormat)
+	return p.UpperBound.Format(getBoundFormat(config))
 }
 
 func partitionResultToPartition(t *testing.T, partitions []partition.Partition, dateFormat string) (result []postgresql.PartitionResult) {

scripts/bats/20_check.bats

@@ -25,7 +25,7 @@ setup() {
   local PREPROVISIONED=2
 
   # Create partioned table 2 retention days
-  create_partioned_table ${TABLE} ${INTERVAL} ${RETENTION} ${PREPROVISIONED}
+  create_daily_partioned_table ${TABLE} ${RETENTION} ${PREPROVISIONED}
 
   local CONFIGURATION=$(cat << EOF
 partitions:
@@ -54,7 +54,7 @@ EOF
   local NEW_RETENTION=2
   local PREPROVISIONED=1
 
-  create_partioned_table ${TABLE} ${TABLE} ${INTERVAL} ${INITIAL_RETENTION} ${PREPROVISIONED}
+  create_daily_partioned_table ${TABLE} ${TABLE} ${INITIAL_RETENTION} ${PREPROVISIONED}
 
   # Generate configuration with only 1 retention
   local CONFIGURATION=$(cat << EOF
@@ -84,7 +84,7 @@ EOF
   local INITIAL_PREPROVISIONED=2
   local NEW_PREPROVISIONED=3
 
-  create_partioned_table ${TABLE} ${INTERVAL} ${RETENTION} ${INITIAL_PREPROVISIONED}
+  create_daily_partioned_table ${TABLE} ${RETENTION} ${INITIAL_PREPROVISIONED}
 
   # Increase preProvisioned partitions
   local CONFIGURATION=$(cat << EOF

scripts/bats/30_provisioning.bats

@@ -2,6 +2,7 @@ load 'test/libs/dependencies'
 load 'test/libs/partitions'
 load 'test/libs/seeds'
 load 'test/libs/sql'
+load 'test/libs/time'
 
 setup() {
   bats_load_library bats-support
@@ -67,7 +68,6 @@ EOF
 
   run postgresql-partition-manager run provisioning -c ${CONFIGURATION_FILE}
 
-
   assert_success
   assert_output --partial "All partitions are correctly provisioned"
   assert_table_exists public $(generate_daily_partition_name ${TABLE} -1) # retention partition
@@ -87,3 +87,111 @@ EOF
   assert_table_exists public $(generate_daily_partition_name ${TABLE} -${NEW_RETENTION}) # New retention partition
   assert_table_exists public $(generate_daily_partition_name ${TABLE} ${NEW_PREPROVISIONED}) # New preProvisioned partition
 }
+
+@test "Test monthly partitions" {
+  local TABLE=$(generate_table_name)
+  local INTERVAL=monthly
+  local RETENTION=1
+  local PREPROVISIONED=1
+  local EXPECTED_LAST_TABLE="${TABLE}_$(get_current_date_adjusted_by_month -1)"
+  local EXPECTED_CURRENT_TABLE="${TABLE}_$(get_current_date_adjusted_by_month 0)"
+  local EXPECTED_NEXT_TABLE="${TABLE}_$(get_current_date_adjusted_by_month +1)"
+
+  # Create partioned table
+  create_partioned_table ${TABLE} ${INTERVAL} ${RETENTION} ${PREPROVISIONED}
+
+  local CONFIGURATION=$(cat << EOF
+partitions:
+  unittest:
+    schema: public
+    table: ${TABLE}
+    interval: ${INTERVAL}
+    partitionKey: created_at
+    cleanupPolicy: drop
+    retention: ${RETENTION}
+    preProvisioned: ${PREPROVISIONED}
+EOF
+)
+  local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}")
+
+  cat ${CONFIGURATION_FILE}
+
+  run postgresql-partition-manager run provisioning -c ${CONFIGURATION_FILE}
+
+  assert_success
+  assert_output --partial "All partitions are correctly provisioned"
+  assert_table_exists public ${EXPECTED_LAST_TABLE}
+  assert_table_exists public ${EXPECTED_CURRENT_TABLE}
+  assert_table_exists public ${EXPECTED_NEXT_TABLE}
+}
+
+@test "Test quarterly partitions" {
+  local TABLE=$(generate_table_name)
+  local INTERVAL=quarterly
+  local RETENTION=1
+  local PREPROVISIONED=1
+  local EXPECTED_LAST_TABLE="${TABLE}_$(get_current_date_adjusted_by_quarter -1)"
+  local EXPECTED_CURRENT_TABLE="${TABLE}_$(get_current_date_adjusted_by_quarter 0)"
+  local EXPECTED_NEXT_TABLE="${TABLE}_$(get_current_date_adjusted_by_quarter +1)"
+
+  # Create partioned table
+  create_partioned_table ${TABLE} ${INTERVAL} ${RETENTION} ${PREPROVISIONED}
+
+  local CONFIGURATION=$(cat << EOF
+partitions:
+  unittest:
+    schema: public
+    table: ${TABLE}
+    interval: ${INTERVAL}
+    partitionKey: created_at
+    cleanupPolicy: drop
+    retention: ${RETENTION}
+    preProvisioned: ${PREPROVISIONED}
+EOF
+)
+  local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}")
+
+  run postgresql-partition-manager run provisioning -c ${CONFIGURATION_FILE}
+
+
+  assert_success
+  assert_output --partial "All partitions are correctly provisioned"
+  assert_table_exists public ${EXPECTED_LAST_TABLE}
+  assert_table_exists public ${EXPECTED_CURRENT_TABLE}
+  assert_table_exists public ${EXPECTED_NEXT_TABLE}
+}
+
+@test "Test yearly partitions" {
+  local TABLE=$(generate_table_name)
+  local INTERVAL=yearly
+  local RETENTION=1
+  local PREPROVISIONED=1
+  local EXPECTED_LAST_TABLE="${TABLE}_$(get_current_date_adjusted_by_year -1)"
+  local EXPECTED_CURRENT_TABLE="${TABLE}_$(get_current_date_adjusted_by_year 0)"
+  local EXPECTED_NEXT_TABLE="${TABLE}_$(get_current_date_adjusted_by_year +1)"
+
+  # Create partioned table
+  create_partioned_table ${TABLE} ${INTERVAL} ${RETENTION} ${PREPROVISIONED}
+
+  local CONFIGURATION=$(cat << EOF
+partitions:
+  unittest:
+    schema: public
+    table: ${TABLE}
+    interval: ${INTERVAL}
+    partitionKey: created_at
+    cleanupPolicy: drop
+    retention: ${RETENTION}
+    preProvisioned: ${PREPROVISIONED}
+EOF
+)
+  local CONFIGURATION_FILE=$(generate_configuration_file "${CONFIGURATION}")
+
+  run postgresql-partition-manager run provisioning -c ${CONFIGURATION_FILE}
+
+  assert_success
+  assert_output --partial "All partitions are correctly provisioned"
+  assert_table_exists public ${EXPECTED_LAST_TABLE}
+  assert_table_exists public ${EXPECTED_CURRENT_TABLE}
+  assert_table_exists public ${EXPECTED_NEXT_TABLE}
+}

scripts/bats/40_cleanup.bats

@@ -17,7 +17,7 @@ setup() {
   local INITIAL_PREPROVISIONED=3
 
   # Create partioned table
-  create_partioned_table ${TABLE} ${INTERVAL} ${INITIAL_RETENTION} ${INITIAL_PREPROVISIONED}
+  create_daily_partioned_table ${TABLE} ${INITIAL_RETENTION} ${INITIAL_PREPROVISIONED}
 
   for ((i=1; i<= INITIAL_RETENTION; i++));do
     assert_table_exists public $(generate_daily_partition_name ${TABLE} -${i})

scripts/bats/test/libs/partitions.bash

@@ -5,6 +5,14 @@ create_partioned_table() {
   local PREPROVISIONED=$4
 
   create_table_from_template ${TABLE}
+}
+
+create_daily_partioned_table() {
+  local TABLE=$1
+  local RETENTION=$2
+  local PREPROVISIONED=$3
+
+  create_table_from_template ${TABLE} "daily" ${RETENTION} ${PREPROVISIONED}
   generate_daily_partitions ${TABLE} ${RETENTION} ${PREPROVISIONED}
 }