Update the create_hypertable interface (#79)

* Updated CI matrix to run latest timescaledb version with PG 15,16,17

A few fixes was made to try to avoid flaky specs and deadlocks.

.github/workflows/ci.yml

@@ -1,7 +1,6 @@
 name: CI
 
 on:
-  push:
   pull_request:
   workflow_dispatch:
 #  schedule:
@@ -14,9 +13,9 @@ jobs:
       matrix:
         ruby: [ '3.1.2' ]
         database:
-          - 'pg16.2-ts2.14.2-all'
-          - 'pg15.6-ts2.14.2-all'
-          - 'pg14.11-ts2.14.2-all'
+          - 'pg17-ts2.17-all'
+          - 'pg16-ts2.17-all'
+          - 'pg15-ts2.17-all'
 
     services:
       database:

.ruby-version

@@ -1 +1 @@
-3.1.2
+3.3.3

README.md

@@ -245,12 +245,16 @@ to the function call while also using `create_table` method:
 
 #### create_table with `:hypertable`
 
+You can just pass the options to the `hypertable` keyword:
+
 ```ruby
 hypertable_options = {
   time_column: 'created_at',
   chunk_time_interval: '1 min',
   compress_segmentby: 'identifier',
-  compression_interval: '7 days'
+  compress_after: '7 days',
+  compress_orderby: 'created_at DESC NULLS LAST',
+  drop_after: '6 months'
 }
 
 create_table(:events, id: false, hypertable: hypertable_options) do |t|
@@ -260,6 +264,27 @@ create_table(:events, id: false, hypertable: hypertable_options) do |t|
 end
 ```
 
+And the code above will create a hypertable with the following options:
+
+```sql
+CREATE TABLE events (
+  identifier text NOT NULL,
+  payload jsonb,
+  created_at timestamp with time zone NOT NULL,
+  updated_at timestamp with time zone NOT NULL
+)
+SELECT create_hypertable('events', by_range('created_at', INTERVAL '1 min');
+ALTER TABLE events SET (
+  timescaledb.compress,
+  timescaledb.compress_segmentby = 'identifier',
+  timescaledb.compress_orderby = 'created_at DESC NULLS LAST',
+  timescaledb.drop_after = INTERVAL '6 months'
+);
+
+SELECT add_compression_policy('events', INTERVAL '7 days');
+SELECT add_retention_policy('events', INTERVAL '6 months');
+```
+
 #### create_continuous_aggregate
 
 This example shows a ticks table grouping ticks as OHLCV histograms for every
@@ -271,7 +296,7 @@ hypertable_options = {
   chunk_time_interval: '1 min',
   compress_segmentby: 'symbol',
   compress_orderby: 'created_at',
-  compression_interval: '7 days'
+  compress_after: '7 days'
 }
 create_table :ticks, hypertable: hypertable_options, id: false do |t|
   t.string :symbol

docs/migrations.md

@@ -10,7 +10,7 @@ hypertable_options = {
   time_column: 'created_at',
   chunk_time_interval: '1 min',
   compress_segmentby: 'identifier',
-  compression_interval: '7 days'
+  compress_after: '7 days'
 }
 
 create_table(:events, id: false, hypertable: hypertable_options) do |t|
@@ -43,7 +43,7 @@ hypertable_options = {
   chunk_time_interval: '1 min',
   compress_segmentby: 'symbol',
   compress_orderby: 'created_at',
-  compression_interval: '7 days'
+  compress_after: '7 days'
 }
 create_table :ticks, hypertable: hypertable_options, id: false do |t|
   t.string :symbol

docs/toolkit_candlestick.md

@@ -42,7 +42,7 @@ db do
     chunk_time_interval: "1 day",
     compress_segmentby: "symbol",
     compress_orderby: "time",
-    compression_interval: "1 month"
+    compress_after: "1 month"
   }
   create_table :ticks, hypertable: hypertable_options, id: false do |t|
     t.timestamp :time

docs/toolkit_ohlc.md

@@ -22,7 +22,7 @@ hypertable_options = {
   chunk_time_interval: '1 week',
   compress_segmentby: 'symbol',
   compress_orderby: 'time',
-  compression_interval: '1 month'
+  compress_after: '1 month'
 }
 create_table :ticks, hypertable: hypertable_options, id: false do |t|
   t.timestampt :time

examples/all_in_one/all_in_one.rb

@@ -28,7 +28,7 @@ class Event < ActiveRecord::Base
     time_column: 'created_at',
     chunk_time_interval: '1 day',
     compress_segmentby: 'identifier',
-    compression_interval: '7 days'
+    compress_after: '7 days'
   }
 
   create_table(:events, id: false, hypertable: hypertable_options) do |t|

examples/all_in_one/benchmark_comparison.rb

@@ -41,7 +41,7 @@ class Event2 < ActiveRecord::Base
     time_column: 'created_at',
     chunk_time_interval: '7 day',
     compress_segmentby: 'identifier',
-    compression_interval: '7 days'
+    compress_after: '7 days'
   }
 
   create_table(:events, id: false, hypertable: hypertable_options) do |t|

examples/all_in_one/caggs.rb

@@ -36,7 +36,7 @@ class Tick < ActiveRecord::Base
     chunk_time_interval: '1 day',
     compress_segmentby: 'symbol',
     compress_orderby: 'time',
-    compression_interval: '7 days'
+    compress_after: '7 days'
   }
 
   create_table :ticks, hypertable: hypertable_options, id: false do |t|

examples/all_in_one/query_data.rb

@@ -34,7 +34,7 @@ def self.timestamp_attributes_for_update_in_model
     time_column: 'created_at',
     chunk_time_interval: '7 day',
     compress_segmentby: 'identifier',
-    compression_interval: '7 days'
+    compress_after: '7 days'
   }
 
   create_table(:events, id: false, hypertable: hypertable_options) do |t|

examples/ranking/config/initializers/timescale.rb

@@ -1,2 +1,5 @@
 require 'timescaledb'
 require 'scenic'
+
+ActiveSupport.on_load(:active_record) { extend Timescaledb::ActsAsHypertable }
+

examples/ranking/db/migrate/20220209120910_create_plays.rb

@@ -6,7 +6,7 @@ def change
         chunk_time_interval: '1 day',
         compress_segmentby: 'game_id',
         compress_orderby: 'created_at',
-        compression_interval: '7 days'
+        compress_after: '7 days'
     }
     create_table :plays, hypertable: hypertable_options, id: false do |t|
       t.references :game, null: false, foreign_key: false

examples/toolkit-demo/candlestick.rb

@@ -92,7 +92,7 @@ class Tick < ActiveRecord::Base
       chunk_time_interval: "1 day",
       compress_segmentby: "symbol",
       compress_orderby: "time",
-      compression_interval: "1 week"
+      compress_after: "1 week"
     }
     create_table :ticks, id: false, hypertable: hypertable_options, if_not_exists: true do |t|
       t.timestamptz :time, null: false

examples/toolkit-demo/ohlc.rb

@@ -81,7 +81,7 @@ class Ohlc1d < ActiveRecord::Base
       chunk_time_interval: '1 week',
       compress_segmentby: 'symbol',
       compress_orderby: 'time',
-      compression_interval: '1 month'
+      compress_after: '1 month'
     }
     create_table :ticks, hypertable: hypertable_options, id: false do |t|
       t.column :time , 'timestamp with time zone'

lib/timescaledb/migration_helpers.rb

@@ -16,7 +16,7 @@ module MigrationHelpers
     #    chunk_time_interval: '1 min',
     #    compress_segmentby: 'identifier',
     #    compress_orderby: 'created_at',
-    #    compression_interval: '7 days'
+    #    compress_after: '7 days'
     #  }
     #
     #  create_table(:events, id: false, hypertable: options) do |t|
@@ -41,38 +41,33 @@ def create_hypertable(table_name,
                           chunk_time_interval: '1 week',
                           compress_segmentby: nil,
                           compress_orderby: 'created_at',
-                          compression_interval: nil,
+                          compress_after: nil,
+                          drop_after: nil,
                           partition_column: nil,
                           number_partitions: nil,
                           **hypertable_options)
 
       original_logger = ActiveRecord::Base.logger
       ActiveRecord::Base.logger = Logger.new(STDOUT)
 
-      options = ["chunk_time_interval => #{chunk_time_interval_clause(chunk_time_interval)}"]
-      options += hypertable_options.map { |k, v| "#{k} => #{quote(v)}" }
+      dimension = "by_range(#{quote(time_column)}, #{parse_interval(chunk_time_interval)})"
 
-      arguments = [
-        quote(table_name),
-        quote(time_column),
-        (quote(partition_column) if partition_column),
-        (number_partitions if partition_column),
-        *options
+      arguments = [ quote(table_name), dimension,
+        *hypertable_options.map { |k, v| "#{k} => #{quote(v)}" }
       ]
 
       execute "SELECT create_hypertable(#{arguments.compact.join(', ')})"
 
-      if compress_segmentby
-        execute <<~SQL
-          ALTER TABLE #{table_name} SET (
-            timescaledb.compress,
-            timescaledb.compress_orderby = '#{compress_orderby}',
-            timescaledb.compress_segmentby = '#{compress_segmentby}'
-          )
-        SQL
+      if partition_column && number_partitions
+        execute "SELECT add_dimension('#{table_name}', by_hash(#{quote(partition_column)}, #{number_partitions}))"
       end
-      if compression_interval
-        execute "SELECT add_compression_policy('#{table_name}', INTERVAL '#{compression_interval}')"
+
+      if compress_segmentby || compress_after
+        add_compression_policy(table_name, orderby: compress_orderby, segmentby: compress_segmentby, compress_after: compress_after)
+      end
+
+      if drop_after
+        add_retention_policy(table_name, drop_after: drop_after)
       end
     ensure
       ActiveRecord::Base.logger = original_logger if original_logger
@@ -146,14 +141,40 @@ def remove_continuous_aggregate_policy(table_name)
       execute "SELECT remove_continuous_aggregate_policy('#{table_name}')"
     end
 
-    def create_retention_policy(table_name, interval:)
-      execute "SELECT add_retention_policy('#{table_name}', INTERVAL '#{interval}')"
+    def create_retention_policy(table_name, drop_after:)
+      execute "SELECT add_retention_policy('#{table_name}', drop_after => #{parse_interval(drop_after)})"
     end
 
+    alias_method :add_retention_policy, :create_retention_policy
+
     def remove_retention_policy(table_name)
       execute "SELECT remove_retention_policy('#{table_name}')"
     end
 
+
+    # Enable compression policy.
+    #
+    # @param table_name [String] The name of the table.
+    # @param orderby [String] The column to order by.
+    # @param segmentby [String] The column to segment by.
+    # @param compress_after [String] The interval to compress after.
+    # @param compression_chunk_time_interval [String] In case to merge chunks.
+    #
+    # @see https://docs.timescale.com/api/latest/compression/add_compression_policy/
+    def add_compression_policy(table_name, orderby:, segmentby:, compress_after: nil, compression_chunk_time_interval: nil)
+      options = []
+      options << 'timescaledb.compress'
+      options << "timescaledb.compress_orderby = '#{orderby}'" if orderby
+      options << "timescaledb.compress_segmentby = '#{segmentby}'" if segmentby
+      options << "timescaledb.compression_chunk_time_interval = INTERVAL '#{compression_chunk_time_interval}'" if compression_chunk_time_interval
+      execute <<~SQL
+        ALTER TABLE #{table_name} SET (
+          #{options.join(',')}
+        )
+      SQL
+      execute "SELECT add_compression_policy('#{table_name}', compress_after => INTERVAL '#{compress_after}')" if compress_after
+    end
+
     private
 
     # Build a string for the WITH clause of the CREATE MATERIALIZED VIEW statement.
@@ -166,11 +187,11 @@ def build_with_clause_option_string(option_key, options)
       ",timescaledb.#{option_key}=#{value}"
     end
 
-    def chunk_time_interval_clause(chunk_time_interval)
-      if chunk_time_interval.is_a?(Numeric)
-        chunk_time_interval
+    def parse_interval(interval)
+      if interval.is_a?(Numeric)
+        interval
       else
-        "INTERVAL '#{chunk_time_interval}'"
+        "INTERVAL '#{interval}'"
       end
     end
   end

lib/timescaledb/schema_dumper.rb

@@ -106,14 +106,12 @@ def timescale_compression_settings_for(hypertable)
         if setting.orderby_column_index
           if setting.orderby_asc
             direction = "ASC"
-            if setting.orderby_nullsfirst
-              direction += " NULLS FIRST"
-            end
+            # For ASC, default is NULLS LAST, so only add if explicitly set to FIRST
+            direction += " NULLS FIRST" if setting.orderby_nullsfirst == true
           else
             direction = "DESC"
-            if !setting.orderby_nullsfirst
-              direction += " NULLS LAST"
-            end
+            # For DESC, default is NULLS FIRST, so only add if explicitly set to LAST
+            direction += " NULLS LAST" if setting.orderby_nullsfirst == false
           end
 
           compression_settings[:compress_orderby] << "#{setting.attname} #{direction}"

spec/spec_helper.rb

@@ -5,12 +5,15 @@
 require 'timescaledb/toolkit'
 require "dotenv"
 require "database_cleaner/active_record"
+require "active_support/testing/time_helpers"
 require_relative "support/active_record/models"
 require_relative "support/active_record/schema"
 
-Dotenv.load! if File.exists?(".env")
+Dotenv.load! if File.exist?(".env")
 
-# Establish a connection for testing
+ActiveSupport.on_load(:active_record_postgresqladapter) do
+  self.datetime_type = :timestamptz
+end
 
 ActiveRecord::Base.establish_connection(ENV['PG_URI_TEST'])
 Timescaledb.establish_connection(ENV['PG_URI_TEST'])
@@ -30,6 +33,9 @@ def destroy_all_chunks_for!(klass)
   # Disable RSpec exposing methods globally on `Module` and `main`
   config.disable_monkey_patching!
 
+  config.before(:suite) do
+    Time.zone = 'UTC'
+  end
   config.expect_with :rspec do |c|
     c.syntax = :expect
   end
@@ -40,6 +46,18 @@ def destroy_all_chunks_for!(klass)
   end
 
   config.after(:each) do
-    DatabaseCleaner.clean
+    retries = 3
+    begin
+      DatabaseCleaner.clean
+    rescue ActiveRecord::StatementInvalid => e
+      if e.message =~ /deadlock detected/ && (retries -= 1) > 0
+        sleep 0.1
+        retry
+      else
+        raise
+      end
+    end
   end
+
+  config.include ActiveSupport::Testing::TimeHelpers
 end

spec/support/active_record/models.rb

@@ -1,4 +1,5 @@
-ActiveRecord::Base.extend Timescaledb::ActsAsHypertable
+ActiveSupport.on_load(:active_record) { extend Timescaledb::ActsAsHypertable }
+
 
 class Event < ActiveRecord::Base
   self.primary_key = "identifier"