Skip to content

Commit

Permalink
Switch to AggregatingMergeTree and record insert times
Browse files Browse the repository at this point in the history
The original choice of `ReplacingMergeTree` table engine worked fine,
but it's limiting when it comes to adding more data. The point of
using a non-default table engine is to use ClickHouse to deduplicate
inserted data. But there's more flexible table engines available.

This commit switches to AggregatingMergeTree and adds a new `inserted`
column that will expose the time a reading was first inserted into
ClickHouse. This will help with understanding when the Octopus API
updates and when is best to query it.
  • Loading branch information
46bit committed Nov 24, 2023
1 parent e370b54 commit 5fc8c37
Showing 1 changed file with 25 additions and 14 deletions.
39 changes: 25 additions & 14 deletions schema.sql
Original file line number Diff line number Diff line change
@@ -1,31 +1,42 @@
CREATE TABLE IF NOT EXISTS octopus_30m_data (
fuel Enum('gas' = 1, 'electricity' = 2),
fuel LowCardinality(String),
start DateTime,
consumption Float64
) ENGINE = ReplacingMergeTree
inserted SimpleAggregateFunction(min, DateTime) DEFAULT now(),
consumption SimpleAggregateFunction(max, Float64)
) ENGINE = AggregatingMergeTree
PARTITION BY toYYYYMM(start)
ORDER BY (fuel, start)
PRIMARY KEY (fuel, start);

CREATE VIEW IF NOT EXISTS octopus_gas_30m AS (
CREATE VIEW IF NOT EXISTS octopus_30m AS (
SELECT
fuel,
start,
start + INTERVAL '30' MINUTE AS end,
max(consumption) AS cubic_metres,
cubic_metres * 10 AS approx_kwh
min(inserted) AS inserted,
max(consumption) AS consumption
FROM octopus_30m_data
WHERE fuel = 'gas'
GROUP BY start
ORDER BY start
GROUP BY fuel, start
ORDER BY fuel, start
);

CREATE VIEW IF NOT EXISTS octopus_electricity_30m AS (
SELECT
start,
start + INTERVAL '30' MINUTE AS end,
max(consumption) AS kwh
FROM octopus_30m_data
end,
inserted,
consumption AS kwh
FROM octopus_30m
WHERE fuel = 'electricity'
GROUP BY start
ORDER BY start
);

CREATE VIEW IF NOT EXISTS octopus_gas_30m AS (
SELECT
start,
end,
inserted,
consumption AS cubic_metres,
cubic_metres * 10 AS approx_kwh
FROM octopus_30m
WHERE fuel = 'gas'
);

0 comments on commit 5fc8c37

Please sign in to comment.