diff --git a/frontend/src/pages/Backups/Backups.tsx b/frontend/src/pages/Backups/Backups.tsx
index 7ea1aed..36b662b 100644
--- a/frontend/src/pages/Backups/Backups.tsx
+++ b/frontend/src/pages/Backups/Backups.tsx
@@ -1,7 +1,7 @@
import React, { useEffect, useState } from 'react'
import { usePollingEffect } from '../../utils/usePollingEffect'
import { ColumnType } from 'antd/es/table'
-import { Table, Button, Form, Input, Modal, Tag, Col, Progress, Row, Tooltip, notification } from 'antd'
+import { Table, Button, Form, Input, Checkbox, Modal, Tag, Col, Progress, Row, Tooltip, notification } from 'antd'
import useSWR, { mutate } from 'swr'
interface BackupRow {
@@ -29,6 +29,7 @@ type FieldType = {
table?: string
bucket?: string
path?: string
+ is_sharded?: boolean
aws_access_key_id?: string
aws_secret_access_key?: string
}
@@ -165,6 +166,16 @@ export default function Backups() {
+
+ label="Is Sharded"
+ name="is_sharded"
+ initialValue="false"
+ valuePropName="checked"
+ rules={[{ required: true, message: 'Is this table sharded?' }]}
+ >
+ is sharded
+
+
label="S3 Bucket"
name="bucket"
diff --git a/frontend/src/pages/Backups/ScheduledBackups.tsx b/frontend/src/pages/Backups/ScheduledBackups.tsx
index 275ce3e..439c3bc 100644
--- a/frontend/src/pages/Backups/ScheduledBackups.tsx
+++ b/frontend/src/pages/Backups/ScheduledBackups.tsx
@@ -1,9 +1,24 @@
import React, { useEffect, useState } from 'react'
import { usePollingEffect } from '../../utils/usePollingEffect'
import { ColumnType } from 'antd/es/table'
-import { Switch, Select, Table, Button, Form, Input, Modal, Tag, Col, Progress, Row, Tooltip, notification } from 'antd'
-import DeleteOutlined from '@ant-design/icons'
-import EditOutlined from '@ant-design/icons'
+import {
+ Switch,
+ Select,
+ Table,
+ Button,
+ Form,
+ Input,
+ Checkbox,
+ Modal,
+ Tag,
+ Col,
+ Progress,
+ Row,
+ Tooltip,
+ notification,
+} from 'antd'
+import DeleteOutlined from '@ant-design/icons/DeleteOutlined'
+import EditOutlined from '@ant-design/icons/EditOutlined'
import { Clusters } from '../Clusters/Clusters'
import useSWR, { mutate } from 'swr'
@@ -16,6 +31,7 @@ interface ScheduleRow {
schedule: string
incremental_schedule: string
table: string
+ is_sharded: boolean
database: string
bucket: string
path: string
@@ -32,6 +48,7 @@ type FieldType = {
incremental_schedule?: string
database?: string
table?: string
+ is_sharded?: boolean
bucket?: string
path?: string
aws_access_key_id?: string
@@ -153,12 +170,12 @@ export default function ScheduledBackups() {
{ title: 'Last Run Time', dataIndex: 'last_run_time' },
{ title: 'Database', dataIndex: 'database' },
{ title: 'Table', dataIndex: 'table' },
+ { title: 'Is Sharded', dataIndex: 'is_sharded', render: (_, sched) => (sched.is_sharded ? 'Yes' : 'No') },
{ title: 'S3 Location', dataIndex: 'bucket', render: (_, sched) => 's3://' + sched.bucket + '/' + sched.path },
- { title: 'Created At', dataIndex: 'created_at' },
{
- title: '',
+ title: 'Actions',
dataIndex: 'id',
- render: id => {
+ render: (id: string, rowData: ScheduleRow) => {
const deleteBackup = async () => {
try {
const res = await fetch(`/api/scheduled_backups/${id}`, {
@@ -173,21 +190,10 @@ export default function ScheduledBackups() {
}
}
- return (
-
-
-
- )
- },
- },
- {
- title: 'Actions',
- dataIndex: 'id',
- render: (id: string, rowData: ScheduleRow) => {
return (
<>
handleEdit(rowData)} />
- {/* handleDelete(id)} /> */}
+ deleteBackup()} style={{ marginLeft: '15px' }} />
)
},
@@ -285,6 +291,16 @@ export default function ScheduledBackups() {
+
+ label="Is Sharded"
+ name="is_sharded"
+ initialValue="false"
+ valuePropName="checked"
+ rules={[{ required: true, message: 'Is this table sharded?' }]}
+ >
+ is sharded
+
+
label="S3 Bucket"
name="bucket"
diff --git a/frontend/src/pages/Overview/Overview.tsx b/frontend/src/pages/Overview/Overview.tsx
index a8f3a44..c28939b 100644
--- a/frontend/src/pages/Overview/Overview.tsx
+++ b/frontend/src/pages/Overview/Overview.tsx
@@ -1,7 +1,7 @@
import React, { useEffect, useState } from 'react'
import { Line } from '@ant-design/charts'
import { Card, Col, Row, Tooltip, notification } from 'antd'
-import InfoCircleOutlined from '@ant-design/icons'
+import InfoCircleOutlined from '@ant-design/icons/InfoCircleOutlined'
import { clickhouseTips } from './tips'
import useSWR from 'swr'
diff --git a/frontend/src/pages/QueryEditor/QueryEditor.tsx b/frontend/src/pages/QueryEditor/QueryEditor.tsx
index 5d0b5d7..66919ba 100644
--- a/frontend/src/pages/QueryEditor/QueryEditor.tsx
+++ b/frontend/src/pages/QueryEditor/QueryEditor.tsx
@@ -6,7 +6,7 @@ import 'prismjs/components/prism-sql'
import 'prismjs/themes/prism.css'
import Editor from 'react-simple-code-editor'
import { v4 as uuidv4 } from 'uuid'
-import SaveOutlined from '@ant-design/icons'
+import SaveOutlined from '@ant-design/icons/SaveOutlined'
function CreateSavedQueryModal({
modalOpen = false,
diff --git a/frontend/src/pages/QueryEditor/SavedQueries.tsx b/frontend/src/pages/QueryEditor/SavedQueries.tsx
index 30a9320..d1ab985 100644
--- a/frontend/src/pages/QueryEditor/SavedQueries.tsx
+++ b/frontend/src/pages/QueryEditor/SavedQueries.tsx
@@ -2,7 +2,7 @@ import { Table, Button, Row, Col, Tooltip } from 'antd'
import React, { useEffect, useState } from 'react'
import { ColumnType } from 'antd/es/table'
import SavedQuery from './SavedQuery'
-import ReloadOutlined from '@ant-design/icons'
+import ReloadOutlined from '@ant-design/icons/ReloadOutlined'
import { useHistory } from 'react-router-dom'
import { isoTimestampToHumanReadable } from '../../utils/dateUtils'
diff --git a/frontend/src/pages/SlowQueries/MetricsTab.tsx b/frontend/src/pages/SlowQueries/MetricsTab.tsx
index f744bab..b76ed18 100644
--- a/frontend/src/pages/SlowQueries/MetricsTab.tsx
+++ b/frontend/src/pages/SlowQueries/MetricsTab.tsx
@@ -2,7 +2,7 @@ import React, { useEffect, useState } from 'react'
import { Line } from '@ant-design/plots'
// @ts-ignore
import { Card, Col, Row, Tooltip, notification } from 'antd'
-import InfoCircleOutlined from '@ant-design/icons'
+import InfoCircleOutlined from '@ant-design/icons/InfoCircleOutlined'
import { NoDataSpinner, QueryDetailData } from './QueryDetail'
export default function MetricsTab({ query_hash }: { query_hash: string }) {
diff --git a/housewatch/clickhouse/backups.py b/housewatch/clickhouse/backups.py
index 54bdc3e..063fdc3 100644
--- a/housewatch/clickhouse/backups.py
+++ b/housewatch/clickhouse/backups.py
@@ -5,6 +5,7 @@
from uuid import uuid4
from housewatch.clickhouse.client import run_query
from housewatch.models.backup import ScheduledBackup, ScheduledBackupRun
+from housewatch.clickhouse.table import table_engine_full
from housewatch.clickhouse.clusters import get_node_per_shard
from django.conf import settings
@@ -15,7 +16,7 @@
logger = structlog.get_logger(__name__)
-def execute_backup_on_shards(
+def execute_backup(
query: str,
params: Dict[str, str | int] = {},
query_settings: Dict[str, str | int] = {},
@@ -25,6 +26,7 @@ def execute_backup_on_shards(
aws_key: Optional[str] = None,
aws_secret: Optional[str] = None,
base_backup: Optional[str] = None,
+ is_sharded: bool = False,
):
"""
This function will execute a backup on each shard in a cluster
@@ -57,6 +59,8 @@ def execute_backup_on_shards(
item[key[0]] = res[index]
response.append(item)
responses.append((shard, response))
+ if not is_sharded:
+ return response
return response
@@ -78,7 +82,9 @@ def get_backup(backup, cluster=None):
return run_query(QUERY, {"uuid": backup}, use_cache=False)
-def create_table_backup(database, table, bucket, path, cluster=None, aws_key=None, aws_secret=None, base_backup=None):
+def create_table_backup(
+ database, table, bucket, path, cluster=None, aws_key=None, aws_secret=None, base_backup=None, is_sharded=False
+):
if aws_key is None or aws_secret is None:
aws_key = settings.AWS_ACCESS_KEY_ID
aws_secret = settings.AWS_SECRET_ACCESS_KEY
@@ -87,7 +93,7 @@ def create_table_backup(database, table, bucket, path, cluster=None, aws_key=Non
QUERY = """BACKUP TABLE %(database)s.%(table)s
TO S3('https://%(bucket)s.s3.amazonaws.com/%(path)s/%(shard)s', '%(aws_key)s', '%(aws_secret)s')
ASYNC"""
- return execute_backup_on_shards(
+ return execute_backup(
QUERY,
{
"database": database,
@@ -102,6 +108,7 @@ def create_table_backup(database, table, bucket, path, cluster=None, aws_key=Non
aws_key=aws_key,
aws_secret=aws_secret,
base_backup=base_backup,
+ is_sharded=is_sharded,
)
QUERY = """BACKUP TABLE %(database)s.%(table)s
TO S3('https://%(bucket)s.s3.amazonaws.com/%(path)s', '%(aws_key)s', '%(aws_secret)s')
@@ -133,7 +140,7 @@ def create_database_backup(database, bucket, path, cluster=None, aws_key=None, a
TO S3('https://%(bucket)s.s3.amazonaws.com/%(path)s/%(shard)s', '%(aws_key)s', '%(aws_secret)s')
ASYNC"""
- return execute_backup_on_shards(
+ return execute_backup(
QUERY,
{
"database": database,
@@ -199,6 +206,7 @@ def run_backup(backup_id, incremental=False):
backup.aws_access_key_id,
backup.aws_secret_access_key,
base_backup=base_backup,
+ is_sharded=backup.is_sharded,
)
uuid = str(uuid4())
br = ScheduledBackupRun.objects.create(
diff --git a/housewatch/clickhouse/table.py b/housewatch/clickhouse/table.py
new file mode 100644
index 0000000..cb89af2
--- /dev/null
+++ b/housewatch/clickhouse/table.py
@@ -0,0 +1,21 @@
+from housewatch.clickhouse.client import run_query
+
+
+def is_replicated_table(database, table):
+ QUERY = """SELECT engine FROM system.tables WHERE database = '%(database)s' AND name = '%(table)s'"""
+ return "replicated" in run_query(QUERY, {"database": database, "table": table})[0]["engine"].lower()
+
+
+def table_engine_full(database, table):
+ QUERY = """SELECT engine_full FROM system.tables WHERE database = '%(database)s' AND name = '%(table)s'"""
+ return run_query(QUERY, {"database": database, "table": table})[0]["engine_full"]
+
+
+def parse_engine(engine_full):
+ engine = engine_full.split("(")[0].strip()
+ params = engine_full.split("(")[1].split(")")[0].split(",")
+ return engine, params
+
+
+def is_sharded_table(database, table):
+ return "sharded" in table_engine_full(database, table).lower()
diff --git a/housewatch/migrations/0011_scheduledbackup_is_sharded_and_more.py b/housewatch/migrations/0011_scheduledbackup_is_sharded_and_more.py
new file mode 100644
index 0000000..ae32402
--- /dev/null
+++ b/housewatch/migrations/0011_scheduledbackup_is_sharded_and_more.py
@@ -0,0 +1,23 @@
+# Generated by Django 4.1.1 on 2024-01-31 06:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('housewatch', '0010_scheduledbackup_incremental_schedule_and_more'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='scheduledbackup',
+ name='is_sharded',
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AlterField(
+ model_name='scheduledbackup',
+ name='table',
+ field=models.CharField(blank=True, max_length=255, null=True),
+ ),
+ ]
diff --git a/housewatch/models/backup.py b/housewatch/models/backup.py
index b512b00..c19b61e 100644
--- a/housewatch/models/backup.py
+++ b/housewatch/models/backup.py
@@ -19,6 +19,7 @@ class ScheduledBackup(models.Model):
incremental_schedule: models.CharField = models.CharField(max_length=255, null=True)
table: models.CharField = models.CharField(max_length=255, null=True, blank=True)
database: models.CharField = models.CharField(max_length=255)
+ is_sharded: models.BooleanField = models.BooleanField(default=False)
cluster: models.CharField = models.CharField(max_length=255, null=True)
bucket: models.CharField = models.CharField(max_length=255)
path: models.CharField = models.CharField(max_length=255)