Merge branch 'master' of https://github.com/jij1949/querybook into ex…

…ternal/group-permissions

Dockerfile

@@ -40,7 +40,9 @@ RUN pip install -r requirements/base.txt \
     pip install -r requirements/${PACKAGE}; \
     done \
     fi \
-    && pip install -r requirements/local.txt || true
+    && if [ -f requirements/local.txt ]; then \
+    pip install -r requirements/local.txt; \
+    fi
 
 COPY package.json yarn.lock ./
 RUN yarn install --pure-lockfile

package.json

@@ -1,6 +1,6 @@
 {
     "name": "querybook",
-    "version": "3.29.0",
+    "version": "3.30.0",
     "description": "A Big Data Webapp",
     "private": true,
     "scripts": {

querybook/migrations/versions/299e24dcfd29_allow_unique_table_ownership_per_type.py

@@ -0,0 +1,87 @@
+"""allow unique_table_ownership per type
+
+Revision ID: 299e24dcfd29
+Revises: c00f08f16065
+Create Date: 2024-01-31 14:39:13.601013
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "299e24dcfd29"
+down_revision = "c00f08f16065"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    ownership_table = sa.Table(
+        "data_table_ownership", sa.MetaData(op.get_bind()), autoload=True
+    )
+
+    for constraint in ownership_table.foreign_key_constraints:
+        op.drop_constraint(constraint.name, "data_table_ownership", type_="foreignkey")
+
+    op.drop_constraint("unique_table_ownership", "data_table_ownership", type_="unique")
+
+    op.create_foreign_key(
+        "data_table_ownership_data_table_id_fk",
+        "data_table_ownership",
+        "data_table",
+        ["data_table_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+    op.create_foreign_key(
+        "data_table_ownership_uid_fk",
+        "data_table_ownership",
+        "user",
+        ["uid"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+    op.create_unique_constraint(
+        "unique_table_ownership",
+        "data_table_ownership",
+        ["data_table_id", "uid", "type"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    ownership_table = sa.Table(
+        "data_table_ownership", sa.MetaData(op.get_bind()), autoload=True
+    )
+
+    for constraint in ownership_table.foreign_key_constraints:
+        op.drop_constraint(constraint.name, "data_table_ownership", type_="foreignkey")
+
+    op.drop_constraint("unique_table_ownership", "data_table_ownership", type_="unique")
+
+    op.create_foreign_key(
+        "data_table_ownership_data_table_id_fk",
+        "data_table_ownership",
+        "data_table",
+        ["data_table_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+    op.create_foreign_key(
+        "data_table_ownership_uid_fk",
+        "data_table_ownership",
+        "user",
+        ["uid"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+    op.create_unique_constraint(
+        "unique_table_ownership",
+        "data_table_ownership",
+        ["data_table_id", "uid"],
+    )
+    # ### end Alembic commands ###

querybook/server/datasources/metastore.py

@@ -1,35 +1,29 @@
 from typing import Tuple, Union
-from flask_login import current_user
 
 from app.auth.permission import (
-    verify_query_engine_environment_permission,
-    verify_environment_permission,
-    verify_metastore_permission,
+    verify_data_column_permission,
     verify_data_schema_permission,
     verify_data_table_permission,
-    verify_data_column_permission,
+    verify_environment_permission,
+    verify_metastore_permission,
+    verify_query_engine_environment_permission,
 )
+from app.datasource import admin_only, api_assert, register, with_impression
 from app.db import DBSession
-from app.datasource import register, api_assert, with_impression, admin_only
 from app.flask_app import cache, limiter
+from const.datasources import RESOURCE_NOT_FOUND_STATUS_CODE
 from const.impression import ImpressionItemType
 from const.metastore import DataTableWarningSeverity, MetadataType
 from const.time import seconds_in_a_day
-from const.datasources import RESOURCE_NOT_FOUND_STATUS_CODE
+from flask_login import current_user
 from lib.lineage.utils import lineage
-from lib.metastore.utils import DataTableFinder
 from lib.metastore import get_metastore_loader
+from lib.metastore.utils import DataTableFinder
 from lib.query_analysis.samples import make_samples_query
 from lib.utils import mysql_cache
-from logic import metastore as logic
 from logic import admin as admin_logic
-from logic import data_element as data_element_logic
-from logic import tag as tag_logic
-from models.metastore import (
-    DataTableWarning,
-    DataTableStatistics,
-    DataTableColumnStatistics,
-)
+from logic import metastore as logic
+from models.metastore import DataTableStatistics, DataTableWarning
 from tasks.run_sample_query import run_sample_query
 
 
@@ -269,6 +263,13 @@ def get_columns_from_table(table_id):
         return logic.get_column_by_table_id(table_id, session=session)
 
 
+@register("/table/<int:table_id>/detailed_column/", methods=["GET"])
+def get_detailed_columns_from_table(table_id):
+    with DBSession() as session:
+        verify_data_table_permission(table_id, session=session)
+        return logic.get_detailed_columns_dict_by_table_id(table_id, session=session)
+
+
 @register("/table/<int:table_id>/raw_samples_query/", methods=["GET"])
 def get_table_samples_raw_query(
     table_id,
@@ -380,14 +381,7 @@ def get_column_by_table(table_id, column_name, with_table=False):
 def get_column(column_id, with_table=False):
     column = logic.get_column_by_id(column_id)
     verify_data_table_permission(column.table_id)
-    column_dict = column.to_dict(with_table)
-
-    column_dict["stats"] = DataTableColumnStatistics.get_all(column_id=column_id)
-    column_dict["tags"] = tag_logic.get_tags_by_column_id(column_id=column_id)
-    column_dict[
-        "data_element_association"
-    ] = data_element_logic.get_data_element_association_by_column_id(column_id)
-    return column_dict
+    return logic.get_detailed_column_dict(column, with_table=with_table)
 
 
 @register("/column/<int:column_id>/", methods=["PUT"])

querybook/server/lib/ai_assistant/tools/table_schema.py

@@ -1,9 +1,9 @@
 from typing import Callable
 
 from app.db import with_session
+from lib.vector_store import get_vector_store
 from logic import metastore as m_logic
 from models.metastore import DataTable, DataTableColumn
-from lib.vector_store import get_vector_store
 
 
 def get_table_documentation(table: DataTable) -> str:
@@ -33,6 +33,11 @@ def _get_column(column: DataTableColumn) -> dict[str, str]:
         column_json["description"] = column.data_elements[0].description
         column_json["data_element"] = column.data_elements[0].name
 
+    if len(column.statistics):
+        column_json["statistics"] = {
+            stat.key: stat.value for stat in column.statistics if stat.value is not None
+        }
+
     return column_json
 
 

querybook/server/lib/export/exporters/gspread_exporter.py

@@ -18,7 +18,7 @@
 from env import QuerybookSettings
 from logic.user import get_user_by_id, update_user_properties
 from lib.export.base_exporter import BaseExporter
-from lib.form import StructFormField, FormField
+from lib.form import StructFormField, FormField, FormFieldType
 from logic.query_execution import (
     get_statement_execution_by_id,
     update_statement_execution,
@@ -118,6 +118,13 @@ def export_form(self):
                     regex="^[A-Z]{1,3}[1-9][0-9]*$",
                 ),
             ),
+            (
+                "clear_sheet",
+                FormField(
+                    field_type=FormFieldType.Boolean,
+                    helper="If checked, the sheet will be cleared before writing the result",
+                ),
+            ),
         )
 
     @with_session
@@ -159,6 +166,7 @@ def export(
         sheet_url=None,
         worksheet_title="Sheet1",
         start_cell="A1",
+        clear_sheet=False,
     ):
         sheet = None
         try:
@@ -174,6 +182,7 @@ def export(
                     statement_execution_id,
                     worksheet_title,
                     start_cell,
+                    clear_sheet,
                 )
             sheet_url = f"https://docs.google.com/spreadsheets/d/{sheet.id}"
             self._save_sheet_to_statement_meta(sheet_url, statement_execution_id)
@@ -213,6 +222,7 @@ def write_csv_to_sheet(
         statement_execution_id: int,
         worksheet_title: str,
         start_cell: str,
+        clear_sheet: bool,
     ):
         with DBSession() as session:
             max_rows = self._get_max_rows(
@@ -236,6 +246,8 @@ def write_csv_to_sheet(
             with gspread_worksheet(
                 sheet, worksheet_title, end_cell_coord[0], end_cell_coord[1]
             ) as worksheet:
+                if clear_sheet:
+                    worksheet.clear()
                 csv = self._get_statement_execution_result_iter(
                     statement_execution_id, number_of_lines=max_rows, session=session
                 )

querybook/server/logic/metastore.py

@@ -5,6 +5,7 @@
 from const.metastore import DataOwner, DataTableWarningSeverity
 from lib.logger import get_logger
 from lib.sqlalchemy import update_model_fields
+from logic import data_element as data_element_logic
 from logic.user import get_user_by_name
 from models.admin import QueryEngineEnvironment
 from models.metastore import (
@@ -500,6 +501,38 @@ def get_column_by_table_id(table_id, session=None):
     )
 
 
+@with_session
+def get_detailed_column_dict(column: DataTableColumn, with_table=False, session=None):
+    from logic import tag as tag_logic
+
+    column_dict = column.to_dict(with_table)
+    column_dict["stats"] = DataTableColumnStatistics.get_all(
+        column_id=column.id, session=session
+    )
+    column_dict["tags"] = tag_logic.get_tags_by_column_id(
+        column_id=column.id, session=session
+    )
+    column_dict[
+        "data_element_association"
+    ] = data_element_logic.get_data_element_association_by_column_id(
+        column.id, session=session
+    )
+    return column_dict
+
+
+@with_session
+def get_detailed_columns_dict_by_table_id(table_id, session=None):
+    data_table_columns = (
+        session.query(DataTableColumn)
+        .filter(DataTableColumn.table_id == table_id)
+        .all()
+    )
+    columns_info = []
+    for col in data_table_columns:
+        columns_info.append(get_detailed_column_dict(col, session=session))
+    return columns_info
+
+
 @with_session
 def get_all_column_name_by_table_id(table_id, session=None):
     return (

querybook/server/models/metastore.py

@@ -1,18 +1,17 @@
 import sqlalchemy as sql
-from sqlalchemy.orm import backref, relationship
-
 from app import db
 from const.db import (
-    utf8mb4_name_length,
-    name_length,
-    now,
     description_length,
-    url_length,
     mediumtext_length,
+    name_length,
+    now,
     type_length,
+    url_length,
+    utf8mb4_name_length,
 )
 from const.metastore import DataTableWarningSeverity
 from lib.sqlalchemy import CRUDMixin, TruncateString
+from sqlalchemy.orm import backref, relationship
 
 Base = db.Base
 
@@ -301,6 +300,11 @@ class DataTableColumn(TruncateString("name", "type", "comment"), Base):
     data_elements = relationship(
         "DataElement", secondary="data_element_association", uselist=True, viewonly=True
     )
+    statistics = relationship(
+        "DataTableColumnStatistics",
+        uselist=True,
+        viewonly=True,
+    )
 
     def to_dict(self, include_table=False):
         column_dict = {
@@ -320,7 +324,9 @@ def to_dict(self, include_table=False):
 class DataTableOwnership(Base):
     __tablename__ = "data_table_ownership"
     __table_args__ = (
-        sql.UniqueConstraint("data_table_id", "uid", name="unique_table_ownership"),
+        sql.UniqueConstraint(
+            "data_table_id", "uid", "type", name="unique_table_ownership"
+        ),
     )
 
     id = sql.Column(sql.Integer, primary_key=True)