Separate ibis server by programming language

ibis-server/Dockerfile

@@ -21,11 +21,8 @@ WORKDIR /app
 
 COPY pyproject.toml ./
 COPY poetry.lock ./
-COPY rust/Cargo.toml ./
-COPY rust/Cargo.lock ./
 
 RUN poetry install --without dev
-RUN poetry run maturin develop
 
 FROM python:3.11-slim-buster as runtime
 
@@ -38,7 +35,7 @@ ENV VIRTUAL_ENV=/app/.venv \
     PATH="/app/.venv/bin:$PATH"
 
 COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
-COPY python/wren_engine app
+COPY app app
 
 EXPOSE 8000
 

ibis-server/README.md

@@ -42,6 +42,7 @@ docker compose up
 Requirements:
 - Python 3.11
 - [poetry](https://github.com/python-poetry/poetry) 1.7.1 (see [Environment Setup](#Environment-Setup))
+- [Rust](https://www.rust-lang.org/tools/install) and [Cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html)
 
 Clone the repository
 ```bash
@@ -64,10 +65,6 @@ Install the dependencies
 ```bash
 just install
 ```
-Build rust code be a python dependency
-```bash
-just maturin
-```
 Run the server
 ```bash
 just run
@@ -82,7 +79,6 @@ just run
 - Install [casey/just](https://github.com/casey/just)
 - Install [pre-commit](https://pre-commit.com) hooks: `just pre-commit-install`
 - Execute `just install` to install the dependencies
-- Execute `just maturin` to make rust be a python dependency.
 - Execute `just test` to run the tests
 - Create `.env` file and fill in the environment variables
 
@@ -98,3 +94,6 @@ just run
 ### Docker
 - Build the image: `just docker-build`
 - Run the container: `just docker-run`
+
+### Modeling Core
+The modeling core is written in Rust and convert to Python package. You can find the [wren-modeling-py](../wren-modeling-py) and the source code in the [wren-modeling-rs](../wren-modeling-rs) directory.

ibis-server/python/wren_engine/logger.py → ibis-server/app/logger.py

@@ -1,7 +1,7 @@
 import logging
 from functools import wraps
 
-from wren_engine.config import get_config
+from app.config import get_config
 
 logging.basicConfig(level=get_config().log_level)
 

ibis-server/python/wren_engine/main.py → ibis-server/app/main.py

@@ -2,10 +2,10 @@
 from fastapi.responses import RedirectResponse
 from starlette.responses import PlainTextResponse
 
-from wren_engine.config import get_config
-from wren_engine.model.connector import QueryDryRunError
-from wren_engine.model.validator import ValidationError
-from wren_engine.routers import v2
+from app.config import get_config
+from app.model.connector import QueryDryRunError
+from app.model.validator import ValidationError
+from app.routers import v2
 
 app = FastAPI()
 app.include_router(v2.router)

ibis-server/python/wren_engine/mdl/analyzer.py → ibis-server/app/mdl/analyzer.py

@@ -1,7 +1,7 @@
 import httpx
 import orjson
 
-from wren_engine.config import get_config
+from app.config import get_config
 
 wren_engine_endpoint = get_config().wren_engine_endpoint
 

ibis-server/python/wren_engine/mdl/rewriter.py → ibis-server/app/mdl/rewriter.py

@@ -1,8 +1,8 @@
 import httpx
 import orjson
 
-from wren_engine.config import get_config
-from wren_engine.logger import log_rewritten
+from app.config import get_config
+from app.logger import log_rewritten
 
 wren_engine_endpoint = get_config().wren_engine_endpoint
 

ibis-server/python/wren_engine/model/connector.py → ibis-server/app/model/connector.py

@@ -3,8 +3,8 @@
 import pandas as pd
 from pydantic import BaseModel, Field
 
-from wren_engine.mdl.rewriter import rewrite
-from wren_engine.model.data_source import (
+from app.mdl.rewriter import rewrite
+from app.model.data_source import (
     BigQueryConnectionInfo,
     ConnectionInfo,
     DataSource,

ibis-server/python/wren_engine/model/metadata/bigquery.py → ibis-server/app/model/metadata/bigquery.py

@@ -1,15 +1,15 @@
 from json import loads
 
-from wren_engine.model.data_source import BigQueryConnectionInfo
-from wren_engine.model.data_source import DataSource
-from wren_engine.model.metadata.dto import (
+from app.model.data_source import BigQueryConnectionInfo
+from app.model.data_source import DataSource
+from app.model.metadata.dto import (
     Table,
     Constraint,
     TableProperties,
     Column,
     ConstraintType,
 )
-from wren_engine.model.metadata.metadata import Metadata
+from app.model.metadata.metadata import Metadata
 
 
 class BigQueryMetadata(Metadata):

ibis-server/python/wren_engine/model/metadata/dto.py → ibis-server/app/model/metadata/dto.py

@@ -3,7 +3,7 @@
 
 from pydantic import BaseModel, Field
 
-from wren_engine.model.data_source import ConnectionInfo
+from app.model.data_source import ConnectionInfo
 
 
 class MetadataDTO(BaseModel):

ibis-server/python/wren_engine/model/metadata/factory.py → ibis-server/app/model/metadata/factory.py

@@ -1,13 +1,13 @@
 from json import loads
 
-from wren_engine.model.data_source import DataSource, ConnectionInfo
-from wren_engine.model.metadata.bigquery import BigQueryMetadata
-from wren_engine.model.metadata.dto import (
+from app.model.data_source import DataSource, ConnectionInfo
+from app.model.metadata.bigquery import BigQueryMetadata
+from app.model.metadata.dto import (
     Table,
     Constraint,
 )
-from wren_engine.model.metadata.metadata import Metadata
-from wren_engine.model.metadata.postgres import PostgresMetadata
+from app.model.metadata.metadata import Metadata
+from app.model.metadata.postgres import PostgresMetadata
 
 
 class MetadataFactory:

ibis-server/python/wren_engine/model/metadata/metadata.py → ibis-server/app/model/metadata/metadata.py

@@ -1,5 +1,5 @@
-from wren_engine.model.connector import ConnectionInfo
-from wren_engine.model.metadata.dto import Table, Constraint
+from app.model.connector import ConnectionInfo
+from app.model.metadata.dto import Table, Constraint
 
 
 class Metadata:

ibis-server/python/wren_engine/model/metadata/postgres.py → ibis-server/app/model/metadata/postgres.py

@@ -1,15 +1,15 @@
 from json import loads
 
-from wren_engine.model.data_source import DataSource, PostgresConnectionInfo
-from wren_engine.model.metadata.dto import (
+from app.model.data_source import DataSource, PostgresConnectionInfo
+from app.model.metadata.dto import (
     Table,
     Constraint,
     TableProperties,
     Column,
     ConstraintType,
     WrenEngineColumnType,
 )
-from wren_engine.model.metadata.metadata import Metadata
+from app.model.metadata.metadata import Metadata
 
 
 class PostgresMetadata(Metadata):

ibis-server/python/wren_engine/model/validator.py → ibis-server/app/model/validator.py

@@ -2,8 +2,8 @@
 
 from pydantic import BaseModel, Field
 
-from wren_engine.model.connector import Connector
-from wren_engine.model.data_source import ConnectionInfo
+from app.model.connector import Connector
+from app.model.data_source import ConnectionInfo
 
 rules = ["column_is_valid"]
 

ibis-server/python/wren_engine/routers/v2/__init__.py → ibis-server/app/routers/v2/__init__.py

@@ -1,6 +1,6 @@
 from fastapi import APIRouter
 
-from wren_engine.routers.v2 import ibis, analysis
+from app.routers.v2 import ibis, analysis
 
 prefix = "/v2"
 

ibis-server/python/wren_engine/routers/v2/analysis.py → ibis-server/app/routers/v2/analysis.py

@@ -1,8 +1,8 @@
 from fastapi import APIRouter
 from pydantic import BaseModel, Field
 
-from wren_engine.logger import log_dto
-from wren_engine.mdl.analyzer import analyze
+from app.logger import log_dto
+from app.mdl.analyzer import analyze
 
 router = APIRouter(prefix="/analysis", tags=["analysis"])
 

ibis-server/python/wren_engine/routers/v2/ibis/__init__.py → ibis-server/app/routers/v2/ibis/__init__.py

@@ -1,6 +1,6 @@
 from fastapi import APIRouter
 
-from wren_engine.routers.v2.ibis import bigquery, mysql, postgres, snowflake
+from app.routers.v2.ibis import bigquery, mysql, postgres, snowflake
 
 prefix = "/ibis"
 

ibis-server/python/wren_engine/routers/v2/ibis/bigquery.py → ibis-server/app/routers/v2/ibis/bigquery.py

@@ -3,12 +3,12 @@
 from fastapi import APIRouter, Query, Response
 from fastapi.responses import JSONResponse
 
-from wren_engine.logger import log_dto
-from wren_engine.model.connector import Connector, QueryBigQueryDTO, to_json
-from wren_engine.model.data_source import DataSource
-from wren_engine.model.metadata.dto import MetadataDTO, Table, Constraint
-from wren_engine.model.metadata.factory import MetadataFactory
-from wren_engine.model.validator import ValidateDTO, Validator
+from app.logger import log_dto
+from app.model.connector import Connector, QueryBigQueryDTO, to_json
+from app.model.data_source import DataSource
+from app.model.metadata.dto import MetadataDTO, Table, Constraint
+from app.model.metadata.factory import MetadataFactory
+from app.model.validator import ValidateDTO, Validator
 
 router = APIRouter(prefix="/bigquery", tags=["bigquery"])
 

ibis-server/python/wren_engine/routers/v2/ibis/mysql.py → ibis-server/app/routers/v2/ibis/mysql.py

@@ -3,10 +3,10 @@
 from fastapi import APIRouter, Query, Response
 from fastapi.responses import JSONResponse
 
-from wren_engine.logger import log_dto
-from wren_engine.model.connector import Connector, to_json, QueryMySqlDTO
-from wren_engine.model.data_source import DataSource
-from wren_engine.model.validator import ValidateDTO, Validator
+from app.logger import log_dto
+from app.model.connector import Connector, to_json, QueryMySqlDTO
+from app.model.data_source import DataSource
+from app.model.validator import ValidateDTO, Validator
 
 router = APIRouter(prefix="/mysql", tags=["mysql"])
 

ibis-server/python/wren_engine/routers/v2/ibis/postgres.py → ibis-server/app/routers/v2/ibis/postgres.py

@@ -3,12 +3,12 @@
 from fastapi import APIRouter, Query, Response
 from fastapi.responses import JSONResponse
 
-from wren_engine.logger import log_dto
-from wren_engine.model.connector import Connector, QueryPostgresDTO, to_json
-from wren_engine.model.data_source import DataSource
-from wren_engine.model.metadata.dto import MetadataDTO, Table, Constraint
-from wren_engine.model.metadata.factory import MetadataFactory
-from wren_engine.model.validator import ValidateDTO, Validator
+from app.logger import log_dto
+from app.model.connector import Connector, QueryPostgresDTO, to_json
+from app.model.data_source import DataSource
+from app.model.metadata.dto import MetadataDTO, Table, Constraint
+from app.model.metadata.factory import MetadataFactory
+from app.model.validator import ValidateDTO, Validator
 
 router = APIRouter(prefix="/postgres", tags=["postgres"])
 

ibis-server/python/wren_engine/routers/v2/ibis/snowflake.py → ibis-server/app/routers/v2/ibis/snowflake.py

@@ -3,10 +3,10 @@
 from fastapi import APIRouter, Query, Response
 from fastapi.responses import JSONResponse
 
-from wren_engine.logger import log_dto
-from wren_engine.model.connector import Connector, QuerySnowflakeDTO, to_json
-from wren_engine.model.data_source import DataSource
-from wren_engine.model.validator import ValidateDTO, Validator
+from app.logger import log_dto
+from app.model.connector import Connector, QuerySnowflakeDTO, to_json
+from app.model.data_source import DataSource
+from app.model.validator import ValidateDTO, Validator
 
 router = APIRouter(prefix="/snowflake", tags=["snowflake"])
 

ibis-server/justfile

@@ -1,8 +1,21 @@
 default:
     @just --list --unsorted
 
+build-core:
+    cd ../wren-modeling-py && just build
+
+core-wheel-path := "../wren-modeling-py/target/wheels/wren_modeling_py-*.whl"
+
+install-core: build-core
+    # Using pip install to avoid adding wheel to pyproject.toml
+    poetry run pip install {{core-wheel-path}}
+
+update-core: build-core
+    poetry run pip install --force-reinstall {{core-wheel-path}}
+
 install:
     poetry install
+    just install-core
 
 pre-commit-install:
     poetry run pre-commit install