From b4c87f2615ab3acdb941510f2e457d19380c39f4 Mon Sep 17 00:00:00 2001 From: Xiaoying Wang Date: Mon, 22 Apr 2024 14:46:28 -0700 Subject: [PATCH] remove mssql from ci --- .github/workflows/ci.yml | 40 +++--- Justfile | 2 +- .../connectorx/tests/test_mssql.py | 119 +++++++++++++----- 3 files changed, 106 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fcfd72a4f..06633fbcb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -165,18 +165,18 @@ jobs: --health-interval 10s --health-timeout 10s --health-retries 5 - mssql: - image: mcr.microsoft.com/mssql/server:2019-latest - env: - ACCEPT_EULA: y - SA_PASSWORD: mssql!Password - ports: - - 1433:1433 - options: >- - --health-cmd "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"$SA_PASSWORD\" -Q 'SELECT 1' || exit 1" - --health-interval 10s - --health-timeout 5s - --health-retries 20 + # mssql: + # image: mcr.microsoft.com/mssql/server:2019-latest + # env: + # ACCEPT_EULA: y + # SA_PASSWORD: mssql!Password + # ports: + # - 1433:1433 + # options: >- + # --health-cmd "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"$SA_PASSWORD\" -Q 'SELECT 1' || exit 1" + # --health-interval 10s + # --health-timeout 5s + # --health-retries 20 steps: - uses: actions/checkout@v4 @@ -190,8 +190,8 @@ jobs: apt-get install -y python${{ matrix.python-version }} python${{ matrix.python-version }}-distutils wget curl postgresql-client build-essential pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client libmysqlclient-dev libicu66 libkrb5-dev libclang-dev ln -s `which python${{ matrix.python-version }}` /usr/bin/python wget -qO- https://bootstrap.pypa.io/get-pip.py | python - python -m pip install mssql-cli six - python -m pip install cli-helpers==2.3.1 + # python -m pip install mssql-cli + # python -m pip install cli-helpers==2.2.0 env: DEBIAN_FRONTEND: noninteractive @@ -232,11 +232,11 @@ jobs: MYSQL_DB: mysql MYSQL_USER: root MYSQL_PASSWORD: mysql - MSSQL_HOST: mssql - MSSQL_PORT: 1433 - MSSQL_DB: tempdb - MSSQL_USER: sa - MSSQL_PASSWORD: mssql!Password + # MSSQL_HOST: mssql + # MSSQL_PORT: 1433 + # MSSQL_DB: tempdb + # MSSQL_USER: sa + # MSSQL_PASSWORD: mssql!Password - name: Clippy linting uses: actions-rs/cargo@v1 @@ -259,7 +259,7 @@ jobs: POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres" SQLITE_URL: "sqlite:///tmp/test.db" MYSQL_URL: "mysql://root:mysql@mysql:3306/mysql" - MSSQL_URL: "mssql://sa:mssql!Password@mssql:1433/tempdb" + # MSSQL_URL: "mssql://sa:mssql!Password@mssql:1433/tempdb" DB1: "postgresql://postgres:postgres@postgres:5432/postgres" DB2: "postgresql://postgres:postgres@postgres:5432/postgres" FED_CONFIG_PATH: ${{ github.workspace }}/.github/config diff --git a/Justfile b/Justfile index f188c51a7..9d617ddb1 100644 --- a/Justfile +++ b/Justfile @@ -54,10 +54,10 @@ seed-db: psql $POSTGRES_URL -f scripts/postgres.sql sqlite3 ${SQLITE_URL#sqlite://} < scripts/sqlite.sql mysql --protocol tcp -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB < scripts/mysql.sql - mssql-cli -S$MSSQL_HOST -U$MSSQL_USER -P$MSSQL_PASSWORD -d$MSSQL_DB -i scripts/mssql.sql # dbs not included in ci seed-db-more: + mssql-cli -S$MSSQL_HOST -U$MSSQL_USER -P$MSSQL_PASSWORD -d$MSSQL_DB -i scripts/mssql.sql mysql --protocol tcp -h$CLICKHOUSE_HOST -P$CLICKHOUSE_PORT -u$CLICKHOUSE_USER -p$CLICKHOUSE_PASSWORD $CLICKHOUSE_DB < scripts/clickhouse.sql psql $REDSHIFT_URL -f scripts/redshift.sql ORACLE_URL_SCRIPT=`echo ${ORACLE_URL#oracle://} | sed "s/:/\//"` diff --git a/connectorx-python/connectorx/tests/test_mssql.py b/connectorx-python/connectorx/tests/test_mssql.py index 6c9403396..43d5095cc 100644 --- a/connectorx-python/connectorx/tests/test_mssql.py +++ b/connectorx-python/connectorx/tests/test_mssql.py @@ -16,12 +16,17 @@ def mssql_url() -> str: @pytest.mark.xfail -def test_on_non_select(mssql_url: str) -> None: +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) +def test_mssql_on_non_select(mssql_url: str) -> None: query = "CREATE TABLE non_select(id INTEGER NOT NULL)" df = read_sql(mssql_url, query) - -def test_aggregation(mssql_url: str) -> None: +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) +def test_mssql_aggregation(mssql_url: str) -> None: query = ( "SELECT test_bool, SUM(test_float) as sum FROM test_table GROUP BY test_bool" ) @@ -35,8 +40,10 @@ def test_aggregation(mssql_url: str) -> None: ) assert_frame_equal(df, expected, check_names=True) - -def test_partition_on_aggregation(mssql_url: str) -> None: +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) +def test_mssql_partition_on_aggregation(mssql_url: str) -> None: query = ( "SELECT test_bool, SUM(test_int) AS test_int FROM test_table GROUP BY test_bool" ) @@ -51,8 +58,10 @@ def test_partition_on_aggregation(mssql_url: str) -> None: df.sort_values(by="test_int", inplace=True, ignore_index=True) assert_frame_equal(df, expected, check_names=True) - -def test_aggregation2(mssql_url: str) -> None: +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) +def test_mssql_aggregation2(mssql_url: str) -> None: query = "select DISTINCT(test_bool) from test_table" df = read_sql(mssql_url, query) expected = pd.DataFrame( @@ -63,8 +72,10 @@ def test_aggregation2(mssql_url: str) -> None: ) assert_frame_equal(df, expected, check_names=True) - -def test_partition_on_aggregation2(mssql_url: str) -> None: +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) +def test_mssql_partition_on_aggregation2(mssql_url: str) -> None: query = "select MAX(test_int) as max, MIN(test_int) as min from test_table" df = read_sql(mssql_url, query, partition_on="max", partition_num=2) expected = pd.DataFrame( @@ -76,7 +87,9 @@ def test_partition_on_aggregation2(mssql_url: str) -> None: ) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_udf(mssql_url: str) -> None: query = ( "SELECT dbo.increment(test_int) AS test_int FROM test_table ORDER BY test_int" @@ -91,8 +104,10 @@ def test_mssql_udf(mssql_url: str) -> None: df.sort_values(by="test_int", inplace=True, ignore_index=True) assert_frame_equal(df, expected, check_names=True) - -def test_manual_partition(mssql_url: str) -> None: +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) +def test_mssql_manual_partition(mssql_url: str) -> None: queries = [ "SELECT * FROM test_table WHERE test_int < 2", "SELECT * FROM test_table WHERE test_int >= 2", @@ -117,7 +132,9 @@ def test_manual_partition(mssql_url: str) -> None: df.sort_values(by="test_int", inplace=True, ignore_index=True) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_without_partition(mssql_url: str) -> None: query = "SELECT * FROM test_table" df = read_sql(mssql_url, query) @@ -137,7 +154,9 @@ def test_mssql_without_partition(mssql_url: str) -> None: ) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_limit_without_partition(mssql_url: str) -> None: query = "SELECT top 3 * FROM test_table" df = read_sql(mssql_url, query) @@ -153,7 +172,9 @@ def test_mssql_limit_without_partition(mssql_url: str) -> None: ) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_limit_large_without_partition(mssql_url: str) -> None: query = "SELECT top 10 * FROM test_table" df = read_sql(mssql_url, query) @@ -173,7 +194,9 @@ def test_mssql_limit_large_without_partition(mssql_url: str) -> None: ) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_with_partition(mssql_url: str) -> None: query = "SELECT * FROM test_table" df = read_sql( @@ -200,7 +223,9 @@ def test_mssql_with_partition(mssql_url: str) -> None: df.sort_values(by="test_int", inplace=True, ignore_index=True) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_limit_with_partition(mssql_url: str) -> None: query = "SELECT top 3 * FROM test_table" df = read_sql( @@ -223,7 +248,9 @@ def test_mssql_limit_with_partition(mssql_url: str) -> None: df.sort_values(by="test_int", inplace=True, ignore_index=True) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_limit_large_with_partition(mssql_url: str) -> None: query = "SELECT top 10 * FROM test_table" df = read_sql( @@ -250,7 +277,9 @@ def test_mssql_limit_large_with_partition(mssql_url: str) -> None: df.sort_values(by="test_int", inplace=True, ignore_index=True) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_with_partition_without_partition_range(mssql_url: str) -> None: query = "SELECT * FROM test_table where test_float > 3" df = read_sql( @@ -273,7 +302,9 @@ def test_mssql_with_partition_without_partition_range(mssql_url: str) -> None: df.sort_values(by="test_int", inplace=True, ignore_index=True) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_with_partition_and_selection(mssql_url: str) -> None: query = "SELECT * FROM test_table WHERE 1 = 3 OR 2 = 2" df = read_sql( @@ -300,7 +331,9 @@ def test_mssql_with_partition_and_selection(mssql_url: str) -> None: df.sort_values(by="test_int", inplace=True, ignore_index=True) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_with_partition_and_projection(mssql_url: str) -> None: query = "SELECT test_int, test_float, test_str FROM test_table" df = read_sql( @@ -323,7 +356,9 @@ def test_mssql_with_partition_and_projection(mssql_url: str) -> None: df.sort_values(by="test_int", inplace=True, ignore_index=True) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_with_partition_and_spja(mssql_url: str) -> None: query = """ SELECT test_bool, AVG(test_float) AS avg, SUM(test_int) AS sum @@ -344,8 +379,10 @@ def test_mssql_with_partition_and_spja(mssql_url: str) -> None: df = df.sort_values("sum").reset_index(drop=True) assert_frame_equal(df, expected, check_names=True) - -def test_empty_result(mssql_url: str) -> None: +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) +def test_mssql_empty_result(mssql_url: str) -> None: query = "SELECT * FROM test_table where test_int < -100" df = read_sql(mssql_url, query) expected = pd.DataFrame( @@ -359,8 +396,10 @@ def test_empty_result(mssql_url: str) -> None: ) assert_frame_equal(df, expected, check_names=True) - -def test_empty_result_on_partition(mssql_url: str) -> None: +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) +def test_mssql_empty_result_on_partition(mssql_url: str) -> None: query = "SELECT * FROM test_table where test_int < -100" df = read_sql(mssql_url, query, partition_on="test_int", partition_num=3) expected = pd.DataFrame( @@ -374,8 +413,10 @@ def test_empty_result_on_partition(mssql_url: str) -> None: ) assert_frame_equal(df, expected, check_names=True) - -def test_empty_result_on_some_partition(mssql_url: str) -> None: +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) +def test_mssql_empty_result_on_some_partition(mssql_url: str) -> None: query = "SELECT * FROM test_table where test_int < 1" df = read_sql(mssql_url, query, partition_on="test_int", partition_num=3) expected = pd.DataFrame( @@ -389,7 +430,9 @@ def test_empty_result_on_some_partition(mssql_url: str) -> None: ) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_types(mssql_url: str) -> None: query = "SELECT * FROM test_types" df = read_sql(mssql_url, query) @@ -454,7 +497,9 @@ def test_mssql_types(mssql_url: str) -> None: ) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_unicode(mssql_url: str) -> None: query = "SELECT test_hello FROM test_str where 1 <= id and id <= 4" df = read_sql(mssql_url, query) @@ -468,7 +513,9 @@ def test_mssql_unicode(mssql_url: str) -> None: ) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_cte(mssql_url: str) -> None: query = "with test_cte (test_int, test_str) as (select test_int, test_str from test_table where test_float > 0) select test_int, test_str from test_cte" df = read_sql(mssql_url, query, partition_on="test_int", partition_num=3) @@ -482,7 +529,9 @@ def test_mssql_cte(mssql_url: str) -> None: ) assert_frame_equal(df, expected, check_names=True) - +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) def test_mssql_offset(mssql_url: str) -> None: query = "SELECT * FROM (SELECT * FROM test_table) AS _ ORDER BY(SELECT NULL) OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY" df = read_sql(mssql_url, query) @@ -497,6 +546,8 @@ def test_mssql_offset(mssql_url: str) -> None: ) assert_frame_equal(df, expected, check_names=True) - -def test_connection_url(mssql_url: str) -> None: +@pytest.mark.skipif( + not os.environ.get("MSSQL_URL"), reason="Test mssql only when `MSSQL_URL` is set" +) +def test_mssql_connection_url(mssql_url: str) -> None: test_mssql_offset(ConnectionUrl(mssql_url)) \ No newline at end of file