integration-tests-against-emulator.yaml

# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

on:
  schedule:
    - cron: "0 0 * * *"
  push:
    branches:
      - master
  pull_request:
name: integration-tests-against-emulator
jobs:
  integration-tests:
    runs-on: ubuntu-latest
    env:
      # set PostgreSQL related environment variables
      PGHOST: localhost
      PGPORT: 5432
      PGUSER: postgres
      PGDATABASE: postgres
      PGPASSWORD: postgres

      # set MySQL related environment variables
      DB_HOST: localhost
      MYSQLHOST: localhost
      MYSQLPORT: 3306
      MYSQLUSER: root
      MYSQLDATABASE: test_interleave_table_data
      MYSQLDB_FKACTION: test_foreign_key_action_data
      MYSQLPWD: root

      # set DynamoDB related environment variables
      AWS_ACCESS_KEY_ID: dummyId
      AWS_SECRET_ACCESS_KEY: dummyKey
      AWS_REGION: local
      DYNAMODB_ENDPOINT_OVERRIDE: http://localhost:8000

      # sql server envs
      # Password for connection as SA
      MSSQL_SA_PASSWORD: tCUE9c1&Ucp0

    services:
      spanner_emulator:
        image: gcr.io/cloud-spanner-emulator/emulator:1.5.28
        ports:
          - 9010:9010
          - 9020:9020
      postgres:
        image: postgres:9.6
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
        ports:
          - 5432:5432
        # needed because the postgres container does not provide a healthcheck
        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
      mariadb:
        image: mariadb:10.11
        env:
          MYSQL_ROOT_PASSWORD: root
          MYSQL_DATABASE: test
        ports:
          - 3306:3306
        # needed because the mysql container does not provide a healthcheck
        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3

      # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocalHistory.html
      # Dynamodb 1.17.0 has deprecated java web shell.
      dynamodb_emulator:
        image: amazon/dynamodb-local:2.5.3
        ports:
          - 8000:8000
          - 8001:8000
        env:
          AWS_ACCESS_KEY_ID: dummyId
          AWS_SECRET_ACCESS_KEY: dummyKey
          AWS_REGION: local
          DYNAMODB_ENDPOINT_OVERRIDE: http://localhost:8000
        options: --workdir /home/dynamodblocal --health-cmd "dynamodb describe-limits --endpoint-url http://dynamodb-local:8000 --region local" --health-interval 10s --health-timeout 5s --health-retries 5
      sqlserver:
        image: mcr.microsoft.com/mssql/server:2022-latest
        env:
          SA_PASSWORD: ${{env.MSSQL_SA_PASSWORD}}
          MSSQL_PID: Express
          ACCEPT_EULA: Y
        ports:
          - 1433:1433
        options:
          --health-cmd "/opt/mssql-tools18/bin/sqlcmd -U sa -P $SA_PASSWORD -Q 'select 1' -b -No -o /dev/null"
          --health-interval 10s --health-timeout 5s --health-retries 3
      oracle:
        image: oracleinanutshell/oracle-xe-11g
        ports:
          - 1521:1521

    steps:
      - uses: actions/checkout@v3

      # init a PostgresSQL database from the test_data
      - name: Install PostgreSQL 12 client required for loading .sql files
        run: |
          sudo apt-get update
          sudo apt-get -yq install postgresql-client
      - run: psql --version
      - run: psql -f test_data/pg_dump.test.out

      # init a MySQL database from the test_data
      - run: mysql --version
      - run: mysql -v -P 3306 --protocol=tcp -u root -proot test < test_data/mysqldump.test.out
      - run: mysql -v -P 3306 --protocol=tcp -u root -proot < test_data/mysql_interleave_dump.test.out
      - run: mysql -v -P 3306 --protocol=tcp -u root -proot < test_data/mysql_foreignkeyaction_dump.test.out

      # init sql server with test_data
      # since we use ubuntu-latest container, we should ensure that the path matches the latest from  https://packages.microsoft.com/config/ubuntu/
      # while its possible to infer the latest from the path in the run script, it will make the run section more complex and hard to maintian.
      - name: Install sqlcmd required for loading .sql files
        run: |
          curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
          curl https://packages.microsoft.com/config/ubuntu/24.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
          sudo apt-get update
          sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
          sudo apt-get install mssql-tools18 unixodbc-dev
          set -x
          ls /opt/mssql-tools18/bin/
          set +x
          echo 'export PATH="$PATH:/opt/mssql-tools18/bin"' >> ~/.bashrc
      - run: /opt/mssql-tools18/bin/sqlcmd -C -?
      - run: /opt/mssql-tools18/bin/sqlcmd -U sa -P ${MSSQL_SA_PASSWORD} -i test_data/sqlserver.test.out -C

      # sqlplus set up init oracle db.
      - name: Install sqlplus required for loading .sql files
        run: |
          sudo apt-get update
          sudo apt-get install -y libaio1t64 rpm2cpio cpio
          sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/libaio.so.1
          curl -O https://download.oracle.com/otn_software/linux/instantclient/2340000/oracle-instantclient-basic-23.4.0.24.05-1.el9.x86_64.rpm
          curl -O https://download.oracle.com/otn_software/linux/instantclient/2340000/oracle-instantclient-sqlplus-23.4.0.24.05-1.el9.x86_64.rpm
          rpm2cpio oracle-instantclient-basic-23.4.0.24.05-1.el9.x86_64.rpm | sudo cpio -idmv
          sudo cp -r usr/* /usr/
          rpm2cpio oracle-instantclient-sqlplus-23.4.0.24.05-1.el9.x86_64.rpm | sudo cpio -idmv
          sudo cp -r usr/* /usr/
          sudo sh -c 'echo /usr/lib/oracle/23/client64/lib/ > /etc/ld.so.conf.d/oracle.conf'
          sudo ldconfig
      - run: sqlplus SYS/oracle@127.0.0.1:1521/xe as sysdba @test_data/oracle.test.out
      # create a spanner instance
      - uses: actions/setup-python@v4
        with:
          python-version: '3.8'
      - uses: google-github-actions/setup-gcloud@v0
        with:
          version: "410.0.0"
      - run: gcloud info
      - run: gcloud config configurations create emulator
      - run: gcloud config set auth/disable_credentials true
      - run: gcloud config set project emulator-test-project
      - run: gcloud config set api_endpoint_overrides/spanner http://localhost:9020/
      - run: gcloud spanner instances create test-instance --config=emulator-config --description="Test Instance" --nodes=1

      - name: Set up Node.js
        uses: actions/setup-node@v2
        with:
          node-version: 16

      - name: Install Dependencies
        run: |
          cd ui
          npm ci

      - name: Start Local Server
        run:  |
          cd ui
          npm start &

      - name: Wait for Local Server to Start
        run: npx wait-on http://localhost:4200 -t 30000

      - name: Cypress run
        run: |
          cd ui
          npx cypress run

      - uses: actions/setup-go@v2
        with:
          go-version: "1.19"
      - run: go version
      - run: go test -v ./...
        env:
          SPANNER_EMULATOR_HOST: localhost:9010
          SPANNER_MIGRATION_TOOL_TESTS_GCLOUD_PROJECT_ID: emulator-test-project
          SPANNER_MIGRATION_TOOL_TESTS_GCLOUD_INSTANCE_ID: test-instance