diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..7de7872 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,16 @@ +# Changelog + +- All notable changes to this project will be documented in this file. +- The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +- We try to follow [semantic versioning](https://semver.org/). +- All changes have a **git tag** available, are build and published to GitHub packages as a docker image. + +## Unreleased + +## v1.1.0 +- First of all, even though this is a **major refactor**, the clientside API is still the same. **There should be no breaking changes!** +- The main goal of this release is to bring IntegreSQL's performance on par with our previous Node.js implementation. Specifially we wanted to eliminate any long-running mutex locks and make the codebase more maintainable and easier to extend in the future. +- ... + +## v1.0.0 +- Initial release May 2020 diff --git a/Dockerfile b/Dockerfile index 8823c0f..b0907ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ # --- https://hub.docker.com/_/golang # --- https://github.com/microsoft/vscode-remote-try-go/blob/master/.devcontainer/Dockerfile ### ----------------------- -FROM golang:1.21.5-bullseye AS development +FROM golang:1.21.6-bullseye AS development # Avoid warnings by switching to noninteractive ENV DEBIAN_FRONTEND=noninteractive @@ -128,7 +128,7 @@ RUN mkdir -p /tmp/watchexec \ RUN mkdir -p /tmp/yq \ && cd /tmp/yq \ && ARCH="$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/)" \ - && wget "https://github.com/mikefarah/yq/releases/download/v4.30.5/yq_linux_${ARCH}.tar.gz" \ + && wget "https://github.com/mikefarah/yq/releases/download/v4.40.5/yq_linux_${ARCH}.tar.gz" \ && tar xzf "yq_linux_${ARCH}.tar.gz" \ && cp "yq_linux_${ARCH}" /usr/local/bin/yq \ && rm -rf /tmp/yq diff --git a/README.md b/README.md index 66056fd..a6e4387 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,16 @@ Do your engineers a favour by allowing them to write fast executing, parallel an [![](https://img.shields.io/docker/image-size/allaboutapps/integresql)](https://hub.docker.com/r/allaboutapps/integresql) [![](https://img.shields.io/docker/pulls/allaboutapps/integresql)](https://hub.docker.com/r/allaboutapps/integresql) [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/allaboutapps/integresql)](https://hub.docker.com/r/allaboutapps/integresql) [![](https://goreportcard.com/badge/github.com/allaboutapps/integresql)](https://goreportcard.com/report/github.com/allaboutapps/integresql) ![](https://github.com/allaboutapps/integresql/workflows/build/badge.svg?branch=master) - [IntegreSQL](#integresql) + - [Integrate by client lib](#integrate-by-client-lib) + - [Integrate by RESTful JSON calls](#integrate-by-restful-json-calls) + - [Demo](#demo) + - [Install](#install) + - [Install using Docker (preferred)](#install-using-docker-preferred) + - [Install locally](#install-locally) + - [Configuration](#configuration) + - [Usage](#usage) + - [Run using Docker (preferred)](#run-using-docker-preferred) + - [Run locally](#run-locally) - [Background](#background) - [Approach 0: Leaking database mutations for subsequent tests](#approach-0-leaking-database-mutations-for-subsequent-tests) - [Approach 1: Isolating by resetting](#approach-1-isolating-by-resetting) @@ -18,16 +28,6 @@ Do your engineers a favour by allowing them to write fast executing, parallel an - [Approach 3c benchmark 1: Baseline](#approach-3c-benchmark-1-baseline) - [Approach 3c benchmark 2: Small project](#approach-3c-benchmark-2-small-project) - [Final approach: IntegreSQL](#final-approach-integresql) - - [Integrate by client lib](#integrate-by-client-lib) - - [Integrate by RESTful JSON calls](#integrate-by-restful-json-calls) - - [Demo](#demo) - - [Install](#install) - - [Install using Docker (preferred)](#install-using-docker-preferred) - - [Install locally](#install-locally) - - [Configuration](#configuration) - - [Usage](#usage) - - [Run using Docker (preferred)](#run-using-docker-preferred) - - [Run locally](#run-locally) - [Contributing](#contributing) - [Development setup](#development-setup) - [Development quickstart](#development-quickstart) @@ -35,6 +35,151 @@ Do your engineers a favour by allowing them to write fast executing, parallel an - [Previous maintainers](#previous-maintainers) - [License](#license) +#### Integrate by client lib + +The flow above might look intimidating at first glance, but trust us, it's simple to integrate especially if there is already an client library available for your specific language. We currently have those: + +* Go: [integresql-client-go](https://github.com/allaboutapps/integresql-client-go) by [Nick Müller - @MorpheusXAUT](https://github.com/MorpheusXAUT) +* Python: [integresql-client-python](https://github.com/msztolcman/integresql-client-python) by [Marcin Sztolcman - @msztolcman](https://github.com/msztolcman) +* .NET: [IntegreSQL.EF](https://github.com/mcctomsk/IntegreSql.EF) by [Artur Drobinskiy - @Shaddix](https://github.com/Shaddix) +* JavaScript/TypeScript: [@devoxa/integresql-client](https://github.com/devoxa/integresql-client) by [Devoxa - @devoxa](https://github.com/devoxa) +* ... *Add your link here and make a PR* + +#### Integrate by RESTful JSON calls + +A really good starting point to write your own integresql-client for a specific language can be found [here (go code)](https://github.com/allaboutapps/integresql-client-go/blob/master/client.go) and [here (godoc)](https://pkg.go.dev/github.com/allaboutapps/integresql-client-go?tab=doc). It's just RESTful JSON after all. + +#### Demo + +If you want to take a look on how we integrate IntegreSQL - 🤭 - please just try our [go-starter](https://github.com/allaboutapps/go-starter) project or take a look at our [test_database setup code](https://github.com/allaboutapps/go-starter/blob/master/internal/test/test_database.go). + +## Install + +### Install using Docker (preferred) + +A minimal Docker image containing a pre-built `IntegreSQL` executable is available at [Docker Hub](https://hub.docker.com/r/allaboutapps/integresql). + +```bash +docker pull allaboutapps/integresql +``` + +### Install locally + +Installing `IntegreSQL` locally requires a working [Go](https://golang.org/dl/) (1.14 or above) environment. Install the `IntegreSQL` executable to your Go bin folder: + +```bash +go get github.com/allaboutapps/integresql/cmd/server +``` + +## Configuration + +`IntegreSQL` requires little configuration, all of which has to be provided via environment variables (due to the intended usage in a Docker environment). The following settings are available: + +| Description | Environment variable | Default | Required | +| ----------------------------------------------------------------- | ------------------------------------- | -------------------- | -------- | +| IntegreSQL: listen address (defaults to all if empty) | `INTEGRESQL_ADDRESS` | `""` | | +| IntegreSQL: port | `INTEGRESQL_PORT` | `5000` | | +| PostgreSQL: host | `INTEGRESQL_PGHOST`, `PGHOST` | `"127.0.0.1"` | Yes | +| PostgreSQL: port | `INTEGRESQL_PGPORT`, `PGPORT` | `5432` | | +| PostgreSQL: username | `INTEGRESQL_PGUSER`, `PGUSER`, `USER` | `"postgres"` | Yes | +| PostgreSQL: password | `INTEGRESQL_PGPASSWORD`, `PGPASSWORD` | `""` | Yes | +| PostgreSQL: database for manager | `INTEGRESQL_PGDATABASE` | `"postgres"` | | +| PostgreSQL: template database to use | `INTEGRESQL_ROOT_TEMPLATE` | `"template0"` | | +| Managed databases: prefix | `INTEGRESQL_DB_PREFIX` | `"integresql"` | | +| Managed *template* databases: prefix `integresql_template_` | `INTEGRESQL_TEMPLATE_DB_PREFIX` | `"template"` | | +| Managed *test* databases: prefix `integresql_test__` | `INTEGRESQL_TEST_DB_PREFIX` | `"test"` | | +| Managed *test* databases: username | `INTEGRESQL_TEST_PGUSER` | PostgreSQL: username | | +| Managed *test* databases: password | `INTEGRESQL_TEST_PGPASSWORD` | PostgreSQL: password | | +| Managed *test* databases: minimal test pool size | `INTEGRESQL_TEST_INITIAL_POOL_SIZE` | `10` | | +| Managed *test* databases: maximal test pool size | `INTEGRESQL_TEST_MAX_POOL_SIZE` | `500` | | + + +## Usage + +### Run using Docker (preferred) + +Simply start the `IntegreSQL` [Docker](https://docs.docker.com/install/) (19.03 or above) container, provide the required environment variables and expose the server port: + +```bash +docker run -d --name integresql -e INTEGRESQL_PORT=5000 -p 5000:5000 allaboutapps/integresql +``` + +`IntegreSQL` can also be included in your project via [Docker Compose](https://docs.docker.com/compose/install/) (1.25 or above): + +```yaml +version: "3.4" +services: + + # Your main service image + service: + depends_on: + - postgres + - integresql + environment: + PGDATABASE: &PGDATABASE "development" + PGUSER: &PGUSER "dbuser" + PGPASSWORD: &PGPASSWORD "9bed16f749d74a3c8bfbced18a7647f5" + PGHOST: &PGHOST "postgres" + PGPORT: &PGPORT "5432" + PGSSLMODE: &PGSSLMODE "disable" + + # optional: env for integresql client testing + # see https://github.com/allaboutapps/integresql-client-go + # INTEGRESQL_CLIENT_BASE_URL: "http://integresql:5000/api" + + # [...] additional main service setup + + integresql: + image: allaboutapps/integresql:1.0.0 + ports: + - "5000:5000" + depends_on: + - postgres + environment: + PGHOST: *PGHOST + PGUSER: *PGUSER + PGPASSWORD: *PGPASSWORD + + postgres: + image: postgres:12.2-alpine # should be the same version as used live + # ATTENTION + # fsync=off, synchronous_commit=off and full_page_writes=off + # gives us a major speed up during local development and testing (~30%), + # however you should NEVER use these settings in PRODUCTION unless + # you want to have CORRUPTED data. + # DO NOT COPY/PASTE THIS BLINDLY. + # YOU HAVE BEEN WARNED. + # Apply some performance improvements to pg as these guarantees are not needed while running locally + command: "postgres -c 'shared_buffers=128MB' -c 'fsync=off' -c 'synchronous_commit=off' -c 'full_page_writes=off' -c 'max_connections=100' -c 'client_min_messages=warning'" + expose: + - "5432" + ports: + - "5432:5432" + environment: + POSTGRES_DB: *PGDATABASE + POSTGRES_USER: *PGUSER + POSTGRES_PASSWORD: *PGPASSWORD + volumes: + - pgvolume:/var/lib/postgresql/data + +volumes: + pgvolume: # declare a named volume to persist DB data +``` + +You may also refer to our [go-starter `docker-compose.yml`](https://github.com/allaboutapps/go-starter/blob/master/docker-compose.yml). + +### Run locally + +Running the `IntegreSQL` server locally requires configuration via exported environment variables (see below): + +```bash +export INTEGRESQL_PORT=5000 +export PGHOST=127.0.0.1 +export PGUSER=test +export PGPASSWORD=testpass +integresql +``` + ## Background We came a long way to realize that something just did not feel right with our PostgreSQL integration testing strategies. @@ -263,150 +408,6 @@ Our flow now finally changed to this: * ... * Subsequent 1..n test runners start/end in parallel and reuse the above logic -#### Integrate by client lib - -The flow above might look intimidating at first glance, but trust us, it's simple to integrate especially if there is already an client library available for your specific language. We currently have those: - -* Go: [integresql-client-go](https://github.com/allaboutapps/integresql-client-go) by [Nick Müller - @MorpheusXAUT](https://github.com/MorpheusXAUT) -* Python: [integresql-client-python](https://github.com/msztolcman/integresql-client-python) by [Marcin Sztolcman - @msztolcman](https://github.com/msztolcman) -* .NET: [IntegreSQL.EF](https://github.com/mcctomsk/IntegreSql.EF) by [Artur Drobinskiy - @Shaddix](https://github.com/Shaddix) -* JavaScript/TypeScript: [@devoxa/integresql-client](https://github.com/devoxa/integresql-client) by [Devoxa - @devoxa](https://github.com/devoxa) -* ... *Add your link here and make a PR* - -#### Integrate by RESTful JSON calls - -A really good starting point to write your own integresql-client for a specific language can be found [here (go code)](https://github.com/allaboutapps/integresql-client-go/blob/master/client.go) and [here (godoc)](https://pkg.go.dev/github.com/allaboutapps/integresql-client-go?tab=doc). It's just RESTful JSON after all. - -#### Demo - -If you want to take a look on how we integrate IntegreSQL - 🤭 - please just try our [go-starter](https://github.com/allaboutapps/go-starter) project or take a look at our [testing setup code](https://github.com/allaboutapps/go-starter/blob/master/internal/test/testing.go). - -## Install - -### Install using Docker (preferred) - -A minimal Docker image containing a pre-built `IntegreSQL` executable is available at [Docker Hub](https://hub.docker.com/r/allaboutapps/integresql). - -```bash -docker pull allaboutapps/integresql -``` - -### Install locally - -Installing `IntegreSQL` locally requires a working [Go](https://golang.org/dl/) (1.14 or above) environment. Install the `IntegreSQL` executable to your Go bin folder: - -```bash -go get github.com/allaboutapps/integresql/cmd/server -``` - -## Configuration - -`IntegreSQL` requires little configuration, all of which has to be provided via environment variables (due to the intended usage in a Docker environment). The following settings are available: - -| Description | Environment variable | Default | Required | -| ----------------------------------------------------------------- | ------------------------------------- | -------------------- | -------- | -| IntegreSQL: listen address (defaults to all if empty) | `INTEGRESQL_ADDRESS` | `""` | | -| IntegreSQL: port | `INTEGRESQL_PORT` | `5000` | | -| PostgreSQL: host | `INTEGRESQL_PGHOST`, `PGHOST` | `"127.0.0.1"` | Yes | -| PostgreSQL: port | `INTEGRESQL_PGPORT`, `PGPORT` | `5432` | | -| PostgreSQL: username | `INTEGRESQL_PGUSER`, `PGUSER`, `USER` | `"postgres"` | Yes | -| PostgreSQL: password | `INTEGRESQL_PGPASSWORD`, `PGPASSWORD` | `""` | Yes | -| PostgreSQL: database for manager | `INTEGRESQL_PGDATABASE` | `"postgres"` | | -| PostgreSQL: template database to use | `INTEGRESQL_ROOT_TEMPLATE` | `"template0"` | | -| Managed databases: prefix | `INTEGRESQL_DB_PREFIX` | `"integresql"` | | -| Managed *template* databases: prefix `integresql_template_` | `INTEGRESQL_TEMPLATE_DB_PREFIX` | `"template"` | | -| Managed *test* databases: prefix `integresql_test__` | `INTEGRESQL_TEST_DB_PREFIX` | `"test"` | | -| Managed *test* databases: username | `INTEGRESQL_TEST_PGUSER` | PostgreSQL: username | | -| Managed *test* databases: password | `INTEGRESQL_TEST_PGPASSWORD` | PostgreSQL: password | | -| Managed *test* databases: minimal test pool size | `INTEGRESQL_TEST_INITIAL_POOL_SIZE` | `10` | | -| Managed *test* databases: maximal test pool size | `INTEGRESQL_TEST_MAX_POOL_SIZE` | `500` | | - - -## Usage - -### Run using Docker (preferred) - -Simply start the `IntegreSQL` [Docker](https://docs.docker.com/install/) (19.03 or above) container, provide the required environment variables and expose the server port: - -```bash -docker run -d --name integresql -e INTEGRESQL_PORT=5000 -p 5000:5000 allaboutapps/integresql -``` - -`IntegreSQL` can also be included in your project via [Docker Compose](https://docs.docker.com/compose/install/) (1.25 or above): - -```yaml -version: "3.4" -services: - - # Your main service image - service: - depends_on: - - postgres - - integresql - environment: - PGDATABASE: &PGDATABASE "development" - PGUSER: &PGUSER "dbuser" - PGPASSWORD: &PGPASSWORD "9bed16f749d74a3c8bfbced18a7647f5" - PGHOST: &PGHOST "postgres" - PGPORT: &PGPORT "5432" - PGSSLMODE: &PGSSLMODE "disable" - - # optional: env for integresql client testing - # see https://github.com/allaboutapps/integresql-client-go - # INTEGRESQL_CLIENT_BASE_URL: "http://integresql:5000/api" - - # [...] additional main service setup - - integresql: - image: allaboutapps/integresql:1.0.0 - ports: - - "5000:5000" - depends_on: - - postgres - environment: - PGHOST: *PGHOST - PGUSER: *PGUSER - PGPASSWORD: *PGPASSWORD - - postgres: - image: postgres:12.2-alpine # should be the same version as used live - # ATTENTION - # fsync=off, synchronous_commit=off and full_page_writes=off - # gives us a major speed up during local development and testing (~30%), - # however you should NEVER use these settings in PRODUCTION unless - # you want to have CORRUPTED data. - # DO NOT COPY/PASTE THIS BLINDLY. - # YOU HAVE BEEN WARNED. - # Apply some performance improvements to pg as these guarantees are not needed while running locally - command: "postgres -c 'shared_buffers=128MB' -c 'fsync=off' -c 'synchronous_commit=off' -c 'full_page_writes=off' -c 'max_connections=100' -c 'client_min_messages=warning'" - expose: - - "5432" - ports: - - "5432:5432" - environment: - POSTGRES_DB: *PGDATABASE - POSTGRES_USER: *PGUSER - POSTGRES_PASSWORD: *PGPASSWORD - volumes: - - pgvolume:/var/lib/postgresql/data - -volumes: - pgvolume: # declare a named volume to persist DB data -``` - -You may also refer to our [go-starter `docker-compose.yml`](https://github.com/allaboutapps/go-starter/blob/master/docker-compose.yml). - -### Run locally - -Running the `IntegreSQL` server locally requires configuration via exported environment variables (see below): - -```bash -export INTEGRESQL_PORT=5000 -export PGHOST=127.0.0.1 -export PGUSER=test -export PGPASSWORD=testpass -integresql -``` ## Contributing @@ -458,9 +459,9 @@ integresql - [Mario Ranftl - @majodev](https://github.com/majodev) -## Previous maintainers +### Previous maintainers -- [Anna Jankowska - @anjankow](https://github.com/anjankow) +- [Anna - @anjankow](https://github.com/anjankow) - [Nick Müller - @MorpheusXAUT](https://github.com/MorpheusXAUT) ## License diff --git a/go.mod b/go.mod index 0e7dfcb..85a522b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/allaboutapps/integresql go 1.20 require ( - github.com/google/uuid v1.5.0 + github.com/google/uuid v1.6.0 github.com/labstack/echo/v4 v4.11.4 github.com/lib/pq v1.10.9 github.com/rs/zerolog v1.31.0 @@ -21,8 +21,8 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/crypto v0.18.0 // indirect + golang.org/x/net v0.20.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect diff --git a/go.sum b/go.sum index 18140f5..2c1be9a 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -41,10 +41,10 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=