fix: ensure recoverability from cluster state changes #950
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
--- | |
# @see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions | |
# | |
# Services feature of GitHub Actions isn't fit for our purposes for testing. | |
# We cannot overwrite arguments of ENTRYPOINT. | |
# @see https://docs.docker.com/engine/reference/commandline/create/#options | |
# | |
# @see https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 | |
name: Test | |
on: | |
push: | |
branches: | |
- master | |
pull_request: | |
branches: | |
- master | |
defaults: | |
run: | |
shell: bash | |
jobs: | |
main: | |
name: Main | |
timeout-minutes: 10 | |
runs-on: ubuntu-latest | |
strategy: | |
fail-fast: false | |
max-parallel: 10 | |
matrix: | |
include: | |
- {redis: '7.2', ruby: '3.3'} | |
- {redis: '7.2', ruby: '3.3', compose: compose.ssl.yaml} | |
- {redis: '7.2', ruby: '3.3', driver: 'hiredis'} | |
- {redis: '7.2', ruby: '3.3', driver: 'hiredis', compose: compose.ssl.yaml} | |
- {redis: '7.2', ruby: '3.3', compose: compose.replica.yaml, replica: '2'} | |
- {redis: '8', ruby: '3.3', compose: compose.valkey.yaml, replica: '2'} | |
- {redis: '7.2', ruby: '3.2', compose: compose.auth.yaml} | |
- {task: test_cluster_broken, restart: 'no', startup: '6'} | |
- {redis: '7.0', ruby: '3.1'} | |
- {redis: '6.2', ruby: '3.0'} | |
- {redis: '5.0', ruby: '2.7'} | |
- {task: test_cluster_scale, pattern: 'Single', compose: compose.scale.yaml, startup: '8'} | |
- {task: test_cluster_scale, pattern: 'Pipeline', compose: compose.scale.yaml, startup: '8'} | |
- {task: test_cluster_scale, pattern: 'Transaction', compose: compose.scale.yaml, startup: '8'} | |
- {task: test_cluster_scale, pattern: 'PubSub', compose: compose.scale.yaml, startup: '8'} | |
- {ruby: 'jruby'} | |
- {ruby: 'truffleruby'} | |
- {task: test_cluster_state, pattern: 'PrimaryOnly', compose: compose.valkey.yaml, redis: '8', replica: '2', startup: '9'} | |
- {task: test_cluster_state, pattern: 'Pooled', compose: compose.valkey.yaml, redis: '8', replica: '2', startup: '9'} | |
- {task: test_cluster_state, pattern: 'ScaleReadRandom', compose: compose.valkey.yaml, redis: '8', replica: '2', startup: '9'} | |
- {task: test_cluster_state, pattern: 'ScaleReadRandomWithPrimary', compose: compose.valkey.yaml, redis: '8', replica: '2', startup: '9'} | |
- {task: test_cluster_state, pattern: 'ScaleReadLatency', compose: compose.valkey.yaml, redis: '8', replica: '2', startup: '9'} | |
env: | |
REDIS_VERSION: ${{ matrix.redis || '7.2' }} | |
DOCKER_COMPOSE_FILE: ${{ matrix.compose || 'compose.yaml' }} | |
REDIS_CONNECTION_DRIVER: ${{ matrix.driver || 'ruby' }} | |
REDIS_REPLICA_SIZE: ${{ matrix.replica || '1' }} | |
RESTART_POLICY: ${{ matrix.restart || 'always' }} | |
REDIS_CLIENT_MAX_STARTUP_SAMPLE: ${{ matrix.startup || '3' }} | |
TEST_CLASS_PATTERN: ${{ matrix.pattern || '' }} | |
steps: | |
- name: Check out code | |
uses: actions/checkout@v4 | |
- name: Set up Ruby | |
uses: ruby/setup-ruby@v1 | |
with: | |
ruby-version: ${{ matrix.ruby || '3.3' }} | |
bundler-cache: true | |
- name: Pull Docker images | |
run: docker compose -f $DOCKER_COMPOSE_FILE pull | |
- name: Run containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE up -d | |
- name: Wait for Redis cluster to be ready | |
run: bundle exec rake wait | |
- name: Print containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE ps | |
- name: Run minitest | |
run: bundle exec rake ${{ matrix.task || 'test' }} | |
- name: Stop containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE down || true | |
nat-ted-env: | |
name: NAT-ted Environments | |
timeout-minutes: 5 | |
runs-on: ubuntu-latest | |
env: | |
REDIS_VERSION: '7.2' | |
DOCKER_COMPOSE_FILE: 'compose.nat.yaml' | |
steps: | |
- name: Check out code | |
uses: actions/checkout@v4 | |
- name: Set up Ruby | |
uses: ruby/setup-ruby@v1 | |
with: | |
ruby-version: '3.3' | |
bundler-cache: true | |
- name: Get IP address of host | |
run: | | |
host_ip_addr=$(ip a | grep eth0 | grep inet | awk '{print $2}' | cut -d'/' -f1) | |
echo "HOST_IP_ADDR=$host_ip_addr" >> $GITHUB_ENV | |
- name: Pull Docker images | |
run: docker compose -f $DOCKER_COMPOSE_FILE pull | |
- name: Run containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE up -d | |
env: | |
HOST_ADDR: ${{ env.HOST_IP_ADDR }} | |
- name: Wait for nodes to be ready | |
run: | | |
node_cnt=$(docker compose -f $DOCKER_COMPOSE_FILE ps --format json | jq -s 'length') | |
i=0 | |
while : | |
do | |
if [[ $i -gt $MAX_ATTEMPTS ]] | |
then | |
echo "Max attempts exceeded: $i times" | |
exit 1 | |
fi | |
healthy_cnt=$(docker compose -f $DOCKER_COMPOSE_FILE ps --format json | jq -s 'map(select(.Health == "healthy")) | length') | |
if [[ $healthy_cnt -eq $node_cnt ]] | |
then | |
break | |
fi | |
echo 'Waiting for nodes to be ready...' | |
sleep 3 | |
: $((++i)) | |
done | |
env: | |
MAX_ATTEMPTS: "10" | |
- name: Print containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE ps | |
- name: Build cluster | |
run: bundle exec rake "build_cluster[$HOST_ADDR]" | |
env: | |
HOST_ADDR: ${{ env.HOST_IP_ADDR }} | |
DEBUG: "1" | |
- name: Run minitest | |
run: bundle exec rake test | |
- name: Stop containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE down || true | |
lint: | |
name: Lint | |
timeout-minutes: 5 | |
runs-on: ubuntu-latest | |
steps: | |
- name: Check out code | |
uses: actions/checkout@v4 | |
- name: Set up Ruby | |
uses: ruby/setup-ruby@v1 | |
with: | |
ruby-version: '3.3' | |
bundler-cache: true | |
- name: Run rubocop | |
run: bundle exec rubocop | |
benchmark: | |
name: Benchmark | |
timeout-minutes: 10 | |
runs-on: ubuntu-latest | |
env: | |
REDIS_VERSION: '7.2' | |
DOCKER_COMPOSE_FILE: 'compose.latency.yaml' | |
REDIS_REPLICA_SIZE: '2' | |
REDIS_CLIENT_MAX_THREADS: '10' | |
DELAY_TIME: '1ms' | |
steps: | |
- name: Check out code | |
uses: actions/checkout@v4 | |
- name: Set up Ruby | |
uses: ruby/setup-ruby@v1 | |
with: | |
ruby-version: '3.3' | |
bundler-cache: true | |
- name: Pull Docker images | |
run: docker compose -f $DOCKER_COMPOSE_FILE pull | |
- name: Run containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE up -d | |
- name: Wait for Redis cluster to be ready | |
run: bundle exec rake wait | |
- name: Print containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE ps | |
- name: Rebuild cluster for balancing of replicas | |
run: bundle exec rake build_cluster_for_bench | |
env: | |
DEBUG: '1' | |
- name: Print topology | |
run: | | |
for i in {1..9} | |
do | |
echo "node$i: $(docker compose -f $DOCKER_COMPOSE_FILE exec node$i redis-cli cluster nodes | grep myself)" | |
done | |
- name: Ping nodes | |
run: | | |
for i in {1..9} | |
do | |
node_addr="$(docker compose -f $DOCKER_COMPOSE_FILE exec node$i redis-cli cluster nodes | grep myself | awk '{print $2}' | cut -d'@' -f1 | cut -d':' -f1)" | |
echo "node$i:" | |
ping -c 5 $node_addr | |
done | |
- name: Print cpu info | |
run: grep 'model name' /proc/cpuinfo | |
- name: Print memory info | |
run: free -w | |
- name: Print disk info | |
run: df -h | |
- name: Run minitest | |
run: bundle exec rake bench | grep BenchCommand | grep -v 'Envoy#bench_pipeline_echo\|Envoy#bench_single_echo' | sort | |
- name: Reset qdisc | |
run: | | |
for i in {5..9..2} | |
do | |
docker compose -f $DOCKER_COMPOSE_FILE exec node$i tc qdisc del dev eth0 root netem || true | |
done | |
- name: Stop containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE down || true | |
ips: | |
name: IPS | |
timeout-minutes: 10 | |
runs-on: ubuntu-latest | |
env: | |
REDIS_VERSION: '7.2' | |
DOCKER_COMPOSE_FILE: 'compose.latency.yaml' | |
REDIS_REPLICA_SIZE: '2' | |
REDIS_CLIENT_MAX_THREADS: '10' | |
DELAY_TIME: '0ms' | |
steps: | |
- name: Check out code | |
uses: actions/checkout@v4 | |
- name: Set up Ruby | |
uses: ruby/setup-ruby@v1 | |
with: | |
ruby-version: '3.3' | |
bundler-cache: true | |
- name: Pull Docker images | |
run: docker compose -f $DOCKER_COMPOSE_FILE pull | |
- name: Run containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE up -d | |
- name: Wait for Redis cluster to be ready | |
run: bundle exec rake wait | |
- name: Print containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE ps | |
- name: Print cpu info | |
run: grep 'model name' /proc/cpuinfo | |
- name: Run iteration per second | |
run: bundle exec rake ips | |
- name: Stop containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE down || true | |
profiling: | |
name: Profiling | |
timeout-minutes: 5 | |
runs-on: ubuntu-latest | |
strategy: | |
fail-fast: false | |
matrix: | |
mode: | |
- single | |
- excessive_pipelining | |
- pipelining_in_moderation | |
- original_mget | |
- emulated_mget | |
env: | |
REDIS_VERSION: '7.2' | |
DOCKER_COMPOSE_FILE: 'compose.yaml' | |
steps: | |
- name: Check out code | |
uses: actions/checkout@v4 | |
- name: Set up Ruby | |
uses: ruby/setup-ruby@v1 | |
with: | |
ruby-version: '3.3' | |
bundler-cache: true | |
- name: Pull Docker images | |
run: docker compose -f $DOCKER_COMPOSE_FILE pull | |
- name: Run containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE up -d | |
- name: Wait for Redis cluster to be ready | |
run: bundle exec rake wait | |
- name: Print containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE ps | |
- name: Run profiler | |
run: bundle exec rake prof | |
env: | |
PROFILE_MODE: ${{ matrix.mode }} | |
- name: Stop containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE down || true | |
massive: | |
name: Massive Cluster | |
timeout-minutes: 10 | |
runs-on: ubuntu-latest | |
env: | |
REDIS_VERSION: '7.2' | |
DOCKER_COMPOSE_FILE: 'compose.massive.yaml' | |
REDIS_SHARD_SIZE: '10' | |
REDIS_REPLICA_SIZE: '2' | |
REDIS_CLIENT_MAX_STARTUP_SAMPLE: '5' | |
steps: | |
- name: Check out code | |
uses: actions/checkout@v4 | |
- name: Set up Ruby | |
uses: ruby/setup-ruby@v1 | |
with: | |
ruby-version: '3.3' | |
bundler-cache: true | |
- name: Print user limits | |
run: ulimit -a | |
- name: Print kernel params | |
run: | | |
sysctl fs.file-max | |
sysctl vm.swappiness | |
sysctl vm.overcommit_memory | |
sysctl net.ipv4.tcp_sack | |
sysctl net.ipv4.tcp_timestamps | |
sysctl net.ipv4.tcp_window_scaling | |
sysctl net.ipv4.tcp_congestion_control | |
sysctl net.ipv4.tcp_syncookies | |
sysctl net.ipv4.tcp_tw_reuse | |
sysctl net.ipv4.tcp_max_syn_backlog | |
sysctl net.core.somaxconn | |
sysctl net.core.rmem_max | |
sysctl net.core.wmem_max | |
- name: Tune kernel params for redis | |
run: | | |
# https://developer.redis.com/operate/redis-at-scale/talking-to-redis/initial-tuning/ | |
sudo sysctl -w vm.overcommit_memory=1 | |
sudo sysctl -w net.ipv4.tcp_tw_reuse=1 # reuse sockets quickly | |
sudo sysctl -w net.ipv4.tcp_max_syn_backlog=1024 # backlog setting | |
sudo sysctl -w net.core.somaxconn=1024 # up the number of connections per port | |
- name: Pull Docker images | |
run: docker compose -f $DOCKER_COMPOSE_FILE pull | |
- name: Run containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE up -d | |
- name: Print memory info | |
run: free -w | |
- name: Wait for Redis cluster to be ready | |
run: bundle exec rake wait | |
env: | |
DEBUG: '1' | |
- name: Print containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE ps | |
- name: Run profiler | |
run: bundle exec rake prof | |
env: | |
PROFILE_MODE: pipelining_in_moderation | |
- name: Stop containers | |
run: docker compose -f $DOCKER_COMPOSE_FILE down || true |