Initial opensource version.

.github/workflows/func-tests.yml

@@ -0,0 +1,37 @@
+name: Func-tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  schedule:
+    - cron: '30 06 * * *'
+
+env:
+  GO_VERSION: 1.21.3
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+
+      - name: Get dependencies
+        run: go get -v -t -d ./...
+
+      - name: Run test
+        run: make test
+
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: logs
+          path: tests/logs

.github/workflows/golangci-lint.yml

@@ -0,0 +1,27 @@
+name: Linters
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+env:
+  GO_VERSION: 1.21.3
+
+permissions:
+  contents: read
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.55

.github/workflows/jepsen-cluster-tests.yml

@@ -0,0 +1,33 @@
+name: Jepsen-cluster-tests
+
+on:
+  schedule:
+    - cron: '30 06 * * *'
+
+env:
+  GO_VERSION: 1.21.3
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+
+      - name: Get dependencies
+        run: go get -v -t -d ./...
+
+      - name: Run test
+        run: make jepsen_cluster_test
+
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: logs
+          path: tests/logs

.github/workflows/jepsen-sentinel-tests.yml

@@ -0,0 +1,33 @@
+name: Jepsen-sentinel-tests
+
+on:
+  schedule:
+    - cron: '30 06 * * *'
+
+env:
+  GO_VERSION: 1.21.3
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+
+      - name: Get dependencies
+        run: go get -v -t -d ./...
+
+      - name: Run test
+        run: make jepsen_sentinel_test
+
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: logs
+          path: tests/logs

.github/workflows/unit-tests.yml

@@ -0,0 +1,34 @@
+name: Unit-tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+env:
+  GO_VERSION: 1.21.3
+
+jobs:
+  unittest:
+    name: all_unittests
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+
+      - name: Test
+        run: make unittests
+        env:
+          TEST_MODIFIER: -race

.gitignore

@@ -0,0 +1,2 @@
+cmd/rdsync/rdsync
+tests/logs/

.golangci.yml

@@ -0,0 +1,69 @@
+linters-settings:
+  modules-download-mode: vendor
+  dupl:
+    threshold: 400
+  funlen:
+    lines: 400
+    statements: 200
+  gocritic:
+    enabled-tags:
+      - performance
+    disabled-tags:
+      - diagnostic
+      - experimental
+      - opinionated
+      - style
+  misspell:
+    locale: US
+  revive:
+    rules:
+      - name: blank-imports
+      - name: context-as-argument
+      - name: context-keys-type
+      - name: dot-imports
+      - name: error-return
+      - name: error-naming
+      - name: exported
+      - name: var-naming
+      - name: var-declaration
+      - name: package-comments
+      - name: range
+      - name: receiver-naming
+      - name: time-naming
+      - name: unexported-return
+      - name: errorf
+      - name: empty-block
+      - name: unreachable-code
+      - name: redefines-builtin-id
+
+linters:
+  disable-all: true
+  enable:
+    - bodyclose
+    - dupl
+    - errcheck
+    - exportloopref
+    - funlen
+    - gocritic
+    - gofmt
+    - goimports
+    - govet
+    - ineffassign
+    - misspell
+    - nakedret
+    - revive
+    - staticcheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - stylecheck
+    - gosimple
+    - whitespace
+
+run:
+  go: 1.21
+  timeout: 5m
+
+severity:
+  default-severity: error

AUTHORS

@@ -0,0 +1,4 @@
+The following authors have created the source code of "rdsync" published and distributed by YANDEX LLC as the owner:
+
+Svyatoslav Ermilin [email protected]
+Evgeny Dyukov [email protected]

CONTRIBUTING.md

@@ -0,0 +1,35 @@
+# Notice to external contributors
+
+
+## General info
+
+Hello! In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the “**CLA**”). The current version of the CLA can be found here:
+1) https://yandex.ru/legal/cla/?lang=en (in English) and 
+2) https://yandex.ru/legal/cla/?lang=ru (in Russian).
+
+By adopting the CLA, you state the following:
+
+* You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA,
+* You have read the terms and conditions of the CLA and agree with them in full,
+* You are legally able to provide and license your contributions as stated,
+* We may use your contributions for our open source projects and for any other our project too,
+* We rely on your assurances concerning the rights of third parties in relation to your contributions.
+
+If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you have already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA.
+
+## Provide contributions 
+
+If you have already adopted terms and conditions of the CLA, you are able to provide your contributions. When you submit your pull request, please add the following information into it:
+
+```
+I hereby agree to the terms of the CLA available at: [link].
+```
+
+Replace the bracketed text as follows:
+* [link] is the link to the current version of the CLA: https://yandex.ru/legal/cla/?lang=en (in English) or https://yandex.ru/legal/cla/?lang=ru (in Russian).
+
+It is enough to provide us such notification once. 
+
+## Other questions
+
+If you have any questions, please mail us at [email protected].

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 YANDEX LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

Makefile

@@ -0,0 +1,77 @@
+.PHONY: format lint unittests recreate_logs test start_sentinel_env run_jepsen_sentinel_test jepsen_sentinel_test start_cluster_env run_jepsen_cluster_test jepsen_cluster_test clean
+PROJECT=rdsync
+ZK_VERSION=3.9.1
+
+cmd/rdsync/rdsync:
+	GOOS=linux go build -tags netgo,osusergo -o ./cmd/rdsync/rdsync ./cmd/rdsync/...
+
+format:
+	gofmt -s -w `find . -name '*.go'`
+	goimports -w `find . -name '*.go'`
+
+lint:
+	docker run --rm -v ${CURDIR}:/app -w /app golangci/golangci-lint:v1.55-alpine golangci-lint run -v
+
+unittests:
+	go test ./cmd/... ./internal/...
+	go test ./cmd/... ./tests/testutil/matchers/
+
+redis/src/redis-server:
+	docker run --rm -v ${CURDIR}:/app -w /app ubuntu:jammy /app/redis_patches/build.sh
+
+test: base_image redis/src/redis-server cmd/rdsync/rdsync recreate_logs
+	rm -rf ./tests/images/redis/rdsync && cp cmd/rdsync/rdsync ./tests/images/redis/rdsync
+	rm -rf ./tests/images/redis/redis-server && cp redis/src/redis-server ./tests/images/redis/redis-server
+	rm -rf ./tests/images/redis/redis-senticache && cp redis/src/redis-senticache ./tests/images/redis/redis-senticache
+	rm -rf ./tests/images/redis/redis-cli && cp redis/src/redis-cli ./tests/images/redis/redis-cli
+	go build ./tests/...
+	(cd tests; go test -timeout 150m)
+
+recreate_logs:
+	@if [ "$(shell ls tests/logs 2>/dev/null | wc -l)" != "0" ]; then\
+		rm -rf ./tests/logs;\
+	fi
+	mkdir -p ./tests/logs
+
+tests/images/zookeeper/zookeeper.tar.gz:
+	wget https://archive.apache.org/dist/zookeeper/zookeeper-$(ZK_VERSION)/apache-zookeeper-$(ZK_VERSION)-bin.tar.gz -nc -O tests/images/zookeeper/zookeeper.tar.gz
+
+base_image: tests/images/zookeeper/zookeeper.tar.gz
+	docker build tests/images/base -t rdsync-base:latest
+
+start_sentinel_env: base_image redis/src/redis-server cmd/rdsync/rdsync recreate_logs
+	rm -rf ./tests/images/redis/rdsync && cp cmd/rdsync/rdsync ./tests/images/redis/rdsync
+	rm -rf ./tests/images/redis/redis-server && cp redis/src/redis-server ./tests/images/redis/redis-server
+	rm -rf ./tests/images/redis/redis-senticache && cp redis/src/redis-senticache ./tests/images/redis/redis-senticache
+	rm -rf ./tests/images/redis/redis-cli && cp redis/src/redis-cli ./tests/images/redis/redis-cli
+	docker-compose -p $(PROJECT) -f ./tests/images/jepsen-compose.yaml up -d --force-recreate --build
+	timeout 600 docker exec rdsync_zoo1_1 setup_zk.sh
+	timeout 600 docker exec rdsync_redis1_1 setup_sentinel.sh
+	timeout 600 docker exec rdsync_redis2_1 setup_sentinel.sh redis1
+	timeout 600 docker exec rdsync_redis3_1 setup_sentinel.sh redis1
+
+run_jepsen_sentinel_test: recreate_logs start_sentinel_env
+	(docker exec rdsync_jepsen_1 /root/jepsen/run.sh >tests/logs/jepsen.log 2>&1 && tail -n 4 tests/logs/jepsen.log) || ./tests/images/jepsen/save_logs.sh
+
+jepsen_sentinel_test: run_jepsen_sentinel_test clean
+
+start_cluster_env: base_image redis/src/redis-server cmd/rdsync/rdsync recreate_logs
+	rm -rf ./tests/images/redis/rdsync && cp cmd/rdsync/rdsync ./tests/images/redis/rdsync
+	rm -rf ./tests/images/redis/redis-server && cp redis/src/redis-server ./tests/images/redis/redis-server
+	rm -rf ./tests/images/redis/redis-senticache && cp redis/src/redis-senticache ./tests/images/redis/redis-senticache
+	rm -rf ./tests/images/redis/redis-cli && cp redis/src/redis-cli ./tests/images/redis/redis-cli
+	docker-compose -p $(PROJECT) -f ./tests/images/jepsen-compose.yaml up -d --force-recreate --build
+	timeout 600 docker exec rdsync_zoo1_1 setup_zk.sh
+	timeout 600 docker exec rdsync_redis1_1 setup_cluster.sh
+	timeout 600 docker exec rdsync_redis2_1 setup_cluster.sh redis1
+	timeout 600 docker exec rdsync_redis3_1 setup_cluster.sh redis1
+
+run_jepsen_cluster_test: recreate_logs start_cluster_env
+	(docker exec rdsync_jepsen_1 /root/jepsen/run.sh >tests/logs/jepsen.log 2>&1 && tail -n 4 tests/logs/jepsen.log) || ./tests/images/jepsen/save_logs.sh
+
+jepsen_cluster_test: run_jepsen_cluster_test clean
+
+clean:
+	docker ps | grep rdsync | awk '{print $$1}' | xargs -r docker rm -f || true
+	docker network ls | grep rdsync | awk '{print $$1}' | xargs -r docker network rm || true
+	rm -rf ./tests/logs