Support s3 bench build during client build (#41)

* add path to s3-benchmark dir * install s3-bench deps * don't build in user space - requires root * update commit IDs for s3-bench * add golang to support s3-bench build * remove yum steps * update ansible commit id * Build client with user-mode * update commit IDs * revert util var
OpenMPDK · Nov 10, 2023 · 65d3104 · 65d3104
1 parent ec16dc2
commit 65d3104
Show file tree

Hide file tree

Showing 5 changed files with 5 additions and 4 deletions.
diff --git a/dss-ansible b/dss-ansible
diff --git a/dss-ecosystem b/dss-ecosystem
diff --git a/dss-minio b/dss-minio
diff --git a/dss-sdk b/dss-sdk
diff --git a/scripts/docker/DSS.Dockerfile b/scripts/docker/DSS.Dockerfile
@@ -24,6 +24,7 @@ RUN set -eux \
         git \
         glibc-devel \
         gmp-devel \
+        golang \
         jemalloc-devel \
         Judy-devel \
         libaio-devel \
+4 −4		README.md
+2 −3		group_vars/onyx.yml
+0 −10		playbooks/configure_vlans.yml
+0 −10		playbooks/remove_vlans.yml
+12 −6		roles/configure_dcqcn/tasks/configure_dcqcn.yml
+0 −2		roles/create_client_library_conf/tasks/create_client_library_conf.yml
+2 −8		roles/create_client_library_conf/tasks/set_endpoints.yml
+12 −1		roles/download_artifacts/tasks/main.yml
+1 −1		roles/format_blobfs/templates/format_disks.sh.j2
+7 −7		roles/get_mlnx_ports/tasks/main.yml
+1 −1		roles/get_mountpoints/tasks/main.yml
+3 −10		roles/start_dss_host/tasks/setup_host_clusters.yml
+1 −1		roles/test_s3_benchmark/defaults/main.yml
+4 −4		roles/test_s3_benchmark/tasks/main.yml
+9 −9		roles/test_s3_benchmark/tasks/s3_benchmark.yml
+20 −2		.github/workflows/main.yml
+3 −0		.gitignore
+50 −16		.gitlab-ci.yml
+3 −1		buildspec/datamover.yml
+8 −13		buildspec/dss-client.yml
+21 −0		buildspec/pytest.yml
+36 −0		buildspec/sonar-scanner.yml
+14 −0		dss_client/scripts/build.sh
+49 −0		dss_datamover/README.md
+32 −15		dss_datamover/client_application.py
+41 −9		dss_datamover/monitor.py
+10 −3		dss_datamover/socket_communication.py
+255 −0		dss_datamover/systemtests/conftest.py
+10 −0		dss_datamover/systemtests/pytest_config.json
+70 −0		dss_datamover/systemtests/test_client.py
+111 −0		dss_datamover/systemtests/test_datamover.py
+64 −0		dss_datamover/systemtests/test_nfs_cluster.py
+0 −38		dss_datamover/tests/README.md
+0 −181		dss_datamover/tests/conftest.py
+0 −5		dss_datamover/tests/pytest_config.json
+0 −58		dss_datamover/tests/test_datamover.py
+ −		dss_datamover/unittests/__pycache__/conftest.cpython-36-pytest-7.0.1.pyc
+ −		dss_datamover/unittests/__pycache__/test_client_application.cpython-36-pytest-7.0.1.pyc
+ −		dss_datamover/unittests/__pycache__/test_dss_client.cpython-36-pytest-7.0.1.pyc
+ −		dss_datamover/unittests/__pycache__/test_logger.cpython-36-pytest-7.0.1.pyc
+ −		dss_datamover/unittests/__pycache__/test_minio_client.cpython-36-pytest-7.0.1.pyc
+ −		dss_datamover/unittests/__pycache__/test_monitor.cpython-36-pytest-7.0.1.pyc
+ −		dss_datamover/unittests/__pycache__/test_nfs_cluster.cpython-36-pytest-7.0.1.pyc
+ −		dss_datamover/unittests/__pycache__/test_s3_client.cpython-36-pytest-7.0.1.pyc
+ −		dss_datamover/unittests/__pycache__/test_socket.cpython-36-pytest-7.0.1.pyc
+ −		dss_datamover/unittests/__pycache__/test_utils.cpython-36-pytest-7.0.1.pyc
+448 −0		dss_datamover/unittests/conftest.py
+67 −0		dss_datamover/unittests/pytest_config.json
+1 −0		dss_datamover/unittests/requirements.txt
+0 −0		dss_datamover/unittests/test_client_application.py
+0 −0		dss_datamover/unittests/test_dss_client.py
+105 −0		dss_datamover/unittests/test_indexing.py
+0 −0		dss_datamover/unittests/test_logger.py
+0 −0		dss_datamover/unittests/test_minio_client.py
+0 −0		dss_datamover/unittests/test_monitor.py
+0 −0		dss_datamover/unittests/test_nfs_cluster.py
+0 −0		dss_datamover/unittests/test_s3_client.py
+274 −0		dss_datamover/unittests/test_socket.py
+1 −8		dss_datamover/unittests/test_utils.py
+3 −0		dss_metrics/config/config.json
+62 −40		dss_metrics/metrics.py
+154 −0		dss_metrics/minio_rest_collector.py
+18 −10		dss_metrics/minio_ustat_collector.py
+0 −141		dss_metrics/nvmftarget_collector.py
+165 −0		dss_metrics/nvmftarget_ustat_collector.py
+2 −1		dss_metrics/requirements.txt
+2 −0		dss_metrics/whitelist.txt
+39 −11		dss_s3benchmark/README.md
+8 −0		dss_s3benchmark/dependencies.txt
+3 −12		build.sh
+2 −2		cmd/debug-storage.go
+119 −0		cmd/erasure-utils.go
+1 −1		cmd/generic-handlers.go
+4 −0		cmd/globals.go
+20 −13		cmd/kv-storage.go
+4 −4		cmd/kv.go
+15 −0		cmd/metrics-router.go
+10 −0		cmd/metrics.go
+1 −0		cmd/object-api-datatypes.go
+1 −1		cmd/posix.go
+1 −1		cmd/storage-interface.go
+1 −1		cmd/storage-rest-client.go
+203 −0		cmd/xl-sets.go
+15 −12		cmd/xl-v1-object.go
+3 −0		cmd/xl-v1.go
+14 −2		.github/workflows/dss-sdk.yml
+3 −0		.gitignore
+53 −35		.gitlab-ci.yml
+36 −0		README.md
+5 −0		__init__.py
+27 −0		buildspec/dss-sdk-build.yml
+15 −0		buildspec/dss-sdk-unit-test.yml
+0 −35		buildspec/dss-sdk.yml
+36 −0		buildspec/sonar-scanner.yml
+0 −0		host/__init__.py
+1 −1		host/config/nkv_config.json
+1 −1		host/config/nkv_config_emul.json
+1 −1		host/config/nkv_config_for_both_remote_local_kv.json
+1 −1		host/config/nkv_config_rdd.json
+1 −1		host/config/nkv_config_remote.json
+0 −0		host/scripts/__init__.py
+37 −13		host/scripts/dss_host.py
+12 −20		target/dss_target.py
+0 −0		tests/__init__.py
+196 −0		tests/conftest.py
+5 −0		tests/requirements.txt
+39 −0		tests/test_disk_stats.py
+67 −0		tests/test_dss_host.py
+54 −0		tests/test_dss_target.py