From 520fecc49dc1af4ae5119fbe5c05212cead694b5 Mon Sep 17 00:00:00 2001 From: Robin Andersson Date: Thu, 12 Oct 2023 11:32:54 +0200 Subject: [PATCH] [HWORKS-571] Run more integration tests in parallel (#1423) (#1412) --- hopsworks-IT/src/test/ruby/.env.example | 1 - hopsworks-IT/src/test/ruby/isolated_tests | 16 ++-- .../src/test/ruby/spec/execution_spec.rb | 31 +------ .../src/test/ruby/spec/experiment_spec.rb | 18 ---- .../src/test/ruby/spec/jupyter_spec.rb | 2 +- .../src/test/ruby/spec/model_registry_spec.rb | 20 ----- hopsworks-IT/src/test/ruby/spec/model_spec.rb | 21 ----- .../serving_default_sklearn_inference_spec.rb | 2 - .../ruby/spec/serving_default_sklearn_spec.rb | 2 - .../test/ruby/spec/serving_default_spec.rb | 2 - ...rving_default_tensorflow_inference_spec.rb | 2 - .../spec/serving_default_tensorflow_spec.rb | 2 - .../test/ruby/spec/serving_inference_spec.rb | 2 - .../src/test/ruby/spec/serving_spec.rb | 66 --------------- .../src/test/ruby/spec/variables_spec.rb | 82 +++++++++++++++++++ 15 files changed, 89 insertions(+), 180 deletions(-) diff --git a/hopsworks-IT/src/test/ruby/.env.example b/hopsworks-IT/src/test/ruby/.env.example index 9b912d8bdd..d6481b2c58 100644 --- a/hopsworks-IT/src/test/ruby/.env.example +++ b/hopsworks-IT/src/test/ruby/.env.example @@ -24,7 +24,6 @@ REMOTE_SSH_USER=vagrant # Username of password-less sudo user REMOTE_SSH_PASSWORD=vagrant # Password of user to SSH into machines REMOTE_SSH_PORT=22 # Port sshd listens on VMs PYTHON_VERSION=3.10 -FAST=true SPARK_EXAMPLE_VERSION="2.12-3.1.1.5" # Only if local diff --git a/hopsworks-IT/src/test/ruby/isolated_tests b/hopsworks-IT/src/test/ruby/isolated_tests index bccadff9a8..f9516f9262 100644 --- a/hopsworks-IT/src/test/ruby/isolated_tests +++ b/hopsworks-IT/src/test/ruby/isolated_tests @@ -1,21 +1,15 @@ +users_spec.rb +variables_spec.rb agent_spec.rb +admin_hosts_spec.rb audit_spec.rb -conda_spec.rb -experiment_spec.rb -execution_spec.rb -execution_kube_spec.rb ee_epipe_spec.rb ee_search_spec.rb ee_tags_spec.rb epipe_spec.rb -projects_spec.rb +ee_epipe_spec.rb prov_ops_spec.rb prov_state_spec.rb search_spec.rb -users_spec.rb docker_image_spec.rb -serving_spec.rb -model_spec.rb -job_alert_spec.rb -job_kube_spec.rb -job_spec.rb +projects_spec.rb \ No newline at end of file diff --git a/hopsworks-IT/src/test/ruby/spec/execution_spec.rb b/hopsworks-IT/src/test/ruby/spec/execution_spec.rb index 168b8ae45f..ecd4f6eca3 100644 --- a/hopsworks-IT/src/test/ruby/spec/execution_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/execution_spec.rb @@ -16,7 +16,6 @@ describe "On #{ENV['OS']}" do before :all do - @max_executions_per_job = getVar('executions_per_job_limit').value @debugOpt=false # Enable the reporting RSpec.configure do |c| @@ -40,12 +39,8 @@ job_types.each do |type| context 'with authentication and executable ' + type do before :all do - setVar('executions_per_job_limit', '2') with_valid_tour_project("spark") end - after :all do - setVar('executions_per_job_limit', @max_executions_per_job) - end after :each do clean_jobs(@project[:id]) end @@ -186,18 +181,14 @@ def run_job_and_get_logs(project, job_name) end end end - describe 'execution extended', if: ENV['FAST'].eql?("false") do + describe 'execution extended' do context 'create' do job_types = ['jar', 'py', 'ipynb'] job_types.each do |type| context 'with authentication and executable ' + type do before :all do - setVar('executions_per_job_limit', '2') with_valid_tour_project("spark") end - after :all do - setVar('executions_per_job_limit', @max_executions_per_job) - end after :each do clean_jobs(@project[:id]) end @@ -640,26 +631,6 @@ def run_job_and_get_logs(project, job_name) create_sparktour_job(@project, "quota5", 'jar') run_execution(@project[:id], "quota5") end - describe "#parallel executions" do - before :all do - setVar("quotas_max_parallel_executions", "1") - @cookies_pe = with_admin_session - with_valid_tour_project("spark") - end - after :all do - setVar("quotas_max_parallel_executions", "-1") - @cookies_pe = nil - end - it "should not launch more than configured max parallel executions" do - create_sparktour_job(@project, "max_parallel_exec", "jar") - start_execution(@project[:id], "max_parallel_exec") - - # reached limit - resp = start_execution(@project[:id], "max_parallel_exec", expected_status: 400) - parsed = JSON.parse(resp) - expect(parsed['usrMsg']).to include("quota") - end - end end describe '#access' do before :all do diff --git a/hopsworks-IT/src/test/ruby/spec/experiment_spec.rb b/hopsworks-IT/src/test/ruby/spec/experiment_spec.rb index aa4f455bb6..3dc85b9631 100644 --- a/hopsworks-IT/src/test/ruby/spec/experiment_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/experiment_spec.rb @@ -19,7 +19,6 @@ @debugOpt = false # ensure data science profile is enabled - @enable_data_science_profile = getVar('enable_data_science_profile') setVar('enable_data_science_profile', "true") wait_on_command_search(repeat: 30) epipe_wait_on_mutations(repeat: 30) @@ -39,23 +38,6 @@ describe 'experiment' do - context 'with data science profile not enabled' do - before :all do - # disable data science profile - setVar('enable_data_science_profile', "false") - with_valid_project - end - - after :all do - setVar('enable_data_science_profile', "true") - end - - it "should fail to get experiments" do - get_experiment(@project[:id], "app_id_4252123_1", nil) - expect_status_details(400, error_code: 120012) - end - end - context 'without authentication' do before :all do with_valid_project diff --git a/hopsworks-IT/src/test/ruby/spec/jupyter_spec.rb b/hopsworks-IT/src/test/ruby/spec/jupyter_spec.rb index ca00befc8a..f11a7d9427 100644 --- a/hopsworks-IT/src/test/ruby/spec/jupyter_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/jupyter_spec.rb @@ -232,7 +232,7 @@ end end - describe "Jupyter extended", if: ENV['FAST'].eql?("false") do + describe "Jupyter extended" do remote_fs_drivers = ["hdfscontentsmanager", "hopsfsmount"] remote_fs_drivers.each do |driver| describe "with jupyter remote filesystem driver: #{driver}" do diff --git a/hopsworks-IT/src/test/ruby/spec/model_registry_spec.rb b/hopsworks-IT/src/test/ruby/spec/model_registry_spec.rb index 41b78475a6..fbb71a9550 100644 --- a/hopsworks-IT/src/test/ruby/spec/model_registry_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/model_registry_spec.rb @@ -17,34 +17,14 @@ before :all do # ensure data science profile is enabled - @enable_data_science_profile = getVar('enable_data_science_profile') setVar('enable_data_science_profile', "true") end after :all do clean_all_test_projects(spec: "model_registry") - setVar('enable_data_science_profile', @enable_data_science_profile[:value]) end describe 'model registry' do - - context 'with data science profile not enabled' do - before :all do - # disable data science profile - setVar('enable_data_science_profile', "false") - with_valid_project - end - - after :all do - setVar('enable_data_science_profile', "true") - end - - it "should fail to get model registries" do - get_model_registries(@project[:id], nil) - expect_status_details(400, error_code: 120012) - end - end - context 'without authentication' do before :all do with_valid_project diff --git a/hopsworks-IT/src/test/ruby/spec/model_spec.rb b/hopsworks-IT/src/test/ruby/spec/model_spec.rb index bc0ef2e21e..417dfa0dd5 100644 --- a/hopsworks-IT/src/test/ruby/spec/model_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/model_spec.rb @@ -20,7 +20,6 @@ @cleanup = true # ensure data science profile is enabled - @enable_data_science_profile = getVar('enable_data_science_profile') setVar('enable_data_science_profile', "true") wait_on_command_search(repeat: 30) epipe_wait_on_mutations(repeat: 30) @@ -40,26 +39,6 @@ describe 'model' do - context 'with data science profile not enabled' do - before :all do - # disable data science profile - setVar('enable_data_science_profile', "false") - - with_valid_project - end - - after :all do - setVar('enable_data_science_profile', "true") - end - - it "should fail to get models" do - get_models(@project[:id], nil) - expect_status_details(400, error_code: 120012) - get_model(@project[:id], "mnist_1") - expect_status_details(400, error_code: 120012) - end - end - context 'without authentication' do before :all do with_valid_project diff --git a/hopsworks-IT/src/test/ruby/spec/serving_default_sklearn_inference_spec.rb b/hopsworks-IT/src/test/ruby/spec/serving_default_sklearn_inference_spec.rb index b7c2c1cea6..a72f76b976 100644 --- a/hopsworks-IT/src/test/ruby/spec/serving_default_sklearn_inference_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/serving_default_sklearn_inference_spec.rb @@ -20,14 +20,12 @@ before :all do # ensure data science profile is enabled - @enable_data_science_profile = getVar('enable_data_science_profile') setVar('enable_data_science_profile', "true") end after :all do clean_all_test_projects(spec: "serving_default_sklearn_inference") purge_all_sklearn_serving_instances - setVar('enable_data_science_profile', @enable_data_science_profile[:value]) end describe 'inference' do diff --git a/hopsworks-IT/src/test/ruby/spec/serving_default_sklearn_spec.rb b/hopsworks-IT/src/test/ruby/spec/serving_default_sklearn_spec.rb index 2c984aa973..71d8e096f6 100644 --- a/hopsworks-IT/src/test/ruby/spec/serving_default_sklearn_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/serving_default_sklearn_spec.rb @@ -22,14 +22,12 @@ before :all do # ensure data science profile is enabled - @enable_data_science_profile = getVar('enable_data_science_profile') setVar('enable_data_science_profile', "true") end after :all do clean_all_test_projects(spec: "serving_default_sklearn") purge_all_sklearn_serving_instances - setVar('enable_data_science_profile', @enable_data_science_profile[:value]) end describe 'sklearn_serving' do diff --git a/hopsworks-IT/src/test/ruby/spec/serving_default_spec.rb b/hopsworks-IT/src/test/ruby/spec/serving_default_spec.rb index 2857503073..85126ad934 100644 --- a/hopsworks-IT/src/test/ruby/spec/serving_default_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/serving_default_spec.rb @@ -22,14 +22,12 @@ before :all do # ensure data science profile is enabled - @enable_data_science_profile = getVar('enable_data_science_profile') setVar('enable_data_science_profile', "true") end after :all do clean_all_test_projects(spec: "serving_default") purge_all_tf_serving_instances - setVar('enable_data_science_profile', @enable_data_science_profile[:value]) end describe "#create" do diff --git a/hopsworks-IT/src/test/ruby/spec/serving_default_tensorflow_inference_spec.rb b/hopsworks-IT/src/test/ruby/spec/serving_default_tensorflow_inference_spec.rb index 6c5759a30a..b34cc101c7 100644 --- a/hopsworks-IT/src/test/ruby/spec/serving_default_tensorflow_inference_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/serving_default_tensorflow_inference_spec.rb @@ -20,14 +20,12 @@ before :all do # ensure data science profile is enabled - @enable_data_science_profile = getVar('enable_data_science_profile') setVar('enable_data_science_profile', "true") end after :all do clean_all_test_projects(spec: "serving_default_tensorflow_inference") purge_all_tf_serving_instances - setVar('enable_data_science_profile', @enable_data_science_profile[:value]) end describe 'inference' do diff --git a/hopsworks-IT/src/test/ruby/spec/serving_default_tensorflow_spec.rb b/hopsworks-IT/src/test/ruby/spec/serving_default_tensorflow_spec.rb index cb3044856d..3791e353e1 100644 --- a/hopsworks-IT/src/test/ruby/spec/serving_default_tensorflow_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/serving_default_tensorflow_spec.rb @@ -22,14 +22,12 @@ before :all do # ensure data science profile is enabled - @enable_data_science_profile = getVar('enable_data_science_profile') setVar('enable_data_science_profile', "true") end after :all do clean_all_test_projects(spec: "serving_default_tensorflow") purge_all_tf_serving_instances - setVar('enable_data_science_profile', @enable_data_science_profile[:value]) end describe "#create" do diff --git a/hopsworks-IT/src/test/ruby/spec/serving_inference_spec.rb b/hopsworks-IT/src/test/ruby/spec/serving_inference_spec.rb index d04589d37a..6d2b6b1433 100755 --- a/hopsworks-IT/src/test/ruby/spec/serving_inference_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/serving_inference_spec.rb @@ -23,14 +23,12 @@ before :all do # ensure data science profile is enabled - @enable_data_science_profile = getVar('enable_data_science_profile') setVar('enable_data_science_profile', "true") end after :all do clean_all_test_projects(spec: "serving_inference") purge_all_tf_serving_instances - setVar('enable_data_science_profile', @enable_data_science_profile[:value]) end let(:test_data) {[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, diff --git a/hopsworks-IT/src/test/ruby/spec/serving_spec.rb b/hopsworks-IT/src/test/ruby/spec/serving_spec.rb index b89585722c..21910b8145 100644 --- a/hopsworks-IT/src/test/ruby/spec/serving_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/serving_spec.rb @@ -19,14 +19,12 @@ before :all do # ensure data science profile is enabled - @enable_data_science_profile = getVar('enable_data_science_profile') setVar('enable_data_science_profile', "true") end after :all do clean_all_test_projects(spec: "serving") purge_all_tf_serving_instances - setVar('enable_data_science_profile', @enable_data_science_profile[:value]) end describe "#create" do @@ -570,25 +568,6 @@ }) expect_status_details(201) end - - describe "with quota enabled" do - before :all do - setVar("quotas_model_deployments_total", "1") - @local_project = create_project - end - after :all do - setVar("quotas_model_deployments_total", "-1") - purge_all_tf_serving_instances - delete_all_servings(@local_project.id) - end - it "should fail to create serving if quota has been reached" do - ## This deployument should go through - with_tensorflow_serving(@local_project.id, @local_project.projectname, @user.username) - - ## Second deployment should fail because quota has been reached - create_tensorflow_serving(@local_project.id, @local_project.projectname, expected_status: 400) - end - end end end @@ -959,31 +938,6 @@ expect_status_details(400, error_code: 240003) end end - - describe "with quota enabled" do - before :all do - setVar("quotas_model_deployments_running", "1") - @local_project = create_project - with_tensorflow_serving(@local_project.id, @local_project.projectname, @user.username) - end - after :all do - setVar("quotas_model_deployments_running", "-1") - purge_all_tf_serving_instances - delete_all_servings(@local_project.id) - end - it "should fail to start serving if quota has been reached" do - ## This deployment should start - start_serving(@local_project, @serving) - - second_serving = create_tensorflow_serving(@local_project.id, @local_project.projectname) - ## Starting this one should fail because quota has beed reached - post "#{ENV['HOPSWORKS_API']}/project/#{@local_project.id}/serving/#{second_serving.id}?action=start" - expect_status_details(400) - parsed = JSON.parse(response) - expect(parsed['devMsg']).to include("quota") - end - end - end describe "#stop", vm: true do @@ -1142,26 +1096,6 @@ end describe "#filter" do - - context 'with data science profile not enabled' do - before :all do - # ensure data science profile is enabled - @enable_data_science_profile = getVar('enable_data_science_profile') - setVar('enable_data_science_profile', "false") - - with_valid_project - end - - after :all do - setVar('enable_data_science_profile', @enable_data_science_profile[:value]) - end - - it "should fail to get servings" do - get_servings(@project, nil) - expect_status_details(400, error_code: 120012) - end - end - context 'without authentication' do before :all do with_valid_project diff --git a/hopsworks-IT/src/test/ruby/spec/variables_spec.rb b/hopsworks-IT/src/test/ruby/spec/variables_spec.rb index 287a77f101..c4dfba28f7 100644 --- a/hopsworks-IT/src/test/ruby/spec/variables_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/variables_spec.rb @@ -67,4 +67,86 @@ expect_status_details(401) end end + + describe 'with variable' do + describe 'quotas_max_parallel_executions' do + context "enabled" do + before :all do + setVar("quotas_max_parallel_executions", "1") + @cookies_pe = with_admin_session + with_valid_tour_project("spark") + end + after :all do + setVar("quotas_max_parallel_executions", "-1") + @cookies_pe = nil + end + it "should not launch more than configured max parallel executions" do + create_sparktour_job(@project, "max_parallel_exec", "jar") + start_execution(@project[:id], "max_parallel_exec") + + # reached limit + resp = start_execution(@project[:id], "max_parallel_exec", expected_status: 400) + parsed = JSON.parse(resp) + expect(parsed['usrMsg']).to include("quota") + end + end + end + + describe 'enable_data_science_profile' do + context 'disabled' do + before :all do + # disable data science profile + setVar('enable_data_science_profile', "false") + with_valid_project + end + + after :all do + setVar('enable_data_science_profile', "true") + end + + it "should fail to get models" do + get_models(@project[:id], nil) + expect_status_details(400, error_code: 120012) + get_model(@project[:id], "mnist_1") + expect_status_details(400, error_code: 120012) + end + + it "should fail to get model registries" do + get_model_registries(@project[:id], nil) + expect_status_details(400, error_code: 120012) + end + + it "should fail to get experiments" do + get_experiment(@project[:id], "app_id_4252123_1", nil) + expect_status_details(400, error_code: 120012) + end + + it "should fail to get servings" do + get_servings(@project, nil) + expect_status_details(400, error_code: 120012) + end + end + end + + describe 'quotas_model_deployments_total' do + context "enabled" do + before :all do + setVar("quotas_model_deployments_total", "1") + @local_project = create_project + end + after :all do + setVar("quotas_model_deployments_total", "-1") + purge_all_tf_serving_instances + delete_all_servings(@local_project.id) + end + it "should fail to create serving if quota has been reached" do + ## This deployument should go through + with_tensorflow_serving(@local_project.id, @local_project.projectname, @user.username) + + ## Second deployment should fail because quota has been reached + create_tensorflow_serving(@local_project.id, @local_project.projectname, expected_status: 400) + end + end + end + end end