diff --git a/hopsworks-IT/src/test/ruby/isolated_tests b/hopsworks-IT/src/test/ruby/isolated_tests index f9516f9262..30d30a604b 100644 --- a/hopsworks-IT/src/test/ruby/isolated_tests +++ b/hopsworks-IT/src/test/ruby/isolated_tests @@ -3,11 +3,9 @@ variables_spec.rb agent_spec.rb admin_hosts_spec.rb audit_spec.rb -ee_epipe_spec.rb ee_search_spec.rb ee_tags_spec.rb epipe_spec.rb -ee_epipe_spec.rb prov_ops_spec.rb prov_state_spec.rb search_spec.rb diff --git a/hopsworks-IT/src/test/ruby/spec/agent_spec.rb b/hopsworks-IT/src/test/ruby/spec/agent_spec.rb index e3829427fd..3ad43587fe 100644 --- a/hopsworks-IT/src/test/ruby/spec/agent_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/agent_spec.rb @@ -38,12 +38,12 @@ end it "should not be able to register" do - post @register_resource, {"host-id": "host0", password: "password"} + post @register_resource, {"host-id": "host0", "private-ip": "10.0.0.15", password: "password"} expect_status_details(401) end it "should not be able to heartbeat" do - post @heartbeat_resource, {"host-id": "host0", agentTime: "1234"} + post @heartbeat_resource, {"host-id": "host0", "private-ip": "10.0.1.15", agentTime: "1234"} expect_status_details(401) end end @@ -72,14 +72,19 @@ before(:all) do @random_host = "host_#{short_random_id}" end - it "should not be able to register" do - post @register_resource, {"host-id": @random_host, password: "some_pass"} - expect_status_details(404, error_code: 100025) + after(:all) do + host = find_by_hostname(@random_host) + host.destroy + end + # [HWORKS-705] allowed any host to register and heartbeat + it "should be able to register random host" do + post @register_resource, {"host-id": @random_host, "private-ip": "10.0.12.15", password: "some_pass"} + expect_status_details(200) end - it "should not be able to heartbeat" do - post @heartbeat_resource, {"host-id": @random_host, agentTime: "1234"} - expect_status_details(404) + it "should be able to heartbeat to random host" do + post @heartbeat_resource, {"host-id": @random_host, "private-ip": "10.0.12.15", "agent-time": "1234"} + expect_status_details(200) end end @@ -97,15 +102,15 @@ it "should be able to register" do host = find_by_hostname(@hostname) expect(host.registered).to eq(false) - post @register_resource, {"host-id": @hostname, password: "pass123"} + post @register_resource, {"host-id": @hostname, "private-ip": "10.0.4.15", password: "pass123"} expect_status_details(200) host = find_by_hostname(@hostname) expect(host.registered).to eq(true) end it "should be able to heartbeat" do - post @register_resource, {"host-id": @hostname, password: "pass123"} - post @heartbeat_resource, {"host-id": @hostname, "num-gpus": 0, "agent-time": 1, + post @register_resource, {"host-id": @hostname, "private-ip": "10.0.5.15", password: "pass123"} + post @heartbeat_resource, {"host-id": @hostname, "private-ip": "10.0.6.15", "num-gpus": 0, "agent-time": 1, "cores": 4, "memory-capacity": 2} expect_status_details(200) host = find_by_hostname(@hostname) diff --git a/hopsworks-IT/src/test/ruby/spec/airflow_spec.rb b/hopsworks-IT/src/test/ruby/spec/airflow_spec.rb index 6c4ded01b0..57fac00525 100644 --- a/hopsworks-IT/src/test/ruby/spec/airflow_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/airflow_spec.rb @@ -52,15 +52,14 @@ end it "should be able to compose DAG" do - get "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/airflow/secretDir" - expect_status_details(200) - secret_dir = response.body - post "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/airflow/dag", @dag_definition expect_status_details(200) - airflow_dir = Variables.find_by(id: "airflow_dir") - dag_file = File.join(airflow_dir.value, "dags", secret_dir, "#{@dag_name}.py") - expect(File.exists?(dag_file)).to be true + + get_dataset_stat(@project, "Airflow/#{@dag_name}.py", datasetType: "&type=DATASET") + expect_status_details(200) + ds = json_body + expect(ds[:attributes][:name]).to eq ("#{@dag_name}.py") + expect(ds[:attributes][:owner]).to eq ("#{@user[:fname]} #{@user[:lname]}") end end end diff --git a/hopsworks-IT/src/test/ruby/spec/audit_spec.rb b/hopsworks-IT/src/test/ruby/spec/audit_spec.rb index 2e6ef300a2..8b889ef64b 100644 --- a/hopsworks-IT/src/test/ruby/spec/audit_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/audit_spec.rb @@ -37,6 +37,12 @@ end end describe "Log" do + before(:all) do + #Logs can end up on any node in HA + if ENV['OS'] == "centos" + skip "These tests do not run on centos" + end + end context 'user login' do it 'should add row to log file' do newUser = create_user diff --git a/hopsworks-IT/src/test/ruby/spec/auxiliary/ml/notebooks/end_to_end_pipeline/sklearn/iris_flower_classifier.py b/hopsworks-IT/src/test/ruby/spec/auxiliary/ml/notebooks/end_to_end_pipeline/sklearn/iris_flower_classifier.py index 6484368aad..d39b92df5c 100755 --- a/hopsworks-IT/src/test/ruby/spec/auxiliary/ml/notebooks/end_to_end_pipeline/sklearn/iris_flower_classifier.py +++ b/hopsworks-IT/src/test/ruby/spec/auxiliary/ml/notebooks/end_to_end_pipeline/sklearn/iris_flower_classifier.py @@ -5,15 +5,10 @@ class Predict(object): def __init__(self): - """ Initializes the serving state, reads a trained model from HDFS""" - self.model_path = "Models/irisflowerclassifier/1/iris_knn.pkl" - print("Copying SKLearn model from HDFS to local directory") - hdfs.copy_to_local(self.model_path) print("Reading local SkLearn model for serving") self.model = joblib.load("./iris_knn.pkl") print("Initialization Complete") - def predict(self, inputs): """ Serves a prediction request usign a trained model""" return self.model.predict(inputs).tolist() # Numpy Arrays are note JSON serializable diff --git a/hopsworks-IT/src/test/ruby/spec/bigquery_storage_connector_spec.rb b/hopsworks-IT/src/test/ruby/spec/bigquery_storage_connector_spec.rb index 5030e0f7db..25ccaaeed3 100644 --- a/hopsworks-IT/src/test/ruby/spec/bigquery_storage_connector_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/bigquery_storage_connector_spec.rb @@ -141,7 +141,7 @@ def create_connector_materializationDataset expect(parsed_result_update['dataset']).to eql(dataset) expect(parsed_result_update['queryTable']).to eql(table) expect(parsed_result_update['queryTable']).to eql(table) - expect(parsed_result_update['arguments']).to eql([]) + expect(parsed_result_update['arguments']).to be_empty end it 'should fail to update non existing connector' do diff --git a/hopsworks-IT/src/test/ruby/spec/dataset_spec.rb b/hopsworks-IT/src/test/ruby/spec/dataset_spec.rb index 391e3e9a49..4c28a73397 100644 --- a/hopsworks-IT/src/test/ruby/spec/dataset_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/dataset_spec.rb @@ -1717,7 +1717,7 @@ end it 'should get 0 result if offset >= len.' do get_datasets_in_path(@project, @dataset[:inode_name], query: "&sort_by=id:asc&limit=10&offset=2500") - expect(json_body[:items]).to be nil + expect(json_body[:items]).to be_empty end end end diff --git a/hopsworks-IT/src/test/ruby/spec/execution_spec.rb b/hopsworks-IT/src/test/ruby/spec/execution_spec.rb index ecd4f6eca3..11023a8210 100644 --- a/hopsworks-IT/src/test/ruby/spec/execution_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/execution_spec.rb @@ -230,20 +230,6 @@ def run_job_and_get_logs(project, job_name) #start execution start_execution(@project[:id], $job_name_2, expected_status: 400) end - it "should not start more than the allowed maximum number of executions per job" do - $job_name_3 = "demo_job_3_" + type - create_sparktour_job(@project, $job_name_3, type) - begin - start_execution(@project[:id], $job_name_3) - execution_id1 = json_body[:id] - start_execution(@project[:id], $job_name_3) - execution_id2 = json_body[:id] - start_execution(@project[:id], $job_name_3, expected_status: 400, error_code: 130040) - ensure - wait_for_execution_completed(@project[:id], $job_name_3, execution_id1, "FINISHED") unless execution_id1.nil? - wait_for_execution_completed(@project[:id], $job_name_3, execution_id2, "FINISHED") unless execution_id2.nil? - end - end it "should start a job and use default args" do $job_name_3 = "demo_job_3_" + type create_sparktour_job(@project, $job_name_3, type) @@ -642,8 +628,10 @@ def run_job_and_get_logs(project, job_name) @project = create_project add_member_to_project(@project, @user_data_scientist[:email], "Data scientist") @job_name = "test_job_#{short_random_id}" - run_job(@user_data_owner, @project, @job_name) + get_executions(@project["id"], @job_name) + execution_id = json_body[:items][0][:id] + wait_for_execution_active(@project[:id], @job_name, execution_id, "FINISHED", "appId") end def setup_job(user, project, job_name) diff --git a/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb b/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb index 7ba7a1eddf..f250543f77 100644 --- a/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb @@ -2077,8 +2077,9 @@ expect_status_details(200) topic = json_body[:items].select{|topic| topic[:name] == topic_name} expect(topic.length).to eq(1) + #disableOnline=true should delete the subject get_subject_schema(project, featuregroup_name + "_" + parsed_json["version"].to_s, 1) - expect_status_details(200) + expect_status_details(404) end it "should be possible to preview from online storage of an on-demand/external feature group" do @@ -2645,8 +2646,8 @@ # add sample ros OnlineFg.db_name = project[:projectname] - OnlineFg.create(testfeature: 1).save - OnlineFg.create(testfeature: 2).save + OnlineFg.create(testfeature: 3).save + OnlineFg.create(testfeature: 4).save get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups/" + featuregroup_id.to_s + "/preview?storage=online&limit=1" expect_status_details(200) @@ -3012,9 +3013,10 @@ topic = [] end expect(topic.length).to eq(1) + #disableOnline=true should delete the subject get_subject_schema(project, featuregroup_name + "_" + parsed_json["version"].to_s, 1) - expect_status_details(200) - expect(json_body[:error_code]).to eql(nil) + expect_status_details(404) + expect(json_body[:error_code]).to eql(40401) end it "should update avro schema when features are appended to existing online feature group" do diff --git a/hopsworks-IT/src/test/ruby/spec/featureview_trainingdataset_spec.rb b/hopsworks-IT/src/test/ruby/spec/featureview_trainingdataset_spec.rb index 0aad2f8319..51b46becda 100644 --- a/hopsworks-IT/src/test/ruby/spec/featureview_trainingdataset_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/featureview_trainingdataset_spec.rb @@ -40,7 +40,6 @@ expect(parsed_json.key?("dataFormat")).to be true expect(parsed_json.key?("trainingDatasetType")).to be true expect(parsed_json.key?("location")).to be true - expect(parsed_json.key?("inodeId")).to be true expect(parsed_json.key?("seed")).to be true expect(parsed_json["featurestoreName"] == featurestore_name).to be true expect(parsed_json["name"] == "#{featureview['name']}_#{featureview['version']}").to be true @@ -319,7 +318,6 @@ expect(parsed_json2.key?("version")).to be true expect(parsed_json2.key?("dataFormat")).to be true expect(parsed_json2.key?("trainingDatasetType")).to be true - expect(parsed_json2.key?("inodeId")).to be true expect(parsed_json2["version"]).to eql(parsed_json["version"]) # make sure the dataformat didn't change diff --git a/hopsworks-IT/src/test/ruby/spec/helpers/command_helper.rb b/hopsworks-IT/src/test/ruby/spec/helpers/command_helper.rb index 1a112e5b5f..41ff10f64e 100644 --- a/hopsworks-IT/src/test/ruby/spec/helpers/command_helper.rb +++ b/hopsworks-IT/src/test/ruby/spec/helpers/command_helper.rb @@ -48,7 +48,7 @@ def wait_on_command(wait_time: 10, repeat: 1, &log_size) result[0] end - def wait_on_command_search(wait_time: 10, repeat: 6) + def wait_on_command_search(wait_time: 20, repeat: 6) result = wait_on_command(wait_time: wait_time, repeat: repeat) do count = CommandSearch.count count diff --git a/hopsworks-IT/src/test/ruby/spec/helpers/jupyter_helper.rb b/hopsworks-IT/src/test/ruby/spec/helpers/jupyter_helper.rb index 2ee16c5dd9..4318dec8e9 100644 --- a/hopsworks-IT/src/test/ruby/spec/helpers/jupyter_helper.rb +++ b/hopsworks-IT/src/test/ruby/spec/helpers/jupyter_helper.rb @@ -130,8 +130,8 @@ def recentnotebooks_search(project, expected_count) pp "#{ENV['HOPSWORKS_API']}/project/#{project[:id]}/jupyter/recent" if (defined?(@debugOpt)) && @debugOpt get "#{ENV['HOPSWORKS_API']}/project/#{project[:id]}/jupyter/recent" expect_status_details(200) - begin - expect(json_body[:items]).not_to be_nil + begin + expect(json_body[:items]).not_to be_nil expect(json_body[:items].length).to eql(expected_count) { 'success' => true } rescue RSpec::Expectations::ExpectationNotMetError => e diff --git a/hopsworks-IT/src/test/ruby/spec/jupyter_spec.rb b/hopsworks-IT/src/test/ruby/spec/jupyter_spec.rb index f11a7d9427..18dbae1a62 100644 --- a/hopsworks-IT/src/test/ruby/spec/jupyter_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/jupyter_spec.rb @@ -85,20 +85,22 @@ secret_dir, staging_dir, settings = start_jupyter(@project) jupyter_running(@project, expected_status: 200) - - jwt_file = File.join(staging_dir, "token.jwt") - expect(File.file? jwt_file).to be true - - jupyter_dir = Variables.find_by(id: "jupyter_dir").value - project_username = "#{@project[:projectname]}__#{@user[:username]}" - path2secret = File.join(jupyter_dir, "Projects", @project[:projectname], project_username, secret_dir, "certificates") - - kstore_file = File.join(path2secret, "#{project_username}__kstore.jks") - expect(File.file? kstore_file).to be true - tstore_file = File.join(path2secret, "#{project_username}__tstore.jks") - expect(File.file? tstore_file).to be true - password_file = File.join(path2secret, "#{project_username}__cert.key") - expect(File.file? password_file).to be true + if ENV['OS'] == "ubuntu" + # Token is in staging_dir if not kube + jwt_file = File.join(staging_dir, "token.jwt") + expect(File.file? jwt_file).to be true + + jupyter_dir = Variables.find_by(id: "jupyter_dir").value + project_username = "#{@project[:projectname]}__#{@user[:username]}" + path2secret = File.join(jupyter_dir, "Projects", @project[:projectname], project_username, secret_dir, "certificates") + + kstore_file = File.join(path2secret, "#{project_username}__kstore.jks") + expect(File.file? kstore_file).to be true + tstore_file = File.join(path2secret, "#{project_username}__tstore.jks") + expect(File.file? tstore_file).to be true + password_file = File.join(path2secret, "#{project_username}__cert.key") + expect(File.file? password_file).to be true + end # Check that the logs are written in the opensearch index. begin @@ -212,17 +214,18 @@ expect_status_details(200) notebook_file = json_body[:items].detect { |d| d[:attributes][:name] == "export_model.ipynb" } expect(notebook_file).to be_present - - create_dir(@project, "Resources/test_dir", query: "&type=DATASET") + # test_dir will be created for each kernel + test_dir = "test_dir#{short_random_id}" + create_dir(@project, "Resources/#{test_dir}", query: "&type=DATASET") expect_status_details(201) - copy_dataset(@project, "Resources/export_model.ipynb", "/Projects/#{@project[:projectname]}/Resources/test_dir/[export model].ipynb", datasetType: "&type=DATASET") + copy_dataset(@project, "Resources/export_model.ipynb", "/Projects/#{@project[:projectname]}/Resources/#{test_dir}/[export model].ipynb", datasetType: "&type=DATASET") expect_status_details(204) - get "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/jupyter/convertIPythonNotebook/Resources/test_dir/%5Bexport%20model%5D.ipynb" + get "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/jupyter/convertIPythonNotebook/Resources/#{test_dir}/%5Bexport%20model%5D.ipynb" expect_status_details(200) - get "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/dataset/Resources/test_dir/?action=listing&expand=inodes" + get "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/dataset/Resources/#{test_dir}/?action=listing&expand=inodes" expect_status_details(200) python_file = json_body[:items].detect { |d| d[:attributes][:name] == "[export model].py" } expect(python_file).to be_present @@ -282,11 +285,13 @@ get_settings(@project) shutdownLevel=6 settings = json_body - settings[:distributionStrategy] = "" settings[:shutdownLevel] = shutdownLevel settings[:pythonKernel] = false settings[:jobConfig][:"spark.executor.memory"] = 1023 start_jupyter(@project, settings: settings, expected_status: 400, error_code: 130029) + # set back spark.executor.memory=1024 + settings[:jobConfig][:"spark.executor.memory"] = 1024 + update_jupyter(@project, settings) end it "should not allow starting multiple notebook servers" do @@ -301,11 +306,10 @@ secret_dir, staging_dir, settings = start_jupyter(@project, shutdownLevel=6, baseDir=nil, noLimit=true) jupyter_running(@project, expected_status: 200) - parsed_json = JSON.parse(json_result) - expect(parsed_json["noLimit"]).to eq(true) + expect(json_body[:noLimit]).to eq(true) stop_jupyter(@project) - jupyter_runningt(@project, expected_status: 404) + jupyter_running(@project, expected_status: 404) end it "should allow multiple restarts" do @@ -326,11 +330,15 @@ jupyter_running(@project, expected_status: 200) json_body[:minutesUntilExpiration].should be < 2 - wait_for_me_time(90, 5) do - jupyter_running(@project) + wait_for_me_time(180, 5) do + jupyter_running(@project, expected_status: 200) is_running = response.code == resolve_status(200, response.code) { 'success' => is_running } end + # JupyterNotebookCleaner is running every 30min it does not make sense to wait 30min in a test + get "#{ENV['HOPSWORKS_TESTING']}/test/jupyter/cleanup" + expect_status(200) # No detail returned + jupyter_running(@project, expected_status: 404) end @@ -343,6 +351,9 @@ json_body[:minutesUntilExpiration].should be > initial_minutes_left-3 sleep(90) + get "#{ENV['HOPSWORKS_TESTING']}/test/jupyter/cleanup" + expect_status(200) # No detail returned + jupyter_running(@project, expected_status: 200) json_body[:minutesUntilExpiration].should be < initial_minutes_left diff --git a/hopsworks-IT/src/test/ruby/spec/model_spec.rb b/hopsworks-IT/src/test/ruby/spec/model_spec.rb index 417dfa0dd5..ad85e5ee40 100644 --- a/hopsworks-IT/src/test/ruby/spec/model_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/model_spec.rb @@ -267,84 +267,6 @@ end end - describe 'shared' do - before :all do - setup - setup_shared - # setup_debug - # setup_shared_debug - epipe_wait_on_provenance - - @test_synth_model_job = "test_synth_model" - @model_name_1 = "model_1" - end - - def setup() - @user1_params = {email: "user1_#{random_id}@email.com", first_name: "User", last_name: "1", password: "Pass123"} - @user1 = create_user_with_role(@user1_params, "HOPS_ADMIN") - pp "user email: #{@user1[:email]}" if defined?(@debugOpt) && @debugOpt - - create_session(@user1[:email], @user1_params[:password]) - @project1 = create_project - pp @project1[:projectname] if defined?(@debugOpt) && @debugOpt - end - def setup_shared() - @user2_params = {email: "user2_#{random_id}@email.com", first_name: "User", last_name: "2", password: "Pass123"} - @user2 = create_user_with_role(@user2_params, "HOPS_ADMIN") - pp "user email: #{@user2[:email]}" if defined?(@debugOpt) && @debugOpt - - create_session(@user2[:email], @user2_params[:password]) - @project2 = create_project - pp @project2[:projectname] if defined?(@debugOpt) && @debugOpt - - create_session(@user1_params[:email], @user1_params[:password]) - share_dataset_checked(@project1, "Models", @project2[:projectname], datasetType: "DATASET") - - create_session(@user2_params[:email], @user2_params[:password]) - accept_dataset_checked(@project2, "#{@project1[:projectname]}::Models", datasetType: "DATASET") - share_dataset_checked(@project2, "Models", @project1[:projectname], datasetType: "DATASET") - - create_session(@user1_params[:email], @user1_params[:password]) - accept_dataset_checked(@project1, "#{@project2[:projectname]}::Models", datasetType: "DATASET") - end - def setup_debug() - @user1_params = {email: "user1_798d9d633133c3202679b0834bbccd41d44fcca1@email.com", first_name: "User", last_name: "1", password: "Pass123"} - @user1 = get_user_by_mail(@user1_params[:email]) - - create_session(@user1_params[:email], @user1_params[:password]) - @project1 = get_project_by_name("ProJect_8a439873") - end - def setup_shared_debug() - @user2_params = {email: "user2_6a7d2e29a708f15a5febd74fc36edc6a091a25f4@email.com", first_name: "User", last_name: "2", password: "Pass123"} - @user2 = get_user_by_mail(@user2_params[:email]) - - create_session(@user2_params[:email], @user2_params[:password]) - @project2 = get_project_by_name("ProJect_28edf9c2") - end - - it 'should setup model in shared models dataset' do - create_session(@user2_params[:email], @user2_params[:password]) - if job_exists(@project2[:id], @test_synth_model_job) - pp "job exists - skipping" - else - prepare_spark_job(@project2, @user2[:username], @test_synth_model_job, "py") - end - expect(job_exists(@project2[:id], @test_synth_model_job)).to be(true) - - create_session(@user1_params[:email], @user1_params[:password]) - if model_exists(@project1, @model_name_1) - pp "model exists - skipping" - else - create_session(@user2_params[:email], @user2_params[:password]) - args = [@project1[:projectname], @model_name_1] - run_job(@project2, @test_synth_model_job, args: args) - end - - create_session(@user1_params[:email], @user1_params[:password]) - expect(model_exists(@project1, @model_name_1)).to be(true) - end - end - describe 'model export' do def setup_user(email, password) if email.nil? || password.nil? diff --git a/hopsworks-IT/src/test/ruby/spec/preparedstatements_spec.rb b/hopsworks-IT/src/test/ruby/spec/preparedstatements_spec.rb index 7eaf163f9a..465ed7c2a2 100644 --- a/hopsworks-IT/src/test/ruby/spec/preparedstatements_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/preparedstatements_spec.rb @@ -36,6 +36,7 @@ json_result, fg_name = create_cached_featuregroup(@project.id, featurestore_id, features: features, featuregroup_name: "test_fg_a_#{short_random_id}", online:true) parsed_json = JSON.parse(json_result) + expect_status_details(201) fg_id = parsed_json["id"] fg_type = parsed_json["type"] # create queryDTO object @@ -77,6 +78,7 @@ ] json_result, fg_name = create_cached_featuregroup(@project.id, featurestore_id, features: features, featuregroup_name: "test_fg_a_#{short_random_id}", online:true) parsed_json = JSON.parse(json_result) + expect_status_details(201) fg_id = parsed_json["id"] fg_type = parsed_json["type"] # create second feature group @@ -86,6 +88,7 @@ ] json_result_b, fg_name_b = create_cached_featuregroup(@project.id, featurestore_id, features: features, featuregroup_name: "test_fg_b_#{short_random_id}", online:true) parsed_json_b = JSON.parse(json_result_b) + expect_status_details(201) fg_id_b = parsed_json_b["id"] fg_b_type = parsed_json_b["type"] # create queryDTO object @@ -135,6 +138,7 @@ ] json_result, fg_name = create_cached_featuregroup(@project.id, featurestore_id, features: features, featuregroup_name: "test_fg_a_#{short_random_id}", online:true) parsed_json = JSON.parse(json_result) + expect_status_details(201) fg_id = parsed_json["id"] fg_type = parsed_json["type"] # create second feature group @@ -144,6 +148,7 @@ ] json_result_b, fg_name_b = create_cached_featuregroup(@project.id, featurestore_id, features: features, featuregroup_name: "test_fg_b_#{short_random_id}", online:true) parsed_json_b = JSON.parse(json_result_b) + expect_status_details(201) fg_id_b = parsed_json_b["id"] fg_b_type = parsed_json_b["type"] # create third feature group @@ -153,6 +158,7 @@ ] json_result_c, fg_name_c = create_cached_featuregroup(@project.id, featurestore_id, features: features, featuregroup_name: "test_fg_c_#{short_random_id}", online:true) parsed_json_c = JSON.parse(json_result_c) + expect_status_details(201) fg_id_c = parsed_json_c["id"] fg_c_type = parsed_json_c["type"] # create queryDTO object @@ -218,6 +224,7 @@ ] json_result, fg_name = create_cached_featuregroup(@project.id, featurestore_id, features: features, featuregroup_name: "test_fg_a_#{short_random_id}", online:true) parsed_json = JSON.parse(json_result) + expect_status_details(201) fg_id = parsed_json["id"] fg_type = parsed_json["type"] # create second feature group @@ -227,6 +234,7 @@ ] json_result_b, fg_name_b = create_cached_featuregroup(@project.id, featurestore_id, features: features, featuregroup_name: "test_fg_b_#{short_random_id}", online:true) parsed_json_b = JSON.parse(json_result_b) + expect_status_details(201) fg_id_b = parsed_json_b["id"] fg_b_type = parsed_json_b["type"] # create queryDTO object @@ -286,6 +294,7 @@ ] json_result, fg_name_a = create_cached_featuregroup(@project.id, featurestore_id, features: features, featuregroup_name: "test_fg_a_#{short_random_id}", online:true) parsed_json = JSON.parse(json_result) + expect_status_details(201) fg_id = parsed_json["id"] fg_type = parsed_json["type"] # create second feature group @@ -295,6 +304,7 @@ ] json_result_b, fg_name_b = create_cached_featuregroup(@project.id, featurestore_id, features: features, featuregroup_name: "test_fg_b_#{short_random_id}", online:false) parsed_json_b = JSON.parse(json_result_b) + expect_status_details(201) fg_id_b = parsed_json_b["id"] fg_b_type = parsed_json_b["type"] # create queryDTO object @@ -337,6 +347,7 @@ ] json_result, fg_name_a = create_cached_featuregroup(@project.id, featurestore_id, features: features, featuregroup_name: "test_fg_a_#{short_random_id}", online:true) parsed_json = JSON.parse(json_result) + expect_status_details(201) fg_id = parsed_json["id"] fg_type = parsed_json["type"] # create second feature group @@ -346,6 +357,7 @@ ] json_result_b, fg_name_b = create_cached_featuregroup(@project.id, featurestore_id, features: features, featuregroup_name: "test_fg_b_#{short_random_id}", online:false) parsed_json_b = JSON.parse(json_result_b) + expect_status_details(201) fg_id_b = parsed_json_b["id"] fg_b_type = parsed_json_b["type"] # create queryDTO object @@ -391,6 +403,7 @@ ] json_result, fg_name = create_cached_featuregroup(@project.id, featurestore_id, features: features, featuregroup_name: "test_fg_a_#{short_random_id}", online:true) parsed_json = JSON.parse(json_result) + expect_status_details(201) fg_id = parsed_json["id"] fg_type = parsed_json["type"] diff --git a/hopsworks-IT/src/test/ruby/spec/serving_spec.rb b/hopsworks-IT/src/test/ruby/spec/serving_spec.rb index 21910b8145..965a830ed8 100644 --- a/hopsworks-IT/src/test/ruby/spec/serving_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/serving_spec.rb @@ -338,7 +338,7 @@ servingTool: "DEFAULT", requestedInstances: 1 }) - expect_json(errorMsg: "Maximum topic replication factor exceeded") + expect_json(errorMsg: "Invalid number of partitions") expect_status_details(400) end diff --git a/hopsworks-IT/src/test/ruby/spec/session_spec.rb b/hopsworks-IT/src/test/ruby/spec/session_spec.rb index c729c4c19f..ec27b8b127 100644 --- a/hopsworks-IT/src/test/ruby/spec/session_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/session_spec.rb @@ -85,15 +85,14 @@ expect_status_details(200) end - it "should fail to login if false login > 20 for agent" do + it "should not fail to login if false login > 20 for agent" do params={} user = create_user_with_role(params, "AGENT") set_false_login(user, 21) try_login(user, "Pass1234") expect_status_details(401) try_login(user, "Pass123") - expect_json(errorCode: 160007) - expect_status_details(401) + expect_status_details(200) end it "should fail to login with blocked account (status 4)" do diff --git a/hopsworks-IT/src/test/ruby/spec/spec_helper.rb b/hopsworks-IT/src/test/ruby/spec/spec_helper.rb index 5feb2fd50e..dbbac464b9 100644 --- a/hopsworks-IT/src/test/ruby/spec/spec_helper.rb +++ b/hopsworks-IT/src/test/ruby/spec/spec_helper.rb @@ -122,6 +122,8 @@ config.example_status_persistence_file_path = "#{ENV['PROJECT_DIR']}#{ENV['RSPEC_TEST_STATUS']}" # uncomment next line if you need to clean hdfs and hopsworks db before test. # config.before(:suite) { clean_test_data } + config.before(:suite) { try_start_all_services } # make sure all services are running + config.after(:suite) { # If we are not using Jenkins, then clean the data if ARGV.grep(/spec\.rb/).empty? && (!ENV['JENKINS'] || ENV['JENKINS'] == "false") @@ -145,6 +147,22 @@ config.timeout = 120 end +def try_start_all_services + begin + command = "/srv/hops/kagent/kagent/bin/start-all-local-services.sh" + if ENV['OS'] == 'ubuntu' + sh.exec!(command) # hopsworks0 + else + sh.exec!(command) # hopsworks0 + sh.exec!("su vagrant -c \"ssh -o StrictHostKeyChecking=no hopsworks1 -t 'sudo #{command}'\"") + sh.exec!("su vagrant -c \"ssh -o StrictHostKeyChecking=no hopsworks2 -t 'sudo #{command}'\"") + sh.exec!("su vagrant -c \"ssh -o StrictHostKeyChecking=no hopsworks3 -t 'sudo #{command}'\"") + end + rescue + puts 'Failed to start all services.' + end +end + def clean_test_data puts "Cleaning test data ..." require 'net/ssh' diff --git a/hopsworks-IT/src/test/ruby/spec/storage_connector_spec.rb b/hopsworks-IT/src/test/ruby/spec/storage_connector_spec.rb index 3d10d36f6c..695d441802 100644 --- a/hopsworks-IT/src/test/ruby/spec/storage_connector_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/storage_connector_spec.rb @@ -526,7 +526,7 @@ # StorageConnectorUtils.isStorageConnectorTypeEnabled() decides which connector type is enabled, and Unit tests can be found in the corresponding folder. before :all do - # enable kafka storage connectors temporarily + # enable kafka storage connectors if not enabled @enable_kafka_storage_connectors = getVar('enable_kafka_storage_connectors') setVar('enable_kafka_storage_connectors', "true") @@ -557,7 +557,7 @@ @connector_name = parsed_json["name"] # disable kafka storage connectors - setVar('enable_kafka_storage_connectors',"false") + setVar('enable_kafka_storage_connectors', "false") create_session(@project[:username], "Pass123") end @@ -574,7 +574,7 @@ delete_connector(project.id, featurestore_id, @connector_name) expect_status_details(200) - # disable kafka storage connectors + # set back default value setVar('enable_kafka_storage_connectors', @enable_kafka_storage_connectors[:value]) create_session(@project[:username], "Pass123") @@ -966,7 +966,9 @@ expect(parsed_json["featurestoreId"]).to eql(featurestore_id) expect(parsed_json["name"]).to eql(@connector_name) expect(parsed_json["storageConnectorType"]).to eql("KAFKA") - expect(parsed_json["bootstrapServers"]).to eql("10.0.2.15:9091") + if ENV['OS'] == "ubuntu" + expect(parsed_json["bootstrapServers"]).to eql("10.0.2.15:9091") + end expect(parsed_json["securityProtocol"]).to eql("SSL") expect(parsed_json.key?("sslTruststoreLocation")).to be false expect(parsed_json.key?("sslTruststorePassword")).to be false @@ -1027,7 +1029,9 @@ expect(parsed_json["featurestoreId"]).to eql(featurestore_id) expect(parsed_json["name"]).to eql(@connector_name) expect(parsed_json["storageConnectorType"]).to eql("KAFKA") - expect(parsed_json["bootstrapServers"]).to eql("10.0.2.15:9091") + if ENV['OS'] == "ubuntu" + expect(parsed_json["bootstrapServers"]).to eql("10.0.2.15:9091") + end expect(parsed_json["securityProtocol"]).to eql("SSL") expect(parsed_json.key?("sslTruststoreLocation")).to be false expect(parsed_json.key?("sslTruststorePassword")).to be false @@ -1062,7 +1066,9 @@ expect(parsed_json["featurestoreId"]).to eql(featurestore_id) expect(parsed_json["name"]).to eql(@connector_name) expect(parsed_json["storageConnectorType"]).to eql("KAFKA") - expect(parsed_json["bootstrapServers"]).to eql("10.0.2.15:9091") + if ENV['OS'] == "ubuntu" + expect(parsed_json["bootstrapServers"]).to eql("10.0.2.15:9091") + end expect(parsed_json["securityProtocol"]).to eql("SSL") expect(parsed_json.key?("sslTruststoreLocation")).to be false expect(parsed_json.key?("sslTruststorePassword")).to be false diff --git a/hopsworks-IT/src/test/ruby/spec/stream_feature_group_spec.rb b/hopsworks-IT/src/test/ruby/spec/stream_feature_group_spec.rb index ea8d6f04f7..afa3b5d5be 100644 --- a/hopsworks-IT/src/test/ruby/spec/stream_feature_group_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/stream_feature_group_spec.rb @@ -717,8 +717,8 @@ "(SELECT `fg1`.`a_testfeature` `a_testfeature`, `fg1`.`a_testfeature1` `a_testfeature1`, `fg1`.`a_testfeature` `join_pk_a_testfeature`, `fg1`.`event_time` `join_evt_event_time`, `fg0`.`b_testfeature1` `b_testfeature1`, RANK() OVER (PARTITION BY `fg0`.`a_testfeature`, `fg1`.`event_time` ORDER BY `fg0`.`event_time` DESC) pit_rank_hopsworks\n" + "FROM `#{project_name.downcase}_featurestore`.`#{fg_a_name}_1` `fg1`\n" + "INNER JOIN `#{project_name.downcase}_featurestore`.`#{fg_b_name}_1` `fg0` ON `fg1`.`a_testfeature` = `fg0`.`a_testfeature` AND `fg1`.`event_time` >= `fg0`.`event_time`\n" + - "WHERE (`fg1`.`a_testfeature` = 10 OR `fg0`.`b_testfeature1` = 10) AND `fg0`.`b_testfeature2` = 10) NA\n" + - "WHERE `pit_rank_hopsworks` = 1) (SELECT `right_fg0`.`a_testfeature` `a_testfeature`, `right_fg0`.`a_testfeature1` `a_testfeature1`, `right_fg0`.`b_testfeature1` `b_testfeature1`\nFROM right_fg0)") + "WHERE `fg0`.`b_testfeature2` = 10) NA\n" + + "WHERE `pit_rank_hopsworks` = 1) (SELECT `right_fg0`.`a_testfeature` `a_testfeature`, `right_fg0`.`a_testfeature1` `a_testfeature1`, `right_fg0`.`b_testfeature1` `b_testfeature1`\nFROM right_fg0\nWHERE `right_fg0`.`a_testfeature` = 10 OR `right_fg0`.`b_testfeature1` = 10)") end it "should be able to create stream feature group with many features and get features in specific order on get" do diff --git a/hopsworks-IT/src/test/ruby/spec/variables_spec.rb b/hopsworks-IT/src/test/ruby/spec/variables_spec.rb index c4dfba28f7..e470a14a78 100644 --- a/hopsworks-IT/src/test/ruby/spec/variables_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/variables_spec.rb @@ -148,5 +148,53 @@ end end end + describe "allowed maximum number of executions per job" do + before :all do + @exec_per_job = getVar('executions_per_job_limit') + setVar('executions_per_job_limit', 2) + with_valid_tour_project("spark") + end + after :all do + setVar('executions_per_job_limit', @exec_per_job [:value]) + clean_jobs(@project[:id]) + end + it "should not start more than the allowed maximum number of executions per job" do + job_name = "demo_job_3" + create_sparktour_job(@project, job_name, 'jar') + begin + start_execution(@project[:id], job_name) + execution_id1 = json_body[:id] + start_execution(@project[:id], job_name) + execution_id2 = json_body[:id] + start_execution(@project[:id], job_name, expected_status: 400, error_code: 130040) + ensure + wait_for_execution_completed(@project[:id], job_name, execution_id1, "FINISHED") unless execution_id1.nil? + wait_for_execution_completed(@project[:id], job_name, execution_id2, "FINISHED") unless execution_id2.nil? + end + end + end + describe "with quota enabled" do + before :all do + setVar("quotas_model_deployments_running", "1") + @local_project = create_project + with_tensorflow_serving(@local_project.id, @local_project.projectname, @user.username) + end + after :all do + setVar("quotas_model_deployments_running", "-1") + purge_all_tf_serving_instances + delete_all_servings(@local_project.id) + end + it "should fail to start serving if quota has been reached" do + ## This deployment should start + start_serving(@local_project, @serving) + + second_serving = create_tensorflow_serving(@local_project.id, @local_project.projectname) + ## Starting this one should fail because quota has beed reached + post "#{ENV['HOPSWORKS_API']}/project/#{@local_project.id}/serving/#{second_serving.id}?action=start" + expect_status_details(400) + parsed = JSON.parse(response) + expect(parsed['devMsg']).to include("quota") + end + end end end diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/agent/AgentView.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/agent/AgentView.java index 967814104e..49355ad632 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/agent/AgentView.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/agent/AgentView.java @@ -17,6 +17,7 @@ package io.hops.hopsworks.api.agent; +import com.fasterxml.jackson.annotation.JsonAlias; import io.hops.hopsworks.common.agent.AgentController; import io.hops.hopsworks.persistence.entity.command.SystemCommand; import io.swagger.annotations.ApiModel; @@ -33,25 +34,30 @@ public class AgentView { // Register @XmlElement(name = "host-id") + @JsonAlias({"host-id"}) private String hostId; private String password; private String hadoopHome; // Heartbeat @XmlElement(name = "agent-time") + @JsonAlias({"agent-time"}) private Long agentTime; @XmlElement(name = "num-gpus") + @JsonAlias({"num-gpus"}) private Integer numGpus; @XmlElement(name = "memory-capacity") + @JsonAlias({"memory-capacity"}) private Long memoryCapacity; private Integer cores; @XmlElement(name = "private-ip") + @JsonAlias({"private-ip"}) private String privateIp; - @XmlElement(name = "services") + private List services; @XmlElement(name = "system-commands") + @JsonAlias({"system-commands"}) private List systemCommands; - @XmlElement(name = "recover") private Boolean recover; public AgentView() { diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/JupyterNotebookCleaner.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/JupyterNotebookCleaner.java index 734af7de21..313ff44566 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/JupyterNotebookCleaner.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/jupyter/JupyterNotebookCleaner.java @@ -68,8 +68,7 @@ @DependsOn("Settings") public class JupyterNotebookCleaner { - private static final Logger LOGGER = Logger.getLogger( - JupyterNotebookCleaner.class.getName()); + private static final Logger LOGGER = Logger.getLogger(JupyterNotebookCleaner.class.getName()); @EJB private JupyterFacade jupyterFacade; @@ -108,6 +107,11 @@ public void execute(Timer timer) { if (!payaraClusterManager.amIThePrimary()) { return; } + doCleanup(); + } + + @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED) + public void doCleanup() { try { LOGGER.log(Level.FINE, "Running JupyterNotebookCleaner."); // 1. Get all Running Jupyter Notebook Servers @@ -120,8 +124,8 @@ public void execute(Timer timer) { if (!jp.isNoLimit() && jp.getExpires().before(currentDate)) { try { LOGGER.log(Level.FINE, - "Shutting down expired notebook user: " + jp.getUser().getUsername() - + " project: " + jp.getProject().getName()); + "Shutting down expired notebook user: " + jp.getUser().getUsername() + + " project: " + jp.getProject().getName()); jupyterController.shutdown(jp.getProject(), jp.getUser(), jp.getSecret(), jp.getCid(), jp.getPort()); } catch (Exception e) { LOGGER.log(Level.SEVERE, "Failed to cleanup notebook with port " + jp.getPort(), e); diff --git a/hopsworks-testing/src/main/java/io/hops/hopsworks/testing/ApplicationConfig.java b/hopsworks-testing/src/main/java/io/hops/hopsworks/testing/ApplicationConfig.java index 37c090dc54..98dd46cb72 100644 --- a/hopsworks-testing/src/main/java/io/hops/hopsworks/testing/ApplicationConfig.java +++ b/hopsworks-testing/src/main/java/io/hops/hopsworks/testing/ApplicationConfig.java @@ -30,6 +30,7 @@ public ApplicationConfig() { register(io.hops.hopsworks.testing.provenance.TestProjectProvenanceResource.class); register(io.hops.hopsworks.testing.provenance.TestProvenanceService.class); register(io.hops.hopsworks.testing.user.TestAuditedUserAdministration.class); + register(io.hops.hopsworks.testing.project.TestJupyterService.class); //swagger register(io.swagger.jaxrs.listing.ApiListingResource.class); diff --git a/hopsworks-testing/src/main/java/io/hops/hopsworks/testing/project/TestJupyterService.java b/hopsworks-testing/src/main/java/io/hops/hopsworks/testing/project/TestJupyterService.java new file mode 100644 index 0000000000..47ec54073e --- /dev/null +++ b/hopsworks-testing/src/main/java/io/hops/hopsworks/testing/project/TestJupyterService.java @@ -0,0 +1,51 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2023, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.testing.project; + +import io.hops.hopsworks.api.filter.Audience; +import io.hops.hopsworks.common.jupyter.JupyterNotebookCleaner; +import io.hops.hopsworks.jwt.annotation.JWTRequired; +import io.swagger.annotations.Api; + +import javax.ejb.EJB; +import javax.ejb.Stateless; +import javax.ejb.TransactionAttribute; +import javax.ejb.TransactionAttributeType; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.logging.Logger; + +@Path("/test/jupyter") +@Stateless +@JWTRequired(acceptedTokens = {Audience.API}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) +@Produces(MediaType.TEXT_PLAIN) +@Api(value = "Jupyter Testing Service", description = "Jupyter Testing Service") +@TransactionAttribute(TransactionAttributeType.NEVER) +public class TestJupyterService { + private static final Logger LOGGER = Logger.getLogger(TestJupyterService.class.getName()); + @EJB + private JupyterNotebookCleaner jupyterNotebookCleaner; + + @GET + @Path("/cleanup") + public Response doCleanup() { + jupyterNotebookCleaner.doCleanup(); + return Response.ok("Cleaned").build(); + } +}