[HWORKS-777] fix tests (#1419)

hopsworks-IT/src/test/ruby/isolated_tests

@@ -3,11 +3,9 @@ variables_spec.rb
 agent_spec.rb
 admin_hosts_spec.rb
 audit_spec.rb
-ee_epipe_spec.rb
 ee_search_spec.rb
 ee_tags_spec.rb
 epipe_spec.rb
-ee_epipe_spec.rb
 prov_ops_spec.rb
 prov_state_spec.rb
 search_spec.rb

hopsworks-IT/src/test/ruby/spec/agent_spec.rb

@@ -38,12 +38,12 @@
       end
 
       it "should not be able to register" do
-        post @register_resource, {"host-id": "host0", password: "password"}
+        post @register_resource, {"host-id": "host0", "private-ip": "10.0.0.15", password: "password"}
         expect_status_details(401)
       end
 
       it "should not be able to heartbeat" do
-        post @heartbeat_resource, {"host-id": "host0", agentTime: "1234"}
+        post @heartbeat_resource, {"host-id": "host0", "private-ip": "10.0.1.15", agentTime: "1234"}
         expect_status_details(401)
       end
     end
@@ -72,14 +72,19 @@
         before(:all) do
           @random_host = "host_#{short_random_id}"
         end
-        it "should not be able to register" do
-          post @register_resource, {"host-id": @random_host, password: "some_pass"}
-          expect_status_details(404, error_code: 100025)
+        after(:all) do
+          host = find_by_hostname(@random_host)
+          host.destroy
+        end
+        # [HWORKS-705] allowed any host to register and heartbeat
+        it "should be able to register random host" do
+          post @register_resource, {"host-id": @random_host, "private-ip": "10.0.12.15", password: "some_pass"}
+          expect_status_details(200)
         end
 
-        it "should not be able to heartbeat" do
-          post @heartbeat_resource, {"host-id": @random_host, agentTime: "1234"}
-          expect_status_details(404)
+        it "should be able to heartbeat to random host" do
+          post @heartbeat_resource, {"host-id": @random_host, "private-ip": "10.0.12.15", "agent-time": "1234"}
+          expect_status_details(200)
         end
       end
 
@@ -97,15 +102,15 @@
         it "should be able to register" do
           host = find_by_hostname(@hostname)
           expect(host.registered).to eq(false)
-          post @register_resource, {"host-id": @hostname, password: "pass123"}
+          post @register_resource, {"host-id": @hostname, "private-ip": "10.0.4.15", password: "pass123"}
           expect_status_details(200)
           host = find_by_hostname(@hostname)
           expect(host.registered).to eq(true)
         end
 
         it "should be able to heartbeat" do
-          post @register_resource, {"host-id": @hostname, password: "pass123"}
-          post @heartbeat_resource, {"host-id": @hostname, "num-gpus": 0, "agent-time": 1,
+          post @register_resource, {"host-id": @hostname, "private-ip": "10.0.5.15", password: "pass123"}
+          post @heartbeat_resource, {"host-id": @hostname, "private-ip": "10.0.6.15", "num-gpus": 0, "agent-time": 1,
                                      "cores": 4, "memory-capacity": 2}
           expect_status_details(200)
           host = find_by_hostname(@hostname)

hopsworks-IT/src/test/ruby/spec/airflow_spec.rb

@@ -52,15 +52,14 @@
       end
 
       it "should be able to compose DAG" do
-        get "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/airflow/secretDir"
-        expect_status_details(200)
-        secret_dir = response.body
-
         post "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/airflow/dag", @dag_definition
         expect_status_details(200)
-        airflow_dir = Variables.find_by(id: "airflow_dir")
-        dag_file = File.join(airflow_dir.value, "dags", secret_dir, "#{@dag_name}.py")
-        expect(File.exists?(dag_file)).to be true
+
+        get_dataset_stat(@project, "Airflow/#{@dag_name}.py", datasetType: "&type=DATASET")
+        expect_status_details(200)
+        ds = json_body
+        expect(ds[:attributes][:name]).to eq ("#{@dag_name}.py")
+        expect(ds[:attributes][:owner]).to eq ("#{@user[:fname]} #{@user[:lname]}")
       end
     end
   end

hopsworks-IT/src/test/ruby/spec/audit_spec.rb

@@ -37,6 +37,12 @@
     end
   end
   describe "Log" do
+    before(:all) do
+      #Logs can end up on any node in HA
+      if ENV['OS'] == "centos"
+        skip "These tests do not run on centos"
+      end
+    end
     context 'user login' do
       it 'should add row to log file' do
         newUser = create_user

hopsworks-IT/src/test/ruby/spec/auxiliary/ml/notebooks/end_to_end_pipeline/sklearn/iris_flower_classifier.py

@@ -5,15 +5,10 @@
 class Predict(object):
 
     def __init__(self):
-        """ Initializes the serving state, reads a trained model from HDFS"""
-        self.model_path = "Models/irisflowerclassifier/1/iris_knn.pkl"
-        print("Copying SKLearn model from HDFS to local directory")
-        hdfs.copy_to_local(self.model_path)
         print("Reading local SkLearn model for serving")
         self.model = joblib.load("./iris_knn.pkl")
         print("Initialization Complete")
 
-
     def predict(self, inputs):
         """ Serves a prediction request usign a trained model"""
         return self.model.predict(inputs).tolist() # Numpy Arrays are note JSON serializable

hopsworks-IT/src/test/ruby/spec/bigquery_storage_connector_spec.rb

@@ -141,7 +141,7 @@ def create_connector_materializationDataset
     expect(parsed_result_update['dataset']).to eql(dataset)
     expect(parsed_result_update['queryTable']).to eql(table)
     expect(parsed_result_update['queryTable']).to eql(table)
-    expect(parsed_result_update['arguments']).to eql([])
+    expect(parsed_result_update['arguments']).to be_empty
   end
 
   it 'should fail to update non existing connector' do

hopsworks-IT/src/test/ruby/spec/dataset_spec.rb

@@ -1717,7 +1717,7 @@
         end
         it 'should get 0 result if offset >= len.' do
           get_datasets_in_path(@project, @dataset[:inode_name], query: "&sort_by=id:asc&limit=10&offset=2500")
-          expect(json_body[:items]).to be nil
+          expect(json_body[:items]).to be_empty
         end
       end
     end

hopsworks-IT/src/test/ruby/spec/execution_spec.rb

@@ -230,20 +230,6 @@ def run_job_and_get_logs(project, job_name)
           #start execution
           start_execution(@project[:id], $job_name_2, expected_status: 400)
         end
-        it "should not start more than the allowed maximum number of executions per job" do
-          $job_name_3 = "demo_job_3_" + type
-          create_sparktour_job(@project, $job_name_3, type)
-          begin
-            start_execution(@project[:id], $job_name_3)
-            execution_id1 = json_body[:id]
-            start_execution(@project[:id], $job_name_3)
-            execution_id2 = json_body[:id]
-            start_execution(@project[:id], $job_name_3, expected_status: 400, error_code: 130040)
-          ensure
-            wait_for_execution_completed(@project[:id], $job_name_3, execution_id1, "FINISHED") unless execution_id1.nil?
-            wait_for_execution_completed(@project[:id], $job_name_3, execution_id2, "FINISHED") unless execution_id2.nil?
-          end
-        end
         it "should start a job and use default args" do
           $job_name_3 = "demo_job_3_" + type
           create_sparktour_job(@project, $job_name_3, type)
@@ -642,8 +628,10 @@ def run_job_and_get_logs(project, job_name)
         @project = create_project
         add_member_to_project(@project, @user_data_scientist[:email], "Data scientist")
         @job_name = "test_job_#{short_random_id}"
-
         run_job(@user_data_owner, @project, @job_name)
+        get_executions(@project["id"], @job_name)
+        execution_id = json_body[:items][0][:id]
+        wait_for_execution_active(@project[:id], @job_name, execution_id, "FINISHED", "appId")
       end
 
       def setup_job(user, project, job_name)

hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb

@@ -2077,8 +2077,9 @@
         expect_status_details(200)
         topic = json_body[:items].select{|topic| topic[:name] == topic_name}
         expect(topic.length).to eq(1)
+        #disableOnline=true should delete the subject
         get_subject_schema(project, featuregroup_name + "_" + parsed_json["version"].to_s, 1)
-        expect_status_details(200)
+        expect_status_details(404)
       end
 
       it "should be possible to preview from online storage of an on-demand/external feature group" do
@@ -2645,8 +2646,8 @@
 
         # add sample ros
         OnlineFg.db_name = project[:projectname]
-        OnlineFg.create(testfeature: 1).save
-        OnlineFg.create(testfeature: 2).save
+        OnlineFg.create(testfeature: 3).save
+        OnlineFg.create(testfeature: 4).save
 
         get "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups/" + featuregroup_id.to_s + "/preview?storage=online&limit=1"
         expect_status_details(200)
@@ -3012,9 +3013,10 @@
           topic = []
         end
         expect(topic.length).to eq(1)
+        #disableOnline=true should delete the subject
         get_subject_schema(project, featuregroup_name + "_" + parsed_json["version"].to_s, 1)
-        expect_status_details(200)
-        expect(json_body[:error_code]).to eql(nil)
+        expect_status_details(404)
+        expect(json_body[:error_code]).to eql(40401)
       end
 
        it "should update avro schema when features are appended to existing online feature group" do

hopsworks-IT/src/test/ruby/spec/featureview_trainingdataset_spec.rb

@@ -40,7 +40,6 @@
           expect(parsed_json.key?("dataFormat")).to be true
           expect(parsed_json.key?("trainingDatasetType")).to be true
           expect(parsed_json.key?("location")).to be true
-          expect(parsed_json.key?("inodeId")).to be true
           expect(parsed_json.key?("seed")).to be true
           expect(parsed_json["featurestoreName"] == featurestore_name).to be true
           expect(parsed_json["name"] == "#{featureview['name']}_#{featureview['version']}").to be true
@@ -319,7 +318,6 @@
           expect(parsed_json2.key?("version")).to be true
           expect(parsed_json2.key?("dataFormat")).to be true
           expect(parsed_json2.key?("trainingDatasetType")).to be true
-          expect(parsed_json2.key?("inodeId")).to be true
 
           expect(parsed_json2["version"]).to eql(parsed_json["version"])
           # make sure the dataformat didn't change

hopsworks-IT/src/test/ruby/spec/helpers/command_helper.rb

@@ -48,7 +48,7 @@ def wait_on_command(wait_time: 10, repeat: 1, &log_size)
     result[0]
   end
 
-  def wait_on_command_search(wait_time: 10, repeat: 6)
+  def wait_on_command_search(wait_time: 20, repeat: 6)
     result = wait_on_command(wait_time: wait_time, repeat: repeat) do
       count = CommandSearch.count
       count

hopsworks-IT/src/test/ruby/spec/helpers/jupyter_helper.rb

@@ -130,8 +130,8 @@ def recentnotebooks_search(project, expected_count)
       pp "#{ENV['HOPSWORKS_API']}/project/#{project[:id]}/jupyter/recent" if (defined?(@debugOpt)) && @debugOpt
       get "#{ENV['HOPSWORKS_API']}/project/#{project[:id]}/jupyter/recent"
       expect_status_details(200)
-	  begin
-		expect(json_body[:items]).not_to be_nil
+	    begin
+		    expect(json_body[:items]).not_to be_nil
         expect(json_body[:items].length).to eql(expected_count)
         { 'success' => true }
       rescue RSpec::Expectations::ExpectationNotMetError => e

hopsworks-IT/src/test/ruby/spec/jupyter_spec.rb

@@ -85,20 +85,22 @@
             secret_dir, staging_dir, settings = start_jupyter(@project)
 
             jupyter_running(@project, expected_status: 200)
-
-            jwt_file = File.join(staging_dir, "token.jwt")
-            expect(File.file? jwt_file).to be true
-
-            jupyter_dir = Variables.find_by(id: "jupyter_dir").value
-            project_username = "#{@project[:projectname]}__#{@user[:username]}"
-            path2secret = File.join(jupyter_dir, "Projects", @project[:projectname], project_username, secret_dir, "certificates")
-
-            kstore_file = File.join(path2secret, "#{project_username}__kstore.jks")
-            expect(File.file? kstore_file).to be true
-            tstore_file = File.join(path2secret, "#{project_username}__tstore.jks")
-            expect(File.file? tstore_file).to be true
-            password_file = File.join(path2secret, "#{project_username}__cert.key")
-            expect(File.file? password_file).to be true
+            if ENV['OS'] == "ubuntu"
+              # Token is in staging_dir if not kube
+              jwt_file = File.join(staging_dir, "token.jwt")
+              expect(File.file? jwt_file).to be true
+
+              jupyter_dir = Variables.find_by(id: "jupyter_dir").value
+              project_username = "#{@project[:projectname]}__#{@user[:username]}"
+              path2secret = File.join(jupyter_dir, "Projects", @project[:projectname], project_username, secret_dir, "certificates")
+
+              kstore_file = File.join(path2secret, "#{project_username}__kstore.jks")
+              expect(File.file? kstore_file).to be true
+              tstore_file = File.join(path2secret, "#{project_username}__tstore.jks")
+              expect(File.file? tstore_file).to be true
+              password_file = File.join(path2secret, "#{project_username}__cert.key")
+              expect(File.file? password_file).to be true
+            end
 
             # Check that the logs are written in the opensearch index.
             begin
@@ -212,17 +214,18 @@
             expect_status_details(200)
             notebook_file = json_body[:items].detect { |d| d[:attributes][:name] == "export_model.ipynb" }
             expect(notebook_file).to be_present
-
-            create_dir(@project, "Resources/test_dir", query: "&type=DATASET")
+            # test_dir will be created for each kernel
+            test_dir = "test_dir#{short_random_id}"
+            create_dir(@project, "Resources/#{test_dir}", query: "&type=DATASET")
             expect_status_details(201)
 
-            copy_dataset(@project, "Resources/export_model.ipynb", "/Projects/#{@project[:projectname]}/Resources/test_dir/[export model].ipynb", datasetType: "&type=DATASET")
+            copy_dataset(@project, "Resources/export_model.ipynb", "/Projects/#{@project[:projectname]}/Resources/#{test_dir}/[export model].ipynb", datasetType: "&type=DATASET")
             expect_status_details(204)
 
-            get "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/jupyter/convertIPythonNotebook/Resources/test_dir/%5Bexport%20model%5D.ipynb"
+            get "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/jupyter/convertIPythonNotebook/Resources/#{test_dir}/%5Bexport%20model%5D.ipynb"
             expect_status_details(200)
 
-            get "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/dataset/Resources/test_dir/?action=listing&expand=inodes"
+            get "#{ENV['HOPSWORKS_API']}/project/#{@project[:id]}/dataset/Resources/#{test_dir}/?action=listing&expand=inodes"
             expect_status_details(200)
             python_file = json_body[:items].detect { |d| d[:attributes][:name] == "[export model].py" }
             expect(python_file).to be_present
@@ -282,11 +285,13 @@
             get_settings(@project)
             shutdownLevel=6
             settings = json_body
-            settings[:distributionStrategy] = ""
             settings[:shutdownLevel] = shutdownLevel
             settings[:pythonKernel] = false
             settings[:jobConfig][:"spark.executor.memory"] = 1023
             start_jupyter(@project, settings: settings, expected_status: 400, error_code: 130029)
+            # set back spark.executor.memory=1024
+            settings[:jobConfig][:"spark.executor.memory"] = 1024
+            update_jupyter(@project, settings)
           end
 
           it "should not allow starting multiple notebook servers" do
@@ -301,11 +306,10 @@
             secret_dir, staging_dir, settings = start_jupyter(@project, shutdownLevel=6, baseDir=nil, noLimit=true)
             jupyter_running(@project, expected_status: 200)
 
-            parsed_json = JSON.parse(json_result)
-            expect(parsed_json["noLimit"]).to eq(true)
+            expect(json_body[:noLimit]).to eq(true)
 
             stop_jupyter(@project)
-            jupyter_runningt(@project, expected_status: 404)
+            jupyter_running(@project, expected_status: 404)
           end
 
           it "should allow multiple restarts" do
@@ -326,11 +330,15 @@
             jupyter_running(@project, expected_status: 200)
             json_body[:minutesUntilExpiration].should be < 2
 
-            wait_for_me_time(90, 5) do
-              jupyter_running(@project)
+            wait_for_me_time(180, 5) do
+              jupyter_running(@project, expected_status: 200)
               is_running = response.code == resolve_status(200, response.code)
               { 'success' => is_running }
             end
+            # JupyterNotebookCleaner is running every 30min it does not make sense to wait 30min in a test
+            get "#{ENV['HOPSWORKS_TESTING']}/test/jupyter/cleanup"
+            expect_status(200) # No detail returned
+
             jupyter_running(@project, expected_status: 404)
           end
 
@@ -343,6 +351,9 @@
             json_body[:minutesUntilExpiration].should be > initial_minutes_left-3
 
             sleep(90)
+            get "#{ENV['HOPSWORKS_TESTING']}/test/jupyter/cleanup"
+            expect_status(200) # No detail returned
+
             jupyter_running(@project, expected_status: 200)
 
             json_body[:minutesUntilExpiration].should be < initial_minutes_left