Implementing the lab lockdown (demisto#11149)

* update sdk version

* Implementing lab lockdown

Co-authored-by: content-bot <[email protected]>
Co-authored-by: hod-alpert <[email protected]>

.circleci/config.yml

@@ -5,12 +5,70 @@ commands:
     parameters:
       instance_role:
         type: string
+      instance_created:
+        type: string
+        default: ""
     steps:
       - run:
           name: Set INSTANCE_ROLE env variable
           command: |
             echo 'export INSTANCE_ROLE="<< parameters.instance_role >>"' >> $BASH_ENV
-            source $BASH_ENV
+            echo 'export INSTANCE_WAS_CREATED="<< parameters.instance_created >>"' >> $BASH_ENV
+  start-tunnel:
+    description: "Opens an ssh tunnel to the demisto servers and wait until the tunnels are established"
+    parameters:
+      timeout:
+        type: integer
+        default: 10
+    steps:
+      - run:
+          name: add ssh configurations
+          shell: /bin/bash
+          command: |
+            if [ -z $INSTANCE_WAS_CREATED ];
+              then
+                echo "Skipping - instance was not created"
+                exit 0
+            fi
+            # Modifying ssh config file
+            echo "Host 10.0.*
+              StrictHostKeyChecking no
+              LogLevel ERROR
+              ProxyJump [email protected]  # disable-secrets-detection
+             Host content-build-lb.demisto.works
+              UserKnownHostsFile /dev/null
+              StrictHostKeyChecking no
+              LogLevel ERROR" >> ~/.ssh/config
+      - run:
+          name: Open SSH Tunnel
+          command: |
+            if [ -z $INSTANCE_WAS_CREATED ];
+              then
+                echo "Skipping - instance was not created"
+                exit 0
+            fi
+            # Generating the ips and ports with the following form: <instance-ip> <tunnel-port>
+            IPS_AND_PORTS=$(cat ./env_results.json | jq ".[] | select(.Role==\"$INSTANCE_ROLE\")" | jq -r '[.InstanceDNS, .TunnelPort] | @tsv' | sed "s/\"//g")
+            # Handling the ip & port pairs line by line
+            echo $IPS_AND_PORTS | grep -o -E "[0-9\.]+ [0-9]{4}" | while read IP_AND_PORT;
+            do
+              # Capturing the IP
+              IP=$(echo $IP_AND_PORT | grep -o -E "10\.0\.[0-9]{1,3}\.[0-9]{1,3}")
+              # Capturing the port
+              PORT=$(echo $IP_AND_PORT | grep -o -E "[0-9]{4}")
+              echo "Opening a tunnel for ip $IP with port $PORT"
+              ssh -4 -o "ServerAliveInterval=15" -f -N "[email protected]" -L "$PORT:$IP:443"  # disable-secrets-detection
+              echo "Waiting for tunnel to be established"
+              until nc -z 127.0.0.1 $PORT -v; do
+                if [ $COUNT -ge << parameters.timeout >> ]; then
+                  echo "ssh tunnel set up timeout on instance with ip $IP";
+                  exit 1;
+                fi;
+                ((COUNT++))
+                sleep 1
+              done
+            done
+
 parameters:
   non_ami_run:
     type: string
@@ -760,6 +818,7 @@ jobs:
             - set-instance-role-env-variable:
                 instance_role: Server 5.0
             - *check_if_instance_was_created
+            - start-tunnel
             - *wait_until_server_ready
             - *install_content_and_configure_integrations_on_server
             - run:
@@ -809,6 +868,7 @@ jobs:
             - set-instance-role-env-variable:
                 instance_role: Server 5.5
             - *check_if_instance_was_created
+            - start-tunnel
             - *wait_until_server_ready
             - *install_content_and_configure_integrations_on_server
             - run:
@@ -858,6 +918,7 @@ jobs:
             - set-instance-role-env-variable:
                 instance_role: Server 6.0
             - *check_if_instance_was_created
+            - start-tunnel
             - *wait_until_server_ready
             - *install_content_and_configure_integrations_on_server
             - run:
@@ -898,6 +959,7 @@ jobs:
       - set-instance-role-env-variable:
           instance_role: Server Master
       - *check_if_instance_was_created
+      - start-tunnel
       - *wait_until_server_ready
       - *install_content_and_configure_integrations_on_server
       - run:
@@ -1013,25 +1075,12 @@ jobs:
       - *prepare_environment
       - set-instance-role-env-variable:
           instance_role: Server 5.5
-      - run:
-          name: Wait until server ready
-          shell: /bin/bash
-          when: always
-          command: |
-            if [ -z $INSTANCE_TESTS ]
-            then
-              echo "Skipping, contributor branch not given."
-              exit 0
-            fi
-            python3 ./Tests/scripts/wait_until_server_ready.py "$INSTANCE_ROLE"
+          instance_created: "true"
+      - start-tunnel
+      - *wait_until_server_ready
       - run:
           name: Instance Test
           command: |
-            if [ -z $INSTANCE_TESTS ]
-            then
-              echo "Skipping, contributor branch not given."
-              exit 0
-            fi
             ./Tests/scripts/instance_test.sh
             export RETVAL=$?
             cp ./Tests/failed_instances.txt $CIRCLE_ARTIFACTS/failed_instances.txt
@@ -1082,20 +1131,18 @@ jobs:
       - *create_instance
       - set-instance-role-env-variable:
           instance_role: Server Master
-      - *wait_until_server_ready
-      - run:
-          name: Wait until server ready
-          command: |
-            python3 ./Tests/scripts/wait_until_server_ready.py "Server Master"
+          instance_created: "true"
+      - start-tunnel
       - run:
           name: Set Environment Variables
           command: |
             echo "export DEMISTO_API_KEY=$(cat $SECRET_CONF_PATH | jq -r '.temp_apikey')" >> $BASH_ENV
-            echo "export DEMISTO_BASE_URL=\"https://$(cat env_results.json | jq -r '.[0].InstanceDNS')\"" >> $BASH_ENV
+            echo "export DEMISTO_BASE_URL=\"https://localhost:$(cat env_results.json | jq -r '.[0].TunnelPort')\"" >> $BASH_ENV
             echo "Server URL: $DEMISTO_BASE_URL"
       - run:
           name: Install SDK Master
           command: pip3 install git+https://github.com/demisto/demisto-sdk.git
+      - *wait_until_server_ready
       - run:
           name: Unlock HelloWorld integration and playbook
           command: python3 Tests/sdknightly/unlock_entity.py integration HelloWorld
@@ -1140,6 +1187,8 @@ jobs:
       - *prepare_environment
       - set-instance-role-env-variable:
           instance_role: Server 6.0
+          instance_created: "true"
+      - start-tunnel
       - *wait_until_server_ready
       - run:
           name: Install Packs
@@ -1171,6 +1220,8 @@ jobs:
       - *prepare_environment
       - set-instance-role-env-variable:
           instance_role: Server Master
+          instance_created: "true"
+      - start-tunnel
       - *wait_until_server_ready
       - run:
           name: Install Packs

Tests/Marketplace/configure_and_install_packs.py

@@ -28,28 +28,28 @@ def main():
     options = options_handler()
 
     # Get the host by the ami env
-    hosts, server_version = Build.get_servers(ami_env=options.ami_env)
+    server_to_port_mapping, server_version = Build.get_servers(ami_env=options.ami_env)
 
     logging.info('Retrieving the credentials for Cortex XSOAR server')
     secret_conf_file = get_json_file(path=options.secret)
     username: str = secret_conf_file.get('username')
     password: str = secret_conf_file.get('userPassword')
 
     # Configure the Servers
-    for host in hosts:
-        server = Server(host=host, user_name=username, password=password)
-        logging.info(f'Adding Marketplace configuration to {host}')
+    for server_url, port in server_to_port_mapping.items():
+        server = Server(internal_ip=server_url, port=port, user_name=username, password=password)
+        logging.info(f'Adding Marketplace configuration to {server_url}')
         error_msg: str = 'Failed to set marketplace configuration.'
         server.add_server_configuration(config_dict=MARKET_PLACE_CONFIGURATION, error_msg=error_msg)
         set_marketplace_url(servers=[server], branch_name=options.branch, ci_build_number=options.build_number)
 
         # Acquire the server's host and install all content packs (one threaded execution)
-        logging.info(f'Starting to install all content packs in {host}')
+        logging.info(f'Starting to install all content packs in {server_url}')
         server_host: str = server.client.api_client.configuration.host
         success_flag = install_all_content_packs(client=server.client, host=server_host, server_version=server_version)
 
         if success_flag:
-            logging.success(f'Finished installing all content packs in {host}')
+            logging.success(f'Finished installing all content packs in {server_url}')
         else:
             logging.error('Failed to install all packs.')
             sys.exit(1)