diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 81b6ca68..fc3d82c9 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -43,7 +43,7 @@ jobs: - name: Test with pytest run: | conda install pytest - source activate base & pytest tests + source activate base & pytest tests -rsx build-windows: @@ -52,26 +52,36 @@ jobs: max-parallel: 5 steps: + # - uses: actions/checkout@v3 + # - name: Set up Python 3.10 + # uses: conda-incubator/setup-miniconda@v2 + # with: + # miniconda-version: "latest" - uses: actions/checkout@v3 - name: Set up Python 3.10 - uses: conda-incubator/setup-miniconda@v2 + uses: actions/setup-python@v3 with: - miniconda-version: "latest" + python-version: '3.10' # - name: Add conda to system path # run: | # # $CONDA is an environment variable pointing to the root of the miniconda directory # echo $CONDA/bin >> $GITHUB_PATH - - name: Add extra channels - run: | - conda config --add channels conda-forge - conda config --add channels defaults + # - name: Add extra channels + # run: | + # conda config --add channels conda-forge + # conda config --add channels defaults - - name: Install dependencies + # - name: Install dependencies + # run: | + # conda env update --file environment.yml --name base + - name: set up env3 run: | - conda env update --file environment.yml --name base + python -m venv ENV3 + - name: Lint with flake8 run: | - conda install flake8 + .\ENV3\Scripts\activate.ps1 + pip install flake8 # stop the build if there are Python syntax errors or undefined names flake8 --exclude deprecated . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide @@ -79,10 +89,11 @@ jobs: - name: Install current Python library run: | - conda activate base # Activate the conda environment + .\ENV3\Scripts\activate.ps1 pip install -e . # Install the current Python library in editable mode pip install cloudmesh-vpn - name: Test with pytest run: | - conda install pytest - source activate base & pytest tests + .\ENV3\Scripts\activate.ps1 + pip install pytest + pytest tests -rsx diff --git a/cloudmesh/common/Host.py b/cloudmesh/common/Host.py index 0903212a..63be69c5 100644 --- a/cloudmesh/common/Host.py +++ b/cloudmesh/common/Host.py @@ -11,6 +11,7 @@ from cloudmesh.common.parameter import Parameter from cloudmesh.common.util import path_expand from cloudmesh.common.util import readfile +from cloudmesh.common.systeminfo import os_is_windows from pprint import pprint from cloudmesh.common.Printer import Printer @@ -233,6 +234,7 @@ def ssh(hosts=None, ssh_command = ['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null', + '-o', 'PreferredAuthentications=publickey', '-i', key, '{host}', f'{command}'] @@ -325,10 +327,17 @@ def _ping(args): """ ip = args['ip'] count = str(args['count']) - count_flag = '-n' if platform == 'windows' else '-c' - command = ['ping', count_flag, count, ip] + + count_flag = '-n' if os_is_windows() else '-c' + if os_is_windows(): + # adding ipv4 enforce for windows + # for some reason -4 is required or hosts + # fail. we need ipv4 + command = ['ping', '-4', ip, count_flag, count] + else: + command = ['ping', count_flag, count, ip] + # command = ['ping', '-4', ip, count_flag, count] result = subprocess.run(command, capture_output=True) - try: timers = result.stdout \ .decode("utf-8", "ignore") \ diff --git a/cloudmesh/common/Shell.py b/cloudmesh/common/Shell.py index 6ffc4b16..0bfd24b2 100755 --- a/cloudmesh/common/Shell.py +++ b/cloudmesh/common/Shell.py @@ -249,11 +249,25 @@ def locale(): @staticmethod def ssh_enabled(): if os_is_linux(): - r = Shell.run("service sshd status | fgrep running").strip() + try: + r = Shell.run("which sshd") + except RuntimeError as e: + raise RuntimeError("You do not have OpenSSH installed. " + "sudo apt-get install openssh-client openssh-server " + "Automatic installation will be implemented in future cloudmesh version.") + # the reason why we do it like this is because WSL + # does not have sshd as a status. this works fine + r = Shell.run("service ssh status | fgrep running").strip() + return len(r) > 0 elif os_is_windows(): - r = Shell.run("ps | grep -F ssh") - return "ssh" in r + # r = Shell.run("ps | grep -F ssh") + # return "ssh" in r + processes = psutil.process_iter(attrs=['name']) + # Filter the processes for 'ssh' + ssh_processes = [p.info for p in processes if 'ssh' in p.info['name']] + return len(ssh_processes) > 0 + elif os_is_mac(): r = Shell.run("ps -ef") if "sshd" in r: @@ -281,7 +295,7 @@ def run_timed(label, command, encoding=None, service=None): return str(result) @staticmethod - def run(command, exit="; exit 0", encoding='utf-8', replace=True, timeout=None): + def run(command, exitcode="", encoding='utf-8', replace=True, timeout=None): """ executes the command and returns the output as string :param command: @@ -295,18 +309,21 @@ def run(command, exit="; exit 0", encoding='utf-8', replace=True, timeout=None): else: c = ";" command = f"{command}".replace(";", c) - else: - command = f"{command} {exit}" + elif exitcode: + command = f"{command} {exitcode}" - if timeout is not None: - r = subprocess.check_output(command, - stderr=subprocess.STDOUT, - shell=True, - timeout=timeout) - else: - r = subprocess.check_output(command, - stderr=subprocess.STDOUT, - shell=True) + try: + if timeout is not None: + r = subprocess.check_output(command, + stderr=subprocess.STDOUT, + shell=True, + timeout=timeout) + else: + r = subprocess.check_output(command, + stderr=subprocess.STDOUT, + shell=True) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"{e.returncode} {e.output.decode()}") if encoding is None or encoding == 'utf-8': return str(r, 'utf-8') else: diff --git a/tests/ssh/test_ssh.py b/tests/ssh/test_ssh.py index 4278515d..9ee4ae88 100644 --- a/tests/ssh/test_ssh.py +++ b/tests/ssh/test_ssh.py @@ -4,7 +4,10 @@ # pytest -v --capture=no tests/ssh/test_ssh..py::Test_name:: ############################################################### +# https://github.com/actions/runner-images/issues/1519 ping does not work in github runner so we skip it. import os +from distutils.util import strtobool +github_action = strtobool(os.getenv('GITHUB_ACTIONS', 'false')) import pytest from cloudmesh.common.Benchmark import Benchmark @@ -49,7 +52,8 @@ def craete_location(host): -@pytest.mark.skipif(not Shell.ssh_enabled(), reason="SSH is not aenabled") +@pytest.mark.skipif(not Shell.ssh_enabled(), reason="SSH is not enabled") +@pytest.mark.skipif(github_action, reason='GitHub Runner uses Azure and Azure does not have an ssh key set up!') @pytest.mark.incremental class TestSsh: @@ -71,6 +75,9 @@ def test_ssh_processors(self): results = self.ssh(processors=processors) print(Printer.write(results)) for result in results: + if "denied (publickey)" in result["stderr"].decode(): + pytest.skip("ssh test cannot proceed because ssh-copy-id not yet " + "done.") assert result["success"] # diff --git a/tests/test_base.py b/tests/test_base.py index 3e55f51f..cf82cb39 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -37,12 +37,12 @@ def test_custom_path(self): def test_environment_variable_path(self): HEADING() - os.environ["CLOUDMESH"] = "./tmp/.cloudmesh" + os.environ["CLOUDMESH_CONFIG_DIR"] = "./tmp/.cloudmesh" cloudmesh = Base() assert cloudmesh.path == "./tmp/.cloudmesh" assert cloudmesh.config == f"{cloudmesh.path}/cloudmesh.yaml" shutil.rmtree("./tmp") - del os.environ["CLOUDMESH"] + del os.environ["CLOUDMESH_CONFIG_DIR"] def test_cloudmesh_in_cwd(self): HEADING() diff --git a/tests/test_ping.py b/tests/test_ping.py index 5401312d..acf60b0c 100644 --- a/tests/test_ping.py +++ b/tests/test_ping.py @@ -17,6 +17,11 @@ cloud = "local" +# https://github.com/actions/runner-images/issues/1519 ping does not work in github runner so we skip it. +import os +from distutils.util import strtobool +github_action = strtobool(os.getenv('GITHUB_ACTIONS', 'false')) + # multiping only works if you have root, so we can not use it # from multiping import MultiPing @@ -33,6 +38,7 @@ @pytest.mark.incremental +@pytest.mark.skipif(github_action, reason='GitHub Runner uses Azure and Azure disables ping. :( Too bad!') class Test_ping: def ping(self, processors=1): diff --git a/tests/test_shell.py b/tests/test_shell.py index 8c6753fd..73f04d30 100755 --- a/tests/test_shell.py +++ b/tests/test_shell.py @@ -110,7 +110,7 @@ def test_pwd(self): Benchmark.Start() r = Shell.pwd() Benchmark.Stop() - assert 'cm/cloudmesh-common' in r or 'cm\\cloudmesh-common' in r + assert 'cloudmesh-common' in r def test_open(self): HEADING() diff --git a/tests/test_shell_tests.py b/tests/test_shell_tests.py index 1b2cf7d8..b6d76668 100644 --- a/tests/test_shell_tests.py +++ b/tests/test_shell_tests.py @@ -21,7 +21,10 @@ from cloudmesh.common.systeminfo import os_is_windows, os_is_linux, os_is_mac from pathlib import Path -import time +# https://github.com/actions/runner-images/issues/1519 ping does not work in github runner so we skip it. +import os +from distutils.util import strtobool +github_action = strtobool(os.getenv('GITHUB_ACTIONS', 'false')) class TestShell: @@ -139,7 +142,7 @@ def test_map_filename(self): assert os.path.exists(path_expand('./tmp')) == False Benchmark.Stop() - + @pytest.mark.skipif(github_action, reason='GitHub Runner is headless, and GUI is not possible, so this is skipped.') def test_open(self): HEADING() Benchmark.Start() @@ -188,6 +191,7 @@ def test_shell_cat(self): assert '#' in r assert 'tabulate' in r + @pytest.mark.skipif(github_action, reason='GitHub Runner uses Azure and Azure disables ping. :( Too bad!') def test_shell_ping(self): HEADING() Benchmark.Start()