From 51fb6d7d9f0fcee70fa1125437f9bca352472973 Mon Sep 17 00:00:00 2001 From: ShiyanWangMS Date: Tue, 12 Sep 2023 17:34:57 +0800 Subject: [PATCH] Upgrade Ansible to 6.7.0 and make Python3 as the default interpreter in sonic-mgmt-docker (#15836) Why I did it This PR is part of sonic-mgmt-docker Python3 migration project. Work item tracking Microsoft ADO (number only): 24397943 How I did it Upgrade Ansible to 6.7.0 Make Python3 as the default interpreter. python is a soft link to python3. If you want to use python2, use the command python2 explicitly. Upgrade some pip packages to higher version in order to meet security requirement. How to verify it Build a private sonic-mgmt-docker successfully. Verify python is python3. Verify python2 is working with 202012 and 202205 branch. Verify python3 is working with master branch. --- dockers/docker-sonic-mgmt/Dockerfile.j2 | 274 +++++++++++------------- 1 file changed, 131 insertions(+), 143 deletions(-) diff --git a/dockers/docker-sonic-mgmt/Dockerfile.j2 b/dockers/docker-sonic-mgmt/Dockerfile.j2 index 2f7ca018e068..16628da32142 100755 --- a/dockers/docker-sonic-mgmt/Dockerfile.j2 +++ b/dockers/docker-sonic-mgmt/Dockerfile.j2 @@ -38,67 +38,67 @@ RUN apt-get update && apt-get install -y apt-transport-https \ telnet \ vim -RUN pip3 install --upgrade pip setuptools wheel -RUN pip3 install aiohttp \ - allure-pytest==2.8.22 \ - ansible==2.9.27 \ - azure-storage-blob==12.9.0 \ - azure-kusto-data \ - azure-kusto-ingest \ - defusedxml \ - celery[redis]==5.2.7 \ - cffi \ - contextlib2==0.6.0.post1 \ - cryptography==3.3.2 \ - dpkt \ - "future>=0.16.0" \ - gitpython \ - ipaddr \ - ipython==8.12.2 \ - ixnetwork-restpy==1.0.64 \ - ixnetwork-open-traffic-generator==0.0.79 \ - jinja2==2.10.1 \ - jsonpatch \ - lxml \ - markupsafe==2.0.1 \ - mock \ - msrest==0.6.21 \ - natsort \ - ncclient \ - netaddr \ - netmiko==2.4.2 \ - paramiko==2.7.1 \ - passlib \ - pexpect \ - prettytable \ - psutil \ - ptf \ - pyasn1==0.4.8 \ - pycryptodome==3.9.8 \ - pyfiglet \ - pylint==1.8.1 \ - pyro4 \ - pysnmp==4.4.12 \ - pysubnettree \ - pytest-ansible \ - pytest-html \ - pytest-repeat \ - pytest-xdist==1.28.0 \ - python-dateutil \ - pytest==7.1.3 \ - redis \ - requests \ - retry \ - rpyc \ - scandir \ - scapy==2.4.5 \ - setuptools-rust \ - six \ - snappi[ixnetwork,convergence]==0.7.44 \ - tabulate \ - textfsm==1.1.2 \ - thrift==0.11.0 \ - virtualenv \ +RUN python3 -m pip install --upgrade pip setuptools wheel +RUN python3 -m pip install aiohttp \ + allure-pytest==2.8.22 \ + ansible==6.7.0 \ + azure-storage-blob==12.9.0 \ + azure-kusto-data \ + azure-kusto-ingest \ + defusedxml \ + celery[redis]==5.2.7 \ + cffi \ + contextlib2==0.6.0.post1 \ + cryptography==41.0.2 \ + dpkt \ + "future>=0.16.0" \ + gitpython \ + ipaddr \ + ipython==8.12.2 \ + ixnetwork-restpy==1.0.64 \ + ixnetwork-open-traffic-generator==0.0.79 \ + jinja2==3.1.2 \ + jsonpatch \ + lxml \ + markupsafe==2.0.1 \ + mock \ + msrest==0.6.21 \ + natsort \ + ncclient \ + netaddr \ + netmiko==2.4.2 \ + paramiko==2.7.1 \ + passlib \ + pexpect \ + prettytable \ + psutil \ + ptf \ + pyasn1==0.4.8 \ + pycryptodome==3.9.8 \ + pyfiglet \ + pylint==1.8.1 \ + pyro4 \ + pysnmp==4.4.12 \ + pysubnettree \ + pytest-ansible==4.0.0 \ + pytest-html \ + pytest-repeat \ + pytest-xdist==1.28.0 \ + python-dateutil \ + pytest==7.4.0 \ + redis \ + requests \ + retry \ + rpyc \ + scandir \ + scapy==2.4.5 \ + setuptools-rust \ + six \ + snappi[ixnetwork,convergence]==0.11.16 \ + tabulate \ + textfsm==1.1.2 \ + thrift==0.11.0 \ + virtualenv \ && wget https://github.com/nanomsg/nanomsg/archive/1.2.tar.gz \ && tar xvfz 1.2.tar.gz \ && cd nanomsg-1.2 \ @@ -109,81 +109,72 @@ RUN pip3 install aiohttp \ && ldconfig \ && cd ../.. \ && rm -fr nanomsg-1.2 \ - && rm -f 1.2.tar.gz \ - && pip3 install nnpy - -RUN curl -fsSL http://archive.ubuntu.com/ubuntu/pool/universe/s/scapy/python-scapy_2.3.3-3_all.deb \ - --output python-scapy_2.3.3-3_all.deb \ - && dpkg -i python-scapy_2.3.3-3_all.deb \ - && rm -f python-scapy_2.3.3-3_all.deb + && rm -f 1.2.tar.gz \ + && python3 -m pip install nnpy RUN curl -fsSL https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py \ && python2 get-pip.py \ - && rm -f get-pip.py \ - && ln -sf `which pip2` /usr/bin/pip + && rm -f get-pip.py -RUN pip install --upgrade pip setuptools wheel -RUN pip install allure-pytest==2.8.22 \ - ansible==2.8.12 \ - azure-storage-blob==12.9.0 \ - celery[redis]==4.4.7 \ - cffi==1.12.0 \ - contextlib2==0.6.0.post1 \ - cryptography==3.3.2 \ - dpkt \ - "future>=0.16.0" \ - gitpython \ - ipaddr \ - ipython==5.4.1 \ - ixnetwork-restpy==1.0.64 \ - ixnetwork-open-traffic-generator==0.0.79 \ - jinja2==2.10.1 \ - jsonpatch \ - lazy-object-proxy==1.6.0 \ - lxml \ - mock \ - msrest==0.6.21 \ - natsort \ - netaddr \ - netmiko==2.4.2 \ - nnpy \ - paramiko==2.7.1 \ - passlib \ - pexpect \ - prettytable \ - psutil \ - pyaml==21.10.1 \ - pyasn1==0.1.9 \ - pycryptodome==3.9.8 \ - pyfiglet \ - pylint==1.8.1 \ - pyro4 \ - pysnmp==4.2.5 \ - pysubnettree \ - pytest==4.6.11 \ - pytest-ansible \ - pytest-repeat \ - pytest-html \ - pytest-xdist==1.28.0 \ - python-dateutil \ - redis \ - requests \ - retry \ - rpyc \ - scandir \ - six \ - snappi[ixnetwork,convergence]==0.7.44 \ - statistics \ - tabulate \ - textfsm==1.1.3 \ - thrift==0.11.0 \ - virtualenv \ - && git clone https://github.com/p4lang/scapy-vxlan.git \ - && cd scapy-vxlan \ - && python setup.py install \ - && cd .. \ - && rm -fr scapy-vxlan \ - && pip install scapy==2.4.5 --upgrade --ignore-installed +RUN python2 -m pip install --upgrade pip setuptools wheel +RUN python2 -m pip install allure-pytest==2.8.22 \ + ansible==2.8.20 \ + azure-storage-blob==12.9.0 \ + celery[redis]==4.4.7 \ + cffi==1.12.0 \ + contextlib2==0.6.0.post1 \ + cryptography==3.3.2 \ + dpkt \ + "future>=0.16.0" \ + gitpython \ + ipaddr \ + ipython==5.4.1 \ + ixnetwork-restpy==1.0.64 \ + ixnetwork-open-traffic-generator==0.0.79 \ + jinja2==2.10.1 \ + jsonpatch \ + lazy-object-proxy==1.6.0 \ + lxml \ + mock \ + msrest==0.6.21 \ + natsort \ + netaddr \ + netmiko==2.4.2 \ + nnpy \ + paramiko==2.7.1 \ + passlib \ + pexpect \ + prettytable \ + protobuf==3.15.0 \ + psutil \ + ptf \ + pyaml==21.10.1 \ + pyasn1==0.1.9 \ + pycryptodome==3.9.8 \ + pyfiglet \ + pylint==1.8.1 \ + pyro4 \ + pysnmp==4.2.5 \ + pysubnettree \ + pytest==4.6.11 \ + pytest-ansible \ + pytest-repeat \ + pytest-html \ + pytest-xdist==1.28.0 \ + python-dateutil \ + redis \ + requests \ + retry \ + rpyc \ + scandir \ + scapy==2.4.5 \ + six \ + snappi[ixnetwork,convergence]==0.11.16 \ + statistics \ + tabulate \ + textfsm==1.1.3 \ + thrift==0.11.0 \ + virtualenv # Install docker-ce-cli RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ @@ -199,16 +190,9 @@ RUN mkdir -p /etc/apt/keyrings \ && apt-get update && apt-get install -y azure-cli ## Copy and install sonic-mgmt docker dependencies -COPY \ -{% for deb in docker_sonic_mgmt_debs.split(' ') -%} -debs/{{ deb }}{{' '}} -{%- endfor -%} -debs/ +COPY debs/sonic-device-data_*.deb debs/ -RUN dpkg -i \ -{% for deb in docker_sonic_mgmt_debs.split(' ') -%} -debs/{{ deb }}{{' '}} -{%- endfor %} +RUN dpkg -i debs/sonic-device-data_*.deb # Install protobuf 3.21.12 RUN mkdir -p /tmp/protobuf \ @@ -283,7 +267,7 @@ RUN python3 -m pip install aiohttp \ celery[redis]==5.2.7 \ cffi \ contextlib2==0.6.0.post1 \ - cryptography==3.3.2 \ + cryptography==41.0.2 \ dpkt \ "future>=0.16.0" \ gitpython \ @@ -329,7 +313,7 @@ RUN python3 -m pip install aiohttp \ scapy==2.4.5 \ setuptools-rust \ six \ - snappi[ixnetwork,convergence]==0.7.44 \ + snappi[ixnetwork,convergence]==0.11.16 \ sshconf==0.2.5 \ tabulate \ textfsm==1.1.2 \ @@ -344,5 +328,9 @@ WORKDIR /azp COPY ./start.sh . RUN chmod +x start.sh +# Install python3 as default python +RUN ln -sf /usr/bin/python3 /usr/bin/python \ + && ln -sf `which pip3` /usr/bin/pip + USER $user WORKDIR /var/$user