diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f63af1de0..2bd51d6e02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,6 @@ jobs: - py37-linux,docs,mypy,tests - general_itests env: - PIP_INDEX_URL: https://pypi.python.org/simple DOCKER_REGISTRY: "" steps: - uses: actions/checkout@v2 @@ -29,7 +28,7 @@ jobs: python-version: 3.7 - run: python -m pip install --upgrade pip - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve - - run: tox -i https://pypi.python.org/simple -e ${{ matrix.toxenv }} + - run: tox -e ${{ matrix.toxenv }} k8s_itests: runs-on: ubuntu-20.04 env: diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 788966e87b..1e5c33b110 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -16,7 +16,6 @@ jobs: - py37-linux,docs,mypy,tests - general_itests env: - PIP_INDEX_URL: https://pypi.python.org/simple DOCKER_REGISTRY: "" steps: - uses: actions/checkout@v2 @@ -25,13 +24,12 @@ jobs: python-version: 3.7 - run: python -m pip install --upgrade pip - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve - - run: tox -i https://pypi.python.org/simple -e ${{ matrix.toxenv }} + - run: tox -e ${{ matrix.toxenv }} pypi: # lets run tests before we push anything to pypi, much like we do internally needs: tox runs-on: ubuntu-20.04 env: - PIP_INDEX_URL: https://pypi.python.org/simple DOCKER_REGISTRY: "" steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index 1f1a5a7539..252379a84b 100644 --- a/Makefile +++ b/Makefile @@ -22,42 +22,41 @@ else endif ifeq ($(PAASTA_ENV),YELP) - export PIP_INDEX_URL ?= https://pypi.yelpcorp.com/simple export DOCKER_REGISTRY ?= docker-dev.yelpcorp.com/ else - export PIP_INDEX_URL ?= https://pypi.python.org/simple export DOCKER_REGISTRY ?= "" + export INDEX_URL_BUILD_ARG ?= PIP_INDEX_URL endif .PHONY: all docs test itest k8s_itests quick-test dev: .paasta/bin/activate - .paasta/bin/tox -i $(PIP_INDEX_URL) + .paasta/bin/tox docs: .paasta/bin/activate - .paasta/bin/tox -i $(PIP_INDEX_URL) -e docs + .paasta/bin/tox -e docs test: .paasta/bin/activate if [ "$(PAASTA_ENV)" != "YELP" ]; then \ - .paasta/bin/tox -i $(PIP_INDEX_URL) -e tests; \ + .paasta/bin/tox -e tests; \ else \ - .paasta/bin/tox -i $(PIP_INDEX_URL) -e tests-yelpy; \ + .paasta/bin/tox -e tests-yelpy; \ fi test-yelpy: .paasta/bin/activate - .paasta/bin/tox -i $(PIP_INDEX_URL) -e tests-yelpy + .paasta/bin/tox -e tests-yelpy test-not-yelpy: .paasta/bin/activate - .paasta/bin/tox -i $(PIP_INDEX_URL) -e tests + .paasta/bin/tox -e tests quick-test: .tox/py37-linux TZ=UTC .tox/py37-linux/bin/py.test --last-failed -x -- tests .tox/py37-linux: .paasta/bin/activate - .paasta/bin/tox -i $(PIP_INDEX_URL) + .paasta/bin/tox dev-api: .tox/py37-linux - .paasta/bin/tox -i $(PIP_INDEX_URL) -e dev-api + .paasta/bin/tox -e dev-api .paasta/bin/activate: requirements.txt requirements-dev.txt test -d .paasta/bin/activate || virtualenv -p python3.7 .paasta @@ -69,7 +68,7 @@ dev-api: .tox/py37-linux touch .paasta/bin/activate itest: test .paasta/bin/activate - .paasta/bin/tox -i $(PIP_INDEX_URL) -e general_itests + .paasta/bin/tox -e general_itests itest_%: # See the makefile in yelp_package/Makefile for packaging stuff @@ -150,7 +149,7 @@ generate_deployments_for_service: | soa_config_playground .tox/py37-linux .PHONY: playground-api playground-api: .tox/py37-linux | soa_config_playground - .paasta/bin/tox -i $(PIP_INDEX_URL) -e playground-api + .paasta/bin/tox -e playground-api .PHONY: setup-kubernetes-job setup-kubernetes-job: k8s_fake_cluster generate_deployments_for_service diff --git a/debian/rules b/debian/rules index 6b37c72bd9..d0b72bb95d 100755 --- a/debian/rules +++ b/debian/rules @@ -1,8 +1,6 @@ #!/usr/bin/make -f # -*- makefile -*- -PIP_INDEX_URL ?= https://pypi.yelpcorp.com/simple - %: dh $@ --with python-virtualenv @@ -21,9 +19,8 @@ PACKAGE=$(shell dh_listpackages) DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs DH_VENV_DIR=debian/$(PACKAGE)$(DH_VIRTUALENV_INSTALL_ROOT)/$(PACKAGE) override_dh_virtualenv: - dh_virtualenv -i $(PIP_INDEX_URL) \ + dh_virtualenv \ --python=/usr/bin/python3.7 \ --preinstall no-manylinux1 \ - --preinstall=-rrequirements-bootstrap.txt \ - --pip-tool pip-custom-platform + --preinstall=-rrequirements-bootstrap.txt cp yelp_package/gopath/paasta_go $(DH_VENV_DIR)/bin/paasta_go diff --git a/general_itests/fake_simple_service/Dockerfile b/general_itests/fake_simple_service/Dockerfile index 3b7196ae25..2249026bef 100644 --- a/general_itests/fake_simple_service/Dockerfile +++ b/general_itests/fake_simple_service/Dockerfile @@ -14,6 +14,3 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:xenial - -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple -ENV PIP_INDEX_URL=${PIP_INDEX_URL} diff --git a/general_itests/fake_simple_service/Makefile b/general_itests/fake_simple_service/Makefile index c723c5c479..dace326e85 100644 --- a/general_itests/fake_simple_service/Makefile +++ b/general_itests/fake_simple_service/Makefile @@ -16,13 +16,11 @@ DOCKER_TAG ?= fake_simple_service-$(USER)-dev ifeq ($(findstring .yelpcorp.com,$(shell hostname -f)), .yelpcorp.com) DOCKER_REGISTRY ?= docker-dev.yelpcorp.com/ - PIP_INDEX_URL ?= https://pypi.yelpcorp.com/simple else DOCKER_REGISTRY ?= "" - PIP_INDEX_URL ?= https://pypi.python.org/simple endif .PHONY: cook-image cook-image: - docker build --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) -t $(DOCKER_TAG) . + docker build --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) -t $(DOCKER_TAG) . diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt index dc06bbadb8..b24f00fb9b 100644 --- a/requirements-bootstrap.txt +++ b/requirements-bootstrap.txt @@ -1,5 +1,4 @@ pip==18.1 -pip-custom-platform==0.5.0 setuptools==39.0.1 venv-update==3.2.4 wheel==0.32.3 diff --git a/tox.ini b/tox.ini index 09bf738423..df9a404684 100644 --- a/tox.ini +++ b/tox.ini @@ -1,19 +1,14 @@ [tox] skipsdist=True envlist=py37-linux -tox_pip_extensions_ext_pip_custom_platform = true tox_pip_extensions_ext_venv_update = true docker_compose_version = 1.26.2 [testenv] -# The Makefile and override the indexserver to the public one when -# running outside of Yelp. -indexserver = https://pypi.yelpcorp.com/simple basepython = python3.7 passenv = SSH_AUTH_SOCK setenv = TZ = UTC - PIP_INDEX_URL = {env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple} deps = --requirement={toxinidir}/requirements.txt --requirement={toxinidir}/requirements-dev.txt @@ -24,7 +19,7 @@ commands = # that said, most of the time people will run make test which will use tox to install these in a # faster way (using venv-update) - so this is really just here for anyone that like to just invoke # `tox` directly and with no explicit env - -pip-custom-platform install -i https://pypi.yelpcorp.com/simple -r yelp_package/extra_requirements_yelp.txt + -pip install -r yelp_package/extra_requirements_yelp.txt [testenv:dev-api] envdir = .tox/py37-linux/ @@ -105,24 +100,25 @@ passenv = DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH + INDEX_URL_BUILD_ARG changedir=k8s_itests/ commands = # Build /etc/paasta used by docker-compose {toxinidir}/k8s_itests/scripts/setup.sh # Run paasta-tools k8s_itests in docker docker-compose down - docker-compose --verbose build --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg PIP_INDEX_URL={env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple} + docker-compose --verbose build --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg {env:INDEX_URL_BUILD_ARG:UNUSED}=https://pypi.org/simple docker-compose up \ --abort-on-container-exit [testenv:example_cluster] changedir=example_cluster/ -passenv = DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH +passenv = DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH INDEX_URL_BUILD_ARG deps = docker-compose=={[tox]docker_compose_version} commands = docker-compose down - docker-compose --verbose build --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg PIP_INDEX_URL={env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple} + docker-compose --verbose build --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg {env:INDEX_URL_BUILD_ARG:UNUSED}=https://pypi.org/simple # Fire up the marathon cluster in background # Run the paastatools container in foreground to catch the output # the `docker-compose run` vs `docker-compose up` is important here, as docker-compose run will diff --git a/yelp_package/Makefile b/yelp_package/Makefile index 7ce492138a..c900b069ea 100644 --- a/yelp_package/Makefile +++ b/yelp_package/Makefile @@ -19,7 +19,7 @@ SHELL=/bin/bash UID:=`id -u` GID:=`id -g` -DOCKER_RUN=docker run -t -v $(CURDIR)/../:/work:rw -e PIP_INDEX_URL=$(PIP_INDEX_URL) yelp/paastatools_$*_container +DOCKER_RUN=docker run -t -v $(CURDIR)/../:/work:rw yelp/paastatools_$*_container NOOP = true ifeq ($(PAASTA_ENV),YELP) @@ -36,7 +36,8 @@ build_%_docker: [ -d ../dist ] || mkdir ../dist docker pull "yelp/paastatools_$*_container" || true cd dockerfiles/$*/ && docker build --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) \ - --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) -t "yelp/paastatools_$*_container" . + $(if $(filter-out $(PAASTA_ENV),YELP), --build-arg PIP_INDEX_URL=https://pypi.org/simple,) \ + -t "yelp/paastatools_$*_container" . .SECONDEXPANSION: itest_%: package_$$* diff --git a/yelp_package/dockerfiles/bionic/Dockerfile b/yelp_package/dockerfiles/bionic/Dockerfile index add3aa8ba5..be04501e19 100644 --- a/yelp_package/dockerfiles/bionic/Dockerfile +++ b/yelp_package/dockerfiles/bionic/Dockerfile @@ -15,7 +15,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:bionic -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN rm /etc/dpkg/dpkg.cfg.d/excludes RUN apt-get update && apt-get install -yq gnupg2 diff --git a/yelp_package/dockerfiles/gitremote/Dockerfile b/yelp_package/dockerfiles/gitremote/Dockerfile index 173fcfada5..a9ef3d7bbc 100644 --- a/yelp_package/dockerfiles/gitremote/Dockerfile +++ b/yelp_package/dockerfiles/gitremote/Dockerfile @@ -1,8 +1,6 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:xenial -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple -ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN apt-get update > /dev/null && \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ git \ diff --git a/yelp_package/dockerfiles/hacheck-sidecar/Dockerfile b/yelp_package/dockerfiles/hacheck-sidecar/Dockerfile index 2c95294e93..b69a133e34 100644 --- a/yelp_package/dockerfiles/hacheck-sidecar/Dockerfile +++ b/yelp_package/dockerfiles/hacheck-sidecar/Dockerfile @@ -1,9 +1,6 @@ FROM docker-dev.yelpcorp.com/bionic_yelp ARG HACHECK_VERSION=0.18.2-yelp1 -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple -ENV PIP_INDEX_URL=$PIP_INDEX_URL - RUN apt-get update && \ apt-get install -y hacheck=${HACHECK_VERSION} paasta-tools python3-distutils && \ mkdir -p /etc/paasta diff --git a/yelp_package/dockerfiles/itest/api/Dockerfile b/yelp_package/dockerfiles/itest/api/Dockerfile index a80924b9b4..4c30ff9466 100644 --- a/yelp_package/dockerfiles/itest/api/Dockerfile +++ b/yelp_package/dockerfiles/itest/api/Dockerfile @@ -15,7 +15,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:bionic -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN apt-get update > /dev/null && \ diff --git a/yelp_package/dockerfiles/itest/hacheck/Dockerfile b/yelp_package/dockerfiles/itest/hacheck/Dockerfile index 2c1aadf91d..9533ac3121 100644 --- a/yelp_package/dockerfiles/itest/hacheck/Dockerfile +++ b/yelp_package/dockerfiles/itest/hacheck/Dockerfile @@ -15,7 +15,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:bionic -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN apt-get update > /dev/null && \ diff --git a/yelp_package/dockerfiles/itest/httpdrain/Dockerfile b/yelp_package/dockerfiles/itest/httpdrain/Dockerfile index b425f7aaa6..0bb1aef38f 100644 --- a/yelp_package/dockerfiles/itest/httpdrain/Dockerfile +++ b/yelp_package/dockerfiles/itest/httpdrain/Dockerfile @@ -1,7 +1,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:bionic -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN apt-get update > /dev/null && \ diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile index 0c79a789bc..2184c2699d 100644 --- a/yelp_package/dockerfiles/itest/k8s/Dockerfile +++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile @@ -15,7 +15,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:bionic -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL # Need Python 3.7 diff --git a/yelp_package/dockerfiles/itest/marathon/Dockerfile b/yelp_package/dockerfiles/itest/marathon/Dockerfile index 69d0308660..32318c06de 100644 --- a/yelp_package/dockerfiles/itest/marathon/Dockerfile +++ b/yelp_package/dockerfiles/itest/marathon/Dockerfile @@ -15,9 +15,6 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:xenial -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple -ENV PIP_INDEX_URL=$PIP_INDEX_URL - RUN apt-get update > /dev/null && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ apt-transport-https \ diff --git a/yelp_package/dockerfiles/itest/mesos/Dockerfile b/yelp_package/dockerfiles/itest/mesos/Dockerfile index f808cac49c..d14f3ba96a 100644 --- a/yelp_package/dockerfiles/itest/mesos/Dockerfile +++ b/yelp_package/dockerfiles/itest/mesos/Dockerfile @@ -15,9 +15,6 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}xenial_pkgbuild -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple -ENV PIP_INDEX_URL=$PIP_INDEX_URL - # Install packages to allow apt to use a repository over HTTPS # https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#xenial-1604 RUN apt-get update > /dev/null && \ diff --git a/yelp_package/dockerfiles/itest/zookeeper/Dockerfile b/yelp_package/dockerfiles/itest/zookeeper/Dockerfile index 3768984d44..c91b541f06 100644 --- a/yelp_package/dockerfiles/itest/zookeeper/Dockerfile +++ b/yelp_package/dockerfiles/itest/zookeeper/Dockerfile @@ -15,9 +15,6 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:xenial -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple -ENV PIP_INDEX_URL=$PIP_INDEX_URL - RUN apt-get update > /dev/null && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ zookeeper > /dev/null && \ diff --git a/yelp_package/dockerfiles/jammy/Dockerfile b/yelp_package/dockerfiles/jammy/Dockerfile index 98ebee7de8..a5bb90b118 100644 --- a/yelp_package/dockerfiles/jammy/Dockerfile +++ b/yelp_package/dockerfiles/jammy/Dockerfile @@ -15,7 +15,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:jammy -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/jammy/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN rm /etc/dpkg/dpkg.cfg.d/excludes RUN apt-get update && apt-get install -yq gnupg2 diff --git a/yelp_package/dockerfiles/mesos-paasta/Dockerfile b/yelp_package/dockerfiles/mesos-paasta/Dockerfile index 1b13e466c5..7a97968860 100644 --- a/yelp_package/dockerfiles/mesos-paasta/Dockerfile +++ b/yelp_package/dockerfiles/mesos-paasta/Dockerfile @@ -1,6 +1,6 @@ FROM example_cluster_mesosbase -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/xenial/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN apt-get update > /dev/null && \ diff --git a/yelp_package/dockerfiles/playground/Dockerfile b/yelp_package/dockerfiles/playground/Dockerfile index 7a4a1f76bd..b0318f9bb5 100644 --- a/yelp_package/dockerfiles/playground/Dockerfile +++ b/yelp_package/dockerfiles/playground/Dockerfile @@ -1,6 +1,6 @@ FROM example_cluster_itest_xenial -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/xenial/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN apt-get update > /dev/null && \ diff --git a/yelp_package/dockerfiles/xenial/Dockerfile b/yelp_package/dockerfiles/xenial/Dockerfile index 8846bf00b9..2062eeff5e 100644 --- a/yelp_package/dockerfiles/xenial/Dockerfile +++ b/yelp_package/dockerfiles/xenial/Dockerfile @@ -15,7 +15,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}xenial_pkgbuild -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/xenial/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN echo "deb http://repos.mesosphere.com/ubuntu xenial main" > /etc/apt/sources.list.d/mesosphere.list && \ diff --git a/yelp_package/extra_requirements_yelp.txt b/yelp_package/extra_requirements_yelp.txt index 9e6521c0bd..959d7b80f9 100644 --- a/yelp_package/extra_requirements_yelp.txt +++ b/yelp_package/extra_requirements_yelp.txt @@ -1,4 +1,3 @@ ---index-url=https://pypi.yelpcorp.com/simple atomicfile==1.0 cached-property==1.3.1 cffi==1.15.0