diff --git a/.codeclimate.yml b/.codeclimate.yml index 7d16259710c..8639c28e972 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -42,4 +42,3 @@ exclude_paths: - "go/vt/proto/" - "go/vt/sqlparser/sql.go" - "py/util/grpc_with_metadata.py" -- "travis/install_grpc.sh" diff --git a/Makefile b/Makefile index ebd18b938b5..62c3faf233a 100644 --- a/Makefile +++ b/Makefile @@ -120,13 +120,15 @@ install_protoc-gen-go: cp -a vendor/github.com/golang/protobuf $${GOPATH}/src/github.com/golang/ go install github.com/golang/protobuf/protoc-gen-go -PROTOC_BINARY := $(shell type -p $(VTROOT)/dist/grpc/usr/local/bin/protoc) -ifeq (,$(PROTOC_BINARY)) - PROTOC_BINARY := $(shell which protoc) -endif - -ifneq (,$(PROTOC_BINARY)) - PROTOC_DIR := $(dir $(PROTOC_BINARY)) +# Find protoc compiler. +# NOTE: We are *not* using the "protoc" binary (as suggested by the grpc Go +# quickstart for example). Instead, we run "protoc" via the Python +# wrapper script which is provided by the "grpcio-tools" PyPi package. +# (The package includes the compiler as library, but not as binary. +# Therefore, we have to use the wrapper script they provide.) +ifneq ($(wildcard $(VTROOT)/dist/grpc/usr/local/lib/python2.7/site-packages/grpc_tools/protoc.py),) +# IMPORTANT: The next line must not be indented. +PROTOC_COMMAND := python -m grpc_tools.protoc endif PROTO_SRCS = $(wildcard proto/*.proto) @@ -139,8 +141,8 @@ PROTO_GO_TEMPS = $(foreach name, $(PROTO_SRC_NAMES), go/vt/.proto.tmp/$(name).pb proto: proto_banner $(PROTO_GO_OUTS) $(PROTO_PY_OUTS) proto_banner: -ifeq (,$(PROTOC_DIR)) - $(error "Cannot find protoc binary. Did bootstrap.sh succeed, and did you execute 'source dev.env'?") +ifeq (,$(PROTOC_COMMAND)) + $(error "Cannot find protoc compiler. Did bootstrap.sh succeed, and did you execute 'source dev.env'?") endif ifndef NOBANNER @@ -148,7 +150,7 @@ ifndef NOBANNER endif $(PROTO_PY_OUTS): py/vtproto/%_pb2.py: proto/%.proto - $(PROTOC_DIR)/protoc -Iproto $< --python_out=py/vtproto --grpc_out=py/vtproto --plugin=protoc-gen-grpc=$(PROTOC_DIR)/grpc_python_plugin + $(PROTOC_COMMAND) -Iproto $< --python_out=py/vtproto --grpc_python_out=py/vtproto $(PROTO_GO_OUTS): $(PROTO_GO_TEMPS) for name in $(PROTO_SRC_NAMES); do \ @@ -160,7 +162,7 @@ $(PROTO_GO_TEMPS): install_protoc-gen-go $(PROTO_GO_TEMPS): go/vt/.proto.tmp/%.pb.go: proto/%.proto mkdir -p go/vt/.proto.tmp - $(PROTOC_DIR)/protoc -Iproto $< --go_out=plugins=grpc:go/vt/.proto.tmp + $(PROTOC_COMMAND) -Iproto $< --plugin=grpc=protoc-gen-go --go_out=plugins=grpc:go/vt/.proto.tmp sed -i -e 's,import \([a-z0-9_]*\) ".",import \1 "vitess.io/vitess/go/vt/proto/\1",g' $@ # Helper targets for building Docker images. diff --git a/bootstrap.sh b/bootstrap.sh index 2c1e6f5eac6..733d94b8223 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -101,38 +101,47 @@ else fi ln -snf $consul_dist/consul $VTROOT/bin/consul -# Install gRPC proto compilers. There is no download for grpc_python_plugin. -# So, we need to build it. -export grpc_dist=$VTROOT/dist/grpc -export grpc_ver="v1.10.0" -if [[ -f $grpc_dist/.build_finished && "$(cat $grpc_dist/.build_finished)" == "$grpc_ver" ]]; then - echo "skipping gRPC build. remove $grpc_dist to force rebuild." +# Install the gRPC Python library (grpcio) and the protobuf gRPC Python plugin (grpcio-tools) from PyPI. +# Dependencies like the Python protobuf package will be installed automatically. +grpc_dist=$VTROOT/dist/grpc +grpc_ver="1.10.0" +grpc_version_file="$grpc_dist/.build_finished" +if [[ -f "$grpc_version_file" && "$(cat "$grpc_version_file")" == "$grpc_ver" ]]; then + echo "skipping gRPC build. remove $grpc_dist to force reinstall." else - echo "installing grpc $grpc_ver" - # unlink homebrew's protobuf, to be able to compile the downloaded protobuf package - if [[ `uname -s` == "Darwin" && "$(brew list -1 | grep google-protobuf)" ]]; then - brew unlink grpc/grpc/google-protobuf - fi + echo "installing gRPC $grpc_ver" + # Cleanup any existing data and re-create the directory. + rm -rf "$grpc_dist" - # protobuf used to be a separate package, now we use the gRPC one. - rm -rf $VTROOT/dist/protobuf + trap "fail 'gRPC build failed'; exit 1" ERR + mkdir -p "$grpc_dist" + pushd "$grpc_dist" >/dev/null - # Cleanup any existing data and re-create the directory. - rm -rf $grpc_dist - mkdir -p $grpc_dist + # Python requires a very recent version of virtualenv. + # We also require a recent version of pip, as we use it to + # upgrade the other tools. + # For instance, setuptools doesn't work with pip 6.0: + # https://github.com/pypa/setuptools/issues/945 + # (and setuptools is used by grpc install). + grpc_virtualenv="$grpc_dist/usr/local" + $VIRTUALENV -v "$grpc_virtualenv" + PIP=$grpc_virtualenv/bin/pip + $PIP install --upgrade pip + $PIP install --upgrade --ignore-installed virtualenv + + grpcio_ver=$grpc_ver + $PIP install --upgrade grpcio==$grpcio_ver grpcio-tools==$grpcio_ver - ./travis/install_grpc.sh $grpc_dist || fail "gRPC build failed" - echo "$grpc_ver" > $grpc_dist/.build_finished + popd >/dev/null + trap - ERR - # link homebrew's protobuf back - if [[ `uname -s` == "Darwin" && "$(brew list -1 | grep google-protobuf)" ]]; then - brew link grpc/grpc/google-protobuf - fi + echo "$grpc_ver" > "$grpc_version_file" # Add newly installed Python code to PYTHONPATH such that other Python module # installations can reuse it. (Once bootstrap.sh has finished, run # source dev.env instead to set the correct PYTHONPATH.) - export PYTHONPATH=$(prepend_path $PYTHONPATH $grpc_dist/usr/local/lib/python2.7/dist-packages) + PYTHONPATH=$(prepend_path "$PYTHONPATH" "$grpc_virtualenv/lib/python2.7/dist-packages") + export PYTHONPATH fi # Install third-party Go tools used as part of the development workflow. diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index 6d48a2737f2..d397176f976 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -29,9 +29,6 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins libz-dev \ && rm -rf /var/lib/apt/lists/* -# Not sure if this is needed, but it was part of php init, which we don't do any more. -RUN mkdir -p /vt/bin - # Install Maven 3.1+ RUN mkdir -p /vt/dist && \ cd /vt/dist && \ @@ -61,10 +58,6 @@ COPY tools /vt/src/vitess.io/vitess/tools COPY travis /vt/src/vitess.io/vitess/travis COPY vendor/vendor.json /vt/src/vitess.io/vitess/vendor/ -# grpcio runtime is needed for python tests. -RUN grpcio_ver="1.7.0" && \ - pip install --upgrade grpcio==$grpcio_ver - # Download vendored Go dependencies RUN cd /vt/src/vitess.io/vitess && \ go get -u github.com/kardianos/govendor && \ diff --git a/travis/install_grpc.sh b/travis/install_grpc.sh deleted file mode 100755 index 4202b2c84c0..00000000000 --- a/travis/install_grpc.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script downloads and installs the grpc library, for -# go and python, in the root of the image. It assumes we're running -# as root in the image. -set -ex - -# Import prepend_path function. -dir="$(dirname "${BASH_SOURCE[0]}")" -source "${dir}/../tools/shell_functions.inc" -if [ $? -ne 0 ]; then - echo "failed to load ../tools/shell_functions.inc" - return 1 -fi - -cd $grpc_dist - -# Python requires a very recent version of virtualenv. -# We also require a recent version of pip, as we use it to -# upgrade the other tools. -# For instance, setuptools doesn't work with pip 6.0: -# https://github.com/pypa/setuptools/issues/945 -# (and setuptools is used by grpc install). -$VIRTUALENV -v $grpc_dist/usr/local -PIP=$grpc_dist/usr/local/bin/pip -$PIP install --upgrade pip -$PIP install --upgrade --ignore-installed virtualenv - -# clone the repository, setup the submodules -git clone https://github.com/grpc/grpc.git -cd grpc -git checkout $grpc_ver -git submodule update --init - -# OSX specific setting + dependencies -if [ `uname -s` == "Darwin" ]; then - export GRPC_PYTHON_BUILD_WITH_CYTHON=1 - $PIP install Cython - - # Work-around macOS Sierra blocker, see: https://github.com/vitessio/vitess/issues/2115 - # TODO(mberlin): Remove this when the underlying issue is fixed and available - # in the gRPC version used by Vitess. - # See: https://github.com/google/protobuf/issues/2182 - export CPPFLAGS="-Wno-deprecated-declarations" -fi - -# Install gRPC python libraries from PyPI. -# Dependencies like protobuf python will be installed automatically. -grpcio_ver=1.10.0 -$PIP install --upgrade grpcio==$grpcio_ver - -# now install grpc_python_plugin, which also builds the protoc compiler. -make grpc_python_plugin -ln -sf $grpc_dist/grpc/bins/opt/protobuf/protoc $VTROOT/bin/protoc -ln -sf $grpc_dist/grpc/bins/opt/grpc_python_plugin $VTROOT/bin/grpc_python_plugin