Patch Kube-in-Kube setup: use internal DNS and support SSL terminatio…

…n in proxies

workflows/pipe-common/pipeline/autoscaling/kubeprovider.py

@@ -16,6 +16,7 @@
 
 import pykube
 from pipeline.autoscaling import utils
+from pipeline.common.noverify_kubernetes import NoVerify_Kube_Client
 import re
 
 RUN_ID_LABEL = 'runid'
@@ -27,9 +28,9 @@ class KubeProvider(object):
 
     def __init__(self):
         try:
-            self.api = pykube.HTTPClient(pykube.KubeConfig.from_service_account())
+            self.api = NoVerify_Kube_Client.get_client(pykube.KubeConfig.from_service_account())
         except Exception:
-            self.api = pykube.HTTPClient(pykube.KubeConfig.from_file("~/.kube/config"))
+            self.api = NoVerify_Kube_Client.get_client(pykube.KubeConfig.from_file("~/.kube/config"))
         self.api.session.verify = False
 
     def get_nodename(self, nodename):

workflows/pipe-common/pipeline/common/__init__.py

@@ -19,3 +19,4 @@
 from .container import *
 from .kubernetes import *
 from .utils import *
+from .noverify_kubernetes import *

workflows/pipe-common/pipeline/common/kubernetes.py

@@ -13,10 +13,10 @@
 # limitations under the License.
 
 import os
+from pipeline.common.noverify_kubernetes import NoVerify_Kube_Client
 
 try:
     from pykube.config import KubeConfig
-    from pykube.http import HTTPClient
     from pykube.http import HTTPError
     from pykube.objects import Pod
     from pykube.objects import Event
@@ -39,9 +39,7 @@ def __init__(self, obj, run_id):
 class Kubernetes:
 
     def __init__(self):
-        if os.getenv('CP_PYKUBE_SKIP_REQUESTS_CA_BUNDLE', None) == "true":
-            os.environ['REQUESTS_CA_BUNDLE'] = ''
-        self.__kube_api = HTTPClient(KubeConfig.from_service_account())
+        self.__kube_api = NoVerify_Kube_Client.get_client(KubeConfig.from_service_account())
         self.__kube_api.session.verify = False
 
     def get_pod(self, run_id):

workflows/pipe-common/pipeline/common/noverify_kubernetes.py

@@ -0,0 +1,33 @@
+import os
+import pykube
+import requests
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+
+class NoVerify_Requests_Session(requests.Session):
+    """A wrapper for requests.Session to override 'verify' property, ignoring REQUESTS_CA_BUNDLE environment variable.
+    This is a workaround for https://github.com/kennethreitz/requests/issues/3829
+    """
+    def merge_environment_settings(self, url, proxies, stream, verify, *args, **kwargs):
+        if self.verify is False:
+            verify = False
+        return super(NoVerify_Requests_Session, self).merge_environment_settings(url, proxies, stream, verify, *args, **kwargs)
+
+
+def _noverify_session_object(strategy=None, config=None, gcloud_file=None):
+    if strategy in ["token", "client-certificate", "basic-auth"]:
+        return NoVerify_Requests_Session()
+    elif strategy in ["gcp"]:
+        return GCPSession(config, gcloud_file).create()
+    else:
+        return NoVerify_Requests_Session()
+
+class NoVerify_Kube_Client(object):
+    """
+    Wrapper for PyKube interface which prioritizes session.verify before REQUESTS_CA_BUNDLE
+    Default version will ignoe session.verify=False if REQUESTS_CA_BUNDLE is set
+    """
+    @staticmethod
+    def get_client(config):
+        pykube.session._session_object = _noverify_session_object
+        requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+        return pykube.HTTPClient(config)

workflows/pipe-common/pipeline/common/pod.py

@@ -17,6 +17,8 @@
 import os
 import uuid
 
+from pipeline.common.noverify_kubernetes import NoVerify_Kube_Client
+
 try:
     from pykube.config import KubeConfig
     from pykube.http import HTTPClient
@@ -34,7 +36,7 @@ class PodLauncher:
     __RUNNING_STATUS = 'Running'
 
     def __init__(self, task_name=None):
-        self.kube_api = HTTPClient(KubeConfig.from_service_account())
+        self.kube_api = NoVerify_Kube_Client.get_client(KubeConfig.from_service_account())
         self.kube_api.session.verify = False
         if task_name:
             self.task_name = task_name

workflows/pipe-common/pipeline/hpc/engine/kube.py

@@ -35,15 +35,18 @@
     GridEngineLaunchAdapter, GridEngineResourceParser
 from pipeline.hpc.logger import Logger
 from pipeline.hpc.resource import IntegralDemand, ResourceSupply
+from pipeline.common.noverify_kubernetes import NoVerify_Kube_Client
 
 
 def get_kube_client():
+    client = None
     try:
-        return pykube.HTTPClient(pykube.KubeConfig.from_service_account())
+        client = NoVerify_Kube_Client.get_client(pykube.KubeConfig.from_service_account())
     except Exception:
         kube_config_path = os.path.join(os.path.expanduser('~'), '.kube', 'config')
-        return pykube.HTTPClient(pykube.KubeConfig.from_file(kube_config_path))
-
+        client =  NoVerify_Kube_Client.get_client(pykube.KubeConfig.from_file(kube_config_path))
+    client.session.verify = False
+    return client
 
 class KubeGridEngine(GridEngine):
 

workflows/pipe-common/pipeline/pipeline.py

@@ -29,10 +29,10 @@
 
 from . import api
 from .storage import S3Bucket
+from .common import NoVerify_Kube_Client
 
 try:
     from pykube.config import KubeConfig
-    from pykube.http import HTTPClient
     from pykube.http import HTTPError
     from pykube.objects import Pod
     from pykube.objects import Event
@@ -221,11 +221,12 @@ def __init__(self, *args, **kwargs):
 
     def _init_kubernetes(self):
         if self.auth_method == "kubeconfig":
-            self.__kube_api = HTTPClient(KubeConfig.from_file(self.kubeconfig_path))
+            self.__kube_api = NoVerify_Kube_Client.get_client(KubeConfig.from_file(self.kubeconfig_path))
         elif self.auth_method == "service-account":
-            self.__kube_api = HTTPClient(KubeConfig.from_service_account())
+            self.__kube_api = NoVerify_Kube_Client.get_client(KubeConfig.from_service_account())
         else:
             raise ValueError("Illegal auth_method")
+        self.__kube_api.session.verify = False
         self.create_id()
 
     @property

workflows/pipe-common/scripts/cluster_setup.py

@@ -13,14 +13,14 @@
 # limitations under the License.
 
 from pipeline import Logger, TaskStatus, PipelineAPI, StatusEntry
+from pipeline.common.noverify_kubernetes import NoVerify_Kube_Client
 import argparse
 import os
 import subprocess
 import time
 
 try:
     from pykube.config import KubeConfig
-    from pykube.http import HTTPClient
     from pykube.http import HTTPError
     from pykube.objects import Pod
     from pykube.objects import Event
@@ -54,7 +54,7 @@ def __init__(self, obj, run_id):
 class Kubernetes:
 
     def __init__(self):
-        self.__kube_api = HTTPClient(KubeConfig.from_service_account())
+        self.__kube_api = NoVerify_Kube_Client.get_client(KubeConfig.from_service_account())
         self.__kube_api.session.verify = False
 
     def get_pod(self, run_id):

workflows/pipe-common/shell/kube_setup_master

@@ -186,7 +186,7 @@ bkp_http_proxy="$http_proxy"
 bkp_https_proxy="$https_proxy"
 bkp_no_proxy="$no_proxy"
 if [ "$CP_KUBE_KEEP_KUBEADM_PROXIES" != "1" ]; then
-  unset http_proxy https_proxy no_proxy
+  unset http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY
 fi
 
 ######################################
@@ -421,11 +421,45 @@ fi
 # Update the /etc/resolv.conf with the Kube's DNS IP
 #############################################################
 if [ "$CP_CAP_KUBE_DNS_ADD_TO_HOST" == "true" ]; then
+    _external_dns=$(cat /etc/resolv.conf | grep -im 1 '^nameserver' | cut -d ' ' -f2)
+    if [ $? -ne 0 ] || [ -z "$_external_dns" ]; then
+        _external_dns="10.96.0.10"
+    fi
+
     \cp /etc/resolv.conf /tmp/resolv.conf
+    \cp /etc/resolv.conf /tmp/resolv.conf.orig
     sed -i "/$CP_CAP_KUBE_DNS_IP/d" /tmp/resolv.conf
-    echo "nameserver $CP_CAP_KUBE_DNS_IP" >> /tmp/resolv.conf
-    echo "options rotate" >> /tmp/resolv.conf
+    sed -i "/options rotate/d" /tmp/resolv.conf
+
+    if [ "$_dns_plugin" == "coredns" ]; then
+        # Here we fix "forward" option of CoreDNS to point to the external IP instead of a defailt /etc/resolv.conf
+        # https://coredns.io/plugins/forward/
+        kubectl get cm coredns -n kube-system -o yaml > /tmp/coredns.yaml.orig
+        sed "s|/etc/resolv.conf|$_external_dns|g" /tmp/coredns.yaml.orig > /tmp/coredns.yaml
+        kubectl replace -n kube-system -f /tmp/coredns.yaml
+        if [ $? -eq 0 ]; then
+            pipe_log_info "Core DNS is configured to use $_external_dns upstream" "$KUBE_MASTER_SETUP_TASK"
+        else
+            pipe_log_warn "[WARN] Cannot set $_external_dns upstream for Core DNS" "$KUBE_MASTER_SETUP_TASK"
+        fi
+        kubectl rollout restart -n kube-system deployment/coredns
+        pipe_log_info "Core DNS has been restarted to pickup any changes to the configuration" "$KUBE_MASTER_SETUP_TASK"
+
+        # Once done - we can use Core DNS resolver as for the whole node, as it will proxy requests
+        # to the external DNS
+        sed -i "1 i\nameserver $CP_CAP_KUBE_DNS_IP" /tmp/resolv.conf
+    else
+        echo "nameserver $CP_CAP_KUBE_DNS_IP" >> /tmp/resolv.conf
+        echo "options rotate" >> /tmp/resolv.conf
+    fi
+
     \cp /tmp/resolv.conf /etc/resolv.conf
+
+    if [ $? -eq 0 ]; then
+        pipe_log_info "$CP_CAP_KUBE_DNS_IP is set as a default DNS resolver" "$KUBE_MASTER_SETUP_TASK"
+    else
+        pipe_log_warn "[WARN] Cannot set $CP_CAP_KUBE_DNS_IP as a default DNS resolver" "$KUBE_MASTER_SETUP_TASK"
+    fi
 fi
 
 (

workflows/pipe-common/shell/kube_setup_worker

@@ -81,7 +81,7 @@ systemctl start kubepods-besteffort.slice
 systemctl start kubepods-burstable.slice
 
 # Removing the proxies to connect to the master directly
-unset http_proxy https_proxy no_proxy
+unset http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY
 
 kubeadm join --token $CP_CAP_KUBE_MASTER_TOKEN $_kube_master_ip:$CP_CAP_KUBE_MASTER_API_PORT --discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors all
 if [ $? -ne 0 ]; then