Changed to keep map of DNS addresses due to 5 second DNS delays.

lsst · Dec 13, 2019 · ec0d6d7 · ec0d6d7
1 parent 914341a
commit ec0d6d7
Show file tree

Hide file tree

Showing 43 changed files with 467 additions and 496 deletions.
diff --git a/admin/templates/configuration/etc/log4cxx.index.properties b/admin/templates/configuration/etc/log4cxx.index.properties
@@ -5,8 +5,9 @@
 # export LSST_LOG_CONFIG=$HOME/.lsst/log4cxx.unittest.properties
 #
 
-#log4j.rootLogger=INFO, CONSOLE
-log4j.rootLogger=DEBUG, CONSOLE
+log4j.rootLogger=INFO, CONSOLE
+#log4j.rootLogger=DEBUG, CONSOLE
+#log4j.rootLogger=WARN, CONSOLE
 
 log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
 log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout

diff --git a/admin/templates/configuration/etc/log4cxx.index_master.properties b/admin/templates/configuration/etc/log4cxx.index_master.properties
@@ -0,0 +1,18 @@
+#
+# Configuration file for log4cxx
+# can be used for unit test
+# by launching next command before unit tests:
+# export LSST_LOG_CONFIG=$HOME/.lsst/log4cxx.unittest.properties
+#
+
+#log4j.rootLogger=INFO, CONSOLE
+log4j.rootLogger=DEBUG, CONSOLE
+#log4j.rootLogger=WARN, CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+#log4j.appender.CONSOLE.layout.ConversionPattern=[%d{yyyy-MM-ddTHH:mm:ss.SSSZ}] [%t] %-5p %c{2} (%F:%L) - %m%n
+log4j.appender.CONSOLE.layout.ConversionPattern=[%d{ddTHH:mm:ss.SSSZ}] [%t] %-5p %c{2} (%F:%L) - %m%n
+
+# Tune log at the module level
+#log4j.logger.lsst.qserv.util=DEBUG
diff --git a/admin/tools/docker/index/container/dev/README b/admin/tools/docker/index/container/dev/README
@@ -6,8 +6,6 @@
 # These are the optional building steps (in case if this container needs to
 # be rebuilt from scratch (which is going to take ~2 hours or longer)
 
-#docker build -t qserv/replica:dev .        &&&
-#docker push qserv/replica:dev              &&&
 docker build -t qserv/loaderbase:dev .
 docker push qserv/loaderbase:dev
 

diff --git a/admin/tools/docker/index/container/dev/master/appMaster.bash b/admin/tools/docker/index/container/dev/master/appMaster.bash
@@ -13,7 +13,7 @@ source /qserv/stack/loadLSST.bash
 cd /home/qserv/dev/qserv
 setup -r . -t qserv-dev
 
-export LSST_LOG_CONFIG=/home/qserv/dev/qserv/admin/templates/configuration/etc/log4cxx.index.properties
+export LSST_LOG_CONFIG=/home/qserv/dev/qserv/admin/templates/configuration/etc/log4cxx.index_master.properties
 
 /home/qserv/dev/qserv/build/loader/appMaster /home/qserv/dev/qserv/core/modules/loader/config/master.cnf
 

diff --git a/admin/tools/docker/index/index-k8-10m.yaml b/admin/tools/docker/index/index-k8-10m.yaml
@@ -1,5 +1,3 @@
-apiVersion: v1
-kind: Service
 metadata:
   name: imaster-svc
   labels:
@@ -61,7 +59,7 @@ metadata:
 spec:
   serviceName: iworker-svc
   podManagementPolicy: Parallel
-  replicas: 3
+  replicas: 14
   selector:
     matchLabels:
       app: iworker-pod
@@ -116,7 +114,7 @@ spec:
       - name: iclientnum-ctr
         image: qserv/indexclientnum:dev
         imagePullPolicy: Always
-        args: ["1000000", "1", "client-k8s-a1.cnf"]
+        args: ["10000000", "1", "client-k8s-a1.cnf"]
         ports:
         - containerPort: 10050
           protocol: UDP
@@ -157,7 +155,7 @@ spec:
       - name: iclientnum2-ctr
         image: qserv/indexclientnum:dev
         imagePullPolicy: Always
-        args: ["2000001", "3000001", "client-k8s-a2.cnf"]
+        args: ["20000001", "30000001", "client-k8s-a2.cnf"]
         ports:
         - containerPort: 10050
           protocol: UDP
@@ -198,9 +196,218 @@ spec:
       - name: iclientnum3-ctr
         image: qserv/indexclientnum:dev
         imagePullPolicy: Always
-        args: ["1000001", "2000000", "client-k8s-a3.cnf"]
+        args: ["10000001", "20000000", "client-k8s-a3.cnf"]
+        ports:
+        - containerPort: 10050
+          protocol: UDP
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: iclientnum4-svc
+  labels:
+    app: index
+spec:
+  ports:
+  - port: 10050
+    protocol: UDP
+  clusterIP: None
+  selector:
+    app: iclientnum4-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: iclientnum4-sts
+  labels:
+    app: index
+spec:
+  serviceName: iclientnum4-svc
+  podManagementPolicy: Parallel
+  replicas: 1
+  selector:
+    matchLabels:
+      app: iclientnum4-pod
+  template:
+    metadata:
+      labels:
+        app: iclientnum4-pod
+    spec:
+      containers:
+      - name: iclientnum3-ctr
+        image: qserv/indexclientnum:dev
+        imagePullPolicy: Always
+        args: ["40000001", "50000000", "client-k8s-a1.cnf"]
         ports:
         - containerPort: 10050
-          protocol: UDP          
+          protocol: UDP
 
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: iclientnum5-svc
+  labels:
+    app: index
+spec:
+  ports:
+  - port: 10050
+    protocol: UDP
+  clusterIP: None
+  selector:
+    app: iclientnum5-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: iclientnum5-sts
+  labels:
+    app: index
+spec:
+  serviceName: iclientnum5-svc
+  podManagementPolicy: Parallel
+  replicas: 1
+  selector:
+    matchLabels:
+      app: iclientnum5-pod
+  template:
+    metadata:
+      labels:
+        app: iclientnum5-pod
+    spec:
+      containers:
+      - name: iclientnum5-ctr
+        image: qserv/indexclientnum:dev
+        imagePullPolicy: Always
+        args: ["50000001", "60000000", "client-k8s-a1.cnf"]
+        ports:
+        - containerPort: 10050
+          protocol: UDP
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: iclientnum6-svc
+  labels:
+    app: index
+spec:
+  ports:
+  - port: 10050
+    protocol: UDP
+  clusterIP: None
+  selector:
+    app: iclientnum6-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: iclientnum6-sts
+  labels:
+    app: index
+spec:
+  serviceName: iclientnum6-svc
+  podManagementPolicy: Parallel
+  replicas: 1
+  selector:
+    matchLabels:
+      app: iclientnum6-pod
+  template:
+    metadata:
+      labels:
+        app: iclientnum6-pod
+    spec:
+      containers:
+      - name: iclientnum6-ctr
+        image: qserv/indexclientnum:dev
+        imagePullPolicy: Always
+        args: ["60000001", "70000000", "client-k8s-a1.cnf"]
+        ports:
+        - containerPort: 10050
+          protocol: UDP
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: iclientnum7-svc
+  labels:
+    app: index
+spec:
+  ports:
+  - port: 10050
+    protocol: UDP
+  clusterIP: None
+  selector:
+    app: iclientnum7-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: iclientnum7-sts
+  labels:
+    app: index
+spec:
+  serviceName: iclientnum7-svc
+  podManagementPolicy: Parallel
+  replicas: 1
+  selector:
+    matchLabels:
+      app: iclientnum7-pod
+  template:
+    metadata:
+      labels:
+        app: iclientnum7-pod
+    spec:
+      containers:
+      - name: iclientnum7-ctr
+        image: qserv/indexclientnum:dev
+        imagePullPolicy: Always
+        args: ["70000001", "80000000", "client-k8s-a1.cnf"]
+        ports:
+        - containerPort: 10050
+          protocol: UDP
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: iclientnum8-svc
+  labels:
+    app: index
+spec:
+  ports:
+  - port: 10050
+    protocol: UDP
+  clusterIP: None
+  selector:
+    app: iclientnum8-pod
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: iclientnum8-sts
+  labels:
+    app: index
+spec:
+  serviceName: iclientnum8-svc
+  podManagementPolicy: Parallel
+  replicas: 1
+  selector:
+    matchLabels:
+      app: iclientnum8-pod
+  template:
+    metadata:
+      labels:
+        app: iclientnum8-pod
+    spec:
+      containers:
+      - name: iclientnum8-ctr
+        image: qserv/indexclientnum:dev
+        imagePullPolicy: Always
+        args: ["80000001", "90000000", "client-k8s-a1.cnf"]
+        ports:
+        - containerPort: 10050
+          protocol: UDP
 
diff --git a/core/modules/loader/BufferUdp.cc b/core/modules/loader/BufferUdp.cc
@@ -48,7 +48,7 @@ MsgElement::Ptr BufferUdp::readFromSocket(boost::asio::ip::tcp::socket& socket,
 
         // If there's something in the buffer already, get it and return.
         // This can happen when the previous read of socket read multiple elements.
-        MsgElement::Ptr msgElem = _safeRetrieve("1readFromSocket&&&" + note);
+        MsgElement::Ptr msgElem = _safeRetrieve("1readFromSocket" + note);
         if (msgElem != nullptr) {
             return msgElem;
         }
@@ -69,7 +69,7 @@ MsgElement::Ptr BufferUdp::readFromSocket(boost::asio::ip::tcp::socket& socket,
 
         /// Try to retrieve an element (there's no guarantee that an entire element got read in a single read.
         // Store original cursor positions so they can be restored if the read fails.
-        msgElem = _safeRetrieve("2readFromSocket&&&" + note);
+        msgElem = _safeRetrieve("2readFromSocket" + note);
         if (msgElem != nullptr) {
             return msgElem;
         }
@@ -117,11 +117,11 @@ void BufferUdp::advanceReadCursor(size_t len) {
 }
 
 
-std::shared_ptr<MsgElement> BufferUdp::_safeRetrieve(std::string const& note) { // &&& delete note, maybe
+std::shared_ptr<MsgElement> BufferUdp::_safeRetrieve(std::string const& note) {
     auto wCursorOriginal = _wCursor;
     auto rCursorOriginal = _rCursor;
     // throwOnMissing=false since missing data is possible with TCP.
-    MsgElement::Ptr msgElem = MsgElement::retrieve(*this, note + " _safeRetrieve &&&", false);
+    MsgElement::Ptr msgElem = MsgElement::retrieve(*this, note + " _safeRetrieve", false);
     if (msgElem != nullptr) {
         return msgElem;
     } else {
@@ -134,20 +134,7 @@ std::shared_ptr<MsgElement> BufferUdp::_safeRetrieve(std::string const& note) {
 
 bool BufferUdp::isRetrieveSafe(size_t len) const {
     auto newLen = (_rCursor + len);
-    // &&&return (newLen <= _end && newLen <= _wCursor);
-    bool res = (newLen <= _end && newLen <= _wCursor); // &&&
-    if (!res) { // &&&
-        LOGS(_log, LOG_LVL_WARN, "&&& BufferUdp::isRetrieveSafe not safe len=" << len <<
-                                 " rCursor=" << (void*)_rCursor <<
-                                 " newLen=" << (void*)newLen <<
-                                 " wCursor=" << (void*)_wCursor <<
-                                 " _end=" << (void*)_end <<
-                                 " (newLen<=end)=" << (newLen <= _end) <<
-                                 " (newLen<=_wCursor)=" << (newLen <= _wCursor) <<
-                                 " res=" << res);
-        LOGS(_log, LOG_LVL_WARN, "&&& BufferUdp::isRetrieveSafe " << dumpStr(false));
-    }
-    return res;
+    return (newLen <= _end && newLen <= _wCursor);
 }
 
 
@@ -157,7 +144,7 @@ bool BufferUdp::retrieve(void* out, size_t len) {
         _rCursor += len;
         return true;
     }
-    LOGS(_log, LOG_LVL_WARN, "&&& BufferUdp::retrieve not safe len=" << len);
+    LOGS(_log, LOG_LVL_DEBUG, "BufferUdp::retrieve not safe len=" << len);
     return false;
 }
 

diff --git a/core/modules/loader/BufferUdp.h b/core/modules/loader/BufferUdp.h
@@ -46,7 +46,7 @@ class MsgElement;
 
 /// A buffer for reading and writing. Nothing can be read from the buffer until
 /// something has been written to it.
-/// TODO: rename BufferUdp is not really accurate anymore. &&&
+/// TODO: rename BufferUdp is not really accurate anymore.
 class BufferUdp {
 public:
     using Ptr = std::shared_ptr<BufferUdp>;

diff --git a/core/modules/loader/Central.cc b/core/modules/loader/Central.cc
@@ -77,7 +77,6 @@ void Central::_checkDoList() {
     while(_loop) {
         // Run and then sleep for a second. TODO A more advanced timer should be used
         doList->checkList();
-        LOGS(_log, LOG_LVL_INFO, "&&& SLEEP");
         usleep(_loopSleepTime);
     }
 }