Merge branch 'master' into dev-logrotate-config

.azure-pipelines/build-swss-template.yml

@@ -107,6 +107,10 @@ jobs:
         target/debs/${{ parameters.debian_version }}/libnl-nf*.deb
         target/debs/${{ parameters.debian_version }}/libyang-*.deb
         target/debs/${{ parameters.debian_version }}/libyang_*.deb
+        target/debs/${{ parameters.debian_version }}/libprotobuf*.deb
+        target/debs/${{ parameters.debian_version }}/libprotoc*.deb
+        target/debs/${{ parameters.debian_version }}/protobuf-compiler*.deb
+        target/debs/${{ parameters.debian_version }}/libdashapi*.deb
     displayName: "Download common libs"
 
   - script: |

.azure-pipelines/docker-sonic-vs/Dockerfile

@@ -4,4 +4,8 @@ ARG docker_container_name
 
 COPY ["debs", "/debs"]
 
+# Remove existing packages first before installing the new/current packages. This is to overcome limitations with
+# Docker's diff detection mechanism, where only the file size and the modification timestamp (which will remain the
+# same, even though contents have changed) are checked between the previous and current layer.
+RUN dpkg --purge libswsscommon python3-swsscommon sonic-db-cli libsaimetadata libsairedis libsaivs syncd-vs swss sonic-eventd
 RUN dpkg -i /debs/libswsscommon_1.0.0_amd64.deb /debs/python3-swsscommon_1.0.0_amd64.deb /debs/sonic-db-cli_1.0.0_amd64.deb /debs/libsaimetadata_1.0.0_amd64.deb /debs/libsairedis_1.0.0_amd64.deb /debs/libsaivs_1.0.0_amd64.deb /debs/syncd-vs_1.0.0_amd64.deb /debs/swss_1.0.0_amd64.deb

.azure-pipelines/test-docker-sonic-vs-template.yml

@@ -6,6 +6,10 @@ parameters:
 - name: log_artifact_name
   type: string
 
+- name: sonic_buildimage_ubuntu20_04
+  type: string
+  default: '$(BUILD_BRANCH)'
+
 jobs:
 - job:
   displayName: vstest
@@ -35,13 +39,25 @@ jobs:
       artifact: sonic-swss-common.amd64.ubuntu20_04
       path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download pre-stage built sonic-swss-common.amd64.ubuntu20_04"
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: sonic-net.sonic-buildimage-ubuntu20.04
+      artifact: sonic-buildimage.amd64.ubuntu20_04
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/${{ parameters.sonic_buildimage_ubuntu20_04 }}'
+      path: $(Build.ArtifactStagingDirectory)/download
+    displayName: "Download sonic buildimage ubuntu20.04 deb packages"
 
   - script: |
       set -ex
       ls -l
       sudo sonic-swss-common/.azure-pipelines/build_and_install_module.sh
 
       sudo apt-get install -y libhiredis0.14 libyang0.16
+      sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libprotobuf*_amd64.deb $(Build.ArtifactStagingDirectory)/download/libprotobuf-lite*_amd64.deb $(Build.ArtifactStagingDirectory)/download/python3-protobuf*_amd64.deb
+      sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libdashapi*.deb
       sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f
       sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb
 

common/binaryserializer.h

@@ -3,6 +3,8 @@
 
 #include "common/armhelper.h"
 
+using namespace std;
+
 namespace swss {
 
 class BinarySerializer {
@@ -64,7 +66,7 @@ class BinarySerializer {
                                                                                             size);
             }
 
-            auto pkey = tmp_buffer;
+            auto pkey = string(tmp_buffer, *pkeylen);
             tmp_buffer += *pkeylen;
 
             WARNINGS_NO_CAST_ALIGN;
@@ -79,7 +81,7 @@ class BinarySerializer {
                                                                                             size);
             }
 
-            auto pval = tmp_buffer;
+            auto pval = string(tmp_buffer, *pvallen);
             tmp_buffer += *pvallen;
 
             values.push_back(std::make_pair(pkey, pval));
@@ -107,9 +109,8 @@ class BinarySerializer {
     void setKeyAndValue(const char* key, size_t klen,
                         const char* value, size_t vlen)
     {
-        // to improve deserialize performance, copy null-terminated string. 
-        setData(key, klen + 1);
-        setData(value, vlen + 1);
+        setData(key, klen);
+        setData(value, vlen);
 
         m_kvp_count++;
     }

common/configdb.h

@@ -105,8 +105,11 @@ class ConfigDBConnector_Native : public SonicV2Connector_Native
                     try:
                         (table, row) = key.split(self.TABLE_NAME_SEPARATOR, 1)
                         if table in self.handlers:
-                            client = self.get_redis_client(self.db_name)
-                            data = self.raw_to_typed(client.hgetall(key))
+                            if item['data'] == 'del':
+                                data = None
+                            else:
+                                client = self.get_redis_client(self.db_name)
+                                data = self.raw_to_typed(client.hgetall(key))
                             if table in init_data and row in init_data[table]:
                                 cache_hit = init_data[table][row] == data
                                 del init_data[table][row]
@@ -120,7 +123,7 @@ class ConfigDBConnector_Native : public SonicV2Connector_Native
         ## Dynamic typed functions used in python
         @staticmethod
         def raw_to_typed(raw_data):
-            if not raw_data or not raw_data.keys():
+            if raw_data is None:
                 return None
             typed_data = {}
             for raw_key in raw_data:

common/events_common.h

@@ -258,7 +258,9 @@ struct serialization
     int
     deserialize(const string& s, Map& data)
     {
-
+        if (s.size() < 2) { // zmq identifying message of length 1
+            return 0;
+        }
         try {
             istringstream ss(s);
             boost::archive::text_iarchive iarch(ss);

common/rediscommand.cpp

@@ -25,16 +25,18 @@ void RedisCommand::format(const char *fmt, ...)
         redisFreeCommand(temp);
         temp = nullptr;
     }
+    len = 0;
 
     va_list ap;
     va_start(ap, fmt);
-    len = redisvFormatCommand(&temp, fmt, ap);
+    int ret = redisvFormatCommand(&temp, fmt, ap);
     va_end(ap);
-    if (len == -1) {
+    if (ret == -1) {
         throw std::bad_alloc();
-    } else if (len == -2) {
+    } else if (ret == -2) {
         throw std::invalid_argument("fmt");
     }
+    len = ret;
 }
 
 void RedisCommand::formatArgv(int argc, const char **argv, const size_t *argvlen)
@@ -44,11 +46,13 @@ void RedisCommand::formatArgv(int argc, const char **argv, const size_t *argvlen
         redisFreeCommand(temp);
         temp = nullptr;
     }
+    len = 0;
 
-    len = redisFormatCommandArgv(&temp, argc, argv, argvlen);
-    if (len == -1) {
+    int ret = redisFormatCommandArgv(&temp, argc, argv, argvlen);
+    if (ret == -1) {
         throw std::bad_alloc();
     }
+    len = ret;
 }
 
 void RedisCommand::format(const vector<string> &commands)
@@ -135,6 +139,8 @@ std::string RedisCommand::toPrintableString() const
 
 const char *RedisCommand::c_str() const
 {
+    if (len == 0)
+        return nullptr;
     return temp;
 }
 

common/schema.h

@@ -147,6 +147,7 @@ namespace swss {
 #define APP_DASH_ACL_OUT_TABLE_NAME         "DASH_ACL_OUT_TABLE"
 #define APP_DASH_ACL_GROUP_TABLE_NAME       "DASH_ACL_GROUP_TABLE"
 #define APP_DASH_ACL_RULE_TABLE_NAME        "DASH_ACL_RULE_TABLE"
+#define APP_DASH_PREFIX_TAG_TABLE_NAME      "DASH_PREFIX_TAG_TABLE"
 #define APP_DASH_ROUTING_TYPE_TABLE_NAME    "DASH_ROUTING_TYPE_TABLE"
 #define APP_DASH_APPLIANCE_TABLE_NAME       "DASH_APPLIANCE_TABLE"
 #define APP_DASH_ROUTE_TABLE_NAME           "DASH_ROUTE_TABLE"

tests/Makefile.am

@@ -5,6 +5,7 @@ LDADD_GTEST = -L/usr/src/gtest -lgtest -lgtest_main -lgmock -lgmock_main
 
 tests_tests_SOURCES = tests/redis_ut.cpp                \
                       tests/redis_piped_ut.cpp          \
+                      tests/redis_command_ut.cpp        \
                       tests/redis_state_ut.cpp          \
                       tests/redis_piped_state_ut.cpp    \
                       tests/tokenize_ut.cpp             \
@@ -44,5 +45,5 @@ tests_tests_SOURCES = tests/redis_ut.cpp                \
                       tests/main.cpp
 
 tests_tests_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_GTEST) $(LIBNL_CFLAGS)
-tests_tests_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_GTEST) $(LIBNL_CFLAGS)
+tests_tests_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_GTEST) $(LIBNL_CFLAGS) -fno-access-control
 tests_tests_LDADD = $(LDADD_GTEST) -lpthread common/libswsscommon.la $(LIBNL_LIBS) $(CODE_COVERAGE_LIBS) sonic-db-cli/libsonicdbcli.la -lzmq -luuid -lboost_serialization

tests/binary_serializer_ut.cpp

@@ -29,7 +29,7 @@ TEST(BinarySerializer, serialize_deserialize)
                                                                 test_table);
     string serialized_str(buffer);
 
-    EXPECT_EQ(serialized_len, 107);
+    EXPECT_EQ(serialized_len, 101);
 
     auto ptr = std::make_shared<KeyOpFieldsValuesTuple>();
     KeyOpFieldsValuesTuple &kco = *ptr;
@@ -84,3 +84,48 @@ TEST(BinarySerializer, deserialize_overflow)
     auto& deserialized_values = kfvFieldsValues(kco);
     EXPECT_THROW(BinarySerializer::deserializeBuffer(buffer, serialized_len - 10, deserialized_values), runtime_error);
 }
+
+TEST(BinarySerializer, protocol_buffer)
+{
+    string test_entry_key = "test_key";
+    string test_command = "test_command";
+    string test_db = "test_db";
+    string test_table = "test_table";
+    string test_key = "key";
+    unsigned char binary_proto_buf[] = {0x0a, 0x05, 0x0d, 0x0a, 0x01, 0x00, 0x20, 0x10, 0xe1, 0x21};
+    string proto_buf_val = string((const char *)binary_proto_buf, sizeof(binary_proto_buf));
+    EXPECT_TRUE(proto_buf_val.length() == sizeof(binary_proto_buf));
+
+    char buffer[200];
+    std::vector<FieldValueTuple> values;
+    values.push_back(std::make_pair(test_key, proto_buf_val));
+    int serialized_len = (int)BinarySerializer::serializeBuffer(
+                                                                buffer,
+                                                                sizeof(buffer),
+                                                                test_entry_key,
+                                                                values,
+                                                                test_command,
+                                                                test_db,
+                                                                test_table);
+    string serialized_str(buffer);
+
+    EXPECT_EQ(serialized_len, 106);
+
+    auto ptr = std::make_shared<KeyOpFieldsValuesTuple>();
+    KeyOpFieldsValuesTuple &kco = *ptr;
+    auto& deserialized_values = kfvFieldsValues(kco);
+    BinarySerializer::deserializeBuffer(buffer, serialized_len, deserialized_values);
+
+    swss::FieldValueTuple fvt = deserialized_values.at(0);
+    EXPECT_TRUE(fvField(fvt) == test_db);
+    EXPECT_TRUE(fvValue(fvt) == test_table);
+
+    fvt = deserialized_values.at(1);
+    EXPECT_TRUE(fvField(fvt) == test_entry_key);
+    EXPECT_TRUE(fvValue(fvt) == test_command);
+
+    fvt = deserialized_values.at(2);
+    EXPECT_TRUE(fvField(fvt) == test_key);
+    EXPECT_TRUE(fvValue(fvt) == proto_buf_val);
+    EXPECT_TRUE(fvValue(fvt).length() == sizeof(binary_proto_buf));
+}

tests/events_common_ut.cpp

@@ -81,17 +81,20 @@ TEST(events_common, send_recv)
     void *sock_p1 = zmq_socket (zmq_ctx, ZMQ_PAIR);
     EXPECT_EQ(0, zmq_bind (sock_p1, path));
 
-    string source("Hello"), source1;
+    string source("Hello"), source1, source2("#");
 
     map<string, string> m = {{"foo", "bar"}, {"hello", "world"}, {"good", "day"}};
-    map<string, string> m1;
+    map<string, string> m1, m2;
 
     EXPECT_EQ(0, zmq_message_send(sock_p0, source, m));
 
     EXPECT_EQ(0, zmq_message_read(sock_p1, 0, source1, m1));
 
     EXPECT_EQ(source, source1);
     EXPECT_EQ(m, m1);
+
+    EXPECT_EQ(0, deserialize(source2, m2));
+
     zmq_close(sock_p0);
     zmq_close(sock_p1);
     zmq_ctx_term(zmq_ctx);

tests/redis_command_ut.cpp

@@ -0,0 +1,12 @@
+#include "gtest/gtest.h"
+#include "common/rediscommand.h"
+
+TEST(RedisCommand, invalid_redis_command)
+{
+    swss::RedisCommand cmd;
+    EXPECT_THROW(cmd.format("Invalid redis command %l^", 1), std::invalid_argument);
+    EXPECT_EQ(cmd.c_str(), nullptr);
+    EXPECT_EQ(cmd.length(), 0);
+    EXPECT_EQ(cmd.len, 0);
+    EXPECT_EQ(cmd.temp, nullptr);
+}

tests/stringutility_ut.cpp

@@ -8,7 +8,7 @@
 
 TEST(STRINGUTILITY, cast_int)
 {
-    int i;
+    int i = 0;
 
     EXPECT_NO_THROW(swss::lexical_convert("123", i));
     EXPECT_EQ(i, 123);
@@ -39,7 +39,7 @@ TEST(STRINGUTILITY, cast_alpha_bool)
 
 TEST(STRINGUTILITY, cast_mix)
 {
-    int i;
+    int i = 0;
     swss::AlphaBoolean b;
     std::string s;