sonic-net · mssonicbld · Jul 6, 2023 · May 30, 2023
@@ -28,7 +28,9 @@ PTCH_LIST  = $(TEMP_HW_MGMT_DIR)/series
 KCFG_LIST = $(TEMP_HW_MGMT_DIR)/kconfig
 HWMGMT_NONUP_LIST = $(BUILD_WORKDIR)/$($(MLNX_HW_MANAGEMENT)_SRC_PATH)/hwmgmt_nonup_patches
 HWMGMT_USER_OUTFILE = $(BUILD_WORKDIR)/integrate-mlnx-hw-mgmt_user.out
+SDK_USER_OUTFILE = $(BUILD_WORKDIR)/integrate-mlnx-sdk_user.out
 TMPFILE_OUT := $(shell mktemp)
+SDK_TMPDIR := $(shell mktemp -d)
 SB_COM_MSG := $(shell mktemp -t sb_commit_msg_file_XXXXX.log)
 SLK_COM_MSG := $(shell mktemp -t slk_commit_msg_file_XXXXX.log)
 SB_HEAD = $(shell git rev-parse --short HEAD)
@@ -140,4 +142,54 @@ endif
 
 	popd $(LOG_SIMPLE)
 
-SONIC_PHONY_TARGETS += integrate-mlnx-hw-mgmt
+integrate-mlnx-sdk:
+	$(FLUSH_LOG)
+	rm -rf $(MLNX_SDK_VERSION).zip sx_kernel-$(MLNX_SDK_VERSION)-$(MLNX_SDK_ISSU_VERSION).tar.gz
+
+ifeq ($(SDK_FROM_SRC),y)
+	wget $(MLNX_SDK_SOURCE_BASE_URL)/sx_kernel-$(MLNX_SDK_VERSION)-$(MLNX_SDK_ISSU_VERSION).tar.gz $(LOG_SIMPLE)
+	tar -xf sx_kernel-$(MLNX_SDK_VERSION)-$(MLNX_SDK_ISSU_VERSION).tar.gz --strip-components=1 -C $(SDK_TMPDIR) $(LOG_SIMPLE)
+else
+	# Download from upstream repository
+	wget $(MLNX_SDK_DRIVERS_GITHUB_URL)/archive/refs/heads/$(MLNX_SDK_VERSION).zip $(LOG_SIMPLE)
+	unzip $(MLNX_SDK_VERSION).zip -d $(SDK_TMPDIR) $(LOG_SIMPLE)
+	mv $(SDK_TMPDIR)/Spectrum-SDK-Drivers-$(MLNX_SDK_VERSION)/* $(SDK_TMPDIR) $(LOG_SIMPLE)
+endif
+
+	pushd $(BUILD_WORKDIR)/src/sonic-linux-kernel; git clean -f -- patch/; git stash -- patch/
+ifeq ($(CREATE_BRANCH), y)
+	git checkout -B "$(BRANCH_SONIC)_$(SLK_HEAD)_integrate_$(MLNX_SDK_VERSION)" HEAD
+	echo $(BRANCH_SONIC)_$(SLK_HEAD)_integrate_$(MLNX_SDK_VERSION) branch created in sonic-linux-kernel $(LOG_SIMPLE)
+endif
+	popd
+
+	echo "#### Integrate SDK $(MLNX_SDK_VERSION) Kernel Patches into SONiC" > ${SDK_USER_OUTFILE}
+
+	pushd $(BUILD_WORKDIR)/$(PLATFORM_PATH) $(LOG_SIMPLE)
+
+    # Run tests
+	pushd integration-scripts/tests; pytest-3 -v; popd
+
+	integration-scripts/sdk_kernel_patches.py \
+                            --sonic_kernel_ver $(KERNEL_VERSION) \
+                            --patches $(SDK_TMPDIR) \
+                            --slk_msg $(SLK_COM_MSG) \
+                            --sdk_ver $(MLNX_SDK_VERSION) \
+                            --build_root $(BUILD_WORKDIR) $(LOG_SIMPLE)
+
+    # Commit the changes in linux kernel and and log the diff
+	pushd $(BUILD_WORKDIR)/src/sonic-linux-kernel
+	git add -- patch/
+
+	echo -en "\n###-> series file changes in sonic-linux-kernel <-###\n" >> ${SDK_USER_OUTFILE}
+	git diff --no-color --staged -- patch/series >> ${SDK_USER_OUTFILE}
+
+	echo -en "\n###-> summary of files updated in sonic-linux-kernel <-###\n" >> ${SDK_USER_OUTFILE}
+	git diff --no-color --staged --stat >> ${SDK_USER_OUTFILE}
+
+	git diff --staged --quiet || git commit -m "$$(cat $(SLK_COM_MSG))"
+	popd
+
+	popd $(LOG_SIMPLE)
+
+SONIC_PHONY_TARGETS += integrate-mlnx-hw-mgmt integrate-mlnx-sdk
@@ -4,6 +4,7 @@
 
 MARK_ID = "###->"
 MLNX_KFG_MARKER = "mellanox"
+SDK_MARKER = "mellanox_sdk"
 HW_MGMT_MARKER = "mellanox_hw_mgmt"
 SLK_PATCH_LOC = "src/sonic-linux-kernel/patch/"
 SLK_KCONFIG = SLK_PATCH_LOC + "kconfig-inclusions"
@@ -13,6 +14,7 @@
 NON_UP_PATCH_LOC = NON_UP_PATCH_DIR + "patches"
 NON_UP_PATCH_DIFF = NON_UP_PATCH_DIR + "series.patch"
 KCFG_HDR_RE = "\[(.*)\]"
+KERNEL_BACKPORTS = "kernel_backports"
  # kconfig_inclusion headers to consider
 HDRS = ["common", "amd64"]
 
@@ -169,3 +171,22 @@ def perform(self):
 
     def write_user_out(self):
         pass
+
+
+def build_commit_description(changes):
+    if not changes:
+        return ""
+    content = "\n"
+    content = content + " ## Patch List\n"
+    for key, value in changes.items():
+        content = content + f"* {key} : {value}\n"
+    return content
+
+def parse_id(id_):
+    if id_ and id_ != "N/A":
+        id_ = "https://github.com/torvalds/linux/commit/" + id_
+
+    if id_ == "N/A":
+        id_ = ""
+
+    return id_
@@ -94,14 +94,6 @@ def load_patch_table(path, k_version):
                     table.append(table_line)
     return table
 
-def build_commit_description(changes):
-    if not changes:
-        return ""
-    content = "\n"
-    content = content + " ## Patch List\n"
-    for key, value in changes.items():
-        content = content + f"* {key} : {value}\n"
-    return content
 
 class Data:
     # list of new upstream patches
@@ -378,17 +370,12 @@ def list_patches(self):
         old_non_up_patches = [ptch.strip() for ptch in Data.old_non_up]
         return old_up_patches, old_non_up_patches
 
-    def parse_id(self, id_):
-        if id_:
-            id_ = "https://github.com/gregkh/linux/commit/" + id_
-        return id_
-
     def create_commit_msg(self, table):
         title = COMMIT_TITLE.format(self.args.hw_mgmt_ver) 
         changes_slk, changes_sb = {}, {}
         old_up_patches, old_non_up_patches = self.list_patches()
         for patch in table:
-            id_ = self.parse_id(patch.get(COMMIT_ID, ""))
+            id_ = parse_id(patch.get(COMMIT_ID, ""))
             patch_ = patch.get(PATCH_NAME)
             if patch_ in Data.new_up and patch_ not in old_up_patches:
                 changes_slk[patch_] = id_

@@ -0,0 +1,214 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+# Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import shutil
+import argparse
+import copy
+
+from helper import *
+
+COMMIT_TITLE = "Integrate SDK {} Kernel Patches"
+
+DELIMITTER = "========================"
+
+class Data:
+    # old sonic-linux-kernel series file 
+    old_series = list()
+    # updated sonic-linux-kernel series file
+    new_series = list()
+    # list of any new SDK patches to add
+    new_patches = list()
+    # list of current patch list in sonic-linux-kernel
+    old_patches = list()
+    # index of the mlnx_hw_mgmt patches start marker in old_series
+    i_sdk_start = -1 
+    # index of the mlnx_hw_mgmt patches end marker in old_series
+    i_sdk_end = -1
+    # kernel directory to consider the patches
+    k_dir = ""
+
+
+class SDKAction(Action):
+    def __init__(self, args):
+        super().__init__(args)
+
+    def check(self):
+        if not (self.args.patches and os.path.exists(self.args.patches)):
+            print("-> ERR: patch directory is missing ")
+            return False
+
+        if not (self.args.build_root and os.path.exists(self.args.build_root)):
+            print("-> ERR: build_root is missing")
+            return False
+
+        return True
+
+    def read_data(self):
+        Data.old_series = FileHandler.read_raw(os.path.join(self.args.build_root, SLK_SERIES))
+
+    def find_sdk_patches(self):
+        for index in range(Data.i_sdk_start+1, Data.i_sdk_end):
+            patch = Data.old_series[index].strip()
+            if patch:
+                Data.old_patches.append(Data.old_series[index].strip())
+        print("-> INFO Current mellanox sdk upstream patches. \n{}".format("\n".join(Data.old_patches)))
+
+    def get_kernel_dir(self):
+        # Get the kernel dir name to get the patches
+        try:
+            (kernel, major, minor) = self.args.sonic_kernel_ver.split(".")
+            minor_int = int(minor)
+        except Exception as e:
+            print("-> FATAL Kernel version formatting error: " + str(e))
+            sys.exit(1)
+
+        major_kernel_path = os.path.join(KERNEL_BACKPORTS, "{}.{}".format(kernel, major))
+
+        # if the k_dir with actual minor doesn't exit, use the closest minor version
+        for minor_i in range(minor_int, 0, -1):
+            path = os.path.join(major_kernel_path, "{}.{}.{}".format(kernel, major, minor_i))
+            if os.path.exists(os.path.join(self.args.patches, path)):
+                minor = str(minor_i)
+                Data.k_dir = path
+                print(f"-> INFO Kernel minor version dir {Data.k_dir} found ")
+                break
+
+        if not Data.k_dir:
+            print("-> Couldn't find the relevant kernel directory")
+            sys.exit(1)
+
+        path_to_check = os.path.join(self.args.patches, Data.k_dir)
+        if not os.path.exists(path_to_check):
+            print("-> FATAL Kernel dir with patches doesn't exist: {}".format(path_to_check))
+            sys.exit(1)
+
+    def update_series(self):
+         # Remove patches if they are already present in existing series file
+        patch_ids_remove = set()
+        for id in range(0, len(Data.old_series)): 
+            for patch in Data.new_patches:
+                if patch in Data.old_series[id]:
+                    patch_ids_remove.add(id)
+
+        # Modify the series files
+        temp_series = copy.deepcopy(Data.old_series)
+        Data.old_series.clear()
+        for id in range(0, len(temp_series)):
+            if id not in patch_ids_remove:
+                Data.old_series.append(temp_series[id])
+            else:
+                print("-> INFO Patch {} will be removed from the existing series".format(temp_series[id]))
+        print("-> INFO Updated Series file after removing SDK patches: \n{}".format("".join(Data.old_series)))
+
+    def add_new_patch_series(self):
+        new_patches = [patch + "\n" for patch in Data.new_patches]
+        Data.new_series = Data.old_series[0:Data.i_sdk_start+1] + new_patches + Data.old_series[Data.i_sdk_end:]
+
+    def process_update(self):
+        src_path = os.path.join(self.args.patches, Data.k_dir)
+        dst_path = os.path.join(self.args.build_root, SLK_PATCH_LOC)
+
+        for patch in Data.new_patches:
+            print(f"-> Moving patch: {patch}")
+            shutil.copy(os.path.join(src_path, patch), dst_path)
+
+        FileHandler.write_lines(os.path.join(self.args.build_root, SLK_SERIES), Data.new_series, True)
+        print("-> INFO Written sonic-linux-kernel series file \n{}".format("".join(Data.new_series)))
+
+    def get_new_patches(self):
+        patches_path = os.path.join(self.args.patches, Data.k_dir)
+        Data.new_patches = FileHandler.read_dir(patches_path, "*.patch")
+        Data.new_patches.sort()
+
+    def refresh_markers(self):
+        print("-> INFO Refreshing Markers ")
+        (Data.i_sdk_start, Data.i_sdk_end) = FileHandler.find_marker_indices(Data.old_series, SDK_MARKER)
+        if Data.i_sdk_start < 0 or Data.i_sdk_end > len(Data.old_series):
+            print("-> FATAL mellanox_sdk marker not found. Couldn't continue.. exiting")
+            sys.exit(1)
+        print("-> INFO mellanox_sdk markers found. start: {}, end: {}".format(Data.i_sdk_start, Data.i_sdk_end))
+
+    def fetch_patch_table(self, root_dir):
+        lines = FileHandler.read_strip(os.path.join(root_dir, "README"))
+        delim_id = 0
+        for id in range(0, len(lines)):
+            if DELIMITTER in lines[id]:
+                delim_id = id
+                break
+
+        table = dict()
+        for id in range(delim_id+1, len(lines)):
+            tokens = lines[id].split()
+            if len(tokens) != 3:
+                print("-> INFO Error Formatted line, {}".format(" ".join(lines[id])))
+                continue 
+            patch, commit = tokens[0], tokens[1]
+            if ".patch" not in patch:
+                print(f"-> INFO Unexpected Patch name {patch}")
+                continue
+            table[patch.strip()] = commit.strip() 
+
+        print(f"-> INFO Final Patch Table: {table}")
+        return table
+
+    def create_commit_msg(self, patch_table):
+        changes = {}
+        for patch in Data.new_patches:
+            if patch not in Data.old_patches:
+                changes[patch] = parse_id(patch_table.get(patch.strip(), ""))
+        slk_commit_msg = COMMIT_TITLE.format(self.args.sdk_ver) + "\n" + build_commit_description(changes)
+        print(f"-> INFO Commit Message: {slk_commit_msg}")
+        return slk_commit_msg
+
+    def perform(self):
+        self.read_data()
+        self.refresh_markers()
+        self.find_sdk_patches()
+        self.get_kernel_dir()
+        self.get_new_patches()
+        self.update_series()
+        self.refresh_markers()
+        self.add_new_patch_series()
+        self.process_update()
+        patch_table = self.fetch_patch_table(os.path.join(self.args.patches, Data.k_dir))
+        slk_msg = self.create_commit_msg(patch_table)
+        if self.args.slk_msg:
+            with open(self.args.slk_msg, 'w') as f:
+                f.write(slk_msg) 
+
+
+
+def create_parser():
+    # Create argument parser
+    parser = argparse.ArgumentParser()
+
+    # Optional arguments
+    parser.add_argument("--sonic_kernel_ver", type=str)
+    parser.add_argument("--patches", type=str)
+    parser.add_argument("--slk_msg", type=str)
+    parser.add_argument("--sdk_ver", type=str)
+    parser.add_argument("--build_root", type=str)
+    return parser
+
+if __name__ == '__main__':
+    parser = create_parser()
+    action = SDKAction(parser.parse_args())
+    action.check()
+    action.perform()
@@ -37,15 +37,15 @@
 TEST_SLK_COMMIT = """\
 Intgerate HW-MGMT 7.0030.0937 Changes
  ## Patch List
-* 0002-i2c-mlxcpld-Decrease-polling-time-for-performance-im.patch : https://github.com/gregkh/linux/commit/cb9744178f33
-* 0003-i2c-mlxcpld-Add-support-for-I2C-bus-frequency-settin.patch : https://github.com/gregkh/linux/commit/66b0c2846ba8
-* 0005-i2c-mux-mlxcpld-Move-header-file-out-of-x86-realm.patch : https://github.com/gregkh/linux/commit/98d29c410475
-* 0006-i2c-mux-mlxcpld-Convert-driver-to-platform-driver.patch : https://github.com/gregkh/linux/commit/84af1b168c50
-* 0007-i2c-mux-mlxcpld-Prepare-mux-selection-infrastructure.patch : https://github.com/gregkh/linux/commit/81566938083a
-* 0008-i2c-mux-mlxcpld-Get-rid-of-adapter-numbers-enforceme.patch : https://github.com/gregkh/linux/commit/cae5216387d1
-* 0009-i2c-mux-mlxcpld-Extend-driver-to-support-word-addres.patch : https://github.com/gregkh/linux/commit/c52a1c5f5db5
-* 0010-i2c-mux-mlxcpld-Extend-supported-mux-number.patch : https://github.com/gregkh/linux/commit/699c0506543e
-* 0011-i2c-mux-mlxcpld-Add-callback-to-notify-mux-creation-.patch : https://github.com/gregkh/linux/commit/a39bd92e92b9
+* 0002-i2c-mlxcpld-Decrease-polling-time-for-performance-im.patch : https://github.com/torvalds/linux/commit/cb9744178f33
+* 0003-i2c-mlxcpld-Add-support-for-I2C-bus-frequency-settin.patch : https://github.com/torvalds/linux/commit/66b0c2846ba8
+* 0005-i2c-mux-mlxcpld-Move-header-file-out-of-x86-realm.patch : https://github.com/torvalds/linux/commit/98d29c410475
+* 0006-i2c-mux-mlxcpld-Convert-driver-to-platform-driver.patch : https://github.com/torvalds/linux/commit/84af1b168c50
+* 0007-i2c-mux-mlxcpld-Prepare-mux-selection-infrastructure.patch : https://github.com/torvalds/linux/commit/81566938083a
+* 0008-i2c-mux-mlxcpld-Get-rid-of-adapter-numbers-enforceme.patch : https://github.com/torvalds/linux/commit/cae5216387d1
+* 0009-i2c-mux-mlxcpld-Extend-driver-to-support-word-addres.patch : https://github.com/torvalds/linux/commit/c52a1c5f5db5
+* 0010-i2c-mux-mlxcpld-Extend-supported-mux-number.patch : https://github.com/torvalds/linux/commit/699c0506543e
+* 0011-i2c-mux-mlxcpld-Add-callback-to-notify-mux-creation-.patch : https://github.com/torvalds/linux/commit/a39bd92e92b9
 
 """