diff --git a/.artifactignore b/.artifactignore index 1126a160d..af97bce55 100644 --- a/.artifactignore +++ b/.artifactignore @@ -1,2 +1,3 @@ **/* !*.deb +!kconfig-diff-*.rst diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 0b5233552..e47575aa7 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -32,6 +32,9 @@ parameters: - name: artifact_name type: string +- name: linux_deb_pattern + type: string + jobs: - job: pool: ${{ parameters.pool }} @@ -44,7 +47,17 @@ jobs: steps: - checkout: self clean: true - displayName: 'Checkout code' + displayName: 'Checkout code' + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipelineId: 426041 + artifact: ${{ parameters.artifact_name }} + runVersion: 'specific' + itemPattern: ${{ parameters.linux_deb_pattern }} + targetPath: $(Agent.TempDirectory)/ + displayName: "Download the linux-image artifact from the last build with a different kernel version" - script: | git config --global user.email "lguohan@build.com" git config --global user.name "Guohan Lu" @@ -52,6 +65,19 @@ jobs: cat /proc/cpuinfo CONFIGURED_ARCH=${{ parameters.arch }} CONFIGURED_PLATFORM=${{ parameters.platform }} make displayName: "Compile sonic kernel" + - script: | + dpkg-deb -x $(Agent.TempDirectory)/${{ parameters.linux_deb_pattern }} $(Agent.TempDirectory)/old + dpkg-deb -x $(System.DefaultWorkingDirectory)/${{ parameters.linux_deb_pattern }} $(Agent.TempDirectory)/new + pip3 install tabulate + python3 $(System.DefaultWorkingDirectory)/.azure-pipelines/kcfg-diff.py \ + --buildid $(Build.BuildId) \ + --ref_buildid 426041 \ + --arch ${{ parameters.arch }} \ + --old_kcfg $(Agent.TempDirectory)/old/boot/ \ + --new_kcfg $(Agent.TempDirectory)/new/boot/ \ + --output $(System.DefaultWorkingDirectory)/kconfig-diff-${{ parameters.platform }}-${{ parameters.arch }}.rst + cat $(System.DefaultWorkingDirectory)/kconfig-diff-${{ parameters.platform }}-${{ parameters.arch }}.rst + displayName: "Compute the kconfig diff" - publish: $(System.DefaultWorkingDirectory)/ artifact: ${{ parameters.artifact_name }} displayName: "Archive sonic kernel debian packages" diff --git a/.azure-pipelines/kcfg-diff.py b/.azure-pipelines/kcfg-diff.py new file mode 100644 index 000000000..3b4c8d814 --- /dev/null +++ b/.azure-pipelines/kcfg-diff.py @@ -0,0 +1,160 @@ +import argparse +import os +import glob +import sys +import re +from tabulate import tabulate + +COLUMN_WIDTH_MAX = 40 +MODIF_COLUMN_WIDTH_MAX = 80 + +RST_TEMPLATE = '''\ +SONiC-linux-kernel KConfig Difference: +========== + +Reference Kernel +------------ +- Version: {} +- BuildID: https://dev.azure.com/mssonic/build/_build/results?buildId={}&view=results + +Latest Kernel +------------ +- Version: {} +- BuildID: https://dev.azure.com/mssonic/build/_build/results?buildId={}&view=results + + +Additions & Deletions +------------ +{} + +Modifications +------------ +{} +''' + +def read_data(file1): + data = [] + try: + with open(file1, 'r') as f1: + data = f1.readlines() + except Exception as e: + print("ABORT: Reading failed from {}, {}".format(file1, str(e))) + sys.exit(1) + + data = filter(lambda line: not(line.startswith('#')), data) + ret = dict() + for line in data: + tokens = line.split('=') + if len(tokens) == 2: + key, val = tokens[0].strip(), tokens[-1].strip() + ret[key] = val + return ret + +def write_data(fname, data): + try: + with open(fname, 'w') as f: + f.write(data) + except Exception as e: + print("ABORT: Writing to the file {} failed {}".format(fname, str(e))) + sys.exit(1) + +def generate_diff(file1, file2): + data_f1 = read_data(file1) + data_f2 = read_data(file2) + + additions = [] + modifications = [] + deletions = [] + + for key_old, val_old in data_f1.items(): + val_new = data_f2.get(key_old, None) + if not val_new: + deletions.append("{}={}".format(key_old, val_old)) + elif val_old != val_new: + modifications.append("{}={}->{}".format(key_old, val_old, val_new)) + if val_new: + del data_f2[key_old] + + for key, val in data_f2.items(): + additions.append("{}={}".format(key, val)) + return additions, modifications, deletions + +def restrict_column_width(lis, width): + for i in range(0, len(lis)): + curr_width = len(lis[i]) + new_val = '' + num_newlines = int(curr_width/width) + for j in range(0, num_newlines+1): + if (j+1)*width < curr_width: + new_val += lis[i][j*width:(j+1)*width] + new_val += "\n" + else: + new_val += lis[i][j*width:] + lis[i] = new_val + +def format_diff_table(additions, modifications, deletions): + max_len = max(len(additions), len(deletions)) + additions += [''] * (max_len - len(additions)) + deletions += [''] * (max_len - len(deletions)) + + restrict_column_width(additions, COLUMN_WIDTH_MAX) + restrict_column_width(deletions, COLUMN_WIDTH_MAX) + restrict_column_width(modifications, MODIF_COLUMN_WIDTH_MAX) + + table_data = list(zip(additions, deletions)) + headers = ["ADDITIONS", "DELETIONS"] + + add_del = tabulate(table_data, headers=headers, tablefmt="grid") + mod = tabulate(list(zip(modifications)), headers=["MODIFICATIONS"], tablefmt="grid") + return add_del, mod + +def parse_kver(loc): + files = glob.glob(os.path.join(loc, "config-*")) + if len(files) > 1: + print("WARNING: Multiple config- files present under {}, {}".format(loc, files)) + result = re.search(r"config-(.*)", os.path.basename(files[-1])) + return result.group(1) + +def create_parser(): + # Create argument parser + parser = argparse.ArgumentParser() + + # Optional arguments + parser.add_argument("--arch", type=str, required=True) + parser.add_argument("--old_kcfg", type=str, required=True) + parser.add_argument("--new_kcfg", type=str, required=True) + parser.add_argument("--output", type=str, required=True) + parser.add_argument("--ref_buildid", type=str, required=True) + parser.add_argument("--buildid", type=str, required=True) + return parser + +def verify_args(args): + if not glob.glob(os.path.join(args.old_kcfg, "config-*")): + print("ABORT: config file missing under {}".format(args.old_kcfg)) + return False + + if not glob.glob(os.path.join(args.new_kcfg, "config-*")): + print("ABORT: config file missing under {}".format(args.new_kcfg)) + return False + + if not os.path.exists(os.path.dirname(args.output)): + print("ABORT: Output Folder {} doesn't exist".format(args.output)) + return False + return True + +if __name__ == "__main__": + parser = create_parser() + args = parser.parse_args() + if not verify_args(args): + sys.exit(1) + + ver_old = parse_kver(args.old_kcfg) + ver_new = parse_kver(args.new_kcfg) + f_old = os.path.join(args.old_kcfg, 'config-{}'.format(ver_old)) + f_new = os.path.join(args.new_kcfg, 'config-{}'.format(ver_new)) + + additions, modifications, deletions = generate_diff(f_old, f_new) + add_del, mod = format_diff_table(additions, modifications, deletions) + + diff = RST_TEMPLATE.format(ver_old, args.ref_buildid, ver_new, args.buildid, add_del, mod) + write_data(args.output, diff) diff --git a/README.md b/README.md index 20ed7b4d8..0e60a5d6e 100644 --- a/README.md +++ b/README.md @@ -50,3 +50,9 @@ If the files patch/kconfig-exclusions and patch/kconfig-inclusions exist, they w Also, the final kernel configuration will be checked to verify that: - all options asked to be excluded are effectively not present in the kernel, - and all options asked to be included are effectively present in the kernel, using the exact type (module or built-in) or string or number. + +## Kernel Configuration Difference during Upgrades + +During Kernel Upgrades, the maintainer must update the configuration diff in the wiki of this repo. This acts as a guide for keeping track of configuration changes between Kernel upgrades. Applies for minor version upgrades as well + +The diff is saved as kconfig-diff-{platform}-{arch}.rst under the artifacts of Azure.sonic-linux-kernel job runs. \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c87c512bd..e333e460f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,6 +17,7 @@ stages: arch: amd64 sonic_slave: sonic-slave-bookworm artifact_name: sonic-linux-kernel + linux_deb_pattern: linux-image-*-unsigned_*deb - template: .azure-pipelines/build-template.yml parameters: @@ -24,6 +25,7 @@ stages: pool: sonicbld-arm64 sonic_slave: sonic-slave-bookworm-arm64 artifact_name: sonic-linux-kernel.arm64 + linux_deb_pattern: linux-image-*-unsigned_*deb - template: .azure-pipelines/build-template.yml parameters: @@ -31,6 +33,7 @@ stages: pool: sonicbld-armhf sonic_slave: sonic-slave-bookworm-armhf artifact_name: sonic-linux-kernel.armhf + linux_deb_pattern: linux-image-*-armmp_*deb - template: .azure-pipelines/build-template.yml parameters: @@ -39,4 +42,5 @@ stages: pool: sonicbld-arm64 sonic_slave: sonic-slave-bookworm-arm64 artifact_name: sonic-linux-kernel.pensando.arm64 + linux_deb_pattern: linux-image-*-unsigned_*deb