From 99b0acbced7244e49236c2909053b45078fe5ebc Mon Sep 17 00:00:00 2001 From: Brian Ting Date: Wed, 21 Jun 2023 15:26:48 -0700 Subject: [PATCH] Creates build_info.json alongside cobalt_build_id. Creates build_info.json alongside the logic that creates cobalt_build_id.h. This offloads generating build_info.json from the CI system to GN and updates build_info.json with the build_id, build_rev, git_rev, build_time, author, and commit fields. On Git repos, git_rev is equivalent to build_rev. On Gerrit repos, git_rev represents the last Git hash synced, tied with Git build_id. b/284367797 --- cobalt/browser/BUILD.gn | 2 +- cobalt/build/BUILD.gn | 29 ++++- .../{build_id.py => build_id_and_info.py} | 33 ++++- cobalt/build/get_build_id.py | 63 --------- cobalt/build/get_build_info.py | 121 ++++++++++++++++++ ...uild_id_test.py => get_build_info_test.py} | 32 ++--- cobalt/h5vcc/BUILD.gn | 2 +- cobalt/network/BUILD.gn | 2 +- 8 files changed, 190 insertions(+), 94 deletions(-) rename cobalt/build/{build_id.py => build_id_and_info.py} (53%) delete mode 100755 cobalt/build/get_build_id.py create mode 100755 cobalt/build/get_build_info.py rename cobalt/build/{get_build_id_test.py => get_build_info_test.py} (75%) diff --git a/cobalt/browser/BUILD.gn b/cobalt/browser/BUILD.gn index 8d3adbc0d3f7..ce7a36ec9dfb 100644 --- a/cobalt/browser/BUILD.gn +++ b/cobalt/browser/BUILD.gn @@ -158,7 +158,7 @@ static_library("browser") { "//cobalt/base", "//cobalt/browser/memory_settings:browser_memory_settings", "//cobalt/browser/memory_tracker:memory_tracker_tool", - "//cobalt/build:cobalt_build_id", + "//cobalt/build:cobalt_build_info", "//cobalt/cache", "//cobalt/configuration", "//cobalt/css_parser", diff --git a/cobalt/build/BUILD.gn b/cobalt/build/BUILD.gn index 6c6e1e87202a..5cb8e7aade43 100644 --- a/cobalt/build/BUILD.gn +++ b/cobalt/build/BUILD.gn @@ -1,4 +1,4 @@ -# Copyright 2021 The Cobalt Authors. All Rights Reserved. +# Copyright 2023 The Cobalt Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -action("cobalt_build_id") { - cobalt_version = exec_script("get_build_id.py", [], "trim string") +action("cobalt_build_info") { + output = exec_script("get_build_info.py", [], "trim string") + build_info = string_split(output, "|") - script = "build_id.py" - outputs = [ "$root_gen_dir/cobalt_build_id.h" ] + build_number = build_info[0] + build_rev = build_info[1] + git_rev = build_info[2] + build_time = build_info[3] + author = build_info[4] + encoded_commit = build_info[5] + + script = "build_id_and_info.py" + outputs = [ + "$root_gen_dir/cobalt_build_id.h", + "$root_gen_dir/build_info.json", + ] args = [ rebase_path(outputs[0], root_build_dir), - cobalt_version, + rebase_path(outputs[1], root_build_dir), + build_number, + build_rev, + git_rev, + build_time, + author, + encoded_commit, ] } diff --git a/cobalt/build/build_id.py b/cobalt/build/build_id_and_info.py similarity index 53% rename from cobalt/build/build_id.py rename to cobalt/build/build_id_and_info.py index 1c6791f944ca..9793212c7530 100755 --- a/cobalt/build/build_id.py +++ b/cobalt/build/build_id_and_info.py @@ -13,6 +13,7 @@ # limitations under the License. """Generate a Cobalt build ID header.""" +import json import sys BUILD_ID_HEADER_TEMPLATE = """ @@ -20,7 +21,7 @@ #define _COBALT_BUILD_ID_H_ #ifndef COBALT_BUILD_VERSION_NUMBER -#define COBALT_BUILD_VERSION_NUMBER "{version_number}" +#define COBALT_BUILD_VERSION_NUMBER "{build_id}" #endif // COBALT_BUILD_VERSION_NUMBER @@ -28,18 +29,38 @@ """ -def BuildId(output_path, version_number): - """Write a Cobalt build_id header file version info. +def BuildId(output_path, build_id): + """Write a Cobalt build_id header file. Args: output_path: Location of the build id header to write. - version_number: Build version number, generated when gn gen is run. + build_id: Build version number, generated when gn gen is run. Returns: 0 on success. """ with open(output_path, 'w', encoding='utf-8') as f: - f.write(BUILD_ID_HEADER_TEMPLATE.format(version_number=version_number)) + f.write(BUILD_ID_HEADER_TEMPLATE.format(build_id=build_id)) + + +def BuildInfo(output_path, info): + """Write a Cobalt build_info json file.""" + build_id, build_rev, git_rev, build_time, author, encoded_commit = info + + info_json = {} + info_json['build_id'] = build_id + info_json['build_rev'] = build_rev + info_json['git_rev'] = git_rev + info_json['build_time'] = build_time + info_json['author'] = author + + commit = ''.join([chr(int(c)) for c in encoded_commit.split(',')]) + + info_json['commit'] = commit + + with open(output_path, 'w', encoding='utf-8') as f: + f.write(json.dumps(info_json, indent=4)) if __name__ == '__main__': - BuildId(sys.argv[1], sys.argv[2]) + BuildId(sys.argv[1], sys.argv[3]) + BuildInfo(sys.argv[2], sys.argv[3:]) diff --git a/cobalt/build/get_build_id.py b/cobalt/build/get_build_id.py deleted file mode 100755 index 5e5c717154d1..000000000000 --- a/cobalt/build/get_build_id.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# Copyright 2017 The Cobalt Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Prints out the Cobalt Build ID.""" - -import os -import re -import subprocess - -_FILE_DIR = os.path.dirname(__file__) -COMMIT_COUNT_BUILD_NUMBER_OFFSET = 1000000 - -# Matches numbers > 1000000. The pattern is basic so git log --grep is able to -# interpret it. -GIT_BUILD_NUMBER_PATTERN = r'[1-9]' + r'[0-9]' * 6 + r'[0-9]*' -BUILD_NUMBER_TAG_PATTERN = r'^BUILD_NUMBER={}$' - -# git log --grep can't handle capture groups. -BUILD_NUBER_PATTERN_WITH_CAPTURE = f'({GIT_BUILD_NUMBER_PATTERN})' - - -def get_build_number_from_commits(cwd=_FILE_DIR): - full_pattern = BUILD_NUMBER_TAG_PATTERN.format(GIT_BUILD_NUMBER_PATTERN) - output = subprocess.check_output( - ['git', 'log', '--grep', full_pattern, '-1', '--pretty=%b'], - cwd=cwd).decode() - - full_pattern_with_capture = re.compile( - BUILD_NUMBER_TAG_PATTERN.format(BUILD_NUBER_PATTERN_WITH_CAPTURE), - flags=re.MULTILINE) - match = full_pattern_with_capture.search(output) - return match.group(1) if match else None - - -def get_build_number_from_commit_count(cwd=_FILE_DIR): - output = subprocess.check_output(['git', 'rev-list', '--count', 'HEAD'], - cwd=cwd) - build_number = int(output.strip().decode('utf-8')) - return build_number + COMMIT_COUNT_BUILD_NUMBER_OFFSET - - -def main(cwd=_FILE_DIR): - build_number = get_build_number_from_commits(cwd=cwd) - - if not build_number: - build_number = get_build_number_from_commit_count(cwd=cwd) - - return build_number - - -if __name__ == '__main__': - print(main()) diff --git a/cobalt/build/get_build_info.py b/cobalt/build/get_build_info.py new file mode 100755 index 000000000000..5ff0fd55b28b --- /dev/null +++ b/cobalt/build/get_build_info.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# Copyright 2023 The Cobalt Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Prints out Cobalt Git Build Info.""" + +import argparse +import datetime +import os +import re +import subprocess +import sys + +_FILE_DIR = os.path.dirname(__file__) +COMMIT_COUNT_BUILD_NUMBER_OFFSET = 1000000 + +# Matches numbers > 1000000. The pattern is basic so git log --grep is able to +# interpret it. +GIT_BUILD_NUMBER_PATTERN = r'[1-9]' + r'[0-9]' * 6 + r'[0-9]*' +GIT_HASH_PATTERN = r'[0-9a-f]' * 40 + +# git log --grep can't handle capture groups. +GIT_BUILD_NUMBER_PATTERN_WITH_CAPTURE = f'({GIT_BUILD_NUMBER_PATTERN})' +GIT_HASH_PATTERN_WITH_CAPTURE = f'({GIT_HASH_PATTERN})' + +BUILD_NUMBER_PATTERN = r'^BUILD_NUMBER={}$' +HASH_PATTERN = r'^GitOrigin-RevId: {}$' + + +def get_build_number_and_hash_from_commits(cwd): + grep_pattern = BUILD_NUMBER_PATTERN.format(GIT_BUILD_NUMBER_PATTERN) + output = subprocess.check_output( + ['git', 'log', '--grep', grep_pattern, '-1', '--pretty=%b'], + cwd=cwd).decode() + + # Gets git build number. + compiled_build_number_pattern = re.compile( + BUILD_NUMBER_PATTERN.format(GIT_BUILD_NUMBER_PATTERN_WITH_CAPTURE), + flags=re.MULTILINE) + match_build_number = compiled_build_number_pattern.search(output) + + if match_build_number is None: + return None + build_number = match_build_number.group(1) + + # Gets matching git hash. + compiled_hash_pattern = re.compile( + HASH_PATTERN.format(GIT_HASH_PATTERN_WITH_CAPTURE), flags=re.MULTILINE) + match_hash = compiled_hash_pattern.search(output) + + if match_hash is None: + git_hash = None + else: + git_hash = match_hash.group(1) + + return (build_number, git_hash) + + +def get_build_number_from_commit_count(cwd): + output = subprocess.check_output(['git', 'rev-list', '--count', 'HEAD'], + cwd=cwd) + build_number = int(output.strip().decode('utf-8')) + return build_number + COMMIT_COUNT_BUILD_NUMBER_OFFSET + + +def get_from_last_commit(placeholder, cwd): + output = subprocess.check_output( + ['git', 'log', '-1', f'--pretty=format:{placeholder}'], cwd=cwd) + return output.strip().decode('utf-8') + + +def get_hash_from_last_commit(cwd): + return get_from_last_commit(r'%H', cwd=cwd) + + +def get_author_from_last_commit(cwd): + return get_from_last_commit(r'%an', cwd=cwd) + + +def get_subject_from_last_commit(cwd): + return get_from_last_commit(r'%s', cwd=cwd) + + +def main(build_number_only=False, cwd=_FILE_DIR): + build_rev = get_hash_from_last_commit(cwd=cwd) + + build_number_hash = get_build_number_and_hash_from_commits(cwd=cwd) + if build_number_hash is not None: + build_number = build_number_hash[0] + git_rev = build_number_hash[1] + else: + build_number = get_build_number_from_commit_count(cwd=cwd) + git_rev = build_rev + + build_time = datetime.datetime.now().ctime() + author = get_author_from_last_commit(cwd=cwd) + commit = get_subject_from_last_commit(cwd=cwd) + + encoded_commit = ','.join([str(ord(c)) for c in commit]) + + if build_number_only: + return build_number + return (f'{build_number}|{build_rev}|{git_rev}|{build_time}|{author}|' + f'{encoded_commit}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--build_number', action='store_true') + args = parser.parse_args(sys.argv[1:]) + print(main(args.build_number)) diff --git a/cobalt/build/get_build_id_test.py b/cobalt/build/get_build_info_test.py similarity index 75% rename from cobalt/build/get_build_id_test.py rename to cobalt/build/get_build_info_test.py index f9ea8c3c3033..e20db444a522 100644 --- a/cobalt/build/get_build_id_test.py +++ b/cobalt/build/get_build_info_test.py @@ -14,16 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Tests the get_build_id module.""" +"""Tests the get_build_info module.""" import os import subprocess import tempfile import unittest -from cobalt.build import get_build_id +from cobalt.build import get_build_info -_TEST_BUILD_NUMBER = 1234 + get_build_id.COMMIT_COUNT_BUILD_NUMBER_OFFSET +_TEST_BUILD_NUMBER = 1234 + get_build_info.COMMIT_COUNT_BUILD_NUMBER_OFFSET # TODO(b/282040638): fix and re-enabled this @@ -58,53 +58,53 @@ def testSanity(self): def testGetBuildNumberFromCommitsSunnyDay(self): self.make_commit_with_build_number() - build_number = get_build_id.get_build_number_from_commits( - cwd=self.test_dir.name) + build_number = get_build_info.get_build_number_and_hash_from_commits( + cwd=self.test_dir.name)[0] self.assertEqual(int(build_number), _TEST_BUILD_NUMBER) def testGetBuildNumberFromCommitsSunnyDayGetMostRecent(self): num_commits = 5 for i in range(num_commits): self.make_commit_with_build_number( - get_build_id.COMMIT_COUNT_BUILD_NUMBER_OFFSET + i) - build_number = get_build_id.get_build_number_from_commits( - cwd=self.test_dir.name) + get_build_info.COMMIT_COUNT_BUILD_NUMBER_OFFSET + i) + build_number = get_build_info.get_build_number_and_hash_from_commits( + cwd=self.test_dir.name)[0] self.assertEqual( int(build_number), - num_commits + get_build_id.COMMIT_COUNT_BUILD_NUMBER_OFFSET - 1) + num_commits + get_build_info.COMMIT_COUNT_BUILD_NUMBER_OFFSET - 1) def testGetBuildNumberFromCommitsRainyDayInvalidBuildNumber(self): self.make_commit() self.make_commit(f'BUILD_NUMBER={_TEST_BUILD_NUMBER}') - build_number = get_build_id.get_build_number_from_commits( + build_number_hash = get_build_info.get_build_number_and_hash_from_commits( cwd=self.test_dir.name) - self.assertIsNone(build_number) + self.assertIsNone(build_number_hash) def testGetBuildNumberFromCommitCountSunnyDay(self): num_commits = 5 for _ in range(num_commits): self.make_commit() - build_number = get_build_id.get_build_number_from_commit_count( + build_number = get_build_info.get_build_number_from_commit_count( cwd=self.test_dir.name) self.assertEqual( build_number, - num_commits + get_build_id.COMMIT_COUNT_BUILD_NUMBER_OFFSET) + num_commits + get_build_info.COMMIT_COUNT_BUILD_NUMBER_OFFSET) def testCommitsOutrankCommitCount(self): self.make_commit() self.make_commit_with_build_number() self.make_commit() - build_number = get_build_id.main(cwd=self.test_dir.name) + build_number = get_build_info.main(True, cwd=self.test_dir.name) self.assertEqual(int(build_number), _TEST_BUILD_NUMBER) def testFallbackToCommitCount(self): num_commits = 5 for _ in range(num_commits): self.make_commit() - build_number = get_build_id.main(cwd=self.test_dir.name) + build_number = get_build_info.main(True, cwd=self.test_dir.name) self.assertEqual( build_number, - num_commits + get_build_id.COMMIT_COUNT_BUILD_NUMBER_OFFSET) + num_commits + get_build_info.COMMIT_COUNT_BUILD_NUMBER_OFFSET) if __name__ == '__main__': diff --git a/cobalt/h5vcc/BUILD.gn b/cobalt/h5vcc/BUILD.gn index f081ce4e09ca..a5f89f036140 100644 --- a/cobalt/h5vcc/BUILD.gn +++ b/cobalt/h5vcc/BUILD.gn @@ -76,7 +76,7 @@ static_library("h5vcc") { "//cobalt/base", "//cobalt/browser:browser_switches", "//cobalt/browser/metrics", - "//cobalt/build:cobalt_build_id", + "//cobalt/build:cobalt_build_info", "//cobalt/cache", "//cobalt/configuration", "//cobalt/dom", diff --git a/cobalt/network/BUILD.gn b/cobalt/network/BUILD.gn index adf09751503b..ef41ca2e0739 100644 --- a/cobalt/network/BUILD.gn +++ b/cobalt/network/BUILD.gn @@ -48,7 +48,7 @@ static_library("network") { deps = [ ":copy_ssl_certificates", "//cobalt/base", - "//cobalt/build:cobalt_build_id", + "//cobalt/build:cobalt_build_info", "//cobalt/configuration", "//cobalt/network_bridge", "//cobalt/persistent_storage:persistent_settings",