commit 5776799296e0eea6156ac10f7cc161658754db03 Author: Marco Miller Date: Sun Oct 11 20:23:05 2020 -0400 Tools: Start a release notes initializer script in python List excluded change subject keywords at the beginning of the python script, for frequent update convenience. That list could be moved to a file if eventually more convenient. Excluded subjects will likely be listed again but in their own section, for a human editor to ponder. Base the script on regex patterns matching. Summarize usage through a README and help for the options. Provide main usage examples in README. The '-c' (check) option takes much more runtime and is in beta. That option tries to exclude commits contained in previous release tags. It is yet to be proven, as it had some visible effect only once, so far. Generated notes list every change in revision range, in git log order. Deeming a subject as noteworthy remains a manual, post editing step. Issues follow their referring change subject rather than the reverse. The latter could be added eventually as another generated view or so. Redundant information is prevented as much as currently implemented. The script currently only lists change subjects and the related issues. Manual editing is expected then to add more, edit or remove information from the generated release_noter.md (markdown file). The latter file is not to be used as is for Gerrit but rather manually imported in parts. Black is used for automatic code formatting. Flake8 was tried but did the opposite of black, locally. It is therefore not used yet. Adding logic to group specific subjects in specific component sections is yet to be done. The script already does it for the submodule (core) plugins. The other large section currently is for all core changes and their issues. Splitting into more component sections, to do soon, will help handling the large generated release notes. JGit may be split too. Another upcoming backlogged item is to add links to the listed change commits nearby issues. Goal being, to be able to double-check the notes manually or at will. This script is more complex than [1] but was originally inspired by it. This assumes the current way of formatting Gerrit git commit messages, without expecting more headers or footers. Near-term plan is to consider refactoring release_noter's main parsing loop, based on [2]'s approach. [1] https://git.eclipse.org/r/c/tracecompass.incubator/org.eclipse.tracecompass.incubator/+/144581 [2] https://www.pydanny.com/why-doesnt-python-have-switch-case.html Bug: Issue 13483 Bug: Issue 13484 Feature: Issue 11123 Change-Id: Ifadc5e742f8a2a7bc0b1c2bc7e021c330c010ac2 diff --git a/tools/release_noter/.gitignore b/tools/release_noter/.gitignore new file mode 100644 index 0000000..6bb75bd --- /dev/null +++ b/tools/release_noter/.gitignore @@ -0,0 +1,2 @@ +/.idea/ +/release_noter.md diff --git a/tools/release_noter/Pipfile b/tools/release_noter/Pipfile new file mode 100644 index 0000000..32907e1 --- /dev/null +++ b/tools/release_noter/Pipfile @@ -0,0 +1,12 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +black = { version = "==20.8b1", markers = "python_version >= '3.8'" } + +[packages] + +[requires] +python_version = "3.8" diff --git a/tools/release_noter/Pipfile.lock b/tools/release_noter/Pipfile.lock new file mode 100644 index 0000000..3eaa8ed --- /dev/null +++ b/tools/release_noter/Pipfile.lock @@ -0,0 +1,131 @@ +{ + "_meta": { + "hash": { + "sha256": "74d854d459fb0a00626d273f07059839465b87197e7f960a9ecf17d22eddaac3" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": {}, + "develop": { + "appdirs": { + "hashes": [ + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" + ], + "version": "==1.4.4" + }, + "black": { + "hashes": [ + "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==20.8b1" + }, + "click": { + "hashes": [ + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==7.1.2" + }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "version": "==0.4.3" + }, + "pathspec": { + "hashes": [ + "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0", + "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061" + ], + "version": "==0.8.0" + }, + "regex": { + "hashes": [ + "sha256:1a16afbfadaadc1397353f9b32e19a65dc1d1804c80ad73a14f435348ca017ad", + "sha256:2308491b3e6c530a3bb38a8a4bb1dc5fd32cbf1e11ca623f2172ba17a81acef1", + "sha256:39a5ef30bca911f5a8a3d4476f5713ed4d66e313d9fb6755b32bec8a2e519635", + "sha256:3d5a8d007116021cf65355ada47bf405656c4b3b9a988493d26688275fde1f1c", + "sha256:4302153abb96859beb2c778cc4662607a34175065fc2f33a21f49eb3fbd1ccd3", + "sha256:463e770c48da76a8da82b8d4a48a541f314e0df91cbb6d873a341dbe578efafd", + "sha256:46ab6070b0d2cb85700b8863b3f5504c7f75d8af44289e9562195fe02a8dd72d", + "sha256:4f5c0fe46fb79a7adf766b365cae56cafbf352c27358fda811e4a1dc8216d0db", + "sha256:60c4f64d9a326fe48e8738c3dbc068e1edc41ff7895a9e3723840deec4bc1c28", + "sha256:671c51d352cfb146e48baee82b1ee8d6ffe357c292f5e13300cdc5c00867ebfc", + "sha256:6cf527ec2f3565248408b61dd36e380d799c2a1047eab04e13a2b0c15dd9c767", + "sha256:7c4fc5a8ec91a2254bb459db27dbd9e16bba1dabff638f425d736888d34aaefa", + "sha256:850339226aa4fec04916386577674bb9d69abe0048f5d1a99f91b0004bfdcc01", + "sha256:8ba3efdd60bfee1aa784dbcea175eb442d059b576934c9d099e381e5a9f48930", + "sha256:8c8c42aa5d3ac9a49829c4b28a81bebfa0378996f9e0ca5b5ab8a36870c3e5ee", + "sha256:8e7ef296b84d44425760fe813cabd7afbb48c8dd62023018b338bbd9d7d6f2f0", + "sha256:a2a31ee8a354fa3036d12804730e1e20d58bc4e250365ead34b9c30bbe9908c3", + "sha256:a63907332531a499b8cdfd18953febb5a4c525e9e7ca4ac147423b917244b260", + "sha256:a8240df4957a5b0e641998a5d78b3c4ea762c845d8cb8997bf820626826fde9a", + "sha256:b8806649983a1c78874ec7e04393ef076805740f6319e87a56f91f1767960212", + "sha256:c077c9d04a040dba001cf62b3aff08fd85be86bccf2c51a770c77377662a2d55", + "sha256:c529ba90c1775697a65b46c83d47a2d3de70f24d96da5d41d05a761c73b063af", + "sha256:d537e270b3e6bfaea4f49eaf267984bfb3628c86670e9ad2a257358d3b8f0955", + "sha256:d629d750ebe75a88184db98f759633b0a7772c2e6f4da529f0027b4a402c0e2f", + "sha256:d9d53518eeed12190744d366ec4a3f39b99d7daa705abca95f87dd8b442df4ad", + "sha256:e490f08897cb44e54bddf5c6e27deca9b58c4076849f32aaa7a0b9f1730f2c20", + "sha256:f579caecbbca291b0fcc7d473664c8c08635da2f9b1567c22ea32311c86ef68c" + ], + "version": "==2020.10.11" + }, + "toml": { + "hashes": [ + "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", + "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88" + ], + "version": "==0.10.1" + }, + "typed-ast": { + "hashes": [ + "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355", + "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919", + "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa", + "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652", + "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75", + "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01", + "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d", + "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1", + "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907", + "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c", + "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3", + "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b", + "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614", + "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb", + "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b", + "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41", + "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6", + "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34", + "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe", + "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", + "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" + ], + "version": "==1.4.1" + }, + "typing-extensions": { + "hashes": [ + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + ], + "version": "==3.7.4.3" + } + } +} diff --git a/tools/release_noter/README.md b/tools/release_noter/README.md new file mode 100644 index 0000000..56b1253 --- /dev/null +++ b/tools/release_noter/README.md @@ -0,0 +1,27 @@ +# Release Noter + +## Setup + +```bash +pipenv install --dev --deploy +``` + +## Usage + +```bash +pipenv run python release_noter.py -h +``` + +## Examples + +```bash +pipenv run python release_noter.py v3.2.3..HEAD +pipenv run python release_noter.py v3.2.3..v3.3.0-rc0 +pipenv run python release_noter.py v3.2.3..v3.3.0-rc0 -c +``` + +## Coding + +```bash +pipenv run black release_noter.py +``` diff --git a/tools/release_noter/release_noter.py b/tools/release_noter/release_noter.py new file mode 100644 index 0000000..bc39d36 --- /dev/null +++ b/tools/release_noter/release_noter.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python + +import argparse +import re +import subprocess + +from enum import Enum + +EXCLUDED_SUBJECTS = { + "AutoValue", + "avadoc", + "avaDoc", + "ava-doc", + "baz", # bazel, bazlet(s) + "Baz", + "class", + "efactor", + "format", + "Format", + "getter", + "gr-", + "immutab", + "IT", + "js", + "lint", + "method", + "module", + "naming", + "nits", + "nongoogle", + "prone", # error prone &co. + "register", + "Register", + "remove", + "Remove", + "rename", + "Rename", + "Revert", + "serializ", + "setter", + "spell", + "Spell", + "test", # testing, tests; unit or else + "Test", + "thread", + "tsetse", + "typescript", + "version", +} + +COMMIT_SHA1_PATTERN = r"^commit ([a-z0-9]+)$" +DATE_HEADER_PATTERN = r"Date: .+" +SUBJECT_SUBMODULES_PATTERN = r"^Update git submodules$" +UPDATE_SUBMODULE_PATTERN = r"\* Update ([a-z/\-]+) from branch '.+'" +SUBMODULE_SUBJECT_PATTERN = r"^- (.+)" +SUBMODULE_MERGE_PATTERN = r".+Merge .+" +ISSUE_ID_PATTERN = r"[a-zA-Z]+: [Ii]ssue ([0-9]+)" +CHANGE_ID_PATTERN = r"^Change-Id: [I0-9a-z]+$" +PLUGIN_PATTERN = r"plugins/([a-z\-]+)" +RELEASE_OPTION_PATTERN = r".+\.\.(v.+)" +RELEASE_TAG_PATTERN = r"v[0-9]+\.[0-9]+\.[0-9]+$" + +ISSUE_URL = "https://bugs.chromium.org/p/gerrit/issues/detail?id=" +CHECK_DISCLAIMER = "experimental and much slower" +GIT_COMMAND = "git" +UTF8 = "UTF-8" + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Generate an initial release notes markdown file.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "-c", + "--check", + dest="check", + required=False, + default=False, + action="store_true", + help=f"check commits for previous releases; {CHECK_DISCLAIMER}", + ) + parser.add_argument("range", help="git log revision range") + return parser.parse_args() + + +def check_args(options): + if not options.check: + return None + release_option = re.search(RELEASE_OPTION_PATTERN, options.range) + if release_option is None: + print("Check option ignored; range doesn't end with release tag.") + return None + print(f"Check option used; {CHECK_DISCLAIMER}.") + return release_option.group(1) + + +def newly_released(commit_sha1, release): + if release is None: + return True + git_tag = [ + GIT_COMMAND, + "tag", + "--contains", + commit_sha1, + ] + process = subprocess.Popen(git_tag, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + verdict = True + for line in iter(process.stdout.readline, ""): + if process.poll() is not None: + break + line = line.strip().decode(UTF8) + if not re.match(rf"{re.escape(release)}$", line): + # Wrongfully pushed or malformed tags ignored. + # Preceding release-candidate (-rcN) tags treated as newly released. + verdict = not re.match(RELEASE_TAG_PATTERN, line) + return verdict + + +def open_git_log(options): + git_log = [ + GIT_COMMAND, + "log", + "--no-merges", + options.range, + ] + return subprocess.Popen(git_log, stdout=subprocess.PIPE) + + +class Change: + subject = None + issues = set() + + +class Task(Enum): + start_commit = 1 + finish_headers = 2 + capture_subject = 3 + capture_submodule = 4 + capture_submodule_subject = 5 + finish_submodule_change = 6 + finish_commit = 7 + + +class Commit: + sha1 = None + subject = None + submodule = None + issues = set() + + def reset(self, signature, task): + if signature is not None: + self.sha1 = signature.group(1) + self.subject = None + self.submodule = None + self.issues = set() + return Task.finish_headers + return task + + +def parse_log(process, release): + commit = Commit() + commits = [] + submodules = dict() + submodule_change = None + task = Task.start_commit + for line in iter(process.stdout.readline, ""): + if process.poll() is not None: + break + line = line.strip().decode(UTF8) + if not line: + continue + if task == Task.start_commit: + task = commit.reset(re.search(COMMIT_SHA1_PATTERN, line), task) + elif task == Task.finish_headers: + if re.match(DATE_HEADER_PATTERN, line): + task = Task.capture_subject + elif task == Task.capture_subject: + if re.match(SUBJECT_SUBMODULES_PATTERN, line): + task = Task.capture_submodule + else: + commit.subject = line + task = Task.finish_commit + elif task == Task.capture_submodule: + commit.submodule = re.search(UPDATE_SUBMODULE_PATTERN, line).group(1) + if commit.submodule not in submodules: + submodules[commit.submodule] = [] + task = Task.capture_submodule_subject + elif task == Task.capture_submodule_subject: + submodule_subject = re.search(SUBMODULE_SUBJECT_PATTERN, line) + if submodule_subject is not None: + if not re.match(SUBMODULE_MERGE_PATTERN, line): + submodule_change = change(submodule_subject, submodules, commit) + task = Task.finish_submodule_change + else: + task = update_task(line, commit, task) + elif task == Task.finish_submodule_change: + submodule_issue = re.search(ISSUE_ID_PATTERN, line) + if submodule_issue is not None: + if submodule_change is not None: + issue_id = submodule_issue.group(1) + submodule_change.issues.add(issue_id) + else: + task = update_task(line, commit, task) + elif task == Task.finish_commit: + commit_issue = re.search(ISSUE_ID_PATTERN, line) + if commit_issue is not None: + commit.issues.add(commit_issue.group(1)) + else: + commit_end = re.match(CHANGE_ID_PATTERN, line) + if commit_end is not None: + commit = finish(commit, commits, release) + task = Task.start_commit + else: + raise RuntimeError("FIXME") + return commits, submodules + + +def change(submodule_subject, submodules, commit): + submodule_change = Change() + submodule_change.subject = submodule_subject.group(1) + for exclusion in EXCLUDED_SUBJECTS: + if exclusion in submodule_change.subject: + return None + for noted_change in submodules[commit.submodule]: + if noted_change.subject == submodule_change.subject: + return noted_change + submodule_change.issues = set() + submodules[commit.submodule].append(submodule_change) + return submodule_change + + +def update_task(line, commit, task): + update_end = re.search(COMMIT_SHA1_PATTERN, line) + if update_end is not None: + task = commit.reset(update_end, task) + return task + + +def finish(commit, commits, release): + if len(commit.issues) == 0: + for exclusion in EXCLUDED_SUBJECTS: + if exclusion in commit.subject: + return Commit() + for noted_commit in commits: + if noted_commit.subject == commit.subject: + return Commit() + if newly_released(commit.sha1, release): + commits.append(commit) + else: + print(f"Previously released: commit {commit.sha1}") + return Commit() + + +def print_commits(commits, md): + md.write("\n## Core Changes\n") + for commit in commits: + md.write(f"\n* {commit.subject}\n") + for issue in sorted(commit.issues): + md.write(f" [Issue {issue}]({ISSUE_URL}{issue})\n") + + +def print_submodules(submodules, md): + md.write("\n## Plugin Changes\n") + for submodule in sorted(submodules): + plugin = re.search(PLUGIN_PATTERN, submodule) + md.write(f"\n### {plugin.group(1)}\n") + for submodule_change in submodules[submodule]: + md.write(f"\n* {submodule_change.subject}\n") + for issue in sorted(submodule_change.issues): + md.write(f" [Issue {issue}]({ISSUE_URL}{issue})\n") + + +def print_notes(commits, submodules): + md = open("release_noter.md", "w") + md.write("# Release Notes\n") + print_submodules(submodules, md) + print_commits(commits, md) + md.close() + + +if __name__ == "__main__": + script_options = parse_args() + release_tag = check_args(script_options) + change_log = open_git_log(script_options) + core_changes, submodule_changes = parse_log(change_log, release_tag) + print_notes(core_changes, submodule_changes)