From 2525545a41fd13f1e844a814b4883cb03a75f24d Mon Sep 17 00:00:00 2001 From: Jiajie Zhong Date: Wed, 12 Oct 2022 09:00:59 +0800 Subject: [PATCH] [dev] Easier release: cherry-pick, changelog, contributor (#11478) Add script for easier release including cherry pick, generating changelog and contributor list * Auto cherry-pick: `python release.py cherry-pick` * Generate changelog: `python release.py changelog` * Generate contributor: `python release.py contributor` close: #11289 related: #12222 --- tools/release/README.md | 38 +++++++ tools/release/github/__init__.py | 16 +++ tools/release/github/changelog.py | 151 +++++++++++++++++++++++++++ tools/release/github/git.py | 70 +++++++++++++ tools/release/github/pull_request.py | 64 ++++++++++++ tools/release/github/resp_get.py | 67 ++++++++++++ tools/release/github/user.py | 43 ++++++++ tools/release/release.py | 106 +++++++++++++++++++ tools/release/requirements.txt | 19 ++++ 9 files changed, 574 insertions(+) create mode 100644 tools/release/README.md create mode 100644 tools/release/github/__init__.py create mode 100644 tools/release/github/changelog.py create mode 100644 tools/release/github/git.py create mode 100644 tools/release/github/pull_request.py create mode 100644 tools/release/github/resp_get.py create mode 100644 tools/release/github/user.py create mode 100644 tools/release/release.py create mode 100644 tools/release/requirements.txt diff --git a/tools/release/README.md b/tools/release/README.md new file mode 100644 index 0000000000..b4596f0f88 --- /dev/null +++ b/tools/release/README.md @@ -0,0 +1,38 @@ +# Tools Release + +A tools for convenient release DolphinScheduler. + +## Prepare + +* python: python 3.6 or above +* pip: latest version of pip is better + +To install dependence, you should run command + +```shell +python -m pip install -r requirements.txt +``` + +## Usage + +### Export Environment Variable + +You can create new token in [create token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token), +it is only need all permission under `repo` + +```shell +export GH_ACCESS_TOKEN="" +export GH_REPO_MILESTONE="" +``` + +### Help + +```shell +python release.py -h +``` + +### Action + +* Auto cherry-pick: `python release.py cherry-pick`, will cause error when your default branch is not up-to-date, or cherry-pick with conflict. But if you fix you can directly re-run this command, it will continue the pick +* Generate changelog: `python release.py changelog` +* Generate contributor: `python release.py contributor` diff --git a/tools/release/github/__init__.py b/tools/release/github/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/tools/release/github/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tools/release/github/changelog.py b/tools/release/github/changelog.py new file mode 100644 index 0000000000..517d6b0b49 --- /dev/null +++ b/tools/release/github/changelog.py @@ -0,0 +1,151 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Github utils for release changelog.""" + +from typing import Dict, List + + +class Changelog: + """Generate changelog according specific pull requests list. + + Each pull requests will only once in final result. If pull requests have more than one label we need, + will classify to high priority label type, currently priority is + `feature > bug > improvement > document > chore`. pr will into feature section if it with both `feature`, + `improvement`, `document` label. + + :param prs: pull requests list. + """ + + key_number = "number" + key_labels = "labels" + key_name = "name" + + label_feature = "feature" + label_bug = "bug" + label_improvement = "improvement" + label_document = "document" + label_chore = "chore" + + changelog_prefix = "\n\n
Click to expand\n\n" + changelog_suffix = "\n\n
\n" + + def __init__(self, prs: List[Dict]): + self.prs = prs + self.features = [] + self.bugfixs = [] + self.improvements = [] + self.documents = [] + self.chores = [] + + def generate(self) -> str: + """Generate changelog.""" + self.classify() + final = [] + if self.features: + detail = f"## Feature{self.changelog_prefix}{self._convert(self.features)}{self.changelog_suffix}" + final.append(detail) + if self.improvements: + detail = ( + f"## Improvement{self.changelog_prefix}" + f"{self._convert(self.improvements)}{self.changelog_suffix}" + ) + final.append(detail) + if self.bugfixs: + detail = f"## Bugfix{self.changelog_prefix}{self._convert(self.bugfixs)}{self.changelog_suffix}" + final.append(detail) + if self.documents: + detail = ( + f"## Document{self.changelog_prefix}" + f"{self._convert(self.documents)}{self.changelog_suffix}" + ) + final.append(detail) + if self.chores: + detail = f"## Chore{self.changelog_prefix}{self._convert(self.chores)}{self.changelog_suffix}" + final.append(detail) + return "\n".join(final) + + @staticmethod + def _convert(prs: List[Dict]) -> str: + """Convert pull requests into changelog item text.""" + return "\n".join( + [f"- {pr['title']} (#{pr['number']}) @{pr['user']['login']}" for pr in prs] + ) + + def classify(self) -> None: + """Classify pull requests different kinds of section in changelog. + + Each pull requests only belongs to one single classification. + """ + for pr in self.prs: + if self.key_labels not in pr: + raise KeyError("PR %s do not have labels", pr[self.key_number]) + if self._is_feature(pr): + self.features.append(pr) + elif self._is_bugfix(pr): + self.bugfixs.append(pr) + elif self._is_improvement(pr): + self.improvements.append(pr) + elif self._is_document(pr): + self.documents.append(pr) + elif self._is_chore(pr): + self.chores.append(pr) + else: + raise KeyError( + "There must at least one of labels `feature|bug|improvement|document|chore`" + "but it do not, pr: %s", + pr["html_url"], + ) + + def _is_feature(self, pr: Dict) -> bool: + """Belong to feature pull requests.""" + return any( + [ + label[self.key_name] == self.label_feature + for label in pr[self.key_labels] + ] + ) + + def _is_bugfix(self, pr: Dict) -> bool: + """Belong to bugfix pull requests.""" + return any( + [label[self.key_name] == self.label_bug for label in pr[self.key_labels]] + ) + + def _is_improvement(self, pr: Dict) -> bool: + """Belong to improvement pull requests.""" + return any( + [ + label[self.key_name] == self.label_improvement + for label in pr[self.key_labels] + ] + ) + + def _is_document(self, pr: Dict) -> bool: + """Belong to document pull requests.""" + return any( + [ + label[self.key_name] == self.label_document + for label in pr[self.key_labels] + ] + ) + + def _is_chore(self, pr: Dict) -> bool: + """Belong to chore pull requests.""" + return any( + [label[self.key_name] == self.label_chore for label in pr[self.key_labels]] + ) diff --git a/tools/release/github/git.py b/tools/release/github/git.py new file mode 100644 index 0000000000..66ef6596e1 --- /dev/null +++ b/tools/release/github/git.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Github utils for git operations.""" + +from pathlib import Path +from typing import Dict, Optional + +from git import Repo + +git_dir_path: Path = Path(__file__).parent.parent.parent.parent.joinpath(".git") + + +class Git: + """Operator to handle git object. + + :param path: git repository path + :param branch: branch you want to query + """ + + def __init__( + self, path: Optional[str] = git_dir_path, branch: Optional[str] = None + ): + self.path = path + self.branch = branch + + @property + def repo(self) -> Repo: + """Get git repo object.""" + return Repo(self.path) + + def has_commit_current(self, sha: str) -> bool: + """Whether SHA in current branch.""" + branches = self.repo.git.branch("--contains", sha) + return f"* {self.repo.active_branch.name}" in branches + + def has_commit_global(self, sha: str) -> bool: + """Whether SHA in all branches.""" + try: + self.repo.commit(sha) + return True + except ValueError: + return False + + def cherry_pick_pr(self, pr: Dict) -> None: + """Run command `git cherry-pick -x `.""" + sha = pr["merge_commit_sha"] + if not self.has_commit_global(sha): + raise RuntimeError( + "Cherry-pick SHA %s error because SHA not exists," + "please make sure you local default branch is up-to-date", + sha, + ) + if self.has_commit_current(sha): + print("SHA %s already in current active branch, skip it.", sha) + self.repo.git.cherry_pick("-x", sha) diff --git a/tools/release/github/pull_request.py b/tools/release/github/pull_request.py new file mode 100644 index 0000000000..74dcd80b32 --- /dev/null +++ b/tools/release/github/pull_request.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Optional, List, Dict +from github.resp_get import RespGet + + +class PullRequest: + """Pull request to filter the by specific condition. + + :param token: token to request GitHub API entrypoint. + :param repo: GitHub repository identify, use `user/repo` or `org/repo`. + """ + url_search = "https://api.github.com/search/issues" + url_pr = "https://api.github.com/repos/{}/pulls/{}" + + def __init__(self, token: str, repo: Optional[str] = "apache/dolphinscheduler"): + self.token = token + self.repo = repo + self.headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"token {token}", + } + + def get_merged_detail(self, number: str) -> Dict: + """Get all merged pull requests detail by pr number. + + :param number: pull requests number you want to get detail. + """ + return RespGet(url=self.url_pr.format(self.repo, number), headers=self.headers).get_single() + + def get_merged_detail_by_milestone(self, milestone: str) -> List[Dict]: + """Get all merged requests pull request detail by specific milestone. + + :param milestone: query by specific milestone. + """ + detail = [] + numbers = {pr.get("number") for pr in self.search_merged_by_milestone(milestone)} + for number in numbers: + pr_dict = RespGet(url=self.url_pr.format(self.repo, number), headers=self.headers).get_single() + detail.append(pr_dict) + return detail + + def search_merged_by_milestone(self, milestone: str) -> List[Dict]: + """Get all merged requests pull request by specific milestone. + + :param milestone: query by specific milestone. + """ + params = {"q": f"repo:{self.repo} is:pr is:merged milestone:{milestone}"} + return RespGet(url=self.url_search, headers=self.headers, param=params).get_total() diff --git a/tools/release/github/resp_get.py b/tools/release/github/resp_get.py new file mode 100644 index 0000000000..a249e4c758 --- /dev/null +++ b/tools/release/github/resp_get.py @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Github utils get HTTP response.""" + +import copy +import json +from typing import Dict, List, Optional + +import requests + + +class RespGet: + """Get response from GitHub restful API. + + :param url: URL to requests GET method. + :param headers: headers for HTTP requests. + :param param: param for HTTP requests. + """ + + def __init__(self, url: str, headers: dict, param: Optional[dict] = None): + self.url = url + self.headers = headers + self.param = param + + @staticmethod + def get(url: str, headers: dict, params: Optional[dict] = None) -> Dict: + """Get single response dict from HTTP requests by given condition.""" + resp = requests.get(url=url, headers=headers, params=params) + if not resp.ok: + raise ValueError("Requests error with", resp.reason) + return json.loads(resp.content) + + def get_single(self) -> Dict: + """Get single response dict from HTTP requests by given condition.""" + return self.get(url=self.url, headers=self.headers, params=self.param) + + def get_total(self) -> List[Dict]: + """Get all response dict from HTTP requests by given condition. + + Will change page number until no data return. + """ + total = [] + curr_param = copy.deepcopy(self.param) + while True: + curr_param["page"] = curr_param.setdefault("page", 0) + 1 + content_dict = self.get( + url=self.url, headers=self.headers, params=curr_param + ) + data = content_dict.get("items") + if not data: + return total + total.extend(data) diff --git a/tools/release/github/user.py b/tools/release/github/user.py new file mode 100644 index 0000000000..152169d2ef --- /dev/null +++ b/tools/release/github/user.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Github utils for user.""" + +from typing import Dict, List, Set + + +class User: + """Get users according specific pull requests list. + + :param prs: pull requests list. + """ + + def __init__(self, prs: List[Dict]): + self.prs = prs + + def contribution_num(self) -> Dict: + """Get unique contributor with name and commit number.""" + res = dict() + for pr in self.prs: + user_id = pr["user"]["login"] + res[user_id] = res.setdefault(user_id, 0) + 1 + return res + + def contributors(self) -> Set[str]: + """Get unique contributor with name.""" + cn = self.contribution_num() + return {contributor for contributor in cn} diff --git a/tools/release/release.py b/tools/release/release.py new file mode 100644 index 0000000000..122e57caaf --- /dev/null +++ b/tools/release/release.py @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Main function for releasing.""" + +import argparse +import os + +from github.changelog import Changelog +from github.git import Git +from github.pull_request import PullRequest +from github.user import User + + +def get_changelog(access_token: str, milestone: str) -> str: + """Get changelog in specific milestone from GitHub Restful API.""" + pr = PullRequest(token=access_token) + pr_merged = pr.search_merged_by_milestone(milestone) + # Sort according to merged time ascending + pr_merged_sort = sorted(pr_merged, key=lambda p: p["closed_at"]) + + changelog = Changelog(pr_merged_sort) + changelog_text = changelog.generate() + return changelog_text + + +def get_contributor(access_token: str, milestone: str) -> str: + """Get contributor in specific milestone from GitHub Restful API.""" + pr = PullRequest(token=access_token) + pr_merged = pr.search_merged_by_milestone(milestone) + + users = User(prs=pr_merged) + contributor = users.contributors() + # Sort according alphabetical + return ", ".join(sorted(contributor)) + + +def auto_cherry_pick(access_token: str, milestone: str) -> None: + """Do git cherry-pick in specific milestone, require update dev branch.""" + pr = PullRequest(token=access_token) + pr_merged = pr.search_merged_by_milestone(milestone) + # Sort according to merged time ascending + pr_merged_sort = sorted(pr_merged, key=lambda p: p["closed_at"]) + + for p in pr_merged_sort: + pr_detail = pr.get_merged_detail(p["number"]) + print(f"git cherry-pick -x {pr_detail['merge_commit_sha']}") + Git().cherry_pick_pr(pr_detail) + + +def build_argparse() -> argparse.ArgumentParser: + """Build argparse.ArgumentParser with specific configuration.""" + parser = argparse.ArgumentParser(prog="release") + + subparsers = parser.add_subparsers( + title="subcommands", + dest="subcommand", + help="Choose one of the subcommand you want to run.", + ) + parser_check = subparsers.add_parser( + "changelog", help="Generate changelog from specific milestone." + ) + parser_check.set_defaults(func=get_changelog) + + parser_prune = subparsers.add_parser( + "contributor", help="List all contributors from specific milestone." + ) + parser_prune.set_defaults(func=get_contributor) + + parser_prune = subparsers.add_parser( + "cherry-pick", + help="Auto cherry pick pr to current branch from specific milestone.", + ) + parser_prune.set_defaults(func=auto_cherry_pick) + + return parser + + +if __name__ == "__main__": + arg_parser = build_argparse() + # args = arg_parser.parse_args(["cherry-pick"]) + args = arg_parser.parse_args() + + ENV_ACCESS_TOKEN = os.environ.get("GH_ACCESS_TOKEN", None) + ENV_MILESTONE = os.environ.get("GH_REPO_MILESTONE", None) + + if ENV_ACCESS_TOKEN is None or ENV_MILESTONE is None: + raise RuntimeError( + "Environment variable `GH_ACCESS_TOKEN` and `GH_REPO_MILESTONE` must provider" + ) + + print(args.func(ENV_ACCESS_TOKEN, ENV_MILESTONE)) diff --git a/tools/release/requirements.txt b/tools/release/requirements.txt new file mode 100644 index 0000000000..2cb12bfa09 --- /dev/null +++ b/tools/release/requirements.txt @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +requests~=2.28 +GitPython~=3.1