X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=scripts%2Fmove_mrs_to_monorepo.py;h=cdd343feaee115745d62b852ee9a5491a406d954;hb=304352ac178c6101db70288078d7f9fe981f908f;hp=8a71c88e7a85faa34517817111613c6dff1e7b99;hpb=085a4a9ec4ab9034cfe36dc62c89a834e9dca7de;p=platform%2Fupstream%2Fgstreamer.git diff --git a/scripts/move_mrs_to_monorepo.py b/scripts/move_mrs_to_monorepo.py index 8a71c88..cdd343f 100755 --- a/scripts/move_mrs_to_monorepo.py +++ b/scripts/move_mrs_to_monorepo.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -from pathlib import Path from urllib.parse import urlparse from contextlib import contextmanager import os @@ -10,7 +9,15 @@ try: import gitlab except ModuleNotFoundError: print("========================================================================", file=sys.stderr) - print("ERROR: Install python-gitlab with `python3 -m pip install python-gitlab`", file=sys.stderr) + print("ERROR: Install python-gitlab with `python3 -m pip install python-gitlab python-dateutil`", file=sys.stderr) + print("========================================================================", file=sys.stderr) + sys.exit(1) + +try: + from dateutil import parser as dateparse +except ModuleNotFoundError: + print("========================================================================", file=sys.stderr) + print("ERROR: Install dateutil with `python3 -m pip install dateutil`", file=sys.stderr) print("========================================================================", file=sys.stderr) sys.exit(1) import argparse @@ -43,7 +50,45 @@ PARSER.add_argument("--skip-branch", action="store", nargs="*", help="Ignore MRs for branches which match those names.", dest="skipped_branches") PARSER.add_argument("--skip-on-failure", action="store_true", default=False) PARSER.add_argument("--dry-run", "-n", action="store_true", default=False) -PARSER.add_argument("--use-branch-if-exists", action="store_true", default=False) +PARSER.add_argument("--use-branch-if-exists", + action="store_true", default=False) +PARSER.add_argument("--list-mrs-only", action="store_true", default=False) +PARSER.add_argument( + "-c", + "--config-file", + action="append", + dest='config_files', + help="Configuration file to use. Can be used multiple times.", + required=False, +) +PARSER.add_argument( + "-g", + "--gitlab", + help=( + "Which configuration section should " + "be used. If not defined, the default selection " + "will be used." + ), + required=False, +) +PARSER.add_argument( + "-m", + "--module", + help="GStreamer module to move MRs for. All if none specified. Can be used multiple times.", + dest='modules', + action="append", + required=False, +) +PARSER.add_argument( + "-mr", + "--mr-url", + default=None, + type=str, + help=( + "URL of the MR to work on." + ), + required=False, +) GST_PROJECTS = [ 'gstreamer', @@ -59,19 +104,40 @@ GST_PROJECTS = [ 'gst-omx', 'gst-editing-services', 'gst-devtools', - 'gst-integration-testsuites', 'gst-docs', 'gst-examples', 'gst-build', 'gst-ci', ] +GST_PROJECTS_ID = { + 'gstreamer': 1357, + 'gst-rtsp-server': 1362, + 'gstreamer-vaapi': 1359, + 'gstreamer-sharp': 1358, + 'gst-python': 1355, + 'gst-plugins-ugly': 1354, + 'gst-plugins-good': 1353, + 'gst-plugins-base': 1352, + 'gst-plugins-bad': 1351, + 'gst-omx': 1350, + 'gst-libav': 1349, + 'gst-integration-testsuites': 1348, + 'gst-examples': 1347, + 'gst-editing-services': 1346, + 'gst-docs': 1345, + 'gst-devtools': 1344, + 'gst-ci': 1343, + 'gst-build': 1342, +} + # We do not want to deal with LFS os.environ["GIT_LFS_SKIP_SMUDGE"] = "1" log_depth = [] # type: T.List[str] + @contextmanager def nested(name=''): global log_depth @@ -81,18 +147,23 @@ def nested(name=''): finally: log_depth.pop() + def bold(text: str): return f"\033[1m{text}\033[0m" + def green(text: str): return f"\033[1;32m{text}\033[0m" + def red(text: str): return f"\033[1;31m{text}\033[0m" + def yellow(text: str): return f"\033[1;33m{text}\033[0m" + def fprint(msg, nested=True): if log_depth: prepend = log_depth[-1] + ' | ' if nested else '' @@ -106,8 +177,12 @@ def fprint(msg, nested=True): class GstMRMover: def __init__(self): - self.gl = self.connect() - self.gl.auth() + self.modules = [] + self.gitlab = None + self.config_files = [] + self.gl = None + self.mr = None + self.mr_url = None self.all_projects = [] self.skipped_branches = [] self.git_rename_limit = None @@ -116,8 +191,13 @@ class GstMRMover: def connect(self): fprint("Logging into gitlab...") - gitlab_api_token = os.environ.get('GITLAB_API_TOKEN') + if self.gitlab: + gl = gitlab.Gitlab.from_config(self.gitlab, self.config_files) + fprint(f"{green(' OK')}\n", nested=False) + return gl + + gitlab_api_token = os.environ.get('GITLAB_API_TOKEN') if gitlab_api_token: gl = gitlab.Gitlab(URL, private_token=gitlab_api_token) fprint(f"{green(' OK')}\n", nested=False) @@ -125,8 +205,8 @@ class GstMRMover: session = requests.Session() sign_in_page = session.get(SIGN_IN_URL).content.decode() - for l in sign_in_page.split('\n'): - m = re.search('name="authenticity_token" value="([^"]+)"', l) + for line in sign_in_page.split('\n'): + m = re.search('name="authenticity_token" value="([^"]+)"', line) if m: break @@ -138,7 +218,6 @@ class GstMRMover: fprint(f"{red('Unable to find the authenticity token')}\n") sys.exit(1) - for data, url in [ ({'user[login]': 'login_or_email', 'user[password]': 'SECRET', @@ -159,12 +238,12 @@ class GstMRMover: return gl sys.exit(bold(f"{red('FAILED')}.\n\nPlease go to:\n\n" - ' https://gitlab.freedesktop.org/-/profile/personal_access_tokens\n\n' - f'and generate a token {bold("with read/write access to all but the registry")},' - ' then set it in the "GITLAB_API_TOKEN" environment variable:"' - f'\n\n $ GITLAB_API_TOKEN= {" ".join(sys.argv)}\n')) + ' https://gitlab.freedesktop.org/-/profile/personal_access_tokens\n\n' + f'and generate a token {bold("with read/write access to all but the registry")},' + ' then set it in the "GITLAB_API_TOKEN" environment variable:"' + f'\n\n $ GITLAB_API_TOKEN= {" ".join(sys.argv)}\n')) - def git(self, *args, can_fail=False, interaction_message=None, call=False): + def git(self, *args, can_fail=False, interaction_message=None, call=False, revert_operation=None): cwd = ROOT_DIR retry = True while retry: @@ -173,11 +252,12 @@ class GstMRMover: if not call: try: return subprocess.check_output(["git"] + list(args), cwd=cwd, - stdin=subprocess.DEVNULL, - stderr=subprocess.STDOUT).decode() - except: + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT).decode() + except subprocess.CalledProcessError: if not can_fail: - fprint(f"\n\n{bold(red('ERROR'))}: `git {' '.join(args)}` failed" + "\n", nested=False) + fprint( + f"\n\n{bold(red('ERROR'))}: `git {' '.join(args)}` failed" + "\n", nested=False) raise else: subprocess.call(["git"] + list(args), cwd=cwd) @@ -192,15 +272,15 @@ class GstMRMover: else: out = "????" fprint(f"\n```" - f"\n{out}\n" - f"Entering a shell in {cwd} to fix:\n\n" - f" {bold(interaction_message)}\n\n" - f"You should then exit with the following codes:\n\n" - f" - {bold('`exit 0`')}: once you have fixed the problem and we can keep moving the merge request\n" - f" - {bold('`exit 1`')}: {bold('retry')}: once you have let the repo in a state where the operation should be to retried\n" - f" - {bold('`exit 2`')}: to skip that merge request\n" - f" - {bold('`exit 3`')}: stop the script and abandon moving your MRs\n" - "\n```\n", nested=False) + f"\n{out}\n" + f"Entering a shell in {cwd} to fix:\n\n" + f" {bold(interaction_message)}\n\n" + f"You should then exit with the following codes:\n\n" + f" - {bold('`exit 0`')}: once you have fixed the problem and we can keep moving the merge request\n" + f" - {bold('`exit 1`')}: {bold('retry')}: once you have let the repo in a state where the operation should be to retried\n" + f" - {bold('`exit 2`')}: to skip that merge request\n" + f" - {bold('`exit 3`')}: stop the script and abandon moving your MRs\n" + "\n```\n", nested=False) try: if os.name == 'nt': shell = os.environ.get( @@ -214,10 +294,14 @@ class GstMRMover: retry = True continue elif e.returncode == 2: + if revert_operation: + self.git(*revert_operation, can_fail=True) return "SKIP" elif e.returncode == 3: + if revert_operation: + self.git(*revert_operation, can_fail=True) sys.exit(3) - except: + except Exception: # Result of subshell does not really matter pass @@ -228,7 +312,47 @@ class GstMRMover: raise e + def cleanup_args(self): + if self.mr_url: + self.modules.append(GST_PROJECTS[0]) + (namespace, module, _, _, mr) = os.path.normpath(urlparse(self.mr_url).path).split('/')[1:] + self.modules.append(module) + self.mr = int(mr) + elif not self.modules: + if self.mr: + sys.exit(f"{red(f'Merge request #{self.mr} specified without module')}\n\n" + f"{bold(' -> Use `--module` to specify which module the MR is from.')}") + + self.modules = GST_PROJECTS + else: + VALID_PROJECTS = GST_PROJECTS[1:] + for m in self.modules: + if m not in VALID_PROJECTS: + projects = '\n- '.join(VALID_PROJECTS) + sys.exit( + f"{red(f'Unknown module {m}')}\nModules are:\n- {projects}") + if self.mr and len(self.modules) > 1: + sys.exit(f"{red(f'Merge request #{self.mr} specified but several modules where specified')}\n\n" + f"{bold(' -> Use `--module` only once to specify an merge request.')}") + self.modules.append(GST_PROJECTS[0]) + def run(self): + self.cleanup_args() + self.gl = self.connect() + self.gl.auth() + + # Skip pre-commit hooks when migrating. Some users may have a + # different version of gnu indent and that can lead to cherry-pick + # failing. + os.environ["GST_DISABLE_PRE_COMMIT_HOOKS"] = "1" + + try: + prevbranch = self.git( + "rev-parse", "--abbrev-ref", "HEAD", can_fail=True).strip() + except Exception: + fprint(bold(yellow("Not on a branch?\n")), indent=False) + prevbranch = None + try: self.setup_repo() @@ -238,25 +362,50 @@ class GstMRMover: self.move_mrs(from_projects, to_project) finally: if self.git_rename_limit is not None: - self.git("config", "merge.renameLimit", str(self.git_rename_limit)) + self.git("config", "merge.renameLimit", + str(self.git_rename_limit)) + if prevbranch: + fprint(f'Back to {prevbranch}\n') + self.git("checkout", prevbranch) def fetch_projects(self): fprint("Fetching projects... ") self.all_projects = [proj for proj in self.gl.projects.list( - membership=1, all=True) if proj.name in GST_PROJECTS] - self.user_project, = [p for p in self.all_projects if p.namespace['path'] == self.gl.user.username and p.name == MONOREPO_NAME] + membership=1, all=True) if proj.name in self.modules] + + try: + self.user_project, = [p for p in self.all_projects + if p.namespace['path'] == self.gl.user.username + and p.name == MONOREPO_NAME] + except ValueError: + fprint( + f"{red(f'ERROR')}\n\nCould not find repository {self.gl.user.name}/{MONOREPO_NAME}") + fprint(f"{red(f'Got to https://gitlab.freedesktop.org/gstreamer/gstreamer/ and create a fork so we can move your Merge requests.')}") + sys.exit(1) fprint(f"{green(' OK')}\n", nested=False) - from_projects = [proj for proj in self.all_projects if proj.namespace['path'] - == NAMESPACE and proj.name != "gstreamer"] + from_projects = [] + user_projects_name = [proj.name for proj in self.all_projects if proj.namespace['path'] + == self.gl.user.username and proj.name in GST_PROJECTS] + for project, id in GST_PROJECTS_ID.items(): + if project not in user_projects_name or project == 'gstreamer': + continue + + projects = [p for p in self.all_projects if p.id == id] + if not projects: + upstream_project = self.gl.projects.get(id) + else: + upstream_project, = projects + assert project + + from_projects.append(upstream_project) + fprint(f"\nMoving MRs from:\n") fprint(f"----------------\n") for p in from_projects: fprint(f" - {bold(p.path_with_namespace)}\n") - to_project, = [p for p in self.all_projects if p.path_with_namespace == - MOVING_NAMESPACE + "/gstreamer"] - + to_project = self.gl.projects.get(GST_PROJECTS_ID['gstreamer']) fprint(f"To: {bold(to_project.path_with_namespace)}\n\n") return from_projects, to_project @@ -268,13 +417,17 @@ class GstMRMover: description = f"**Copied from {URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}**\n\n{mr.description}" + title = mr.title + if ':' not in mr.title: + title = f"{project.name}: {mr.title}" + new_mr_dict = { 'source_branch': branch, 'allow_collaboration': True, 'remove_source_branch': True, 'target_project_id': to_project.id, 'target_branch': MONOREPO_BRANCH, - 'title': mr.title, + 'title': title, 'labels': mr.labels, 'description': description, } @@ -309,15 +462,29 @@ class GstMRMover: new_discussion = None for note in notes: - note_url = f"{mr_url}#note_{note['id']}" - body = f"**{note['author']['name']} - {PING_SIGN}{note['author']['username']} wrote [here]({note_url})**:\n\n" - body += '\n'.join([l for l in note['body'].split('\n')]) + note = discussion.notes.get(note['id']) + + note_url = f"{mr_url}#note_{note.id}" + when = dateparse.parse( + note.created_at).strftime('on %d, %b %Y') + body = f"**{note.author['name']} - {PING_SIGN}{note.author['username']} wrote [here]({note_url})** {when}:\n\n" + body += '\n'.join([line for line in note.body.split('\n')]) + + obj = { + 'body': body, + 'type': note.type, + 'resolvable': note.resolvable, + } - obj = {'body': body, 'type': note['type']} if new_discussion: new_discussion.notes.create(obj) else: new_discussion = new_mr.discussions.create(obj) + + if not note.resolvable or note.resolved: + new_discussion.resolved = True + new_discussion.save() + fprint(f"{green(' OK')}\n", nested=False) print(f"New MR available at: {bold(new_mr_url)}\n") @@ -325,7 +492,8 @@ class GstMRMover: return new_mr def push_branch(self, branch): - fprint(f"-> Pushing branch {branch} to remote {self.gl.user.username}...") + fprint( + f"-> Pushing branch {branch} to remote {self.gl.user.username}...") if self.git("push", "--no-verify", self.gl.user.username, branch, interaction_message=f"pushing {branch} to {self.gl.user.username} with:\n " f" `$git push {self.gl.user.username} {branch}`") == "SKIP": @@ -361,15 +529,20 @@ class GstMRMover: if self.git("checkout", remote_branch, "-b", branch, interaction_message=f"checking out branch with `git checkout {remote_branch} -b {branch}`") == "SKIP": - fprint(bold(f"{red('SKIPPED')} (couldn't checkout)\n"), nested=False) + fprint( + bold(f"{red('SKIPPED')} (couldn't checkout)\n"), nested=False) return False + # unset upstream to avoid to push to main (ie push.default = tracking) + self.git("branch", branch, "--unset-upstream") + for commit in reversed([c for c in mr.commits()]): if self.git("cherry-pick", commit.id, interaction_message=f"cherry-picking {commit.id} onto {branch} with:\n " - f" `$ git cherry-pick {commit.id}`") == "SKIP": - fprint(f"{yellow('SKIPPED')} (couldn't cherry-pick).", nested=False) - self.git("cherry-pick", "--abort", can_fail=True) + f" `$ git cherry-pick {commit.id}`", + revert_operation=["cherry-pick", "--abort"]) == "SKIP": + fprint( + f"{yellow('SKIPPED')} (couldn't cherry-pick).", nested=False) return False self.git("show", remote_branch + "..", call=True) @@ -385,6 +558,7 @@ class GstMRMover: def move_mrs(self, from_projects, to_project): failed_mrs = [] + found_mr = None for from_project in from_projects: with nested(f'{bold(from_project.path_with_namespace)}'): fprint(f'Fetching mrs') @@ -397,12 +571,20 @@ class GstMRMover: fprint(f"{green(' DONE')}\n", nested=False) for mr in mrs: - fprint(f'Moving {mr.source_branch} "{mr.title}": {URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}... ') + if self.mr: + if self.mr != mr.iid: + continue + found_mr = True + fprint( + f'Moving {mr.source_branch} "{mr.title}": {URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}... ') if mr.source_branch in self.skipped_branches: print(f"{yellow('SKIPPED')} (blacklisted branch)") failed_mrs.append( f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}") continue + if self.list_mrs_only: + fprint("\n"f"List only: {yellow('SKIPPED')}\n") + continue with nested(f'{bold(from_project.path_with_namespace)}: {mr.iid}'): new_mr = self.recreate_mr(from_project, to_project, mr) @@ -415,10 +597,15 @@ class GstMRMover: self.close_mr(from_project, to_project, mr, new_mr) - fprint(f"\n{yellow('DONE')} with {from_project.path_with_namespace}\n\n", nested=False) + fprint( + f"\n{yellow('DONE')} with {from_project.path_with_namespace}\n\n", nested=False) + + if self.mr and not found_mr: + sys.exit( + bold(red(f"\n==> Couldn't find MR {self.mr} in {self.modules[0]}\n"))) for mr in failed_mrs: - print(f"Didn't move MR: {mr}") + fprint(f"Didn't move MR: {mr}\n") def close_mr(self, project, to_project, mr, new_mr): if new_mr: @@ -434,7 +621,8 @@ class GstMRMover: if new_mr_url: obj = {'body': f"Moved to: {new_mr_url}"} else: - ret = input(f"Write a comment to add while closing MR {mr.iid} '{bold(mr.title)}':\n\n").strip() + ret = input( + f"Write a comment to add while closing MR {mr.iid} '{bold(mr.title)}':\n\n").strip() if ret: obj = {'body': ret} @@ -445,7 +633,8 @@ class GstMRMover: mr.discussions.create(obj) mr.state_event = 'close' mr.save() - fprint(f'Old MR {mr_url} "{bold(mr.title)}" {yellow("CLOSED")}\n') + fprint( + f'Old MR {mr_url} "{bold(mr.title)}" {yellow("CLOSED")}\n') def setup_repo(self): fprint(f"Setting up '{bold(ROOT_DIR)}'...") @@ -453,7 +642,8 @@ class GstMRMover: try: out = self.git("status", "--porcelain") if out: - fprint("\n" + red('Git repository is not clean:') + "\n```\n" + out + "\n```\n") + fprint("\n" + red('Git repository is not clean:') + + "\n```\n" + out + "\n```\n") sys.exit(1) except Exception as e: @@ -476,7 +666,8 @@ class GstMRMover: git_rename_limit = 0 if int(git_rename_limit) < 999999: self.git_rename_limit = git_rename_limit - fprint("-> Setting git rename limit to 999999 so we can properly cherry-pick between repos") + fprint( + "-> Setting git rename limit to 999999 so we can properly cherry-pick between repos\n") self.git("config", "merge.renameLimit", "999999")