scripts/move_mrs_to_monorepo.py

   1 #!/usr/bin/env python3
   2
   3 from pathlib import Path
   4 from urllib.parse import urlparse
   5 from contextlib import contextmanager
   6 import os
   7 import re
   8 import sys
   9 try:
  10     import gitlab
  11 except ModuleNotFoundError:
  12     print("========================================================================", file=sys.stderr)
  13     print("ERROR: Install python-gitlab with `python3 -m pip install python-gitlab`", file=sys.stderr)
  14     print("========================================================================", file=sys.stderr)
  15     sys.exit(1)
  16 import argparse
  17 import requests
  18
  19 import subprocess
  20
  21 ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
  22
  23 URL = "https://gitlab.freedesktop.org/"
  24 SIGN_IN_URL = URL + 'sign_in'
  25 LOGIN_URL = URL + 'users/sign_in'
  26 LOGIN_URL_LDAP = URL + '/users/auth/ldapmain/callback'
  27
  28 MONOREPO_REMOTE_NAME = 'origin'
  29 NAMESPACE = "gstreamer"
  30 MONOREPO_NAME = 'gstreamer'
  31 MONOREPO_REMOTE = URL + f'{NAMESPACE}/{MONOREPO_NAME}'
  32 MONOREPO_BRANCH = 'main'
  33 PING_SIGN = '@'
  34 MOVING_NAMESPACE = NAMESPACE
  35
  36 PARSER = argparse.ArgumentParser(
  37     description="Move merge request from old GStreamer module to the new"
  38                 "GStreamer 'monorepo'.\n"
  39                 " All your pending merge requests from all GStreamer modules will"
  40                 " be moved the the mono repository."
  41 )
  42 PARSER.add_argument("--skip-branch", action="store", nargs="*",
  43                     help="Ignore MRs for branches which match those names.", dest="skipped_branches")
  44 PARSER.add_argument("--skip-on-failure", action="store_true", default=False)
  45 PARSER.add_argument("--dry-run", "-n", action="store_true", default=False)
  46 PARSER.add_argument("--use-branch-if-exists", action="store_true", default=False)
  47
  48 GST_PROJECTS = [
  49     'gstreamer',
  50     'gst-plugins-base',
  51     'gst-plugins-good',
  52     'gst-plugins-bad',
  53     'gst-plugins-ugly',
  54     'gst-libav',
  55     'gst-rtsp-server',
  56     'gstreamer-vaapi',
  57     'gstreamer-sharp',
  58     'gst-python',
  59     'gst-omx',
  60     'gst-editing-services',
  61     'gst-devtools',
  62     'gst-integration-testsuites',
  63     'gst-docs',
  64     'gst-examples',
  65     'gst-build',
  66     'gst-ci',
  67 ]
  68
  69 # We do not want to deal with LFS
  70 os.environ["GIT_LFS_SKIP_SMUDGE"] = "1"
  71
  72
  73 log_depth = []               # type: T.List[str]
  74
  75 @contextmanager
  76 def nested(name=''):
  77     global log_depth
  78     log_depth.append(name)
  79     try:
  80         yield
  81     finally:
  82         log_depth.pop()
  83
  84 def bold(text: str):
  85     return f"\033[1m{text}\033[0m"
  86
  87 def green(text: str):
  88     return f"\033[1;32m{text}\033[0m"
  89
  90 def red(text: str):
  91     return f"\033[1;31m{text}\033[0m"
  92
  93 def yellow(text: str):
  94     return f"\033[1;33m{text}\033[0m"
  95
  96 def fprint(msg, nested=True):
  97     if log_depth:
  98         prepend = log_depth[-1] + ' | ' if nested else ''
  99     else:
 100         prepend = ''
 101
 102     print(prepend + msg, end="")
 103     sys.stdout.flush()
 104
 105
 106 class GstMRMover:
 107     def __init__(self):
 108
 109         self.gl = self.connect()
 110         self.gl.auth()
 111         self.all_projects = []
 112         self.skipped_branches = []
 113         self.git_rename_limit = None
 114         self.skip_on_failure = None
 115         self.dry_run = False
 116
 117     def connect(self):
 118         fprint("Logging into gitlab...")
 119         gitlab_api_token = os.environ.get('GITLAB_API_TOKEN')
 120
 121         if gitlab_api_token:
 122             gl = gitlab.Gitlab(URL, private_token=gitlab_api_token)
 123             fprint(f"{green(' OK')}\n", nested=False)
 124             return gl
 125
 126         session = requests.Session()
 127         sign_in_page = session.get(SIGN_IN_URL).content.decode()
 128         for l in sign_in_page.split('\n'):
 129             m = re.search('name="authenticity_token" value="([^"]+)"', l)
 130             if m:
 131                 break
 132
 133         token = None
 134         if m:
 135             token = m.group(1)
 136
 137         if not token:
 138             fprint(f"{red('Unable to find the authenticity token')}\n")
 139             sys.exit(1)
 140
 141
 142         for data, url in [
 143             ({'user[login]': 'login_or_email',
 144               'user[password]': 'SECRET',
 145               'authenticity_token': token}, LOGIN_URL),
 146             ({'username': 'login_or_email',
 147               'password': 'SECRET',
 148               'authenticity_token': token}, LOGIN_URL_LDAP)]:
 149
 150             r = session.post(url, data=data)
 151             if r.status_code != 200:
 152                 continue
 153
 154             try:
 155                 gl = gitlab.Gitlab(URL, api_version=4, session=session)
 156                 gl.auth()
 157             except gitlab.exceptions.GitlabAuthenticationError as e:
 158                 continue
 159             return gl
 160
 161         sys.exit(bold(f"{red('FAILED')}.\n\nPlease go to:\n\n"
 162             '   https://gitlab.freedesktop.org/-/profile/personal_access_tokens\n\n'
 163             f'and generate a token {bold("with read/write access to all but the registry")},'
 164             ' then set it in the "GITLAB_API_TOKEN" environment variable:"'
 165             f'\n\n  $ GITLAB_API_TOKEN=<your token> {" ".join(sys.argv)}\n'))
 166
 167     def git(self, *args, can_fail=False, interaction_message=None, call=False):
 168         cwd = ROOT_DIR
 169         retry = True
 170         while retry:
 171             retry = False
 172             try:
 173                 if not call:
 174                     try:
 175                         return subprocess.check_output(["git"] + list(args), cwd=cwd,
 176                                                     stdin=subprocess.DEVNULL,
 177                                                     stderr=subprocess.STDOUT).decode()
 178                     except:
 179                         if not can_fail:
 180                             fprint(f"\n\n{bold(red('ERROR'))}: `git {' '.join(args)}` failed" + "\n", nested=False)
 181                         raise
 182                 else:
 183                     subprocess.call(["git"] + list(args), cwd=cwd)
 184                     return "All good"
 185             except Exception as e:
 186                 if interaction_message:
 187                     if self.skip_on_failure:
 188                         return "SKIP"
 189                     output = getattr(e, "output", b"")
 190                     if output is not None:
 191                         out = output.decode()
 192                     else:
 193                         out = "????"
 194                     fprint(f"\n```"
 195                           f"\n{out}\n"
 196                           f"Entering a shell in {cwd} to fix:\n\n"
 197                           f" {bold(interaction_message)}\n\n"
 198                           f"You should then exit with the following codes:\n\n"
 199                           f"  - {bold('`exit 0`')}: once you have fixed the problem and we can keep moving the merge request\n"
 200                           f"  - {bold('`exit 1`')}: {bold('retry')}: once you have let the repo in a state where the operation should be to retried\n"
 201                           f"  - {bold('`exit 2`')}: to skip that merge request\n"
 202                           f"  - {bold('`exit 3`')}: stop the script and abandon moving your MRs\n"
 203                           "\n```\n", nested=False)
 204                     try:
 205                         if os.name == 'nt':
 206                             shell = os.environ.get(
 207                                 "COMSPEC", r"C:\WINDOWS\system32\cmd.exe")
 208                         else:
 209                             shell = os.environ.get(
 210                                 "SHELL", os.path.realpath("/bin/sh"))
 211                         subprocess.check_call(shell, cwd=cwd)
 212                     except subprocess.CalledProcessError as e:
 213                         if e.returncode == 1:
 214                             retry = True
 215                             continue
 216                         elif e.returncode == 2:
 217                             return "SKIP"
 218                         elif e.returncode == 3:
 219                             sys.exit(3)
 220                     except:
 221                         # Result of subshell does not really matter
 222                         pass
 223
 224                     return "User fixed it"
 225
 226                 if can_fail:
 227                     return "Failed but we do not care"
 228
 229                 raise e
 230
 231     def run(self):
 232         try:
 233             self.setup_repo()
 234
 235             from_projects, to_project = self.fetch_projects()
 236
 237             with nested('  '):
 238                 self.move_mrs(from_projects, to_project)
 239         finally:
 240             if self.git_rename_limit is not None:
 241                 self.git("config", "merge.renameLimit", str(self.git_rename_limit))
 242
 243     def fetch_projects(self):
 244         fprint("Fetching projects... ")
 245         self.all_projects = [proj for proj in self.gl.projects.list(
 246             membership=1, all=True) if proj.name in GST_PROJECTS]
 247         self.user_project, = [p for p in self.all_projects if p.namespace['path'] == self.gl.user.username and p.name == MONOREPO_NAME]
 248         fprint(f"{green(' OK')}\n", nested=False)
 249
 250         from_projects = [proj for proj in self.all_projects if proj.namespace['path']
 251                          == NAMESPACE and proj.name != "gstreamer"]
 252         fprint(f"\nMoving MRs from:\n")
 253         fprint(f"----------------\n")
 254         for p in from_projects:
 255             fprint(f"  - {bold(p.path_with_namespace)}\n")
 256
 257         to_project, = [p for p in self.all_projects if p.path_with_namespace ==
 258                        MOVING_NAMESPACE + "/gstreamer"]
 259
 260         fprint(f"To: {bold(to_project.path_with_namespace)}\n\n")
 261
 262         return from_projects, to_project
 263
 264     def recreate_mr(self, project, to_project, mr):
 265         branch = f"{project.name}-{mr.source_branch}"
 266         if not self.create_branch_for_mr(branch, project, mr):
 267             return None
 268
 269         description = f"**Copied from {URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}**\n\n{mr.description}"
 270
 271         new_mr_dict = {
 272             'source_branch': branch,
 273             'allow_collaboration': True,
 274             'remove_source_branch': True,
 275             'target_project_id': to_project.id,
 276             'target_branch': MONOREPO_BRANCH,
 277             'title': mr.title,
 278             'labels': mr.labels,
 279             'description': description,
 280         }
 281
 282         try:
 283             fprint(f"-> Recreating MR '{bold(mr.title)}'...")
 284             if self.dry_run:
 285                 fprint(f"\nDry info:\n{new_mr_dict}\n")
 286             else:
 287                 new_mr = self.user_project.mergerequests.create(new_mr_dict)
 288                 fprint(f"{green(' OK')}\n", nested=False)
 289         except gitlab.exceptions.GitlabCreateError as e:
 290             fprint(f"{yellow('SKIPPED')} (An MR already exists)\n", nested=False)
 291             return None
 292
 293         fprint(f"-> Adding discussings from MR '{mr.title}'...")
 294         if self.dry_run:
 295             fprint(f"{green(' OK')}\n", nested=False)
 296             return None
 297
 298         new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}"
 299         for issue in mr.closes_issues():
 300             obj = {'body': f'Fixing MR moved to: {new_mr_url}'}
 301             issue.discussions.create(obj)
 302
 303         mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}"
 304         for discussion in mr.discussions.list():
 305             # FIXME notes = [n for n in discussion.attributes['notes'] if n['type'] is not None]
 306             notes = [n for n in discussion.attributes['notes']]
 307             if not notes:
 308                 continue
 309
 310             new_discussion = None
 311             for note in notes:
 312                 note_url = f"{mr_url}#note_{note['id']}"
 313                 body = f"**{note['author']['name']} - {PING_SIGN}{note['author']['username']} wrote [here]({note_url})**:\n\n"
 314                 body += '\n'.join([l for l in note['body'].split('\n')])
 315
 316                 obj = {'body': body, 'type': note['type']}
 317                 if new_discussion:
 318                     new_discussion.notes.create(obj)
 319                 else:
 320                     new_discussion = new_mr.discussions.create(obj)
 321         fprint(f"{green(' OK')}\n", nested=False)
 322
 323         print(f"New MR available at: {bold(new_mr_url)}\n")
 324
 325         return new_mr
 326
 327     def push_branch(self, branch):
 328         fprint(f"-> Pushing branch {branch} to remote {self.gl.user.username}...")
 329         if self.git("push", "--no-verify", self.gl.user.username, branch,
 330                     interaction_message=f"pushing {branch} to {self.gl.user.username} with:\n  "
 331                     f" `$git push {self.gl.user.username} {branch}`") == "SKIP":
 332             fprint(yellow("'SKIPPED' (couldn't push)"), nested=False)
 333
 334             return False
 335
 336         fprint(f"{green(' OK')}\n", nested=False)
 337
 338         return True
 339
 340     def create_branch_for_mr(self, branch, project, mr):
 341         remote_name = project.name + '-' + self.gl.user.username
 342         remote_branch = f"{MONOREPO_REMOTE_NAME}/{MONOREPO_BRANCH}"
 343         if self.use_branch_if_exists:
 344             try:
 345                 self.git("checkout", branch)
 346                 self.git("show", remote_branch + "..", call=True)
 347                 if self.dry_run:
 348                     fprint("Dry run... not creating MR")
 349                     return True
 350                 cont = input('\n     Create MR [y/n]? ')
 351                 if cont.strip().lower() != 'y':
 352                     fprint("Cancelled")
 353                     return False
 354                 return self.push_branch(branch)
 355             except subprocess.CalledProcessError as e:
 356                 pass
 357
 358         self.git("remote", "add", remote_name,
 359                  f"{URL}{self.gl.user.username}/{project.name}.git", can_fail=True)
 360         self.git("fetch", remote_name)
 361
 362         if self.git("checkout", remote_branch, "-b", branch,
 363                     interaction_message=f"checking out branch with `git checkout {remote_branch} -b {branch}`") == "SKIP":
 364             fprint(bold(f"{red('SKIPPED')} (couldn't checkout)\n"), nested=False)
 365             return False
 366
 367         for commit in reversed([c for c in mr.commits()]):
 368             if self.git("cherry-pick", commit.id,
 369                         interaction_message=f"cherry-picking {commit.id} onto {branch} with:\n  "
 370                         f" `$ git cherry-pick {commit.id}`") == "SKIP":
 371                 fprint(f"{yellow('SKIPPED')} (couldn't cherry-pick).", nested=False)
 372                 self.git("cherry-pick", "--abort", can_fail=True)
 373                 return False
 374
 375         self.git("show", remote_branch + "..", call=True)
 376         if self.dry_run:
 377             fprint("Dry run... not creating MR\n")
 378             return True
 379         cont = input('\n     Create MR [y/n]? ')
 380         if cont.strip().lower() != 'y':
 381             fprint(f"{red('Cancelled')}\n", nested=False)
 382             return False
 383
 384         return self.push_branch(branch)
 385
 386     def move_mrs(self, from_projects, to_project):
 387         failed_mrs = []
 388         for from_project in from_projects:
 389             with nested(f'{bold(from_project.path_with_namespace)}'):
 390                 fprint(f'Fetching mrs')
 391                 mrs = [mr for mr in from_project.mergerequests.list(
 392                     all=True, author_id=self.gl.user.id) if mr.author['username'] == self.gl.user.username and mr.state == "opened"]
 393                 if not mrs:
 394                     fprint(f"{yellow(' None')}\n", nested=False)
 395                     continue
 396
 397                 fprint(f"{green(' DONE')}\n", nested=False)
 398
 399                 for mr in mrs:
 400                     fprint(f'Moving {mr.source_branch} "{mr.title}": {URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}... ')
 401                     if mr.source_branch in self.skipped_branches:
 402                         print(f"{yellow('SKIPPED')} (blacklisted branch)")
 403                         failed_mrs.append(
 404                             f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}")
 405                         continue
 406
 407                     with nested(f'{bold(from_project.path_with_namespace)}: {mr.iid}'):
 408                         new_mr = self.recreate_mr(from_project, to_project, mr)
 409                         if not new_mr:
 410                             if not self.dry_run:
 411                                 failed_mrs.append(
 412                                     f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}")
 413                         else:
 414                             fprint(f"{green(' OK')}\n", nested=False)
 415
 416                         self.close_mr(from_project, to_project, mr, new_mr)
 417
 418             fprint(f"\n{yellow('DONE')} with {from_project.path_with_namespace}\n\n", nested=False)
 419
 420         for mr in failed_mrs:
 421             print(f"Didn't move MR: {mr}")
 422
 423     def close_mr(self, project, to_project, mr, new_mr):
 424         if new_mr:
 425             new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}"
 426         else:
 427             new_mr_url = None
 428         mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}"
 429         cont = input(f'\n  Close old MR {mr_url} "{bold(mr.title)}" ? [y/n]')
 430         if cont.strip().lower() != 'y':
 431             fprint(f"{yellow('Not closing old MR')}\n")
 432         else:
 433             obj = None
 434             if new_mr_url:
 435                 obj = {'body': f"Moved to: {new_mr_url}"}
 436             else:
 437                 ret = input(f"Write a comment to add while closing MR {mr.iid} '{bold(mr.title)}':\n\n").strip()
 438                 if ret:
 439                     obj = {'body': ret}
 440
 441             if self.dry_run:
 442                 fprint(f"{bold('Dry run, not closing')}\n", nested=False)
 443             else:
 444                 if obj:
 445                     mr.discussions.create(obj)
 446                 mr.state_event = 'close'
 447                 mr.save()
 448                 fprint(f'Old MR {mr_url} "{bold(mr.title)}" {yellow("CLOSED")}\n')
 449
 450     def setup_repo(self):
 451         fprint(f"Setting up '{bold(ROOT_DIR)}'...")
 452
 453         try:
 454             out = self.git("status", "--porcelain")
 455             if out:
 456                 fprint("\n" + red('Git repository is not clean:') + "\n```\n" + out + "\n```\n")
 457                 sys.exit(1)
 458
 459         except Exception as e:
 460             exit(
 461                 f"Git repository{ROOT_DIR} is not clean. Clean it up before running {sys.argv[0]}\n ({e})")
 462
 463         self.git('remote', 'add', MONOREPO_REMOTE_NAME,
 464                  MONOREPO_REMOTE, can_fail=True)
 465         self.git('fetch', MONOREPO_REMOTE_NAME)
 466
 467         self.git('remote', 'add', self.gl.user.username,
 468                  f"git@gitlab.freedesktop.org:{self.gl.user.username}/gstreamer.git", can_fail=True)
 469         self.git('fetch', self.gl.user.username,
 470                  interaction_message=f"Setup your fork of {URL}gstreamer/gstreamer as remote called {self.gl.user.username}")
 471         fprint(f"{green(' OK')}\n", nested=False)
 472
 473         try:
 474             git_rename_limit = int(self.git("config", "merge.renameLimit"))
 475         except subprocess.CalledProcessError:
 476             git_rename_limit = 0
 477         if int(git_rename_limit) < 999999:
 478             self.git_rename_limit = git_rename_limit
 479             fprint("-> Setting git rename limit to 999999 so we can properly cherry-pick between repos")
 480             self.git("config", "merge.renameLimit", "999999")
 481
 482
 483 def main():
 484     mover = GstMRMover()
 485     PARSER.parse_args(namespace=mover)
 486     mover.run()
 487
 488
 489 if __name__ == '__main__':
 490     main()