scripts/move_mrs_to_monorepo.py

   1 #!/usr/bin/env python3
   2
   3 from urllib.parse import urlparse
   4 from contextlib import contextmanager
   5 import os
   6 import re
   7 import sys
   8 try:
   9     import gitlab
  10 except ModuleNotFoundError:
  11     print("========================================================================", file=sys.stderr)
  12     print("ERROR: Install python-gitlab with `python3 -m pip install python-gitlab dateutil`", file=sys.stderr)
  13     print("========================================================================", file=sys.stderr)
  14     sys.exit(1)
  15
  16 try:
  17     from dateutil import parser as dateparse
  18 except ModuleNotFoundError:
  19     print("========================================================================", file=sys.stderr)
  20     print("ERROR: Install dateutil with `python3 -m pip install dateutil`", file=sys.stderr)
  21     print("========================================================================", file=sys.stderr)
  22     sys.exit(1)
  23 import argparse
  24 import requests
  25
  26 import subprocess
  27
  28 ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
  29
  30 URL = "https://gitlab.freedesktop.org/"
  31 SIGN_IN_URL = URL + 'sign_in'
  32 LOGIN_URL = URL + 'users/sign_in'
  33 LOGIN_URL_LDAP = URL + '/users/auth/ldapmain/callback'
  34
  35 MONOREPO_REMOTE_NAME = 'origin'
  36 NAMESPACE = "gstreamer"
  37 MONOREPO_NAME = 'gstreamer'
  38 MONOREPO_REMOTE = URL + f'{NAMESPACE}/{MONOREPO_NAME}'
  39 MONOREPO_BRANCH = 'main'
  40 PING_SIGN = '@'
  41 MOVING_NAMESPACE = NAMESPACE
  42
  43 PARSER = argparse.ArgumentParser(
  44     description="Move merge request from old GStreamer module to the new"
  45                 "GStreamer 'monorepo'.\n"
  46                 " All your pending merge requests from all GStreamer modules will"
  47                 " be moved the the mono repository."
  48 )
  49 PARSER.add_argument("--skip-branch", action="store", nargs="*",
  50                     help="Ignore MRs for branches which match those names.", dest="skipped_branches")
  51 PARSER.add_argument("--skip-on-failure", action="store_true", default=False)
  52 PARSER.add_argument("--dry-run", "-n", action="store_true", default=False)
  53 PARSER.add_argument("--use-branch-if-exists", action="store_true", default=False)
  54
  55 GST_PROJECTS = [
  56     'gstreamer',
  57     'gst-plugins-base',
  58     'gst-plugins-good',
  59     'gst-plugins-bad',
  60     'gst-plugins-ugly',
  61     'gst-libav',
  62     'gst-rtsp-server',
  63     'gstreamer-vaapi',
  64     'gstreamer-sharp',
  65     'gst-python',
  66     'gst-omx',
  67     'gst-editing-services',
  68     'gst-devtools',
  69     'gst-integration-testsuites',
  70     'gst-docs',
  71     'gst-examples',
  72     'gst-build',
  73     'gst-ci',
  74 ]
  75
  76 # We do not want to deal with LFS
  77 os.environ["GIT_LFS_SKIP_SMUDGE"] = "1"
  78
  79
  80 log_depth = []               # type: T.List[str]
  81
  82 @contextmanager
  83 def nested(name=''):
  84     global log_depth
  85     log_depth.append(name)
  86     try:
  87         yield
  88     finally:
  89         log_depth.pop()
  90
  91 def bold(text: str):
  92     return f"\033[1m{text}\033[0m"
  93
  94 def green(text: str):
  95     return f"\033[1;32m{text}\033[0m"
  96
  97 def red(text: str):
  98     return f"\033[1;31m{text}\033[0m"
  99
 100 def yellow(text: str):
 101     return f"\033[1;33m{text}\033[0m"
 102
 103 def fprint(msg, nested=True):
 104     if log_depth:
 105         prepend = log_depth[-1] + ' | ' if nested else ''
 106     else:
 107         prepend = ''
 108
 109     print(prepend + msg, end="")
 110     sys.stdout.flush()
 111
 112
 113 class GstMRMover:
 114     def __init__(self):
 115
 116         self.gl = self.connect()
 117         self.gl.auth()
 118         self.all_projects = []
 119         self.skipped_branches = []
 120         self.git_rename_limit = None
 121         self.skip_on_failure = None
 122         self.dry_run = False
 123
 124     def connect(self):
 125         fprint("Logging into gitlab...")
 126         gitlab_api_token = os.environ.get('GITLAB_API_TOKEN')
 127
 128         if gitlab_api_token:
 129             gl = gitlab.Gitlab(URL, private_token=gitlab_api_token)
 130             fprint(f"{green(' OK')}\n", nested=False)
 131             return gl
 132
 133         session = requests.Session()
 134         sign_in_page = session.get(SIGN_IN_URL).content.decode()
 135         for l in sign_in_page.split('\n'):
 136             m = re.search('name="authenticity_token" value="([^"]+)"', l)
 137             if m:
 138                 break
 139
 140         token = None
 141         if m:
 142             token = m.group(1)
 143
 144         if not token:
 145             fprint(f"{red('Unable to find the authenticity token')}\n")
 146             sys.exit(1)
 147
 148
 149         for data, url in [
 150             ({'user[login]': 'login_or_email',
 151               'user[password]': 'SECRET',
 152               'authenticity_token': token}, LOGIN_URL),
 153             ({'username': 'login_or_email',
 154               'password': 'SECRET',
 155               'authenticity_token': token}, LOGIN_URL_LDAP)]:
 156
 157             r = session.post(url, data=data)
 158             if r.status_code != 200:
 159                 continue
 160
 161             try:
 162                 gl = gitlab.Gitlab(URL, api_version=4, session=session)
 163                 gl.auth()
 164             except gitlab.exceptions.GitlabAuthenticationError as e:
 165                 continue
 166             return gl
 167
 168         sys.exit(bold(f"{red('FAILED')}.\n\nPlease go to:\n\n"
 169             '   https://gitlab.freedesktop.org/-/profile/personal_access_tokens\n\n'
 170             f'and generate a token {bold("with read/write access to all but the registry")},'
 171             ' then set it in the "GITLAB_API_TOKEN" environment variable:"'
 172             f'\n\n  $ GITLAB_API_TOKEN=<your token> {" ".join(sys.argv)}\n'))
 173
 174     def git(self, *args, can_fail=False, interaction_message=None, call=False):
 175         cwd = ROOT_DIR
 176         retry = True
 177         while retry:
 178             retry = False
 179             try:
 180                 if not call:
 181                     try:
 182                         return subprocess.check_output(["git"] + list(args), cwd=cwd,
 183                                                     stdin=subprocess.DEVNULL,
 184                                                     stderr=subprocess.STDOUT).decode()
 185                     except:
 186                         if not can_fail:
 187                             fprint(f"\n\n{bold(red('ERROR'))}: `git {' '.join(args)}` failed" + "\n", nested=False)
 188                         raise
 189                 else:
 190                     subprocess.call(["git"] + list(args), cwd=cwd)
 191                     return "All good"
 192             except Exception as e:
 193                 if interaction_message:
 194                     if self.skip_on_failure:
 195                         return "SKIP"
 196                     output = getattr(e, "output", b"")
 197                     if output is not None:
 198                         out = output.decode()
 199                     else:
 200                         out = "????"
 201                     fprint(f"\n```"
 202                           f"\n{out}\n"
 203                           f"Entering a shell in {cwd} to fix:\n\n"
 204                           f" {bold(interaction_message)}\n\n"
 205                           f"You should then exit with the following codes:\n\n"
 206                           f"  - {bold('`exit 0`')}: once you have fixed the problem and we can keep moving the merge request\n"
 207                           f"  - {bold('`exit 1`')}: {bold('retry')}: once you have let the repo in a state where the operation should be to retried\n"
 208                           f"  - {bold('`exit 2`')}: to skip that merge request\n"
 209                           f"  - {bold('`exit 3`')}: stop the script and abandon moving your MRs\n"
 210                           "\n```\n", nested=False)
 211                     try:
 212                         if os.name == 'nt':
 213                             shell = os.environ.get(
 214                                 "COMSPEC", r"C:\WINDOWS\system32\cmd.exe")
 215                         else:
 216                             shell = os.environ.get(
 217                                 "SHELL", os.path.realpath("/bin/sh"))
 218                         subprocess.check_call(shell, cwd=cwd)
 219                     except subprocess.CalledProcessError as e:
 220                         if e.returncode == 1:
 221                             retry = True
 222                             continue
 223                         elif e.returncode == 2:
 224                             return "SKIP"
 225                         elif e.returncode == 3:
 226                             sys.exit(3)
 227                     except:
 228                         # Result of subshell does not really matter
 229                         pass
 230
 231                     return "User fixed it"
 232
 233                 if can_fail:
 234                     return "Failed but we do not care"
 235
 236                 raise e
 237
 238     def run(self):
 239         try:
 240             self.setup_repo()
 241
 242             from_projects, to_project = self.fetch_projects()
 243
 244             with nested('  '):
 245                 self.move_mrs(from_projects, to_project)
 246         finally:
 247             if self.git_rename_limit is not None:
 248                 self.git("config", "merge.renameLimit", str(self.git_rename_limit))
 249
 250     def fetch_projects(self):
 251         fprint("Fetching projects... ")
 252         self.all_projects = [proj for proj in self.gl.projects.list(
 253             membership=1, all=True) if proj.name in GST_PROJECTS]
 254         self.user_project, = [p for p in self.all_projects if p.namespace['path'] == self.gl.user.username and p.name == MONOREPO_NAME]
 255         fprint(f"{green(' OK')}\n", nested=False)
 256
 257         from_projects = [proj for proj in self.all_projects if proj.namespace['path']
 258                          == NAMESPACE and proj.name != "gstreamer"]
 259         fprint(f"\nMoving MRs from:\n")
 260         fprint(f"----------------\n")
 261         for p in from_projects:
 262             fprint(f"  - {bold(p.path_with_namespace)}\n")
 263
 264         to_project, = [p for p in self.all_projects if p.path_with_namespace ==
 265                        MOVING_NAMESPACE + "/gstreamer"]
 266
 267         fprint(f"To: {bold(to_project.path_with_namespace)}\n\n")
 268
 269         return from_projects, to_project
 270
 271     def recreate_mr(self, project, to_project, mr):
 272         branch = f"{project.name}-{mr.source_branch}"
 273         if not self.create_branch_for_mr(branch, project, mr):
 274             return None
 275
 276         description = f"**Copied from {URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}**\n\n{mr.description}"
 277
 278         new_mr_dict = {
 279             'source_branch': branch,
 280             'allow_collaboration': True,
 281             'remove_source_branch': True,
 282             'target_project_id': to_project.id,
 283             'target_branch': MONOREPO_BRANCH,
 284             'title': mr.title,
 285             'labels': mr.labels,
 286             'description': description,
 287         }
 288
 289         try:
 290             fprint(f"-> Recreating MR '{bold(mr.title)}'...")
 291             if self.dry_run:
 292                 fprint(f"\nDry info:\n{new_mr_dict}\n")
 293             else:
 294                 new_mr = self.user_project.mergerequests.create(new_mr_dict)
 295                 fprint(f"{green(' OK')}\n", nested=False)
 296         except gitlab.exceptions.GitlabCreateError as e:
 297             fprint(f"{yellow('SKIPPED')} (An MR already exists)\n", nested=False)
 298             return None
 299
 300         fprint(f"-> Adding discussings from MR '{mr.title}'...")
 301         if self.dry_run:
 302             fprint(f"{green(' OK')}\n", nested=False)
 303             return None
 304
 305         new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}"
 306         for issue in mr.closes_issues():
 307             obj = {'body': f'Fixing MR moved to: {new_mr_url}'}
 308             issue.discussions.create(obj)
 309
 310         mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}"
 311         for discussion in mr.discussions.list():
 312             # FIXME notes = [n for n in discussion.attributes['notes'] if n['type'] is not None]
 313             notes = [n for n in discussion.attributes['notes']]
 314             if not notes:
 315                 continue
 316
 317             new_discussion = None
 318             for note in notes:
 319                 note = discussion.notes.get(note['id'])
 320
 321                 note_url = f"{mr_url}#note_{note.id}"
 322                 when = dateparse.parse(note.created_at).strftime('on %d, %b %Y')
 323                 body = f"**{note.author['name']} - {PING_SIGN}{note.author['username']} wrote [here]({note_url})** {when}:\n\n"
 324                 body += '\n'.join([l for l in note.body.split('\n')])
 325
 326                 obj = {
 327                     'body': body,
 328                     'type': note.type,
 329                     'resolvable': note.resolvable,
 330                 }
 331
 332                 if new_discussion:
 333                     new_discussion.notes.create(obj)
 334                 else:
 335                     new_discussion = new_mr.discussions.create(obj)
 336
 337                 if not note.resolvable or note.resolved:
 338                     new_discussion.resolved = True
 339                     new_discussion.save()
 340
 341         fprint(f"{green(' OK')}\n", nested=False)
 342
 343         print(f"New MR available at: {bold(new_mr_url)}\n")
 344
 345         return new_mr
 346
 347     def push_branch(self, branch):
 348         fprint(f"-> Pushing branch {branch} to remote {self.gl.user.username}...")
 349         if self.git("push", "--no-verify", self.gl.user.username, branch,
 350                     interaction_message=f"pushing {branch} to {self.gl.user.username} with:\n  "
 351                     f" `$git push {self.gl.user.username} {branch}`") == "SKIP":
 352             fprint(yellow("'SKIPPED' (couldn't push)"), nested=False)
 353
 354             return False
 355
 356         fprint(f"{green(' OK')}\n", nested=False)
 357
 358         return True
 359
 360     def create_branch_for_mr(self, branch, project, mr):
 361         remote_name = project.name + '-' + self.gl.user.username
 362         remote_branch = f"{MONOREPO_REMOTE_NAME}/{MONOREPO_BRANCH}"
 363         if self.use_branch_if_exists:
 364             try:
 365                 self.git("checkout", branch)
 366                 self.git("show", remote_branch + "..", call=True)
 367                 if self.dry_run:
 368                     fprint("Dry run... not creating MR")
 369                     return True
 370                 cont = input('\n     Create MR [y/n]? ')
 371                 if cont.strip().lower() != 'y':
 372                     fprint("Cancelled")
 373                     return False
 374                 return self.push_branch(branch)
 375             except subprocess.CalledProcessError as e:
 376                 pass
 377
 378         self.git("remote", "add", remote_name,
 379                  f"{URL}{self.gl.user.username}/{project.name}.git", can_fail=True)
 380         self.git("fetch", remote_name)
 381
 382         if self.git("checkout", remote_branch, "-b", branch,
 383                     interaction_message=f"checking out branch with `git checkout {remote_branch} -b {branch}`") == "SKIP":
 384             fprint(bold(f"{red('SKIPPED')} (couldn't checkout)\n"), nested=False)
 385             return False
 386
 387         for commit in reversed([c for c in mr.commits()]):
 388             if self.git("cherry-pick", commit.id,
 389                         interaction_message=f"cherry-picking {commit.id} onto {branch} with:\n  "
 390                         f" `$ git cherry-pick {commit.id}`") == "SKIP":
 391                 fprint(f"{yellow('SKIPPED')} (couldn't cherry-pick).", nested=False)
 392                 self.git("cherry-pick", "--abort", can_fail=True)
 393                 return False
 394
 395         self.git("show", remote_branch + "..", call=True)
 396         if self.dry_run:
 397             fprint("Dry run... not creating MR\n")
 398             return True
 399         cont = input('\n     Create MR [y/n]? ')
 400         if cont.strip().lower() != 'y':
 401             fprint(f"{red('Cancelled')}\n", nested=False)
 402             return False
 403
 404         return self.push_branch(branch)
 405
 406     def move_mrs(self, from_projects, to_project):
 407         failed_mrs = []
 408         for from_project in from_projects:
 409             with nested(f'{bold(from_project.path_with_namespace)}'):
 410                 fprint(f'Fetching mrs')
 411                 mrs = [mr for mr in from_project.mergerequests.list(
 412                     all=True, author_id=self.gl.user.id) if mr.author['username'] == self.gl.user.username and mr.state == "opened"]
 413                 if not mrs:
 414                     fprint(f"{yellow(' None')}\n", nested=False)
 415                     continue
 416
 417                 fprint(f"{green(' DONE')}\n", nested=False)
 418
 419                 for mr in mrs:
 420                     fprint(f'Moving {mr.source_branch} "{mr.title}": {URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}... ')
 421                     if mr.source_branch in self.skipped_branches:
 422                         print(f"{yellow('SKIPPED')} (blacklisted branch)")
 423                         failed_mrs.append(
 424                             f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}")
 425                         continue
 426
 427                     with nested(f'{bold(from_project.path_with_namespace)}: {mr.iid}'):
 428                         new_mr = self.recreate_mr(from_project, to_project, mr)
 429                         if not new_mr:
 430                             if not self.dry_run:
 431                                 failed_mrs.append(
 432                                     f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}")
 433                         else:
 434                             fprint(f"{green(' OK')}\n", nested=False)
 435
 436                         self.close_mr(from_project, to_project, mr, new_mr)
 437
 438             fprint(f"\n{yellow('DONE')} with {from_project.path_with_namespace}\n\n", nested=False)
 439
 440         for mr in failed_mrs:
 441             print(f"Didn't move MR: {mr}")
 442
 443     def close_mr(self, project, to_project, mr, new_mr):
 444         if new_mr:
 445             new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}"
 446         else:
 447             new_mr_url = None
 448         mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}"
 449         cont = input(f'\n  Close old MR {mr_url} "{bold(mr.title)}" ? [y/n]')
 450         if cont.strip().lower() != 'y':
 451             fprint(f"{yellow('Not closing old MR')}\n")
 452         else:
 453             obj = None
 454             if new_mr_url:
 455                 obj = {'body': f"Moved to: {new_mr_url}"}
 456             else:
 457                 ret = input(f"Write a comment to add while closing MR {mr.iid} '{bold(mr.title)}':\n\n").strip()
 458                 if ret:
 459                     obj = {'body': ret}
 460
 461             if self.dry_run:
 462                 fprint(f"{bold('Dry run, not closing')}\n", nested=False)
 463             else:
 464                 if obj:
 465                     mr.discussions.create(obj)
 466                 mr.state_event = 'close'
 467                 mr.save()
 468                 fprint(f'Old MR {mr_url} "{bold(mr.title)}" {yellow("CLOSED")}\n')
 469
 470     def setup_repo(self):
 471         fprint(f"Setting up '{bold(ROOT_DIR)}'...")
 472
 473         try:
 474             out = self.git("status", "--porcelain")
 475             if out:
 476                 fprint("\n" + red('Git repository is not clean:') + "\n```\n" + out + "\n```\n")
 477                 sys.exit(1)
 478
 479         except Exception as e:
 480             exit(
 481                 f"Git repository{ROOT_DIR} is not clean. Clean it up before running {sys.argv[0]}\n ({e})")
 482
 483         self.git('remote', 'add', MONOREPO_REMOTE_NAME,
 484                  MONOREPO_REMOTE, can_fail=True)
 485         self.git('fetch', MONOREPO_REMOTE_NAME)
 486
 487         self.git('remote', 'add', self.gl.user.username,
 488                  f"git@gitlab.freedesktop.org:{self.gl.user.username}/gstreamer.git", can_fail=True)
 489         self.git('fetch', self.gl.user.username,
 490                  interaction_message=f"Setup your fork of {URL}gstreamer/gstreamer as remote called {self.gl.user.username}")
 491         fprint(f"{green(' OK')}\n", nested=False)
 492
 493         try:
 494             git_rename_limit = int(self.git("config", "merge.renameLimit"))
 495         except subprocess.CalledProcessError:
 496             git_rename_limit = 0
 497         if int(git_rename_limit) < 999999:
 498             self.git_rename_limit = git_rename_limit
 499             fprint("-> Setting git rename limit to 999999 so we can properly cherry-pick between repos")
 500             self.git("config", "merge.renameLimit", "999999")
 501
 502
 503 def main():
 504     mover = GstMRMover()
 505     PARSER.parse_args(namespace=mover)
 506     mover.run()
 507
 508
 509 if __name__ == '__main__':
 510     main()