scripts/move_mrs_to_monorepo.py

   1 #!/usr/bin/env python3
   2
   3 from urllib.parse import urlparse
   4 from contextlib import contextmanager
   5 import os
   6 import re
   7 import sys
   8 try:
   9     import gitlab
  10 except ModuleNotFoundError:
  11     print("========================================================================", file=sys.stderr)
  12     print("ERROR: Install python-gitlab with `python3 -m pip install python-gitlab dateutil`", file=sys.stderr)
  13     print("========================================================================", file=sys.stderr)
  14     sys.exit(1)
  15
  16 try:
  17     from dateutil import parser as dateparse
  18 except ModuleNotFoundError:
  19     print("========================================================================", file=sys.stderr)
  20     print("ERROR: Install dateutil with `python3 -m pip install dateutil`", file=sys.stderr)
  21     print("========================================================================", file=sys.stderr)
  22     sys.exit(1)
  23 import argparse
  24 import requests
  25
  26 import subprocess
  27
  28 ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
  29
  30 URL = "https://gitlab.freedesktop.org/"
  31 SIGN_IN_URL = URL + 'sign_in'
  32 LOGIN_URL = URL + 'users/sign_in'
  33 LOGIN_URL_LDAP = URL + '/users/auth/ldapmain/callback'
  34
  35 MONOREPO_REMOTE_NAME = 'origin'
  36 NAMESPACE = "gstreamer"
  37 MONOREPO_NAME = 'gstreamer'
  38 MONOREPO_REMOTE = URL + f'{NAMESPACE}/{MONOREPO_NAME}'
  39 MONOREPO_BRANCH = 'main'
  40 PING_SIGN = '@'
  41 MOVING_NAMESPACE = NAMESPACE
  42
  43 PARSER = argparse.ArgumentParser(
  44     description="Move merge request from old GStreamer module to the new"
  45                 "GStreamer 'monorepo'.\n"
  46                 " All your pending merge requests from all GStreamer modules will"
  47                 " be moved the the mono repository."
  48 )
  49 PARSER.add_argument("--skip-branch", action="store", nargs="*",
  50                     help="Ignore MRs for branches which match those names.", dest="skipped_branches")
  51 PARSER.add_argument("--skip-on-failure", action="store_true", default=False)
  52 PARSER.add_argument("--dry-run", "-n", action="store_true", default=False)
  53 PARSER.add_argument("--use-branch-if-exists", action="store_true", default=False)
  54 PARSER.add_argument(
  55     "-c",
  56     "--config-file",
  57     action="append",
  58     dest='config_files',
  59     help="Configuration file to use. Can be used multiple times.",
  60     required=False,
  61 )
  62 PARSER.add_argument(
  63     "-g",
  64     "--gitlab",
  65     help=(
  66         "Which configuration section should "
  67         "be used. If not defined, the default selection "
  68         "will be used."
  69     ),
  70     required=False,
  71 )
  72 PARSER.add_argument(
  73     "-m",
  74     "--module",
  75     help="GStreamer module to move MRs for. All if none specified. Can be used multiple times.",
  76     dest='modules',
  77     action="append",
  78     required=False,
  79 )
  80 PARSER.add_argument(
  81     "--mr",
  82     default=None,
  83     type=int,
  84     help=(
  85         "Id of the MR to work on."
  86         " One (and only one) module must be specified with `--module`."
  87     ),
  88     required=False,
  89 )
  90
  91 GST_PROJECTS = [
  92     'gstreamer',
  93     'gst-plugins-base',
  94     'gst-plugins-good',
  95     'gst-plugins-bad',
  96     'gst-plugins-ugly',
  97     'gst-libav',
  98     'gst-rtsp-server',
  99     'gstreamer-vaapi',
 100     'gstreamer-sharp',
 101     'gst-python',
 102     'gst-omx',
 103     'gst-editing-services',
 104     'gst-devtools',
 105     'gst-integration-testsuites',
 106     'gst-docs',
 107     'gst-examples',
 108     'gst-build',
 109     'gst-ci',
 110 ]
 111
 112 # We do not want to deal with LFS
 113 os.environ["GIT_LFS_SKIP_SMUDGE"] = "1"
 114
 115
 116 log_depth = []               # type: T.List[str]
 117
 118 @contextmanager
 119 def nested(name=''):
 120     global log_depth
 121     log_depth.append(name)
 122     try:
 123         yield
 124     finally:
 125         log_depth.pop()
 126
 127 def bold(text: str):
 128     return f"\033[1m{text}\033[0m"
 129
 130 def green(text: str):
 131     return f"\033[1;32m{text}\033[0m"
 132
 133 def red(text: str):
 134     return f"\033[1;31m{text}\033[0m"
 135
 136 def yellow(text: str):
 137     return f"\033[1;33m{text}\033[0m"
 138
 139 def fprint(msg, nested=True):
 140     if log_depth:
 141         prepend = log_depth[-1] + ' | ' if nested else ''
 142     else:
 143         prepend = ''
 144
 145     print(prepend + msg, end="")
 146     sys.stdout.flush()
 147
 148
 149 class GstMRMover:
 150     def __init__(self):
 151
 152         self.modules = []
 153         self.gitlab = None
 154         self.config_files = []
 155         self.gl = None
 156         self.mr = None
 157         self.all_projects = []
 158         self.skipped_branches = []
 159         self.git_rename_limit = None
 160         self.skip_on_failure = None
 161         self.dry_run = False
 162
 163     def connect(self):
 164         fprint("Logging into gitlab...")
 165
 166         if self.gitlab:
 167             gl = gitlab.Gitlab.from_config(self.gitlab, self.config_files)
 168             fprint(f"{green(' OK')}\n", nested=False)
 169             return gl
 170
 171         gitlab_api_token = os.environ.get('GITLAB_API_TOKEN')
 172         if gitlab_api_token:
 173             gl = gitlab.Gitlab(URL, private_token=gitlab_api_token)
 174             fprint(f"{green(' OK')}\n", nested=False)
 175             return gl
 176
 177         session = requests.Session()
 178         sign_in_page = session.get(SIGN_IN_URL).content.decode()
 179         for l in sign_in_page.split('\n'):
 180             m = re.search('name="authenticity_token" value="([^"]+)"', l)
 181             if m:
 182                 break
 183
 184         token = None
 185         if m:
 186             token = m.group(1)
 187
 188         if not token:
 189             fprint(f"{red('Unable to find the authenticity token')}\n")
 190             sys.exit(1)
 191
 192
 193         for data, url in [
 194             ({'user[login]': 'login_or_email',
 195               'user[password]': 'SECRET',
 196               'authenticity_token': token}, LOGIN_URL),
 197             ({'username': 'login_or_email',
 198               'password': 'SECRET',
 199               'authenticity_token': token}, LOGIN_URL_LDAP)]:
 200
 201             r = session.post(url, data=data)
 202             if r.status_code != 200:
 203                 continue
 204
 205             try:
 206                 gl = gitlab.Gitlab(URL, api_version=4, session=session)
 207                 gl.auth()
 208             except gitlab.exceptions.GitlabAuthenticationError as e:
 209                 continue
 210             return gl
 211
 212         sys.exit(bold(f"{red('FAILED')}.\n\nPlease go to:\n\n"
 213             '   https://gitlab.freedesktop.org/-/profile/personal_access_tokens\n\n'
 214             f'and generate a token {bold("with read/write access to all but the registry")},'
 215             ' then set it in the "GITLAB_API_TOKEN" environment variable:"'
 216             f'\n\n  $ GITLAB_API_TOKEN=<your token> {" ".join(sys.argv)}\n'))
 217
 218     def git(self, *args, can_fail=False, interaction_message=None, call=False, revert_operation=None):
 219         cwd = ROOT_DIR
 220         retry = True
 221         while retry:
 222             retry = False
 223             try:
 224                 if not call:
 225                     try:
 226                         return subprocess.check_output(["git"] + list(args), cwd=cwd,
 227                                                     stdin=subprocess.DEVNULL,
 228                                                     stderr=subprocess.STDOUT).decode()
 229                     except:
 230                         if not can_fail:
 231                             fprint(f"\n\n{bold(red('ERROR'))}: `git {' '.join(args)}` failed" + "\n", nested=False)
 232                         raise
 233                 else:
 234                     subprocess.call(["git"] + list(args), cwd=cwd)
 235                     return "All good"
 236             except Exception as e:
 237                 if interaction_message:
 238                     if self.skip_on_failure:
 239                         return "SKIP"
 240                     output = getattr(e, "output", b"")
 241                     if output is not None:
 242                         out = output.decode()
 243                     else:
 244                         out = "????"
 245                     fprint(f"\n```"
 246                           f"\n{out}\n"
 247                           f"Entering a shell in {cwd} to fix:\n\n"
 248                           f" {bold(interaction_message)}\n\n"
 249                           f"You should then exit with the following codes:\n\n"
 250                           f"  - {bold('`exit 0`')}: once you have fixed the problem and we can keep moving the merge request\n"
 251                           f"  - {bold('`exit 1`')}: {bold('retry')}: once you have let the repo in a state where the operation should be to retried\n"
 252                           f"  - {bold('`exit 2`')}: to skip that merge request\n"
 253                           f"  - {bold('`exit 3`')}: stop the script and abandon moving your MRs\n"
 254                           "\n```\n", nested=False)
 255                     try:
 256                         if os.name == 'nt':
 257                             shell = os.environ.get(
 258                                 "COMSPEC", r"C:\WINDOWS\system32\cmd.exe")
 259                         else:
 260                             shell = os.environ.get(
 261                                 "SHELL", os.path.realpath("/bin/sh"))
 262                         subprocess.check_call(shell, cwd=cwd)
 263                     except subprocess.CalledProcessError as e:
 264                         if e.returncode == 1:
 265                             retry = True
 266                             continue
 267                         elif e.returncode == 2:
 268                             if revert_operation:
 269                                 self.git(*revert_operation, can_fail=True)
 270                             return "SKIP"
 271                         elif e.returncode == 3:
 272                             if revert_operation:
 273                                 self.git(*revert_operation, can_fail=True)
 274                             sys.exit(3)
 275                     except:
 276                         # Result of subshell does not really matter
 277                         pass
 278
 279                     return "User fixed it"
 280
 281                 if can_fail:
 282                     return "Failed but we do not care"
 283
 284                 raise e
 285
 286     def cleanup_args(self):
 287         if not self.modules:
 288             if self.mr:
 289                 sys.exit(f"{red(f'Merge request #{self.mr} specified without module')}\n\n"
 290                          f"{bold(' -> Use `--module` to specify which module the MR is from.')}")
 291
 292             self.modules = GST_PROJECTS
 293         else:
 294             VALID_PROJECTS = GST_PROJECTS[1:]
 295             for m in self.modules:
 296                 if m not in VALID_PROJECTS:
 297                     projects = '\n- '.join(VALID_PROJECTS)
 298                     sys.exit(f"{red(f'Unknown module {m}')}\nModules are:\n- {projects}")
 299             if self.mr and len(self.modules) > 1:
 300                 sys.exit(f"{red(f'Merge request #{self.mr} specified but several modules where specified')}\n\n"
 301                          f"{bold(' -> Use `--module` only once to specify an merge request.')}")
 302             self.modules.append(GST_PROJECTS[0])
 303
 304     def run(self):
 305         self.cleanup_args()
 306         self.gl = self.connect()
 307         self.gl.auth()
 308
 309         try:
 310             prevbranch = self.git("rev-parse", "--abbrev-ref", "HEAD", can_fail=True).strip()
 311         except:
 312             fprint(bold(yellow("Not on a branch?\n")), indent=False)
 313             prevbranch = None
 314
 315         try:
 316             self.setup_repo()
 317
 318             from_projects, to_project = self.fetch_projects()
 319
 320             with nested('  '):
 321                 self.move_mrs(from_projects, to_project)
 322         finally:
 323             if self.git_rename_limit is not None:
 324                 self.git("config", "merge.renameLimit", str(self.git_rename_limit))
 325             if prevbranch:
 326                 fprint(f'Back to {prevbranch}\n')
 327                 self.git("checkout", prevbranch)
 328
 329     def fetch_projects(self):
 330         fprint("Fetching projects... ")
 331         self.all_projects = [proj for proj in self.gl.projects.list(
 332             membership=1, all=True) if proj.name in self.modules]
 333
 334         try:
 335             self.user_project, = [p for p in self.all_projects
 336                                     if p.namespace['path'] == self.gl.user.username
 337                                         and p.name == MONOREPO_NAME]
 338         except ValueError:
 339             fprint(f"{red(f'ERROR')}\n\nCould not find repository {self.gl.user.name}/{MONOREPO_NAME}")
 340             fprint(f"{red(f'Got to https://gitlab.freedesktop.org/gstreamer/gstreamer/ and create a fork so we can move your Merge requests.')}")
 341             sys.exit(1)
 342         fprint(f"{green(' OK')}\n", nested=False)
 343
 344         from_projects = [proj for proj in self.all_projects if proj.namespace['path']
 345                          == NAMESPACE and proj.name != "gstreamer"]
 346         fprint(f"\nMoving MRs from:\n")
 347         fprint(f"----------------\n")
 348         for p in from_projects:
 349             fprint(f"  - {bold(p.path_with_namespace)}\n")
 350
 351         to_project, = [p for p in self.all_projects if p.path_with_namespace ==
 352                        MOVING_NAMESPACE + "/gstreamer"]
 353
 354         fprint(f"To: {bold(to_project.path_with_namespace)}\n\n")
 355
 356         return from_projects, to_project
 357
 358     def recreate_mr(self, project, to_project, mr):
 359         branch = f"{project.name}-{mr.source_branch}"
 360         if not self.create_branch_for_mr(branch, project, mr):
 361             return None
 362
 363         description = f"**Copied from {URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}**\n\n{mr.description}"
 364
 365         title = mr.title
 366         if ':' not in mr.title:
 367             title = f"{project.name}: {mr.title}"
 368
 369         new_mr_dict = {
 370             'source_branch': branch,
 371             'allow_collaboration': True,
 372             'remove_source_branch': True,
 373             'target_project_id': to_project.id,
 374             'target_branch': MONOREPO_BRANCH,
 375             'title': title,
 376             'labels': mr.labels,
 377             'description': description,
 378         }
 379
 380         try:
 381             fprint(f"-> Recreating MR '{bold(mr.title)}'...")
 382             if self.dry_run:
 383                 fprint(f"\nDry info:\n{new_mr_dict}\n")
 384             else:
 385                 new_mr = self.user_project.mergerequests.create(new_mr_dict)
 386                 fprint(f"{green(' OK')}\n", nested=False)
 387         except gitlab.exceptions.GitlabCreateError as e:
 388             fprint(f"{yellow('SKIPPED')} (An MR already exists)\n", nested=False)
 389             return None
 390
 391         fprint(f"-> Adding discussings from MR '{mr.title}'...")
 392         if self.dry_run:
 393             fprint(f"{green(' OK')}\n", nested=False)
 394             return None
 395
 396         new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}"
 397         for issue in mr.closes_issues():
 398             obj = {'body': f'Fixing MR moved to: {new_mr_url}'}
 399             issue.discussions.create(obj)
 400
 401         mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}"
 402         for discussion in mr.discussions.list():
 403             # FIXME notes = [n for n in discussion.attributes['notes'] if n['type'] is not None]
 404             notes = [n for n in discussion.attributes['notes']]
 405             if not notes:
 406                 continue
 407
 408             new_discussion = None
 409             for note in notes:
 410                 note = discussion.notes.get(note['id'])
 411
 412                 note_url = f"{mr_url}#note_{note.id}"
 413                 when = dateparse.parse(note.created_at).strftime('on %d, %b %Y')
 414                 body = f"**{note.author['name']} - {PING_SIGN}{note.author['username']} wrote [here]({note_url})** {when}:\n\n"
 415                 body += '\n'.join([l for l in note.body.split('\n')])
 416
 417                 obj = {
 418                     'body': body,
 419                     'type': note.type,
 420                     'resolvable': note.resolvable,
 421                 }
 422
 423                 if new_discussion:
 424                     new_discussion.notes.create(obj)
 425                 else:
 426                     new_discussion = new_mr.discussions.create(obj)
 427
 428                 if not note.resolvable or note.resolved:
 429                     new_discussion.resolved = True
 430                     new_discussion.save()
 431
 432         fprint(f"{green(' OK')}\n", nested=False)
 433
 434         print(f"New MR available at: {bold(new_mr_url)}\n")
 435
 436         return new_mr
 437
 438     def push_branch(self, branch):
 439         fprint(f"-> Pushing branch {branch} to remote {self.gl.user.username}...")
 440         if self.git("push", "--no-verify", self.gl.user.username, branch,
 441                     interaction_message=f"pushing {branch} to {self.gl.user.username} with:\n  "
 442                     f" `$git push {self.gl.user.username} {branch}`") == "SKIP":
 443             fprint(yellow("'SKIPPED' (couldn't push)"), nested=False)
 444
 445             return False
 446
 447         fprint(f"{green(' OK')}\n", nested=False)
 448
 449         return True
 450
 451     def create_branch_for_mr(self, branch, project, mr):
 452         remote_name = project.name + '-' + self.gl.user.username
 453         remote_branch = f"{MONOREPO_REMOTE_NAME}/{MONOREPO_BRANCH}"
 454         if self.use_branch_if_exists:
 455             try:
 456                 self.git("checkout", branch)
 457                 self.git("show", remote_branch + "..", call=True)
 458                 if self.dry_run:
 459                     fprint("Dry run... not creating MR")
 460                     return True
 461                 cont = input('\n     Create MR [y/n]? ')
 462                 if cont.strip().lower() != 'y':
 463                     fprint("Cancelled")
 464                     return False
 465                 return self.push_branch(branch)
 466             except subprocess.CalledProcessError as e:
 467                 pass
 468
 469         self.git("remote", "add", remote_name,
 470                  f"{URL}{self.gl.user.username}/{project.name}.git", can_fail=True)
 471         self.git("fetch", remote_name)
 472
 473         if self.git("checkout", remote_branch, "-b", branch,
 474                     interaction_message=f"checking out branch with `git checkout {remote_branch} -b {branch}`") == "SKIP":
 475             fprint(bold(f"{red('SKIPPED')} (couldn't checkout)\n"), nested=False)
 476             return False
 477
 478         for commit in reversed([c for c in mr.commits()]):
 479             if self.git("cherry-pick", commit.id,
 480                         interaction_message=f"cherry-picking {commit.id} onto {branch} with:\n  "
 481                         f" `$ git cherry-pick {commit.id}`",
 482                         revert_operation=["cherry-pick", "--abort"]) == "SKIP":
 483                 fprint(f"{yellow('SKIPPED')} (couldn't cherry-pick).", nested=False)
 484                 return False
 485
 486         self.git("show", remote_branch + "..", call=True)
 487         if self.dry_run:
 488             fprint("Dry run... not creating MR\n")
 489             return True
 490         cont = input('\n     Create MR [y/n]? ')
 491         if cont.strip().lower() != 'y':
 492             fprint(f"{red('Cancelled')}\n", nested=False)
 493             return False
 494
 495         return self.push_branch(branch)
 496
 497     def move_mrs(self, from_projects, to_project):
 498         failed_mrs = []
 499         found_mr = None
 500         for from_project in from_projects:
 501             with nested(f'{bold(from_project.path_with_namespace)}'):
 502                 fprint(f'Fetching mrs')
 503                 mrs = [mr for mr in from_project.mergerequests.list(
 504                     all=True, author_id=self.gl.user.id) if mr.author['username'] == self.gl.user.username and mr.state == "opened"]
 505                 if not mrs:
 506                     fprint(f"{yellow(' None')}\n", nested=False)
 507                     continue
 508
 509                 fprint(f"{green(' DONE')}\n", nested=False)
 510
 511                 for mr in mrs:
 512                     if self.mr:
 513                         if self.mr != mr.iid:
 514                             continue
 515                         found_mr = True
 516                     fprint(f'Moving {mr.source_branch} "{mr.title}": {URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}... ')
 517                     if mr.source_branch in self.skipped_branches:
 518                         print(f"{yellow('SKIPPED')} (blacklisted branch)")
 519                         failed_mrs.append(
 520                             f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}")
 521                         continue
 522
 523                     with nested(f'{bold(from_project.path_with_namespace)}: {mr.iid}'):
 524                         new_mr = self.recreate_mr(from_project, to_project, mr)
 525                         if not new_mr:
 526                             if not self.dry_run:
 527                                 failed_mrs.append(
 528                                     f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}")
 529                         else:
 530                             fprint(f"{green(' OK')}\n", nested=False)
 531
 532                         self.close_mr(from_project, to_project, mr, new_mr)
 533
 534             fprint(f"\n{yellow('DONE')} with {from_project.path_with_namespace}\n\n", nested=False)
 535
 536         if self.mr and not found_mr:
 537             sys.exit(bold(red(f"\n==> Couldn't find MR {self.mr} in {self.modules[0]}\n")))
 538
 539         for mr in failed_mrs:
 540             fprint(f"Didn't move MR: {mr}\n")
 541
 542     def close_mr(self, project, to_project, mr, new_mr):
 543         if new_mr:
 544             new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}"
 545         else:
 546             new_mr_url = None
 547         mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}"
 548         cont = input(f'\n  Close old MR {mr_url} "{bold(mr.title)}" ? [y/n]')
 549         if cont.strip().lower() != 'y':
 550             fprint(f"{yellow('Not closing old MR')}\n")
 551         else:
 552             obj = None
 553             if new_mr_url:
 554                 obj = {'body': f"Moved to: {new_mr_url}"}
 555             else:
 556                 ret = input(f"Write a comment to add while closing MR {mr.iid} '{bold(mr.title)}':\n\n").strip()
 557                 if ret:
 558                     obj = {'body': ret}
 559
 560             if self.dry_run:
 561                 fprint(f"{bold('Dry run, not closing')}\n", nested=False)
 562             else:
 563                 if obj:
 564                     mr.discussions.create(obj)
 565                 mr.state_event = 'close'
 566                 mr.save()
 567                 fprint(f'Old MR {mr_url} "{bold(mr.title)}" {yellow("CLOSED")}\n')
 568
 569     def setup_repo(self):
 570         fprint(f"Setting up '{bold(ROOT_DIR)}'...")
 571
 572         try:
 573             out = self.git("status", "--porcelain")
 574             if out:
 575                 fprint("\n" + red('Git repository is not clean:') + "\n```\n" + out + "\n```\n")
 576                 sys.exit(1)
 577
 578         except Exception as e:
 579             exit(
 580                 f"Git repository{ROOT_DIR} is not clean. Clean it up before running {sys.argv[0]}\n ({e})")
 581
 582         self.git('remote', 'add', MONOREPO_REMOTE_NAME,
 583                  MONOREPO_REMOTE, can_fail=True)
 584         self.git('fetch', MONOREPO_REMOTE_NAME)
 585
 586         self.git('remote', 'add', self.gl.user.username,
 587                  f"git@gitlab.freedesktop.org:{self.gl.user.username}/gstreamer.git", can_fail=True)
 588         self.git('fetch', self.gl.user.username,
 589                  interaction_message=f"Setup your fork of {URL}gstreamer/gstreamer as remote called {self.gl.user.username}")
 590         fprint(f"{green(' OK')}\n", nested=False)
 591
 592         try:
 593             git_rename_limit = int(self.git("config", "merge.renameLimit"))
 594         except subprocess.CalledProcessError:
 595             git_rename_limit = 0
 596         if int(git_rename_limit) < 999999:
 597             self.git_rename_limit = git_rename_limit
 598             fprint("-> Setting git rename limit to 999999 so we can properly cherry-pick between repos\n")
 599             self.git("config", "merge.renameLimit", "999999")
 600
 601
 602 def main():
 603     mover = GstMRMover()
 604     PARSER.parse_args(namespace=mover)
 605     mover.run()
 606
 607
 608 if __name__ == '__main__':
 609     main()