3 from urllib.parse import urlparse
4 from contextlib import contextmanager
10 except ModuleNotFoundError:
11 print("========================================================================", file=sys.stderr)
12 print("ERROR: Install python-gitlab with `python3 -m pip install python-gitlab python-dateutil`", file=sys.stderr)
13 print("========================================================================", file=sys.stderr)
17 from dateutil import parser as dateparse
18 except ModuleNotFoundError:
19 print("========================================================================", file=sys.stderr)
20 print("ERROR: Install dateutil with `python3 -m pip install dateutil`", file=sys.stderr)
21 print("========================================================================", file=sys.stderr)
28 ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
30 URL = "https://gitlab.freedesktop.org/"
31 SIGN_IN_URL = URL + 'sign_in'
32 LOGIN_URL = URL + 'users/sign_in'
33 LOGIN_URL_LDAP = URL + '/users/auth/ldapmain/callback'
35 MONOREPO_REMOTE_NAME = 'origin'
36 NAMESPACE = "gstreamer"
37 MONOREPO_NAME = 'gstreamer'
38 MONOREPO_REMOTE = URL + f'{NAMESPACE}/{MONOREPO_NAME}'
39 MONOREPO_BRANCH = 'main'
41 MOVING_NAMESPACE = NAMESPACE
43 PARSER = argparse.ArgumentParser(
44 description="Move merge request from old GStreamer module to the new"
45 "GStreamer 'monorepo'.\n"
46 " All your pending merge requests from all GStreamer modules will"
47 " be moved the the mono repository."
49 PARSER.add_argument("--skip-branch", action="store", nargs="*",
50 help="Ignore MRs for branches which match those names.", dest="skipped_branches")
51 PARSER.add_argument("--skip-on-failure", action="store_true", default=False)
52 PARSER.add_argument("--dry-run", "-n", action="store_true", default=False)
53 PARSER.add_argument("--use-branch-if-exists",
54 action="store_true", default=False)
55 PARSER.add_argument("--list-mrs-only", action="store_true", default=False)
61 help="Configuration file to use. Can be used multiple times.",
68 "Which configuration section should "
69 "be used. If not defined, the default selection "
77 help="GStreamer module to move MRs for. All if none specified. Can be used multiple times.",
88 "URL of the MR to work on."
105 'gst-editing-services',
115 'gst-rtsp-server': 1362,
116 'gstreamer-vaapi': 1359,
117 'gstreamer-sharp': 1358,
119 'gst-plugins-ugly': 1354,
120 'gst-plugins-good': 1353,
121 'gst-plugins-base': 1352,
122 'gst-plugins-bad': 1351,
125 'gst-integration-testsuites': 1348,
126 'gst-examples': 1347,
127 'gst-editing-services': 1346,
129 'gst-devtools': 1344,
134 # We do not want to deal with LFS
135 os.environ["GIT_LFS_SKIP_SMUDGE"] = "1"
138 log_depth = [] # type: T.List[str]
144 log_depth.append(name)
152 return f"\033[1m{text}\033[0m"
155 def green(text: str):
156 return f"\033[1;32m{text}\033[0m"
160 return f"\033[1;31m{text}\033[0m"
163 def yellow(text: str):
164 return f"\033[1;33m{text}\033[0m"
167 def fprint(msg, nested=True):
169 prepend = log_depth[-1] + ' | ' if nested else ''
173 print(prepend + msg, end="")
182 self.config_files = []
186 self.all_projects = []
187 self.skipped_branches = []
188 self.git_rename_limit = None
189 self.skip_on_failure = None
193 fprint("Logging into gitlab...")
196 gl = gitlab.Gitlab.from_config(self.gitlab, self.config_files)
197 fprint(f"{green(' OK')}\n", nested=False)
200 gitlab_api_token = os.environ.get('GITLAB_API_TOKEN')
202 gl = gitlab.Gitlab(URL, private_token=gitlab_api_token)
203 fprint(f"{green(' OK')}\n", nested=False)
206 session = requests.Session()
207 sign_in_page = session.get(SIGN_IN_URL).content.decode()
208 for line in sign_in_page.split('\n'):
209 m = re.search('name="authenticity_token" value="([^"]+)"', line)
218 fprint(f"{red('Unable to find the authenticity token')}\n")
222 ({'user[login]': 'login_or_email',
223 'user[password]': 'SECRET',
224 'authenticity_token': token}, LOGIN_URL),
225 ({'username': 'login_or_email',
226 'password': 'SECRET',
227 'authenticity_token': token}, LOGIN_URL_LDAP)]:
229 r = session.post(url, data=data)
230 if r.status_code != 200:
234 gl = gitlab.Gitlab(URL, api_version=4, session=session)
236 except gitlab.exceptions.GitlabAuthenticationError as e:
240 sys.exit(bold(f"{red('FAILED')}.\n\nPlease go to:\n\n"
241 ' https://gitlab.freedesktop.org/-/profile/personal_access_tokens\n\n'
242 f'and generate a token {bold("with read/write access to all but the registry")},'
243 ' then set it in the "GITLAB_API_TOKEN" environment variable:"'
244 f'\n\n $ GITLAB_API_TOKEN=<your token> {" ".join(sys.argv)}\n'))
246 def git(self, *args, can_fail=False, interaction_message=None, call=False, revert_operation=None):
254 return subprocess.check_output(["git"] + list(args), cwd=cwd,
255 stdin=subprocess.DEVNULL,
256 stderr=subprocess.STDOUT).decode()
257 except subprocess.CalledProcessError:
260 f"\n\n{bold(red('ERROR'))}: `git {' '.join(args)}` failed" + "\n", nested=False)
263 subprocess.call(["git"] + list(args), cwd=cwd)
265 except Exception as e:
266 if interaction_message:
267 if self.skip_on_failure:
269 output = getattr(e, "output", b"")
270 if output is not None:
271 out = output.decode()
276 f"Entering a shell in {cwd} to fix:\n\n"
277 f" {bold(interaction_message)}\n\n"
278 f"You should then exit with the following codes:\n\n"
279 f" - {bold('`exit 0`')}: once you have fixed the problem and we can keep moving the merge request\n"
280 f" - {bold('`exit 1`')}: {bold('retry')}: once you have let the repo in a state where the operation should be to retried\n"
281 f" - {bold('`exit 2`')}: to skip that merge request\n"
282 f" - {bold('`exit 3`')}: stop the script and abandon moving your MRs\n"
283 "\n```\n", nested=False)
286 shell = os.environ.get(
287 "COMSPEC", r"C:\WINDOWS\system32\cmd.exe")
289 shell = os.environ.get(
290 "SHELL", os.path.realpath("/bin/sh"))
291 subprocess.check_call(shell, cwd=cwd)
292 except subprocess.CalledProcessError as e:
293 if e.returncode == 1:
296 elif e.returncode == 2:
298 self.git(*revert_operation, can_fail=True)
300 elif e.returncode == 3:
302 self.git(*revert_operation, can_fail=True)
305 # Result of subshell does not really matter
308 return "User fixed it"
311 return "Failed but we do not care"
315 def cleanup_args(self):
317 self.modules.append(GST_PROJECTS[0])
318 (namespace, module, _, _, mr) = os.path.normpath(urlparse(self.mr_url).path).split('/')[1:]
319 self.modules.append(module)
321 elif not self.modules:
323 sys.exit(f"{red(f'Merge request #{self.mr} specified without module')}\n\n"
324 f"{bold(' -> Use `--module` to specify which module the MR is from.')}")
326 self.modules = GST_PROJECTS
328 VALID_PROJECTS = GST_PROJECTS[1:]
329 for m in self.modules:
330 if m not in VALID_PROJECTS:
331 projects = '\n- '.join(VALID_PROJECTS)
333 f"{red(f'Unknown module {m}')}\nModules are:\n- {projects}")
334 if self.mr and len(self.modules) > 1:
335 sys.exit(f"{red(f'Merge request #{self.mr} specified but several modules where specified')}\n\n"
336 f"{bold(' -> Use `--module` only once to specify an merge request.')}")
337 self.modules.append(GST_PROJECTS[0])
341 self.gl = self.connect()
344 # Skip pre-commit hooks when migrating. Some users may have a
345 # different version of gnu indent and that can lead to cherry-pick
347 os.environ["GST_DISABLE_PRE_COMMIT_HOOKS"] = "1"
350 prevbranch = self.git(
351 "rev-parse", "--abbrev-ref", "HEAD", can_fail=True).strip()
353 fprint(bold(yellow("Not on a branch?\n")), indent=False)
359 from_projects, to_project = self.fetch_projects()
362 self.move_mrs(from_projects, to_project)
364 if self.git_rename_limit is not None:
365 self.git("config", "merge.renameLimit",
366 str(self.git_rename_limit))
368 fprint(f'Back to {prevbranch}\n')
369 self.git("checkout", prevbranch)
371 def fetch_projects(self):
372 fprint("Fetching projects... ")
373 self.all_projects = [proj for proj in self.gl.projects.list(
374 membership=1, all=True) if proj.name in self.modules]
377 self.user_project, = [p for p in self.all_projects
378 if p.namespace['path'] == self.gl.user.username
379 and p.name == MONOREPO_NAME]
382 f"{red(f'ERROR')}\n\nCould not find repository {self.gl.user.name}/{MONOREPO_NAME}")
383 fprint(f"{red(f'Got to https://gitlab.freedesktop.org/gstreamer/gstreamer/ and create a fork so we can move your Merge requests.')}")
385 fprint(f"{green(' OK')}\n", nested=False)
388 user_projects_name = [proj.name for proj in self.all_projects if proj.namespace['path']
389 == self.gl.user.username and proj.name in GST_PROJECTS]
390 for project, id in GST_PROJECTS_ID.items():
391 if project not in user_projects_name or project == 'gstreamer':
394 projects = [p for p in self.all_projects if p.id == id]
396 upstream_project = self.gl.projects.get(id)
398 upstream_project, = projects
401 from_projects.append(upstream_project)
403 fprint(f"\nMoving MRs from:\n")
404 fprint(f"----------------\n")
405 for p in from_projects:
406 fprint(f" - {bold(p.path_with_namespace)}\n")
408 to_project = self.gl.projects.get(GST_PROJECTS_ID['gstreamer'])
409 fprint(f"To: {bold(to_project.path_with_namespace)}\n\n")
411 return from_projects, to_project
413 def recreate_mr(self, project, to_project, mr):
414 branch = f"{project.name}-{mr.source_branch}"
415 if not self.create_branch_for_mr(branch, project, mr):
418 description = f"**Copied from {URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}**\n\n{mr.description}"
421 if ':' not in mr.title:
422 title = f"{project.name}: {mr.title}"
425 'source_branch': branch,
426 'allow_collaboration': True,
427 'remove_source_branch': True,
428 'target_project_id': to_project.id,
429 'target_branch': MONOREPO_BRANCH,
432 'description': description,
436 fprint(f"-> Recreating MR '{bold(mr.title)}'...")
438 fprint(f"\nDry info:\n{new_mr_dict}\n")
440 new_mr = self.user_project.mergerequests.create(new_mr_dict)
441 fprint(f"{green(' OK')}\n", nested=False)
442 except gitlab.exceptions.GitlabCreateError as e:
443 fprint(f"{yellow('SKIPPED')} (An MR already exists)\n", nested=False)
446 fprint(f"-> Adding discussings from MR '{mr.title}'...")
448 fprint(f"{green(' OK')}\n", nested=False)
451 new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}"
452 for issue in mr.closes_issues():
453 obj = {'body': f'Fixing MR moved to: {new_mr_url}'}
454 issue.discussions.create(obj)
456 mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}"
457 for discussion in mr.discussions.list():
458 # FIXME notes = [n for n in discussion.attributes['notes'] if n['type'] is not None]
459 notes = [n for n in discussion.attributes['notes']]
463 new_discussion = None
465 note = discussion.notes.get(note['id'])
467 note_url = f"{mr_url}#note_{note.id}"
468 when = dateparse.parse(
469 note.created_at).strftime('on %d, %b %Y')
470 body = f"**{note.author['name']} - {PING_SIGN}{note.author['username']} wrote [here]({note_url})** {when}:\n\n"
471 body += '\n'.join([line for line in note.body.split('\n')])
476 'resolvable': note.resolvable,
480 new_discussion.notes.create(obj)
482 new_discussion = new_mr.discussions.create(obj)
484 if not note.resolvable or note.resolved:
485 new_discussion.resolved = True
486 new_discussion.save()
488 fprint(f"{green(' OK')}\n", nested=False)
490 print(f"New MR available at: {bold(new_mr_url)}\n")
494 def push_branch(self, branch):
496 f"-> Pushing branch {branch} to remote {self.gl.user.username}...")
497 if self.git("push", "--no-verify", self.gl.user.username, branch,
498 interaction_message=f"pushing {branch} to {self.gl.user.username} with:\n "
499 f" `$git push {self.gl.user.username} {branch}`") == "SKIP":
500 fprint(yellow("'SKIPPED' (couldn't push)"), nested=False)
504 fprint(f"{green(' OK')}\n", nested=False)
508 def create_branch_for_mr(self, branch, project, mr):
509 remote_name = project.name + '-' + self.gl.user.username
510 remote_branch = f"{MONOREPO_REMOTE_NAME}/{MONOREPO_BRANCH}"
511 if self.use_branch_if_exists:
513 self.git("checkout", branch)
514 self.git("show", remote_branch + "..", call=True)
516 fprint("Dry run... not creating MR")
518 cont = input('\n Create MR [y/n]? ')
519 if cont.strip().lower() != 'y':
522 return self.push_branch(branch)
523 except subprocess.CalledProcessError as e:
526 self.git("remote", "add", remote_name,
527 f"{URL}{self.gl.user.username}/{project.name}.git", can_fail=True)
528 self.git("fetch", remote_name)
530 if self.git("checkout", remote_branch, "-b", branch,
531 interaction_message=f"checking out branch with `git checkout {remote_branch} -b {branch}`") == "SKIP":
533 bold(f"{red('SKIPPED')} (couldn't checkout)\n"), nested=False)
536 # unset upstream to avoid to push to main (ie push.default = tracking)
537 self.git("branch", branch, "--unset-upstream")
539 for commit in reversed([c for c in mr.commits()]):
540 if self.git("cherry-pick", commit.id,
541 interaction_message=f"cherry-picking {commit.id} onto {branch} with:\n "
542 f" `$ git cherry-pick {commit.id}`",
543 revert_operation=["cherry-pick", "--abort"]) == "SKIP":
545 f"{yellow('SKIPPED')} (couldn't cherry-pick).", nested=False)
548 self.git("show", remote_branch + "..", call=True)
550 fprint("Dry run... not creating MR\n")
552 cont = input('\n Create MR [y/n]? ')
553 if cont.strip().lower() != 'y':
554 fprint(f"{red('Cancelled')}\n", nested=False)
557 return self.push_branch(branch)
559 def move_mrs(self, from_projects, to_project):
562 for from_project in from_projects:
563 with nested(f'{bold(from_project.path_with_namespace)}'):
564 fprint(f'Fetching mrs')
565 mrs = [mr for mr in from_project.mergerequests.list(
566 all=True, author_id=self.gl.user.id) if mr.author['username'] == self.gl.user.username and mr.state == "opened"]
568 fprint(f"{yellow(' None')}\n", nested=False)
571 fprint(f"{green(' DONE')}\n", nested=False)
575 if self.mr != mr.iid:
579 f'Moving {mr.source_branch} "{mr.title}": {URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}... ')
580 if mr.source_branch in self.skipped_branches:
581 print(f"{yellow('SKIPPED')} (blacklisted branch)")
583 f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}")
585 if self.list_mrs_only:
586 fprint("\n"f"List only: {yellow('SKIPPED')}\n")
589 with nested(f'{bold(from_project.path_with_namespace)}: {mr.iid}'):
590 new_mr = self.recreate_mr(from_project, to_project, mr)
594 f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}")
596 fprint(f"{green(' OK')}\n", nested=False)
598 self.close_mr(from_project, to_project, mr, new_mr)
601 f"\n{yellow('DONE')} with {from_project.path_with_namespace}\n\n", nested=False)
603 if self.mr and not found_mr:
605 bold(red(f"\n==> Couldn't find MR {self.mr} in {self.modules[0]}\n")))
607 for mr in failed_mrs:
608 fprint(f"Didn't move MR: {mr}\n")
610 def close_mr(self, project, to_project, mr, new_mr):
612 new_mr_url = f"{URL}/{to_project.path_with_namespace}/-/merge_requests/{new_mr.iid}"
615 mr_url = f"{URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}"
616 cont = input(f'\n Close old MR {mr_url} "{bold(mr.title)}" ? [y/n]')
617 if cont.strip().lower() != 'y':
618 fprint(f"{yellow('Not closing old MR')}\n")
622 obj = {'body': f"Moved to: {new_mr_url}"}
625 f"Write a comment to add while closing MR {mr.iid} '{bold(mr.title)}':\n\n").strip()
630 fprint(f"{bold('Dry run, not closing')}\n", nested=False)
633 mr.discussions.create(obj)
634 mr.state_event = 'close'
637 f'Old MR {mr_url} "{bold(mr.title)}" {yellow("CLOSED")}\n')
639 def setup_repo(self):
640 fprint(f"Setting up '{bold(ROOT_DIR)}'...")
643 out = self.git("status", "--porcelain")
645 fprint("\n" + red('Git repository is not clean:')
646 + "\n```\n" + out + "\n```\n")
649 except Exception as e:
651 f"Git repository{ROOT_DIR} is not clean. Clean it up before running {sys.argv[0]}\n ({e})")
653 self.git('remote', 'add', MONOREPO_REMOTE_NAME,
654 MONOREPO_REMOTE, can_fail=True)
655 self.git('fetch', MONOREPO_REMOTE_NAME)
657 self.git('remote', 'add', self.gl.user.username,
658 f"git@gitlab.freedesktop.org:{self.gl.user.username}/gstreamer.git", can_fail=True)
659 self.git('fetch', self.gl.user.username,
660 interaction_message=f"Setup your fork of {URL}gstreamer/gstreamer as remote called {self.gl.user.username}")
661 fprint(f"{green(' OK')}\n", nested=False)
664 git_rename_limit = int(self.git("config", "merge.renameLimit"))
665 except subprocess.CalledProcessError:
667 if int(git_rename_limit) < 999999:
668 self.git_rename_limit = git_rename_limit
670 "-> Setting git rename limit to 999999 so we can properly cherry-pick between repos\n")
671 self.git("config", "merge.renameLimit", "999999")
676 PARSER.parse_args(namespace=mover)
680 if __name__ == '__main__':