#!/usr/bin/env python3
-from pathlib import Path
from urllib.parse import urlparse
from contextlib import contextmanager
import os
import gitlab
except ModuleNotFoundError:
print("========================================================================", file=sys.stderr)
- print("ERROR: Install python-gitlab with `python3 -m pip install python-gitlab`", file=sys.stderr)
+ print("ERROR: Install python-gitlab with `python3 -m pip install python-gitlab python-dateutil`", file=sys.stderr)
+ print("========================================================================", file=sys.stderr)
+ sys.exit(1)
+
+try:
+ from dateutil import parser as dateparse
+except ModuleNotFoundError:
+ print("========================================================================", file=sys.stderr)
+ print("ERROR: Install dateutil with `python3 -m pip install dateutil`", file=sys.stderr)
print("========================================================================", file=sys.stderr)
sys.exit(1)
import argparse
help="Ignore MRs for branches which match those names.", dest="skipped_branches")
PARSER.add_argument("--skip-on-failure", action="store_true", default=False)
PARSER.add_argument("--dry-run", "-n", action="store_true", default=False)
-PARSER.add_argument("--use-branch-if-exists", action="store_true", default=False)
+PARSER.add_argument("--use-branch-if-exists",
+ action="store_true", default=False)
+PARSER.add_argument("--list-mrs-only", action="store_true", default=False)
+PARSER.add_argument(
+ "-c",
+ "--config-file",
+ action="append",
+ dest='config_files',
+ help="Configuration file to use. Can be used multiple times.",
+ required=False,
+)
+PARSER.add_argument(
+ "-g",
+ "--gitlab",
+ help=(
+ "Which configuration section should "
+ "be used. If not defined, the default selection "
+ "will be used."
+ ),
+ required=False,
+)
+PARSER.add_argument(
+ "-m",
+ "--module",
+ help="GStreamer module to move MRs for. All if none specified. Can be used multiple times.",
+ dest='modules',
+ action="append",
+ required=False,
+)
+PARSER.add_argument(
+ "-mr",
+ "--mr-url",
+ default=None,
+ type=str,
+ help=(
+ "URL of the MR to work on."
+ ),
+ required=False,
+)
GST_PROJECTS = [
'gstreamer',
'gst-omx',
'gst-editing-services',
'gst-devtools',
- 'gst-integration-testsuites',
'gst-docs',
'gst-examples',
'gst-build',
'gst-ci',
]
+GST_PROJECTS_ID = {
+ 'gstreamer': 1357,
+ 'gst-rtsp-server': 1362,
+ 'gstreamer-vaapi': 1359,
+ 'gstreamer-sharp': 1358,
+ 'gst-python': 1355,
+ 'gst-plugins-ugly': 1354,
+ 'gst-plugins-good': 1353,
+ 'gst-plugins-base': 1352,
+ 'gst-plugins-bad': 1351,
+ 'gst-omx': 1350,
+ 'gst-libav': 1349,
+ 'gst-integration-testsuites': 1348,
+ 'gst-examples': 1347,
+ 'gst-editing-services': 1346,
+ 'gst-docs': 1345,
+ 'gst-devtools': 1344,
+ 'gst-ci': 1343,
+ 'gst-build': 1342,
+}
+
# We do not want to deal with LFS
os.environ["GIT_LFS_SKIP_SMUDGE"] = "1"
log_depth = [] # type: T.List[str]
+
@contextmanager
def nested(name=''):
global log_depth
finally:
log_depth.pop()
+
def bold(text: str):
return f"\033[1m{text}\033[0m"
+
def green(text: str):
return f"\033[1;32m{text}\033[0m"
+
def red(text: str):
return f"\033[1;31m{text}\033[0m"
+
def yellow(text: str):
return f"\033[1;33m{text}\033[0m"
+
def fprint(msg, nested=True):
if log_depth:
prepend = log_depth[-1] + ' | ' if nested else ''
class GstMRMover:
def __init__(self):
- self.gl = self.connect()
- self.gl.auth()
+ self.modules = []
+ self.gitlab = None
+ self.config_files = []
+ self.gl = None
+ self.mr = None
+ self.mr_url = None
self.all_projects = []
self.skipped_branches = []
self.git_rename_limit = None
def connect(self):
fprint("Logging into gitlab...")
- gitlab_api_token = os.environ.get('GITLAB_API_TOKEN')
+ if self.gitlab:
+ gl = gitlab.Gitlab.from_config(self.gitlab, self.config_files)
+ fprint(f"{green(' OK')}\n", nested=False)
+ return gl
+
+ gitlab_api_token = os.environ.get('GITLAB_API_TOKEN')
if gitlab_api_token:
gl = gitlab.Gitlab(URL, private_token=gitlab_api_token)
fprint(f"{green(' OK')}\n", nested=False)
session = requests.Session()
sign_in_page = session.get(SIGN_IN_URL).content.decode()
- for l in sign_in_page.split('\n'):
- m = re.search('name="authenticity_token" value="([^"]+)"', l)
+ for line in sign_in_page.split('\n'):
+ m = re.search('name="authenticity_token" value="([^"]+)"', line)
if m:
break
fprint(f"{red('Unable to find the authenticity token')}\n")
sys.exit(1)
-
for data, url in [
({'user[login]': 'login_or_email',
'user[password]': 'SECRET',
return gl
sys.exit(bold(f"{red('FAILED')}.\n\nPlease go to:\n\n"
- ' https://gitlab.freedesktop.org/-/profile/personal_access_tokens\n\n'
- f'and generate a token {bold("with read/write access to all but the registry")},'
- ' then set it in the "GITLAB_API_TOKEN" environment variable:"'
- f'\n\n $ GITLAB_API_TOKEN=<your token> {" ".join(sys.argv)}\n'))
+ ' https://gitlab.freedesktop.org/-/profile/personal_access_tokens\n\n'
+ f'and generate a token {bold("with read/write access to all but the registry")},'
+ ' then set it in the "GITLAB_API_TOKEN" environment variable:"'
+ f'\n\n $ GITLAB_API_TOKEN=<your token> {" ".join(sys.argv)}\n'))
- def git(self, *args, can_fail=False, interaction_message=None, call=False):
+ def git(self, *args, can_fail=False, interaction_message=None, call=False, revert_operation=None):
cwd = ROOT_DIR
retry = True
while retry:
if not call:
try:
return subprocess.check_output(["git"] + list(args), cwd=cwd,
- stdin=subprocess.DEVNULL,
- stderr=subprocess.STDOUT).decode()
- except:
+ stdin=subprocess.DEVNULL,
+ stderr=subprocess.STDOUT).decode()
+ except subprocess.CalledProcessError:
if not can_fail:
- fprint(f"\n\n{bold(red('ERROR'))}: `git {' '.join(args)}` failed" + "\n", nested=False)
+ fprint(
+ f"\n\n{bold(red('ERROR'))}: `git {' '.join(args)}` failed" + "\n", nested=False)
raise
else:
subprocess.call(["git"] + list(args), cwd=cwd)
else:
out = "????"
fprint(f"\n```"
- f"\n{out}\n"
- f"Entering a shell in {cwd} to fix:\n\n"
- f" {bold(interaction_message)}\n\n"
- f"You should then exit with the following codes:\n\n"
- f" - {bold('`exit 0`')}: once you have fixed the problem and we can keep moving the merge request\n"
- f" - {bold('`exit 1`')}: {bold('retry')}: once you have let the repo in a state where the operation should be to retried\n"
- f" - {bold('`exit 2`')}: to skip that merge request\n"
- f" - {bold('`exit 3`')}: stop the script and abandon moving your MRs\n"
- "\n```\n", nested=False)
+ f"\n{out}\n"
+ f"Entering a shell in {cwd} to fix:\n\n"
+ f" {bold(interaction_message)}\n\n"
+ f"You should then exit with the following codes:\n\n"
+ f" - {bold('`exit 0`')}: once you have fixed the problem and we can keep moving the merge request\n"
+ f" - {bold('`exit 1`')}: {bold('retry')}: once you have let the repo in a state where the operation should be to retried\n"
+ f" - {bold('`exit 2`')}: to skip that merge request\n"
+ f" - {bold('`exit 3`')}: stop the script and abandon moving your MRs\n"
+ "\n```\n", nested=False)
try:
if os.name == 'nt':
shell = os.environ.get(
retry = True
continue
elif e.returncode == 2:
+ if revert_operation:
+ self.git(*revert_operation, can_fail=True)
return "SKIP"
elif e.returncode == 3:
+ if revert_operation:
+ self.git(*revert_operation, can_fail=True)
sys.exit(3)
- except:
+ except Exception:
# Result of subshell does not really matter
pass
raise e
+ def cleanup_args(self):
+ if self.mr_url:
+ self.modules.append(GST_PROJECTS[0])
+ (namespace, module, _, _, mr) = os.path.normpath(urlparse(self.mr_url).path).split('/')[1:]
+ self.modules.append(module)
+ self.mr = int(mr)
+ elif not self.modules:
+ if self.mr:
+ sys.exit(f"{red(f'Merge request #{self.mr} specified without module')}\n\n"
+ f"{bold(' -> Use `--module` to specify which module the MR is from.')}")
+
+ self.modules = GST_PROJECTS
+ else:
+ VALID_PROJECTS = GST_PROJECTS[1:]
+ for m in self.modules:
+ if m not in VALID_PROJECTS:
+ projects = '\n- '.join(VALID_PROJECTS)
+ sys.exit(
+ f"{red(f'Unknown module {m}')}\nModules are:\n- {projects}")
+ if self.mr and len(self.modules) > 1:
+ sys.exit(f"{red(f'Merge request #{self.mr} specified but several modules where specified')}\n\n"
+ f"{bold(' -> Use `--module` only once to specify an merge request.')}")
+ self.modules.append(GST_PROJECTS[0])
+
def run(self):
+ self.cleanup_args()
+ self.gl = self.connect()
+ self.gl.auth()
+
+ # Skip pre-commit hooks when migrating. Some users may have a
+ # different version of gnu indent and that can lead to cherry-pick
+ # failing.
+ os.environ["GST_DISABLE_PRE_COMMIT_HOOKS"] = "1"
+
+ try:
+ prevbranch = self.git(
+ "rev-parse", "--abbrev-ref", "HEAD", can_fail=True).strip()
+ except Exception:
+ fprint(bold(yellow("Not on a branch?\n")), indent=False)
+ prevbranch = None
+
try:
self.setup_repo()
self.move_mrs(from_projects, to_project)
finally:
if self.git_rename_limit is not None:
- self.git("config", "merge.renameLimit", str(self.git_rename_limit))
+ self.git("config", "merge.renameLimit",
+ str(self.git_rename_limit))
+ if prevbranch:
+ fprint(f'Back to {prevbranch}\n')
+ self.git("checkout", prevbranch)
def fetch_projects(self):
fprint("Fetching projects... ")
self.all_projects = [proj for proj in self.gl.projects.list(
- membership=1, all=True) if proj.name in GST_PROJECTS]
- self.user_project, = [p for p in self.all_projects if p.namespace['path'] == self.gl.user.username and p.name == MONOREPO_NAME]
+ membership=1, all=True) if proj.name in self.modules]
+
+ try:
+ self.user_project, = [p for p in self.all_projects
+ if p.namespace['path'] == self.gl.user.username
+ and p.name == MONOREPO_NAME]
+ except ValueError:
+ fprint(
+ f"{red(f'ERROR')}\n\nCould not find repository {self.gl.user.name}/{MONOREPO_NAME}")
+ fprint(f"{red(f'Got to https://gitlab.freedesktop.org/gstreamer/gstreamer/ and create a fork so we can move your Merge requests.')}")
+ sys.exit(1)
fprint(f"{green(' OK')}\n", nested=False)
- from_projects = [proj for proj in self.all_projects if proj.namespace['path']
- == NAMESPACE and proj.name != "gstreamer"]
+ from_projects = []
+ user_projects_name = [proj.name for proj in self.all_projects if proj.namespace['path']
+ == self.gl.user.username and proj.name in GST_PROJECTS]
+ for project, id in GST_PROJECTS_ID.items():
+ if project not in user_projects_name or project == 'gstreamer':
+ continue
+
+ projects = [p for p in self.all_projects if p.id == id]
+ if not projects:
+ upstream_project = self.gl.projects.get(id)
+ else:
+ upstream_project, = projects
+ assert project
+
+ from_projects.append(upstream_project)
+
fprint(f"\nMoving MRs from:\n")
fprint(f"----------------\n")
for p in from_projects:
fprint(f" - {bold(p.path_with_namespace)}\n")
- to_project, = [p for p in self.all_projects if p.path_with_namespace ==
- MOVING_NAMESPACE + "/gstreamer"]
-
+ to_project = self.gl.projects.get(GST_PROJECTS_ID['gstreamer'])
fprint(f"To: {bold(to_project.path_with_namespace)}\n\n")
return from_projects, to_project
description = f"**Copied from {URL}/{project.path_with_namespace}/-/merge_requests/{mr.iid}**\n\n{mr.description}"
+ title = mr.title
+ if ':' not in mr.title:
+ title = f"{project.name}: {mr.title}"
+
new_mr_dict = {
'source_branch': branch,
'allow_collaboration': True,
'remove_source_branch': True,
'target_project_id': to_project.id,
'target_branch': MONOREPO_BRANCH,
- 'title': mr.title,
+ 'title': title,
'labels': mr.labels,
'description': description,
}
new_discussion = None
for note in notes:
- note_url = f"{mr_url}#note_{note['id']}"
- body = f"**{note['author']['name']} - {PING_SIGN}{note['author']['username']} wrote [here]({note_url})**:\n\n"
- body += '\n'.join([l for l in note['body'].split('\n')])
+ note = discussion.notes.get(note['id'])
+
+ note_url = f"{mr_url}#note_{note.id}"
+ when = dateparse.parse(
+ note.created_at).strftime('on %d, %b %Y')
+ body = f"**{note.author['name']} - {PING_SIGN}{note.author['username']} wrote [here]({note_url})** {when}:\n\n"
+ body += '\n'.join([line for line in note.body.split('\n')])
+
+ obj = {
+ 'body': body,
+ 'type': note.type,
+ 'resolvable': note.resolvable,
+ }
- obj = {'body': body, 'type': note['type']}
if new_discussion:
new_discussion.notes.create(obj)
else:
new_discussion = new_mr.discussions.create(obj)
+
+ if not note.resolvable or note.resolved:
+ new_discussion.resolved = True
+ new_discussion.save()
+
fprint(f"{green(' OK')}\n", nested=False)
print(f"New MR available at: {bold(new_mr_url)}\n")
return new_mr
def push_branch(self, branch):
- fprint(f"-> Pushing branch {branch} to remote {self.gl.user.username}...")
+ fprint(
+ f"-> Pushing branch {branch} to remote {self.gl.user.username}...")
if self.git("push", "--no-verify", self.gl.user.username, branch,
interaction_message=f"pushing {branch} to {self.gl.user.username} with:\n "
f" `$git push {self.gl.user.username} {branch}`") == "SKIP":
if self.git("checkout", remote_branch, "-b", branch,
interaction_message=f"checking out branch with `git checkout {remote_branch} -b {branch}`") == "SKIP":
- fprint(bold(f"{red('SKIPPED')} (couldn't checkout)\n"), nested=False)
+ fprint(
+ bold(f"{red('SKIPPED')} (couldn't checkout)\n"), nested=False)
return False
+ # unset upstream to avoid to push to main (ie push.default = tracking)
+ self.git("branch", branch, "--unset-upstream")
+
for commit in reversed([c for c in mr.commits()]):
if self.git("cherry-pick", commit.id,
interaction_message=f"cherry-picking {commit.id} onto {branch} with:\n "
- f" `$ git cherry-pick {commit.id}`") == "SKIP":
- fprint(f"{yellow('SKIPPED')} (couldn't cherry-pick).", nested=False)
- self.git("cherry-pick", "--abort", can_fail=True)
+ f" `$ git cherry-pick {commit.id}`",
+ revert_operation=["cherry-pick", "--abort"]) == "SKIP":
+ fprint(
+ f"{yellow('SKIPPED')} (couldn't cherry-pick).", nested=False)
return False
self.git("show", remote_branch + "..", call=True)
def move_mrs(self, from_projects, to_project):
failed_mrs = []
+ found_mr = None
for from_project in from_projects:
with nested(f'{bold(from_project.path_with_namespace)}'):
fprint(f'Fetching mrs')
fprint(f"{green(' DONE')}\n", nested=False)
for mr in mrs:
- fprint(f'Moving {mr.source_branch} "{mr.title}": {URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}... ')
+ if self.mr:
+ if self.mr != mr.iid:
+ continue
+ found_mr = True
+ fprint(
+ f'Moving {mr.source_branch} "{mr.title}": {URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}... ')
if mr.source_branch in self.skipped_branches:
print(f"{yellow('SKIPPED')} (blacklisted branch)")
failed_mrs.append(
f"{URL}{from_project.path_with_namespace}/merge_requests/{mr.iid}")
continue
+ if self.list_mrs_only:
+ fprint("\n"f"List only: {yellow('SKIPPED')}\n")
+ continue
with nested(f'{bold(from_project.path_with_namespace)}: {mr.iid}'):
new_mr = self.recreate_mr(from_project, to_project, mr)
self.close_mr(from_project, to_project, mr, new_mr)
- fprint(f"\n{yellow('DONE')} with {from_project.path_with_namespace}\n\n", nested=False)
+ fprint(
+ f"\n{yellow('DONE')} with {from_project.path_with_namespace}\n\n", nested=False)
+
+ if self.mr and not found_mr:
+ sys.exit(
+ bold(red(f"\n==> Couldn't find MR {self.mr} in {self.modules[0]}\n")))
for mr in failed_mrs:
- print(f"Didn't move MR: {mr}")
+ fprint(f"Didn't move MR: {mr}\n")
def close_mr(self, project, to_project, mr, new_mr):
if new_mr:
if new_mr_url:
obj = {'body': f"Moved to: {new_mr_url}"}
else:
- ret = input(f"Write a comment to add while closing MR {mr.iid} '{bold(mr.title)}':\n\n").strip()
+ ret = input(
+ f"Write a comment to add while closing MR {mr.iid} '{bold(mr.title)}':\n\n").strip()
if ret:
obj = {'body': ret}
mr.discussions.create(obj)
mr.state_event = 'close'
mr.save()
- fprint(f'Old MR {mr_url} "{bold(mr.title)}" {yellow("CLOSED")}\n')
+ fprint(
+ f'Old MR {mr_url} "{bold(mr.title)}" {yellow("CLOSED")}\n')
def setup_repo(self):
fprint(f"Setting up '{bold(ROOT_DIR)}'...")
try:
out = self.git("status", "--porcelain")
if out:
- fprint("\n" + red('Git repository is not clean:') + "\n```\n" + out + "\n```\n")
+ fprint("\n" + red('Git repository is not clean:')
+ + "\n```\n" + out + "\n```\n")
sys.exit(1)
except Exception as e:
git_rename_limit = 0
if int(git_rename_limit) < 999999:
self.git_rename_limit = git_rename_limit
- fprint("-> Setting git rename limit to 999999 so we can properly cherry-pick between repos")
+ fprint(
+ "-> Setting git rename limit to 999999 so we can properly cherry-pick between repos\n")
self.git("config", "merge.renameLimit", "999999")
projects / platform / upstream / gstreamer.git / blobdiff