#!/usr/bin/env python3 import argparse from datetime import datetime import json import os import requests import sys from urllib3.util.retry import Retry GITLAB_API = 'https://gitlab.com/api/v4' GITLAB_API_UPLOAD = GITLAB_API + '/projects/{repo}/uploads' GITLAB_API_ISSUE_CREATE = GITLAB_API + '/projects/{repo}/issues' GITLAB_API_ISSUE_UPDATE = GITLAB_API + '/projects/{repo}/issues/{issue}' GITLAB_API_ISSUE_NOTE = GITLAB_API + '/projects/{repo}/issues/{issue}/notes' GITLAB_API_ISSUE_UNSCRIBE = GITLAB_API + '/projects/{repo}/issues/{issue}/unsubscribe' GITLAB_ISSUE = 'https://gitlab.com/{repo}/issues/{issue}' SF_ISSUE = 'https://sourceforge.net{url}{issue}' SF_USER = 'https://sourceforge.net/u/{user}' TEMPLATE_USER = '[{user}](%s)' % SF_USER ATTACHMENT_TEMPLATE = '![{file}]({url})' ATTACHMENTS = '\n\n**Attachments**: \n' DESCRIPTION_TEMPLATE_AUTHOR = '*Originally reported by:* {author}\n\n{description}' DESCRIPTION_TEMPLATE_URL = '{description}\n\n---\n*Original URL*: {url}' COMMENT_TEMPLATE_AUTHOR = '*Originally posted by:* {author}\n\n{text}' def get_parser(): parser = argparse.ArgumentParser( description='Migrate LilyPond issues to GitLab', ) parser.add_argument( '--token', help=( 'GitLab token, get it at ' + 'https://gitlab.com/profile/personal_access_tokens' ), required=True, ) parser.add_argument( '--repo', help='GitLab repository where to import', required=True ) parser.add_argument( 'export', help='Directory with export from SourceForge' ) return parser parser = get_parser() args = parser.parse_args(sys.argv[1:]) api_repo = args.repo.replace('/', '%2F') export_dir = args.export json_file = os.path.join(export_dir, 'issues.json') if not os.path.isfile(json_file): print('No export found in `%s`' % export) sys.exit(1) with open(json_file, 'r') as f: issues = json.load(f) tickets = issues['tickets'] sf_url = issues['tracker_config']['options']['url'] # issues['milestones'] is empty issues_custom_fields = issues['custom_fields'] open_status_names = issues['open_status_names'].split(' ') closed_status_names = issues['closed_status_names'].split(' ') # issues['saved_bins'] contains our pre-defined search queries api = requests.Session() # Supply token api.headers = {'Private-Token': args.token} # Retry on "Internal Server Error" retries = Retry( method_whitelist=['POST','PUT'], status_forcelist=[500], ) adapter = requests.adapters.HTTPAdapter(max_retries=retries) api.mount(GITLAB_API, adapter) def get_ticket_num(ticket): return ticket['ticket_num'] def is_author(author): return author not in ['*anonymous', 'googleimporter'] def process_body(body): # Normalize line breaks. body = body.replace('\r\n', '\n') # Remove useless information. body = body.replace('*Originally created by:* *anonymous\n\n', '') # Put two blanks before \n, are rendered as line break in GitLab. body = body.replace('\n', ' \n') return body def process_datetime(created_at): created_at = datetime.fromisoformat(created_at) created_at = created_at.isoformat() + 'Z' return created_at def handle_error(r, expected): if r.status_code == expected: return print(r.status_code, expected) print(r.text) sys.exit(1) def upload_file(path): print('>> upload_file(\'%s\')' % path) path = os.path.join(export_dir, path) with open(path, 'rb') as f: files = {'file': f} upload_file_url = GITLAB_API_UPLOAD.format( repo=api_repo, ) r = api.post(upload_file_url, files=files) handle_error(r, 201) r = json.loads(r.text) return ATTACHMENT_TEMPLATE.format( file=os.path.basename(path), url=r['url'], ) def process_attachments(attachments): if len(attachments) == 0: return '' attachments_string = ATTACHMENTS for a in attachments: attachments_string += upload_file(a['path']) + ' ' return attachments_string def create_issue(issue, ticket): print('> create_issue(%d)' % issue) title = ticket['summary'] if title == '': title = '-- no title --' author = ticket['reported_by'] description = process_body(ticket['description']) created_at = process_datetime(ticket['created_date']) if is_author(author): description = DESCRIPTION_TEMPLATE_AUTHOR.format( author=TEMPLATE_USER.format(user=author), description=description, ) description = DESCRIPTION_TEMPLATE_URL.format( description=description, url=SF_ISSUE.format(url=sf_url, issue=issue), ) attachments = process_attachments(ticket['attachments']) description = description + attachments attributes = { 'iid': issue, 'title': title, 'description': description, 'created_at': created_at, } create_issue_url = GITLAB_API_ISSUE_CREATE.format( repo=api_repo, ) r = api.post(create_issue_url, data=attributes) handle_error(r, 201) def create_comment(issue, post): print('> create_comment({issue}, {timestamp})'.format( issue=issue, timestamp=post['timestamp'], )) author = post['author'] text = process_body(post['text']) created_at = process_datetime(post['timestamp']) if is_author(author): text = COMMENT_TEMPLATE_AUTHOR.format( author=TEMPLATE_USER.format(user=author), text=text, ) attachments = process_attachments(post['attachments']) text = text + attachments attributes = { 'body': text, 'created_at': created_at, } create_issue_note_url = GITLAB_API_ISSUE_NOTE.format( repo=api_repo, issue=issue, ) r = api.post(create_issue_note_url, data=attributes) handle_error(r, 201) def update_issue_status(issue, ticket): print('> update_issue_status(%d)' % issue) updated_at = process_datetime(ticket['mod_date']) labels = ticket['labels'] status = ticket['status'] custom_fields = ticket['custom_fields'] labels += [ 'Status: {status}'.format(status=status) ] for f in issues_custom_fields: value = custom_fields.get(f['name'], '').strip() if value != '': labels += [ '{label}: {value}'.format( label=f['label'], value=value )] attributes = { 'labels': ','.join(labels), 'updated_at': updated_at, } if status in closed_status_names: attributes['state_event'] = 'close' update_issue_url = GITLAB_API_ISSUE_UPDATE.format( repo=api_repo, issue=issue, ) r = api.put(update_issue_url, data=attributes) handle_error(r, 200) def unscribe_issue(issue): print('> unscribe_issue(%d)' % issue) unscribe_url = GITLAB_API_ISSUE_UNSCRIBE.format( repo=api_repo, issue=issue, ) r = api.post(unscribe_url) handle_error(r, 201) def migrate_issue(issue, ticket): print('URL: %s' % GITLAB_ISSUE.format(repo=args.repo, issue=issue)) create_issue(issue, ticket) for post in ticket['discussion_thread']['posts']: create_comment(issue, post) update_issue_status(issue, ticket) unscribe_issue(issue) tickets.sort(key=get_ticket_num) for idx, t in enumerate(tickets): issue = get_ticket_num(t) print('Migrating issue #{issue}, idx {idx} ...'.format( issue=issue, idx=idx, )) migrate_issue(issue, t) print('') print('DONE')