From b5e8592a3713251d62453fc83c4d585e07910437 Mon Sep 17 00:00:00 2001 From: Bojan Markovic Date: Mon, 25 Jun 2018 18:06:21 +0200 Subject: [PATCH] First working release, seems to do everything I know how to --- LICENSE | 21 ++ README.md | 156 +++++++++++++++ index.js | 499 ++++++++++++----------------------------------- package.json | 2 +- src/ajax.js | 54 +++++ src/config.js | 68 +++++++ src/gitlab.js | 124 ++++++++++++ src/jira.js | 66 +++++++ src/transform.js | 67 +++++++ src/util.js | 88 +++++++++ 10 files changed, 771 insertions(+), 374 deletions(-) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 src/ajax.js create mode 100644 src/config.js create mode 100644 src/gitlab.js create mode 100644 src/jira.js create mode 100644 src/transform.js create mode 100644 src/util.js diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5c20e9f --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2018 Bojan Markovic http://bmarkovic.github.io + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f2ac2b1 --- /dev/null +++ b/README.md @@ -0,0 +1,156 @@ +# jira2gitlab + +2018 Bojan Markovic + + +Imports issues from JIRA to Gitlab. + +> Note: Use at your own peril + +## Installation + +It is a Node.js CLI application. It uses modern ES features so the latest LTS +(currently Node 10) is strongly recommended. + +Provided that you already have Node on your system +([otherwise see here](https://nodejs.org/en/download/package-manager/)) +you can simply pull dependencies with: + + $ npm install + +amd you're ready to go. + +## Usage + +The program is started with: + + $ nodw . [go] + +If you can't be bothered to deal with Node (I don't blame you) you can use the +provided binary releases. The usage is similar except taht instead of the line +above you use + + $ jira2gitlab [go] + +If you don't provide the `go` CLI parameter it will not attempt to write to the +gitlab instance but will only dump all the downloaded content into the +`payloads` directory. You can examine the JSON files, and `gitlab-issues.json` +in particular, to see how the posted Gitlab issues will look before posting +them to the Gitlab project. + +If the file `config.json` is not present in the program directory it will exit +with an error and dump a default `config.json` to the filesystem, that you can +then edit with your required data. The configuration is commented below: + +```javascript +{ + "jira": { // JIRA config settings + "url": "http://jira.attlasian.net", // JIRA instance base URL + "project": "PROJ", // JIRA project key / short name + "account": { // JIRA Authentication + "username": "user@example.org", // JIRA username + "password": "1234" // JIRA password + } + }, + "gitlab": { // Gitlab config settings + "url": "https://gitlab.com", // Gitlab base URL + "project": "namespace/project", // Gitlab namespaced project name + "token": "" // Gitlab personal access token + }, + "settings": { // jira2gitlab settings + + "rejectUnauthorized": false, // Set to true to reject self-signed + // certificates for either service + + "ignoreDone": false, // Whether not to upload issues that + // were closed in JIRA. Note that if + // you decide to upload an issue it will + // be opened due to Gitlab API + // limitations + + "sudo": true // Gitlab API sudo. Use this to post + // issues to gitla + } +} +``` + +## Notes + +### User Matching + +The program will try to match users if they exist in both JIRA and Gitlab +instances, by email address. If no match is found the user field, if mandatory +will be infered to belong to the token owner (see: Gitlab Personal Access +Token). + +For this to work fully (issues and comments appearing as if they were created +by the actual JIRA author) all users must exist in both services, with same +email adresses, and the user whose token is used on Gitlab must be a Gitlab +admin. + +Finally, the token must have sudo rights (chosen when generating token), and +the `settings.sudo` config option must be set to `true`. + +### Closed/Fixed issues + +As of this writing there isn't a way to close an issue through the API that is +known to me, nor to relay the clossed state for a newly created issue through +it. + +The `settings.ignoreDone` flag let's you configure whether you want to ignore +the resolved issues (i.e. not copy them at all) or if you want them copied even +if that means that the issue will be open in Gitlab. + +Set the option to `true` if you don't want resolved issues copied over. + +### Gitlab Personal Access Token + +Gitlab API authentication uses a token which you can issue yourself by visiting + + https:///profile/personal_access_tokens + +On that page you need to name the token, tick all the access restriction boxes +and **copy the issued token string somewhere safe**, as AFAIK you cannot see it +again after you close the page (but you can issue another one). + +Ticking all access restriction boxes is important. Unless you are an Admin user +of the Gitlab instance and + +This string is what you need to paste into the `token` key in the config. + +### Debug Mode + +You can get very verbose output about the activities of the script by starting +the script with `DEBUG` environment variable set to a positive numerical value: + + $ DEBUG=1 node . go + +The program uses [chalk](https://github.com/chalk/chalk) to colorize the output. +If you want to preserve the colorized output (for example for future inspection +with something like `less`) use the `FORCE_COLOR` environment variable: + + $ DEBUG=1 FORCE_COLOR=1 node . go | tee jira2gitlab.log + +Then you can replay the logfile with colors using e.g. `less jira2gitlab.log` + +## License + +Licensed under MIT License. See `LICENSE` file in repository root. + +## Acknowledgment / Rationale + +Some of the code is based on the information from the following +[blog post](https://about.gitlab.com/2017/08/21/migrating-your-jira-issues-into-gitlab/), +which, in turn, is originally from +[here](https://medium.com/linagora-engineering/gitlab-rivals-winter-is-here-584eacf1fe9a) + +Unfortunately, some of the original information is somewhat missleading and in +case of Gitlab API in particular, often wrong. The code from this project is +the working version of ideas presented in the post. + +I've also chosen to use [`needle`](https://github.com/tomas/needle) over +`axios` due to it's much terser and simpler support for binary file I/O and the +fact that it doesn't needlessly translate to/from Node `http` idioms and API +that it wraps, making me rely on it's own documentation less (and, as tradition +dictates, almost everyone has made one AJAX lib for Node, and failed to +document it apart from the "look how nice my API is" marketing). diff --git a/index.js b/index.js index 4bf2251..b7abd83 100644 --- a/index.js +++ b/index.js @@ -1,352 +1,54 @@ -const fs = require('fs') -const path = require('path') -const needle = require('needle') const chalk = require('chalk') -const ora = require('ora') const _ = require('lodash') -const print = console.debug.bind(console) +const { print, debug, error, spinner, writeJSON } = require('./src/util') -const defaultConfig = JSON.parse(` -{ - "jira": { - "url": "http://jira.attlasian.net", - "project": "PROJ", - "account": { - "username": "user@example.org", - "password": "1234" - } - }, - "gitlab": { - "url": "https://gitlab.com", - "project": "", - "token": "" - } -} -`) - -// Debug print, but only in debug mode -const debug = ( - (process.env.NODE_ENV && process.env.NODE_ENV.includes('devel')) || - (process.env.DEBUG && process.env.DEBUG > 0) - ) - ? s => print(s) - : Function.prototype // javascript for 'noop' - -const error = s => print(chalk.bold.red(s)) -const info = s => print(chalk.yellow(s)) - -const spinner = ( - (process.env.NODE_ENV && process.env.NODE_ENV.includes('devel')) || - (process.env.DEBUG && process.env.DEBUG > 0) -) - // in debug mode emulate ora API but print debug info - ? { - __txt: '', - start(x) { - this.__txt = x - print(x) - }, - get text() { - return this.__txt - }, - set text(x) { - this.__txt = x - print(x) - }, - succeed() { - print(chalk.green('✔ Success:') + ' ' + this.__txt) - this.__txt = '' - } - } - // in production mode return ora instance - : ora() /** - * Promisified fs.readFile + * GITLAB specific functions */ -const readFile = (filepath, encoding) => new Promise((resolve, reject) => { - debug('Reading file: ' + chalk.cyan(filepath)) - fs.readFile(filepath, encoding, (err, data) => { - if (err) reject(err) - else resolve(data) - }) -}) +const { + getNewUploadUrl, + getGitlabUsers, + searchGitlabProjects, + uploadBinaryToGitlab, + postGitlabIssue, + postGitlabNote +} = require('./src/gitlab') /** - * Promisified fs.readFile + * JIRA specific functions */ -const writeFile = (filepath, data, encoding) => new Promise((resolve, reject) => { - debug('Writing to file: ' + chalk.cyan(filepath)) - fs.writeFile(filepath, data, encoding, (err, data) => { - if (err) reject(err) - else { - debug(chalk.green('Written: ') + chalk.cyan(filepath)) - resolve(data) - } - }) -}) - -// wrapper, to curry JSON reading -const readJSON = async filepath => JSON.parse(String( - await readFile(filepath, 'utf8') -)) - -// wrapper, curries JSON writing -const writeJSON = async (filepath, data) => await writeFile( - filepath, JSON.stringify(data, null, 2), 'utf8' -) - -const jiraIssuesUrl = (jira) => - `${jira.url}/rest/api/2/search?jql=project=${jira.project}%20ORDER%20BY%20id%20ASC&maxResults=1000` - -const jiraAttachementsUrl = (jira, jiraIssue) => - `${jira.url}/rest/api/2/issue/${jiraIssue.id}/?fields=attachment,comment` - -const gitlabUsersUrl = (gitlab) => - `${gitlab.url}/api/v4/users?active=true&search=&per_page=10000` - -const gitlabSearchProjectsUrl = (gitlab) => { - let projectName = gitlab.project.split('/').pop() - return `${gitlab.url}/api/v4/projects/?search=${encodeURIComponent(projectName)}` -} - -const gitlabBinaryUploadUrl = gitlab => - `${gitlab.url}/api/v4/projects/${gitlab.projectId}/uploads` - -const gitlabIssueUrl = gitlab => - `${gitlab.url}/api/v4/projects/${gitlab.projectId}/issues` - -const getNewUploadUrl = (gitlab, upload) => - `${gitlab.url}/${gitlab.project}${upload.url}` - -const jiraToGitlabUser = (jiraUser, gitlabUsers) => - jiraUser - ? _.find(gitlabUsers, { email: jiraUser.emailAddress }) - : null - -const jiraToGitlabIssue = (jiraIssue, jiraAttachments, jiraComments, gitlabUsers) => ({ - title: jiraIssue.fields.summary, - description: `> JIRA issue: ${jiraIssue.key}\n\n${jiraIssue.fields.description}`, - labels: [jiraIssue.fields.issuetype.name, ...( - jiraIssue.fields.fixVersions.length > 0 - ? jiraIssue.fields.fixVersions.map(f => f.name) - : [] - )].join(','), - created_at: jiraIssue.fields.created, - updated_at: jiraIssue.fields.updated, - done: ( - jiraIssue.fields.resolution && - ['Fixed', 'Done', 'Duplicate'].includes(jiraIssue.fields.resolution.name) - ) ? true : false, - assignee: jiraToGitlabUser(jiraIssue.fields.assignee, gitlabUsers), - reporter: jiraToGitlabUser(jiraIssue.fields.reporter, gitlabUsers), - comments: jiraComments.map(jiraComment => ({ - author: jiraToGitlabUser(jiraComment.author, gitlabUsers), - comment: jiraComment.body, - created_at: jiraComment.created - })), - attachments: jiraAttachments.map(jiraAttachment => ({ - author: jiraToGitlabUser(jiraAttachment.author, gitlabUsers), - filename: jiraAttachment.filename, - content: jiraAttachment.content, - created_at: jiraAttachment.created, - mimeType: jiraAttachment.mimeType - })), - jira_key: jiraIssue.key -}) - -const get = async req => { - - debug(chalk.bold.cyan('GET') + ' from ' + chalk.cyan(req.url)) - - let { headers, auth, username, password } = req - let options = { headers, auth, username, password, ...req.options } - - if (!(req.options && req.options.output)) options.parse = true - - let res = await needle('get', req.url, options) - - if (res && res.statusCode === 200) { - debug(chalk.green('GET Success!: ') + chalk.cyan(req.url)) - debug( - 'Got data of type: ' + - chalk.cyan(typeof res.body) + - ' with length: ' + - chalk.cyan(res.body.length)) - } - return res.body -} - -const post = async (req, data) => { - - debug(chalk.magenta('POST') + ' to ' + chalk.cyan(req.url)) - - let { headers, auth, username, password } = req - let options = { headers, auth, username, password, ...req.options } - - let res = await needle('post', req.url, data, options) - - if (res && res.statusCode === 200) { - debug(chalk.violet('POST Success!: ') + chalk.cyan(req.url)) - debug( - 'Got data of type: ' + - chalk.cyan(typeof res.body) + - ' with length: ' + - chalk.cyan(res.body.length)) - } - return res.body -} - -const getJiraIssues = async (jira) => { - let req = { - url: jiraIssuesUrl(jira), - auth: 'basic', - username: jira.account.username, - password: jira.account.password - } - - return get(req) -} - -const getJiraAttachements = async (jira, jiraIssue) => { - let req = { - url: jiraAttachementsUrl(jira, jiraIssue), - auth: 'basic', - username: jira.account.username, - password: jira.account.password - } - - let resp = await get(req) - // TODO: return attachments and comments - return resp -} - -const getBinaryLocalFilename = (key, jiraAttachment) => - 'payloads/' + key + '_' + jiraAttachment.filename - -const getJiraAttachementBinary = async (jira, jiraIssue, jiraAttachment) => { - let req = { - url: jiraAttachment.content, - auth: 'basic', - username: jira.account.username, - password: jira.account.password, - options: { - output: getBinaryLocalFilename(jiraIssue.key, jiraAttachment) - } - } - - let resp = await get(req) - // TODO: return attachments and comments - return resp -} - -const getGitlabUsers = async (gitlab) => { - let req = { - url: gitlabUsersUrl(gitlab), - headers: { - 'PRIVATE-TOKEN': gitlab.token - } - } - - return get(req) -} - -const searchGitlabProjects = async (gitlab) => { - let req = { - url: gitlabSearchProjectsUrl(gitlab), - headers: { - 'PRIVATE-TOKEN': gitlab.token - } - } - - return get(req) -} - -const uploadBinaryToGitlab = async (gitlab, filename, mimeType) => { - debug('Uoloading file: ' + chalk.cyan(filename) + ' mime: ' + chalk.cyan(mimeType)) - let req = { - url: gitlabBinaryUploadUrl(gitlab), - headers: { - 'PRIVATE-TOKEN': gitlab.token - }, - options: { - multipart: true - }, - } - let data = { - file: { - file: filename, - content_type: mimeType - } - } - - return post(req, data) -} - -const postGitlabIssue = async (gitlab, issue) => { - let req = { - url: gitlabIssueUrl(gitlab), - headers: { - 'PRIVATE-TOKEN': gitlab.token - }, - options: { json: true } - } - - let data = issue - - return post(req, data) -} - +const { + getJiraIssues, + getJiraAttachements, + getJiraAttachementBinary, + getBinaryLocalFilename +} = require('./src/jira') /** - * Main script body + * Transformations */ -const main = async () => { - print( - chalk.bold('\njira2gitlab\n\n') + - 'Imports JIRA issues into Gitlab\n\n' + - chalk.bold.cyan(' .. use at your own peril .. \n') - ) - - /** - * GETTING CONFIGURATION - */ - - spinner.start(chalk.yellow(' Getting configuration..')) - const fileConfig = await (() => readJSON('./config.json') - .then(config => { return config }) - .catch(async err => { - // We will complain and write defaults to the - // Filesystem - - error('\nMissing "config.json" configuration file\n') - await writeJSON('config.json', defaultConfig) - info(`We've written "config.json" with the defaults for you.\n`) - print( - `Edit the file with your instance information before running\n` + - `the program again.` - ) - process.exit(1) - }) - )() - - const config = _.merge({}, defaultConfig, fileConfig) - - spinner.succeed() +const { + originalAuthor, + jiraToGitlabIssue, + attachmentLine +} = require('./src/transform') +const { getConfig } = require('./src/config') - /** - * GETTING BASIC INSTANCE DATA - * - * Using Promise.all we'll parallel download - * - * - Gitlab Users - * - Gitlab projects with similar name to given project - * - All Jira issues for the given JIRA project - * - */ +/** + * GETTING BASIC INSTANCE DATA + * + * Using Promise.all we'll parallel download + * + * - Gitlab Users + * - Gitlab projects with similar name to given project + * - All Jira issues for the given JIRA project + * + */ +const getInstanceData = async () => { - spinner.start(chalk.yellow(' Getting base Gitlab and JIRA instance data..')) + const { config } = global let [ gitlabUsers, gitlabProjects, { issues: jiraIssues } ] = await Promise.all([ await getGitlabUsers(config.gitlab), @@ -382,26 +84,28 @@ const main = async () => { // Update spinner with some data stats spinner.text = chalk.yellow(' Getting base Gitlab and JIRA instance data.. ') + chalk.cyan(jiraIssues.length + ' issues') - spinner.succeed() - /** - * GETTING JIRA ISSUE ATTACHMENTS - * - * Using Promise.all we'll parallel download - * - * - Issue attachment and comment metadata - * - All Jira attachment binaries, and store them in the filesystem - * - */ + return { gitlabUsers, gitlabProjects, jiraIssues } +} + +/** + * GETTING JIRA ISSUE ATTACHMENTS + * + * Using Promise.all we'll parallel download + * + * - Issue attachment and comment metadata + * - All Jira attachment binaries, and store them in the filesystem + * + */ +const getJiraAttachments = async (jiraIssues, gitlabUsers) => { + + const { config } = global let attComm = [] let atts = 0 let procAtts = 0 - let comms = 0 let curKey = '' - spinner.start(chalk.yellow(' Getting Jira Issue attachments..')) - // spinner updating local closure let updAtts = (key) => { @@ -426,7 +130,7 @@ const main = async () => { error(`Couldn't find fields for ${jiraIssue.key} on JIRA instance`) attComm.push({issue: jiraIssue.key, attachments: [], comments: []}) - return jiraToGitlabIssue(jiraIssue, [], [], gitlabUsers) + return jiraToGitlabIssue(jiraIssue, [], [], gitlabUsers, config.sudo) } else { @@ -454,7 +158,7 @@ const main = async () => { procAtts += binaries.length updAtts() - return jiraToGitlabIssue(jiraIssue, jiraAttachments, jiraComments, gitlabUsers) + return jiraToGitlabIssue(jiraIssue, jiraAttachments, jiraComments, gitlabUsers, config.sudo) } } @@ -467,11 +171,37 @@ const main = async () => { `binaries from the instance.`) } else { debug( - 'Downloaded '+ chalk.bold.cyan(atts) + ' attachments from project: ' + + 'Downloaded ' + chalk.bold.cyan(atts) + ' attachments from project: ' + chalk.cyan(config.jira.project) ) } + return { attComm, gitlabIssues } +} + +// async main +const main = async () => { + + print( + chalk.bold('\njira2gitlab v0.6.0\n\n') + + 'Imports JIRA issues into Gitlab\n\n' + + chalk.bold.cyan(' .. use at your own peril .. \n') + ) + + // Getting configuration + spinner.start(chalk.yellow(' Getting configuration..')) + const config = await getConfig() + global.config = config + spinner.succeed() + + // getting instance data + spinner.start(chalk.yellow(' Getting base Gitlab and JIRA instance data..')) + let { gitlabUsers, gitlabProjects, jiraIssues } = await getInstanceData() + spinner.succeed() + + // Getting JIRA issue attachments, comments and attached binaries + spinner.start(chalk.yellow(' Getting Jira Issue attachments..')) + let { attComm, gitlabIssues } = await getJiraAttachments(jiraIssues, gitlabUsers) spinner.succeed() /** @@ -480,7 +210,6 @@ const main = async () => { * Using Promise.all we'll parallel save all the data so far to disk * */ - spinner.start( chalk.yellow(' Storing downloaded data to ') + chalk.cyan('./payloads/') @@ -509,21 +238,21 @@ const main = async () => { * */ + let atts = 0 + let procAtts = 0 + let comms = 0 + let procComms = 0 + let curKey = '' let counter = 0 let issueCounter = 0 let gitlabPosts = [] + let postedComments = [] // if 'go' CLI parameter was given, post issues to Gitlab try { if (process.argv[2] === 'go') { - spinner.start(chalk.yellow(' Posting Gitlab issues to Gitlab..')) - - // reset all counter values - atts = 0 - procAtts = 0 - comms = 0 - curKey = '' + spinner.start(chalk.yellow(' Posting issues to Gitlab..')) // spinner updating local closure let updGlab = (key) => { @@ -531,31 +260,40 @@ const main = async () => { if (key) curKey = key spinner.text = ( - chalk.yellow(' Posting Gitlab issues to Gitlab.. Processing ') + - chalk.magenta(curKey + ': ') + + chalk.yellow(' Posting issues to Gitlab.. Key: ') + + chalk.magenta(curKey) + ' ' + + chalk.cyan(issueCounter + '/' + gitlabIssues.length) + + chalk.yellow(' issues ') + chalk.cyan(procAtts + '/' + atts) + - chalk.yellow(' attachments') + chalk.yellow(' atts ') + + chalk.cyan(procComms + '/' + comms) + + chalk.yellow(' comms.') ) } // I had to do it synchronously for the logic to hold - for (let issue of gitlabIssues) { - if (issueCounter > 4) throw new Error('Enough!') + if (issue.done && config.settings.ignoreDone) continue let jiraKey = issue.jira_key let newIssue = _.cloneDeep(issue) delete newIssue.jira_key - if (jiraKey !== 'STAT-4') continue - atts += issue.attachments.length let attachments = [] + let tailDescription = '' if (issue.attachments.length > 0) { + + tailDescription += '\n\n' + + '### Attachments\n\n\n' + + + '|Filename|Uploader|Attachment|\n' + + '|---|---|---|\n' + for (let attachment of issue.attachments) { let attach = _.cloneDeep(attachment) @@ -579,28 +317,45 @@ const main = async () => { procAtts++ updGlab() + if (upload.markdown) tailDescription += attachmentLine(config.gitlab, upload, attachment) + attachments.push(attach) } - } else { - attachments = issue.attachments } + delete newIssue.attachments + + let comments = newIssue.comments + delete newIssue.comments + + comms += (comments && comments instanceof Array) ? comments.length : 0 + updGlab(jiraKey) - newIssue.attachments = _.cloneDeep(attachments) - let issueResp = await postGitlabIssue(config.gitlab, newIssue) - debug(issueResp) + newIssue.description += tailDescription + + let issueResp = await postGitlabIssue(config.gitlab, newIssue, config.sudo) if (issueResp.error) { throw new Error(issueResp.error) } else issueCounter += 1 gitlabPosts.push(newIssue) + let commentsUrl = issueResp['_links']['notes'] + + for (let comment of comments) { + + comment.body = (config.sudo ? originalAuthor(comment.author) : '') + comment.body + let resp = await postGitlabNote(config.gitlab, commentsUrl, comment) + procComms += 1 + if (!resp.error) postedComments.push(resp) + } } await writeJSON('payloads/gitlab-issues.json', gitlabPosts) + await writeJSON('payloads/gitlab-notes.json', postedComments) spinner.succeed() } } catch (err) { @@ -609,13 +364,11 @@ const main = async () => { // rethrow throw err } - - } if ( (process.env.NODE_ENV && process.env.NODE_ENV.includes('devel')) || - (process.env.DEBUG && process.env.DEBUG > 0) + (process.env.DEBUG && Number(process.env.DEBUG) > 0) ) { main() .then() diff --git a/package.json b/package.json index 57d2603..8d88968 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jira2gitlab", - "version": "0.1.0", + "version": "0.6.0", "description": "JIRA to GitLab migration tool", "main": "index.js", "scripts": { diff --git a/src/ajax.js b/src/ajax.js new file mode 100644 index 0000000..ea7d6e9 --- /dev/null +++ b/src/ajax.js @@ -0,0 +1,54 @@ +const needle = require('needle') +const { debug } = require('./util.js') +const chalk = require('chalk') + +const get = async req => { + + debug(chalk.bold.cyan('GET') + ' from ' + chalk.cyan(req.url)) + + let { headers, auth, username, password } = req + let options = { headers, auth, username, password, ...req.options } + + if (!(req.options && req.options.output)) options.parse = true + + let res = await needle('get', req.url, options) + + if (res && res.statusCode === 200) { + debug(chalk.green('GET Success!: ') + chalk.cyan(req.url)) + debug( + 'Got data of type: ' + + chalk.cyan(typeof res.body) + + ' with length: ' + + chalk.cyan(res.body.length)) + } + return res.body +} + +const post = async (req, data) => { + + debug( + chalk.magenta('POST') + ' to ' + chalk.cyan(req.url) + + (req.headers.Sudo ? ' as user ' + chalk.magenta(req.headers.Sudo) : '') + ) + + let { headers, auth, username, password } = req + let options = { headers, auth, username, password, ...req.options } + + let res = await needle('post', req.url, data, options) + + if (res && res.statusCode === 200) { + debug(chalk.violet('POST Success!: ') + chalk.cyan(req.url)) + debug( + 'Got data of type: ' + + chalk.cyan(typeof res.body) + + ' with length: ' + + chalk.cyan(res.body.length)) + } + return res.body +} + +module.exports = { + get, + post, + needle +} diff --git a/src/config.js b/src/config.js new file mode 100644 index 0000000..40895e0 --- /dev/null +++ b/src/config.js @@ -0,0 +1,68 @@ + +const _ = require('lodash') +const { readJSON, writeJSON, error, info, print } = require('./util.js') +const fs = require('fs') + +const defaultConfig = JSON.parse(` +{ + "jira": { + "url": "http://jira.attlasian.net", + "project": "PROJ", + "account": { + "username": "user@example.org", + "password": "1234" + } + }, + "gitlab": { + "url": "https://gitlab.com", + "project": "", + "token": "" + }, + "settings": { + "rejectUnauthorized": false, + "ignoreDone": false, + "sudo": true + } +} +`) + +const getConfig = async () => { + + let fileConfig = await (() => + readJSON('./config.json') + .then(config => { return config }) + .catch(async err => { + // We will complain and write defaults to the + // Filesystem + error('\nMissing "config.json" configuration file\n') + + await writeJSON('config.json', defaultConfig) + await new Promise((resolve, reject) => fs.mkdir('./payloads/', (err) => { + if (err) reject(err) + else resolve() + })) + + info(`We've written "config.json" with the defaults for you.\n`) + print( + `Edit the file with your instance information before running\n` + + `the program again.` + ) + process.exit(1) + }) + )() + + let config = _.merge({}, defaultConfig, fileConfig) + + // globalize settings + config.gitlab.rejectUnauthorized = config.settings.rejectUnauthorized + config.gitlab.sudo = config.settings.sudo + config.jira.rejectUnauthorized = config.settings.rejectUnauthorized + + return config + +} + +module.exports = { + defaultConfig, + getConfig +} diff --git a/src/gitlab.js b/src/gitlab.js new file mode 100644 index 0000000..527e83c --- /dev/null +++ b/src/gitlab.js @@ -0,0 +1,124 @@ +const chalk = require('chalk') +const { get, post } = require('./ajax') +const { debug } = require('./util') + +const gitlabUsersUrl = (gitlab) => + `${gitlab.url}/api/v4/users?active=true&search=&per_page=10000` + +const gitlabSearchProjectsUrl = (gitlab) => { + let projectName = gitlab.project.split('/').pop() + return `${gitlab.url}/api/v4/projects/?search=${encodeURIComponent(projectName)}` +} + +const gitlabBinaryUploadUrl = gitlab => + `${gitlab.url}/api/v4/projects/${gitlab.projectId}/uploads` + +const gitlabIssueUrl = gitlab => + `${gitlab.url}/api/v4/projects/${gitlab.projectId}/issues` + +const getNewUploadUrl = (gitlab, upload) => + `${gitlab.url}/${gitlab.project}${upload.url}` + +const getGitlabUsers = async (gitlab) => { + let req = { + url: gitlabUsersUrl(gitlab), + headers: { + 'PRIVATE-TOKEN': gitlab.token + }, + options: { + rejectUnauthorized: gitlab.rejectUnauthorized + } + } + + return get(req) +} + +const searchGitlabProjects = async (gitlab) => { + let req = { + url: gitlabSearchProjectsUrl(gitlab), + headers: { + 'PRIVATE-TOKEN': gitlab.token + }, + options: { + rejectUnauthorized: gitlab.rejectUnauthorized + } + } + + return get(req) +} + +const uploadBinaryToGitlab = async (gitlab, filename, mimeType) => { + debug('Uoloading file: ' + chalk.cyan(filename) + ' mime: ' + chalk.cyan(mimeType)) + let req = { + url: gitlabBinaryUploadUrl(gitlab), + headers: { + 'PRIVATE-TOKEN': gitlab.token + }, + options: { + multipart: true, + rejectUnauthorized: gitlab.rejectUnauthorized + } + } + let data = { + file: { + file: filename, + content_type: mimeType + } + } + + return post(req, data) +} + +const postGitlabIssue = async (gitlab, issue) => { + let req = { + url: gitlabIssueUrl(gitlab), + headers: { + 'PRIVATE-TOKEN': gitlab.token + }, + options: { + json: true, + rejectUnauthorized: gitlab.rejectUnauthorized + } + } + + if (gitlab.sudo && issue.author.username) { + req.headers['Sudo'] = issue.author.username + } + + let data = issue + + return post(req, data) +} + +const postGitlabNote = async (gitlab, url, note) => { + let req = { + url, + headers: { + 'PRIVATE-TOKEN': gitlab.token + }, + options: { + json: true, + rejectUnauthorized: gitlab.rejectUnauthorized + } + } + + if (gitlab.sudo && note.author.username) { + req.headers['Sudo'] = note.author.username + } + + let data = note + + return post(req, data) +} + +module.exports = { + gitlabUsersUrl, + gitlabSearchProjectsUrl, + gitlabBinaryUploadUrl, + getNewUploadUrl, + getGitlabUsers, + searchGitlabProjects, + uploadBinaryToGitlab, + postGitlabIssue, + postGitlabNote +} diff --git a/src/jira.js b/src/jira.js new file mode 100644 index 0000000..018af19 --- /dev/null +++ b/src/jira.js @@ -0,0 +1,66 @@ +const { get } = require('./ajax') + +const jiraIssuesUrl = (jira) => + `${jira.url}/rest/api/2/search?jql=project=${jira.project}%20ORDER%20BY%20id%20ASC&maxResults=1000` + +const jiraAttachementsUrl = (jira, jiraIssue) => + `${jira.url}/rest/api/2/issue/${jiraIssue.id}/?fields=attachment,comment` + +const getJiraIssues = async (jira) => { + let req = { + url: jiraIssuesUrl(jira), + auth: 'basic', + username: jira.account.username, + password: jira.account.password, + options: { + rejectUnauthorized: jira.rejectUnauthorized + } + } + + return get(req) +} + +const getBinaryLocalFilename = (key, jiraAttachment) => + 'payloads/' + key + '_' + jiraAttachment.filename + +const getJiraAttachements = async (jira, jiraIssue) => { + let req = { + url: jiraAttachementsUrl(jira, jiraIssue), + auth: 'basic', + username: jira.account.username, + password: jira.account.password, + options: { + rejectUnauthorized: jira.rejectUnauthorized + } + } + + let resp = await get(req) + // TODO: return attachments and comments + return resp +} + +const getJiraAttachementBinary = async (jira, jiraIssue, jiraAttachment) => { + let req = { + url: jiraAttachment.content, + auth: 'basic', + username: jira.account.username, + password: jira.account.password, + options: { + output: getBinaryLocalFilename(jiraIssue.key, jiraAttachment), + rejectUnauthorized: jira.rejectUnauthorized + } + } + + let resp = await get(req) + // TODO: return attachments and comments + return resp +} + +module.exports = { + jiraIssuesUrl, + jiraAttachementsUrl, + getJiraIssues, + getJiraAttachements, + getJiraAttachementBinary, + getBinaryLocalFilename +} diff --git a/src/transform.js b/src/transform.js new file mode 100644 index 0000000..35c2cae --- /dev/null +++ b/src/transform.js @@ -0,0 +1,67 @@ +const _ = require('lodash') +const { getNewUploadUrl } = require('./gitlab') + +const jiraToGitlabUser = (jiraUser, gitlabUsers) => +jiraUser + ? _.find(gitlabUsers, { email: jiraUser.emailAddress }) || { id: null } + : { id: null } + +const userSnippet = user => `@${ + (user && user.id) ? user.username : 'Unknown' + } ${ + (user && user.id) ? '"' + user.name + '"' : '' + }` + +const originalAuthor = author => `> Originally by ${userSnippet(author)}\n\n` + +const attachmentLine = (gitlab, upload, attachment) => + `|${ + upload.url.split('/').pop() + }|${ + userSnippet(attachment.author + )}|${ + upload.markdown.startsWith('!') + ? '' + : upload.markdown + }|\n` + +const jiraToGitlabIssue = (jiraIssue, jiraAttachments, jiraComments, gitlabUsers, sudo) => ({ + title: jiraIssue.fields.summary, + description: `> JIRA issue: ${jiraIssue.key}\n\n${ + sudo ? '' : originalAuthor(jiraToGitlabUser(jiraIssue.fields.reporter, gitlabUsers)) + }${jiraIssue.fields.description}`, + labels: [jiraIssue.fields.issuetype.name, ...( + jiraIssue.fields.fixVersions.length > 0 + ? jiraIssue.fields.fixVersions.map(f => f.name) + : [] + )].join(','), + created_at: jiraIssue.fields.created, + updated_at: jiraIssue.fields.updated, + done: ( + jiraIssue.fields.resolution && + ['Fixed', 'Done', 'Duplicate'].includes(jiraIssue.fields.resolution.name) + ) ? true : false, + assignee: jiraToGitlabUser(jiraIssue.fields.assignee, gitlabUsers), + author: jiraToGitlabUser(jiraIssue.fields.reporter, gitlabUsers), + comments: jiraComments.map(jiraComment => ({ + author: jiraToGitlabUser(jiraComment.author, gitlabUsers), + body: jiraComment.body, + created_at: jiraComment.created + })), + attachments: jiraAttachments.map(jiraAttachment => ({ + author: jiraToGitlabUser(jiraAttachment.author, gitlabUsers), + filename: jiraAttachment.filename, + content: jiraAttachment.content, + created_at: jiraAttachment.created, + mimeType: jiraAttachment.mimeType + })), + jira_key: jiraIssue.key + }) + + module.exports = { + jiraToGitlabUser, + userSnippet, + originalAuthor, + jiraToGitlabIssue, + attachmentLine +} diff --git a/src/util.js b/src/util.js new file mode 100644 index 0000000..c36bbf8 --- /dev/null +++ b/src/util.js @@ -0,0 +1,88 @@ +const fs = require('fs') +const chalk = require('chalk') +const ora = require('ora') + +const print = console.debug.bind(console) + +// Debug print, but only in debug mode +const debug = ( + (process.env.NODE_ENV && process.env.NODE_ENV.includes('devel')) || + (process.env.DEBUG && Number(process.env.DEBUG) > 0) + ) + ? s => print(s) + : Function.prototype // javascript for 'noop' + +const error = s => print(chalk.bold.red(s)) +const info = s => print(chalk.yellow(s)) + +const spinner = ( + (process.env.NODE_ENV && process.env.NODE_ENV.includes('devel')) || + (process.env.DEBUG && Number(process.env.DEBUG) > 0) +) + // in debug mode emulate ora API but print debug info + ? { + __txt: '', + start(x) { + this.__txt = x + print(x) + }, + get text() { + return this.__txt + }, + set text(x) { + this.__txt = x + print(x) + }, + succeed() { + print(chalk.green('✔ Success:') + ' ' + this.__txt) + this.__txt = '' + } + } + // in production mode return ora instance + : ora() +/** + * Promisified fs.readFile + */ +const readFile = (filepath, encoding) => new Promise((resolve, reject) => { + debug('Reading file: ' + chalk.cyan(filepath)) + fs.readFile(filepath, encoding, (err, data) => { + if (err) reject(err) + else resolve(data) + }) +}) + +/** + * Promisified fs.readFile + */ +const writeFile = (filepath, data, encoding) => new Promise((resolve, reject) => { + debug('Writing to file: ' + chalk.cyan(filepath)) + fs.writeFile(filepath, data, encoding, (err, data) => { + if (err) reject(err) + else { + debug(chalk.green('Written: ') + chalk.cyan(filepath)) + resolve(data) + } + }) +}) + +// wrapper, to curry JSON reading +const readJSON = async filepath => JSON.parse(String( + await readFile(filepath, 'utf8') +)) + +// wrapper, curries JSON writing +const writeJSON = async (filepath, data) => await writeFile( + filepath, JSON.stringify(data, null, 2), 'utf8' +) + +module.exports = { + print, + debug, + error, + info, + spinner, + readFile, + writeFile, + readJSON, + writeJSON +}