-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.js
77 lines (66 loc) · 1.88 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
/**
* Gitlab Project Repository Downloader
* @author Rheza Satria
* BTPN Jenius
*/
process.env.NODE_DEBUG='request';
const fs = require('fs');
const mkdirp = require('mkdirp');
const request = require('request');
const async = require('async');
const repoList = require('./repo.json');
const BASE_PATH = process.env.GITLAB_REPO_URL + '/api/v3/';
const PRIVATE_TOKEN = process.env.GITLAB_PRIVATE_TOKEN;
const EXTENSION = process.env.OUT_EXTENSION || 'tar';
const DOWNLOAD_CONCURRENCY = process.env.DOWNLOAD_CONCURRENCY || 1;
const DEST_DIR = process.env.DEST_DIR || 'download';
mkdirp.sync(DEST_DIR, function (err) {
if (err) console.error(err)
else console.log('new directory created!');
});
const URL = `${BASE_PATH}`;
console.log('Creating status file');
const gitlabRequest = request.defaults({
baseUrl: BASE_PATH,
qs: {
private_token: PRIVATE_TOKEN,
},
});
function workerDownloader(task, callback) {
console.log('Begin downloading ' + task.name);
gitlabRequest({
url: `/projects/${task.id}/repository/archive.${EXTENSION}`
})
.on('error', (err) => {
// Keep continuing if error persist, since we may also want to consume other data in queue
console.log(err);
})
.on('response', () => {
callback();
})
.pipe(fs.createWriteStream(`${DEST_DIR}/${task.name}.${EXTENSION}`));
}
gitlabRequest({
url: '/projects',
qs: {
archived: false,
per_page: 999,
order_by: 'name',
sort: 'asc'
},
json: true
}, (err, res, body) => {
if (err) {
throw err;
}
const q = async.queue(workerDownloader, Number(DOWNLOAD_CONCURRENCY));
repoList.map((repo) => {
const foundRepo = body.find((_repo) => repo.name === _repo.name);
if (!foundRepo) {
console.log(`Cannot find ${repo.name} in gitlab repository list`);
}
q.push(foundRepo, function(err) {
console.log(`Finished processing ${foundRepo.name}`);
});
});
});