Merge pull request #502 from Medable/incomplete-export-fix

detect end of export stream

packages/mdctl-cli/lib/env/export.js

@@ -46,7 +46,15 @@ const fs = require('fs'),
  adapter = options.adapter || new ExportFileTreeAdapter(outputDir, streamOptions),
  // eslint-disable-next-line max-len
  lockUnlock = new LockUnlock(outputDir, client.environment.endpoint, client.environment.env),
- memo = {}
+ memo = {},
+ logStream = new Transform({
+ objectMode: true,
+ transform(chunk, encoding, cb) {
+ console.log(`[${new Date().toISOString()}] Exporting ${chunk.key}: ${chunk.name || chunk.id}`)
+ this.push(chunk)
+ cb()
+ }
+ })
 
  let manifestFile,
  inputStream,
@@ -152,15 +160,6 @@ const fs = require('fs'),
  inputStream = options.stream.pipe(ndjson.parse())
  }
 
- const logStream = new Transform({
- objectMode: true,
- transform(chunk, encoding, cb) {
- console.log(`[${new Date().toISOString()}] Exporting ${chunk.key}: ${chunk.name || chunk.id}`)
- this.push(chunk);
- cb();
- }
- })
-
  return new Promise((resolve, reject) => {
  const resultStream = pump(inputStream, streamTransform, logStream, adapter, async(err) => {
 
@@ -176,6 +175,10 @@ const fs = require('fs'),
  return reject(err)
  }
 
+ if (!streamTransform.complete()) {
+ return reject(new Error('Export not complete!'))
+ }
+
  if (options.docs) {
  console.log('Documenting env')
  return Docs.generateDocumentation({

packages/mdctl-core/streams/export_stream.js

@@ -11,6 +11,11 @@ class ExportStream extends Transform {
  objectMode: true
  }, options))
  this.runtimes = []
+ this.completed = false
+ }
+
+ complete() {
+ return this.completed
  }
 
  checkKeys(name) {
@@ -24,6 +29,9 @@ class ExportStream extends Transform {
  } else if (chunk.object === 'fault') {
  callback(Fault.from(chunk))
  } else {
+ if (chunk.object === 'manifest-exports') {
+ this.completed = true
+ }
  if (this.checkKeys(chunk.object)) {
  const section = new ExportSection(chunk, chunk.object)
  this.push(section)