#1596

- better error handling

grails-app/controllers/au/org/ala/ecodata/MetadataController.groovy

@@ -9,6 +9,7 @@ import org.springframework.web.multipart.MultipartFile
 import static au.org.ala.ecodata.Status.DELETED
 
 class MetadataController {
+ static responseFormats = ['json']
 
  def metadataService, activityService, commonService, projectService, webService
 
@@ -235,17 +236,22 @@ class MetadataController {
  if (file && outputName) {
 
  def data = metadataService.excelWorkbookToMap(file.inputStream, outputName, true)
+ def status = 200
 
  def result
- if (!data) {
- response.status = 400
+ if (data.error) {
+ status = 500
+ data.status = 500
+ result = data
+ }
+ else if (!data.success) {
+ status = 400
  result = [status:400, error:'No data was found that matched the output description identified by the type parameter, please check the template you used to upload the data. ']
  }
  else {
- result = [status: 200, data:data]
+ result = [status: 200, data:data.success]
  }
- render result as JSON
-
+ respond(result, status: status)
  }
  else {
  response.status = 400

grails-app/i18n/messages.properties

@@ -309,4 +309,8 @@ projectAcitivity.attribution={0}. ({1}) {2} dataset download. Retrieved from {3}
 activityForm.invalidIndex=Form template {0} has invalid index fields {1}
 activityForm.latestVersionIsInDraft = Cannot create a new draft of a form when the current version is a draft
 
-user.userHubs.hubId.unique=Each UserHub must have a unique hubId
+user.userHubs.hubId.unique=Each UserHub must have a unique hubId
+
+bulkimport.conversionToObjectError=Error parsing data into an object in serial number {0}
+bulkimport.errorGroupBySerialNumber=Error making nested object from multiple rows in serial number {0}
+bulkimport.conversionToFormSectionError=Error parsing data for form section (output) - {0}

grails-app/services/au/org/ala/ecodata/MetadataService.groovy

@@ -12,6 +12,7 @@ import org.apache.poi.ss.usermodel.Sheet
 import org.apache.poi.ss.usermodel.Workbook
 import org.apache.poi.ss.usermodel.WorkbookFactory
 import org.apache.poi.ss.util.CellReference
+import grails.web.mime.MimeType
 
 import java.text.SimpleDateFormat
 import java.util.zip.ZipEntry
@@ -707,48 +708,71 @@ class MetadataService {
  excelImportService.mapSheet(workbook, config)
  }
 
- List excelWorkbookToMap(InputStream excelWorkbookIn, String activityFormName, Boolean normalise, Integer formVersion = null) {
+ Map excelWorkbookToMap(InputStream excelWorkbookIn, String activityFormName, Boolean normalise, Integer formVersion = null) {
  List result = []
+ List errors = []
  Workbook workbook = WorkbookFactory.create(excelWorkbookIn)
  ActivityForm form = activityFormService.findActivityForm(activityFormName, formVersion)
  form?.sections?.each { FormSection section ->
- String sectionName = section.name
- List model = annotatedOutputDataModel(sectionName)
- String sheetName = XlsExporter.sheetName(sectionName)
- Sheet sheet = workbook.getSheet(sheetName)
- def columnMap = excelImportService.getDataHeaders(sheet)
- def config = [
- sheet:sheetName,
- startRow:2,
- columnMap:columnMap
- ]
- List data = excelImportService.mapSheet(workbook, config)
- List normalisedData = []
- if(normalise) {
- data.collect { Map row ->
- def normalisedRow = [:]
- row.each { cell ->
- excelImportService.convertDotNotationToObject(normalisedRow, cell.key, cell.value)
- excelImportService.removeEmptyObjects(normalisedRow)
+ try {
+ String sectionName = section.name
+ List model = annotatedOutputDataModel(sectionName)
+ String sheetName = XlsExporter.sheetName(sectionName)
+ Sheet sheet = workbook.getSheet(sheetName)
+ def columnMap = excelImportService.getDataHeaders(sheet)
+ def config = [
+ sheet:sheetName,
+ startRow:2,
+ columnMap:columnMap
+ ]
+ List data = excelImportService.mapSheet(workbook, config)
+ List normalisedData = []
+ if(normalise) {
+ data.collect { Map row ->
+ def normalisedRow = [:]
+ try {
+ row.each { cell ->
+ excelImportService.convertDotNotationToObject(normalisedRow, cell.key, cell.value)
+ excelImportService.removeEmptyObjects(normalisedRow)
+ }
+
+ addOutputSpeciesIdToSpeciesData(normalisedRow, model)
+ normalisedData << normalisedRow
+ }
+ catch (Exception ex) {
+ errors << [message: messageSource.getMessage('bulkimport.conversionToObjectError', [row.serial].toArray(), '', Locale.default), error: ex.message]
+ }
  }
+ }
 
- addOutputSpeciesIdToSpeciesData(normalisedRow, model)
- normalisedData << normalisedRow
+
+ List rollUpData = []
+ Map groupedBySerial = normalisedData.groupBy {it[OutputUploadTemplateBuilder.SERIAL_NUMBER_DATA]}
+ groupedBySerial.each { key, List rows ->
+ try {
+ rollUpData << rollUpDataIntoSingleElement(rows, model)
+ }
+ catch (Exception ex) {
+ errors << [message: messageSource.getMessage('bulkimport.errorGroupBySerialNumber', [key].toArray(), '', Locale.default), error: ex.message]
+ }
  }
- }
 
- List rollUpData = []
- Map groupedBySerial = normalisedData.groupBy {it[OutputUploadTemplateBuilder.SERIAL_NUMBER_DATA]}
- groupedBySerial.each { key, List rows ->
- rollUpData << rollUpDataIntoSingleElement(rows, model)
+ result.addAll(rollUpData.collect {
+ [[outputName: activityFormName, data: it]]
+ })
+ }
+ catch (Exception ex) {
+ errors << [message: messageSource.getMessage('bulkimport.conversionToFormSectionError', [section.name].toArray(), '', Locale.default), error: ex.message]
  }
-
- result.addAll(rollUpData.collect {
- [[outputName: activityFormName, data: it]]
- })
  }
 
- result
+ if (errors) {
+ String errorMessage = errors.collect { it.message + " - " + it.error }.join("<br/>")
+ return [error: errorMessage]
+ }
+ else {
+ return [success: result]
+ }
  }
 
  boolean isRowValidNextMemberOfArray(Map row, List models) {

src/test/groovy/au/org/ala/ecodata/MetadataServiceSpec.groovy

@@ -270,7 +270,8 @@ class MetadataServiceSpec extends MongoSpec implements ServiceUnitTest<MetadataS
  def file = new File("src/test/resources/bulk_import_example.xlsx").newInputStream()
 
  when:
- def content = service.excelWorkbookToMap(file, 'form1', true, null)
+ def resp = service.excelWorkbookToMap(file, 'form1', true, null)
+ def content = resp.success
 
  then:
  content.size() == 2