This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository coselmar. See https://gitlab.nuiton.org/codelutin/coselmar.git commit fdce731043b1840716f3cf526ff78523dbfbee9a Author: Yannick Martel <martel@©odelutin.com> Date: Tue Jun 6 12:05:37 2017 +0200 refs-50 #9206 First draft for error management in Documents Zip import --- .../beans/MassiveDocumentsImportResult.java | 47 +++++++++++++++ .../coselmar/services/v1/DocumentsWebService.java | 65 +++++++++++---------- .../services/v1/DocumentsWebServiceTest.java | 37 +++++++++++- .../src/test/resources/documents_errors.zip | Bin 0 -> 87789 bytes 4 files changed, 116 insertions(+), 33 deletions(-) diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/MassiveDocumentsImportResult.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/MassiveDocumentsImportResult.java new file mode 100644 index 0000000..fff4400 --- /dev/null +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/MassiveDocumentsImportResult.java @@ -0,0 +1,47 @@ +package fr.ifremer.coselmar.beans; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * @author ymartel (martel@codelutin.com) + */ +public class MassiveDocumentsImportResult { + + protected List<String> missingFiles; + protected boolean errorSystem; + protected boolean invalidFile; + + public MassiveDocumentsImportResult() { + this.missingFiles = new ArrayList<>(); + } + + public List<String> getMissingFiles() { + return missingFiles; + } + + public void setMissingFiles(List<String> missingFiles) { + this.missingFiles = missingFiles; + } + + public void addMissingFile(String missingFileName) { + this.missingFiles.add(missingFileName); + } + + public boolean isErrorSystem() { + return errorSystem; + } + + public void setErrorSystem(boolean errorSystem) { + this.errorSystem = errorSystem; + } + + public boolean isInvalidFile() { + return invalidFile; + } + + public void setInvalidFile(boolean invalidFile) { + this.invalidFile = invalidFile; + } +} diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java index 80be99e..6455deb 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java @@ -35,6 +35,7 @@ import fr.ifremer.coselmar.beans.DocumentImportModel; import fr.ifremer.coselmar.beans.DocumentSearchBean; import fr.ifremer.coselmar.beans.DocumentSearchExample; import fr.ifremer.coselmar.beans.FileInfos; +import fr.ifremer.coselmar.beans.MassiveDocumentsImportResult; import fr.ifremer.coselmar.beans.QuestionBean; import fr.ifremer.coselmar.beans.UserBean; import fr.ifremer.coselmar.beans.UserWebToken; @@ -685,7 +686,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { return types; } - public void uploadZipDocuments(UploadFile uploadFile) throws InvalidCredentialException, UnauthorizedException { + public MassiveDocumentsImportResult uploadZipDocuments(UploadFile uploadFile) throws InvalidCredentialException, UnauthorizedException { Preconditions.checkNotNull(uploadFile); @@ -702,24 +703,8 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } throw new UnauthorizedException(message); } - List<String> missingFiles = importFromZip(uploadFile.getFile(), currentUser); + return importFromZip(uploadFile.getFile(), currentUser); - if (missingFiles.isEmpty()) { - // All is ok ! - commit(); - - } else { - // Something wrong happened... rollback, and refresh lucene to avoid new data - rollback(); - AdminWebService adminWebService = getServicesContext().newService(AdminWebService.class); - try { - adminWebService.refreshDocumentsIndex(); - } catch (IOException e) { - if (log.isErrorEnabled()) { - log.error("Unable to refresh Lucene Documents index. Data should be corrupted", e); - } - } - } } //////////////////////////////////////////////////////////////////////////// @@ -980,7 +965,8 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } - protected List<String> importFromZip(File file, CoselmarUser currentUser) { + protected MassiveDocumentsImportResult importFromZip(File file, CoselmarUser currentUser) { + MassiveDocumentsImportResult importResult = new MassiveDocumentsImportResult(); // File should be a Zip ZipFile zipFile; try { @@ -1002,7 +988,9 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { if (log.isErrorEnabled()) { log.error(message, e); } - throw new CoselmarTechnicalException(message, e); + importResult.setInvalidFile(true); + importResult.addMissingFile(DESCRIPTION_CSV_FILE_NAME); + return importResult; // Direct break : cannot continue without this file } // Now, read CSV ... @@ -1023,7 +1011,6 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { throw new CoselmarTechnicalException("Unable to unzip file : error with unzip directory", e); } - List<String> missingFiles = new ArrayList<>(); // ... and read each entries and retrieve associated File try { for (DocumentBean documentBean : importer) { @@ -1031,8 +1018,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { String fileName = documentBean.getFileName(); ZipEntry zipFileEntry = zipFile.getEntry(fileName); if (zipFileEntry == null) { - // TODO ymartel 20170601 : manage errors + regenerate Lucene Index - missingFiles.add(fileName); + importResult.addMissingFile(fileName); } else { try { @@ -1068,8 +1054,8 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { if (log.isErrorEnabled()) { log.error(message, e); } - // TODO ymartel 20170601 : manage errors + regenerate Lucene Index - missingFiles.add(fileName); + importResult.setErrorSystem(true); + importResult.addMissingFile(fileName); } } @@ -1078,6 +1064,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { if (log.isErrorEnabled()) { log.error("Error with CSV file", ire); } + importResult.setErrorSystem(true); throw new CoselmarTechnicalException("Error with CSV file", ire); } finally { // Close importer @@ -1087,19 +1074,37 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { descriptionInputStream.close(); } catch (IOException e) { if (log.isErrorEnabled()) { - log.error("Unable to close 'descriptions.csv' input stream from zip"); + log.error("Unable to close 'descriptions.csv' input stream from zip", e); } } } - if (missingFiles.isEmpty()) { - // Ok, let move all files from temp storage to real one ! + if (!importResult.isErrorSystem() && (importResult.getMissingFiles() == null || importResult.getMissingFiles().isEmpty())) { + // All is ok ! try { + // Ok, let move all files from temp storage to real one ! FileUtils.moveDirectory(new File(zipTempPath), new File(getUserDocumentPath(currentUser))); + commit(); } catch (IOException e) { // Big problem if we can't move files into real folder ! - e.printStackTrace(); + if (log.isErrorEnabled()) { + log.error("Unable to move files from temp folder '%s' to final folder", e); + } + importResult.setErrorSystem(true); + rollback(); + } + + } else { + // Something wrong happened... rollback, and refresh lucene to avoid new data + rollback(); + AdminWebService adminWebService = getServicesContext().newService(AdminWebService.class); + try { + adminWebService.refreshDocumentsIndex(); + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("Unable to refresh Lucene Documents index. Data should be corrupted", e); + } } } - return missingFiles; + return importResult; } } diff --git a/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java b/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java index 5357c1c..ca3562c 100644 --- a/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java +++ b/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java @@ -24,6 +24,7 @@ package fr.ifremer.coselmar.services.v1; * #L% */ +import fr.ifremer.coselmar.beans.MassiveDocumentsImportResult; import fr.ifremer.coselmar.persistence.entity.CoselmarUser; import fr.ifremer.coselmar.persistence.entity.CoselmarUserImpl; import fr.ifremer.coselmar.persistence.entity.CoselmarUserTopiaDao; @@ -80,9 +81,9 @@ public class DocumentsWebServiceTest extends AbstractCoselmarServiceTest { coselmarUser.setFirstname("Jean"); getServiceContext().getPersistenceContext().commit(); - List<String> errors = documentsWebService.importFromZip(zipFile, coselmarUser); - Assert.assertNotNull(errors); - Assert.assertTrue(errors.isEmpty()); + MassiveDocumentsImportResult importResult = documentsWebService.importFromZip(zipFile, coselmarUser); + Assert.assertNotNull(importResult); + Assert.assertTrue(importResult.getMissingFiles().isEmpty()); DocumentTopiaDao documentDao = getServiceContext().getPersistenceContext().getDocumentDao(); List<Document> documents = documentDao.findAll(); Assert.assertEquals(3, documents.size()); @@ -90,4 +91,34 @@ public class DocumentsWebServiceTest extends AbstractCoselmarServiceTest { File oneFile = new File(oneFilePath); Assert.assertTrue(oneFile.exists()); } + + @Test + public void testUploadZipDocumentsWithErrors() throws Exception { + + DocumentsWebService documentsWebService = getServiceContext().newService(DocumentsWebService.class); + InputStream resourceAsStream = this.getClass().getResourceAsStream("/documents_errors.zip"); + Path tempZipPath = Files.createTempFile(null, null); + Files.copy(resourceAsStream, tempZipPath, StandardCopyOption.REPLACE_EXISTING); + resourceAsStream.close(); + File zipFile = tempZipPath.toFile(); + Assert.assertNotNull(zipFile); + Assert.assertTrue(zipFile.exists()); + + // create a test user + CoselmarUserTopiaDao coselmarUserDao = getServiceContext().getPersistenceContext().getCoselmarUserDao(); + CoselmarUser coselmarUser = coselmarUserDao.create(); + coselmarUser.setName("Peuplu"); + coselmarUser.setFirstname("Jean"); + getServiceContext().getPersistenceContext().commit(); + + MassiveDocumentsImportResult importResult = documentsWebService.importFromZip(zipFile, coselmarUser); + Assert.assertNotNull(importResult); + Assert.assertFalse(importResult.getMissingFiles().isEmpty()); + Assert.assertEquals(1, importResult.getMissingFiles().size()); + Assert.assertEquals("Reunion-v0.3.pdf", importResult.getMissingFiles().get(0)); + // Should import have been rollback + DocumentTopiaDao documentDao = getServiceContext().getPersistenceContext().getDocumentDao(); + List<Document> documents = documentDao.findAll(); + Assert.assertEquals(0, documents.size()); + } } diff --git a/coselmar-rest/src/test/resources/documents_errors.zip b/coselmar-rest/src/test/resources/documents_errors.zip new file mode 100644 index 0000000..836cde6 Binary files /dev/null and b/coselmar-rest/src/test/resources/documents_errors.zip differ -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.