This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository coselmar. See https://gitlab.nuiton.org/codelutin/coselmar.git commit dc2afeb93523010de541f4cba659d38420b1a51a Author: Yannick Martel <martel@©odelutin.com> Date: Thu Jun 1 14:29:22 2017 +0200 Prepare import massif de document --- .../fr/ifremer/coselmar/beans/DocumentBean.java | 6 + .../coselmar/beans/DocumentImportModel.java | 44 +++++ .../java/fr/ifremer/coselmar/beans/FileInfos.java | 35 ++++ .../coselmar/converter/BeanEntityConverter.java | 4 +- .../coselmar/services/indexation/TikaUtils.java | 1 + .../coselmar/services/v1/AdminWebService.java | 10 +- .../coselmar/services/v1/DocumentsWebService.java | 216 ++++++++++++--------- 7 files changed, 215 insertions(+), 101 deletions(-) diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java index f8c855f..cc724c8 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentBean.java @@ -64,6 +64,12 @@ public class DocumentBean implements Serializable { // If restricted, could have a list of restricted user protected Set<UserBean> authorizedUsers; + public static DocumentBean newEmptyInstance() { + return new DocumentBean(); + } + + private DocumentBean(){} + public DocumentBean(String id, String name, String ownerName, String ownerId, String privacy, Date depositDate, Collection<String> keywords, String type, String summary, String language, Date publicationDate, diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java new file mode 100644 index 0000000..f8b5749 --- /dev/null +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/DocumentImportModel.java @@ -0,0 +1,44 @@ +package fr.ifremer.coselmar.beans; + +import fr.ifremer.coselmar.persistence.entity.Privacy; +import org.nuiton.csv.ValueParser; +import org.nuiton.csv.ext.AbstractImportModel; + +import java.text.ParseException; + +/** + * @author ymartel (martel@codelutin.com) + */ +public class DocumentImportModel extends AbstractImportModel<DocumentBean> { + + protected static final ValueParser<Privacy> DOCUMENT_PRIVACY_PARSER = new ValueParser<Privacy>() { + @Override + public Privacy parse(String value) throws ParseException { + return Privacy.valueOf(value.toUpperCase()); + } + }; + + public DocumentImportModel() { + super(';'); + + newMandatoryColumn("name", "name"); + newMandatoryColumn("type", "type"); + newMandatoryColumn("privacy", "privacy", DOCUMENT_PRIVACY_PARSER); + newMandatoryColumn("keywords", "keywords"); + newOptionalColumn("authors", "authors"); + newMandatoryColumn("summary", "summary"); + newOptionalColumn("license", "license"); + newOptionalColumn("copyright", "copyright"); + newOptionalColumn("language", "language"); + newOptionalColumn("publicationDate", "publicationDate"); + newOptionalColumn("comment", "comment"); + newMandatoryColumn("fileName", "fileName"); + newMandatoryColumn("citation", "citation"); + } + + @Override + public DocumentBean newEmptyInstance() { + return DocumentBean.newEmptyInstance(); + } + +} diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/FileInfos.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/FileInfos.java new file mode 100644 index 0000000..5bebc04 --- /dev/null +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/FileInfos.java @@ -0,0 +1,35 @@ +package fr.ifremer.coselmar.beans; + +/** + * @author ymartel (martel@codelutin.com) + */ +public class FileInfos { + + protected String FileName; + protected String FilePath; + protected String MimeType; + + public String getFileName() { + return FileName; + } + + public void setFileName(String fileName) { + FileName = fileName; + } + + public String getFilePath() { + return FilePath; + } + + public void setFilePath(String filePath) { + FilePath = filePath; + } + + public String getMimeType() { + return MimeType; + } + + public void setMimeType(String mimeType) { + MimeType = mimeType; + } +} diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/converter/BeanEntityConverter.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/converter/BeanEntityConverter.java index 9af8254..b8d8e2a 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/converter/BeanEntityConverter.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/converter/BeanEntityConverter.java @@ -309,7 +309,7 @@ public class BeanEntityConverter { * </ul> * If documentBean is null, simply return null Document. */ - public static Document fromBean(DocumentBean documentBean) { + public static Document fromSearchBean(DocumentBean documentBean) { if (documentBean == null) { return null; } @@ -337,7 +337,7 @@ public class BeanEntityConverter { String privacy = documentBean.getPrivacy(); if (StringUtils.isNotBlank(privacy)) { - document.setPrivacy(Privacy.valueOf(privacy)); + document.setPrivacy(Privacy.valueOf(privacy.toUpperCase())); } Date publicationDate = documentBean.getPublicationDate(); diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java index 4a517a4..68e9ca4 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/indexation/TikaUtils.java @@ -51,4 +51,5 @@ public class TikaUtils { } return fileContent; } + } diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java index c4ad5f1..40c4534 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/AdminWebService.java @@ -24,18 +24,14 @@ package fr.ifremer.coselmar.services.v1; * #L% */ -import java.io.File; -import java.io.IOException; -import java.util.List; - import fr.ifremer.coselmar.beans.DocumentBean; import fr.ifremer.coselmar.beans.QuestionBean; import fr.ifremer.coselmar.beans.UserWebToken; import fr.ifremer.coselmar.converter.BeanEntityConverter; +import fr.ifremer.coselmar.exceptions.CoselmarTechnicalException; import fr.ifremer.coselmar.persistence.entity.CoselmarUserRole; import fr.ifremer.coselmar.persistence.entity.Document; import fr.ifremer.coselmar.persistence.entity.Question; -import fr.ifremer.coselmar.exceptions.CoselmarTechnicalException; import fr.ifremer.coselmar.services.CoselmarWebServiceSupport; import fr.ifremer.coselmar.services.errors.InvalidCredentialException; import fr.ifremer.coselmar.services.errors.UnauthorizedException; @@ -44,7 +40,9 @@ import fr.ifremer.coselmar.services.indexation.QuestionsIndexationService; import fr.ifremer.coselmar.services.indexation.TikaUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; -import org.apache.tika.exception.TikaException; + +import java.io.IOException; +import java.util.List; import static org.apache.commons.logging.LogFactory.getLog; diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java index 63e133d..1e52d9c 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java @@ -33,6 +33,7 @@ import com.google.common.collect.Lists; import fr.ifremer.coselmar.beans.DocumentBean; import fr.ifremer.coselmar.beans.DocumentSearchBean; import fr.ifremer.coselmar.beans.DocumentSearchExample; +import fr.ifremer.coselmar.beans.FileInfos; import fr.ifremer.coselmar.beans.QuestionBean; import fr.ifremer.coselmar.beans.UserBean; import fr.ifremer.coselmar.beans.UserWebToken; @@ -52,10 +53,8 @@ import fr.ifremer.coselmar.services.indexation.DocumentsIndexationService; import fr.ifremer.coselmar.services.indexation.TikaUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.logging.Log; import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.tika.exception.TikaException; import org.debux.webmotion.server.call.UploadFile; import org.debux.webmotion.server.render.Render; import org.nuiton.topia.persistence.TopiaNoResultException; @@ -67,7 +66,6 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -97,6 +95,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { CoselmarUserRole.SUPERVISOR.name(), CoselmarUserRole.EXPERT.name() ); + protected static final String DESCRIPTION_CSV_FILE_NAME = "description.csv"; public DocumentBean getDocument(String documentId) throws InvalidCredentialException, UnauthorizedException { @@ -181,7 +180,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } searchExample.setFullTextSearch(searchBean.getFullTextSearch()); - Document example = BeanEntityConverter.fromBean(searchBean); + Document example = BeanEntityConverter.fromSearchBean(searchBean); searchExample.setExample(example); searchExample.setOwnerName(searchBean.getOwnerName()); @@ -276,7 +275,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { searchExample.setFullTextSearch(searchBean.getFullTextSearch()); - Document example = BeanEntityConverter.fromBean(searchBean); + Document example = BeanEntityConverter.fromSearchBean(searchBean); searchExample.setExample(example); searchExample.setOwnerName(searchBean.getOwnerName()); @@ -376,91 +375,13 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { throw new InvalidCredentialException(message); } - String documentName = document.getName(); - String contentType = null; - String filePath = null; - String fileContent = null; - - // If document has a file, manage it ! - if (uploadFile != null) { - Pair<String, String> pathAndContentType = managerDocumentFile(uploadFile, owner); - filePath = pathAndContentType.getLeft(); - contentType = pathAndContentType.getRight(); - fileContent = TikaUtils.getFileContent(filePath); - } - - // Document Metadata - Document documentEntity = getDocumentDao().create(); - - documentEntity.setOwner(owner); - - documentEntity.setName(documentName); - - String privacy = document.getPrivacy().toUpperCase(); - documentEntity.setPrivacy(Privacy.valueOf(privacy)); - - // Manage privacy : if restricted, create an UserGroup with authorized users - if (StringUtils.equals(Privacy.RESTRICTED.name(), privacy)) { - Set<UserBean> authorizedUsers = document.getAuthorizedUsers(); - Set<CoselmarUser> coselmarUsers = retrieveUsers(authorizedUsers); - CoselmarUserGroup restrictedUsers = getCoselmarUserGroupDao().create(); - restrictedUsers.setName(documentEntity.getTopiaId()); - - restrictedUsers.addAllMembers(coselmarUsers); - documentEntity.addRestrictedList(restrictedUsers); - - } - - documentEntity.addAllKeywords(document.getKeywords()); - - Date depositDate = document.getDepositDate(); - if (depositDate != null) { - documentEntity.setDepositDate(new Date(depositDate.getTime())); - } else { - documentEntity.setDepositDate(new Date()); - } - - documentEntity.setType(document.getType()); - documentEntity.setSummary(document.getSummary()); - documentEntity.setLanguage(document.getLanguage()); - documentEntity.setPublicationDate(document.getPublicationDate()); - - - // Legal / copyright part - documentEntity.setAuthors(document.getAuthors()); - documentEntity.setCopyright(document.getCopyright()); - documentEntity.setLicense(document.getLicense()); - - // Document resource part + FileInfos fileInfos = null; if (uploadFile != null) { - documentEntity.setWithFile(true); - documentEntity.setMimeType(contentType); - documentEntity.setFilePath(filePath); - documentEntity.setFileContent(fileContent); - } else { - documentEntity.setWithFile(false); + fileInfos = manageDocumentFile(uploadFile, owner); } - documentEntity.setExternalUrl(document.getExternalUrl()); - - documentEntity.setComment(document.getComment()); - - documentEntity.setCitation(document.getCitation()); + DocumentBean result = createDocument(document, fileInfos, owner); commit(); - DocumentBean result = BeanEntityConverter.toBean(getPersistenceContext().getTopiaIdFactory(), documentEntity); - - DocumentsIndexationService documentsIndexationService = getServicesContext().newService(DocumentsIndexationService.class); - try { - documentsIndexationService.indexDocument(result, fileContent); - if (log.isDebugEnabled()) { - String message = String.format("Document '%s' added to index", documentName); - log.debug(message); - } - } catch (IOException e) { - if (log.isErrorEnabled()) { - log.error("Unable to index new document", e); - } - } return result; @@ -492,9 +413,9 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { // Get owner to place correctly the file CoselmarUser owner = documentEntity.getOwner(); - Pair<String, String> pathAndContentType = managerDocumentFile(uploadFile, owner); - String filePath = pathAndContentType.getLeft(); - String contentType = pathAndContentType.getRight(); + FileInfos fileInfos = manageDocumentFile(uploadFile, owner); + String filePath = fileInfos.getFilePath(); + String contentType = fileInfos.getMimeType(); // Read file content String fileContent = TikaUtils.getFileContent(filePath); @@ -757,13 +678,117 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { //////////////////////////////////////////////////////////////////////////// /** + * From a {@link DocumentBean}, create a {@link Document}. + * If there is a {@link FileInfos}, get the file data. + * + * @return a DocumentBean with all created {@link Document} data. + */ + protected DocumentBean createDocument(DocumentBean documentBean, FileInfos fileInfos, CoselmarUser owner) { + + // Document Metadata + Document documentEntity = createDocumentMetadataFromBean(documentBean, owner); + + // If document has a file, manage it ! + String fileContent = ""; + + if (fileInfos != null) { + String filePath = fileInfos.getFilePath(); + String contentType = fileInfos.getMimeType(); + fileContent = TikaUtils.getFileContent(filePath); + + documentEntity.setWithFile(true); + documentEntity.setMimeType(contentType); + documentEntity.setFilePath(filePath); + documentEntity.setFileContent(fileContent); + } else { + documentEntity.setWithFile(false); + } + + DocumentBean result = BeanEntityConverter.toBean(getPersistenceContext().getTopiaIdFactory(), documentEntity); + + // Indexation job + DocumentsIndexationService documentsIndexationService = getServicesContext().newService(DocumentsIndexationService.class); + try { + documentsIndexationService.indexDocument(result, fileContent); + if (log.isDebugEnabled()) { + String message = String.format("Document '%s' added to index", documentBean.getName()); + log.debug(message); + } + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("Unable to index new document", e); + } + } + return result; + } + + /** + * Convert metadata fields from {@link DocumentBean} into a {@link Document} and put default values where no given (such as {@link Document#PROPERTY_DEPOSIT_DATE} for example). + * This also manage owner and restriction but <strong>does not manage</strong> attached file part. + * No commit are done here. + */ + public Document createDocumentMetadataFromBean(DocumentBean documentBean, CoselmarUser owner) { + Preconditions.checkNotNull(documentBean); + + // Document Metadata + Document document = getDocumentDao().create(); + + document.setOwner(owner); + + document.setName(documentBean.getName()); + + String privacy = documentBean.getPrivacy().toUpperCase(); + document.setPrivacy(Privacy.valueOf(privacy)); + + // Manage privacy : if restricted, create an UserGroup with authorized users + if (StringUtils.equals(Privacy.RESTRICTED.name(), privacy)) { + Set<UserBean> authorizedUsers = documentBean.getAuthorizedUsers(); + Set<CoselmarUser> coselmarUsers = retrieveUsers(authorizedUsers); + CoselmarUserGroup restrictedUsers = getCoselmarUserGroupDao().create(); + restrictedUsers.setName(document.getTopiaId()); + + restrictedUsers.addAllMembers(coselmarUsers); + document.addRestrictedList(restrictedUsers); + } + + document.addAllKeywords(documentBean.getKeywords()); + + Date depositDate = documentBean.getDepositDate(); + if (depositDate != null) { + document.setDepositDate(new Date(depositDate.getTime())); + } else { + document.setDepositDate(new Date()); + } + + document.setType(documentBean.getType()); + document.setSummary(documentBean.getSummary()); + document.setLanguage(documentBean.getLanguage()); + Date publicationDate = documentBean.getPublicationDate(); + if (publicationDate != null) { + document.setPublicationDate(new Date(publicationDate.getTime())); + } + + document.setCitation(documentBean.getCitation()); + document.setComment(documentBean.getComment()); + + // Legal / copyright part + document.setAuthors(documentBean.getAuthors()); + document.setCopyright(documentBean.getCopyright()); + document.setLicense(documentBean.getLicense()); + + document.setExternalUrl(documentBean.getExternalUrl()); + + return document; + } + + /** * When a Document is sent, it could have a File part : this manage the * upload file. The file is stored in the user specific directory, and the * contentType of document is returned, cause need in Document Metadata. * * @return the upload file Metadata */ - protected Pair<String, String> managerDocumentFile(UploadFile uploadFile, CoselmarUser owner) { + protected FileInfos manageDocumentFile(UploadFile uploadFile, CoselmarUser owner) { Preconditions.checkNotNull(uploadFile); // Document File @@ -771,9 +796,9 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { File uploadedFile = uploadFile.getFile(); String contentType = uploadFile.getContentType(); - if (log.isInfoEnabled()) { + if (log.isDebugEnabled()) { String message = String.format("File name : %s, content-type : %s", fileName, contentType); - log.info(message); + log.debug(message); } // put the document in the good directory @@ -792,7 +817,12 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } throw new CoselmarTechnicalException("Internal error during file transfer"); } - return Pair.of(destFile.getAbsolutePath(), contentType); + + FileInfos fileInfos = new FileInfos(); + fileInfos.setFileName(fileName); + fileInfos.setMimeType(contentType); + fileInfos.setFilePath(destFile.getAbsolutePath()); + return fileInfos; } protected String getUserDocumentPath(CoselmarUser user) { -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.

This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository coselmar. See https://gitlab.nuiton.org/codelutin/coselmar.git commit fdce731043b1840716f3cf526ff78523dbfbee9a Author: Yannick Martel <martel@©odelutin.com> Date: Tue Jun 6 12:05:37 2017 +0200 refs-50 #9206 First draft for error management in Documents Zip import --- .../beans/MassiveDocumentsImportResult.java | 47 +++++++++++++++ .../coselmar/services/v1/DocumentsWebService.java | 65 +++++++++++---------- .../services/v1/DocumentsWebServiceTest.java | 37 +++++++++++- .../src/test/resources/documents_errors.zip | Bin 0 -> 87789 bytes 4 files changed, 116 insertions(+), 33 deletions(-) diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/MassiveDocumentsImportResult.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/MassiveDocumentsImportResult.java new file mode 100644 index 0000000..fff4400 --- /dev/null +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/MassiveDocumentsImportResult.java @@ -0,0 +1,47 @@ +package fr.ifremer.coselmar.beans; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * @author ymartel (martel@codelutin.com) + */ +public class MassiveDocumentsImportResult { + + protected List<String> missingFiles; + protected boolean errorSystem; + protected boolean invalidFile; + + public MassiveDocumentsImportResult() { + this.missingFiles = new ArrayList<>(); + } + + public List<String> getMissingFiles() { + return missingFiles; + } + + public void setMissingFiles(List<String> missingFiles) { + this.missingFiles = missingFiles; + } + + public void addMissingFile(String missingFileName) { + this.missingFiles.add(missingFileName); + } + + public boolean isErrorSystem() { + return errorSystem; + } + + public void setErrorSystem(boolean errorSystem) { + this.errorSystem = errorSystem; + } + + public boolean isInvalidFile() { + return invalidFile; + } + + public void setInvalidFile(boolean invalidFile) { + this.invalidFile = invalidFile; + } +} diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java index 80be99e..6455deb 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java @@ -35,6 +35,7 @@ import fr.ifremer.coselmar.beans.DocumentImportModel; import fr.ifremer.coselmar.beans.DocumentSearchBean; import fr.ifremer.coselmar.beans.DocumentSearchExample; import fr.ifremer.coselmar.beans.FileInfos; +import fr.ifremer.coselmar.beans.MassiveDocumentsImportResult; import fr.ifremer.coselmar.beans.QuestionBean; import fr.ifremer.coselmar.beans.UserBean; import fr.ifremer.coselmar.beans.UserWebToken; @@ -685,7 +686,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { return types; } - public void uploadZipDocuments(UploadFile uploadFile) throws InvalidCredentialException, UnauthorizedException { + public MassiveDocumentsImportResult uploadZipDocuments(UploadFile uploadFile) throws InvalidCredentialException, UnauthorizedException { Preconditions.checkNotNull(uploadFile); @@ -702,24 +703,8 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } throw new UnauthorizedException(message); } - List<String> missingFiles = importFromZip(uploadFile.getFile(), currentUser); + return importFromZip(uploadFile.getFile(), currentUser); - if (missingFiles.isEmpty()) { - // All is ok ! - commit(); - - } else { - // Something wrong happened... rollback, and refresh lucene to avoid new data - rollback(); - AdminWebService adminWebService = getServicesContext().newService(AdminWebService.class); - try { - adminWebService.refreshDocumentsIndex(); - } catch (IOException e) { - if (log.isErrorEnabled()) { - log.error("Unable to refresh Lucene Documents index. Data should be corrupted", e); - } - } - } } //////////////////////////////////////////////////////////////////////////// @@ -980,7 +965,8 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { } - protected List<String> importFromZip(File file, CoselmarUser currentUser) { + protected MassiveDocumentsImportResult importFromZip(File file, CoselmarUser currentUser) { + MassiveDocumentsImportResult importResult = new MassiveDocumentsImportResult(); // File should be a Zip ZipFile zipFile; try { @@ -1002,7 +988,9 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { if (log.isErrorEnabled()) { log.error(message, e); } - throw new CoselmarTechnicalException(message, e); + importResult.setInvalidFile(true); + importResult.addMissingFile(DESCRIPTION_CSV_FILE_NAME); + return importResult; // Direct break : cannot continue without this file } // Now, read CSV ... @@ -1023,7 +1011,6 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { throw new CoselmarTechnicalException("Unable to unzip file : error with unzip directory", e); } - List<String> missingFiles = new ArrayList<>(); // ... and read each entries and retrieve associated File try { for (DocumentBean documentBean : importer) { @@ -1031,8 +1018,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { String fileName = documentBean.getFileName(); ZipEntry zipFileEntry = zipFile.getEntry(fileName); if (zipFileEntry == null) { - // TODO ymartel 20170601 : manage errors + regenerate Lucene Index - missingFiles.add(fileName); + importResult.addMissingFile(fileName); } else { try { @@ -1068,8 +1054,8 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { if (log.isErrorEnabled()) { log.error(message, e); } - // TODO ymartel 20170601 : manage errors + regenerate Lucene Index - missingFiles.add(fileName); + importResult.setErrorSystem(true); + importResult.addMissingFile(fileName); } } @@ -1078,6 +1064,7 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { if (log.isErrorEnabled()) { log.error("Error with CSV file", ire); } + importResult.setErrorSystem(true); throw new CoselmarTechnicalException("Error with CSV file", ire); } finally { // Close importer @@ -1087,19 +1074,37 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { descriptionInputStream.close(); } catch (IOException e) { if (log.isErrorEnabled()) { - log.error("Unable to close 'descriptions.csv' input stream from zip"); + log.error("Unable to close 'descriptions.csv' input stream from zip", e); } } } - if (missingFiles.isEmpty()) { - // Ok, let move all files from temp storage to real one ! + if (!importResult.isErrorSystem() && (importResult.getMissingFiles() == null || importResult.getMissingFiles().isEmpty())) { + // All is ok ! try { + // Ok, let move all files from temp storage to real one ! FileUtils.moveDirectory(new File(zipTempPath), new File(getUserDocumentPath(currentUser))); + commit(); } catch (IOException e) { // Big problem if we can't move files into real folder ! - e.printStackTrace(); + if (log.isErrorEnabled()) { + log.error("Unable to move files from temp folder '%s' to final folder", e); + } + importResult.setErrorSystem(true); + rollback(); + } + + } else { + // Something wrong happened... rollback, and refresh lucene to avoid new data + rollback(); + AdminWebService adminWebService = getServicesContext().newService(AdminWebService.class); + try { + adminWebService.refreshDocumentsIndex(); + } catch (IOException e) { + if (log.isErrorEnabled()) { + log.error("Unable to refresh Lucene Documents index. Data should be corrupted", e); + } } } - return missingFiles; + return importResult; } } diff --git a/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java b/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java index 5357c1c..ca3562c 100644 --- a/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java +++ b/coselmar-rest/src/test/java/fr/ifremer/coselmar/services/v1/DocumentsWebServiceTest.java @@ -24,6 +24,7 @@ package fr.ifremer.coselmar.services.v1; * #L% */ +import fr.ifremer.coselmar.beans.MassiveDocumentsImportResult; import fr.ifremer.coselmar.persistence.entity.CoselmarUser; import fr.ifremer.coselmar.persistence.entity.CoselmarUserImpl; import fr.ifremer.coselmar.persistence.entity.CoselmarUserTopiaDao; @@ -80,9 +81,9 @@ public class DocumentsWebServiceTest extends AbstractCoselmarServiceTest { coselmarUser.setFirstname("Jean"); getServiceContext().getPersistenceContext().commit(); - List<String> errors = documentsWebService.importFromZip(zipFile, coselmarUser); - Assert.assertNotNull(errors); - Assert.assertTrue(errors.isEmpty()); + MassiveDocumentsImportResult importResult = documentsWebService.importFromZip(zipFile, coselmarUser); + Assert.assertNotNull(importResult); + Assert.assertTrue(importResult.getMissingFiles().isEmpty()); DocumentTopiaDao documentDao = getServiceContext().getPersistenceContext().getDocumentDao(); List<Document> documents = documentDao.findAll(); Assert.assertEquals(3, documents.size()); @@ -90,4 +91,34 @@ public class DocumentsWebServiceTest extends AbstractCoselmarServiceTest { File oneFile = new File(oneFilePath); Assert.assertTrue(oneFile.exists()); } + + @Test + public void testUploadZipDocumentsWithErrors() throws Exception { + + DocumentsWebService documentsWebService = getServiceContext().newService(DocumentsWebService.class); + InputStream resourceAsStream = this.getClass().getResourceAsStream("/documents_errors.zip"); + Path tempZipPath = Files.createTempFile(null, null); + Files.copy(resourceAsStream, tempZipPath, StandardCopyOption.REPLACE_EXISTING); + resourceAsStream.close(); + File zipFile = tempZipPath.toFile(); + Assert.assertNotNull(zipFile); + Assert.assertTrue(zipFile.exists()); + + // create a test user + CoselmarUserTopiaDao coselmarUserDao = getServiceContext().getPersistenceContext().getCoselmarUserDao(); + CoselmarUser coselmarUser = coselmarUserDao.create(); + coselmarUser.setName("Peuplu"); + coselmarUser.setFirstname("Jean"); + getServiceContext().getPersistenceContext().commit(); + + MassiveDocumentsImportResult importResult = documentsWebService.importFromZip(zipFile, coselmarUser); + Assert.assertNotNull(importResult); + Assert.assertFalse(importResult.getMissingFiles().isEmpty()); + Assert.assertEquals(1, importResult.getMissingFiles().size()); + Assert.assertEquals("Reunion-v0.3.pdf", importResult.getMissingFiles().get(0)); + // Should import have been rollback + DocumentTopiaDao documentDao = getServiceContext().getPersistenceContext().getDocumentDao(); + List<Document> documents = documentDao.findAll(); + Assert.assertEquals(0, documents.size()); + } } diff --git a/coselmar-rest/src/test/resources/documents_errors.zip b/coselmar-rest/src/test/resources/documents_errors.zip new file mode 100644 index 0000000..836cde6 Binary files /dev/null and b/coselmar-rest/src/test/resources/documents_errors.zip differ -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.

This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository coselmar. See https://gitlab.nuiton.org/codelutin/coselmar.git commit 64faa458eb0ff6864a34d108a4d0646acf69ce69 Author: Yannick Martel <martel@©odelutin.com> Date: Tue Jun 6 20:48:24 2017 +0200 refs-65 #9206 Start UI for documents zip file --- .../beans/MassiveDocumentsImportResult.java | 14 ++---- .../coselmar/services/v1/DocumentsWebService.java | 2 +- coselmar-rest/src/main/resources/mapping | 1 + coselmar-ui/src/main/webapp/i18n/en.js | 10 ++++ coselmar-ui/src/main/webapp/i18n/fr.js | 10 ++++ .../src/main/webapp/js/coselmar-admin-services.js | 18 ++++++++ .../src/main/webapp/js/coselmar-controllers.js | 20 +++++++- .../src/main/webapp/views/admin/admintools.html | 54 ++++++++++++++++++++++ 8 files changed, 118 insertions(+), 11 deletions(-) diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/MassiveDocumentsImportResult.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/MassiveDocumentsImportResult.java index 20495f4..3f8f4ab 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/MassiveDocumentsImportResult.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/beans/MassiveDocumentsImportResult.java @@ -8,15 +8,19 @@ import java.util.List; */ public class MassiveDocumentsImportResult { + protected boolean success; protected List<String> missingFiles; protected boolean errorSystem; protected ImportSystemErrorType systemErrorType; - protected boolean invalidFile; public MassiveDocumentsImportResult() { this.missingFiles = new ArrayList<>(); } + public boolean isSuccess() { + return missingFiles.isEmpty() && !errorSystem; + } + public List<String> getMissingFiles() { return missingFiles; } @@ -38,14 +42,6 @@ public class MassiveDocumentsImportResult { this.errorSystem = true; } - public boolean isInvalidFile() { - return invalidFile; - } - - public void setInvalidFile(boolean invalidFile) { - this.invalidFile = invalidFile; - } - public enum ImportSystemErrorType { UNABLE_TO_READ_ZIP, UNABLE_TO_READ_CSV, diff --git a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java index d54a7a0..d3b676a 100644 --- a/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java +++ b/coselmar-rest/src/main/java/fr/ifremer/coselmar/services/v1/DocumentsWebService.java @@ -989,8 +989,8 @@ public class DocumentsWebService extends CoselmarWebServiceSupport { if (log.isErrorEnabled()) { log.error(message, e); } - importResult.setInvalidFile(true); importResult.addMissingFile(DESCRIPTION_CSV_FILE_NAME); + importResult.setSystemErrorType(MassiveDocumentsImportResult.ImportSystemErrorType.UNABLE_TO_READ_CSV); return importResult; // Direct break : cannot continue without this file } diff --git a/coselmar-rest/src/main/resources/mapping b/coselmar-rest/src/main/resources/mapping index b498a97..154cae2 100644 --- a/coselmar-rest/src/main/resources/mapping +++ b/coselmar-rest/src/main/resources/mapping @@ -78,6 +78,7 @@ GET /v1/general/topwords GeneralWebService.getTopWords # Admin API POST /v1/admin/lucene/index AdminWebService.refreshLuceneIndex +POST /v1/admin/documents/zip DocumentsWebService.uploadZipDocuments # Export diff --git a/coselmar-ui/src/main/webapp/i18n/en.js b/coselmar-ui/src/main/webapp/i18n/en.js index 54eee19..e6b38f5 100644 --- a/coselmar-ui/src/main/webapp/i18n/en.js +++ b/coselmar-ui/src/main/webapp/i18n/en.js @@ -318,6 +318,16 @@ var translateEN = { "admin.tools.message.refresh.inprogress" : "Refresh in progress.", "admin.tools.message.refresh.done" : "Search index has been well refreshed !", "admin.tools.message.refresh.errors" : "Error during search index refresh. Please try again. If problem persist, see server administrator.", +"admin.tools.label.documentsImport" : "Import Documents Zip file", +"admin.tools.button.documentsimport" : "Import !", +"admin.tools.message.massimport.inprogress" : "Import in progress.", +"admin.tools.message.massimport.done" : "Documents have been well imported !", +"admin.tools.message.massimport.errors" : "Error during documents import. Please try again. If problem persist, see server administrator.", +"admin.tools.message.massimport.fail" : "Some error occur during import.", +"admin.tools.message.massimport.zipReadFail" : "Unable to open zipfile, maybe it is corrupt ?", +"admin.tools.message.massimport.csvReadFail" : "Unable to read 'description.csv' file, please verify file format.", +"admin.tools.message.massimport.fileReadFail" : "Unable to read some files from zip.", +"admin.tools.message.massimport.missingFiles" : "Files not found from zip : ", //Common part diff --git a/coselmar-ui/src/main/webapp/i18n/fr.js b/coselmar-ui/src/main/webapp/i18n/fr.js index b4b398b..14f0d06 100644 --- a/coselmar-ui/src/main/webapp/i18n/fr.js +++ b/coselmar-ui/src/main/webapp/i18n/fr.js @@ -318,6 +318,16 @@ var translateFR = { "admin.tools.message.refresh.inprogress" : "Mise à jour en cours.", "admin.tools.message.refresh.done" : "L'index de recherche à bien été raffraichi !", "admin.tools.message.refresh.errors" : "Une erreur est survenue durant la mise à jour de l'index de recherche. Veuillez contacter l'administrateur serveur.", +"admin.tools.label.documentsImport" : "Importer un Zip de Documents", +"admin.tools.button.documentsimport" : "Importer !", +"admin.tools.message.massimport.inprogress" : "Import en cours.", +"admin.tools.message.massimport.done" : "Les documents ont bien été importés !", +"admin.tools.message.massimport.errors" : "Des erreurs sont survenues durant l'import. Veuillez contacter l'administrateur serveur.", +"admin.tools.message.massimport.fail" : "Des erreurs sont survenues durant le traitement des fichiers.", +"admin.tools.message.massimport.zipReadFail" : "Impossible de lire le fichier zip, peut-être est-il corrompu ?", +"admin.tools.message.massimport.csvReadFail" : "Erreur dans le traitement du fichier 'description.csv'. Veuillez vérifier le format des données.", +"admin.tools.message.massimport.fileReadFail" : "Erreur dans le traitement de fichiers contenus dans le zip.", +"admin.tools.message.massimport.missingFiles" : "Fichiers non retrouvés dans le zip : ", //Common part diff --git a/coselmar-ui/src/main/webapp/js/coselmar-admin-services.js b/coselmar-ui/src/main/webapp/js/coselmar-admin-services.js index 9a2da38..430fae4 100644 --- a/coselmar-ui/src/main/webapp/js/coselmar-admin-services.js +++ b/coselmar-ui/src/main/webapp/js/coselmar-admin-services.js @@ -47,4 +47,22 @@ function Admin(resource, config){ adminResource.save(null, successFunction, failFunction); } + this.importDocumentsZip = function(file, successFunction, failFunction){ + + var formData = new FormData(); + formData.append("uploadFile", file); + + var serviceURL = baseURL + "/documents/zip"; + + // Launch import ! + var adminResource = resource(serviceURL, null, { + 'upload': { + method:'POST', + transformRequest: angular.identity, + headers: {'Content-Type': undefined, 'Content-Transfer-Encoding': 'utf-8'} + } + }); + adminResource.upload(null, formData, successFunction, failFunction); + } + }; \ No newline at end of file diff --git a/coselmar-ui/src/main/webapp/js/coselmar-controllers.js b/coselmar-ui/src/main/webapp/js/coselmar-controllers.js index d0110e1..4db9a5b 100644 --- a/coselmar-ui/src/main/webapp/js/coselmar-controllers.js +++ b/coselmar-ui/src/main/webapp/js/coselmar-controllers.js @@ -2164,7 +2164,8 @@ coselmarControllers.controller("ReferentialCtrl", ['$scope', '$routeParams', '$l coselmarControllers.controller("AdminCtrl", ['$scope', '$routeParams', '$location', 'adminService', function($scope, $routeParams, $location, adminService){ - $scope.status = { 'indexRefresh' : 'none'}; + $scope.status = { 'indexRefresh' : 'none', 'massimport' : 'none', 'importResult' : undefined }; + $scope.importResult = undefined; $scope.refreshIndex = function() { $scope.status.indexRefresh = "inprogress"; @@ -2178,6 +2179,23 @@ coselmarControllers.controller("AdminCtrl", ['$scope', '$routeParams', '$locatio }; + $scope.importDocumentsZip = function() { + $scope.status.massimport = "inprogress"; + + adminService.importDocumentsZip($scope.uploadFile, function(importResult) { + if (importResult.isSuccess) { + $scope.status.massimport = "finish"; + } else { + $scope.status.massimport = "fail"; + $scope.status.importResult = importResult; + } + + },function(error) { + $scope.status.massimport = "error"; + }); + + }; + }]); /** diff --git a/coselmar-ui/src/main/webapp/views/admin/admintools.html b/coselmar-ui/src/main/webapp/views/admin/admintools.html index b337f8f..bdb9f25 100644 --- a/coselmar-ui/src/main/webapp/views/admin/admintools.html +++ b/coselmar-ui/src/main/webapp/views/admin/admintools.html @@ -52,4 +52,58 @@ </div> <br/> </div> + + <div class="form-group"> + <div class="form-group"> + + <label class="col-md-5 control-label">{{ 'admin.tools.label.documentsImport' | translate }}</label> + + <div class="col-md-7 "> + + <input class="col-md-8" type="file" class="form-control" name="uploadFile" ng-file-model="uploadFile" accept=".zip"/> + + <a class="form-inline btn btn-primary col-md-4" + ng-click="importDocumentsZip()" + ng-disabled="!uploadFile">{{ 'admin.tools.button.documentsimport' | translate }}</a> + </div> + </div> + + <div ng-show="status.massImport != 'none'" > + <div ng-show="status.massimport === 'inprogress'"> + {{ 'admin.tools.message.massimport.inprogress' | translate }} + </div> + <div ng-show="status.massimport === 'finish'"> + {{ 'admin.tools.message.massimport.done' | translate }} + </div> + <div ng-show="status.massimport === 'error'"> + {{ 'admin.tools.message.massimport.error' | translate }} + </div> + + <div ng-show="status.massimport === 'fail'"> + {{ 'admin.tools.message.massimport.fail' | translate }} + <!-- Unable to read Zip --> + <div ng-show="status.importResult && status.importResult.errorSystem && status.importResult.systemErrorType == 'UNABLE_TO_READ_ZIP'"> + {{ 'admin.tools.message.massimport.zipReadFail' | translate }} + </div> + <!-- Unable to read CSV data --> + <div ng-show="status.importResult && status.importResult.errorSystem && status.importResult.systemErrorType == 'UNABLE_TO_READ_CSV'"> + {{ 'admin.tools.message.massimport.csvReadFail' | translate }} + </div> + <!-- Unable to read a file entry in Zip --> + <div ng-show="status.importResult && status.importResult.errorSystem && status.importResult.systemErrorType == 'UNABLE_TO_READ_ZIP_ENTRY'"> + {{ 'admin.tools.message.massimport.fileReadFail' | translate }} + </div> + <!-- Missing files in Zip --> + <div ng-show="status.importResult && status.importResult.missingFiles && status.importResult.missingFiles.length > 0"> + {{ 'admin.tools.message.massimport.missingFiles' | translate }} + <ul> + <li ng-repeat="missingFile in status.importResult.missingFiles">{{missingFile}}</li> + </ul> + </div> + </div> + + + </div> + <br/> + </div> </div> \ No newline at end of file -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.