Author: chatellier
Date: 2010-10-28 15:07:48 +0000 (Thu, 28 Oct 2010)
New Revision: 116
Log:
Validation plus completes (sur plusieurs lignes)
Modified:
trunk/coser-business/src/main/java/fr/ifremer/coser/CoserBusinessConfig.java
trunk/coser-business/src/main/java/fr/ifremer/coser/services/ValidationService.java
trunk/coser-business/src/main/resources/i18n/coser-business-en_GB.properties
trunk/coser-business/src/main/resources/i18n/coser-business-fr_FR.properties
trunk/coser-business/src/test/java/fr/ifremer/coser/services/ValidationServiceTest.java
Modified: trunk/coser-business/src/main/java/fr/ifremer/coser/CoserBusinessConfig.java
===================================================================
--- trunk/coser-business/src/main/java/fr/ifremer/coser/CoserBusinessConfig.java 2010-10-28 13:42:45 UTC (rev 115)
+++ trunk/coser-business/src/main/java/fr/ifremer/coser/CoserBusinessConfig.java 2010-10-28 15:07:48 UTC (rev 116)
@@ -82,6 +82,11 @@
return result;
}
+ public double getObervationNobsmin() {
+ double result = getOptionAsDouble(CoserBusinessOption.OBSERVATION_NOBSMIN.key);
+ return result;
+ }
+
public static enum CoserBusinessOption implements OptionDef {
DATABASE_DIRECTORY("coser.database.directory", _("coser.config.database.directory.description"), "${user.home}" + File.separator + "coser", String.class, false, false),
@@ -89,8 +94,10 @@
VALIDATOR_DIRECTORY("coser.validator.directory", _("coser.config.validator.directory.description"), "${" + DATABASE_DIRECTORY.key + "}" + File.separator + "validators", String.class, false, false),
REFERENCE_SPECIES("coser.reference.species", _("coser.config.reference.species.description"), "", String.class, false, false),
- REFERENCE_ZONES("coser.reference.zones", _("coser.config.reference.zones.description"), "", String.class, false, false);
+ REFERENCE_ZONES("coser.reference.zones", _("coser.config.reference.zones.description"), "", String.class, false, false),
+ OBSERVATION_NOBSMIN("coser.observation.nobsmin", _("coser.config.observation.nobsmin.description"), "", Double.class, false, false);
+
protected String key;
protected String description;
protected String defaultValue;
Modified: trunk/coser-business/src/main/java/fr/ifremer/coser/services/ValidationService.java
===================================================================
--- trunk/coser-business/src/main/java/fr/ifremer/coser/services/ValidationService.java 2010-10-28 13:42:45 UTC (rev 115)
+++ trunk/coser-business/src/main/java/fr/ifremer/coser/services/ValidationService.java 2010-10-28 15:07:48 UTC (rev 116)
@@ -28,6 +28,7 @@
import static org.nuiton.i18n.I18n._;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -175,7 +176,24 @@
* @return les erreurs de validation
*/
public List<ValidationError> validateCategory(Control control, Category category, ProgressMonitor progress) {
+ List<ValidationError> errors = validateCategoryXWork(control, category, progress);
+ List<ValidationError> specificErrors = validateCategorySpecific(control, category, progress);
+ if (specificErrors != null) {
+ errors.addAll(specificErrors);
+ }
+ return errors;
+ }
+ /**
+ * Valide une category entière d'un project.
+ *
+ * @param control control a valider
+ * @param category category a valider
+ * @param progress progress monitor
+ * @return les erreurs de validation (not null)
+ */
+ public List<ValidationError> validateCategoryXWork(Control control, Category category, ProgressMonitor progress) {
+
// instance des bean utilisé lors de la validation
Catch beanCatch = new Catch();
Haul beanHaul = new Haul();
@@ -248,7 +266,6 @@
// check for duplicated lines
String uniqueDataKey = getSignificantData(line);
- System.out.println(uniqueDataKey);
if (uniqueDataKeys.contains(uniqueDataKey)) {
ValidationError error = new ValidationError();
error.setLevel(ValidationLevel.ERROR);
@@ -266,8 +283,8 @@
/**
* Retourne sous forme de clé la partie significative des données.
- *
* Donc sans le numéro de ligne.
+ * Utilisé pour la detection des doublons.
*
* @param data data
* @return string key
@@ -281,4 +298,253 @@
}
return sb.toString();
}
+
+ /**
+ * Effectue un calcul global, mais specific a chaque categorie.
+ *
+ * @param control control
+ * @param category category
+ * @param progress progress monitor
+ * @return
+ */
+ protected List<ValidationError> validateCategorySpecific(Control control,
+ Category category, ProgressMonitor progress) {
+
+ List<ValidationError> validationErrors = null;
+ switch (category) {
+ case CATCH:
+ validationErrors = validateCategorySpecificCatch(control, progress);
+ break;
+ case HAUL:
+ validationErrors = validateCategorySpecificHaul(control, progress);
+ break;
+ case LENGTH:
+ validationErrors = validateCategorySpecificLength(control, progress);
+ break;
+ case STRATA:
+ validationErrors = validateCategorySpecificStrata(control, progress);
+ break;
+ }
+ return validationErrors;
+ }
+
+ /**
+ * Alerte si Somme(CAPTURES$Nombre par CAPTURES$Annee|Strate|Espece) < nobsmin.
+ * Erreur si CAPT$Nombre n'est pas 0 si le poids dans CAPT$Poids>0
+ *
+ * @param control
+ * @param progress
+ * @return
+ */
+ protected List<ValidationError> validateCategorySpecificCatch(
+ Control control, ProgressMonitor progress) {
+
+ List<ValidationError> validationErrors = new ArrayList<ValidationError>();
+
+ int total = control.getCatch().size() - 1;
+ if (progress != null) {
+ progress.setText(_("coser.business.control.step.observation", 0));
+ progress.setTotal(total);
+ }
+
+ // "Campagne","Annee","Trait","Espece","Nombre","Poids"
+ // regroupement par campagne/annee/trait
+ Map<String, Double> nombreForKey = new HashMap<String, Double>();
+ Map<String, String> firstLineForKey = new HashMap<String, String>();
+
+ // parcours des elements
+ Iterator<String[]> itTuple = control.getCatch().iterator();
+ itTuple.next(); // skip header
+ int lineIndex = 1; // skip header
+ while (itTuple.hasNext()) {
+ String[] tuple = itTuple.next();
+
+ // update progress
+ if (progress != null) {
+ int progressPercent = (int)((double)lineIndex / (double)total * 100.0);
+ progress.setText(_("coser.business.control.step.observation", progressPercent));
+ progress.setCurrent(lineIndex);
+ ++lineIndex;
+ }
+
+ // compute key
+ StringBuffer sb = new StringBuffer();
+ for (int tupleIndex = 0 ; tupleIndex < tuple.length; ++tupleIndex) {
+ if (tupleIndex == Catch.INDEX_YEAR || tupleIndex == Catch.INDEX_HAUL ||
+ tupleIndex == Catch.INDEX_SPECIES) {
+ sb.append(tuple[tupleIndex]).append(';');
+ }
+ }
+
+ // store number sum
+ String nombreValue = tuple[Catch.INDEX_NUMBER];
+ String lineNumber = tuple[AbstractDataEntity.INDEX_LINE];
+ try {
+ Double nombre = Double.valueOf(nombreValue);
+ String key = sb.toString();
+ if (nombreForKey.containsKey(key)) {
+ Double oldValue = nombreForKey.get(key);
+ Double newValue = oldValue + nombre;
+ nombreForKey.put(key, newValue);
+
+ // sauvegarde la premiere ligne qui correspond a un regroupement de clé
+ firstLineForKey.put(key, lineNumber);
+ }
+ else {
+ nombreForKey.put(key, nombre);
+ }
+
+
+ // autre validation (rien a voir avec le reste attention)
+ // Vérifier que le nombre d'individus dans CAPT$Nombre n'est pas 0 si le poids dans CAPT$Poids>0.
+ String poidsValue = tuple[Catch.INDEX_WEIGHT];
+ Double poids = Double.valueOf(poidsValue);
+ if (poids > 0 && nombre <= 0) {
+ ValidationError error = new ValidationError();
+ error.setLevel(ValidationLevel.ERROR);
+ error.setLineNumber(lineNumber);
+ error.setMessage(_("coser.business.control.error.noCatchNumberWithWeight"));
+ validationErrors.add(error);
+ }
+ }
+ catch (NumberFormatException ex) {
+ // par trop grave, normalement les données deviennet
+ // valide au fil de la validation
+ if (log.isWarnEnabled()) {
+ log.warn("Can't parse " + nombreValue + " as double");
+ }
+ }
+ }
+
+ // now look for invalid data
+ for (Map.Entry<String, Double> sumObservation : nombreForKey.entrySet()) {
+ String key = sumObservation.getKey();
+ Double value = sumObservation.getValue();
+ if (value < config.getObervationNobsmin()) {
+
+ String lineNumber = firstLineForKey.get(key);
+
+ ValidationError error = new ValidationError();
+ error.setLevel(ValidationLevel.ERROR);
+ error.setLineNumber(lineNumber);
+ error.setMessage(_("coser.business.control.error.minObservationCount", key));
+ validationErrors.add(error);
+ }
+ }
+
+ return validationErrors;
+ }
+
+ /**
+ * Alerte si Somme(TAILLES$Nombre par TAILLES$Annee|Strate|Espece) < nobsmin
+ *
+ * @param control
+ * @param progress
+ * @return
+ */
+ protected List<ValidationError> validateCategorySpecificLength(
+ Control control, ProgressMonitor progress) {
+ List<ValidationError> validationErrors = new ArrayList<ValidationError>();
+
+ int total = control.getLength().size() - 1;
+ if (progress != null) {
+ progress.setText(_("coser.business.control.step.observation", 0));
+ progress.setTotal(total);
+ }
+
+ // "Campagne","Annee","Trait","Espece","Nombre","Poids"
+ // regroupement par campagne/annee/trait
+ Map<String, Double> nombreForKey = new HashMap<String, Double>();
+ Map<String, String> firstLineForKey = new HashMap<String, String>();
+
+ // parcours des elements
+ Iterator<String[]> itTuple = control.getLength().iterator();
+ itTuple.next(); // skip header
+ int lineIndex = 1; // skip header
+ while (itTuple.hasNext()) {
+ String[] tuple = itTuple.next();
+
+ // update progress
+ if (progress != null) {
+ int progressPercent = (int)((double)lineIndex / (double)total * 100.0);
+ progress.setText(_("coser.business.control.step.observation", progressPercent));
+ progress.setCurrent(lineIndex);
+ ++lineIndex;
+ }
+
+ // compute key
+ StringBuffer sb = new StringBuffer();
+ for (int tupleIndex = 0 ; tupleIndex < tuple.length; ++tupleIndex) {
+ if (tupleIndex == Length.INDEX_YEAR || tupleIndex == Length.INDEX_HAUL ||
+ tupleIndex == Length.INDEX_SPECIES) {
+ sb.append(tuple[tupleIndex]).append(';');
+ }
+ }
+
+ // store number sum
+ String nombreValue = tuple[Length.INDEX_NUMBER];
+ String lineNumber = tuple[AbstractDataEntity.INDEX_LINE];
+ try {
+ Double nombre = Double.valueOf(nombreValue);
+ String key = sb.toString();
+ if (nombreForKey.containsKey(key)) {
+ Double oldValue = nombreForKey.get(key);
+ Double newValue = oldValue + nombre;
+ nombreForKey.put(key, newValue);
+
+ // sauvegarde la premiere ligne qui correspond a un regroupement de clé
+ firstLineForKey.put(key, lineNumber);
+ }
+ else {
+ nombreForKey.put(key, nombre);
+ }
+ }
+ catch (NumberFormatException ex) {
+ // par trop grave, normalement les données deviennet
+ // valide au fil de la validation
+ if (log.isWarnEnabled()) {
+ log.warn("Can't parse " + nombreValue + " as double");
+ }
+ }
+ }
+
+ // now look for invalid data
+ for (Map.Entry<String, Double> sumObservation : nombreForKey.entrySet()) {
+ String key = sumObservation.getKey();
+ Double value = sumObservation.getValue();
+ if (value < config.getObervationNobsmin()) {
+
+ String lineNumber = firstLineForKey.get(key);
+
+ ValidationError error = new ValidationError();
+ error.setLevel(ValidationLevel.WARNING);
+ error.setLineNumber(lineNumber);
+ error.setMessage(_("coser.business.control.error.minObservationCount", key));
+ validationErrors.add(error);
+ }
+ }
+
+ return validationErrors;
+ }
+
+ /**
+ * @param control
+ * @param progress
+ * @return
+ */
+ protected List<ValidationError> validateCategorySpecificHaul(Control control,
+ ProgressMonitor progress) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /**
+ * @param control
+ * @param progress
+ * @return
+ */
+ protected List<ValidationError> validateCategorySpecificStrata(
+ Control control, ProgressMonitor progress) {
+ return null;
+ }
}
Modified: trunk/coser-business/src/main/resources/i18n/coser-business-en_GB.properties
===================================================================
--- trunk/coser-business/src/main/resources/i18n/coser-business-en_GB.properties 2010-10-28 13:42:45 UTC (rev 115)
+++ trunk/coser-business/src/main/resources/i18n/coser-business-en_GB.properties 2010-10-28 15:07:48 UTC (rev 116)
@@ -1,3 +1,4 @@
+Age\ attribute\ is\ required=
Can't\ create\ project=
Can't\ find\ line\ %s\ for\ deletion=
Can't\ find\ line\ %s\ for\ undeletion=
@@ -2,10 +3,28 @@
Can't\ read\ file\ '%s'.\ Check\ CSV\ file\ separator\ (%c)=
+Depth\ attribute\ is\ not\ a\ valid\ double=
+Maturity\ attribute\ is\ required=
Missing\ file\ %s=
+Number\ attribute\ is\ not\ a\ valid\ double=
Original\ line\ already\ exists\!=
Project\ %s\ already\ exist=
Project\ %s\ doesn't\ exists\ \!=
+Project\ name\ is\ required=
Selection\ %s\ already\ exists=
+Selection\ name\ is\ required=
+Sex\ is\ mandatory=
Specy\ %s\ doesn't\ exist\ in\ referential=
+Weight\ attribute\ is\ not\ a\ valid\ double=
+Weight\ n'est\ pas\ compris\ entre\ ${min}\ en\ ${max}=
Wrong\ header\ detected\ in\ file.\ Find\ \:\ %s,\ expected\ %s=
Wrong\ header\ detected\ in\ file.\ Find\ \:\ %s,\ expected\ %s\ or\ %s=
+You\ must\ enter\ a\ campagne\ name.=
+You\ must\ enter\ a\ espece\ name.=
+You\ must\ enter\ a\ haul\ name.=
+You\ must\ enter\ a\ month\ name.=
+You\ must\ enter\ a\ strata\ name.=
+You\ must\ enter\ a\ stratum\ name.=
+You\ must\ enter\ a\ survey\ name.=
+You\ must\ enter\ a\ sweptSurface\ name.=
+You\ must\ enter\ a\ trait\ name.=
+You\ must\ enter\ a\ year\ name.=
coser.business.category.catch=Catch
@@ -15,9 +34,23 @@
coser.business.category.length=Length
coser.business.category.reftax.species=
coser.business.category.strata=Stata
+coser.business.control.error.duplicatedLine=
+coser.business.control.error.minObservationCount=
+coser.business.control.error.noCatchNumberWithWeight=
+coser.business.control.step.observation=
+coser.business.control.step.observation.catch=
+coser.business.control.step.xworks=
coser.business.line=Line
coser.config.database.directory.description=
+coser.config.observation.nobsmin.description=
coser.config.projects.directory.description=
coser.config.reference.species.description=
coser.config.reference.zones.description=
coser.config.validator.directory.description=
+lat\ attribute\ is\ not\ a\ valid\ double=
+lat\ must\ contains\ at\ least\ 5\ decimal=
+length\ attribute\ is\ not\ a\ valid\ double=
+long\ attribute\ is\ not\ a\ valid\ double=
+long\ must\ contains\ at\ least\ 5\ decimal=
+number\ attribute\ is\ not\ a\ valid\ double=
+sweptSurface\ attribute\ is\ not\ a\ valid\ double=
Modified: trunk/coser-business/src/main/resources/i18n/coser-business-fr_FR.properties
===================================================================
--- trunk/coser-business/src/main/resources/i18n/coser-business-fr_FR.properties 2010-10-28 13:42:45 UTC (rev 115)
+++ trunk/coser-business/src/main/resources/i18n/coser-business-fr_FR.properties 2010-10-28 15:07:48 UTC (rev 116)
@@ -1,3 +1,4 @@
+Age\ attribute\ is\ required=
Can't\ create\ project=Impossible de cr\u00E9er le projet
Can't\ find\ line\ %s\ for\ deletion=
Can't\ find\ line\ %s\ for\ undeletion=
@@ -2,10 +3,28 @@
Can't\ read\ file\ '%s'.\ Check\ CSV\ file\ separator\ (%c)=Impossible de lire le fichier '%s'.\nMerci de v\u00E9rifier le s\u00E9parateur utilis\u00E9 (%c).
+Depth\ attribute\ is\ not\ a\ valid\ double=
+Maturity\ attribute\ is\ required=
Missing\ file\ %s=Fichier manquant \: %s
+Number\ attribute\ is\ not\ a\ valid\ double=
Original\ line\ already\ exists\!=
Project\ %s\ already\ exist=Le projet %s existe d\u00E9j\u00E0 \!
Project\ %s\ doesn't\ exists\ \!=Le projet %s n'existe pas \!
+Project\ name\ is\ required=
Selection\ %s\ already\ exists=La s\u00E9lection %s existe d\u00E9j\u00E0 \!
+Selection\ name\ is\ required=
+Sex\ is\ mandatory=
Specy\ %s\ doesn't\ exist\ in\ referential=L'esp\u00E8ce %s n'existe pas dans le r\u00E9f\u00E9rentiel
+Weight\ attribute\ is\ not\ a\ valid\ double=
+Weight\ n'est\ pas\ compris\ entre\ ${min}\ en\ ${max}=
Wrong\ header\ detected\ in\ file.\ Find\ \:\ %s,\ expected\ %s=Mauvais ent\u00EAte de fichier d\u00E9tect\u00E9.\n\nTrouv\u00E9\u2009\:\n\t%s\nAttendu\u2009\:\n\t%s.
Wrong\ header\ detected\ in\ file.\ Find\ \:\ %s,\ expected\ %s\ or\ %s=Mauvais ent\u00EAte de fichier d\u00E9tect\u00E9.\n\nTrouv\u00E9\u2009\:\n\t%s\nAttendu\u2009\:\n\t%s\nou\u2009\:\n\t%s.
+You\ must\ enter\ a\ campagne\ name.=
+You\ must\ enter\ a\ espece\ name.=
+You\ must\ enter\ a\ haul\ name.=
+You\ must\ enter\ a\ month\ name.=
+You\ must\ enter\ a\ strata\ name.=
+You\ must\ enter\ a\ stratum\ name.=
+You\ must\ enter\ a\ survey\ name.=
+You\ must\ enter\ a\ sweptSurface\ name.=
+You\ must\ enter\ a\ trait\ name.=
+You\ must\ enter\ a\ year\ name.=
coser.business.category.catch=Captures
@@ -15,9 +34,22 @@
coser.business.category.length=Tailles
coser.business.category.reftax.species=Reftax (esp\u00E8ce)
coser.business.category.strata=Strates
+coser.business.control.error.duplicatedLine=Ligne en doublon
+coser.business.control.error.minObservationCount=Nombre minimal d'observation non atteint (%s) \: %.2f
+coser.business.control.error.noCatchNumberWithWeight=Poids renseign\u00E9s, mais capture non pr\u00E9sente
+coser.business.control.step.observation=Verfication du nombre d'obervation (%d%%)
+coser.business.control.step.xworks=Validation par lignes (%d%%)
coser.business.line=Ligne
coser.config.database.directory.description=Emplacement de la base de campagnes de coser
+coser.config.observation.nobsmin.description=Nombre minimal d'observation
coser.config.projects.directory.description=Emplacement des projets Coser
coser.config.reference.species.description=Emplacement du fichier de r\u00E9f\u00E9rence des esp\u00E8ces
coser.config.reference.zones.description=Emplacement du fichier de r\u00E9f\u00E9rence des zones
coser.config.validator.directory.description=Emplacement des fichiers de validations
+lat\ attribute\ is\ not\ a\ valid\ double=
+lat\ must\ contains\ at\ least\ 5\ decimal=
+length\ attribute\ is\ not\ a\ valid\ double=
+long\ attribute\ is\ not\ a\ valid\ double=
+long\ must\ contains\ at\ least\ 5\ decimal=
+number\ attribute\ is\ not\ a\ valid\ double=
+sweptSurface\ attribute\ is\ not\ a\ valid\ double=
Modified: trunk/coser-business/src/test/java/fr/ifremer/coser/services/ValidationServiceTest.java
===================================================================
--- trunk/coser-business/src/test/java/fr/ifremer/coser/services/ValidationServiceTest.java 2010-10-28 13:42:45 UTC (rev 115)
+++ trunk/coser-business/src/test/java/fr/ifremer/coser/services/ValidationServiceTest.java 2010-10-28 15:07:48 UTC (rev 116)
@@ -32,9 +32,13 @@
import org.junit.Assert;
import org.junit.Test;
+import fr.ifremer.coser.CoserConstants.Category;
+import fr.ifremer.coser.bean.Control;
import fr.ifremer.coser.control.ValidationError;
import fr.ifremer.coser.data.Catch;
import fr.ifremer.coser.data.Haul;
+import fr.ifremer.coser.storage.DataStorage;
+import fr.ifremer.coser.storage.MemoryListStorage;
/**
* Test abour validation.
@@ -61,17 +65,11 @@
myCatch.setData(new String[]{"1", "","","","","",""});
List<ValidationError> errors = validationService.validate(myCatch);
- if (log.isDebugEnabled()) {
- log.debug("Validation errors = " + errors);
- }
Assert.assertNotNull(errors);
Assert.assertEquals(4, errors.size());
myCatch.setData(new String[]{"1", "Toto","","","","999",""});
errors = validationService.validate(myCatch);
- if (log.isDebugEnabled()) {
- log.debug("Validation errors = " + errors);
- }
Assert.assertNotNull(errors);
Assert.assertEquals(3, errors.size());
}
@@ -84,10 +82,7 @@
Catch myCatch = new Catch();
myCatch.setData(new String[]{"1", "Test survey","1999","Test trait","Test sp","NA","12"});
List<ValidationError> errors = validationService.validate(myCatch);
- if (log.isDebugEnabled()) {
- log.debug("Validation errors = " + errors);
- }
- Assert.assertNull(errors);
+ Assert.assertNull(errors);
}
/**
@@ -102,7 +97,23 @@
haulBean.setData(new String[]{"1","COSER_TEST","2010","TRAIT3","10","STR3","0.06000","43.89000","-1.73234","115.00"});
errors = validationService.validate(haulBean);
- System.out.println(errors);
Assert.assertNull(errors);
}
+
+ /**
+ * Test, si un poids est reseigné, la capture doit l'etre.
+ */
+ @Test
+ public void testWeigthWithNonCapture() {
+ Control control = new Control();
+ DataStorage dataCatch = new MemoryListStorage();
+ dataCatch.add(new String[]{"Line", "Campagne","Annee","Trait","Espece","Nombre","Poids"});
+ dataCatch.add(new String[]{"1", "Test survey","1999","Test trait","Test sp","0","12"});
+ control.setCatch(dataCatch);
+ List<ValidationError> errors = validationService.validateCategory(control, Category.CATCH, null);
+ if (log.isDebugEnabled()) {
+ log.debug("Validation errors = " + errors);
+ }
+ Assert.assertEquals(1, errors.size());
+ }
}
