001 /*
002 * Copyright (c) 1995-2010, The University of Sheffield. See the file
003 * COPYRIGHT.txt in the software or at http://gate.ac.uk/gate/COPYRIGHT.txt
004 *
005 * This file is part of GATE (see http://gate.ac.uk/), and is free
006 * software, licenced under the GNU Library General Public License,
007 * Version 2, June 1991 (in the distribution as file licence.html,
008 * and also available at http://gate.ac.uk/gate/licence.html).
009 *
010 * Thomas Heitz, 10/Mars/2010
011 *
012 * $Id$
013 */
014
015 package gate.util;
016
017 import gate.*;
018 import gate.util.persistence.PersistenceManager;
019 import junit.framework.Test;
020 import junit.framework.TestCase;
021 import junit.framework.TestSuite;
022
023 import java.io.*;
024 import java.util.*;
025
026 /**
027 * Test an application against its previous run on a corpus.
028 * If the results have changed more than fail.treshold parameter
029 * then compare both previous and current application with the gold standard.
030 */
031 public class TestApplication extends TestCase {
032
033 public static Test suite() {
034 return new TestSuite(TestApplication.class);
035 }
036
037 /**
038 <pre>
039 The config file includes:
040 - the application to be run
041 - three directories containing
042 - a copy of the clean documents
043 - these are annotated with the current application
044 - the annotated version becomes a build artifact (which can be used
045 to update the reference set, in the case of good changes)
046 - a set of documents annotated with the previous version
047 - the gold standard
048 - for each such directory configuration includes the annotation set
049 and annotation types to be used
050 </pre>
051 */
052 @Override
053 protected void setUp() throws Exception {
054
055 // initialisations
056 propertiesFile = new File("test.application.properties");
057 properties = new Properties();
058 annotationFeatures = new HashSet<String>();
059 applicationAnnotationSet = "";
060 goldAnnotationSet = "";
061 failThreshold = 0.7;
062
063 // read values from the properties file
064 Out.prln();
065 Out.prln("Load properties file: " + propertiesFile.getAbsolutePath());
066 FileInputStream propertiesFileInputStream = null;
067 try {
068 propertiesFileInputStream = new FileInputStream(propertiesFile);
069 properties.load(propertiesFileInputStream);
070 String property = properties.getProperty("document.encoding");
071 if (property != null && !property.equals("")) {
072 documentEncoding = property.trim();
073 Out.prln("Document encoding: " + documentEncoding);
074 }
075 property = properties.getProperty("application.file");
076 if (property != null && !property.equals("")) {
077 applicationFile = new File(property.trim());
078 Out.prln("Application path: "+applicationFile.getAbsolutePath());
079 } else {
080 throw new IllegalArgumentException("application.file");
081 }
082 property = properties.getProperty("clean.documents.directory");
083 if (property != null && !property.equals("")) {
084 cleanDocumentsDirectory = new File(property.trim());
085 Out.prln("Clean documents directory: "
086 + cleanDocumentsDirectory.getAbsolutePath());
087 } else {
088 throw new IllegalArgumentException("clean.documents.directory");
089 }
090 property = properties.getProperty("previous.run.directory");
091 if (property != null && !property.equals("")) {
092 previousRunDirectory = new File(property.trim());
093 Out.prln("Previous run directory: "
094 + previousRunDirectory.getAbsolutePath());
095 }
096 property = properties.getProperty("gold.standard.directory");
097 if (property != null && !property.equals("")) {
098 goldStandardDirectory = new File(property.trim());
099 Out.prln("Gold standard directory: "
100 + goldStandardDirectory.getAbsolutePath());
101 }
102 property = properties.getProperty("application.annotation.set");
103 if (property != null && !property.equals("")) {
104 applicationAnnotationSet = property.trim();
105 Out.prln("Application annotation set: " + applicationAnnotationSet);
106 }
107 property = properties.getProperty("gold.annotation.set");
108 if (property != null && !property.equals("")) {
109 goldAnnotationSet = property.trim();
110 Out.prln("Gold annotation set: " + goldAnnotationSet);
111 }
112 property = properties.getProperty("annotation.types");
113 if (property != null && !property.equals("")) {
114 annotationTypes = new HashSet<String>(
115 Arrays.asList(property.trim().split(", ?")));
116 Out.prln("Annotation types: "
117 + Arrays.toString(annotationTypes.toArray()));
118 } else {
119 throw new IllegalArgumentException("annotation.types");
120 }
121 property = properties.getProperty("annotation.features");
122 if (property != null && !property.equals("")) {
123 annotationFeatures = new HashSet<String>(
124 Arrays.asList(property.trim().split(", ?")));
125 Out.prln("Annotation features: "
126 + Arrays.toString(annotationFeatures.toArray()));
127 }
128 property = properties.getProperty("fail.threshold");
129 if (property != null && !property.equals("")) {
130 failThreshold = Double.parseDouble(property.trim());
131 Out.prln("Fail threshold: " + failThreshold);
132 }
133 property = properties.getProperty("results.file");
134 if (property != null && !property.equals("")) {
135 resultsFile = new File(property.trim());
136 Out.prln("Results file: "+resultsFile.getAbsolutePath());
137 }
138 Out.prln();
139
140 } catch (IOException e) {
141 Out.prln("Could not load the properties file:");
142 Out.prln("test.application.properties");
143 Out.prln("The file must be in the directory from" +
144 " where you run the test.");
145 e.printStackTrace();
146
147 } catch (IllegalArgumentException e) {
148 Out.prln("Property " + e.getMessage() + " is empty or not defined!");
149
150 } finally {
151 if (propertiesFileInputStream != null) {
152 propertiesFileInputStream.close();
153 }
154 }
155 }
156
157 @Override
158 protected void tearDown() throws Exception {
159
160 FileOutputStream propertiesFileOutputStream = null;
161 try {
162 propertiesFileOutputStream = new FileOutputStream(propertiesFile);
163 properties.store(propertiesFileOutputStream,
164 " Property file for the JUnit TestApplication test.\n" +
165 "\n" +
166 "# the encoding for all the documents, optional\n" +
167 "#document.encoding=utf-8\n" +
168 "\n" +
169 "# the application to be run on the clean documents directory\n" +
170 "#application.file=/home/thomas/.tmp/gate/plugins/ANNIE/ANNIE_with_defaults.gapp\n" +
171 "\n" +
172 "# documents with only original markups\n" +
173 "# these are annotated with the current application\n" +
174 "#clean.documents.directory=/home/thomas/.tmp/corpus/gatecorpora/business/clean\n" +
175 "\n" +
176 "# the previous run documents directory that will be filled automatically\n" +
177 "# with the current run of the application on the clean documents\n" +
178 "# it can be empty the first time and then you will get no results\n" +
179 "#previous.run.directory=\n" +
180 "\n" +
181 "# annotated documents from the clean documents, optional\n" +
182 "#gold.standard.directory=/home/thomas/.tmp/corpus/gatecorpora/business/marked\n" +
183 "\n" +
184 "# annotation set, types and features\n" +
185 "# if an annotation set is empty then use the default set\n" +
186 "# only annotation.features is optional\n" +
187 "#application.annotation.set=\n" +
188 "#gold.annotation.set=Key\n" +
189 "#annotation.types=Location, Date, Person\n" +
190 "#annotation.features=\n" +
191 "\n" +
192 "# average F1 score threshold, between 0 and 1 to make the test failed\n" +
193 "# it also determines if the previous and current application must be\n" +
194 "# compared with the goldstandard, default value is 0.7\n" +
195 "# 0 will never fail\n" +
196 "# 1 will fail for any difference between documents\n" +
197 "#fail.threshold=0.9\n" +
198 "\n" +
199 "# file where to save the result in HTML format\n" +
200 "# if not set will use a temporary file\n" +
201 "#results.file=");
202 }
203 catch (IOException e) {
204 Out.prln("Could not save the properties file:");
205 Out.prln("test.application.properties");
206 Out.prln("The file must be in the directory from" +
207 " where you run the test.");
208 e.printStackTrace();
209 } finally {
210 if (propertiesFileOutputStream != null) {
211 propertiesFileOutputStream.close();
212 }
213 }
214 }
215
216 /**
217 <pre>
218 The logic is:
219 - annotate the clean docs with the current version
220 - perform anndif between current version and the previous version
221 - if there are any differences:
222 - perform anndif between current version and the GS
223 - perform anndif between previous version and the GS
224 - produce a report containing the evaluation numbers
225 for the 3 diffs performed
226 - produce a build artifact with the detailed changes
227 (actual individual annotations that are different, in e.g. HTML format,
228 similar to what CBT currently produces).
229 </pre>
230 */
231 public void test() {
232
233 Writer writer = null;
234 try {
235 // initialise GATE
236 if (!Gate.isInitialised()) { Gate.init(); }
237
238 // load the application
239 Out.prln("Load the application");
240 CorpusController controller = (CorpusController)
241 PersistenceManager.loadObjectFromFile(applicationFile);
242 controller.init();
243
244 // create a corpus from the clean documents
245 Corpus newCorpus = Factory.newCorpus("New corpus");
246 FileFilter acceptAllFileFilter = new FileFilter() {
247 public boolean accept(File pathname) {
248 return true;
249 }
250 };
251 newCorpus.populate(cleanDocumentsDirectory.toURI().toURL(),
252 acceptAllFileFilter, documentEncoding, false);
253
254 // run the application on the clean documents
255 Out.prln("Run the application on the clean "+newCorpus.size()+" documents");
256 controller.setCorpus(newCorpus);
257 controller.execute();
258
259 // store the resulting documents in a temporary directory
260 Out.prln("Save the documents processed in ");
261 File temporaryDirectory = File.createTempFile("gate-test-application-", null);
262 if (!temporaryDirectory.delete()
263 || !temporaryDirectory.mkdir()) {
264 throw new IOException("Unable to create temporary directory.\n"
265 + temporaryDirectory.getCanonicalPath());
266 }
267 for (Object object : newCorpus) {
268 Document document = (Document) object;
269 writer = new BufferedWriter(new FileWriter(new File(temporaryDirectory,
270 // use the same file name as the original document
271 Files.fileFromURL(document.getSourceUrl()).getName())));
272 writer.write(document.toXml());
273 writer.close();
274 }
275 Out.prln(temporaryDirectory.getPath());
276
277 // save the location of the previous run to be reuse later
278 properties.put("previous.run.directory",
279 temporaryDirectory.getCanonicalPath());
280
281 // if previous directory exist and is not empty
282 if (previousRunDirectory != null
283 && previousRunDirectory.listFiles().length > 0) {
284
285 // create the results file and write an HTML header
286 if (resultsFile == null) {
287 resultsFile = File.createTempFile(
288 "gate-test-application-results-", ".html");
289 properties.put("results.file", resultsFile.getCanonicalPath());
290 }
291 writer = new BufferedWriter(new FileWriter(resultsFile));
292 writer.write(BEGINHTML + nl);
293 writer.write(BEGINHEAD);
294 writer.write("GATE Test Application");
295 writer.write(ENDHEAD + nl);
296 writer.write("<h1>GATE Test Application</h1>" + nl);
297 writer.write("<p>Application: " + applicationFile.getPath() + "<br>" + nl);
298 writer.write("Fail Threshold: " + failThreshold + "</p>" + nl);
299 writer.write("<p> </p>" + nl);
300
301 // compare documents annotations between the previous and new run
302 Out.prln("Compare previous and new application");
303 writer.write("<h2>Compare previous and new application</h2>" + nl);
304 AnnotationDiffer annotationDiffer;
305 List<AnnotationDiffer.Pairing> pairings =
306 new ArrayList<AnnotationDiffer.Pairing>();
307 double averageF1MeasurePerDocument = 0;
308 for (Object object : newCorpus) {
309 Document newDocument = (Document) object;
310 writer.write("<h3>Document: " + newDocument.getName() + "</h3>" + nl);
311 String fileName = Files.fileFromURL(newDocument.getSourceUrl()).getName();
312 Document previousDocument = Factory.newDocument(new File(
313 previousRunDirectory, fileName).toURI().toURL(), documentEncoding);
314 double averageF1MeasurePerType = 0;
315 for (String type : annotationTypes) {
316 writer.write("<h4>Annotation type: " + type + "</h4>" + nl);
317 annotationDiffer = new AnnotationDiffer();
318 annotationDiffer.setSignificantFeaturesSet(annotationFeatures);
319 pairings.clear();
320 pairings.addAll(annotationDiffer.calculateDiff(
321 previousDocument.getAnnotations(applicationAnnotationSet).get(type),
322 newDocument.getAnnotations(applicationAnnotationSet).get(type)));
323 if (annotationDiffer.getCorrectMatches()
324 != annotationDiffer.getKeysCount()) {
325 writer.write(printHTMLForPairings(pairings, newDocument));
326 }
327 averageF1MeasurePerType += annotationDiffer.getFMeasureStrict(1);
328 }
329 averageF1MeasurePerDocument +=
330 averageF1MeasurePerType / annotationTypes.size();
331 writer.write("Average F1 measure is "
332 + averageF1MeasurePerType / annotationTypes.size() + "</p>" + nl);
333 }
334 averageF1MeasurePerDocument =
335 averageF1MeasurePerDocument / newCorpus.size();
336 Out.prln("Average F1 measure is " + averageF1MeasurePerDocument);
337
338 // if different enough then
339 if (averageF1MeasurePerDocument < failThreshold
340 && goldStandardDirectory != null) {
341
342 // compare previous with gold standard
343 Out.prln("Compare previous application and gold standard");
344 writer.write("<h2>Compare previous application and gold standard</h2>" + nl);
345 for (Object object : newCorpus) {
346 Document newDocument = (Document) object;
347 writer.write("<h3>Document: " + newDocument.getName() + "</h3>" + nl);
348 String fileName = Files.fileFromURL(
349 newDocument.getSourceUrl()).getName();
350 Document goldDocument = Factory.newDocument(new File(
351 goldStandardDirectory, fileName).toURI().toURL(), documentEncoding);
352 Document previousDocument = Factory.newDocument(new File(
353 previousRunDirectory, fileName).toURI().toURL(), documentEncoding);
354 for (String type : annotationTypes) {
355 writer.write("<h4>Annotation type: " + type + "</h4>" + nl);
356 annotationDiffer = new AnnotationDiffer();
357 annotationDiffer.setSignificantFeaturesSet(annotationFeatures);
358 pairings.clear();
359 pairings.addAll(annotationDiffer.calculateDiff(
360 goldDocument.getAnnotations(goldAnnotationSet).get(type),
361 previousDocument.getAnnotations(applicationAnnotationSet).get(type)));
362 if (annotationDiffer.getCorrectMatches()
363 != annotationDiffer.getKeysCount()) {
364 writer.write(printHTMLForPairings(pairings, newDocument));
365 }
366 }
367 }
368
369 // compare new with gold standard
370 Out.prln("Compare new application and gold standard");
371 writer.write("<h2>Compare new application and gold standard</h2>" + nl);
372 for (Object object : newCorpus) {
373 Document newDocument = (Document) object;
374 writer.write("<h3>Document: " + newDocument.getName() + "</h3>" + nl);
375 String fileName = Files.fileFromURL(
376 newDocument.getSourceUrl()).getName();
377 Document goldDocument = Factory.newDocument(new File(
378 goldStandardDirectory, fileName).toURI().toURL(), documentEncoding);
379 for (String type : annotationTypes) {
380 writer.write("<h4>Annotation type: " + type + "</h4>" + nl);
381 annotationDiffer = new AnnotationDiffer();
382 annotationDiffer.setSignificantFeaturesSet(annotationFeatures);
383 pairings.clear();
384 pairings.addAll(annotationDiffer.calculateDiff(
385 goldDocument.getAnnotations(goldAnnotationSet).get(type),
386 newDocument.getAnnotations(applicationAnnotationSet).get(type)));
387 if (annotationDiffer.getCorrectMatches()
388 != annotationDiffer.getKeysCount()) {
389 writer.write(printHTMLForPairings(pairings, newDocument));
390 }
391 }
392 }
393 }
394
395 // write an HTML footer in the results file
396 writer.write(ENDHTML + nl);
397 Out.prln("Results have been written to " + resultsFile.getPath());
398
399 if (averageF1MeasurePerDocument < failThreshold) {
400 // this test fail
401 fail("The average F1 measure is " + averageF1MeasurePerDocument
402 + " which is inferior to the fail threshold " + failThreshold);
403 }
404 } else {
405 Out.prln();
406 Out.prln("Previous run directory is missing or empty.");
407 Out.prln("You will need to run again this test with your new application");
408 Out.prln("so it can compute the differences.");
409 }
410
411 } catch (GateException e) {
412 e.printStackTrace();
413 } catch (GateRuntimeException e) {
414 e.printStackTrace();
415 } catch (IOException e) {
416 e.printStackTrace();
417 } finally {
418 if (writer != null) {
419 try {
420 writer.close();
421 } catch (IOException e) {
422 e.printStackTrace();
423 }
424 }
425 }
426 }
427
428 private String printHTMLForPairings(List<AnnotationDiffer.Pairing> pairings,
429 Document document) {
430 final String nl = Strings.getNl();
431 StringBuilder builder = new StringBuilder();
432
433 // table header
434 builder.append("<table cellpadding=\"0\" border=\"1\">").append(nl);
435 builder.append("<tr><th>Start</th><th>End</th><th>Key</th>");
436 if (annotationFeatures != null && !annotationFeatures.isEmpty()) {
437 builder.append("<th>Features</th>");
438 }
439 builder.append("<th>=?</th><th>Start</th><th>End</th><th>Response</th>");
440 if (annotationFeatures != null && !annotationFeatures.isEmpty()) {
441 builder.append("<th>Features</th>");
442 }
443 builder.append("</tr>").append(nl);
444
445 // table content
446 for (AnnotationDiffer.Pairing pairing : pairings) {
447 if (pairing.getType() == AnnotationDiffer.CORRECT_TYPE) {
448 continue;
449 }
450 Annotation key = pairing.getKey();
451 if (key == null) {
452 builder.append("<tr><td></td><td></td><td></td>");
453 if (annotationFeatures != null && !annotationFeatures.isEmpty()) {
454 builder.append("<td></td>");
455 }
456 } else {
457 String keyString;
458 try {
459 keyString = document.getContent().getContent(
460 key.getStartNode().getOffset(),
461 key.getEndNode().getOffset()).toString();
462 } catch(InvalidOffsetException e) {
463 throw new LuckyException(e);
464 }
465 builder.append("<tr>")
466 .append("<td>").append(key.getStartNode().getOffset().toString())
467 .append("</td><td>").append(key.getEndNode().getOffset().toString())
468 .append("</td><td>").append(keyString)
469 .append("</td>");
470 if (annotationFeatures != null && !annotationFeatures.isEmpty()) {
471 builder.append("<td>")
472 .append(key.getFeatures().toString()).append("</td>");
473 }
474 }
475 String type = "";
476 switch(pairing.getType()) {
477 case AnnotationDiffer.CORRECT_TYPE: type = "="; break;
478 case AnnotationDiffer.PARTIALLY_CORRECT_TYPE: type = "~"; break;
479 case AnnotationDiffer.MISSING_TYPE: type = "-?"; break;
480 case AnnotationDiffer.SPURIOUS_TYPE: type = "?-"; break;
481 case AnnotationDiffer.MISMATCH_TYPE: type = "<>"; break;
482 }
483 builder.append("<td>").append(type).append("</td>");
484 Annotation response = pairing.getResponse();
485 if (response == null) {
486 builder.append("<td></td><td></td><td></td>");
487 if (annotationFeatures != null && !annotationFeatures.isEmpty()) {
488 builder.append("<td></td>");
489 }
490 } else {
491 String responseString;
492 try {
493 responseString = document.getContent().getContent(
494 response.getStartNode().getOffset(),
495 response.getEndNode().getOffset()).toString();
496 } catch(InvalidOffsetException e) {
497 throw new LuckyException(e);
498 }
499 builder
500 .append("<td>").append(response.getStartNode().getOffset().toString())
501 .append("</td><td>").append(response.getEndNode().getOffset().toString())
502 .append("</td><td>").append(responseString).append("</td>");
503 if (annotationFeatures != null && !annotationFeatures.isEmpty()) {
504 builder.append("<td>")
505 .append(response.getFeatures().toString()).append("</td>");
506 }
507 }
508 builder.append("</tr>").append(nl);
509 }
510
511 builder.append("</table>").append(nl);
512 builder.append("<p> </p>").append(nl);
513 return builder.toString();
514 }
515
516 protected Properties properties;
517 protected File propertiesFile;
518 protected String documentEncoding;
519 protected File applicationFile;
520 protected File cleanDocumentsDirectory;
521 protected File previousRunDirectory;
522 protected File goldStandardDirectory;
523 protected String applicationAnnotationSet;
524 protected String goldAnnotationSet;
525 protected Set<String> annotationTypes;
526 protected Set<String> annotationFeatures;
527 protected double failThreshold;
528 protected File resultsFile;
529
530 final String nl = Strings.getNl();
531 static final String BEGINHTML =
532 "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">" +
533 "<html>";
534 static final String ENDHTML = "</body></html>";
535 static final String BEGINHEAD = "<head>" +
536 "<meta content=\"text/html; charset=utf-8\" http-equiv=\"content-type\">"
537 + "<title>";
538 static final String ENDHEAD = "</title></head><body>";
539 }
|