001 /**
002 * Copyright (c) 1995-2010, The University of Sheffield. See the file
003 * COPYRIGHT.txt in the software or at http://gate.ac.uk/gate/COPYRIGHT.txt
004 *
005 * This file is part of GATE (see http://gate.ac.uk/), and is free
006 * software, licenced under the GNU Library General Public License,
007 * Version 2, June 1991 (in the distribution as file licence.html,
008 * and also available at http://gate.ac.uk/gate/licence.html).
009 *
010 * $Id: ContingencyTable.java 12125 2010-01-04 14:44:43Z ggorrell $
011 */
012
013 package gate.util;
014
015 import gate.AnnotationSet;
016 import gate.Annotation;
017
018 import java.text.NumberFormat;
019 import java.util.ArrayList;
020 import java.util.Arrays;
021 import java.util.Collection;
022 import java.util.Collections;
023 import java.util.HashMap;
024 import java.util.HashSet;
025 import java.util.List;
026 import java.util.Locale;
027 import java.util.SortedSet;
028 import java.util.TreeSet;
029
030
031 /**
032 * Given two annotation sets, a type and a feature,
033 * compares the feature values. It finds matching annotations and treats
034 * the feature values as classifications. Its purpose is to calculate the
035 * extent of agreement between the feature values in the two annotation
036 * sets. It computes observed agreement and Kappa measures.
037 */
038 public class ClassificationMeasures {
039
040 /** Array of dimensions categories * categories. */
041 private float[][] confusionMatrix;
042
043 /** Cohen's kappa. */
044 private float kappaCohen = 0;
045
046 /** Scott's pi or Siegel & Castellan's kappa */
047 private float kappaPi = 0;
048
049 private boolean isCalculatedKappas = false;
050
051 /** List of feature values that are the labels of the confusion matrix */
052 private TreeSet<String> featureValues;
053
054 public ClassificationMeasures() {
055 // empty constructor
056 }
057
058 /**
059 * Portion of the instances on which the annotators agree.
060 * @return a number between 0 and 1. 1 means perfect agreements.
061 */
062 public float getObservedAgreement()
063 {
064 float agreed = getAgreedTrials();
065 float total = getTotalTrials();
066 if(total>0) {
067 return agreed/total;
068 } else {
069 return 0;
070 }
071 }
072
073 /**
074 * Kappa is defined as the observed agreements minus the agreement
075 * expected by chance.
076 * The CohenâÄôs Kappa is based on the individual distribution of each
077 * annotator.
078 * @return a number between -1 and 1. 1 means perfect agreements.
079 */
080 public float getKappaCohen()
081 {
082 if(!isCalculatedKappas){
083 computeKappaPairwise();
084 isCalculatedKappas = true;
085 }
086 return kappaCohen;
087 }
088
089 /**
090 * Kappa is defined as the observed agreements minus the agreement
091 * expected by chance.
092 * The Siegel & CastellanâÄôs Kappa is based on the assumption that all the
093 * annotators have the same distribution.
094 * @return a number between -1 and 1. 1 means perfect agreements.
095 */
096 public float getKappaPi()
097 {
098 if(!isCalculatedKappas){
099 computeKappaPairwise();
100 isCalculatedKappas = true;
101 }
102 return kappaPi;
103 }
104
105 /**
106 * To understand exactly which types are being confused with which other
107 * types you will need to view this array in conjunction with featureValues,
108 * which gives the class labels (annotation types) in the correct order.
109 * @return confusion matrix describing how annotations in one
110 * set are classified in the other and vice versa
111 */
112 public float[][] getConfusionMatrix(){
113 return confusionMatrix.clone();
114 }
115
116 /**
117 * This is necessary to make sense of the confusion matrix.
118 * @return list of annotation types (class labels) in the
119 * order in which they appear in the confusion matrix
120 */
121 public SortedSet<String> getFeatureValues(){
122 return Collections.unmodifiableSortedSet(featureValues);
123 }
124
125 /**
126 * Create a confusion matrix in which annotations of identical span
127 * bearing the specified feature name are compared in terms of feature value.
128 * Compiles list of classes (feature values) on the fly.
129 *
130 * @param aS1 annotation set to compare to the second
131 * @param aS2 annotation set to compare to the first
132 * @param type annotation type containing the features to compare
133 * @param feature feature name whose values will be compared
134 * @param verbose message error output when ignoring annotations
135 */
136 public void calculateConfusionMatrix(AnnotationSet aS1, AnnotationSet aS2,
137 String type, String feature, boolean verbose)
138 {
139 // We'll accumulate a list of the feature values (a.k.a. class labels)
140 featureValues = new TreeSet<String>();
141
142 // Make a hash of hashes for the counts.
143 HashMap<String, HashMap<String, Float>> countMap =
144 new HashMap<String, HashMap<String, Float>>();
145
146 // Get all the annotations of the correct type containing
147 // the correct feature
148 HashSet<String> featureSet = new HashSet<String>();
149 featureSet.add(feature);
150 AnnotationSet relevantAnns1 = aS1.get(type, featureSet);
151 AnnotationSet relevantAnns2 = aS2.get(type, featureSet);
152
153 // For each annotation in aS1, find the match in aS2
154 for (Annotation relevantAnn1 : relevantAnns1) {
155
156 // First we need to check that this annotation is not identical in span
157 // to anything else in the same set. Duplicates should be excluded.
158 List<Annotation> dupeAnnotations = new ArrayList<Annotation>();
159 for (Annotation aRelevantAnns1 : relevantAnns1) {
160 if (aRelevantAnns1.equals(relevantAnn1)) { continue; }
161 if (aRelevantAnns1.coextensive(relevantAnn1)) {
162 dupeAnnotations.add(aRelevantAnns1);
163 dupeAnnotations.add(relevantAnn1);
164 }
165 }
166
167 if (dupeAnnotations.size() > 1) {
168 if (verbose) {
169 Out.prln("ClassificationMeasures: " +
170 "Same span annotations in set 1 detected! Ignoring.");
171 Out.prln(Arrays.toString(dupeAnnotations.toArray()));
172 }
173 } else {
174 // Find the match in as2
175 List<Annotation> coextensiveAnnotations = new ArrayList<Annotation>();
176 for (Annotation relevantAnn2 : relevantAnns2) {
177 if (relevantAnn2.coextensive(relevantAnn1)) {
178 coextensiveAnnotations.add(relevantAnn2);
179 }
180 }
181
182 if (coextensiveAnnotations.size() == 0) {
183 if (verbose) {
184 Out.prln("ClassificationMeasures: Annotation in set 1 " +
185 "with no counterpart in set 2 detected! Ignoring.");
186 Out.prln(relevantAnn1.toString());
187 }
188 } else if (coextensiveAnnotations.size() == 1) {
189
190 // What are our feature values?
191 String featVal1 = (String) relevantAnn1.getFeatures().get(feature);
192 String featVal2 = (String)
193 coextensiveAnnotations.get(0).getFeatures().get(feature);
194
195 // Make sure both are present in our feature value list
196 featureValues.add(featVal1);
197 featureValues.add(featVal2);
198
199 // Update the matrix hash of hashes
200 // Get the right hashmap for the as1 feature value
201 HashMap<String, Float> subHash = countMap.get(featVal1);
202 if (subHash == null) {
203 // This is a new as1 feature value, since it has no subhash yet
204 HashMap<String, Float> subHashForNewAS1FeatVal =
205 new HashMap<String, Float>();
206
207 // Since it is a new as1 feature value, there can be no existing
208 // as2 feature values paired with it. So we make a new one for this
209 // as2 feature value
210 subHashForNewAS1FeatVal.put(featVal2, (float) 1);
211
212 countMap.put(featVal1, subHashForNewAS1FeatVal);
213 } else {
214 // Increment the count
215 Float count = subHash.get(featVal2);
216 if (count == null) {
217 subHash.put(featVal2, (float) 1);
218 } else {
219 subHash.put(featVal2, (float) count.intValue() + 1);
220 }
221
222 }
223 } else if (coextensiveAnnotations.size() > 1) {
224 if (verbose) {
225 Out.prln("ClassificationMeasures: " +
226 "Same span annotations in set 2 detected! Ignoring.");
227 Out.prln(Arrays.toString(coextensiveAnnotations.toArray()));
228 }
229 }
230 }
231 }
232
233 // Now we have this hash of hashes, but the calculation implementations
234 // require an array of floats. So for now we can just translate it.
235 confusionMatrix = convert2DHashTo2DFloatArray(countMap, featureValues);
236 }
237
238 /**
239 * Given a list of ClassificationMeasures, this will combine to make
240 * a megatable. Then you can use kappa getters to get micro average
241 * figures for the entire set.
242 * @param tables tables to combine
243 */
244 public ClassificationMeasures(Collection<ClassificationMeasures> tables) {
245 /* A hash of hashes for the actual values.
246 * This will later be converted to a 2D float array for
247 * compatibility with the existing code. */
248 HashMap<String, HashMap<String, Float>> countMap =
249 new HashMap<String, HashMap<String, Float>>();
250
251 /* Make a new feature values set which is a superset of all the others */
252 TreeSet<String> newFeatureValues = new TreeSet<String>();
253
254 /* Now we are going to add each new contingency table in turn */
255
256 for (ClassificationMeasures table : tables) {
257 int it1index = 0;
258 for (String featureValue1 : table.featureValues) {
259 newFeatureValues.add(featureValue1);
260 int it2index = 0;
261 for (String featureValue2 : table.featureValues) {
262
263 /* So we have the labels of the count we want to add */
264 /* What is the value we want to add? */
265 Float valtoadd = table.confusionMatrix[it1index][it2index];
266
267 HashMap<String, Float> subHash = countMap.get(featureValue1);
268 if (subHash == null) {
269 /* This is a new as1 feature value, since it has no subhash yet */
270 HashMap<String, Float> subHashForNewAS1FeatVal =
271 new HashMap<String, Float>();
272
273 /* Since it is a new as1 feature value, there can be no existing
274 * as2 feature values paired with it. So we make a new one for this
275 * as2 feature value */
276 subHashForNewAS1FeatVal.put(featureValue2, valtoadd);
277
278 countMap.put(featureValue1, subHashForNewAS1FeatVal);
279 } else {
280 /* Increment the count */
281 Float count = subHash.get(featureValue2);
282 if (count == null) {
283 subHash.put(featureValue2, valtoadd);
284 } else {
285 subHash.put(featureValue2, count.intValue() + valtoadd);
286 }
287 }
288 it2index++;
289 }
290 it1index++;
291 }
292 }
293
294 confusionMatrix = convert2DHashTo2DFloatArray(countMap, newFeatureValues);
295 featureValues = newFeatureValues;
296 isCalculatedKappas = false;
297 }
298
299 /** Compute Cohen's and Pi kappas for two annotators.
300 */
301 protected void computeKappaPairwise()
302 {
303 // Compute the agreement
304 float observedAgreement = getObservedAgreement();
305 int numCats = featureValues.size();
306 // compute the agreement by chance
307 // Get the marginal sum for each annotator
308 float[] marginalArrayC = new float[numCats];
309 float[] marginalArrayR = new float[numCats];
310 float totalSum = 0;
311 for(int i = 0; i < numCats; ++i) {
312 float sum = 0;
313 for(int j = 0; j < numCats; ++j)
314 sum += confusionMatrix[i][j];
315 marginalArrayC[i] = sum;
316 totalSum += sum;
317 sum = 0;
318 for(int j = 0; j < numCats; ++j)
319 sum += confusionMatrix[j][i];
320 marginalArrayR[i] = sum;
321 }
322 // Compute Cohen's p(E)
323 float pE = 0;
324 if(totalSum > 0) {
325 float doubleSum = totalSum * totalSum;
326 for(int i = 0; i < numCats; ++i)
327 pE += (marginalArrayC[i] * marginalArrayR[i]) / doubleSum;
328 }
329 // Compute Cohen's Kappa
330 if(totalSum > 0) // FIXME: division by zero when pE = 1
331 kappaCohen = (observedAgreement - pE) / (1 - pE);
332 else kappaCohen = 0;
333 // Compute S&C's chance agreement
334 pE = 0;
335 if(totalSum > 0) {
336 float doubleSum = 2 * totalSum;
337 for(int i = 0; i < numCats; ++i) {
338 float p = (marginalArrayC[i] + marginalArrayR[i]) / doubleSum;
339 pE += p * p;
340 }
341 }
342 if(totalSum > 0) // FIXME: division by zero when pE = 1
343 kappaPi = (observedAgreement - pE) / (1 - pE);
344 else kappaPi = 0;
345 // Compute the specific agreement for each label using marginal sums
346 float[][] sAgreements = new float[numCats][2];
347 for(int i = 0; i < numCats; ++i) {
348 if(marginalArrayC[i] + marginalArrayR[i]>0)
349 sAgreements[i][0] = (2 * confusionMatrix[i][i])
350 / (marginalArrayC[i] + marginalArrayR[i]);
351 else sAgreements[i][0] = 0.0f;
352 if(2 * totalSum - marginalArrayC[i] - marginalArrayR[i]>0)
353 sAgreements[i][1] = (2 * (totalSum - marginalArrayC[i]
354 - marginalArrayR[i] + confusionMatrix[i][i]))
355 / (2 * totalSum - marginalArrayC[i] - marginalArrayR[i]);
356 else sAgreements[i][1] = 0.0f;
357 }
358 }
359
360 /** Gets the number of annotations for which the two annotation sets
361 * are in agreement with regards to the annotation type.
362 * @return Number of agreed trials
363 */
364 public float getAgreedTrials(){
365 float sumAgreed = 0;
366 for(int i = 0; i < featureValues.size(); ++i) {
367 sumAgreed += confusionMatrix[i][i];
368 }
369 return sumAgreed;
370 }
371
372 /** Gets the total number of annotations in the two sets.
373 * Note that only matched annotations (identical span) are
374 * considered.
375 * @return Number of trials
376 */
377 public float getTotalTrials(){
378 float sumTotal = 0;
379 for(int i = 0; i < featureValues.size(); ++i) {
380 for(int j = 0; j < featureValues.size(); ++j) {
381 sumTotal += confusionMatrix[i][j];
382 }
383 }
384 return sumTotal;
385 }
386
387 /**
388 * @param title matrix title
389 * @return confusion matrix as a list of list of String
390 */
391 public List<List<String>> getConfusionMatrix(String title) {
392 List<List<String>> matrix = new ArrayList<List<String>>();
393 List<String> row = new ArrayList<String>();
394 row.add(" ");
395 matrix.add(row); // spacer
396 row = new ArrayList<String>();
397 row.add(title);
398 matrix.add(row); // title
399 SortedSet<String> features = new TreeSet<String>(getFeatureValues());
400 row = new ArrayList<String>();
401 row.add("");
402 row.addAll(features);
403 matrix.add(row); // heading horizontal
404 for (float[] confusionValues : getConfusionMatrix()) {
405 row = new ArrayList<String>();
406 row.add(features.first()); // heading vertical
407 features.remove(features.first());
408 for (float confusionValue : confusionValues) {
409 row.add(String.valueOf((int) confusionValue));
410 }
411 matrix.add(row); // confusion values
412 }
413 return matrix;
414 }
415
416 public List<String> getMeasuresRow(Object[] measures, String documentName) {
417 NumberFormat f = NumberFormat.getInstance(Locale.ENGLISH);
418 f.setMaximumFractionDigits(2);
419 f.setMinimumFractionDigits(2);
420 List<String> row = new ArrayList<String>();
421 row.add(documentName);
422 row.add(String.valueOf((int) getAgreedTrials()));
423 row.add(String.valueOf((int) getTotalTrials()));
424 for (Object object : measures) {
425 String measure = (String) object;
426 if (measure.equals("Observed agreement")) {
427 row.add(f.format(getObservedAgreement()));
428 }
429 if (measure.equals("Cohen's Kappa")) {
430 float result = getKappaCohen();
431 row.add(Float.isNaN(result) ? "" : f.format(result));
432 }
433 if (measure.equals("Pi's Kappa")) {
434 float result = getKappaPi();
435 row.add(Float.isNaN(result) ? "" : f.format(result));
436 }
437 }
438 return row;
439 }
440
441 /**
442 * Convert between two formats of confusion matrix.
443 * A hashmap of hashmaps is easier to populate but an array is better for
444 * matrix computation.
445 * @param countMap count for each label as in confusion matrix
446 * @param featureValues sorted set of labels that will define the dimensions
447 * @return converted confusion matrix as an 2D array
448 */
449 private float[][] convert2DHashTo2DFloatArray(
450 HashMap<String, HashMap<String, Float>> countMap,
451 TreeSet<String> featureValues)
452 {
453 int dimensionOfContingencyTable = featureValues.size();
454 float[][] matrix =
455 new float[dimensionOfContingencyTable][dimensionOfContingencyTable];
456 int i=0;
457 int j=0;
458 for (String featureValue1 : featureValues) {
459 HashMap<String, Float> hashForThisAS1FeatVal =
460 countMap.get(featureValue1);
461 j = 0;
462 for (String featureValue2 : featureValues) {
463 Float count = null;
464 if (hashForThisAS1FeatVal != null) {
465 count = hashForThisAS1FeatVal.get(featureValue2);
466 }
467 if (count != null) {
468 matrix[i][j] = count;
469 } else {
470 matrix[i][j] = 0;
471 }
472 j++;
473 }
474 i++;
475 }
476 return matrix;
477 }
478
479 }
|