0001 /*
0002 * Copyright (c) 2009-2010, Ontotext AD.
0003 * Copyright (c) 1995-2010, The University of Sheffield. See the file
0004 * COPYRIGHT.txt in the software or at http://gate.ac.uk/gate/COPYRIGHT.txt
0005 *
0006 * This file is part of GATE (see http://gate.ac.uk/), and is free
0007 * software, licenced under the GNU Library General Public License,
0008 * Version 2, June 1991 (in the distribution as file licence.html,
0009 * and also available at http://gate.ac.uk/gate/licence.html).
0010 *
0011 * Thomas Heitz - 10 June 2009
0012 *
0013 * $Id: CorpusQualityAssurance.java 13121 2010-10-06 12:52:51Z thomas_heitz $
0014 */
0015
0016 package gate.gui;
0017
0018 import java.awt.BorderLayout;
0019 import java.awt.Cursor;
0020 import java.awt.Dimension;
0021 import java.awt.GridBagConstraints;
0022 import java.awt.GridBagLayout;
0023 import java.awt.Insets;
0024 import java.awt.event.ActionEvent;
0025 import java.awt.event.KeyEvent;
0026 import java.awt.event.MouseAdapter;
0027 import java.awt.event.MouseEvent;
0028 import java.io.BufferedWriter;
0029 import java.io.File;
0030 import java.io.FileWriter;
0031 import java.io.IOException;
0032 import java.io.Writer;
0033 import java.net.MalformedURLException;
0034 import java.net.URL;
0035 import java.util.*;
0036 import java.util.Timer;
0037 import java.text.NumberFormat;
0038 import java.text.Collator;
0039
0040 import javax.swing.*;
0041 import javax.swing.event.AncestorEvent;
0042 import javax.swing.event.AncestorListener;
0043 import javax.swing.event.ChangeEvent;
0044 import javax.swing.event.ChangeListener;
0045 import javax.swing.event.ListSelectionEvent;
0046 import javax.swing.event.ListSelectionListener;
0047 import javax.swing.table.DefaultTableModel;
0048 import javax.swing.table.JTableHeader;
0049 import javax.swing.text.Position;
0050
0051 import gate.Annotation;
0052 import gate.AnnotationSet;
0053 import gate.Corpus;
0054 import gate.Document;
0055 import gate.Factory;
0056 import gate.Gate;
0057 import gate.Resource;
0058 import gate.creole.AbstractVisualResource;
0059 import gate.event.CorpusEvent;
0060 import gate.creole.metadata.CreoleResource;
0061 import gate.creole.metadata.GuiType;
0062 import gate.event.CorpusListener;
0063 import gate.swing.XJTable;
0064 import gate.swing.XJFileChooser;
0065 import gate.util.AnnotationDiffer;
0066 import gate.util.ClassificationMeasures;
0067 import gate.util.OntologyMeasures;
0068 import gate.util.ExtensionFileFilter;
0069 import gate.util.OptionsMap;
0070 import gate.util.Strings;
0071
0072 /**
0073 * Quality assurance corpus view.
0074 * Compare two sets of annotations with optionally their features
0075 * globally for each annotation and for each document inside a corpus
0076 * with different measures notably precision, recall and F1-score.
0077 */
0078 @CreoleResource(name = "Corpus Quality Assurance", guiType = GuiType.LARGE,
0079 resourceDisplayed = "gate.Corpus", mainViewer = false,
0080 helpURL = "http://gate.ac.uk/userguide/sec:eval:corpusqualityassurance")
0081 public class CorpusQualityAssurance extends AbstractVisualResource
0082 implements CorpusListener {
0083
0084 public Resource init(){
0085 initLocalData();
0086 initGuiComponents();
0087 initListeners();
0088 return this;
0089 }
0090
0091 protected void initLocalData(){
0092 collator = Collator.getInstance(Locale.ENGLISH);
0093 collator.setStrength(Collator.TERTIARY);
0094 documentTableModel = new DefaultTableModel();
0095 documentTableModel.addColumn("Document");
0096 documentTableModel.addColumn("Match");
0097 documentTableModel.addColumn("Only A");
0098 documentTableModel.addColumn("Only B");
0099 documentTableModel.addColumn("Overlap");
0100 annotationTableModel = new DefaultTableModel();
0101 annotationTableModel.addColumn("Annotation");
0102 annotationTableModel.addColumn("Match");
0103 annotationTableModel.addColumn("Only A");
0104 annotationTableModel.addColumn("Only B");
0105 annotationTableModel.addColumn("Overlap");
0106 document2TableModel = new DefaultTableModel();
0107 document2TableModel.addColumn("Document");
0108 document2TableModel.addColumn("Agreed");
0109 document2TableModel.addColumn("Total");
0110 confusionTableModel = new DefaultTableModel();
0111 types = new TreeSet<String>(collator);
0112 corpusChanged = false;
0113 measuresType = FSCORE_MEASURES;
0114 doubleComparator = new Comparator<String>() {
0115 public int compare(String s1, String s2) {
0116 if (s1 == null || s2 == null) {
0117 return 0;
0118 } else if (s1.equals("")) {
0119 return 1;
0120 } else if (s2.equals("")) {
0121 return -1;
0122 } else {
0123 return Double.valueOf(s1).compareTo(Double.valueOf(s2));
0124 }
0125 }
0126 };
0127 totalComparator = new Comparator<String>() {
0128 public int compare(String s1, String s2) {
0129 if (s1 == null || s2 == null) {
0130 return 0;
0131 } else if (s1.equals("Micro summary")) {
0132 return s2.equals("Macro summary") ? -1 : 1;
0133 } else if (s1.equals("Macro summary")) {
0134 return s2.equals("Micro summary") ? -1 : 1;
0135 } else if (s2.equals("Micro summary")) {
0136 return s1.equals("Macro summary") ? 1 : -1;
0137 } else if (s2.equals("Macro summary")) {
0138 return s1.equals("Micro summary") ? 1 : -1;
0139 } else {
0140 return s1.compareTo(s2);
0141 }
0142 }
0143 };
0144 }
0145
0146 protected void initGuiComponents() {
0147 setLayout(new BorderLayout());
0148
0149 JPanel sidePanel = new JPanel(new GridBagLayout());
0150 GridBagConstraints gbc = new GridBagConstraints();
0151 gbc.gridx = 0;
0152 sidePanel.add(Box.createVerticalStrut(5), gbc);
0153
0154 // toolbar
0155 JToolBar toolbar = new JToolBar();
0156 toolbar.setFloatable(false);
0157 toolbar.add(openDocumentAction = new OpenDocumentAction());
0158 openDocumentAction.setEnabled(false);
0159 toolbar.add(openAnnotationDiffAction = new OpenAnnotationDiffAction());
0160 openAnnotationDiffAction.setEnabled(false);
0161 toolbar.add(exportToHtmlAction = new ExportToHtmlAction());
0162 toolbar.add(reloadCacheAction = new ReloadCacheAction());
0163 toolbar.add(new HelpAction());
0164 gbc.anchor = GridBagConstraints.NORTHWEST;
0165 sidePanel.add(toolbar, gbc);
0166 gbc.anchor = GridBagConstraints.NORTH;
0167 sidePanel.add(Box.createVerticalStrut(5), gbc);
0168
0169 // annotation sets list
0170 JLabel label = new JLabel("Annotation Sets A/Key & B/Response");
0171 label.setToolTipText("aka 'Key & Response sets'");
0172 gbc.fill = GridBagConstraints.BOTH;
0173 sidePanel.add(label, gbc);
0174 sidePanel.add(Box.createVerticalStrut(2), gbc);
0175 setList = new JList();
0176 setList.setSelectionModel(new ToggleSelectionABModel(setList));
0177 setList.setPrototypeCellValue("present in every document");
0178 setList.setVisibleRowCount(4);
0179 gbc.weighty = 1;
0180 sidePanel.add(new JScrollPane(setList), gbc);
0181 gbc.weighty = 0;
0182 sidePanel.add(Box.createVerticalStrut(2), gbc);
0183 setCheck = new JCheckBox("present in every document", false);
0184 setCheck.addActionListener(new AbstractAction(){
0185 public void actionPerformed(ActionEvent e) {
0186 updateSetList();
0187 }
0188 });
0189 sidePanel.add(setCheck, gbc);
0190 sidePanel.add(Box.createVerticalStrut(5), gbc);
0191
0192 // annotation types list
0193 label = new JLabel("Annotation Types");
0194 label.setToolTipText("Annotation types to compare");
0195 sidePanel.add(label, gbc);
0196 sidePanel.add(Box.createVerticalStrut(2), gbc);
0197 typeList = new JList();
0198 typeList.setSelectionModel(new ToggleSelectionModel());
0199 typeList.setPrototypeCellValue("present in every document");
0200 typeList.setVisibleRowCount(4);
0201 gbc.weighty = 1;
0202 sidePanel.add(new JScrollPane(typeList), gbc);
0203 gbc.weighty = 0;
0204 sidePanel.add(Box.createVerticalStrut(2), gbc);
0205 typeCheck = new JCheckBox("present in every selected set", false);
0206 typeCheck.addActionListener(new AbstractAction(){
0207 public void actionPerformed(ActionEvent e) {
0208 setList.getListSelectionListeners()[0].valueChanged(null);
0209 }
0210 });
0211 sidePanel.add(typeCheck, gbc);
0212 sidePanel.add(Box.createVerticalStrut(5), gbc);
0213
0214 // annotation features list
0215 label = new JLabel("Annotation Features");
0216 label.setToolTipText("Annotation features to compare");
0217 sidePanel.add(label, gbc);
0218 sidePanel.add(Box.createVerticalStrut(2), gbc);
0219 featureList = new JList();
0220 featureList.setSelectionModel(new ToggleSelectionModel());
0221 featureList.setPrototypeCellValue("present in every document");
0222 featureList.setVisibleRowCount(4);
0223 gbc.weighty = 1;
0224 sidePanel.add(new JScrollPane(featureList), gbc);
0225 gbc.weighty = 0;
0226 sidePanel.add(Box.createVerticalStrut(2), gbc);
0227 featureCheck = new JCheckBox("present in every selected type", false);
0228 featureCheck.addActionListener(new AbstractAction(){
0229 public void actionPerformed(ActionEvent e) {
0230 typeList.getListSelectionListeners()[0].valueChanged(null);
0231 }
0232 });
0233 sidePanel.add(featureCheck, gbc);
0234 sidePanel.add(Box.createVerticalStrut(5), gbc);
0235
0236 // measures tabbed panes
0237 label = new JLabel("Measures");
0238 label.setToolTipText("Measures used to compare annotations");
0239 optionsButton = new JToggleButton("Options");
0240 optionsButton.setMargin(new Insets(1, 1, 1, 1));
0241 JPanel labelButtonPanel = new JPanel(new BorderLayout());
0242 labelButtonPanel.add(label, BorderLayout.WEST);
0243 labelButtonPanel.add(optionsButton, BorderLayout.EAST);
0244 sidePanel.add(labelButtonPanel, gbc);
0245 sidePanel.add(Box.createVerticalStrut(2), gbc);
0246 final JScrollPane measureScrollPane = new JScrollPane();
0247 measureList = new JList();
0248 measureList.setSelectionModel(new ToggleSelectionModel());
0249 String prefix = getClass().getName() + '.';
0250 double beta = (userConfig.getDouble(prefix+"fscorebeta") == null) ?
0251 1.0 : userConfig.getDouble(prefix+"fscorebeta");
0252 double beta2 = (userConfig.getDouble(prefix+"fscorebeta2") == null) ?
0253 0.5 : userConfig.getDouble(prefix+"fscorebeta2");
0254 String fscore = "F" + beta + "-score ";
0255 String fscore2 = "F" + beta2 + "-score ";
0256 measureList.setModel(new ExtendedListModel(new String[]{
0257 fscore+"strict", fscore+"lenient", fscore+"average",
0258 fscore+"strict BDM", fscore+"lenient BDM", fscore+"average BDM",
0259 fscore2+"strict", fscore2+"lenient", fscore2+"average",
0260 fscore2+"strict BDM", fscore2+"lenient BDM", fscore2+"average BDM"}));
0261 measureList.setPrototypeCellValue("present in every document");
0262 measureList.setVisibleRowCount(4);
0263 measureScrollPane.setViewportView(measureList);
0264 final JScrollPane measure2ScrollPane = new JScrollPane();
0265 measure2List = new JList();
0266 measure2List.setSelectionModel(new ToggleSelectionModel());
0267 measure2List.setModel(new ExtendedListModel(new String[]{
0268 "Observed agreement", "Cohen's Kappa" , "Pi's Kappa"}));
0269 measure2List.setPrototypeCellValue("present in every document");
0270 measure2List.setVisibleRowCount(4);
0271 measure2ScrollPane.setViewportView(measure2List);
0272 measureTabbedPane = new JTabbedPane();
0273 measureTabbedPane.addTab("F-Score", null,
0274 measureScrollPane, "Inter-annotator agreement");
0275 measureTabbedPane.addTab("Classification", null,
0276 measure2ScrollPane, "Classification agreement");
0277 gbc.weighty = 1;
0278 sidePanel.add(measureTabbedPane, gbc);
0279 gbc.weighty = 0;
0280 sidePanel.add(Box.createVerticalStrut(5), gbc);
0281 sidePanel.add(Box.createVerticalGlue(), gbc);
0282
0283 // options panel for fscore measures
0284 final JPanel measureOptionsPanel = new JPanel();
0285 measureOptionsPanel.setLayout(
0286 new BoxLayout(measureOptionsPanel, BoxLayout.Y_AXIS));
0287 JPanel betaPanel = new JPanel();
0288 betaPanel.setLayout(new BoxLayout(betaPanel, BoxLayout.X_AXIS));
0289 JLabel betaLabel = new JLabel("Fscore Beta 1:");
0290 final JSpinner betaSpinner =
0291 new JSpinner(new SpinnerNumberModel(beta, 0, 1, 0.1));
0292 betaSpinner.setToolTipText(
0293 "<html>Relative weight of precision and recall." +
0294 "<ul><li>1 weights equally precision and recall" +
0295 "<li>0.5 weights precision twice as much as recall" +
0296 "<li>2 weights recall twice as much as precision</ul></html>");
0297 betaPanel.add(betaLabel);
0298 betaPanel.add(Box.createHorizontalStrut(5));
0299 betaPanel.add(betaSpinner);
0300 betaPanel.add(Box.createHorizontalGlue());
0301 measureOptionsPanel.add(betaPanel);
0302 betaSpinner.setMaximumSize(new Dimension(Integer.MAX_VALUE,
0303 Math.round(betaLabel.getPreferredSize().height*1.5f)));
0304 JPanel beta2Panel = new JPanel();
0305 beta2Panel.setLayout(new BoxLayout(beta2Panel, BoxLayout.X_AXIS));
0306 JLabel beta2Label = new JLabel("Fscore Beta 2:");
0307 final JSpinner beta2Spinner =
0308 new JSpinner(new SpinnerNumberModel(beta2, 0, 1, 0.1));
0309 beta2Spinner.setToolTipText(betaSpinner.getToolTipText());
0310 beta2Panel.add(beta2Label);
0311 beta2Panel.add(Box.createHorizontalStrut(5));
0312 beta2Panel.add(beta2Spinner);
0313 beta2Panel.add(Box.createHorizontalGlue());
0314 measureOptionsPanel.add(beta2Panel);
0315 beta2Spinner.setMaximumSize(new Dimension(Integer.MAX_VALUE,
0316 Math.round(beta2Label.getPreferredSize().height*1.5f)));
0317 JPanel bdmFilePanel = new JPanel();
0318 bdmFilePanel.setLayout(new BoxLayout(bdmFilePanel, BoxLayout.X_AXIS));
0319 JLabel bdmFileLabel = new JLabel("BDM file:");
0320 JButton bdmFileButton = new JButton(new SetBdmFileAction());
0321 bdmFilePanel.add(bdmFileLabel);
0322 bdmFilePanel.add(Box.createHorizontalStrut(5));
0323 bdmFilePanel.add(bdmFileButton);
0324 bdmFilePanel.add(Box.createHorizontalGlue());
0325 measureOptionsPanel.add(bdmFilePanel);
0326
0327 // options panel for classification measures
0328 final JPanel measure2OptionsPanel = new JPanel();
0329 measure2OptionsPanel.setLayout(
0330 new BoxLayout(measure2OptionsPanel, BoxLayout.Y_AXIS));
0331 JPanel verbosePanel = new JPanel();
0332 verbosePanel.setLayout(new BoxLayout(verbosePanel, BoxLayout.X_AXIS));
0333 boolean verbose = (userConfig.getBoolean(prefix+"verbose") == null) ?
0334 false : userConfig.getBoolean(prefix+"verbose");
0335 verboseOptionCheckBox = new JCheckBox("Output ignored annotations",verbose);
0336 verbosePanel.add(verboseOptionCheckBox);
0337 verbosePanel.add(Box.createHorizontalGlue());
0338 measure2OptionsPanel.add(verbosePanel);
0339
0340 // options button action
0341 optionsButton.setAction(new AbstractAction("Options") {
0342 int[] selectedIndices;
0343 public void actionPerformed(ActionEvent e) {
0344 JToggleButton button = (JToggleButton) e.getSource();
0345 // switch measure options panel and measure list
0346 if (button.isSelected()) {
0347 if (measuresType == FSCORE_MEASURES) {
0348 selectedIndices = measureList.getSelectedIndices();
0349 measureScrollPane.setViewportView(measureOptionsPanel);
0350 } else if (measuresType == CLASSIFICATION_MEASURES) {
0351 selectedIndices = measure2List.getSelectedIndices();
0352 measure2ScrollPane.setViewportView(measure2OptionsPanel);
0353 }
0354 } else {
0355 String prefix = getClass().getEnclosingClass().getName() + '.';
0356 if (measuresType == FSCORE_MEASURES) {
0357 // update beta with new values
0358 String fscore = "F" + betaSpinner.getValue() + "-score ";
0359 String fscore2 = "F" + beta2Spinner.getValue() + "-score ";
0360 measureList.setModel(new ExtendedListModel(new String[]{
0361 fscore+"strict", fscore+"lenient", fscore+"average",
0362 fscore+"strict BDM", fscore+"lenient BDM", fscore+"average BDM",
0363 fscore2+"strict", fscore2+"lenient", fscore2+"average",
0364 fscore2+"strict BDM", fscore2+"lenient BDM", fscore2+"average BDM"}));
0365 // save in GATE preferences
0366 userConfig.put(prefix+"fscorebeta", betaSpinner.getValue());
0367 userConfig.put(prefix+"fscorebeta2", beta2Spinner.getValue());
0368 // put back the list and its selection
0369 measureScrollPane.setViewportView(measureList);
0370 measureList.setSelectedIndices(selectedIndices);
0371 } else if (measuresType == CLASSIFICATION_MEASURES) {
0372 userConfig.put(prefix+"verbose",verboseOptionCheckBox.isSelected());
0373 measure2ScrollPane.setViewportView(measure2List);
0374 measure2List.setSelectedIndices(selectedIndices);
0375 }
0376 }
0377 }
0378 });
0379
0380 // compare button and progress bar
0381 JButton compareButton = new JButton(compareAction = new CompareAction());
0382 compareAction.setEnabled(false);
0383 sidePanel.add(compareButton, gbc);
0384 sidePanel.add(Box.createVerticalStrut(5), gbc);
0385 progressBar = new JProgressBar();
0386 progressBar.setStringPainted(true);
0387 progressBar.setString("");
0388 sidePanel.add(progressBar, gbc);
0389 sidePanel.add(Box.createVerticalStrut(5), gbc);
0390
0391 // tables
0392 annotationTable = new XJTable() {
0393 public boolean isCellEditable(int rowIndex, int vColIndex) {
0394 return false;
0395 }
0396 protected JTableHeader createDefaultTableHeader() {
0397 return new JTableHeader(columnModel) {
0398 public String getToolTipText(MouseEvent event) {
0399 int index = columnModel.getColumnIndexAtX(event.getPoint().x);
0400 if (index == -1) { return null; }
0401 int modelIndex = columnModel.getColumn(index).getModelIndex();
0402 String columnName = this.table.getModel().getColumnName(modelIndex);
0403 return createToolTipFromColumnName(columnName);
0404 }
0405 };
0406 }
0407 };
0408 annotationTable.setModel(annotationTableModel);
0409 annotationTable.setSortable(false);
0410 annotationTable.setEnableHidingColumns(true);
0411 annotationTable.setAutoResizeMode(XJTable.AUTO_RESIZE_ALL_COLUMNS);
0412 documentTable = new XJTable() {
0413 public boolean isCellEditable(int rowIndex, int vColIndex) {
0414 return false;
0415 }
0416 protected JTableHeader createDefaultTableHeader() {
0417 return new JTableHeader(columnModel) {
0418 public String getToolTipText(MouseEvent event) {
0419 int index = columnModel.getColumnIndexAtX(event.getPoint().x);
0420 if (index == -1) { return null; }
0421 int modelIndex = columnModel.getColumn(index).getModelIndex();
0422 String columnName = this.table.getModel().getColumnName(modelIndex);
0423 return createToolTipFromColumnName(columnName);
0424 }
0425 };
0426 }
0427 };
0428 documentTable.setModel(documentTableModel);
0429 documentTable.setSortable(false);
0430 documentTable.setEnableHidingColumns(true);
0431 documentTable.setAutoResizeMode(XJTable.AUTO_RESIZE_ALL_COLUMNS);
0432 document2Table = new XJTable() {
0433 public boolean isCellEditable(int rowIndex, int vColIndex) {
0434 return false;
0435 }
0436 };
0437 document2Table.setModel(document2TableModel);
0438 confusionTable = new XJTable() {
0439 public boolean isCellEditable(int rowIndex, int vColIndex) {
0440 return false;
0441 }
0442 };
0443 confusionTable.setModel(confusionTableModel);
0444 confusionTable.setSortable(false);
0445
0446 tableTabbedPane = new JTabbedPane();
0447 tableTabbedPane.addTab("Corpus statistics", null,
0448 new JScrollPane(annotationTable),
0449 "Compare each annotation type for the whole corpus");
0450 tableTabbedPane.addTab("Document statistics", null,
0451 new JScrollPane(documentTable),
0452 "Compare each documents in the corpus with theirs annotations");
0453
0454 JSplitPane splitPane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT);
0455 splitPane.setContinuousLayout(true);
0456 splitPane.setOneTouchExpandable(true);
0457 splitPane.setResizeWeight(0.80);
0458 splitPane.setLeftComponent(tableTabbedPane);
0459 splitPane.setRightComponent(new JScrollPane(sidePanel));
0460
0461 add(splitPane);
0462 }
0463
0464 protected void initListeners() {
0465
0466 // when the view is shown update the tables if the corpus has changed
0467 addAncestorListener(new AncestorListener() {
0468 public void ancestorAdded(AncestorEvent event) {
0469 if (!isShowing() || !corpusChanged) { return; }
0470 if (timerTask != null) { timerTask.cancel(); }
0471 Date timeToRun = new Date(System.currentTimeMillis() + 1000);
0472 timerTask = new TimerTask() { public void run() {
0473 readSetsTypesFeatures(0);
0474 }};
0475 timer.schedule(timerTask, timeToRun); // add a delay before updating
0476 }
0477 public void ancestorRemoved(AncestorEvent event) { /* do nothing */ }
0478 public void ancestorMoved(AncestorEvent event) { /* do nothing */ }
0479 });
0480
0481 // when set list selection change
0482 setList.addListSelectionListener(new ListSelectionListener() {
0483 public void valueChanged(ListSelectionEvent e) {
0484 if (typesSelected == null) {
0485 typesSelected = typeList.getSelectedValues();
0486 }
0487 typeList.setModel(new ExtendedListModel());
0488 keySetName = ((ToggleSelectionABModel)
0489 setList.getSelectionModel()).getSelectedValueA();
0490 responseSetName = ((ToggleSelectionABModel)
0491 setList.getSelectionModel()).getSelectedValueB();
0492 if (keySetName == null
0493 || responseSetName == null
0494 || setList.getSelectionModel().getValueIsAdjusting()) {
0495 compareAction.setEnabled(false);
0496 return;
0497 }
0498 setList.setEnabled(false);
0499 setCheck.setEnabled(false);
0500 // update type UI list
0501 TreeSet<String> someTypes = new TreeSet<String>();
0502 TreeMap<String, TreeSet<String>> typesFeatures;
0503 boolean firstLoop = true; // needed for retainAll to work
0504 synchronized(docsSetsTypesFeatures) {
0505 for (TreeMap<String, TreeMap<String, TreeSet<String>>>
0506 setsTypesFeatures : docsSetsTypesFeatures.values()) {
0507 typesFeatures = setsTypesFeatures.get(
0508 keySetName.equals("[Default set]") ? "" : keySetName);
0509 if (typesFeatures != null) {
0510 if (typeCheck.isSelected() && !firstLoop) {
0511 someTypes.retainAll(typesFeatures.keySet());
0512 } else {
0513 someTypes.addAll(typesFeatures.keySet());
0514 }
0515 } else if (typeCheck.isSelected()) {
0516 // empty set no types to display
0517 break;
0518 }
0519 typesFeatures = setsTypesFeatures.get(
0520 responseSetName.equals("[Default set]") ? "" : responseSetName);
0521 if (typesFeatures != null) {
0522 if (typeCheck.isSelected()) {
0523 someTypes.retainAll(typesFeatures.keySet());
0524 } else {
0525 someTypes.addAll(typesFeatures.keySet());
0526 }
0527 } else if (typeCheck.isSelected()) {
0528 break;
0529 }
0530 firstLoop = false;
0531 }
0532 }
0533 typeList.setModel(new ExtendedListModel(someTypes.toArray()));
0534 if (someTypes.size() > 0) {
0535 for (Object value : typesSelected) {
0536 // put back the selection if possible
0537 int index = typeList.getNextMatch(
0538 (String) value, 0, Position.Bias.Forward);
0539 if (index != -1) {
0540 typeList.setSelectedIndex(index);
0541 }
0542 }
0543 }
0544 typesSelected = null;
0545 setList.setEnabled(true);
0546 setCheck.setEnabled(true);
0547 if (measuresType == FSCORE_MEASURES) {
0548 compareAction.setEnabled(true);
0549 }
0550 }
0551 });
0552
0553 // when type list selection change
0554 typeList.addListSelectionListener(new ListSelectionListener() {
0555 public void valueChanged(ListSelectionEvent e) {
0556 // update feature UI list
0557 if (featuresSelected == null) {
0558 featuresSelected = featureList.getSelectedValues();
0559 }
0560 featureList.setModel(new ExtendedListModel());
0561 if (typeList.getSelectedValues().length == 0
0562 || typeList.getSelectionModel().getValueIsAdjusting()) {
0563 return;
0564 }
0565 final Set<String> typeNames = new HashSet<String>();
0566 for (Object type : typeList.getSelectedValues()) {
0567 typeNames.add((String) type);
0568 }
0569 typeList.setEnabled(false);
0570 typeCheck.setEnabled(false);
0571 TreeSet<String> features = new TreeSet<String>(collator);
0572 TreeMap<String, TreeSet<String>> typesFeatures;
0573 boolean firstLoop = true; // needed for retainAll to work
0574 synchronized(docsSetsTypesFeatures) {
0575 for (TreeMap<String, TreeMap<String, TreeSet<String>>> sets :
0576 docsSetsTypesFeatures.values()) {
0577 typesFeatures = sets.get(keySetName.equals("[Default set]") ?
0578 "" : keySetName);
0579 if (typesFeatures != null) {
0580 for (String typeName : typesFeatures.keySet()) {
0581 if (typeNames.contains(typeName)) {
0582 if (featureCheck.isSelected() && !firstLoop) {
0583 features.retainAll(typesFeatures.get(typeName));
0584 } else {
0585 features.addAll(typesFeatures.get(typeName));
0586 }
0587 }
0588 }
0589 } else if (featureCheck.isSelected()) {
0590 // empty type no features to display
0591 break;
0592 }
0593 typesFeatures = sets.get(responseSetName.equals("[Default set]") ?
0594 "" : responseSetName);
0595 if (typesFeatures != null) {
0596 for (String typeName : typesFeatures.keySet()) {
0597 if (typeNames.contains(typeName)) {
0598 if (featureCheck.isSelected()) {
0599 features.retainAll(typesFeatures.get(typeName));
0600 } else {
0601 features.addAll(typesFeatures.get(typeName));
0602 }
0603 }
0604 }
0605 } else if (featureCheck.isSelected()) {
0606 break;
0607 }
0608 firstLoop = false;
0609 }
0610 }
0611 featureList.setModel(new ExtendedListModel(features.toArray()));
0612 if (features.size() > 0) {
0613 for (Object value : featuresSelected) {
0614 // put back the selection if possible
0615 int index = featureList.getNextMatch(
0616 (String) value, 0, Position.Bias.Forward);
0617 if (index != -1) {
0618 featureList.setSelectedIndex(index);
0619 }
0620 }
0621 }
0622 featuresSelected = null;
0623 typeList.setEnabled(true);
0624 typeCheck.setEnabled(true);
0625 }
0626 });
0627
0628 // when type list selection change
0629 featureList.addListSelectionListener(new ListSelectionListener() {
0630 public void valueChanged(ListSelectionEvent e) {
0631 if (measuresType == CLASSIFICATION_MEASURES) {
0632 if (typeList.getSelectedIndices().length == 1
0633 && featureList.getSelectedIndices().length == 1) {
0634 compareAction.setEnabled(true);
0635 compareAction.putValue(Action.SHORT_DESCRIPTION,
0636 "Compare annotations between sets A and B");
0637 } else {
0638 compareAction.setEnabled(false);
0639 compareAction.putValue(Action.SHORT_DESCRIPTION,
0640 "You must select exactly one type and one feature");
0641 }
0642 }
0643 }
0644 });
0645
0646 // when the measure tab selection change
0647 measureTabbedPane.addChangeListener(new ChangeListener() {
0648 public void stateChanged(ChangeEvent e) {
0649 JTabbedPane tabbedPane = (JTabbedPane) e.getSource();
0650 int selectedTab = tabbedPane.getSelectedIndex();
0651 tableTabbedPane.removeAll();
0652 openDocumentAction.setEnabled(false);
0653 openAnnotationDiffAction.setEnabled(false);
0654 if (optionsButton.isSelected()) {
0655 optionsButton.doClick(); // hide the options panel if shown
0656 }
0657 if (tabbedPane.getTitleAt(selectedTab).equals("F-Score")) {
0658 measuresType = FSCORE_MEASURES;
0659 compareAction.setEnabled(keySetName != null
0660 && responseSetName != null);
0661 compareAction.putValue(Action.SHORT_DESCRIPTION,
0662 "Compare annotations between sets A and B");
0663 tableTabbedPane.addTab("Corpus statistics", null,
0664 new JScrollPane(annotationTable),
0665 "Compare each annotation type for the whole corpus");
0666 tableTabbedPane.addTab("Document statistics", null,
0667 new JScrollPane(documentTable),
0668 "Compare each documents in the corpus with theirs annotations");
0669 } else {
0670 measuresType = CLASSIFICATION_MEASURES;
0671 featureList.getListSelectionListeners()[0].valueChanged(null);
0672 tableTabbedPane.addTab("Document statistics", null,
0673 new JScrollPane(document2Table),
0674 "Compare each documents in the corpus with theirs annotations");
0675 tableTabbedPane.addTab("Confusion Matrices", null,
0676 new JScrollPane(confusionTable), "Describe how annotations in" +
0677 " one set are classified in the other and vice versa.");
0678 }
0679 }
0680 });
0681
0682 // enable/disable toolbar icons according to the document table selection
0683 documentTable.getSelectionModel().addListSelectionListener(
0684 new ListSelectionListener() {
0685 public void valueChanged(ListSelectionEvent e) {
0686 if (e.getValueIsAdjusting()) { return; }
0687 boolean enabled = documentTable.getSelectedRow() != -1
0688 && !((String)documentTableModel.getValueAt(
0689 documentTable.getSelectedRow(), 0)).endsWith("summary");
0690 openDocumentAction.setEnabled(enabled);
0691 openAnnotationDiffAction.setEnabled(enabled);
0692 }
0693 }
0694 );
0695
0696 // enable/disable toolbar icons according to the document 2 table selection
0697 document2Table.getSelectionModel().addListSelectionListener(
0698 new ListSelectionListener() {
0699 public void valueChanged(ListSelectionEvent e) {
0700 if (e.getValueIsAdjusting()) { return; }
0701 boolean enabled = document2Table.getSelectedRow() != -1
0702 && !((String)document2TableModel.getValueAt(
0703 document2Table.getSelectedRow(),
0704 0)).endsWith("summary");
0705 openDocumentAction.setEnabled(enabled);
0706 openAnnotationDiffAction.setEnabled(enabled);
0707 }
0708 }
0709 );
0710
0711 // double click on a document loads it in the document editor
0712 documentTable.addMouseListener(new MouseAdapter() {
0713 public void mouseClicked(MouseEvent e) {
0714 if (!e.isPopupTrigger()
0715 && e.getClickCount() == 2
0716 && openDocumentAction.isEnabled()) {
0717 openDocumentAction.actionPerformed(null);
0718 }
0719 }
0720 });
0721
0722 // double click on a document loads it in the document editor
0723 document2Table.addMouseListener(new MouseAdapter() {
0724 public void mouseClicked(MouseEvent e) {
0725 if (!e.isPopupTrigger()
0726 && e.getClickCount() == 2
0727 && openDocumentAction.isEnabled()) {
0728 openDocumentAction.actionPerformed(null);
0729 }
0730 }
0731 });
0732
0733 InputMap inputMap = getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW);
0734 ActionMap actionMap = getActionMap();
0735 inputMap.put(KeyStroke.getKeyStroke("F1"), "help");
0736 actionMap.put("help", new HelpAction());
0737 }
0738
0739 /**
0740 * Create a table header tool tips from the column name.
0741 * @param columnName name used for creating the tooltip
0742 * @return tooltip value
0743 */
0744 protected String createToolTipFromColumnName(String columnName) {
0745 String tooltip;
0746 if (columnName.equals("Document")
0747 || columnName.equals("Annotation")) {
0748 tooltip = null;
0749 } else if (columnName.equals("Match")) {
0750 tooltip = "aka Correct";
0751 } else if (columnName.equals("Only A")) {
0752 tooltip = "aka Missing";
0753 } else if (columnName.equals("Only B")) {
0754 tooltip = "aka Spurious";
0755 } else if (columnName.equals("Overlap")) {
0756 tooltip = "aka Partial";
0757 } else if (columnName.equals("Rec.B/A")) {
0758 tooltip = "Recall for B relative to A";
0759 } else if (columnName.equals("Prec.B/A")) {
0760 tooltip = "Precision for B relative to A";
0761 } else {
0762 tooltip = columnName
0763 .replaceFirst("s.", "score strict")
0764 .replaceFirst("l.", "score lenient")
0765 .replaceFirst("a.", "score average")
0766 .replaceFirst("B.", " BDM");
0767 }
0768 return tooltip;
0769 }
0770
0771 protected static class ExtendedListModel extends DefaultListModel {
0772 public ExtendedListModel() {
0773 super();
0774 }
0775 public ExtendedListModel(Object[] elements) {
0776 super();
0777 for (Object element : elements) {
0778 super.addElement(element);
0779 }
0780 }
0781 }
0782
0783 protected static class ToggleSelectionModel extends DefaultListSelectionModel {
0784 boolean gestureStarted = false;
0785 public void setSelectionInterval(int index0, int index1) {
0786 if (isSelectedIndex(index0) && !gestureStarted) {
0787 super.removeSelectionInterval(index0, index1);
0788 } else {
0789 super.addSelectionInterval(index0, index1);
0790 }
0791 gestureStarted = true;
0792 }
0793 public void setValueIsAdjusting(boolean isAdjusting) {
0794 if (!isAdjusting) {
0795 gestureStarted = false;
0796 }
0797 }
0798 }
0799
0800 /**
0801 * Add a suffix A and B for the first and second selected item.
0802 * Allows only 2 items to be selected.
0803 */
0804 protected static class ToggleSelectionABModel extends DefaultListSelectionModel {
0805 public ToggleSelectionABModel(JList list) {
0806 this.list = list;
0807 }
0808 public void setSelectionInterval(int index0, int index1) {
0809 ExtendedListModel model = (ExtendedListModel) list.getModel();
0810 String value = (String) model.getElementAt(index0);
0811 if (value.endsWith(" (A)") || value.endsWith(" (B)")) {
0812 // if ends with ' (A)' or ' (B)' then remove the suffix
0813 model.removeElementAt(index0);
0814 model.insertElementAt(value.substring(0,
0815 value.length() - " (A)".length()), index0);
0816 if (value.endsWith(" (A)")) {
0817 selectedValueA = null;
0818 } else {
0819 selectedValueB = null;
0820 }
0821 removeSelectionInterval(index0, index1);
0822 } else {
0823 // suffix with ' (A)' or ' (B)' if not already existing
0824 if (selectedValueA == null) {
0825 model.removeElementAt(index0);
0826 model.insertElementAt(value + " (A)", index0);
0827 selectedValueA = value;
0828 addSelectionInterval(index0, index1);
0829 } else if (selectedValueB == null) {
0830 model.removeElementAt(index0);
0831 model.insertElementAt(value + " (B)", index0);
0832 selectedValueB = value;
0833 addSelectionInterval(index0, index1);
0834 }
0835 }
0836 }
0837 public void clearSelection() {
0838 selectedValueA = null;
0839 selectedValueB = null;
0840 super.clearSelection();
0841 }
0842 public String getSelectedValueA() {
0843 return selectedValueA;
0844 }
0845 public String getSelectedValueB() {
0846 return selectedValueB;
0847 }
0848 JList list;
0849 String selectedValueA, selectedValueB;
0850 }
0851
0852 public void cleanup(){
0853 super.cleanup();
0854 corpus = null;
0855 }
0856
0857 public void setTarget(Object target){
0858 if(corpus != null && corpus != target){
0859 //we already had a different corpus
0860 corpus.removeCorpusListener(this);
0861 }
0862 if(!(target instanceof Corpus)){
0863 throw new IllegalArgumentException(
0864 "This view can only be used with a GATE corpus!\n" +
0865 target.getClass().toString() + " is not a GATE corpus!");
0866 }
0867 this.corpus = (Corpus) target;
0868 corpus.addCorpusListener(this);
0869
0870 corpusChanged = true;
0871 if (!isShowing()) { return; }
0872 if (timerTask != null) { timerTask.cancel(); }
0873 Date timeToRun = new Date(System.currentTimeMillis() + 2000);
0874 timerTask = new TimerTask() { public void run() {
0875 readSetsTypesFeatures(0);
0876 }};
0877 timer.schedule(timerTask, timeToRun); // add a delay before updating
0878 }
0879
0880 public void documentAdded(final CorpusEvent e) {
0881 corpusChanged = true;
0882 if (!isShowing()) { return; }
0883 if (timerTask != null) { timerTask.cancel(); }
0884 Date timeToRun = new Date(System.currentTimeMillis() + 2000);
0885 timerTask = new TimerTask() { public void run() {
0886 readSetsTypesFeatures(0);
0887 }};
0888 timer.schedule(timerTask, timeToRun); // add a delay before updating
0889 }
0890
0891 public void documentRemoved(final CorpusEvent e) {
0892 corpusChanged = true;
0893 if (!isShowing()) { return; }
0894 if (timerTask != null) { timerTask.cancel(); }
0895 Date timeToRun = new Date(System.currentTimeMillis() + 2000);
0896 timerTask = new TimerTask() { public void run() {
0897 readSetsTypesFeatures(0);
0898 }};
0899 timer.schedule(timerTask, timeToRun); // add a delay before updating
0900 }
0901
0902 /**
0903 * Update set lists.
0904 * @param documentStart first document to read in the corpus,
0905 * the first document of the corpus is 0.
0906 */
0907 protected void readSetsTypesFeatures(final int documentStart) {
0908 if (!isShowing()) { return; }
0909 corpusChanged = false;
0910 SwingUtilities.invokeLater(new Runnable(){ public void run() {
0911 progressBar.setMaximum(corpus.size() - 1);
0912 progressBar.setString("Read sets, types, features");
0913 reloadCacheAction.setEnabled(false);
0914 }});
0915 CorpusQualityAssurance.this.setCursor(
0916 Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
0917 Runnable runnable = new Runnable() { public void run() {
0918 if (docsSetsTypesFeatures.size() != corpus.getDocumentNames().size()
0919 || !docsSetsTypesFeatures.keySet().containsAll(corpus.getDocumentNames())) {
0920 if (documentStart == 0) { docsSetsTypesFeatures.clear(); }
0921 TreeMap<String, TreeMap<String, TreeSet<String>>> setsTypesFeatures;
0922 TreeMap<String, TreeSet<String>> typesFeatures;
0923 TreeSet<String> features;
0924 for (int i = documentStart; i < corpus.size(); i++) {
0925 // fill in the lists of document, set, type and feature names
0926 boolean documentWasLoaded = corpus.isDocumentLoaded(i);
0927 Document document = (Document) corpus.get(i);
0928 if (document != null && document.getAnnotationSetNames() != null) {
0929 setsTypesFeatures =
0930 new TreeMap<String, TreeMap<String, TreeSet<String>>>(collator);
0931 HashSet<String> setNames =
0932 new HashSet<String>(document.getAnnotationSetNames());
0933 setNames.add("");
0934 for (String set : setNames) {
0935 typesFeatures = new TreeMap<String, TreeSet<String>>(collator);
0936 AnnotationSet annotations = document.getAnnotations(set);
0937 for (String type : annotations.getAllTypes()) {
0938 features = new TreeSet<String>(collator);
0939 for (Annotation annotation : annotations.get(type)) {
0940 for (Object featureKey : annotation.getFeatures().keySet()) {
0941 features.add((String) featureKey);
0942 }
0943 }
0944 typesFeatures.put(type, features);
0945 }
0946 setsTypesFeatures.put(set, typesFeatures);
0947 }
0948 docsSetsTypesFeatures.put(document.getName(), setsTypesFeatures);
0949 }
0950 if (!documentWasLoaded) {
0951 corpus.unloadDocument(document);
0952 Factory.deleteResource(document);
0953 }
0954 final int progressValue = i + 1;
0955 SwingUtilities.invokeLater(new Runnable(){ public void run() {
0956 progressBar.setValue(progressValue);
0957 if ((progressValue+1) % 5 == 0) {
0958 // update the set list every 5 documents read
0959 updateSetList();
0960 }
0961 }});
0962 if (Thread.interrupted()) { return; }
0963 }
0964 }
0965 updateSetList();
0966 SwingUtilities.invokeLater(new Runnable(){ public void run(){
0967 progressBar.setValue(progressBar.getMinimum());
0968 progressBar.setString("");
0969 CorpusQualityAssurance.this.setCursor(
0970 Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
0971 reloadCacheAction.setEnabled(true);
0972 }});
0973 }};
0974 readSetsTypesFeaturesThread = new Thread(runnable, "readSetsTypesFeatures");
0975 readSetsTypesFeaturesThread.setPriority(Thread.MIN_PRIORITY);
0976 readSetsTypesFeaturesThread.start();
0977 }
0978
0979 protected void updateSetList() {
0980 final TreeSet<String> setsNames = new TreeSet<String>(collator);
0981 Set<String> sets;
0982 boolean firstLoop = true; // needed for retainAll to work
0983 synchronized(docsSetsTypesFeatures) {
0984 for (String document : docsSetsTypesFeatures.keySet()) {
0985 // get the list of set names
0986 sets = docsSetsTypesFeatures.get(document).keySet();
0987 if (!sets.isEmpty()) {
0988 if (setCheck.isSelected() && !firstLoop) {
0989 setsNames.retainAll(sets);
0990 } else {
0991 setsNames.addAll(sets);
0992 }
0993 } else if (setCheck.isSelected()) {
0994 break;
0995 }
0996 firstLoop = false;
0997 }
0998 }
0999 SwingUtilities.invokeLater(new Runnable(){ public void run() {
1000 // update the UI lists of sets
1001 setsNames.remove("");
1002 setsNames.add("[Default set]");
1003 String keySetNamePrevious = keySetName;
1004 String responseSetNamePrevious = responseSetName;
1005 setList.setModel(new ExtendedListModel(setsNames.toArray()));
1006 if (setsNames.size() > 0) {
1007 if (keySetNamePrevious != null) {
1008 // put back the selection if possible
1009 int index = setList.getNextMatch(
1010 keySetNamePrevious, 0, Position.Bias.Forward);
1011 if (index != -1) {
1012 setList.setSelectedIndex(index);
1013 }
1014 }
1015 if (responseSetNamePrevious != null) {
1016 // put back the selection if possible
1017 int index = setList.getNextMatch(
1018 responseSetNamePrevious, 0, Position.Bias.Forward);
1019 if (index != -1) {
1020 setList.setSelectedIndex(index);
1021 }
1022 }
1023 }
1024 }});
1025 }
1026
1027 protected void compareAnnotation() {
1028 int progressValuePrevious = -1;
1029 if (readSetsTypesFeaturesThread != null
1030 && readSetsTypesFeaturesThread.isAlive()) {
1031 // stop the thread that reads the sets, types and features
1032 progressValuePrevious = progressBar.getValue();
1033 readSetsTypesFeaturesThread.interrupt();
1034 }
1035 SwingUtilities.invokeLater(new Runnable() { public void run() {
1036 progressBar.setMaximum(corpus.size() - 1);
1037 progressBar.setString("Compare annotations");
1038 setList.setEnabled(false);
1039 setCheck.setEnabled(false);
1040 typeList.setEnabled(false);
1041 typeCheck.setEnabled(false);
1042 featureList.setEnabled(false);
1043 featureCheck.setEnabled(false);
1044 optionsButton.setEnabled(false);
1045 measureTabbedPane.setEnabled(false);
1046 measureList.setEnabled(false);
1047 exportToHtmlAction.setEnabled(false);
1048 reloadCacheAction.setEnabled(false);
1049 }});
1050
1051 boolean useBdm = false;
1052 if (measuresType == FSCORE_MEASURES) {
1053 differsByDocThenType.clear();
1054 documentNames.clear();
1055 for (Object measure : measureList.getSelectedValues()) {
1056 if (((String) measure).contains("BDM")) { useBdm = true; break; }
1057 }
1058 }
1059 List<ClassificationMeasures> classificationMeasuresList =
1060 new ArrayList<ClassificationMeasures>();
1061 List<OntologyMeasures> documentOntologyMeasuresList =
1062 new ArrayList<OntologyMeasures>();
1063 List<OntologyMeasures> annotationOntologyMeasuresList =
1064 new ArrayList<OntologyMeasures>();
1065
1066 // for each document
1067 for (int row = 0; row < corpus.size(); row++) {
1068 boolean documentWasLoaded = corpus.isDocumentLoaded(row);
1069 Document document = (Document) corpus.get(row);
1070 documentNames.add(document.getName());
1071 Set<Annotation> keys = new HashSet<Annotation>();
1072 Set<Annotation> responses = new HashSet<Annotation>();
1073 // get annotations from selected annotation sets
1074 if (keySetName.equals("[Default set]")) {
1075 keys = document.getAnnotations();
1076 } else if (document.getAnnotationSetNames() != null
1077 && document.getAnnotationSetNames().contains(keySetName)) {
1078 keys = document.getAnnotations(keySetName);
1079 }
1080 if (responseSetName.equals("[Default set]")) {
1081 responses = document.getAnnotations();
1082 } else if (document.getAnnotationSetNames() != null
1083 && document.getAnnotationSetNames()
1084 .contains(responseSetName)) {
1085 responses = document.getAnnotations(responseSetName);
1086 }
1087 if (!documentWasLoaded) { // in case of datastore
1088 corpus.unloadDocument(document);
1089 Factory.deleteResource(document);
1090 }
1091
1092 // add data to the fscore document table
1093 if (measuresType == FSCORE_MEASURES) {
1094 types.clear();
1095 for (Object type : typeList.getSelectedValues()) {
1096 types.add((String) type);
1097 }
1098 if (typeList.isSelectionEmpty()) {
1099 for (int i = 0; i < typeList.getModel().getSize(); i++) {
1100 types.add((String) typeList.getModel().getElementAt(i));
1101 }
1102 }
1103 Set<String> featureSet = new HashSet<String>();
1104 for (Object feature : featureList.getSelectedValues()) {
1105 featureSet.add((String) feature);
1106 }
1107 HashMap<String, AnnotationDiffer> differsByType =
1108 new HashMap<String, AnnotationDiffer>();
1109 AnnotationDiffer differ;
1110 Set<Annotation> keysIter = new HashSet<Annotation>();
1111 Set<Annotation> responsesIter = new HashSet<Annotation>();
1112 for (String type : types) {
1113 if (!keys.isEmpty() && !types.isEmpty()) {
1114 keysIter = ((AnnotationSet)keys).get(type);
1115 }
1116 if (!responses.isEmpty() && !types.isEmpty()) {
1117 responsesIter = ((AnnotationSet)responses).get(type);
1118 }
1119 differ = new AnnotationDiffer();
1120 differ.setSignificantFeaturesSet(featureSet);
1121 differ.calculateDiff(keysIter, responsesIter); // compare
1122 differsByType.put(type, differ);
1123 }
1124 differsByDocThenType.add(differsByType);
1125 differ = new AnnotationDiffer(differsByType.values());
1126 List<String> measuresRow;
1127 if (useBdm) {
1128 OntologyMeasures ontologyMeasures = new OntologyMeasures();
1129 ontologyMeasures.setBdmFile(bdmFileUrl);
1130 ontologyMeasures.calculateBdm(differsByType.values());
1131 documentOntologyMeasuresList.add(ontologyMeasures);
1132 measuresRow = ontologyMeasures.getMeasuresRow(
1133 measureList.getSelectedValues(),
1134 documentNames.get(documentNames.size()-1));
1135 } else {
1136 measuresRow = differ.getMeasuresRow(measureList.getSelectedValues(),
1137 documentNames.get(documentNames.size()-1));
1138 }
1139 documentTableModel.addRow(measuresRow.toArray());
1140
1141 // add data to the classification document table
1142 } else if (measuresType == CLASSIFICATION_MEASURES
1143 && !keys.isEmpty() && !responses.isEmpty()) {
1144 ClassificationMeasures classificationMeasures =
1145 new ClassificationMeasures();
1146 classificationMeasures.calculateConfusionMatrix(
1147 (AnnotationSet) keys, (AnnotationSet) responses,
1148 (String) typeList.getSelectedValue(),
1149 (String) featureList.getSelectedValue(),
1150 verboseOptionCheckBox.isSelected());
1151 classificationMeasuresList.add(classificationMeasures);
1152 List<String> measuresRow = classificationMeasures.getMeasuresRow(
1153 measure2List.getSelectedValues(),
1154 documentNames.get(documentNames.size()-1));
1155 document2TableModel.addRow(measuresRow.toArray());
1156 List<List<String>> matrix = classificationMeasures
1157 .getConfusionMatrix(documentNames.get(documentNames.size()-1));
1158 for (List<String> matrixRow : matrix) {
1159 while (confusionTableModel.getColumnCount() < matrix.size()) {
1160 confusionTableModel.addColumn(" ");
1161 }
1162 confusionTableModel.addRow(matrixRow.toArray());
1163 }
1164 }
1165 final int progressValue = row + 1;
1166 SwingUtilities.invokeLater(new Runnable(){ public void run() {
1167 progressBar.setValue(progressValue);
1168 }});
1169 } // for (int row = 0; row < corpus.size(); row++)
1170
1171 // add data to the fscore annotation table
1172 if (measuresType == FSCORE_MEASURES) {
1173 for (String type : types) {
1174 ArrayList<AnnotationDiffer> differs = new ArrayList<AnnotationDiffer>();
1175 for (HashMap<String, AnnotationDiffer> differsByType :
1176 differsByDocThenType) {
1177 differs.add(differsByType.get(type));
1178 }
1179 List<String> measuresRow;
1180 if (useBdm) {
1181 OntologyMeasures ontologyMeasures = new OntologyMeasures();
1182 ontologyMeasures.setBdmFile(bdmFileUrl);
1183 ontologyMeasures.calculateBdm(differs);
1184 annotationOntologyMeasuresList.add(ontologyMeasures);
1185 measuresRow = ontologyMeasures.getMeasuresRow(
1186 measureList.getSelectedValues(), type);
1187 } else {
1188 AnnotationDiffer differ = new AnnotationDiffer(differs);
1189 measuresRow = differ.getMeasuresRow(
1190 measureList.getSelectedValues(), type);
1191 }
1192 annotationTableModel.addRow(measuresRow.toArray());
1193 }
1194 }
1195
1196 // add summary rows to the fscore tables
1197 if (measuresType == FSCORE_MEASURES) {
1198 if (useBdm) {
1199 OntologyMeasures ontologyMeasures =
1200 new OntologyMeasures(documentOntologyMeasuresList);
1201 printSummary(ontologyMeasures, documentTableModel, 5,
1202 documentTableModel.getRowCount(), measureList.getSelectedValues());
1203 ontologyMeasures = new OntologyMeasures(annotationOntologyMeasuresList);
1204 printSummary(ontologyMeasures, annotationTableModel, 5,
1205 annotationTableModel.getRowCount(), measureList.getSelectedValues());
1206 } else {
1207 List<AnnotationDiffer> differs = new ArrayList<AnnotationDiffer>();
1208 for (Map<String, AnnotationDiffer> differsByType :
1209 differsByDocThenType) {
1210 differs.addAll(differsByType.values());
1211 }
1212 AnnotationDiffer differ = new AnnotationDiffer(differs);
1213 printSummary(differ, documentTableModel, 5,
1214 documentTableModel.getRowCount(), measureList.getSelectedValues());
1215 printSummary(differ, annotationTableModel, 5,
1216 annotationTableModel.getRowCount(), measureList.getSelectedValues());
1217 }
1218
1219 // add summary rows to the classification tables
1220 } else if (measuresType == CLASSIFICATION_MEASURES) {
1221 ClassificationMeasures classificationMeasures =
1222 new ClassificationMeasures(classificationMeasuresList);
1223 printSummary(classificationMeasures, document2TableModel, 3,
1224 document2TableModel.getRowCount(), measure2List.getSelectedValues());
1225 List<List<String>> matrix = classificationMeasures
1226 .getConfusionMatrix("Whole corpus");
1227 int insertionRow = 0;
1228 for (List<String> row : matrix) {
1229 while (confusionTableModel.getColumnCount() < matrix.size()) {
1230 confusionTableModel.addColumn(" ");
1231 }
1232 confusionTableModel.insertRow(insertionRow++, row.toArray());
1233 }
1234 }
1235
1236 SwingUtilities.invokeLater(new Runnable(){ public void run(){
1237 progressBar.setValue(progressBar.getMinimum());
1238 progressBar.setString("");
1239 setList.setEnabled(true);
1240 setCheck.setEnabled(true);
1241 typeList.setEnabled(true);
1242 typeCheck.setEnabled(true);
1243 featureList.setEnabled(true);
1244 featureCheck.setEnabled(true);
1245 optionsButton.setEnabled(true);
1246 measureTabbedPane.setEnabled(true);
1247 measureList.setEnabled(true);
1248 exportToHtmlAction.setEnabled(true);
1249 reloadCacheAction.setEnabled(true);
1250 }});
1251 if (progressValuePrevious > -1) {
1252 // restart the thread where it was interrupted
1253 readSetsTypesFeatures(progressValuePrevious);
1254 }
1255 }
1256
1257 protected void printSummary(Object measureObject,
1258 DefaultTableModel tableModel, int columnGroupSize,
1259 int insertionRow, Object[] measures) {
1260 AnnotationDiffer differ = null;
1261 ClassificationMeasures classificationMeasures = null;
1262 OntologyMeasures ontologyMeasures = null;
1263 if (measureObject instanceof AnnotationDiffer) {
1264 differ = (AnnotationDiffer) measureObject;
1265 } else if (measureObject instanceof ClassificationMeasures) {
1266 classificationMeasures = (ClassificationMeasures) measureObject;
1267 } else if (measureObject instanceof OntologyMeasures) {
1268 ontologyMeasures = (OntologyMeasures) measureObject;
1269 }
1270 NumberFormat f = NumberFormat.getInstance(Locale.ENGLISH);
1271 f.setMaximumFractionDigits(2);
1272 f.setMinimumFractionDigits(2);
1273 List<Object> values = new ArrayList<Object>();
1274
1275 // average measures by document
1276 values.add("Macro summary");
1277 for (int col = 1; col < tableModel.getColumnCount(); col++) {
1278 if (col < columnGroupSize) {
1279 values.add("");
1280 } else {
1281 float sumF = 0;
1282 for (int row = 0; row < tableModel.getRowCount(); row++) {
1283 try {
1284 sumF += Float.parseFloat((String) tableModel.getValueAt(row, col));
1285 } catch(NumberFormatException e) {
1286 // do nothing
1287 }
1288 }
1289 values.add(f.format(sumF / tableModel.getRowCount()));
1290 }
1291 }
1292 tableModel.insertRow(insertionRow, values.toArray());
1293
1294 // sum counts and recalculate measures like the corpus is one document
1295 values.clear();
1296 values.add("Micro summary");
1297 for (int col = 1; col < columnGroupSize; col++) {
1298 int sum = 0;
1299 for (int row = 0; row < tableModel.getRowCount()-1; row++) {
1300 try {
1301 sum += Integer.valueOf((String) tableModel.getValueAt(row, col));
1302 } catch(NumberFormatException e) {
1303 // do nothing
1304 }
1305 }
1306 values.add(Integer.toString(sum));
1307 }
1308 if (measureObject instanceof OntologyMeasures) {
1309 List<AnnotationDiffer> differs = new ArrayList<AnnotationDiffer>(
1310 ontologyMeasures.getDifferByTypeMap().values());
1311 differ = new AnnotationDiffer(differs);
1312 }
1313 for (Object object : measures) {
1314 String measure = (String) object;
1315 int index = measure.indexOf('-');
1316 double beta = (index == -1) ?
1317 1 : Double.valueOf(measure.substring(1, index));
1318 if (measure.endsWith("strict")) {
1319 values.add(f.format(differ.getRecallStrict()));
1320 values.add(f.format(differ.getPrecisionStrict()));
1321 values.add(f.format(differ.getFMeasureStrict(beta)));
1322 } else if (measure.endsWith("strict BDM")) {
1323 values.add(f.format(ontologyMeasures.getRecallStrictBdm()));
1324 values.add(f.format(ontologyMeasures.getPrecisionStrictBdm()));
1325 values.add(f.format(ontologyMeasures.getFMeasureStrictBdm(beta)));
1326 } else if (measure.endsWith("lenient")) {
1327 values.add(f.format(differ.getRecallLenient()));
1328 values.add(f.format(differ.getPrecisionLenient()));
1329 values.add(f.format(differ.getFMeasureLenient(beta)));
1330 } else if (measure.endsWith("lenient BDM")) {
1331 values.add(f.format(ontologyMeasures.getRecallLenientBdm()));
1332 values.add(f.format(ontologyMeasures.getPrecisionLenientBdm()));
1333 values.add(f.format(ontologyMeasures.getFMeasureLenientBdm(beta)));
1334 } else if (measure.endsWith("average")) {
1335 values.add(f.format(differ.getRecallAverage()));
1336 values.add(f.format(differ.getPrecisionAverage()));
1337 values.add(f.format(differ.getFMeasureAverage(beta)));
1338 } else if (measure.endsWith("average BDM")) {
1339 values.add(f.format(ontologyMeasures.getRecallAverageBdm()));
1340 values.add(f.format(ontologyMeasures.getPrecisionAverageBdm()));
1341 values.add(f.format(ontologyMeasures.getFMeasureAverageBdm(beta)));
1342 } else if (measure.equals("Observed agreement")) {
1343 values.add(f.format(classificationMeasures.getObservedAgreement()));
1344 } else if (measure.equals("Cohen's Kappa")) {
1345 float result = classificationMeasures.getKappaCohen();
1346 values.add(Float.isNaN(result) ? "" : f.format(result));
1347 } else if (measure.equals("Pi's Kappa")) {
1348 float result = classificationMeasures.getKappaPi();
1349 values.add(Float.isNaN(result) ? "" : f.format(result));
1350 }
1351 }
1352 tableModel.insertRow(insertionRow + 1, values.toArray());
1353 }
1354
1355 protected class SetBdmFileAction extends AbstractAction {
1356 public SetBdmFileAction() {
1357 super("Browse");
1358 putValue(SHORT_DESCRIPTION, "Choose a BDM file to compute BDM measures");
1359 }
1360 public void actionPerformed(ActionEvent evt) {
1361 XJFileChooser fileChooser = MainFrame.getFileChooser();
1362 fileChooser.setAcceptAllFileFilterUsed(true);
1363 fileChooser.setDialogTitle("Choose a BDM file");
1364 fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY);
1365 fileChooser.setResource(
1366 CorpusQualityAssurance.class.getName() + ".BDMfile");
1367 int res = fileChooser.showOpenDialog(CorpusQualityAssurance.this);
1368 if (res != JFileChooser.APPROVE_OPTION) { return; }
1369 try {
1370 bdmFileUrl = fileChooser.getSelectedFile().toURI().toURL();
1371 } catch (MalformedURLException e) {
1372 e.printStackTrace();
1373 }
1374 }
1375 }
1376
1377 /**
1378 * Update document table.
1379 */
1380 protected class CompareAction extends AbstractAction {
1381 public CompareAction() {
1382 super("Compare");
1383 putValue(SHORT_DESCRIPTION, "Compare annotations between sets A and B");
1384 putValue(MNEMONIC_KEY, KeyEvent.VK_ENTER);
1385 putValue(SMALL_ICON, MainFrame.getIcon("crystal-clear-action-run"));
1386 }
1387 public void actionPerformed(ActionEvent evt) {
1388 boolean useBdm = false;
1389 for (Object measure : measureList.getSelectedValues()) {
1390 if (((String) measure).contains("BDM")) { useBdm = true; break; }
1391 }
1392 if (useBdm && measuresType == FSCORE_MEASURES && bdmFileUrl == null) {
1393 new SetBdmFileAction().actionPerformed(null);
1394 if (bdmFileUrl == null) { return; }
1395 }
1396
1397 CorpusQualityAssurance.this.setCursor(
1398 Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
1399 setEnabled(false);
1400
1401 Runnable runnable = new Runnable() { public void run() {
1402 if (measuresType == FSCORE_MEASURES) {
1403 documentTableModel = new DefaultTableModel();
1404 annotationTableModel = new DefaultTableModel();
1405 documentTableModel.addColumn("Document");
1406 annotationTableModel.addColumn("Annotation");
1407 documentTableModel.addColumn("Match");
1408 annotationTableModel.addColumn("Match");
1409 documentTableModel.addColumn("Only A");
1410 annotationTableModel.addColumn("Only A");
1411 documentTableModel.addColumn("Only B");
1412 annotationTableModel.addColumn("Only B");
1413 documentTableModel.addColumn("Overlap");
1414 annotationTableModel.addColumn("Overlap");
1415 for (Object measure : measureList.getSelectedValues()) {
1416 String measureString = ((String) measure)
1417 .replaceFirst("score strict", "s.")
1418 .replaceFirst("score lenient", "l.")
1419 .replaceFirst("score average", "a.")
1420 .replaceFirst(" BDM", "B.");
1421 documentTableModel.addColumn("Rec.B/A");
1422 annotationTableModel.addColumn("Rec.B/A");
1423 documentTableModel.addColumn("Prec.B/A");
1424 annotationTableModel.addColumn("Prec.B/A");
1425 documentTableModel.addColumn(measureString);
1426 annotationTableModel.addColumn(measureString);
1427 }
1428 compareAnnotation(); // do all the computation
1429 // update data
1430
1431 SwingUtilities.invokeLater(new Runnable() { public void run() {
1432 // redraw document table
1433 documentTable.setModel(documentTableModel);
1434 for (int col = 0; col < documentTable.getColumnCount(); col++) {
1435 documentTable.setComparator(col, doubleComparator);
1436 }
1437 documentTable.setComparator(0, totalComparator);
1438 documentTable.setSortedColumn(0);
1439 // redraw annotation table
1440 annotationTable.setModel(annotationTableModel);
1441 for (int col = 0; col < annotationTable.getColumnCount(); col++) {
1442 annotationTable.setComparator(col, doubleComparator);
1443 }
1444 annotationTable.setComparator(0, totalComparator);
1445 annotationTable.setSortedColumn(0);
1446 CorpusQualityAssurance.this.setCursor(
1447 Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
1448 setEnabled(true);
1449 }});
1450
1451 } else if (measuresType == CLASSIFICATION_MEASURES) {
1452 document2TableModel = new DefaultTableModel();
1453 document2TableModel.addColumn("Document");
1454 document2TableModel.addColumn("Agreed");
1455 document2TableModel.addColumn("Total");
1456 for (Object measure : measure2List.getSelectedValues()) {
1457 document2TableModel.addColumn(measure);
1458 }
1459 confusionTableModel = new DefaultTableModel();
1460 compareAnnotation(); // do all the computation
1461 SwingUtilities.invokeLater(new Runnable() { public void run() {
1462 document2Table.setSortable(false);
1463 document2Table.setModel(document2TableModel);
1464 document2Table.setComparator(0, totalComparator);
1465 document2Table.setComparator(1, doubleComparator);
1466 document2Table.setSortedColumn(0);
1467 document2Table.setSortable(true);
1468 confusionTable.setModel(confusionTableModel);
1469 CorpusQualityAssurance.this.setCursor(
1470 Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
1471 setEnabled(true);
1472 }});
1473 }
1474 }};
1475 Thread thread = new Thread(runnable, "CompareAction");
1476 thread.setPriority(Thread.MIN_PRIORITY);
1477 thread.start();
1478 }
1479 }
1480
1481 class OpenDocumentAction extends AbstractAction{
1482 public OpenDocumentAction(){
1483 super("Open documents", MainFrame.getIcon("document"));
1484 putValue(SHORT_DESCRIPTION,
1485 "Opens document for the selected row in a document editor");
1486 putValue(MNEMONIC_KEY, KeyEvent.VK_UP);
1487 }
1488 public void actionPerformed(ActionEvent e){
1489 final Document document = (Document)
1490 corpus.get(measuresType == FSCORE_MEASURES ?
1491 documentTable.rowViewToModel(documentTable.getSelectedRow())
1492 : document2Table.rowViewToModel(document2Table.getSelectedRow()));
1493 SwingUtilities.invokeLater( new Runnable() { public void run() {
1494 MainFrame.getInstance().select(document);
1495 }});
1496 }
1497 }
1498
1499 class OpenAnnotationDiffAction extends AbstractAction{
1500 public OpenAnnotationDiffAction(){
1501 super("Open annotation diff", MainFrame.getIcon("annDiff"));
1502 putValue(SHORT_DESCRIPTION,
1503 "Opens annotation diff for the selected row in the document table");
1504 putValue(MNEMONIC_KEY, KeyEvent.VK_RIGHT);
1505 }
1506 public void actionPerformed(ActionEvent e){
1507 Document document = (Document)
1508 corpus.get(measuresType == FSCORE_MEASURES ?
1509 documentTable.rowViewToModel(documentTable.getSelectedRow())
1510 : document2Table.rowViewToModel(document2Table.getSelectedRow()));
1511 String documentName = document.getName();
1512 String annotationType = (String) typeList.getSelectedValue();
1513 Set<String> featureSet = new HashSet<String>();
1514 for (Object feature : featureList.getSelectedValues()) {
1515 featureSet.add((String) feature);
1516 }
1517 AnnotationDiffGUI frame = new AnnotationDiffGUI("Annotation Difference",
1518 documentName, documentName, keySetName,
1519 responseSetName, annotationType, featureSet);
1520 frame.pack();
1521 frame.setLocationRelativeTo(MainFrame.getInstance());
1522 frame.setVisible(true);
1523 }
1524 }
1525
1526 protected class ExportToHtmlAction extends AbstractAction{
1527 public ExportToHtmlAction(){
1528 super("Export to HTML");
1529 putValue(SHORT_DESCRIPTION, "Export the tables to HTML");
1530 putValue(SMALL_ICON,
1531 MainFrame.getIcon("crystal-clear-app-download-manager"));
1532 }
1533 public void actionPerformed(ActionEvent evt){
1534 XJFileChooser fileChooser = MainFrame.getFileChooser();
1535 fileChooser.setAcceptAllFileFilterUsed(true);
1536 fileChooser.setDialogTitle("Choose a file where to export the tables");
1537 fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY);
1538 ExtensionFileFilter filter = new ExtensionFileFilter("HTML files","html");
1539 fileChooser.addChoosableFileFilter(filter);
1540 String title = corpus.getName();
1541 title += "_" + keySetName;
1542 title += "-" + responseSetName;
1543 fileChooser.setFileName(title + ".html");
1544 fileChooser.setResource(CorpusQualityAssurance.class.getName());
1545 int res = fileChooser.showSaveDialog(CorpusQualityAssurance.this);
1546 if (res != JFileChooser.APPROVE_OPTION) { return; }
1547
1548 File saveFile = fileChooser.getSelectedFile();
1549 Writer fw = null;
1550 try{
1551 fw = new BufferedWriter(new FileWriter(saveFile));
1552
1553 // Header, Title
1554 fw.write(BEGINHTML + nl);
1555 fw.write(BEGINHEAD);
1556 fw.write(title);
1557 fw.write(ENDHEAD + nl);
1558 fw.write("<H1>Corpus Quality Assurance</H1>" + nl);
1559 fw.write("<P>Corpus: " + corpus.getName() + "<BR>" + nl);
1560 fw.write("Key set: " + keySetName + "<BR>" + nl);
1561 fw.write("Response set: " + responseSetName + "<BR>" + nl);
1562 fw.write("Types: "
1563 + Strings.toString(typeList.getSelectedValues()) + "<BR>" + nl);
1564 fw.write("Features: "
1565 + Strings.toString(featureList.getSelectedValues()) + "</P>" + nl);
1566 fw.write("<P> </P>" + nl);
1567
1568 ArrayList<JTable> tablesToExport = new ArrayList<JTable>();
1569 tablesToExport.add(annotationTable);
1570 tablesToExport.add(documentTable);
1571 tablesToExport.add(document2Table);
1572 tablesToExport.add(confusionTable);
1573 for (JTable table : tablesToExport) {
1574 fw.write(BEGINTABLE + nl + "<TR>" + nl);
1575 for(int col = 0; col < table.getColumnCount(); col++){
1576 fw.write("<TH align=\"left\">"
1577 + table.getColumnName(col) + "</TH>" + nl);
1578 }
1579 fw.write("</TR>" + nl);
1580 for(int row = 0; row < table.getRowCount(); row ++){
1581 fw.write("<TR>" + nl);
1582 for(int col = 0; col < table.getColumnCount(); col++){
1583 String value = (String) table.getValueAt(row, col);
1584 if (value == null) { value = ""; }
1585 fw.write("<TD>" + value + "</TD>" + nl);
1586 }
1587 fw.write("</TR>" + nl);
1588 }
1589 fw.write(ENDTABLE + nl);
1590 fw.write("<P> </P>" + nl);
1591 }
1592
1593 fw.write(ENDHTML + nl);
1594 fw.flush();
1595
1596 } catch(IOException ioe){
1597 JOptionPane.showMessageDialog(CorpusQualityAssurance.this,
1598 ioe.toString(), "GATE", JOptionPane.ERROR_MESSAGE);
1599 ioe.printStackTrace();
1600
1601 } finally {
1602 if (fw != null) {
1603 try {
1604 fw.close();
1605 } catch (IOException e) {
1606 e.printStackTrace();
1607 }
1608 }
1609 }
1610 }
1611
1612 final String nl = Strings.getNl();
1613 static final String BEGINHTML =
1614 "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">" +
1615 "<html>";
1616 static final String ENDHTML = "</body></html>";
1617 static final String BEGINHEAD = "<head>" +
1618 "<meta content=\"text/html; charset=utf-8\" http-equiv=\"content-type\">"
1619 + "<title>";
1620 static final String ENDHEAD = "</title></head><body>";
1621 static final String BEGINTABLE = "<table cellpadding=\"0\" border=\"1\">";
1622 static final String ENDTABLE = "</table>";
1623 }
1624
1625 class ReloadCacheAction extends AbstractAction{
1626 public ReloadCacheAction(){
1627 super("Reload cache", MainFrame.getIcon("crystal-clear-action-reload"));
1628 putValue(SHORT_DESCRIPTION,
1629 "Reload cache for set, type and feature names list");
1630 }
1631 public void actionPerformed(ActionEvent e){
1632 docsSetsTypesFeatures.clear();
1633 readSetsTypesFeatures(0);
1634 }
1635 }
1636
1637 protected class HelpAction extends AbstractAction {
1638 public HelpAction() {
1639 super();
1640 putValue(SHORT_DESCRIPTION, "User guide for this tool");
1641 putValue(SMALL_ICON, MainFrame.getIcon("crystal-clear-action-info"));
1642 putValue(ACCELERATOR_KEY, KeyStroke.getKeyStroke("F1"));
1643 }
1644 public void actionPerformed(ActionEvent e) {
1645 MainFrame.getInstance().showHelpFrame(
1646 "sec:eval:corpusqualityassurance",
1647 CorpusQualityAssurance.class.getName());
1648 }
1649 }
1650
1651 // local variables
1652 protected Corpus corpus;
1653 protected boolean corpusChanged;
1654 protected TreeSet<String> types;
1655 /** cache for document*set*type*feature names */
1656 final protected Map<String, TreeMap<String, TreeMap<String, TreeSet<String>>>>
1657 docsSetsTypesFeatures = Collections.synchronizedMap(new LinkedHashMap
1658 <String, TreeMap<String, TreeMap<String, TreeSet<String>>>>());
1659 /** ordered by document as in the <code>corpus</code>
1660 * then contains (annotation type * AnnotationDiffer) */
1661 protected ArrayList<HashMap<String, AnnotationDiffer>> differsByDocThenType =
1662 new ArrayList<HashMap<String, AnnotationDiffer>>();
1663 protected ArrayList<String> documentNames = new ArrayList<String>();
1664 protected String keySetName;
1665 protected String responseSetName;
1666 protected Object[] typesSelected;
1667 protected Object[] featuresSelected;
1668 protected Timer timer = new Timer("CorpusQualityAssurance", true);
1669 protected TimerTask timerTask;
1670 protected Thread readSetsTypesFeaturesThread;
1671 /** FSCORE_MEASURES or CLASSIFICATION_MEASURES */
1672 protected int measuresType;
1673 protected static final int FSCORE_MEASURES = 0;
1674 protected static final int CLASSIFICATION_MEASURES = 1;
1675 protected Collator collator;
1676 protected Comparator<String> doubleComparator;
1677 protected Comparator<String> totalComparator;
1678 protected OptionsMap userConfig = Gate.getUserConfig();
1679 protected URL bdmFileUrl;
1680
1681 // user interface components
1682 protected XJTable documentTable;
1683 protected DefaultTableModel documentTableModel;
1684 protected XJTable annotationTable;
1685 protected DefaultTableModel annotationTableModel;
1686 protected XJTable document2Table;
1687 protected DefaultTableModel document2TableModel;
1688 protected XJTable confusionTable;
1689 protected DefaultTableModel confusionTableModel;
1690 protected JTabbedPane tableTabbedPane;
1691 protected JList setList;
1692 protected JList typeList;
1693 protected JList featureList;
1694 protected JToggleButton optionsButton;
1695 protected JTabbedPane measureTabbedPane;
1696 protected JList measureList;
1697 protected JList measure2List;
1698 protected JCheckBox setCheck;
1699 protected JCheckBox typeCheck;
1700 protected JCheckBox featureCheck;
1701 protected JProgressBar progressBar;
1702 protected JCheckBox verboseOptionCheckBox;
1703
1704 // actions
1705 protected OpenDocumentAction openDocumentAction;
1706 protected OpenAnnotationDiffAction openAnnotationDiffAction;
1707 protected ExportToHtmlAction exportToHtmlAction;
1708 protected ReloadCacheAction reloadCacheAction;
1709 protected CompareAction compareAction;
1710 }
|