Keep the Misc fields on a CoreLabel

Author: AngledLuffa

itest/src/edu/stanford/nlp/pipeline/CoNLLUReaderITest.java

@@ -52,6 +52,10 @@ public class CoNLLUReaderITest {
     {"sps00", "rg", "da0fp0", "ncfp000", "fp"},
   };
 
+  static final String[][] EXPECTED_MISC = {
+    {null, null, "ArgTem=arg1:tem", null, null, "ArgTem=argM:tmp", "ArgTem=arg1:tem", null, null, null, "ArgTem=argM:loc", "ArgTem=argM:adv", null, null, null, null, null, "ArgTem=arg2:atr", null, "ArgTem=arg0:agt", null},
+    {null, null, null, null, null},
+  };
 
   static final String[][] EXPECTED_FEATS = {
     {
@@ -232,11 +236,12 @@ public void testReadingInCoNLLUFile() throws ClassNotFoundException, IOException
       }
     }
 
-    // check the features and that there are no fields currently unaccounted for
+    // check the features, the misc columns, and that there are no fields currently unaccounted for
     for (int i = 0; i < sentences.size(); ++i) {
       CoreMap sentence = sentences.get(i);
       List<CoreLabel> tokens = sentence.get(CoreAnnotations.TokensAnnotation.class);
       assertEquals(EXPECTED_FEATS[i].length, tokens.size());
+      assertEquals(EXPECTED_MISC[i].length, tokens.size());
       for (int j = 0; j < tokens.size(); ++j) {
         CoreLabel token = tokens.get(j);
 
@@ -251,6 +256,15 @@ public void testReadingInCoNLLUFile() throws ClassNotFoundException, IOException
           assertEquals(expected, feats);
         }
 
+        String expectedMisc = EXPECTED_MISC[i][j];
+        if (expectedMisc == null) {
+          assertFalse(token.containsKey(CoreAnnotations.CoNLLUMisc.class));
+        } else {
+          expectedKeys += 1;
+          String misc = token.get(CoreAnnotations.CoNLLUMisc.class).toString();
+          assertEquals(expectedMisc, misc);
+        }
+
         // Some of the AnCora sentences don't have XPOS
         if (token.containsKey(CoreAnnotations.PartOfSpeechAnnotation.class)) {
           expectedKeys += 1;

src/edu/stanford/nlp/pipeline/CoNLLUReader.java

@@ -427,6 +427,7 @@ public CoreLabel convertLineToCoreLabel(CoNLLUSentence sentence, String line) {
     String spacesBefore = miscKeyValues.get("SpacesBefore");
     if (spacesBefore != null) {
       cl.setBefore(unescapeSpacesAfter(spacesBefore));
+      miscKeyValues.remove("SpacesBefore");
     }
 
     // handle the MWT info and after text
@@ -474,6 +475,21 @@ public CoreLabel convertLineToCoreLabel(CoNLLUSentence sentence, String line) {
       String spaceAfter = miscToSpaceAfter(miscKeyValues);
       cl.setAfter(spaceAfter);
     }
+    miscKeyValues.remove("SpaceAfter");
+    miscKeyValues.remove("SpacesAfter");
+    if (miscKeyValues.size() > 0) {
+      // rebuild the misc, since we have removed the SpaceAfter, SpacesAfter, and SpacesBefore
+      StringBuilder misc = new StringBuilder();
+      for (Map.Entry<String, String> entry : miscKeyValues.entrySet()) {
+        if (misc.length() > 0) {
+          misc.append("|");
+        }
+        misc.append(entry.getKey());
+        misc.append("=");
+        misc.append(entry.getValue());
+      }
+      cl.set(CoreAnnotations.CoNLLUMisc.class, misc.toString());
+    }
     return cl;
   }