Fix the JSONOutputter test

doc/loglinear/QUICKSTART.txt

@@ -2,7 +2,7 @@ loglinear package quickstart:
 
 First, read the ConcatVector section in ARCH.txt.
 
-To jump straight into working code, go read generateSentenceModel() in edu.stanford.nlp.loglinear.CoNLLBenchmark.
+To jump straight into working code, go read generateSentenceModel() in edu.stanford.nlp.loglinear.learning.CoNLLBenchmark.
 
 #####################################################
 

doc/loglinear/README.txt

@@ -1,6 +1,6 @@
 For an explanation of how everything fits together, see ARCH.txt
 
-For a quick runnable object, go run edu.stanford.nlp.loglinear.CoNLLBenchmark in core's test package.
+For a quick runnable object, go run edu.stanford.nlp.loglinear.learning.CoNLLBenchmark in core's test package.
 
 For a tutorial, see QUICKSTART.txt
 

itest/src/edu/stanford/nlp/parser/nndep/DependencyParserITest.java

@@ -46,15 +46,15 @@ public void testDependencyParserEnglishSD() {
  }
 
  // Lower because we're evaluating on PTB + extraDevTest, not just PTB
- private static final double EnglishUdLas = 88.72648417258083;
+ private static final double EnglishUdLas = 84.9873;
 
  /**
  * Test that the NN dependency parser performance doesn't change.
  */
  public void testDependencyParserEnglishUD() {
  DependencyParser parser = new DependencyParser();
  parser.loadModelFile("/u/nlp/data/depparser/nn/distrib-2015-04-16/english_UD.gz");
- double las = parser.testCoNLL("/u/nlp/data/depparser/nn/data/dependency_treebanks/UD-converted/dev.conll", null);
+ double las = parser.testCoNLL("/u/nlp/data/depparser/nn/data/dependency_treebanks/USD/dev.conll", null);
  assertEquals(String.format("English UD LAS should be %.2f but was %.2f",
  EnglishUdLas, las), EnglishUdLas, las, 1e-4);
  }

itest/src/edu/stanford/nlp/pipeline/AnnotationOutputterITest.java

@@ -43,72 +43,72 @@ public void testSimpleSentenceJSON() throws IOException {
  "{\n" +
  " \"sentences\": [\n" +
  " {\n" +
- " \"index\": \"0\",\n" +
+ " \"index\": 0,\n" +
  " \"parse\": \"(ROOT (NP (JJ Bad) (NN wolf)))\",\n" +
  " \"basic-dependencies\": [\n" +
  " {\n" +
  " \"dep\": \"ROOT\",\n" +
- " \"governor\": \"0\",\n" +
+ " \"governor\": 0,\n" +
  " \"governorGloss\": \"ROOT\",\n" +
- " \"dependent\": \"2\",\n" +
+ " \"dependent\": 2,\n" +
  " \"dependentGloss\": \"wolf\"\n" +
  " },\n" +
  " {\n" +
  " \"dep\": \"amod\",\n" +
- " \"governor\": \"2\",\n" +
+ " \"governor\": 2,\n" +
  " \"governorGloss\": \"wolf\",\n" +
- " \"dependent\": \"1\",\n" +
+ " \"dependent\": 1,\n" +
  " \"dependentGloss\": \"Bad\"\n" +
  " }\n" +
  " ],\n" +
  " \"collapsed-dependencies\": [\n" +
  " {\n" +
  " \"dep\": \"ROOT\",\n" +
- " \"governor\": \"0\",\n" +
+ " \"governor\": 0,\n" +
  " \"governorGloss\": \"ROOT\",\n" +
- " \"dependent\": \"2\",\n" +
+ " \"dependent\": 2,\n" +
  " \"dependentGloss\": \"wolf\"\n" +
  " },\n" +
  " {\n" +
  " \"dep\": \"amod\",\n" +
- " \"governor\": \"2\",\n" +
+ " \"governor\": 2,\n" +
  " \"governorGloss\": \"wolf\",\n" +
- " \"dependent\": \"1\",\n" +
+ " \"dependent\": 1,\n" +
  " \"dependentGloss\": \"Bad\"\n" +
  " }\n" +
  " ],\n" +
  " \"collapsed-ccprocessed-dependencies\": [\n" +
  " {\n" +
  " \"dep\": \"ROOT\",\n" +
- " \"governor\": \"0\",\n" +
+ " \"governor\": 0,\n" +
  " \"governorGloss\": \"ROOT\",\n" +
- " \"dependent\": \"2\",\n" +
+ " \"dependent\": 2,\n" +
  " \"dependentGloss\": \"wolf\"\n" +
  " },\n" +
  " {\n" +
  " \"dep\": \"amod\",\n" +
- " \"governor\": \"2\",\n" +
+ " \"governor\": 2,\n" +
  " \"governorGloss\": \"wolf\",\n" +
- " \"dependent\": \"1\",\n" +
+ " \"dependent\": 1,\n" +
  " \"dependentGloss\": \"Bad\"\n" +
  " }\n" +
  " ],\n" +
  " \"tokens\": [\n" +
  " {\n" +
- " \"index\": \"1\",\n" +
+ " \"index\": 1,\n" +
  " \"word\": \"Bad\",\n" +
  " \"lemma\": \"bad\",\n" +
- " \"characterOffsetBegin\": \"0\",\n" +
- " \"characterOffsetEnd\": \"3\",\n" +
+ " \"characterOffsetBegin\": 0,\n" +
+ " \"characterOffsetEnd\": 3,\n" +
  " \"pos\": \"JJ\",\n" +
  " \"ner\": \"O\"\n" +
  " },\n" +
  " {\n" +
- " \"index\": \"2\",\n" +
+ " \"index\": 2,\n" +
  " \"word\": \"wolf\",\n" +
  " \"lemma\": \"wolf\",\n" +
- " \"characterOffsetBegin\": \"4\",\n" +
- " \"characterOffsetEnd\": \"8\",\n" +
+ " \"characterOffsetBegin\": 4,\n" +
+ " \"characterOffsetEnd\": 8,\n" +
  " \"pos\": \"NN\",\n" +
  " \"ner\": \"O\"\n" +
  " }\n" +

src/edu/stanford/nlp/classify/LogConditionalObjectiveFunction.java

@@ -72,6 +72,7 @@ public class LogConditionalObjectiveFunction<L, F> extends AbstractStochasticCac
 
  /** Multithreading gradient calculations is a bit cheaper if you reuse the threads. */
  protected int threads = Execution.threads;
+ protected ExecutorService executorService = Executors.newFixedThreadPool(threads);
 
  @Override
  public int domainDimension() {
@@ -325,7 +326,7 @@ private void calculateCLbatch(double[] x) {
  CountDownLatch latch = new CountDownLatch(threads);
  for (int i = 0; i < threads; i++) {
  runnables[i] = new CLBatchDerivativeCalculation(threads, i, null, x, derivative.length, latch);
- new Thread(runnables[i]).start();
+ executorService.execute(runnables[i]);
  }
  try {
  latch.await();
@@ -683,7 +684,7 @@ public double calculateStochasticUpdate(double[] x, double xscale, int[] batch,
  CountDownLatch latch = new CountDownLatch(threads);
  for (int i = 0; i < threads; i++) {
  runnables[i] = new CLBatchDerivativeCalculation(threads, i, batch, x, x.length, latch);
- new Thread(runnables[i]).start();
+ executorService.execute(runnables[i]);
  }
  try {
  latch.await();
@@ -1004,7 +1005,7 @@ protected void rvfcalculate(double[] x) {
  CountDownLatch latch = new CountDownLatch(threads);
  for (int i = 0; i < threads; i++) {
  runnables[i] = new RVFDerivativeCalculation(threads, i, x, derivative.length, latch);
- new Thread(runnables[i]).start();
+ executorService.execute(runnables[i]);
  }
  try {
  latch.await();

src/edu/stanford/nlp/ie/crf/CRFLogConditionalObjectiveFunction.java

@@ -348,6 +348,18 @@ protected double regularGradientAndValue() {
  return multiThreadGradient(docIDs, false);
  }
 
+ private class GradientCalculator implements Runnable {
+ @Override
+ public void run() {
+ double[][] arr = empty2D();
+ }
+ }
+
+ protected double newMultithreadGradient(List<Integer> docIDs, boolean calculateEmpirical) {
+ double objective = 0.0;
+ return objective;
+ }
+
  protected double multiThreadGradient(List<Integer> docIDs, boolean calculateEmpirical) {
  double objective = 0.0;
  // TODO: This is a bunch of unnecessary heap traffic, should all be on the stack

src/edu/stanford/nlp/ling/CoreLabel.java

@@ -554,7 +554,7 @@ public void setEndPosition(int endPos) {
  public static final String TAG_SEPARATOR = "/";
 
  public enum OutputFormat {
- VALUE_INDEX, VALUE, VALUE_TAG, VALUE_TAG_INDEX, MAP, VALUE_MAP, VALUE_INDEX_MAP, WORD, WORD_INDEX, VALUE_TAG_NER, LEMMA_INDEX, ALL
+ VALUE_INDEX, VALUE, VALUE_TAG, VALUE_TAG_INDEX, MAP, VALUE_MAP, VALUE_INDEX_MAP, WORD, WORD_INDEX, VALUE_TAG_NER, ALL
  }
 
  public static final OutputFormat DEFAULT_FORMAT = OutputFormat.VALUE_INDEX;
@@ -684,13 +684,6 @@ public String toString(OutputFormat format) {
  }
  break;
  }
- case LEMMA_INDEX:
- buf.append(lemma());
- Integer index = this.get(CoreAnnotations.IndexAnnotation.class);
- if (index != null) {
- buf.append('-').append((index).intValue());
- }
- break;
  case ALL:{
  for(Class en: this.keySet()){
  buf.append(";").append(en).append(":").append(this.get(en));