Merge branch 'master' of jamie:/u/nlp/git/javanlp

README.md

@@ -5,6 +5,15 @@ Stanford CoreNLP provides a set of natural language analysis tools written in Ja
 
 The Stanford CoreNLP code is written in Java and licensed under the GNU General Public License (v3 or later). Note that this is the full GPL, which allows many free uses, but not its use in proprietary software that you distribute to others.
 
+#### How To Compile (with ant)
+
+1. cd CoreNLP ; ant
+
+#### How To Create A Jar 
+
+1. compile the code
+2. cd CoreNLP/classes ; jar -cf ../stanford-corenlp.jar edu
+
 You can find releases of Stanford CoreNLP on [Maven Central](http:https://search.maven.org/#browse%7C11864822).
 
 You can find more explanation and documentation on [the Stanford CoreNLP homepage](http:https://nlp.stanford.edu/software/corenlp.shtml#Demo).

doc/corenlp/META-INF/MANIFEST.MF

@@ -0,0 +1,6 @@
+Manifest-Version: 1.0
+Implementation-Version: 3.7.0
+Built-Date: 2016-04-27
+Created-By: Stanford JavaNLP
+Main-class: edu.stanford.nlp.pipeline.StanfordCoreNLP
+

itest/src/edu/stanford/nlp/time/SUTimeITest.java

@@ -1004,8 +1004,8 @@ public void testSUTimeDateTime() throws IOException {
  "It happened late this afternoon.\n" +
  "It happened at 1800 hours.\n" +
  "The early nineteen fifties.\n" +
- "The story broke in the last week of October.\n";
-// "It was 7pm and then 7:20pm."; // TODO: re-enable me
+ "The story broke in the last week of October.\n" +
+ "It was 7pm and then 7:20pm.";
 
  // set up expected results
  Iterator<Timex> expectedTimexes =
@@ -1021,9 +1021,9 @@ public void testSUTimeDateTime() throws IOException {
  Timex.fromXml("<TIMEX3 tid=\"t12\" alt_value=\"THIS AF\" type=\"DATE\" mod=\"LATE\" temporalFunction=\"true\" valueFromFunction=\"tf2\" anchorTimeID=\"t0\">late this afternoon</TIMEX3>"), // TODO: time
  Timex.fromXml("<TIMEX3 tid=\"t13\" value=\"T18:00\" type=\"TIME\">1800 hours</TIMEX3>"),
  Timex.fromXml("<TIMEX3 tid=\"t14\" value=\"195X\" type=\"DATE\" mod=\"EARLY\">The early nineteen fifties</TIMEX3>"),
- Timex.fromXml("<TIMEX3 tid=\"t15\" alt_value=\"PREV_IMMEDIATE P1W INTERSECT XXXX-10\" type=\"DATE\" temporalFunction=\"true\" valueFromFunction=\"tf3\" anchorTimeID=\"t16\">the last week of October</TIMEX3>")
-// Timex.fromXml("<TIMEX3 tid=\"t17\" value=\"T19:00\" type=\"TIME\">7pm</TIMEX3>"),
-// Timex.fromXml("<TIMEX3 tid=\"t18\" value=\"T19:20\" type=\"TIME\">7:20pm.</TIMEX3>") // TODO: the period should be dropped
+ Timex.fromXml("<TIMEX3 tid=\"t15\" alt_value=\"PREV_IMMEDIATE P1W INTERSECT XXXX-10\" type=\"DATE\" temporalFunction=\"true\" valueFromFunction=\"tf3\" anchorTimeID=\"t16\">the last week of October</TIMEX3>"),
+ Timex.fromXml("<TIMEX3 tid=\"t17\" value=\"T19:00\" type=\"TIME\">7pm</TIMEX3>"),
+ Timex.fromXml("<TIMEX3 tid=\"t18\" value=\"T19:20\" type=\"TIME\">7:20pm.</TIMEX3>") // TODO: the period should be dropped
  ).iterator();
 
  Iterator<Timex> expectedTimexesResolved =
@@ -1039,9 +1039,9 @@ public void testSUTimeDateTime() throws IOException {
  Timex.fromXml("<TIMEX3 tid=\"t10\" value=\"2005-08-12TAF\" type=\"TIME\" mod=\"LATE\">late this afternoon</TIMEX3>"),
  Timex.fromXml("<TIMEX3 tid=\"t11\" value=\"2005-08-12T18:00\" type=\"TIME\">1800 hours</TIMEX3>"),
  Timex.fromXml("<TIMEX3 tid=\"t12\" value=\"195X\" type=\"DATE\" mod=\"EARLY\">The early nineteen fifties</TIMEX3>"),
- Timex.fromXml("<TIMEX3 tid=\"t13\" alt_value=\"PREV_IMMEDIATE P1W INTERSECT XXXX-10\" type=\"DATE\" temporalFunction=\"true\" valueFromFunction=\"tf0\" anchorTimeID=\"t14\">the last week of October</TIMEX3>") // TODO: Resolve
-// Timex.fromXml("<TIMEX3 tid=\"t15\" value=\"2005-08-12T19:00\" type=\"TIME\">7pm</TIMEX3>"),
-// Timex.fromXml("<TIMEX3 tid=\"t16\" value=\"2005-08-12T19:20\" type=\"TIME\">7:20pm.</TIMEX3>") // TODO: the period should be dropped
+ Timex.fromXml("<TIMEX3 tid=\"t13\" alt_value=\"PREV_IMMEDIATE P1W INTERSECT XXXX-10\" type=\"DATE\" temporalFunction=\"true\" valueFromFunction=\"tf0\" anchorTimeID=\"t14\">the last week of October</TIMEX3>"), // TODO: Resolve
+ Timex.fromXml("<TIMEX3 tid=\"t15\" value=\"2005-08-12T19:00\" type=\"TIME\">7pm</TIMEX3>"),
+ Timex.fromXml("<TIMEX3 tid=\"t16\" value=\"2005-08-12T19:20\" type=\"TIME\">7:20pm.</TIMEX3>") // TODO: the period should be dropped
  ).iterator();
 
  // create document
@@ -1069,7 +1069,7 @@ public void testSUTimeDateTime() throws IOException {
  }
 
  // TODO: Re-enable me
- public void _testSUTimeDateTime2() throws IOException {
+ public void testSUTimeDateTime2() throws IOException {
  // Set up test text
  String testText = "The meeting is scheduled for 09/18/05 or 18 Sep '05.\n" +
  "1 year ago tomorrow.\n" +

liblocal/README

@@ -13,34 +13,32 @@ DESCRIPTION: ANTLR runtime, for compiled software
 
 URL: http:https://www.antlr.com
 
-USED BY:
-The Quickcheck library
+USED BY: The Quickcheck library (not directly used in Stanford NLP code)
 
 LAST UPDATE: 2015/10/5
 
 LAST UPDATE BY: Keenon Werling
 
 -----------------------------------------------------------------------
-hamcrest-core.jar
+java-hamcrest.jar
 
-ORIGINAL JAR NAME: hamcrest-core-1.3.jar
+ORIGINAL JAR NAME: java-hamcrest-2.0.0.0.jar
 
-VERSION: 1.3
+VERSION: 2.0.0.0
 
-RELEASE DATE: Jul, 2010
+RELEASE DATE: January 2015
 
 SOURCE AVAILABLE: Maven Central
 
 DESCRIPTION: Hamcrest shennanigans, for JUnit
 
 URL: http:https://www.hamcrest.org
 
-USED BY:
-The JUnit library
+USED BY: The JUnit library (not directly used in Stanford NLP code)
 
-LAST UPDATE: 2015/10/5
+LAST UPDATE: 2016-04-30
 
-LAST UPDATE BY: Keenon Werling
+LAST UPDATE BY: John Bauer
 
 -----------------------------------------------------------------------
 javaruntype.jar
@@ -57,8 +55,7 @@ DESCRIPTION: Something for Quickcheck
 
 URL: http:https://www.javaruntype.org
 
-USED BY:
-The Quickcheck library
+USED BY: The Quickcheck library (not directly used in Stanford NLP code)
 
 LAST UPDATE: 2015/10/5
 
@@ -79,8 +76,7 @@ DESCRIPTION: Quickcheck, runs random inputs and validates outputs
 
 URL: https://github.com/pholser/junit-quickcheck
 
-USED BY:
-The Quickcheck library
+USED BY: loglinear package tests
 
 LAST UPDATE: 2015/10/5
 
@@ -97,7 +93,7 @@ RELEASE DATE: Nov, 2013
 
 SOURCE AVAILABLE: Maven Central
 
-DESCRIPTION: Quickcheck, runs random inputs and validates outputs
+DESCRIPTION: loglinear package tests
 
 URL: https://github.com/pholser/junit-quickcheck
 
@@ -123,8 +119,7 @@ DESCRIPTION: JUnit theories run JUnit against a number of inputs
 
 URL: junit.org
 
-USED BY:
-The Quickcheck library
+USED BY: loglinear package tests
 
 LAST UPDATE: 2015/10/5
 
@@ -145,8 +140,7 @@ DESCRIPTION: Object graph navigation library, used by Quickcheck
 
 URL: https://commons.apache.org/proper/commons-ognl/
 
-USED BY:
-The Quickcheck library
+USED BY: The Quickcheck library (not directly used in Stanford NLP code)
 
 LAST UPDATE: 2015/10/5
 

src/edu/stanford/nlp/graph/DirectedMultiGraph.java

@@ -2,7 +2,6 @@
 
 import java.util.*;
 
-import edu.stanford.nlp.semgraph.SemanticGraphEdge;
 import edu.stanford.nlp.util.CollectionUtils;
 import edu.stanford.nlp.util.Generics;
 import edu.stanford.nlp.util.MapFactory;
@@ -41,8 +40,9 @@ public DirectedMultiGraph(MapFactory<V, Map<V, List<E>>> outerMapFactory, MapFac
  }
 
  /**
- * Creates a copy of the given graph. This will copy the entire data structure (this may be slow!), but will not copy
- * any of the edge or vertex objects.
+ * Creates a copy of the given graph. This will copy the entire data
+ * structure (this may be slow!), but will not copy any of the edge
+ * or vertex objects.
  *
  * @param graph The graph to copy into this object.
  */
@@ -477,39 +477,38 @@ public List<Set<V>> getConnectedComponents() {
  /**
  * Deletes all duplicate edges.
  */
-
- public void deleteDuplicateEdges() {
- for (V vertex : getAllVertices()) {
- for (V vertex2 : outgoingEdges.get(vertex).keySet()) {
- List<E> data = outgoingEdges.get(vertex).get(vertex2);
- Set<E> deduplicatedData = new TreeSet<>(data);
- data.clear();
- data.addAll(deduplicatedData);
- }
- for (V vertex2 : incomingEdges.get(vertex).keySet()) {
- List<E> data = incomingEdges.get(vertex).get(vertex2);
- Set<E> deduplicatedData = new TreeSet<>(data);
- data.clear();
- data.addAll(deduplicatedData);
- }
- }
-}
+ public void deleteDuplicateEdges() {
+ for (V vertex : getAllVertices()) {
+ for (V vertex2 : outgoingEdges.get(vertex).keySet()) {
+ List<E> data = outgoingEdges.get(vertex).get(vertex2);
+ Set<E> deduplicatedData = new TreeSet<>(data);
+ data.clear();
+ data.addAll(deduplicatedData);
+ }
+ for (V vertex2 : incomingEdges.get(vertex).keySet()) {
+ List<E> data = incomingEdges.get(vertex).get(vertex2);
+ Set<E> deduplicatedData = new TreeSet<>(data);
+ data.clear();
+ data.addAll(deduplicatedData);
+ }
+ }
+ }
 
 
  public Iterator<E> incomingEdgeIterator(final V vertex) {
- return new EdgeIterator<>(incomingEdges, vertex);
+ return new EdgeIterator<>(vertex, incomingEdges, outgoingEdges);
  }
 
  public Iterable<E> incomingEdgeIterable(final V vertex) {
- return () -> new EdgeIterator<>(incomingEdges, vertex);
+ return () -> new EdgeIterator<>(vertex, incomingEdges, outgoingEdges);
  }
 
  public Iterator<E> outgoingEdgeIterator(final V vertex) {
- return new EdgeIterator<>(outgoingEdges, vertex);
+ return new EdgeIterator<>(vertex, outgoingEdges, incomingEdges);
  }
 
  public Iterable<E> outgoingEdgeIterable(final V vertex) {
- return () -> new EdgeIterator<>(outgoingEdges, vertex);
+ return () -> new EdgeIterator<>(vertex, outgoingEdges, incomingEdges);
  }
 
  public Iterator<E> edgeIterator() {
@@ -520,28 +519,40 @@ public Iterable<E> edgeIterable() {
  return () -> new EdgeIterator<>(DirectedMultiGraph.this);
  }
 
-
+
+ /**
+ * This class handles either iterating over a single vertex's
+ * connections or over all connections in a graph.
+ */
  static class EdgeIterator<V, E> implements Iterator<E> {
- private final Map<V, Map<V, List<E>>> incomingEdges;
- private Iterator<Map<V, List<E>>> vertexIterator;
- private Iterator<List<E>> connectionIterator;
+ private final Map<V, Map<V, List<E>>> reverseEdges;
+ /** when iterating over the whole graph, this iterates over nodes */
+ private Iterator<Map.Entry<V, Map<V, List<E>>>> vertexIterator;
+ /** for a given node, this iterates over its neighbors */
+ private Iterator<Map.Entry<V, List<E>>> connectionIterator;
+ /** given the neighbor of a node, this iterates over all its connections */
  private Iterator<E> edgeIterator;
- private E lastRemoved = null;
+
+ private V currentSource = null;
+ private V currentTarget = null;
+ private E currentEdge = null;
  private boolean hasNext = true;
 
 
  public EdgeIterator(DirectedMultiGraph<V, E> graph) {
- vertexIterator = graph.outgoingEdges.values().iterator();
- incomingEdges = graph.incomingEdges;
+ vertexIterator = graph.outgoingEdges.entrySet().iterator();
+ reverseEdges = graph.incomingEdges;
  }
 
- public EdgeIterator(Map<V, Map<V, List<E>>> source, V startVertex) {
+ public EdgeIterator(V startVertex, Map<V, Map<V, List<E>>> source,
+ Map<V, Map<V, List<E>>> reverseEdges) {
+ currentSource = startVertex;
  Map<V, List<E>> neighbors = source.get(startVertex);
  if (neighbors != null) {
  vertexIterator = null;
- connectionIterator = neighbors.values().iterator();
+ connectionIterator = neighbors.entrySet().iterator();
  }
- incomingEdges = null;
+ this.reverseEdges = reverseEdges;
  }
 
  @Override
@@ -555,18 +566,22 @@ public E next() {
  if (!hasNext()) {
  throw new NoSuchElementException("Graph edge iterator exhausted.");
  }
- lastRemoved = edgeIterator.next();
- return lastRemoved;
+ currentEdge = edgeIterator.next();
+ return currentEdge;
  }
 
  private void primeIterator() {
  if (edgeIterator != null && edgeIterator.hasNext()) {
  hasNext = true; // technically, we shouldn't need to put this here, but let's be safe
  } else if (connectionIterator != null && connectionIterator.hasNext()) {
- edgeIterator = connectionIterator.next().iterator();
+ Map.Entry<V, List<E>> nextConnection = connectionIterator.next();
+ edgeIterator = nextConnection.getValue().iterator();
+ currentTarget = nextConnection.getKey();
  primeIterator();
  } else if (vertexIterator != null && vertexIterator.hasNext()) {
- connectionIterator = vertexIterator.next().values().iterator();
+ Map.Entry<V, Map<V, List<E>>> nextVertex = vertexIterator.next();
+ connectionIterator = nextVertex.getValue().entrySet().iterator();
+ currentSource = nextVertex.getKey();
  primeIterator();
  } else {
  hasNext = false;
@@ -575,22 +590,60 @@ private void primeIterator() {
 
  @Override
  public void remove() {
- if (incomingEdges == null) {
- throw new UnsupportedOperationException("remove() is only valid if iterating over entire graph (Gabor was too lazy to implement the general case...sorry!)");
+ if (currentEdge != null) {
+ reverseEdges.get(currentTarget).get(currentSource).remove(currentEdge);
+ edgeIterator.remove();
+
+ if (reverseEdges.get(currentTarget).get(currentSource) != null &&
+ reverseEdges.get(currentTarget).get(currentSource).size() == 0) {
+ connectionIterator.remove();
+ reverseEdges.get(currentTarget).remove(currentSource);
+ // TODO: may not be necessary to set this to null
+ edgeIterator = null;
+ }
  }
- if (lastRemoved != null) {
- if (lastRemoved instanceof SemanticGraphEdge) {
- SemanticGraphEdge edge = (SemanticGraphEdge) lastRemoved;
- //noinspection unchecked
- incomingEdges.get((V) edge.getDependent()).get((V) edge.getGovernor()).remove((E) edge);
- edgeIterator.remove();
- } else {
- // TODO(from gabor): This passes the tests, but actually leaves the graph in an inconsistent state.
- edgeIterator.remove();
-// throw new UnsupportedOperationException("remove() is only valid if iterating over semantic graph edges (Gabor was too lazy to implement the general case...sorry!)");
+ }
+ }
+
+ /**
+ * Topological sort of the graph.
+ * <br>
+ * This method uses the depth-first search implementation of
+ * topological sort.
+ * Topological sorting only works if the graph is acyclic.
+ *
+ * @return A sorted list of the vertices
+ * @throws IllegalStateException if this graph is not a DAG
+ */
+ public List<V> topologicalSort() {
+ List<V> result = Generics.newArrayList();
+ Set<V> temporary = outerMapFactory.newSet();
+ Set<V> permanent = outerMapFactory.newSet();
+ for (V vertex : getAllVertices()) {
+ if (!temporary.contains(vertex)) {
+ topologicalSortHelper(vertex, temporary, permanent, result);
+ }
+ }
+ Collections.reverse(result);
+ return result;
+ }
+
+ private void topologicalSortHelper(V vertex, Set<V> temporary, Set<V> permanent, List<V> result) {
+ temporary.add(vertex);
+ Map<V, List<E>> neighborMap = outgoingEdges.get(vertex);
+ if (neighborMap != null) {
+ for (V neighbor : neighborMap.keySet()) {
+ if (permanent.contains(neighbor)) {
+ continue;
+ }
+ if (temporary.contains(neighbor)) {
+ throw new IllegalStateException("This graph has cycles. Topological sort not possible: " + this.toString());
  }
+ topologicalSortHelper(neighbor, temporary, permanent, result);
  }
  }
+ result.add(vertex);
+ permanent.add(vertex);
  }
 
  /**