forked from stanfordnlp/CoreNLP
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' of scail:/u/nlp/git/javanlp
- Loading branch information
Gabor Angeli
authored and
Stanford NLP
committed
Dec 20, 2014
1 parent
3e6115e
commit 52cf0e8
Showing
19 changed files
with
438 additions
and
209 deletions.
There are no files selected for viewing
83 changes: 0 additions & 83 deletions
83
itest/src/edu/stanford/nlp/pipeline/PosParserTagCompatibilityITest.java
This file was deleted.
Oops, something went wrong.
124 changes: 124 additions & 0 deletions
124
itest/src/edu/stanford/nlp/pipeline/TaggerParserPosTagCompatibilityITest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| package edu.stanford.nlp.pipeline; | ||
|
|
||
| import java.util.Set; | ||
|
|
||
| import edu.stanford.nlp.parser.shiftreduce.ShiftReduceParser; | ||
| import junit.framework.TestCase; | ||
|
|
||
| import edu.stanford.nlp.parser.lexparser.LexicalizedParser; | ||
| import edu.stanford.nlp.tagger.maxent.MaxentTagger; | ||
|
|
||
| /** | ||
| * @author Christopher Manning | ||
| */ | ||
| public class TaggerParserPosTagCompatibilityITest extends TestCase { | ||
|
|
||
| private static void testTagSet3(String[] lexParsers, String[] maxentTaggers, String[] srParsers) { | ||
| LexicalizedParser lp = LexicalizedParser.loadModel(lexParsers[0]); | ||
| Set<String> tagSet = lp.getLexicon().tagSet(lp.treebankLanguagePack().getBasicCategoryFunction()); | ||
| for (String name : maxentTaggers) { | ||
| MaxentTagger tagger = new MaxentTagger(name); | ||
| assertEquals(lexParsers[0] + " vs. " + name + " tag set mismatch", tagSet, tagger.tagSet()); | ||
| } | ||
| for (String name : lexParsers) { | ||
| LexicalizedParser lp2 = LexicalizedParser.loadModel(name); | ||
| assertEquals(lexParsers[0] + " vs. " + name + " tag set mismatch", | ||
| tagSet, lp2.getLexicon().tagSet(lp.treebankLanguagePack().getBasicCategoryFunction())); | ||
| } | ||
|
|
||
| for (String name : srParsers) { | ||
| ShiftReduceParser srp = ShiftReduceParser.loadModel(name); | ||
|
|
||
| assertEquals(lexParsers[0] + " vs. " + name + " tag set mismatch", | ||
| tagSet, srp.tagSet()); | ||
| } | ||
| } | ||
|
|
||
|
|
||
| private static final String[] englishTaggers = { | ||
| "edu/stanford/nlp/models/pos-tagger/english-left3words/english-left3words-distsim.tagger", | ||
| "edu/stanford/nlp/models/pos-tagger/english-bidirectional/english-bidirectional-distsim.tagger", | ||
| "edu/stanford/nlp/models/pos-tagger/english-caseless-left3words-distsim.tagger", | ||
| }; | ||
|
|
||
| private static final String[] englishParsers = { | ||
| "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz", | ||
| "edu/stanford/nlp/models/lexparser/englishPCFG.caseless.ser.gz", | ||
| "edu/stanford/nlp/models/lexparser/englishRNN.ser.gz", | ||
| "edu/stanford/nlp/models/lexparser/englishFactored.ser.gz", | ||
| }; | ||
|
|
||
| private static final String[] englishSrParsers = { | ||
| "edu/stanford/nlp/models/srparser/englishSR.beam.ser.gz", | ||
| "edu/stanford/nlp/models/srparser/englishSR.ser.gz", | ||
| }; | ||
|
|
||
| public void testEnglishTagSet() { | ||
| testTagSet3(englishParsers, englishTaggers, englishSrParsers); | ||
| } | ||
|
|
||
|
|
||
| private static final String[] germanTaggers = { | ||
| "edu/stanford/nlp/models/pos-tagger/german/german-fast.tagger", | ||
| "edu/stanford/nlp/models/pos-tagger/german/german-fast-caseless.tagger", | ||
| "edu/stanford/nlp/models/pos-tagger/german/german-fast.tagger", | ||
| "edu/stanford/nlp/models/pos-tagger/german/german-hgc.tagger" | ||
| }; | ||
|
|
||
| private static final String[] germanParsers = { | ||
| "edu/stanford/nlp/models/lexparser/germanPCFG.ser.gz", | ||
| "edu/stanford/nlp/models/lexparser/germanFactored.ser.gz", | ||
| }; | ||
| private static final String[] germanSrParsers = { | ||
| "edu/stanford/nlp/models/srparser/germanSR.ser.gz", | ||
| }; | ||
|
|
||
| public void testGermanTagSet() { | ||
| testTagSet3(germanParsers, germanTaggers, germanSrParsers); | ||
| } | ||
|
|
||
|
|
||
| private static final String[] chineseTaggers = { | ||
| "edu/stanford/nlp/models/pos-tagger/chinese-distsim/chinese-distsim.tagger", | ||
| }; | ||
|
|
||
| private static final String[] chineseParsers = { | ||
| // Can't compare Xinhua ones because they have a smaller tag set than the full CTB v6+ | ||
| // "edu/stanford/nlp/models/lexparser/xinhuaPCFG.ser.gz", | ||
| "edu/stanford/nlp/models/lexparser/chineseFactored.ser.gz", | ||
| "edu/stanford/nlp/models/lexparser/chinesePCFG.ser.gz", | ||
| // "edu/stanford/nlp/models/lexparser/xinhuaFactoredSegmenting.ser.gz", | ||
| // "edu/stanford/nlp/models/lexparser/xinhuaFactored.ser.gz", | ||
|
|
||
| }; | ||
| private static final String[] chineseSrParsers = { | ||
| "edu/stanford/nlp/models/srparser/chineseSR.ser.gz", | ||
| }; | ||
|
|
||
| public void testChineseTagSet() { | ||
| testTagSet3(chineseParsers, chineseTaggers, chineseSrParsers); | ||
| } | ||
|
|
||
|
|
||
| private static final String[] spanishTaggers = { | ||
| "edu/stanford/nlp/models/pos-tagger/spanish/spanish.tagger", | ||
| "edu/stanford/nlp/models/pos-tagger/spanish/spanish-distsim.tagger", | ||
| }; | ||
|
|
||
| private static final String[] spanishParsers = { | ||
| "edu/stanford/nlp/models/lexparser/spanishPCFG.ser.gz", | ||
| }; | ||
|
|
||
| private static final String[] spanishSrParsers = { | ||
| // todo [cdm 2014]: For some reason the SR parsers don't have the same tag set, missing 6 tags.... | ||
| // "edu/stanford/nlp/models/srparser/spanishSR.ser.gz", | ||
| // "edu/stanford/nlp/models/srparser/spanishSR.beam.ser.gz", | ||
| }; | ||
|
|
||
| public void testSpanishTagSet() { | ||
| testTagSet3(spanishParsers, spanishTaggers, spanishSrParsers); | ||
| } | ||
|
|
||
| // todo: Add French and Arabic sometime | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.