-
Notifications
You must be signed in to change notification settings - Fork 9
/
pipelines.py
115 lines (76 loc) · 3.46 KB
/
pipelines.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
from abc import abstractmethod
import spacy
from legal_ner.ner.entity_ruler import init_entity_ruler
from legal_ner.ner.pattern_based import PatternBasedMatcher
from legal_ner.ner.regexp_based import RegexpBasedMatcher
from legal_ner.ner.regexps.dates import GermanDates
from legal_ner.ner.regexps.money import GermanEuros
from legal_ner.ner.regexps.percents import GermanPercentages
from legal_ner.ner.matcher_patterns import party, reasoning
from legal_ner.tokenization.special_cases import special_cases
from legal_ner.tokenization.tokenizer import create_custom_tokenizer
class Entity:
# statistical
PER = 'PER'
LOC = 'LOC'
ORG = 'ORG'
MISC = 'MISC'
# pattern based
PARTY = 'PARTY'
REASONING = 'REASONING'
# regexp based
DATE = 'DATE'
EURO = 'EURO'
PERCENT = 'PERCENT'
class Pipeline:
def __init__(self, model):
self.model = model
@abstractmethod
def run(self, text):
pass
class RuleBasedPipeline(Pipeline):
def run(self, text):
nlp = spacy.load(self.model)
nlp.tokenizer = create_custom_tokenizer(nlp)
for word, special_case in special_cases.items():
nlp.tokenizer.add_special_case(word, special_case)
nlp.remove_pipe('parser')
nlp.remove_pipe('ner')
party_matcher = PatternBasedMatcher(nlp, party, Entity.PARTY)
nlp.add_pipe(party_matcher, name='party_matcher')
reasoning_matcher = PatternBasedMatcher(nlp, reasoning, Entity.REASONING)
nlp.add_pipe(reasoning_matcher, name='reasoning_matcher')
date_matcher = RegexpBasedMatcher(nlp, Entity.DATE, GermanDates())
nlp.add_pipe(date_matcher, name='regexp_dates_extractor')
percents_matcher = RegexpBasedMatcher(nlp, Entity.EURO, GermanEuros())
nlp.add_pipe(percents_matcher, name='regexp_euro_extractor')
percents_matcher = RegexpBasedMatcher(nlp, Entity.PERCENT, GermanPercentages())
nlp.add_pipe(percents_matcher, name='regexp_percent_extractor')
doc = nlp(text)
return doc
class StatisticalPipeline(Pipeline):
def run(self, text):
nlp = spacy.load(self.model)
nlp.tokenizer = create_custom_tokenizer(nlp)
for word, special_case in special_cases.items():
nlp.tokenizer.add_special_case(word, special_case)
doc = nlp(text)
return doc
class JoinedPipeline(Pipeline):
def run(self, text):
nlp = spacy.load(self.model)
nlp.tokenizer = create_custom_tokenizer(nlp)
for word, special_case in special_cases.items():
nlp.tokenizer.add_special_case(word, special_case)
party_entity_ruler = init_entity_ruler(nlp, party, Entity.PARTY)
nlp.add_pipe(party_entity_ruler, name='party_entity_ruler', before='ner')
reasoning_entity_ruler = init_entity_ruler(nlp, reasoning, Entity.REASONING)
nlp.add_pipe(reasoning_entity_ruler, name='reasoning_entity_ruler', before='ner')
date_matcher = RegexpBasedMatcher(nlp, Entity.DATE, GermanDates())
nlp.add_pipe(date_matcher, name='regexp_dates_extractor', before='ner')
percents_matcher = RegexpBasedMatcher(nlp, Entity.EURO, GermanEuros())
nlp.add_pipe(percents_matcher, name='regexp_euro_extractor', before='ner')
percents_matcher = RegexpBasedMatcher(nlp, Entity.PERCENT, GermanPercentages())
nlp.add_pipe(percents_matcher, name='regexp_percent_extractor', before='ner')
doc = nlp(text)
return doc