-
Notifications
You must be signed in to change notification settings - Fork 130
/
normalize_embeddings.py
44 lines (35 loc) · 1.82 KB
/
normalize_embeddings.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
# Copyright (C) 2016-2018 Mikel Artetxe <[email protected]>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import embeddings
import argparse
import sys
def main():
# Parse command line arguments
parser = argparse.ArgumentParser(description='Normalize word embeddings')
parser.add_argument('actions', choices=['unit', 'center', 'unitdim', 'centeremb'], nargs='*', default=[], help='the actions to perform in order')
parser.add_argument('-i', '--input', default=sys.stdin.fileno(), help='the input word embedding file (defaults to stdin)')
parser.add_argument('-o', '--output', default=sys.stdout.fileno(), help='the output word embedding file (defaults to stdout)')
parser.add_argument('--encoding', default='utf-8', help='the character encoding for input/output (defaults to utf-8)')
args = parser.parse_args()
# Read input embeddings
f = open(args.input, encoding=args.encoding, errors='surrogateescape')
words, matrix = embeddings.read(f)
# Perform normalization actions
embeddings.normalize(matrix, args.actions)
# Write normalized embeddings
f = open(args.output, mode='w', encoding=args.encoding, errors='surrogateescape')
embeddings.write(words, matrix, f)
if __name__ == '__main__':
main()