forked from HumanSignal/label-studio
-
Notifications
You must be signed in to change notification settings - Fork 0
/
server.py
384 lines (303 loc) · 13.2 KB
/
server.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
"""This file and its contents are licensed under the Apache License 2.0. Please see the included NOTICE for copyright information and LICENSE for a copy of the license.
"""
import sys
import logging
import socket
import pathlib
import os
import io
import json
import getpass
from colorama import init, Fore
if sys.platform == 'win32':
init(convert=True)
# on windows there will be problems with sqlite and json1 support, so fix it
from label_studio.core.utils.windows_sqlite_fix import windows_dll_fix
windows_dll_fix()
from django.core.management import call_command
from django.db import IntegrityError
from django.core.wsgi import get_wsgi_application
from django.db.migrations.executor import MigrationExecutor
from django.db import connections, DEFAULT_DB_ALIAS
from label_studio.core.argparser import parse_input_args
from label_studio.core.utils.params import get_env
logger = logging.getLogger(__name__)
LS_PATH = str(pathlib.Path(__file__).parent.absolute())
DEFAULT_USERNAME = 'default_user@localhost'
def _setup_env():
sys.path.insert(0, LS_PATH)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "label_studio.core.settings.label_studio")
application = get_wsgi_application()
def _app_run(host, port):
http_socket = '{}:{}'.format(host, port)
call_command('runserver', '--noreload', http_socket)
def is_database_synchronized(database):
connection = connections[database]
connection.prepare_database()
executor = MigrationExecutor(connection)
targets = executor.loader.graph.leaf_nodes()
return not executor.migration_plan(targets)
def _apply_database_migrations():
if not is_database_synchronized(DEFAULT_DB_ALIAS):
print('Initializing database..')
call_command('migrate', '--no-color', verbosity=0)
def _get_config(config_path):
with io.open(os.path.abspath(config_path), encoding='utf-8') as c:
config = json.load(c)
return config
def _create_project(title, user, label_config=None, sampling=None, description=None):
from projects.models import Project
from organizations.models import Organization
project = Project.objects.filter(title=title).first()
if project is not None:
print('Project with title "{}" already exists'.format(title))
else:
org = Organization.objects.first()
org.add_user(user)
project = Project.objects.create(title=title, created_by=user, organization=org)
print('Project with title "{}" successfully created'.format(title))
if label_config is not None:
with open(os.path.abspath(label_config)) as c:
project.label_config = c.read()
if sampling is not None:
project.sampling = sampling
if description is not None:
project.description = description
project.save()
return project
def _get_user_info(username):
from users.models import User
from users.serializers import UserSerializer
if not username:
username = DEFAULT_USERNAME
user = User.objects.filter(email=username)
if not user.exists():
print({'status': 'error', 'message': f"user {username} doesn't exist"})
return
user = user.first()
user_data = UserSerializer(user).data
user_data['token'] = user.auth_token.key
user_data['status'] = 'ok'
print('=> User info:')
print(user_data)
return user_data
def _create_user(input_args, config):
from users.models import User
from organizations.models import Organization
username = input_args.username or config.get('username') or get_env('USERNAME')
password = input_args.password or config.get('password') or get_env('PASSWORD')
token = input_args.user_token or config.get('user_token') or get_env('USER_TOKEN')
if not username:
user = User.objects.filter(email=DEFAULT_USERNAME).first()
if user is not None:
if password and not user.check_password(password):
user.set_password(password)
user.save()
print(f'User {DEFAULT_USERNAME} password changed')
return user
print(f'Please enter default user email, or press Enter to use {DEFAULT_USERNAME}')
username = input('Email: ')
if not username:
username = DEFAULT_USERNAME
if not password:
password = getpass.getpass(f'Default user password {DEFAULT_USERNAME}: ')
try:
user = User.objects.create_user(email=username, password=password)
user.is_staff = True
user.is_superuser = True
user.save()
if token and len(token) > 5:
from rest_framework.authtoken.models import Token
Token.objects.filter(key=user.auth_token.key).update(key=token)
else:
print(f"Token {token} is not applied to user {DEFAULT_USERNAME} "
f"because it's empty or len(token) < 5")
except IntegrityError:
print('User {} already exists'.format(username))
user = User.objects.get(email=username)
org = Organization.objects.first()
if not org:
org = Organization.create_organization(created_by=user, title='Label Studio')
else:
org.add_user(user)
user.active_organization = org
user.save(update_fields=['active_organization'])
return user
def _init(input_args, config):
if not _project_exists(input_args.project_name):
from projects.models import Project
sampling_map = {'sequential': Project.SEQUENCE, 'uniform': Project.UNIFORM,
'prediction-score-min': Project.UNCERTAINTY}
user = _create_user(input_args, config)
_create_project(
title=input_args.project_name,
user=user,
label_config=input_args.label_config,
description=input_args.project_desc,
sampling=sampling_map.get(input_args.sampling, 'sequential')
)
else:
print('Project "{0}" already exists'.format(input_args.project_name))
def _reset_password(input_args):
from users.models import User
username = input_args.username
if not username:
username = input('Username: ')
user = User.objects.filter(email=username).first()
if user is None:
print('User with username {} not found'.format(username))
return
password = input_args.password
if not password:
password = getpass.getpass('New password:')
if not password:
print('Can not set empty password')
return
if user.check_password(password):
print('Entered password is the same as current')
return
user.set_password(password)
user.save()
print('Password successfully changed')
def check_port_in_use(host, port):
logger.info('Checking if host & port is available :: ' + str(host) + ':' + str(port))
host = host.replace('https://', '').replace('http:https://', '')
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
return s.connect_ex((host, port)) == 0
def _get_free_port(port, debug):
# check port is busy
if not debug:
original_port = port
# try up to 1000 new ports
while check_port_in_use('localhost', port):
old_port = port
port = int(port) + 1
if port - original_port >= 1000:
raise ConnectionError(
'\n*** WARNING! ***\n Could not find an available port\n'
+ ' to launch label studio. \n Last tested port was '
+ str(port)
+ '\n****************\n'
)
print(
'\n*** WARNING! ***\n* Port '
+ str(old_port)
+ ' is in use.\n'
+ '* Trying to start at '
+ str(port)
+ '\n****************\n'
)
return port
def _project_exists(project_name):
from projects.models import Project
return Project.objects.filter(title=project_name).exists()
def main():
input_args = parse_input_args(sys.argv[1:])
# setup logging level
if input_args.log_level:
os.environ.setdefault("LOG_LEVEL", input_args.log_level)
if input_args.database:
database_path = pathlib.Path(input_args.database)
os.environ.setdefault("DATABASE_NAME", str(database_path.absolute()))
if input_args.data_dir:
data_dir_path = pathlib.Path(input_args.data_dir)
os.environ.setdefault("LABEL_STUDIO_BASE_DATA_DIR", str(data_dir_path.absolute()))
config = _get_config(input_args.config_path)
# set host name
host = input_args.host or config.get('host', '')
if not get_env('HOST'):
os.environ.setdefault('HOST', host) # it will be passed to settings.HOSTNAME as env var
_setup_env()
_apply_database_migrations()
from label_studio.core.utils.common import collect_versions
versions = collect_versions()
if input_args.command == 'reset_password':
_reset_password(input_args)
return
if input_args.command == 'shell':
call_command('shell_plus')
return
# print version
if input_args.command == 'version':
from label_studio import __version__
print('\nLabel Studio version:', __version__, '\n')
print(json.dumps(versions, indent=4))
# init
elif input_args.command == 'user' or getattr(input_args, 'user', None):
_get_user_info(input_args.username)
return
# init
elif input_args.command == 'init' or getattr(input_args, 'init', None):
_init(input_args, config)
print('')
print('Label Studio has been successfully initialized.')
if input_args.command != 'start':
print('Start the server: label-studio start ' + input_args.project_name)
return
# start with migrations from old projects, '.' project_name means 'label-studio start' without project name
elif input_args.command == 'start' and input_args.project_name != '.':
from label_studio.core.old_ls_migration import migrate_existing_project
from projects.models import Project
sampling_map = {'sequential': Project.SEQUENCE, 'uniform': Project.UNIFORM,
'prediction-score-min': Project.UNCERTAINTY}
if not _project_exists(input_args.project_name):
migrated = False
project_path = pathlib.Path(input_args.project_name)
if project_path.exists():
print('Project directory from previous verion of label-studio found')
print('Start migrating..')
config_path = project_path / 'config.json'
config = _get_config(config_path)
user = _create_user(input_args, config)
label_config_path = project_path / 'config.xml'
project = _create_project(
title=input_args.project_name,
user=user,
label_config=label_config_path,
sampling=sampling_map.get(config.get('sampling', 'sequential'), Project.UNIFORM),
description=config.get('description', ''),
)
migrate_existing_project(project_path, project, config)
migrated = True
print(
Fore.LIGHTYELLOW_EX +
'\n*** WARNING! ***\n'
+ f'Project {input_args.project_name} migrated to Label Studio Database\n'
+ "YOU DON'T NEED THIS FOLDER ANYMORE"
+ '\n****************\n' +
Fore.WHITE
)
if not migrated:
print(
'Project "{project_name}" not found. '
'Did you miss create it first with `label-studio init {project_name}` ?'.format(
project_name=input_args.project_name
)
)
return
# on `start` command, launch browser if --no-browser is not specified and start label studio server
if input_args.command == 'start' or input_args.command is None:
from label_studio.core.utils.common import start_browser
if get_env('USERNAME') and get_env('PASSWORD') or input_args.username:
_create_user(input_args, config)
# ssl not supported from now
cert_file = input_args.cert_file or config.get('cert')
key_file = input_args.key_file or config.get('key')
if cert_file or key_file:
logger.error("Label Studio doesn't support SSL web server with cert and key.\n"
'Use nginx or other servers for it.')
return
# internal port and internal host for server start
internal_host = input_args.internal_host or config.get('internal_host', '0.0.0.0')
internal_port = input_args.port or get_env('PORT') or config.get('port', 8080)
internal_port = int(internal_port)
internal_port = _get_free_port(internal_port, input_args.debug)
# save selected port to global settings
from django.conf import settings
settings.INTERNAL_PORT = str(internal_port)
# browser
url = ('http:https://localhost:' + str(internal_port)) if not host else host
start_browser(url, input_args.no_browser)
_app_run(host=internal_host, port=internal_port)
if __name__ == "__main__":
sys.exit(main())