-
Notifications
You must be signed in to change notification settings - Fork 1.6k
/
mx_sdk_vm.py
1274 lines (1129 loc) · 63.5 KB
/
mx_sdk_vm.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#
# Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# The Universal Permissive License (UPL), Version 1.0
#
# Subject to the condition set forth below, permission is hereby granted to any
# person obtaining a copy of this software, associated documentation and/or
# data (collectively the "Software"), free of charge and under any and all
# copyright rights in the Software, and any and all patent rights owned or
# freely licensable by each licensor hereunder covering either (i) the
# unmodified Software as contributed to or provided by such licensor, or (ii)
# the Larger Works (as defined below), to deal in both
#
# (a) the Software, and
#
# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
# one is included with the Software each a "Larger Work" to which the Software
# is contributed by such licensors),
#
# without restriction, including without limitation the rights to copy, create
# derivative works of, display, perform, and distribute the Software and make,
# use, sell, offer for sale, import, export, have made, and have sold the
# Software and the Larger Work(s), and to sublicense the foregoing rights on
# either these or other terms.
#
# This license is subject to the following condition:
#
# The above copyright notice and either this complete permission notice or at a
# minimum a reference to the UPL must be included in all copies or substantial
# portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
from __future__ import print_function
from abc import ABCMeta
import mx
import mx_javamodules
import mx_subst
import mx_util
import os
import re
import shutil
import tempfile
import textwrap
import types
from os.path import join, exists, isfile, isdir, dirname, relpath
from zipfile import ZipFile, ZIP_DEFLATED
from binascii import b2a_hex # pylint: disable=no-name-in-module
from collections import OrderedDict
from argparse import ArgumentParser
from mx_javamodules import as_java_module, JavaModuleDescriptor
# This can be incremented if a new GraalVM release needs to be built on exactly the same JDK as a previous GraalVM release.
# It will be appended after a dot to the JDK build number an appear in the java.vendor.version.
# This should be brought back down to 1 when the JDK version is updated (feature, interim, patch, or even build, see java.lang.Runtime.Version).
release_build = '1'
_suite = mx.suite('sdk')
_graalvm_components = dict() # By short_name
_graalvm_components_by_name = dict()
_vm_configs = []
_graalvm_hostvm_configs = [
('jvm', [], ['--jvm'], 50),
('jvm-no-truffle-compilation', [], ['--jvm', '--experimental-options', '--engine.Compilation=false'], 29),
('native', [], ['--native'], 100),
('native-no-truffle-compilation', [], ['--native', '--experimental-options', '--engine.Compilation=false'], 39),
('jvm-3-compiler-threads', [], ['--jvm', '--engine.CompilerThreads=3'], 50),
('native-3-compiler-threads', [], ['--native', '--engine.CompilerThreads=3'], 100)
]
_base_jdk = None
class AbstractNativeImageConfig(object, metaclass=ABCMeta):
def __init__(self, destination, jar_distributions, build_args, use_modules=None, links=None, is_polyglot=False, dir_jars=False, home_finder=False, build_time=1, build_args_enterprise=None): # pylint: disable=super-init-not-called
"""
:type destination: str
:type jar_distributions: list[str]
:type build_args: list[str]
:param str | None use_modules: Run (with 'launcher') or run and build image with module support (with 'image').
:type links: list[str] | None
:type is_polyglot: bool
:param bool dir_jars: If true, all jars in the component directory are added to the classpath.
:type home_finder: bool
:type build_time: int
:type build_args_enterprise: list[str] | None
"""
self.destination = mx_subst.path_substitutions.substitute(destination)
self.jar_distributions = jar_distributions
self.build_args = build_args
self.use_modules = use_modules
self.links = [mx_subst.path_substitutions.substitute(link) for link in links] if links else []
self.is_polyglot = is_polyglot
self.dir_jars = dir_jars
self.home_finder = home_finder
self.build_time = build_time
self.build_args_enterprise = build_args_enterprise or []
self.relative_home_paths = {}
self.relative_extracted_lib_paths = {}
assert isinstance(self.jar_distributions, list)
assert isinstance(self.build_args, (list, types.GeneratorType))
assert isinstance(self.build_args_enterprise, list)
def __str__(self):
return self.destination
def __repr__(self):
return str(self)
@staticmethod
def get_add_exports_list(required_exports, custom_target_module_str=None):
add_exports = []
for required in required_exports:
target_modules = required_exports[required]
target_modules_str = custom_target_module_str or ','.join(sorted(target_module.name for target_module in target_modules))
required_module_name, required_package_name = required
add_exports.append('--add-exports=' + required_module_name + '/' + required_package_name + "=" + target_modules_str)
return sorted(add_exports)
def get_add_exports(self, missing_jars):
if self.use_modules is None:
return ''
distributions = self.jar_distributions
distributions_transitive = mx.classpath_entries(distributions)
distributions_transitive_clean = [entry for entry in distributions_transitive if str(entry) not in missing_jars]
required_exports = mx_javamodules.requiredExports(distributions_transitive_clean, base_jdk())
return AbstractNativeImageConfig.get_add_exports_list(required_exports)
def add_relative_home_path(self, language, path):
if language in self.relative_home_paths and self.relative_home_paths[language] != path:
raise Exception('the relative home path of {} is already set to {} and cannot also be set to {} for {}'.format(
language, self.relative_home_paths[language], path, self.destination))
self.relative_home_paths[language] = path
def add_relative_extracted_lib_path(self, name, path):
if name in self.relative_extracted_lib_paths and self.relative_extracted_lib_paths[name] != path:
raise Exception('the relative extracted lib path of {} is already set to {} and cannot also be set to {} for {}'.format(
name, self.relative_extracted_lib_paths[name], path, self.destination))
self.relative_extracted_lib_paths[name] = path
class LauncherConfig(AbstractNativeImageConfig):
def __init__(self, destination, jar_distributions, main_class, build_args, is_main_launcher=True,
default_symlinks=True, is_sdk_launcher=False, custom_launcher_script=None, extra_jvm_args=None,
use_modules=None, main_module=None, link_at_build_time=True, option_vars=None, home_finder=True, **kwargs):
"""
:param str main_class
:param bool is_main_launcher
:param bool default_symlinks
:param bool is_sdk_launcher: Whether it uses org.graalvm.launcher.Launcher
:param str custom_launcher_script: Custom launcher script, to be used when not compiled as a native image
:param list[str] | None extra_jvm_args
:param str main_module: Specifies the main module. Mandatory if use_modules is not None
:param bool link_at_build_time
:param list[str] | None option_vars
"""
super(LauncherConfig, self).__init__(destination, jar_distributions, build_args, use_modules=use_modules, home_finder=home_finder, **kwargs)
self.main_module = main_module
assert self.use_modules is None or self.main_module
self.main_class = main_class
self.link_at_build_time = link_at_build_time
self.is_main_launcher = is_main_launcher
self.default_symlinks = default_symlinks
self.is_sdk_launcher = is_sdk_launcher
self.custom_launcher_script = custom_launcher_script
self.extra_jvm_args = [] if extra_jvm_args is None else extra_jvm_args
self.option_vars = [] if option_vars is None else option_vars
class LanguageLauncherConfig(LauncherConfig):
def __init__(self, destination, jar_distributions, main_class, build_args, language,
is_sdk_launcher=True, **kwargs):
"""
:param str language
"""
super(LanguageLauncherConfig, self).__init__(destination, jar_distributions, main_class, build_args,
is_sdk_launcher=is_sdk_launcher, is_polyglot=False, **kwargs)
self.language = language
# Ensure the language launcher can always find the language home
self.add_relative_home_path(language, relpath('.', dirname(destination)))
class LibraryConfig(AbstractNativeImageConfig):
def __init__(self, destination, jar_distributions, build_args, jvm_library=False, use_modules=None, add_to_module=None, home_finder=False, headers=True, **kwargs):
"""
:param bool jvm_library
:param str add_to_module: the simple name of a module that should be modified to include this native library. It must not be a path or end with `.jmod`
:param bool headers: whether headers produced by the native image build should be placed next to the native image.
"""
super(LibraryConfig, self).__init__(destination, jar_distributions, build_args, use_modules=use_modules, home_finder=home_finder, **kwargs)
self.jvm_library = jvm_library
self.add_to_module = add_to_module
self.headers = headers
class LanguageLibraryConfig(LibraryConfig):
def __init__(self, jar_distributions, build_args, language, main_class=None, is_sdk_launcher=True, launchers=None, option_vars=None, default_vm_args=None, headers=False, set_default_relative_home_path=True, **kwargs):
"""
:param str language
:param str main_class
"""
kwargs.pop('destination', None)
super(LanguageLibraryConfig, self).__init__('lib/<lib:' + language + 'vm>', jar_distributions, build_args, home_finder=True, headers=headers, **kwargs)
if not launchers:
assert not main_class
self.is_sdk_launcher = is_sdk_launcher
self.main_class = main_class
self.language = language
self.default_symlinks = None
self.relative_home_paths = {}
self.launchers = [mx_subst.path_substitutions.substitute(l) for l in launchers] if launchers else []
self.option_vars = [] if option_vars is None else option_vars
self.default_vm_args = [] if default_vm_args is None else default_vm_args
assert all(arg.startswith("--vm.") for arg in self.default_vm_args)
if set_default_relative_home_path:
# Ensure the language launcher can always find the language home
self.add_relative_home_path(language, relpath('.', dirname(self.destination)))
class GraalVmComponent(object):
def __init__(self,
suite,
name,
short_name,
license_files,
third_party_license_files,
jar_distributions=None,
builder_jar_distributions=None,
support_distributions=None,
support_headers_distributions=None,
support_libraries_distributions=None,
dir_name=None,
launcher_configs=None,
library_configs=None,
provided_executables=None,
polyglot_lib_build_args=None,
polyglot_lib_jar_dependencies=None,
polyglot_lib_build_dependencies=None,
has_polyglot_lib_entrypoints=False,
boot_jars=None,
jvmci_parent_jars=None,
jlink=True,
priority=None,
installable=None,
post_install_msg=None,
installable_id=None,
standalone=None,
dependencies=None,
supported=None,
early_adopter=False,
stability=None,
extra_installable_qualifiers=None,
has_relative_home=True,
jvm_configs=None,
extra_native_targets=None):
"""
:param suite mx.Suite: the suite this component belongs to
:type name: str
:param str short_name: a short, unique name for this component
:param str | None | False dir_name: the directory name in which this component lives. If `None`, the `short_name` is used. If `False`, files are copied to the root-dir for the component type.
:param installable: Produce a distribution installable via `gu`
:param post_install_msg: Post-installation message to be printed
:param list[str] dependencies: a list of component names
:param list[str | (str, str)] provided_executables: executables to be placed in the appropriate `bin` directory.
In the list, strings represent a path inside the component (e.g., inside a support distribution).
Tuples `(dist, exec)` represent an executable to be copied found in `dist`, at path `exec` (the same basename will be used).
:param jvm_configs: list of dicts that describe changes to the `lib/jvm.cfg` file. Example:
{
'configs': ['-truffle KNOWN'],
'priority': -1, # 0 is invalid; < 0 prepends to the default configs; > 0 appends
}
:param extra_native_targets: list of str, enables extra targets in multi-target projects.
:type license_files: list[str]
:type third_party_license_files: list[str]
:type polyglot_lib_build_args: list[str]
:type polyglot_lib_jar_dependencies: list[str]
:type polyglot_lib_build_dependencies: list[str]
:type has_polyglot_lib_entrypoints: bool
:type boot_jars: list[str]
:type jvmci_parent_jars: list[str]
:type launcher_configs: list[LauncherConfig]
:type library_configs: list[LibraryConfig]
:type jar_distributions: list[str]
:type builder_jar_distributions: list[str]
:type support_distributions: list[str]
:type support_headers_distributions: list[str]
:type support_libraries_distributions: list[str]
:param int priority: priority with a higher value means higher priority
:type installable: bool
:type installable_id: str
:type standalone: bool
:type post_install_msg: str
:type stability: str | None
:type extra_installable_qualifiers: list[str] | None
:type has_relative_home: bool
:type jvm_configs: list[dict] or None
"""
if dependencies is None:
mx.logv('Component {} does not specify dependencies'.format(name))
self.suite = suite
self.name = name
self.short_name = short_name
self.dir_name = dir_name if dir_name is not None else short_name
self.license_files = license_files
self.third_party_license_files = third_party_license_files
self.dependency_names = dependencies or []
self.provided_executables = provided_executables or []
self.polyglot_lib_build_args = polyglot_lib_build_args or []
self.polyglot_lib_jar_dependencies = polyglot_lib_jar_dependencies or []
self.polyglot_lib_build_dependencies = polyglot_lib_build_dependencies or []
self.has_polyglot_lib_entrypoints = has_polyglot_lib_entrypoints
self.boot_jars = boot_jars or []
self.jvmci_parent_jars = jvmci_parent_jars or []
self.jar_distributions = jar_distributions or []
self.builder_jar_distributions = builder_jar_distributions or []
self.support_distributions = support_distributions or []
self.support_headers_distributions = support_headers_distributions or []
self.support_libraries_distributions = support_libraries_distributions or []
self.priority = priority or 0
self.launcher_configs = launcher_configs or []
self.library_configs = library_configs or []
if installable is None:
installable = isinstance(self, GraalVmLanguage)
self.installable = installable
if standalone is None:
standalone = installable
self.standalone = standalone
self.post_install_msg = post_install_msg
self.installable_id = installable_id or self.dir_name
self.extra_installable_qualifiers = extra_installable_qualifiers or []
self.has_relative_home = has_relative_home
self.jvm_configs = jvm_configs or []
self.extra_native_targets = extra_native_targets
if supported is not None or early_adopter:
if stability is not None:
raise mx.abort("{}: Cannot use `stability` attribute in combination with deprecated `supported` and `early_adopter` attributes".format(name))
mx.warn("{}: `supported` and `early_adopter` attributes are deprecated, please use `stability`".format(name))
if stability is None:
if supported:
if early_adopter:
stability = "earlyadopter"
else:
stability = "supported"
else:
if early_adopter:
stability = "experimental-earlyadopter"
else:
stability = "experimental"
self.stability = stability
self.jlink = jlink
assert isinstance(self.jar_distributions, list)
assert isinstance(self.builder_jar_distributions, list)
assert isinstance(self.support_distributions, list)
assert isinstance(self.support_headers_distributions, list)
assert isinstance(self.support_libraries_distributions, list)
assert isinstance(self.license_files, list)
assert isinstance(self.third_party_license_files, list)
assert isinstance(self.provided_executables, list)
assert isinstance(self.polyglot_lib_build_args, list)
assert isinstance(self.polyglot_lib_jar_dependencies, list)
assert isinstance(self.polyglot_lib_build_dependencies, list)
assert isinstance(self.boot_jars, list)
assert isinstance(self.jvmci_parent_jars, list)
assert isinstance(self.launcher_configs, list)
assert isinstance(self.library_configs, list)
assert isinstance(self.extra_installable_qualifiers, list)
assert not any(cp_arg in self.polyglot_lib_build_args for cp_arg in ('-cp', '-classpath')), "the '{}' component passes a classpath argument to libpolylgot: '{}'. Use `polyglot_lib_jar_dependencies` instead".format(self.name, ' '.join(self.polyglot_lib_build_args))
def __str__(self):
return "{} ({})".format(self.name, self.dir_name)
def direct_dependencies(self):
try:
return [graalvm_component_by_name(name) for name in self.dependency_names]
except Exception as e:
raise Exception("{} (required by {})".format(e, self.name))
class GraalVmTruffleComponent(GraalVmComponent):
def __init__(self, suite, name, short_name, license_files, third_party_license_files, truffle_jars,
include_in_polyglot=None, standalone_dir_name=None, standalone_dir_name_enterprise=None,
standalone_dependencies=None, standalone_dependencies_enterprise=None, **kwargs):
"""
:param list[str] truffle_jars: JAR distributions that should be on the classpath for the language implementation.
:param bool include_in_polyglot: whether this component is included in `--language:all` or `--tool:all` and should be part of polyglot images (deprecated).
:param str standalone_dir_name: name for the standalone archive and directory inside
:param str standalone_dir_name_enterprise: like `standalone_dir_name`, but for the EE standalone. Defaults to `standalone_dir_name` if not set.
:param dict[str, (str, list[str])] standalone_dependencies: dict of dependent components to include in the CE standalone in the form {component name: (relative path, excluded_paths)}.
:param dict[str, (str, list[str])] standalone_dependencies_enterprise: like `standalone_dependencies`, but for the EE standalone. Defaults to `standalone_dependencies` if not set.
"""
super(GraalVmTruffleComponent, self).__init__(suite, name, short_name, license_files, third_party_license_files,
jar_distributions=truffle_jars, **kwargs)
if include_in_polyglot is not None:
mx.warn('"include_in_polyglot" is deprecated. Please drop all uses.')
self.standalone_dir_name = standalone_dir_name or '{}-<version>-<graalvm_os>-<arch>'.format(self.dir_name)
self.standalone_dir_name_enterprise = standalone_dir_name_enterprise or self.standalone_dir_name
self.standalone_dependencies = standalone_dependencies or {}
self.standalone_dependencies_enterprise = standalone_dependencies_enterprise or self.standalone_dependencies
assert isinstance(self.standalone_dependencies, dict)
assert isinstance(self.standalone_dependencies_enterprise, dict)
class GraalVmLanguage(GraalVmTruffleComponent):
"""
:param support_distributions: distributions the contents of which is added to the language's home directory.
The contents of support distributions setting the `fileListPurpose` attribute to `native-image-resources` will end up as file list in the `native-image-resources.filelist` file in this language's home directory.
As a part of a native image build that includes this language, the files in the merged file list will be copied as resources to a directory named `resources` next to the produced image.
"""
pass
class GraalVmTool(GraalVmTruffleComponent):
def __init__(self, suite, name, short_name, license_files, third_party_license_files, truffle_jars,
include_by_default=False, **kwargs):
"""
:type include_by_default: bool
"""
super(GraalVmTool, self).__init__(suite, name, short_name, license_files, third_party_license_files,
truffle_jars, **kwargs)
self.include_by_default = include_by_default
class GraalVMSvmMacro(GraalVmComponent):
pass
class GraalVmJdkComponent(GraalVmComponent):
pass
class GraalVmJreComponent(GraalVmComponent):
pass
class GraalVmTruffleLibrary(GraalVmJreComponent):
pass
class GraalVmJvmciComponent(GraalVmJreComponent):
def __init__(self, suite, name, short_name, license_files, third_party_license_files, jvmci_jars, **kwargs):
"""
:type jvmci_jars: list[str]
"""
super(GraalVmJvmciComponent, self).__init__(suite, name, short_name, license_files, third_party_license_files,
**kwargs)
self.jvmci_jars = jvmci_jars or []
assert isinstance(self.jvmci_jars, list)
def register_graalvm_component(component):
"""
:type component: GraalVmComponent
"""
def _log_ignored_component(kept, ignored):
"""
:type kept: GraalVmComponent
:type ignored: GraalVmComponent
"""
mx.logv('Suites \'{}\' and \'{}\' are registering a component with the same short name (\'{}\'), with priority \'{}\' and \'{}\' respectively.'.format(kept.suite.name, ignored.suite.name, kept.short_name, kept.priority, ignored.priority))
mx.logv('Ignoring the one from suite \'{}\'.'.format(ignored.suite.name))
assert ',' not in component.short_name, component.short_name
assert ',' not in component.name, component.name
_prev = _graalvm_components.get(component.short_name, None)
if _prev:
if _prev.priority == component.priority:
mx.abort('Suites \'{}\' and \'{}\' are registering a component with the same short name (\'{}\') and priority (\'{}\')'.format(_prev.suite.name, component.suite.name, _prev.short_name, _prev.priority))
elif _prev.priority < component.priority:
_graalvm_components[component.short_name] = component
del _graalvm_components_by_name[_prev.name]
_graalvm_components_by_name[component.name] = component
_log_ignored_component(component, _prev)
else:
_log_ignored_component(_prev, component)
else:
_graalvm_components[component.short_name] = component
_graalvm_components_by_name[component.name] = component
def graalvm_component_by_name(name, fatalIfMissing=True):
"""
:rtype: GraalVmComponent
"""
if name in _graalvm_components:
return _graalvm_components[name]
elif name in _graalvm_components_by_name:
return _graalvm_components_by_name[name]
else:
if fatalIfMissing:
raise Exception("Unknown component: {}".format(name))
return None
def graalvm_components(opt_limit_to_suite=False):
"""
:rtype: list[GraalVmComponent]
"""
if opt_limit_to_suite and mx.get_opts().specific_suites:
return [c for c in _graalvm_components.values() if c.suite.name in mx.get_opts().specific_suites]
else:
return list(_graalvm_components.values())
def graalvm_home(fatalIfMissing=False):
import mx_sdk_vm_impl
return mx_sdk_vm_impl.graalvm_home(fatalIfMissing=fatalIfMissing)
def add_graalvm_hostvm_config(name, java_args=None, launcher_args=None, priority=0):
"""
:type name: str
:type java_args: list[str] | None
:type launcher_args: list[str] | None
:type priority: int
"""
_graalvm_hostvm_configs.append((name, java_args, launcher_args, priority))
def register_vm_config(config_name, components, suite, dist_name=None, env_file=None):
"""
:type config_name: str
:type components: list[str]
:type suite: mx.Suite
:type dist_name: str
:type env_file: str or None
"""
assert config_name is not None
assert components is not None and len(components)
_dist_name = dist_name or config_name
_vm_configs.append((_dist_name, config_name, components, suite, env_file))
def get_graalvm_hostvm_configs():
return _graalvm_hostvm_configs
def base_jdk():
global _base_jdk
if _base_jdk is None:
_base_jdk = mx.get_jdk(tag='default')
return _base_jdk
def base_jdk_version():
return base_jdk().javaCompliance.value
def _probe_jvmci_info(jdk, attribute_name):
if not hasattr(jdk, '.enables_jvmci_by_default'):
out = mx.LinesOutputCapture()
sink = lambda x: x
mx.run([jdk.java, '-XX:+UnlockExperimentalVMOptions', '-XX:+PrintFlagsFinal', '-version'], out=out, err=sink)
enableJVMCI = False
enableJVMCIProduct = False
jvmciThreadsPerNativeLibraryRuntime = None
for line in out.lines:
if 'EnableJVMCI' in line and 'true' in line:
enableJVMCI = True
if 'EnableJVMCIProduct' in line:
enableJVMCIProduct = True
if 'JVMCIThreadsPerNativeLibraryRuntime' in line:
m = re.search(r'JVMCIThreadsPerNativeLibraryRuntime *= *(\d+)', line)
if not m:
mx.abort(f'Could not extract value of JVMCIThreadsPerNativeLibraryRuntime from "{line}"')
jvmciThreadsPerNativeLibraryRuntime = int(m.group(1))
setattr(jdk, '.enables_jvmci_by_default', enableJVMCI)
setattr(jdk, '.supports_enablejvmciproduct', enableJVMCIProduct)
setattr(jdk, '.jvmciThreadsPerNativeLibraryRuntime', jvmciThreadsPerNativeLibraryRuntime)
return getattr(jdk, attribute_name)
def jdk_enables_jvmci_by_default(jdk):
"""
Gets the default value for the EnableJVMCI VM option in `jdk`.
"""
return _probe_jvmci_info(jdk, '.enables_jvmci_by_default')
def jdk_supports_enablejvmciproduct(jdk):
"""
Determines if the jdk supports flag -XX:+EnableJVMCIProduct which isn't the case
for some OpenJDK 11u distros.
"""
return _probe_jvmci_info(jdk, '.supports_enablejvmciproduct')
def get_JVMCIThreadsPerNativeLibraryRuntime(jdk):
"""
Gets the value of the flag -XX:JVMCIThreadsPerNativeLibraryRuntime.
Returns None if this flag is not supported in `jdk` otherwise returns the default value as an int
"""
return _probe_jvmci_info(jdk, '.jvmciThreadsPerNativeLibraryRuntime')
def _probe_jlink_info(jdk, attribute_name):
"""
Determines if the jlink executable in `jdk` supports various options such
as those added by JDK-8232080 and JDK-8237467.
"""
if not hasattr(jdk, '.supports_JDK_8232080'):
output = mx.OutputCapture()
jlink_exe = jdk.javac.replace('javac', 'jlink')
mx.run([jlink_exe, '--list-plugins'], out=output)
setattr(jdk, '.supports_JDK_8232080', '--add-options=' in output.data or '--add-options ' in output.data)
setattr(jdk, '.supports_save_jlink_argfiles', '--save-jlink-argfiles=' in output.data or '--save-jlink-argfiles ' in output.data)
setattr(jdk, '.supports_copy_files', '--copy-files=' in output.data or '--copy-files ' in output.data)
return getattr(jdk, attribute_name)
def jlink_supports_8232080(jdk):
"""
Determines if the jlink executable in `jdk` supports ``--add-options`` and
``--vendor-[bug-url|vm-bug-url|version]`` added by JDK-8232080.
"""
return _probe_jlink_info(jdk, '.supports_JDK_8232080')
def jlink_has_save_jlink_argfiles(jdk):
"""
Determines if the jlink executable in `jdk` supports ``--save-jlink-argfiles``.
"""
return _probe_jlink_info(jdk, '.supports_save_jlink_argfiles')
def _jdk_omits_warning_for_jlink_set_ThreadPriorityPolicy(jdk): # pylint: disable=invalid-name
"""
Determines if the `jdk` suppresses a warning about ThreadPriorityPolicy when it
is non-zero if the value is set from the jimage.
https://bugs.openjdk.java.net/browse/JDK-8235908.
"""
if not hasattr(jdk, '.omits_ThreadPriorityPolicy_warning'):
out = mx.OutputCapture()
sink = lambda x: x
tmpdir = tempfile.mkdtemp(prefix='jdk_omits_warning_for_jlink_set_ThreadPriorityPolicy')
jlink_exe = jdk.javac.replace('javac', 'jlink')
mx.run([jlink_exe, '--add-options=-XX:ThreadPriorityPolicy=1', '--output=' + join(tmpdir, 'jdk'), '--add-modules=java.base'])
mx.run([mx.exe_suffix(join(tmpdir, 'jdk', 'bin', 'java')), '-version'], out=sink, err=out)
shutil.rmtree(tmpdir)
setattr(jdk, '.omits_ThreadPriorityPolicy_warning', '-XX:ThreadPriorityPolicy=1 may require system level permission' not in out.data)
return getattr(jdk, '.omits_ThreadPriorityPolicy_warning')
def _read_java_base_hashes(jdk):
"""
Read the hashes stored in the ``java.base`` module of `jdk`.
"""
hashes = {}
out = mx.LinesOutputCapture()
mx.run([jdk.exe_path('jmod'), 'describe', join(jdk.home, 'jmods', 'java.base.jmod')], out=out)
lines = out.lines
for line in lines:
if line.startswith('hashes'):
parts = line.split()
assert len(parts) == 4, 'expected hashes line to have 4 fields, got {} fields: {}'.format(len(parts), line)
_, module_name, algorithm, hash_value = parts
hashes[module_name] = (algorithm, hash_value)
return hashes
def _patch_default_security_policy(build_dir, jmods_dir, dst_jdk_dir):
"""
Edits the default security policy in the ``lib/security/default.policy`` entry of
`jmods_dir`/java.base.jmod to grant all permissions to anything loaded from
``${java.home}/languages/-`` as well as the following modules:
com.oracle.graal.graal_enterprise
org.graalvm.truffle
org.graalvm.sdk
Extra permissions are also granted to:
org.graalvm.locator
:returns str: path to the patched version of ``java.base.jmod``
"""
graalvm_policy = textwrap.dedent("""
grant codeBase "jrt:/com.oracle.graal.graal_enterprise" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.truffle" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.sdk" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.truffle.runtime" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.truffle.compiler" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.nativebridge" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.jniutils" {
permission java.security.AllPermission;
};
grant codeBase "jrt:/org.graalvm.locator" {
permission java.io.FilePermission "<<ALL FILES>>", "read";
permission java.util.PropertyPermission "*", "read,write";
permission java.lang.RuntimePermission "createClassLoader";
permission java.lang.RuntimePermission "getClassLoader";
permission java.lang.RuntimePermission "getenv.*";
};
grant codeBase "file:${java.home}/languages/-" {
permission java.security.AllPermission;
};
""")
graalvm_policy_utf8 = graalvm_policy.encode('utf-8')
policy_entry = 'lib/security/default.policy'
patched_java_base = join(build_dir, 'java.base.jmod')
patched_java_base_source = join(build_dir, 'java.base.jmod.source')
dst_patched_java_base = join(dst_jdk_dir, 'jmods', 'java.base.jmod')
def open_jmods(to_read, to_write=None):
in_fp = open(to_read, 'rb')
out_fp = None if to_write is None else open(to_write, 'wb')
jmod_header = in_fp.read(4)
if len(jmod_header) != 4 or jmod_header != b'JM\x01\x00':
raise mx.abort("Unexpected jmod header in {}: {}".format(to_read, b2a_hex(jmod_header).decode('ascii')))
if out_fp:
out_fp.write(jmod_header)
return in_fp, out_fp
return in_fp
def needs_patching(java_base_jmod):
# Return True if a different JDK was used for the current patched java.base
if not exists(patched_java_base_source):
return True
with open(patched_java_base_source) as fp:
source = fp.read()
if source != jmods_dir:
return True
if exists(java_base_jmod):
with open_jmods(java_base_jmod) as fp:
with ZipFile(fp, 'r') as zf:
policy = zf.read(policy_entry).decode('utf-8')
return graalvm_policy not in policy
return True
if not needs_patching(dst_patched_java_base):
shutil.copy(dst_patched_java_base, patched_java_base)
return patched_java_base
# Record the jmods directory of the original java.base module
with mx.open(patched_java_base_source, 'w') as fp:
fp.write(jmods_dir)
fps = open_jmods(join(jmods_dir, 'java.base.jmod'), patched_java_base)
with fps[0] as src_f, fps[1] as dst_f:
policy_found = False
with ZipFile(src_f, 'r') as src_zip, ZipFile(dst_f, 'w', src_zip.compression) as dst_zip:
for i in src_zip.infolist():
if i.filename[-1] == '/':
continue
src_member = src_zip.read(i)
if i.filename == policy_entry:
policy_found = True
if graalvm_policy_utf8 not in src_member:
src_member += graalvm_policy_utf8
dst_zip.writestr(i, src_member)
if not policy_found:
raise mx.abort("Couldn't find `{}` in {}".format(policy_entry, join(jmods_dir, 'java.base.jmod')))
return patched_java_base
def _get_image_root_modules(root_module_names, module_names, jdk_module_names, use_upgrade_module_path):
"""
Gets the argument for the jlink ``--add-modules`` flag (i.e. the roots of the module graph to link into the image)
:return frozenset: the set of root module names
"""
all_names = frozenset(jdk_module_names).union(module_names)
if root_module_names is not None:
missing = frozenset(root_module_names).difference(jdk_module_names).difference(module_names)
if missing:
mx.abort('Invalid module(s): {}.\nAvailable modules: {}'.format(','.join(missing), ','.join(sorted(all_names))))
if use_upgrade_module_path:
# Modules that will be on --upgrade-module-path are excluded from the image
return frozenset((n for n in root_module_names if n not in module_names))
else:
return frozenset(root_module_names)
else:
if use_upgrade_module_path:
# Modules that will be on --upgrade-module-path are excluded from the image
return frozenset((n for n in jdk_module_names if n not in module_names))
else:
return all_names
def _get_image_vm_options(jdk, use_upgrade_module_path, modules, synthetic_modules, default_to_jvmci=False):
"""
Gets the argument for the jlink ``--add-options`` flag.
:param bool | str default_to_jvmci: default to using JVMCI as JIT, without looking at the included modules
When set to 'lib' ensure the JVMCI compiler library is used.
:return list: the list of VM options to cook into the image
"""
vm_options = []
if jlink_supports_8232080(jdk):
if mx.get_env('CONTINUOUS_INTEGRATION', None) == 'true':
is_gate = mx.get_env('BUILD_TARGET', None) == 'gate'
is_bench = 'bench-' in mx.get_env('BUILD_NAME', '')
if is_gate or is_bench:
# For gate and benchmark jobs, we want to know about each compilation failure
# but only exit the VM on systemic compilation failure for gate jobs.
vm_options.append('-Djdk.graal.CompilationFailureAction=Diagnose')
mx.log('Adding -Djdk.graal.CompilationFailureAction=Diagnose VM option to image')
if is_gate:
mx.log('Adding -Djdk.graal.SystemicCompilationFailureRate=-1 VM option to image')
vm_options.append('-Djdk.graal.SystemicCompilationFailureRate=-1')
if use_upgrade_module_path or _jdk_omits_warning_for_jlink_set_ThreadPriorityPolicy(jdk):
vm_options.append('-XX:ThreadPriorityPolicy=1')
else:
mx.logv('[Creating JDK without -XX:ThreadPriorityPolicy=1]')
if jdk_supports_enablejvmciproduct(jdk):
non_synthetic_modules = [m.name for m in modules if m not in synthetic_modules]
if default_to_jvmci or 'jdk.graal.compiler' in non_synthetic_modules:
threads = get_JVMCIThreadsPerNativeLibraryRuntime(jdk)
vm_options.extend(['-XX:+UnlockExperimentalVMOptions', '-XX:+EnableJVMCIProduct'])
if threads is not None and threads != 1:
vm_options.append('-XX:JVMCIThreadsPerNativeLibraryRuntime=1')
if default_to_jvmci == 'lib':
vm_options.append('-XX:+UseJVMCINativeLibrary')
vm_options.extend(['-XX:-UnlockExperimentalVMOptions'])
else:
# Don't default to using JVMCI as JIT unless Graal is being updated in the image.
# This avoids unexpected issues with using the out-of-date Graal compiler in
# the JDK itself.
vm_options.extend(['-XX:+UnlockExperimentalVMOptions', '-XX:+EnableJVMCIProduct', '-XX:-UseJVMCICompiler', '-XX:-UnlockExperimentalVMOptions'])
else:
mx.logv('[Creating JDK without -XX:+EnableJVMCIProduct]')
if modules and use_upgrade_module_path:
vm_options.append('--upgrade-module-path=' + os.pathsep.join((synthetic_modules.get(m, m.jarpath) for m in modules)))
elif use_upgrade_module_path:
mx.abort('Cannot create an image with an --upgrade-module-path setting since jlink does not support the --add-options flag')
return vm_options
def _copy_src_zip(from_jdk, to_jdk, extra_modules, extra_modules_predicate):
"""
Creates `to_jdk`/lib/src.zip from the entries in `from_jdk`/src.zip except for those
in the sources of `extra_modules` plus the sources of the modules in `extra_modules`
unless `extra_modules_predicate` returns False.
"""
from_src_zip = join(from_jdk, 'lib', 'src.zip')
to_src_zip = join(to_jdk, 'lib', 'src.zip')
with ZipFile(to_src_zip, 'w', compression=ZIP_DEFLATED, allowZip64=True) as out_zf:
mx.logv('[Creating ' + to_src_zip + ']')
if isfile(from_src_zip):
extra_module_names = frozenset((jmd.name for jmd in extra_modules))
with ZipFile(from_src_zip, 'r') as in_zf:
for name in in_zf.namelist():
module_name = name.split('/', 1)[0]
if not name.endswith('/') and module_name not in extra_module_names:
out_zf.writestr(name, in_zf.read(name))
else:
mx.warn("'{}' does not exist or is not a file".format(from_src_zip))
for jmd in extra_modules:
jmd_src_zip = jmd.jarpath[0:-len('.jar')] + '.src.zip'
if isfile(jmd_src_zip) and extra_modules_predicate(jmd):
mx.logv('[Extracting ' + jmd_src_zip + ']')
with ZipFile(jmd_src_zip, 'r') as in_zf:
for name in in_zf.namelist():
if not name.endswith('/'):
if 'module-info' in name:
print('2. writing', name)
out_zf.writestr(jmd.name + '/' + name, in_zf.read(name))
# Add module-info.java to sources
out_zf.writestr(jmd.name + '/module-info.java', jmd.as_module_info(extras_as_comments=False))
def _vm_options_match(vm_options, vm_options_path):
"""
Determines if `vm_options` matches the value persisted in `vm_options_path`.
"""
if not exists(vm_options_path):
return False
with open(vm_options_path) as fp:
previous_vm_options = fp.read().strip()
current_vm_options = os.linesep.join(vm_options)
return previous_vm_options == current_vm_options
def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, ignore_dists,
root_module_names=None,
missing_export_target_action='none',
with_source=lambda x: True,
vendor_info=None,
dedup_legal_notices=True,
use_upgrade_module_path=False,
default_to_jvmci=False):
"""
Uses jlink from `jdk` to create a new JDK image in `dst_jdk_dir` with `module_dists` and
their dependencies added to the JDK image, replacing any existing modules of the same name.
:param JDKConfig jdk: source JDK
:param str dst_jdk_dir: path to use for the jlink --output option
:param list module_dists: list of distributions defining modules
:param list ignore_dists: list of distributions that should be ignored for missing_export_target_action
:param list root_module_names: list of strings naming the module root set for the new JDK image.
The named modules must either be in `module_dists` or in `jdk`. If None, then
the root set will be all the modules in ``module_dists` and `jdk`.
:param str missing_export_target_action: the action to perform for a qualified export target that
is not present in `module_dists` and does not have a hash stored in java.base.
The choices are:
"create" - an empty module is created
"error" - raise an error
"warn" - raise a warning
"none" - do nothing
:param lambda with_source: returns True if the sources of a module distribution must be included in the new JDK
:param dict vendor_info: values for the jlink vendor options added by JDK-8232080
:param bool use_upgrade_module_path: if True, then instead of linking `module_dists` into the image, resolve
them via --upgrade-module-path at image runtime
:param bool | str default_to_jvmci: default to using JVMCI as JIT, without looking at the included modules.
When set to 'lib' ensure the JVMCI compiler library is used.
:return bool: False if use_upgrade_module_path == True and the existing image is up to date otherwise True
"""
assert callable(with_source)
import mx_sdk_vm_impl
missing_export_target_action = missing_export_target_action or mx_sdk_vm_impl.default_jlink_missing_export_action()
allowed_actions = ('create', 'error', 'warn', 'none')
assert missing_export_target_action in allowed_actions, f"Unknown missing export action. Expected: {allowed_actions}. Got: {missing_export_target_action}"
if jdk.javaCompliance < '9':
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' with jlink since it is not JDK 9 or later')
exploded_java_base_module = join(jdk.home, 'modules', 'java.base')
if exists(exploded_java_base_module):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since it appears to be a developer build with exploded modules')
jimage = join(jdk.home, 'lib', 'modules')
jmods_dir = join(jdk.home, 'jmods')
if not isfile(jimage):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jimage + ' is missing or is not an ordinary file')
if not isdir(jmods_dir):
mx.abort('Cannot derive a new JDK from ' + jdk.home + ' since ' + jmods_dir + ' is missing or is not a directory')
# Exclude jdk.aot due to GR-10545 and JDK-8255616
jdk_modules = {jmd.name: jmd for jmd in jdk.get_modules() if jmd.name != 'jdk.aot'}
modules = [as_java_module(dist, jdk) for dist in module_dists]
module_names = frozenset((m.name for m in modules))
all_module_names = frozenset(list(jdk_modules.keys())) | module_names
# Read hashes stored in java.base (the only module in the JDK where hashes are stored)
hashes = _read_java_base_hashes(jdk)
build_dir = mx_util.ensure_dir_exists(join(dst_jdk_dir + ".build"))
# Directory under dst_jdk_dir for artifacts related to use_upgrade_module_path
upgrade_dir = join(dst_jdk_dir, 'upgrade_modules_support')
# Map from JavaModuleDescriptors to post-jlink jar location.
synthetic_modules = OrderedDict()
try:
if missing_export_target_action != 'none':
ignore_module_names = set(mx_javamodules.get_module_name(mx.dependency(ignore_dist)) for ignore_dist in ignore_dists)
# Synthesize modules for targets of qualified exports that are not present in `modules`.
# Without this, runtime module resolution will fail due to missing modules.
target_requires = {}
for jmd in modules:
for targets in jmd.exports.values():
for target in targets:
if target not in all_module_names and target not in ignore_module_names and target not in hashes:
target_requires.setdefault(target, set()).add(jmd.name)
if target_requires:
if missing_export_target_action in ('error', 'warn'):
mx.abort_or_warn('Target(s) of qualified exports cannot be resolved:\n* ' + '\n* '.join(f"{k} required by {v}" for k, v in target_requires.items()), missing_export_target_action == 'abort')
assert missing_export_target_action in ('create', 'warn'), 'invalid value for missing_export_target_action: ' + missing_export_target_action
for name, requires in sorted(target_requires.items()):
module_jar = join(build_dir, name + '.jar')
jmd = JavaModuleDescriptor(name, {}, requires={module: [] for module in requires}, uses=set(), provides={}, jarpath=module_jar)
module_build_dir = mx_util.ensure_dir_exists(join(build_dir, name))
module_info = jmd.as_module_info()
module_info_java = join(module_build_dir, 'module-info.java')
module_info_class = join(module_build_dir, 'module-info.class')
dst_module_jar = join(upgrade_dir, name + '.jar')
synthetic_modules[jmd] = dst_module_jar
if use_upgrade_module_path and exists(dst_module_jar):
with ZipFile(dst_module_jar, 'r') as zf:
previous_module_info = zf.read('module-info.java').decode()
if previous_module_info == module_info:
mx.logv('[Reusing synthetic module {}]'.format(name))
os.rename(dst_module_jar, module_jar)
continue
mx.logv('[Rebuilding synthetic module {} as module descriptor changed]'.format(name))
with open(module_info_java, 'w') as fp:
fp.write(module_info)
mx.run([jdk.javac, '-d', module_build_dir,
'--limit-modules=java.base,' + ','.join(jmd.requires.keys()),
'--module-path=' + os.pathsep.join((m.jarpath for m in modules)),
module_info_java])
with ZipFile(module_jar, 'w') as zf: