Refactor build scripts

.gitignore

@@ -21,6 +21,9 @@ python-package/lets_plot/package_data
 __pycache__/
 /python-package/*.ipynb
 
+# Local properties file
+local.properties
+# Old properties file
 build_settings.yml
 
 # Autogenerated folder with yarn.lock file

build.gradle

@@ -10,7 +10,7 @@ plugins {
  id("io.github.gradle-nexus.publish-plugin") version "1.3.0"
 }
 
-import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform
+import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform as platform
 import org.yaml.snakeyaml.Yaml
 
 project.ext.letsPlotTaskGroup = 'lets-plot'
@@ -31,7 +31,7 @@ allprojects {
  }
 
  project.ext.jfx_platform = { -> //getJfxPlatform()
- OperatingSystem os = org.gradle.nativeplatform.platform.internal.DefaultNativePlatform.currentOperatingSystem
+ def os = platform.currentOperatingSystem
  if (os.isWindows()) {
  return "win"
  } else if (os.isLinux()) {
@@ -51,77 +51,77 @@ allprojects {
 }
 
 // Read build settings from commandline parameters (for build_release.py script):
-def readSettingsFromParameters() {
- def settings = [
- build_python_extension: false,
- enable_python_package : false,
- python : [bin_path: "", include_path: ""],
- pypi : [test: [username: null, password: null],
- prod: [username: null, password: null]]
- ]
- if (project.hasProperty("build_python_extension")) {
- settings.build_python_extension = project.getProperty("build_python_extension")
- assert settings.build_python_extension != null
- }
+def readPropertiesFromParameters() {
+ def properties = new Properties()
  if (project.hasProperty("enable_python_package")) {
- settings.enable_python_package = project.getProperty("enable_python_package")
- assert settings.enable_python_package != null
+ properties."enable_python_package" = project.getProperty("enable_python_package").toBoolean()
+ }
+ if (properties.enable_python_package) {
+ properties."python.bin_path" = project.getProperty("python.bin_path")
+ properties."python.include_path" = project.getProperty("python.include_path")
  }
- if (settings.build_python_extension) {
- settings.python.bin_path = project.getProperty("python_bin_path")
- settings.python.include_path = project.getProperty("python_include_path")
- assert !settings.python.bin_path.isEmpty()
- assert !settings.python.include_path.isEmpty()
+ def os = platform.getCurrentOperatingSystem()
+ if (!os.windows) {
+ properties."architecture" = project.getProperty("architecture")
  }
- return settings
+ return properties
 }
 
-// Read build settings from build_settings.yml:
-def readSettingsFromYaml() {
- def build_settings_file = new File(rootDir, "build_settings.yml")
- if (!build_settings_file.canRead()) {
- throw new GradleException("Couldn't read build_settings.yml")
- }
- def settings = new Yaml().load(build_settings_file.newInputStream())
- if (settings.build_python_extension) {
- assert settings.python.include_path != null
+// Read build settings from local.properties:
+def readPropertiesFromFile() {
+ def os = platform.getCurrentOperatingSystem()
+ def properties = new Properties()
+ def localPropsFileName = "local.properties"
+
+ if (project.file(localPropsFileName).exists()) {
+ properties.load(project.file(localPropsFileName).newInputStream())
+ } else {
+ throw new FileNotFoundException("${localPropsFileName} file not found!\n" +
+ "Check ${localPropsFileName}_template file for the template.")
  }
- if (settings.enable_python_package) {
- assert settings.build_python_extension
- assert settings.python.bin_path != null
+ properties."enable_python_package" = properties."enable_python_package".toBoolean()
+
+ if (!os.windows) {
+ assert properties."architecture" != null // Windows only 64bit version can be built, so the arch parameter is not needed and may not be set.
  }
- return settings
-}
 
-// Get Python's arch to define Kotlin targets:
-def getPythonArch() {
- def python_bin_path = project.buildSettings.python.bin_path
- def getArchOutput = new ByteArrayOutputStream()
- exec {
- commandLine "${python_bin_path}/python",
- "-c",
- "import platform; print(platform.machine())"
- standardOutput = getArchOutput
+ if (properties."enable_python_package") {
+
+ def pythonBinPath = properties."python.bin_path"
+ def pythonIncludePath = properties."python.include_path"
+
+ assert pythonBinPath != null
+ assert pythonIncludePath != null
+
+ if (!os.windows) {
+ def getArchOutput = new ByteArrayOutputStream()
+ exec {
+ commandLine "${pythonBinPath}/python",
+ "-c",
+ "import platform; print(platform.machine())"
+ standardOutput = getArchOutput
+ }
+ def currentPythonArch = getArchOutput.toString().trim()
+
+ if (currentPythonArch != properties."architecture") {
+ throw new IllegalArgumentException("Project and Python architectures don't match!\n" +
+ " - Value, from your '${localPropsFileName}' file: ${properties.architecture}\n" +
+ " - Your Python architecture: ${currentPythonArch}\n" +
+ "Check your '${localPropsFileName}' file.")
+ }
+ }
  }
- return getArchOutput.toString().trim()
+ return properties
 }
 
-
 // For build_release.py settings will be read from commandline parameters.
-// If not, settings will be read from build_settings.yml.
+// In other cases, settings will be read from local.properties.
 if (project.hasProperty("build_release")) {
- project.ext.buildSettings = readSettingsFromParameters()
+ ext.localProps = readPropertiesFromParameters()
 } else {
- project.ext.buildSettings = readSettingsFromYaml() as LinkedHashMap
+ ext.localProps = readPropertiesFromFile()
 }
 
-// Read Python arch from commandline parameter if used:
-if (project.hasProperty("build_arch")) {
- project.ext.pythonArch = project.getProperty("build_arch")
- assert !project.pythonArch.isEmpty()
-} else if (project.buildSettings.build_python_extension) {
- project.ext.pythonArch = getPythonArch()
-}
 
 // Maven publication settings
 // define local Maven Repository path:
@@ -131,8 +131,8 @@ project.ext.localMavenRepository = "$rootDir/.maven-publish-dev-repo"
 nexusPublishing {
  repositories {
  maven {
- username = project.buildSettings?.sonatype?.username
- password = project.buildSettings?.sonatype?.password
+ username = localProps."sonatype.username"
+ password = localProps."sonatype.password"
  stagingProfileId = "11c25ff9a87b89"
  nexusUrl.set(uri("https://oss.sonatype.org/service/local/"))
  snapshotRepositoryUrl.set(uri("https://oss.sonatype.org/content/repositories/snapshots/"))
@@ -170,31 +170,29 @@ subprojects {
  ]) {
  apply plugin: "org.jetbrains.kotlin.multiplatform"
 
- def currentOs = DefaultNativePlatform.getCurrentOperatingSystem()
+ def os = platform.getCurrentOperatingSystem()
 
  kotlin {
- if (project.buildSettings.build_python_extension) {
- if (currentOs.macOsX & project.pythonArch == "x86_64") {
+ if (os.macOsX & localProps."architecture" == "x86_64") {
+ macosX64()
+ } else if (os.macOsX & localProps."architecture" == "arm64") {
+ if (project.hasProperty("build_release")) {
  macosX64()
- } else if (currentOs.macOsX & project.pythonArch == "arm64") {
- if (project.hasProperty("build_release")) {
- macosX64()
- macosArm64()
- } else {
- macosArm64()
- }
- } else if (currentOs.linux) {
- if (project.hasProperty("build_release")) {
- linuxX64()
- linuxArm64()
- } else if (project.pythonArch == "x86_64") {
- linuxX64()
- }
- } else if (currentOs.windows) {
- mingwX64()
+ macosArm64()
  } else {
- throw new Exception("Unsupported platform.")
+ macosArm64()
+ }
+ } else if (os.linux) {
+ if (project.hasProperty("build_release")) {
+ linuxX64()
+ linuxArm64()
+ } else if (project.localProps."architecture" == "x86_64") {
+ linuxX64()
  }
+ } else if (os.windows) {
+ mingwX64()
+ } else {
+ throw new Exception("Unsupported platform! Check project settings.")
  }
  }
  }

build_release.py

@@ -79,7 +79,7 @@ def build_python_packages(build_command, arch=None):
  # Runs Python artifacts build commands. If 'arch' argument was passed, adds it to shell command.
  python_extension_build_command = [gradle_script_name, "python-extension:build"]
  if arch is not None:
- python_extension_build_command += [f"-Pbuild_arch={arch}"]
+ python_extension_build_command += [f"-Parchitecture={arch}"]
  command = build_command + [arch]
  else:
  command = build_command
@@ -89,6 +89,17 @@ def build_python_packages(build_command, arch=None):
  run_command(command)
 
 
+def get_python_arch(python_bin_path):
+ get_python_arch_command = [f"{python_bin_path}/python", "-c", "import platform; print(platform.machine())"]
+ process = subprocess.check_output(get_python_arch_command, stderr=None)
+ current_python_arch = process.decode().strip()
+ if current_python_arch == "arm64" or current_python_arch == "x86_64":
+ return current_python_arch
+ else:
+ print_error_and_exit(f"Got wrong Python architecture for {python_bin_path}!\n"
+ f"Check your settings file or Python installation.")
+
+
 # Read Python settings file from script argument.
 # Paths to Python binaries and include directories will be got from here:
 python_settings = read_settings_file()
@@ -119,23 +130,19 @@ def build_python_packages(build_command, arch=None):
  # So the only one Python host installation, defined in the settings file, is needed:
  python_paths = list(python_settings.values())[0]
 
- # And Python package build by Gradle is disabled due the same reason.
- enable_python_package = "false"
-
- # Enable Python Extension. Native artifacts from here are used for Python packages.
- build_python_extension = "true"
+ # Enable Python package to build Python extension module.
+ enable_python_package = "true"
 
  # Collect all predefined parameters:
  build_parameters = [
  "-Pbuild_release=true",
- "-Ppython_bin_path=%s" % (python_paths["bin_path"]),
- "-Ppython_include_path=%s" % (python_paths["include_path"]),
- f"-Penable_python_package={enable_python_package}",
- f"-Pbuild_python_extension={build_python_extension}"
+ "-Ppython.bin_path=%s" % (python_paths["bin_path"]),
+ "-Ppython.include_path=%s" % (python_paths["include_path"]),
+ f"-Penable_python_package={enable_python_package}"
  ]
 
  # Run JS artifact build first:
- gradle_js_build_command = [gradle_script_name, "js-package:jsBrowserProductionWebpack"]
+ gradle_js_build_command = [gradle_script_name, "js-package:jsBrowserProductionWebpack", "-Parchitecture=x86_64"]
  run_command(gradle_js_build_command + build_parameters)
 
  # Run Python 'manylinux' packages build for x64 arch:
@@ -153,7 +160,6 @@ def build_python_packages(build_command, arch=None):
  # for all Python binaries, defined in the settings file.
  # Enable Python Extension and packages build by Gradle:
  enable_python_package = "true"
- build_python_extension = "true"
 
  # Define Gradle command for Python packages build:
  python_package_build_command = [gradle_script_name, "python-package-build:build"]
@@ -163,11 +169,13 @@ def build_python_packages(build_command, arch=None):
  # Collect all predefined parameters:
  build_parameters = [
  "-Pbuild_release=true",
- "-Ppython_bin_path=%s" % (python_paths["bin_path"]),
- "-Ppython_include_path=%s" % (python_paths["include_path"]),
- f"-Penable_python_package={enable_python_package}",
- f"-Pbuild_python_extension={build_python_extension}"
+ "-Ppython.bin_path=%s" % (python_paths["bin_path"]),
+ "-Ppython.include_path=%s" % (python_paths["include_path"]),
+ f"-Penable_python_package={enable_python_package}"
  ]
+ # Add architecture parameter for Mac:
+ if system == "Darwin":
+ build_parameters += ["-Parchitecture=%s" % (get_python_arch(python_paths["bin_path"]))]
 
  # Run Python package build:
  build_python_packages(python_package_build_command + build_parameters)

devdocs/RELEASE.md

@@ -105,7 +105,7 @@ The directory `python-package/dist` must contain Python release wheels:
 
 ## Publish artifacts
 
-Put `build_settings.yml` in the project root. See `build_settings.template.yml` for an example. 
+Put `local.properties` in the project root. See `local.properties.template` for an example. 
 Fill `pypi` and `sonatype` sections with credentials.
 
 ### 1. Python wheels (PyPi):

devdocs/SNAPSHOT_publishing.md

@@ -3,7 +3,7 @@
 ### 1. Prepare for publishing:
 
  - Check `version` in the root `build.gradle` file: version must be like `X.X.X-SNAPSHOT`.
- - Add token to the `sonatype` section of the `build_settings.yml` file.
+ - Add token to the `sonatype` section of the `local.properties` file.
 
 ### 2. Publish:
 

build_settings.template.yml → local.properties.template

@@ -1,31 +1,32 @@
----
+#
+# Copyright (c) 2023. JetBrains s.r.o.
+# Use of this source code is governed by the MIT license that can be found in the LICENSE file.
+#
 
-# option to exclude python extension from build
-build_python_extension: no
+# Set project architecture for Linux or Mac. Windows users can skip this parameter.
+# Available values: 'arm64', 'x86_64'"
+architecture=x86_64
 
-# enable tasks responsible for building and publishing of lets-plot python package
-enable_python_package: no
+# Enable tasks responsible for building and publishing of lets-plot python package.
+# Set 'true' or 'false' here:
+enable_python_package=false
 
 # -------------------------------------------------------------------
 # Python settings
-# -------------------------------------------------------------------
-python:
- bin_path: /anaconda3/bin
- include_path: /anaconda3/include/python3.8
-
+# ---------
+#python.bin_path=
+#python.include_path=
 
 # -------------------------------------------------------------------
 # PyPI settings
 # -------------------------------------------------------------------
-pypi:
- # credentials on test.pypi.org
- test:
- username:
- password:
- # credentials on pypi.org
- prod:
- username:
- password:
+# credentials for test.pypi.org
+#pypi.test.username=
+#pypi.test.password=
+# credentials on pypi.org
+#pypi.prod.username=
+#pypi.prod.password=
+
 
 
 # -------------------------------------------------------------------
@@ -39,6 +40,5 @@ pypi:
 # signing.password=your_password_here
 # signing.secretKeyRingFile=/Users/you/.gnupg/secring.kbx
 # -------------------------------------------------------------------
-sonatype:
- username:
- password:
+#sonatype.username=
+#sonatype.password=