Fixes installation with pre-compile approach on stan files

- Fix installation issue on various platforms fixes #845 #846 ; Now support up to Python 3.11 with macos, linux and windows
- Updates cmdstanpy in setup.py and stan.py to fix issues plaguing installation on various platforms including windows, linux, and macos.
- Update deployment method to sdist to ensure cmdstan requirements are properly installed.
- Update cmdstan to install on default directory for simpler workflow

Co-Authored-By: Edwin Ng <[email protected]>
Co-Authored-By: szmark001 <[email protected]>
Co-Authored-By: swotai <[email protected]>

.github/workflows/black.yaml

@@ -6,11 +6,33 @@ jobs:
  lint:
  runs-on: ubuntu-latest
  steps:
- - uses: actions/checkout@v2
-# Currently only checking instead of auto formatting
- - uses: jpetrucciani/black-check@master
-# - uses: psf/black@stable
- - name: Install Dependencies
- run: pip install -r requirements-test.txt
- - name: black check diff
- run: black --diff .
+ # - uses: actions/checkout@v3
+ # - uses: psf/black@stable
+ # Currently only checking instead of auto formatting
+ # - uses: jpetrucciani/black-check@master
+ # - name: Install Dependencies
+ # run: |
+ # python -m pip install --upgrade pip
+ # pip install -r requirements-test.txt
+ # pip install black --upgrade
+ # - name: black check diff
+ # run: black --diff .
+ - name: Check out repository
+ uses: actions/checkout@v3
+
+ - name: Set up Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.9 # Set to your preferred Python version
+
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install -r requirements-test.txt
+ pip install black --upgrade
+
+ # - name: Run Black Check
+ # run: black --check . # conforms Black's style without changing it
+
+ - name: Show Black Diff
+ run: black --diff . # shows the diff of Black changes, if any

.github/workflows/pypi-deploy.yaml

@@ -24,12 +24,25 @@ jobs:
  pip install setuptools wheel twine
  pip install build
  pip install auditwheel
- - name: Build package
- run:
+ # - name: Build package
+ # run:
+ # python -m build
+ # - name: Convert linux wheel to manylinux wheels
+ # run:
+ # auditwheel repair --plat manylinux2014_x86_64 -w dist dist/*linux_x86_64.whl
+ # - name: Remove platform specific linux wheels
+ # run:
+ # rm dist/*-linux_x86_64.whl
+ # - name: Publish package
+ # uses: pypa/[email protected]
+ # with:
+ # user: __token__
+ # password: ${{ secrets.PYPI_PASSWORD }}
+ - name: Build and publish
+ env:
+ TWINE_USERNAME: __token__
+ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+ run: |
+ pip install .
  python -m build
- auditwheel repair dis/*linux_x86_64.whl
- - name: Publish package
- uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
- with:
- user: __token__
- password: ${{ secrets.PYPI_PASSWORD }}
+ python -m twine upload dist/*.tar.gz

.gitignore

@@ -79,10 +79,12 @@ tramp
 ############
 *_netbeans
 
+.DS_Store
 .arcconfig
 .ipynb_checkpoints
 .vscode
 __pycache__
+.pytest_cache
 build/
 dist/
 orbit.egg-info/
@@ -102,6 +104,7 @@ tutorial/_build/
 !.gitmodules
 !.jenkins/
 !.scaffold.yaml
+!.readthedocs.yml
 
 # Java
 *.class
@@ -124,9 +127,7 @@ ins.xml
 *.bak-*
 
 # ignore stan compiled files due to cmdstanpy
-stan_compiled/
 *.hpp
-orbit/stan_model/
 orbit/stan_compiled/
 orbit/stan/ets
 orbit/stan/dlt

CONTRIBUTING.md

@@ -14,13 +14,15 @@ If you'd like to contribute to something else, open an issue for discussion firs
 # Setup
 
 First fork the repository on `dev` branch
+
 ```bash
 $ git clone --single-branch --branch dev https://github.com/uber/orbit.git
 ```
 
 # Making Changes
 
 Create a new branch to develop new code
+
 ```bash
 $ cd orbit
 $ git checkout feat-my-new-feature # our branch naming convention
@@ -31,14 +33,11 @@ $ pip install -e . # install with dev mode
 # Test
 
 ## Prerequisites
+
 Install orbit required dependencies for test.
-```bash
-$ pip install -r requirements-test.txt
-```
 
-In order to run tests, you need to build the Cython modules
 ```bash
-$ python setup.py build_ext --inplace
+$ pip install -r requirements-test.txt
 ```
 
 ## Testing
@@ -55,23 +54,27 @@ $ pytest -vs tests/ --cov orbit/
 You can run black linting to lint the code style.
 
 ### Linting one single file
+
 ```bash
 $ black <file path>
 ```
 
 ### Linting every file under the current directory
+
 ```bash
 $ black .
 ```
 
 ### Outputting the code change black would have done without actually making change
+
 ```bash
 $ black --diff <file path>
 ```
 
 # Submission
 
 In your PR, please include:
+
 - Changes made
 - Links to related issues/PRs
 - Tests
@@ -81,5 +84,6 @@ In your PR, please include:
 Please add the core Orbit contributors as reviewers.
 
 ## Merging and Releasing versions
-We use squash and merge for changes onto `dev` branch. However, due to history comparison, from `dev` to `release`
-and `master`, we use rebase and merge. For release details, please refer to [RELEASE.md](./RELEASE.md)
+
+We use squash and merge for changes onto `dev` branch. However, due to history comparison, from `dev` to `release`
+and `master`, we use rebase and merge. For release details, please refer to [RELEASE.md](./RELEASE.md)

MANIFEST.in

@@ -4,8 +4,9 @@ include pyproject.toml
 include *.txt
 include requirements.txt
 include requirements-test.txt
-include orbit/stan/*.stan
+include orbit/stan/*
 include orbit/stylelib/*.mplstyle
+include orbit/config.json
 
 recursive-include tests *
 recursive-exclude * __pycache__

install_stan.py

@@ -0,0 +1,62 @@
+import json
+import os
+import platform
+import shutil
+from multiprocessing import cpu_count
+
+import cmdstanpy
+
+with open("orbit/config.json") as f:
+ config = json.load(f)
+CMDSTAN_VERSION = config["CMDSTAN_VERSION"]
+IS_WINDOWS = platform.platform().startswith("Win")
+
+
+def remove_older_cmdstan(cmdstan_version):
+ """
+ Checks the default CmdStan path (~/.cmdstan) for folders with older versions
+ and removes them.
+
+ Args:
+ cmdstan_version (str): The current CmdStan version.
+ """
+ default_path = os.path.expanduser("~/.cmdstan")
+
+ if not os.path.exists(default_path):
+ # Path doesn't exist, nothing to remove
+ return
+
+ for folder in os.listdir(default_path):
+ if folder.startswith("cmdstan-"):
+ # Extract version number from folder name
+ folder_version = folder.split("-")[1]
+ if folder_version < cmdstan_version:
+ # Remove folder if version is older
+ full_path = os.path.join(default_path, folder)
+ try:
+ print(f"Removing older CmdStan version: {folder} : {full_path}")
+ shutil.rmtree(full_path)
+ print(f"Done.")
+ except OSError as e:
+ print(f"Error removing {folder}: {e}")
+
+
+def install_stan():
+ """
+ Compile and install stan backend
+ Reference from prophet
+ """
+
+ remove_older_cmdstan(CMDSTAN_VERSION)
+
+ if not cmdstanpy.install_cmdstan(
+ version=CMDSTAN_VERSION,
+ overwrite=True,
+ verbose=True,
+ cores=cpu_count(),
+ progress=True,
+ compiler=IS_WINDOWS,
+ ):
+ raise RuntimeError("CmdStan failed to install.")
+
+ print(f"Installed cmdstanpy (cmdstan v.{CMDSTAN_VERSION}) and compiler.")

orbit/__version__.py

@@ -1 +1 @@
-__version__ = "1.1.4.3"
+__version__ = "1.1.4.4.dev14"

orbit/config.json

@@ -0,0 +1,11 @@
+{
+ "CMDSTAN_VERSION": "2.34.1",
+ "ORBIT_MODELS": [
+ [
+ "dlt",
+ "ets",
+ "ktrlite",
+ "lgt"
+ ]
+ ]
+}

orbit/constants/constants.py

@@ -147,4 +147,4 @@ class CompiledStanModelPath:
  """
 
  PARENT = "orbit"
- CHILD = "stan_compiled"
+ CHILD = "stan"

orbit/estimators/stan_estimator.py

@@ -1,18 +1,21 @@
-from abc import abstractmethod
-from copy import copy
 import logging
 import multiprocessing
+from abc import abstractmethod
+from copy import copy
 from sys import platform, version_info
 
-from .base_estimator import BaseEstimator
 from ..exceptions import EstimatorException
-from ..utils.stan import get_compiled_stan_model, suppress_stdout_stderr
 from ..utils.general import update_dict
 from ..utils.logger import get_logger
-
+from ..utils.set_cmdstan_path import set_cmdstan_path
+from ..utils.stan import get_compiled_stan_model
+from .base_estimator import BaseEstimator
 
 logger = get_logger("orbit")
 
+# Make sure models are using the right cmdstan folder
+set_cmdstan_path()
+
 
 class StanEstimator(BaseEstimator):
  """Abstract StanEstimator with shared args for all StanEstimator child classes
@@ -133,7 +136,8 @@ def fit(
  )
  logger.info(msg)
 
- compiled_mod = get_compiled_stan_model(model_name)
+ compiled_mod = get_compiled_stan_model(stan_model_name=model_name)
+
  # check https://mc-stan.org/cmdstanpy/api.html#cmdstanpy.CmdStanModel.sample
  # for additional args
  stan_mcmc_fit = compiled_mod.sample(
@@ -194,7 +198,7 @@ def fit(
  fitter=None,
  init_values=None,
  ):
- compiled_mod = get_compiled_stan_model(model_name)
+ compiled_mod = get_compiled_stan_model(stan_model_name=model_name)
  data_input.update({"T_STAR": 1.0})
 
  # passing callable from the model as seen in `initfun1()`

orbit/template/dlt.py

@@ -751,10 +751,9 @@ def predict(
  global_trend_level + global_trend_slope * idx * self._time_delta
  )
  elif self.global_trend_option == GlobalTrendOption.loglinear.name:
- full_global_trend[
- :, idx
- ] = global_trend_level + global_trend_slope * np.log(
- 1 + idx * self._time_delta
+ full_global_trend[:, idx] = (
+ global_trend_level
+ + global_trend_slope * np.log(1 + idx * self._time_delta)
  )
  elif self.global_trend_option == GlobalTrendOption.logistic.name:
  full_global_trend[:, idx] = self.global_floor + (

orbit/template/ktr.py

@@ -1,4 +1,3 @@
-from orbit.models.ktrlite import KTRLite
 import pandas as pd
 import numpy as np
 import math
@@ -14,14 +13,14 @@
  TrainingMetaKeys,
  PredictionMetaKeys,
 )
+from ..constants.palette import OrbitPalette
 from ..exceptions import IllegalArgument, ModelException, PredictionException
 from ..utils.general import is_ordered_datetime
 from ..utils.kernels import gauss_kernel, sandwich_kernel
 from ..utils.features import make_seasonal_regressors
 from .model_template import ModelTemplate
 from ..estimators.pyro_estimator import PyroEstimatorSVI
 from ..models import KTRLite
-from orbit.constants.palette import OrbitPalette
 from ..utils.knots import get_knot_idx, get_knot_dates
 from ..utils.plot import orbit_style_decorator