fix(update-server): Fix issues with 3.2 api on 3.3 system (#2097)

This fixes issues with updating to a 3.2 api on a 3.3 system, and also issues
with running 3.2 on a 3.3 system.

API server wheels from before 3.3.0 do not have resource subdirectories and should not be
provisioned. The update server now inspects the version (from the filename) of the api wheel it is
installing and will not provision wheels whose version is prior to 3.3.0.

In addition, once the 3.2 api is uploaded, it will not have established the
proper setup scripts. This commit changes the container to properly fall back on
the scripts established in /usr/local/lib.

Dockerfile

@@ -89,13 +89,17 @@ STOPSIGNAL SIGTERM
 # For backward compatibility, udev is enabled by default
 ENV UDEV on
 
-RUN echo "export CONTAINER_ID=$(uuidgen)" | tee -a /etc/profile.d/opentrons.sh
-
 # The one link we have to make in the dockerfile still to make sure we get our
 # environment variables
 COPY ./compute/find_python_module_path.py /usr/local/bin/
+COPY ./compute/find_ot_resources.py /usr/local/bin
 RUN ln -sf /data/system/ot-environ.sh /etc/profile.d/00-persistent-ot-environ.sh &&\
- ln -sf `find_python_module_path.py opentrons`/resources/ot-environ.sh /etc/profile.d/01-builtin-ot-environ.sh
+ ln -sf `find_ot_resources.py`/ot-environ.sh /etc/profile.d/01-builtin-ot-environ.sh
+
+# Note: the quoting that defines the PATH echo is very specifically set up to
+# get $PATH in the script literally so it is evaluated at container runtime.
+RUN echo "export CONTAINER_ID=$(uuidgen)" | tee -a /etc/profile.d/opentrons.sh\
+ && echo 'export PATH=$PATH:'"`find_ot_resources.py`/scripts" | tee -a /etc/profile.d/opentrons.sh
 
 
 # This configuration is used both by both the build and runtime so it has to

api/opentrons/resources/ot-environ.sh

@@ -16,7 +16,7 @@ if [ -z $OT_ENVIRON_SET_UP ]; then
  # connecting to Host OS services
  export DBUS_SYSTEM_BUS_ADDRESS=unix:path=/host/run/dbus/system_bus_socket
  export PYTHONPATH=$PYTHONPATH:/data/packages/usr/local/lib/python3.6/site-packages
- export PATH=$PATH:/data/packages/usr/local/bin:$OT_CONFIG_PATH/scripts
+ export PATH=/data/packages/usr/local/bin:$OT_CONFIG_PATH/scripts:$PATH
 
  # TODO(seth, 8/15/2018): These are almost certainly unused and should be hardcoded
  # if they are in fact still used

compute/container_setup.sh

@@ -12,7 +12,7 @@ if [ "$previous_id" != "$current_id" ] ; then
  rm -rf /data/packages/usr/local/lib/python3.6/site-packages/opentrons*
  rm -rf /data/packages/usr/local/lib/python3.6/site-packages/ot2serverlib*
  rm -rf /data/packages/usr/local/lib/python3.6/site-packages/otupdate*
- provision=`find_python_module_path.py opentrons`/resources/scripts/provision-api-resources
+ provision=`find_ot_resources.py`/scripts/provision-api-resources
  echo "[ $0 ] provisioning with $provision"
  python "$provision"
  echo "$current_id" > /data/id

compute/find_ot_resources.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+""" Simple python script to find the most salient opentrons resources directory.
+
+The first thing we try is just using find_python_module.py to find whatever
+opentrons module takes priority. However, if somebody installed a 3.2 api on the
+system, that module may not have a /resources subdirectory. In this case, we
+fall back to what we know will be there in /usr/local/lib.
+"""
+import importlib
+import os
+import sys
+import find_python_module_path
+
+def find_resources_dir():
+ ideal = find_python_module_path.find_module('opentrons')
+ if not os.path.exists(os.path.join(ideal, 'resources')):
+ new_path = []
+ for item in sys.path:
+ if '/data' not in item:
+ new_path += item
+ sys.path = new_path
+ fallback = find_python_module_path.find_module('opentrons')
+ # Here we return whatever we get and trust the caller to fail
+ return os.path.join(fallback, 'resources')
+ else:
+ return os.path.join(ideal, 'resources')
+
+if __name__ == '__main__':
+ print(find_resources_dir())

update-server/otupdate/control.py

@@ -1,21 +1,63 @@
+import asyncio
 import os
 import logging
 from time import sleep
+import aiohttp
 from aiohttp import web
 from threading import Thread
 
 log = logging.getLogger(__name__)
 
 
+def do_restart():
+ """ This is the (somewhat) synchronous method to use to do a restart.
+
+ It actually starts a thread that does the restart. `__wait_and_restart`,
+ on the other hand, should not be called directly, because it will block
+ until the system restarts.
+ """
+ Thread(target=__wait_and_restart).start()
+
+
 def __wait_and_restart():
+ """ Delay and then execute the restart. Do not call directly. Instead, call
+ `do_restart()`.
+ """
  log.info('Restarting server')
  sleep(1)
- os.system('kill 1')
+ # We can use the default event loop here because this
+ # is actually running in a thread. We use aiohttp here because urllib is
+ # painful and we don’t have `requests`.
+ loop = asyncio.new_event_loop()
+ loop.run_until_complete(_resin_supervisor_restart())
+
+
+async def _resin_supervisor_restart():
+ """ Execute a container restart by requesting it from the supervisor.
+
+ Note that failures here are returned but most likely will not be
+ sent back to the caller, since this is run in a separate workthread.
+ If the system is not responding, look for these log messages.
+ """
+ supervisor = os.environ.get('RESIN_SUPERVISOR_ADDRESS',
+ 'https://127.0.0.1:48484')
+ restart_url = supervisor + '/v1/restart'
+ api = os.environ.get('RESIN_SUPERVISOR_API_KEY', 'unknown')
+ app_id = os.environ.get('RESIN_APP_ID', 'unknown')
+ async with aiohttp.ClientSession() as session:
+ async with session.post(restart_url,
+ params={'apikey': api},
+ json={'appId': app_id,
+ 'force': True}) as resp:
+ body = await resp.read()
+ if resp.status != 202:
+ log.error("Could not shut down: {}: {}"
+ .format(resp.status, body))
 
 
 async def restart(request):
  """
  Returns OK, then waits approximately 1 second and restarts container
  """
- Thread(target=__wait_and_restart).start()
+ do_restart()
  return web.json_response({"message": "restarting"})

update-server/otupdate/install.py

@@ -3,12 +3,35 @@
 import shutil
 import asyncio
 import logging
+import re
 import traceback
 from aiohttp import web
 
 log = logging.getLogger(__name__)
 VENV_NAME = 'env'
 
+# This regex should match a PEP427 compliant wheel filename and extract its
+# version into separate groups. Groups 1, 2, 3, and 4 should be the major,
+# minor, patch, and tag respectively. The tag capture group is optional
+WHEEL_VERSION_RE = re.compile('^[\w]+-([\d]+).([\d]+).([\d]+)([\w.]+)?-.*\.whl') # noqa
+FIRST_PROVISIONED_VERSION = (3, 3, 0)
+
+
+def _version_less(version_a, version_b):
+ """ Takes two version as (major, minor, patch) tuples and returns a <= b.
+ """
+ if version_a[0] > version_b[0]:
+ return False
+ elif version_a[0] < version_b[0]:
+ return True
+ else:
+ if version_a[1] > version_b[1]:
+ return False
+ elif version_a[1] < version_b[1]:
+ return True
+ else:
+ return version_a[2] < version_b[2]
+
 
 async def _install(python, filename, loop) -> (str, str, int):
  running_on_pi = os.environ.get('RUNNING_ON_PI') and '/tmp' in python
@@ -158,6 +181,9 @@ async def install_py(python, data, loop) -> (dict, int):
 
 
 async def _provision_container(python, loop) -> (str, int):
+ if not os.environ.get('RUNNING_ON_PI'):
+ return {'message': 'Did not provision (not on pi)',
+ 'filename': '<provision>'}, 0
  provision_command = 'provision-api-resources'
  proc = await asyncio.create_subprocess_shell(
  provision_command, stdout=asyncio.subprocess.PIPE,
@@ -186,16 +212,26 @@ async def update_api(request: web.Request) -> web.Response:
  """
  log.debug('Update request received')
  data = await request.post()
+ # import pdb; pdb.set_trace()
  try:
  res0, rc0 = await install_py(
  sys.executable, data['whl'], request.loop)
  reslist = [res0]
- if rc0 == 0 and os.environ.get('RUNNING_ON_PI'):
- resprov, rcprov = await _provision_container(
- sys.executable, request.loop)
- reslist.append(resprov)
- else:
- rcprov = 0
+ filename = os.path.basename(res0['filename'])
+ version_re_res = re.search(WHEEL_VERSION_RE, filename)
+ rcprov = 0
+ if not version_re_res:
+ log.warning("Wheel version regex didn't match {}: won't provision"
+ .format(filename))
+ elif rc0 == 0:
+ v_maj, v_min, v_pat\
+ = (int(v) for v in version_re_res.group(1, 2, 3))
+ if not _version_less((v_maj, v_min, v_pat),
+ FIRST_PROVISIONED_VERSION):
+ # import pdb; pdb.set_trace()
+ resprov, rcprov = await _provision_container(
+ sys.executable, request.loop)
+ reslist.append(resprov)
  if 'serverlib' in data.keys():
  res1, rc1 = await install_py(
  sys.executable, data['serverlib'], request.loop)

update-server/tests/test_update_api.py

@@ -0,0 +1,112 @@
+""" Test file for updating the API.
+"""
+import os
+import subprocess
+import sys
+import tempfile
+
+import otupdate
+from otupdate import install
+
+
+def build_pkg(package_name, version, in_dir=None):
+ """ Build a fake minor package and return its path """
+ if not in_dir:
+ td = tempfile.mkdtemp()
+ in_dir = os.path.join(td, package_name)
+ os.mkdir(in_dir)
+ test_setup = """
+from setuptools import setup
+
+setup(name='{0}',
+version='{1}',
+description='Test package',
+url='https://github.com/Opentrons/opentrons',
+author='Opentrons',
+author_email='[email protected]',
+license='Apache 2.0',
+packages=['{0}'],
+zip_safe=False)
+""".format(package_name, version)
+ test_setup_file = os.path.join(in_dir, 'setup.py')
+ with open(test_setup_file, 'w') as tsf:
+ tsf.write(test_setup)
+
+ src_dir = os.path.join(in_dir, package_name)
+ try:
+ os.mkdir(src_dir)
+ except FileExistsError:
+ pass
+ test_code = """
+print("all ok")'
+"""
+ test_file = os.path.join(src_dir, '__init__.py')
+
+ with open(test_file, 'w') as tf:
+ tf.write(test_code)
+
+ cmd = '{} setup.py bdist_wheel'.format(sys.executable)
+ subprocess.run(cmd, cwd=in_dir, shell=True)
+ return os.path.join(
+ in_dir, 'dist',
+ '{}-{}-py3-none-any.whl'.format(package_name, version))
+
+
+async def test_provision_version_gate(loop, monkeypatch, test_client):
+
+ async def mock_install_py(executable, data, loop):
+ return {'message': 'ok', 'filename': data.filename}, 0
+
+ monkeypatch.setattr(install, 'install_py', mock_install_py)
+
+ container_provisioned = False
+
+ async def mock_provision_container(executable, loop):
+ nonlocal container_provisioned
+ container_provisioned = True
+ return {'message': 'ok', 'filename': '<provision>'}, 0
+
+ monkeypatch.setattr(install, '_provision_container',
+ mock_provision_container)
+
+ update_package = os.path.join(os.path.abspath(
+ os.path.dirname(otupdate.__file__)), 'package.json')
+ app = otupdate.get_app(
+ api_package=None,
+ update_package=update_package,
+ smoothie_version='not available',
+ loop=loop,
+ test=False)
+ cli = await loop.create_task(test_client(app))
+
+ pkg_name = 'opentrons'
+ td = tempfile.mkdtemp()
+ tmpd = os.path.join(td, pkg_name)
+ os.mkdir(tmpd)
+
+ # First test: API server with a version before 3.3.0 should not provision
+ test_wheel = build_pkg('opentrons', '3.2.0a2', tmpd)
+
+ resp = await cli.post(
+ '/server/update', data={'whl': open(test_wheel, 'rb')})
+
+ assert resp.status == 200
+ assert not container_provisioned
+
+ # Second test: An api server with a version == 3.3.0 (regardless of tag)
+ # should provision
+ test_wheel = build_pkg('opentrons', '3.3.0rc5', tmpd)
+ resp = await cli.post(
+ '/server/update', data={'whl': open(test_wheel, 'rb')})
+
+ assert resp.status == 200
+ assert container_provisioned
+
+ # Third test: An api server with a version > 3.3.0 should provision
+ container_provisioned = False
+ test_wheel = build_pkg('opentrons', '3.4.0', tmpd)
+ resp = await cli.post(
+ '/server/update', data={'whl': open(test_wheel, 'rb')})
+
+ assert resp.status == 200
+ assert container_provisioned