Skip to content

Commit

Permalink
feat(hardware-testing): Added hardware testing usb package (#13140)
Browse files Browse the repository at this point in the history
The hardware-testing package is used for testing various parts of the Flex, however in order to use it you have to have a computer that has all of our packages, and dependencies and has been set up to run our software. This poses some friction as it requires a lot to get up and running. This pr solves this problem by creating a hardware-testing package, that can be run from a USB drive plugged directly into the Flex.


- Added setup-usb-module to hardware-testing Makefile, which creates a USB package
- Added hardware-testing entry to scripts/python_build_utils.py so we can get the package version
- Added correct versioning based on git tags when producing sdist/wheel files
  • Loading branch information
vegano1 committed Jul 24, 2023
1 parent 6584665 commit 6267e73
Show file tree
Hide file tree
Showing 5 changed files with 297 additions and 16 deletions.
65 changes: 50 additions & 15 deletions hardware-testing/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,17 @@ include ../scripts/python.mk

SHX := npx shx

sdist_file = dist/hardware_testing-0.0.1.tar.gz
wheel_file = dist/hardware_testing-0.0.1-py3-none-any.whl

ot_project := $(OPENTRONS_PROJECT)
project_rs_default = $(if $(ot_project),$(ot_project),robot-stack)
project_ot3_default = $(if $(ot_project),$(ot_project),ot3)

package_name = hardware_testing
package_version = $(call python_package_version,hardware-testing,$(project_ot3_default))
wheel_file = dist/$(call python_get_wheelname,hardware-testing,$(project_rs_default),$(package_name),$(BUILD_NUMBER))
sdist_file = dist/$(call python_get_sdistname,hardware-testing,$(project_ot3_default),$(package_name))
usb_file = dist/$(package_name)-usb-$(package_version).tar.gz
# Find the branch, sha, version that will be used to update the VERSION.json file
version_file = $(call python_get_git_version,hardware-testing,$(project_ot3_default),hardware-testing)

# These variables can be overriden when make is invoked to customize the
# behavior of pytest. For instance,
Expand All @@ -27,6 +35,9 @@ ssh_opts ?= $(default_ssh_opts)
ot_py_sources := $(filter %.py,$(shell $(SHX) find hardware_testing/))
ot_sources := $(ot_py_sources)

# usb package options
usb_dir ?=

# Defined separately than the clean target so the wheel file doesn’t have to
# depend on a PHONY target
clean_cmd = $(SHX) rm -rf build dist .coverage coverage.xml '*.egg-info' '**/__pycache__' '**/*.pyc' '**/.mypy_cache'
Expand All @@ -47,22 +58,22 @@ teardown:
clean:
$(clean_cmd)

$(sdist_file): setup.py $(ot_sources) clean
$(python) setup.py sdist
.PHONY: wheel
wheel: export OPENTRONS_PROJECT=$(project_rs_default)
wheel:
rm -rf dist/*.whl
$(python) setup.py $(wheel_opts) bdist_wheel
$(SHX) rm -rf build
$(SHX) ls dist

$(wheel_file): setup.py $(ot_sources) clean
$(python) setup.py bdist_wheel
.PHONY: sdist
sdist: export OPENTRONS_PROJECT=$(project_ot3_default)
sdist:
$(clean_cmd)
$(python) setup.py sdist
$(SHX) rm -rf build
$(SHX) ls dist

.PHONY: sdist
sdist: $(sdist_file)

.PHONY: wheel
wheel: $(wheel_file)

.PHONY: test
test:
-$(MAKE) apply-patches-gravimetric
Expand Down Expand Up @@ -163,7 +174,7 @@ restart:
$(call restart-service,$(host),$(br_ssh_key),$(ssh_opts),"opentrons-robot-server")

.PHONY: push-no-restart
push-no-restart:
push-no-restart: wheel
$(call push-python-package,$(host),$(br_ssh_key),$(ssh_opts),$(wheel_file))

.PHONY: push
Expand All @@ -175,7 +186,7 @@ restart-ot3:

.PHONY: push-no-restart-ot3
push-no-restart-ot3: sdist Pipfile.lock
$(call push-python-sdist,$(host),,$(ssh_opts),$(sdist_file),/opt/opentrons-robot-server,"hardware_testing")
$(call push-python-sdist,$(host),,$(ssh_opts),$(sdist_file),/opt/opentrons-robot-server,"hardware_testing",,,$(version_file))

.PHONY: push-ot3
push-ot3: push-no-restart-ot3 push-plot-webpage-ot3 push-description-ot3
Expand Down Expand Up @@ -268,3 +279,27 @@ push-photometric-ot2:
.PHONY: get-latest-tag
get-latest-tag:
git tag -l --sort=-v:refname "ot3@*" --merged | (head -n 1 || Select -First 1)


# Creates a tarball of the hardware-testing module that can be deployed to a thumb-drive
.PHONY: setup-usb-module-ot3
setup-usb-module: sdist Pipfile.lock
@echo "Creating hardware-testing USB package."

# create git-description
$(shell $(python) -c "from hardware_testing.data import *; create_git_description_file()")

# Copy files to dir
mkdir -p ./dist/hardware_testing_usb/
mv .hardware-testing-description ./dist/hardware_testing_usb/
cp -r $(sdist_file) ./dist/hardware_testing_usb/
cp -r ./hardware_testing/tools/usb-package/* ./dist/hardware_testing_usb/
cp -r ./hardware_testing/tools/plot ./dist/hardware_testing_usb/

# Create the usb-package tar file
tar -zcvf $(usb_file) -C ./dist/ ./hardware_testing_usb/
rm -rf ./dist/hardware_testing_usb

# Lets extract the usb package to the usb device if given
$(if $(usb_dir), tar -xvf $(usb_file) -C $(usb_dir))

79 changes: 79 additions & 0 deletions hardware-testing/hardware_testing/tools/usb-package/readme.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
## OVERVIEW
The usb-package bundles up the hardware_testing module so it can run from a usb thumbdrive on the Flex.

## INSTRUCTIONS

### How to make package:
make -C hardware-testing setup-usb-module
This will produce a tar file **hardware_testing_usb-<version>.tar.gz** in **hardware-testing/dist/**
example:
hardware_testing_usb-0.0.1.tar.gz

You can also make and install the tar file to a target with the **usb_dir** option
This will extract the contents to the specified location
make -C hardware-testing setup-usb-module usb_dir="/Volumes/ENFAIN/"

### How to deploy:
If you use the **usb_dir** option the usb-package will be installed on the usb drive
You can also manually extract the tar file onto a target like so
tar -xvf hardware_testing_usb-0.0.1.tar.gz -C "<drive-location>"

### How to install on the Flex:
Once you have an usb thumdrive with the usb-packge deployed you can plug in the thumbdrive to
a Flex and start the setup script to install the required files like so

0. Make sure the Flex is powered on
1. ssh to the robot with
ssh root@<ip-address>
NOTE: You can also use screen or similar if you are using an FTDI connection
screen /dev/<ftdi-device> 115200
2. Plug in the hardware-testing thumbdrive to the Flex
The device should be mounted to **/media/sda** or similar
Note: You can also manually mount the thumbdrive with
a. From the Linux command line list usb devices with
blkid
Note:
Your drive address should be something like
**/dev/sda: LABEL="ENFAIN"...**
The LABEL should be the name of your usb thumbdrive
b. mount the drive
mount /dev/sda /mnt
c. the device should now be accessed thorugh /mnt
3. Navigate to the mounted drive
a. if drive was automatically mounted, run
cd /media/sdx/hardware_testing_usb where x is the enumerated drive
b. or if mounted manually
cd /mnt/hardware_testing_usb
4. Run the setup script
a. From the usb-packge dir **hardware_testing_usb** run
./setup
5. Re-logging to apply
a. from the command shell run
logout
b. use ssh or screen to log back in
see step 1 above.
6. You should now see
**Hardware-Testing package enabled at <mount-location>/hardware_testing_usb**

Note: **<mount-location>** is where the thumbdrive was mounted typically **/media/sdx**
but could also be **/mnt** or different if mounted manually
7. Done
a. The hardware-testing package is now setup and can be used as normal
example:
python3 -m hardware_testing.scripts.module_calibration


### How to uninstall from the Flex:
With the usb thumbdrive plugged into the Flex
1. Navigate to usb-packge dir
cd <mount-location>/hardware_testing_usb
2. Run the setup script
./setup teardown
You should see **Teardown Success**
3. The usb drive can now be removed


### Troubleshoting

If the usb thumbdrive is removed and replugged it could enumerate differently.
This means you will need to run the ./setup script again
145 changes: 145 additions & 0 deletions hardware-testing/hardware_testing/tools/usb-package/setup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
#! /bin/bash

# This script sets up the hardware-testing module to run from usb or some other dir on the Flex robot. Read the readme.txt file for setup and instructions.

USB_DIR=$(pwd)
MOUNT_DIR=$(dirname ${USB_DIR})
PACKAGE_VERSION=""
PACKAGE_NAME="hardware_testing"
PACKAGE_DIR=$(echo ${USB_DIR}/${PACKAGE_NAME}-*/)
PACKAGE_TAR_FILE=$(echo ${USB_DIR}/${PACKAGE_NAME}-*.tar.gz)
PKG_INFO_FILE=$PACKAGE_DIR/PKG-INFO
SYSTEM_VERSION_FILE="/etc/VERSION.json"
ENV_PROFILE="/etc/profile.d/ot-usb-environ.sh"

# Script entry-point
main() {
echo "Validating files"
validate "$@"

# execute action
case $1 in
teardown)
echo "Teardown"
teardown
;;
*)
echo "Setup"
setup
;;
esac
}

# Make sure we have the correct files before doing anything
validate() {
# Check if we are running on the robot or not
if [ ! -f $SYSTEM_VERSION_FILE ]; then
echo "${SYSTEM_VERSION_FILE} not found, make sure you're running this from a robot!"
exit 1;
fi
is_flex=$(cat $SYSTEM_VERSION_FILE | grep -o OT-3)
if [ -z $is_flex ]; then
echo "Make sure robot is a Flex before running setup."
exit 1
fi

# Extract the tarball
usb_module_filename=$(ls $PACKAGE_TAR_FILE)
if [ ! -f "$usb_module_filename" ]; then
echo "Could not find package tarball - ${PACKAGE_TAR_FILE}"
exit 1;
fi
}

_extract_tarball() {
echo "Extracting tarball ${usb_module_filename}"
tar -xvf $usb_module_filename -C $USB_DIR

# Get the version of the package
if [ ! -f $PKG_INFO_FILE ]; then
echo "error: ${PKG_INFO_FILE} was not found!"
exit 1;
fi
PACKAGE_VERSION=$(cat ${PKG_INFO_FILE} | sed -n '/Version/{n;n;p}')
}


# Helper function to write/delete environment variable profile script
_env_profile() {
# mount the filesystem rw
mount -o remount,rw /

if [[ $1 =~ "delete" ]]; then
echo "Deleting usb-package env file - ${ENV_PROFILE}"
rm -rf $ENV_PROFILE
mount -o remount,ro /
return;
fi

echo "Writing usb-package env profile - $ENV_PROFILE"

cat <<EOF > $ENV_PROFILE
#!/usr/bin/env sh
# Do an auto-teardown if the usb device is not found
if [ ! -d $USB_DIR ]; then
mount -o remount,rw /
rm -rf $ENV_PROFILE
mount -o remount,ro /
return 1;
fi
echo "Hardware-Testing package enabled at $USB_DIR"
export PYTHONPATH=\$PYTHONPATH:$PACKAGE_DIR
# set OT_SYSTEM_VERSION if not set
if [ -z \$OT_SYSTEM_VERSION ]; then
export OT_SYSTEM_VERSION="0.0.0"
fi
# set the TESTING_DATA_DIR
export TESTING_DATA_DIR=$USB_DIR/testing_data
EOF
# remount filesystem as ro
mount -o remount,ro /
}

# Sets up the hardware-testing module to be used from usb location
setup() {
echo "Setting up hardware-testing module ${PACKAGE_VERSION}"

# lets extract that tarball
_extract_tarball

# Lets set up the environment profile
_env_profile

# TODO (ba, 2023-07-19): set the logging location for hardware-testing scripts to usb
# We need to consilidate hardware-testing logging before being able to do this.

# Lets deal with plot-webpage since the sdist file only contains .py files
cp -r $USB_DIR/plot $PACKAGE_DIR/$PACKAGE_NAME/tools/

# Lets deal with hardware-testing-description file, is /data correct?
cp -r $USB_DIR/.hardware-testing-description /data/

echo "Hardware-Testing module has been setup, re-login to apply changes."
}

# Tearsdown the hardare-testing module
teardown() {
echo "Tearing down hardare-testing module ${PACKAGE_VERSION}"

# delete the environment profile
_env_profile delete

# delete the description file
rm -rf /data/.hardware-testing-description

echo "Teardown Success"
}

set -eE -o pipefail
trap teardown ERR

main "$@"
23 changes: 22 additions & 1 deletion hardware-testing/setup.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,31 @@
"""Setup script."""

import os
import sys

from setuptools import setup, find_packages

HERE = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.join(HERE, "..", "scripts"))
from python_build_utils import normalize_version # noqa: E402


def _get_version() -> None:
buildno = os.getenv("BUILD_NUMBER")
project = os.getenv("OPENTRONS_PROJECT", "ot3")
git_dir = os.getenv("OPENTRONS_GIT_DIR", None)
if buildno:
normalize_opts = {"extra_tag": buildno}
else:
normalize_opts = {}
return normalize_version(
"hardware-testing", project, git_dir=git_dir, **normalize_opts
)


setup(
name="hardware_testing",
version="0.0.1",
version=_get_version(),
packages=find_packages(where=".", exclude=["tests.*", "tests"]),
url="",
license="",
Expand Down
1 change: 1 addition & 0 deletions scripts/python_build_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
'shared-data': PackageEntry('shared_data'),
'notify-server': PackageEntry('notify_server'),
'hardware': PackageEntry('opentrons_hardware'),
'hardware-testing': PackageEntry('hardware_testing'),
'usb-bridge': PackageEntry('usb_bridge'),
'system-server': PackageEntry('system_server'),
'server-utils': PackageEntry('server_utils'),
Expand Down

0 comments on commit 6267e73

Please sign in to comment.