Merge pull request #53 from meadej/main

Pipeline updates (allows configuration variables in place of static values) and documentation

.env-example

@@ -0,0 +1,54 @@
+HOSTNAME=skyscan
+
+#Axis-PTZ
+TZ="New York" #Local timezone in "TZ database name" format. https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+LAT=38.889444 # Latitude of the camera
+LONG=-77.035278 # Longitude of the camera
+ALT=200 # Altitude of the camera, this should be in METERS above sea level
+ROLL=0 # Roll Angle of Camera Mount from Inertial
+PITCH=0 # Pitch Angle of Camera Mount from Inertial
+YAW=0 # Yaw Angle of Camera Mount from Inertial
+MIN_ELEVATION=0 # The minimum elevation for the camera.
+FEEDER_ID=long-api-key-goes-here # Your FlightAware feeder ID (required)
+AXIS_USERNAME=root # The username for the Axis camera
+AXIS_PASSWORD=password # The Password for the Axis camera
+AXIS_IP=192.168.1.220 # The IP address of the camera
+CAMERA_MOVE_SPEED=50 # The speed at which the Axis will move for Pan/Tilt (0-100)
+CAMERA_DELAY=0.5 # How many seconds after issuing a Pan/Tilt command should a picture be taken
+CAMERA_ZOOM=1000 # The zoom setting for the camera (0-9999)
+CAMERA_LEAD=0.5 # How many seconds ahead of a plane's predicted location should the camera be positioned
+RTL_DEV=1 # The device ID for the RTL-SDR - set using the rtl_eeprom program
+GPS_SERIAL=/dev/ttyACM0 # GPS module serial port
+
+#Edge-detect 
+WEIGHTS=/data/weights/localizer.pt
+AGNOSTIC_NMS=True
+NOSAVE=True
+CONF_THRES=0.25
+IMG_SIZE=640
+DEVICE=CPU
+IOU_THRES=0.45
+SOURCE_DIR=/data/tosort
+PLANE_DIR=/data/plane
+NOPLANE_DIR=/data/noplane
+LOG_DIR=/data/log
+VIEW_IMG=False
+SAVE_TXT=False
+SAVE_JSON=True
+SAVE_CONF=False
+CLASSES=0
+AUGMENT=False
+MQTT_IP=mqtt
+MQTT_TOPIC=/skyscan/captures/data/processed
+CAMERA_TOPIC=skyscan/${HOSTNAME}/image/edgetech-axis-ptz/JSON
+NO_TRACE=False
+UPDATE=False
+OMP_NUM_THREADS=1
+
+# Coral Detect Variables
+CORAL_WEIGHTS=/data/weights/localizer.tflite
+
+# Tracker Variables
+MQTT_IP=mqtt
+TRACKER_INPUT_TOPIC=skyscan/${HOSTNAME}/ADSB/edgetech-pingusb/JSON
+TRACKER_PUBLISH_TOPIC=skyscan/${HOSTNAME}/flight/edgetech-tracker/JSON

.github/workflows/README.md

@@ -0,0 +1,21 @@
+# SkyScan GitHub Workflows
+
+## Overview
+
+SkyScan and associated projects use a number of GitHub action workflows to verify code security, secrets integrity, and provide project building and deployment capability. These workflows and their required secrets and configuration variables are detailed below. 
+
+For information on adding configuration variables and secrets, see the GitHub Actions documentation [here](https://docs.github.com/en/actions/learn-github-actions/variables#creating-configuration-variables-for-an-environment) and [here](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-a-repository).
+
+## Docker Container Building 
+
+SkyScan uses the action outlined in `dockerbuild.yml` to automatically build Dockerfiles located in the project and push them to Docker Hub. This action is triggered everytime there is a push to the `main` branch. 
+
+### Secrets
+* `DOCKER_USERNAME` (required) - The login username for Docker Hub. 
+* `DOCKER_NAMESPACE` (required) - The Docker Hub account that this image will be pushed to. The `DOCKER_USERNAME` Docker Hub user must have write permissions to `DOCKER_NAMESPACE`. Oftentimes they are the same value. 
+* `DOCKER_TOKEN` (required) - A Docker Hub personal access token associated with the `DOCKER_USERNAME` account. 
+
+### Configuration Variables
+* `PROJECT_NAME` (required) - The overarching name of the project. 
+* `DOCKER_BUILD_PLATFORMS` (optional, defaults to `linux/amd64`) - A comma separated list of the target platforms the images will be built for. 
+* `DOCKER_BUILD_FOLDERS` (optional, defaults to `.`) - A comma separated list of folders under the main repository containing Dockerfiles to be built. 

.github/workflows/dockerbuild.yml

@@ -39,21 +39,21 @@ jobs:
  - name: Build Images
  env: 
  # Every folder in the repo that has a Dockerfile within it, comma separated 
- DOCKERFOLDERS: "tracker,piaware,pan-tilt-pi,notebook-server,egi,axis-ptz"
- PROJECT_NAME: "skyscan"
+ DOCKERFOLDERS: "${{ vars.DOCKER_BUILD_FOLDERS }}"
+ PROJECT_NAME: "${{ vars.PROJECT_NAME }}"
  REPO_NAME: "${{ secrets.DOCKER_NAMESPACE }}"
+ PLATFORMS: "${{ vars.DOCKER_BUILD_PLATFORMS }}"
  run: |
  IFS=","
- read -ra ARR <<< "$DOCKERFOLDERS"
+ read -ra ARR <<< "${DOCKERFOLDERS:-.}"
  for folder in "${ARR[@]}"
  do 
  IFS="/"
  read -ra NAMEFOLDER <<< $folder
  SUBNAME=${NAMEFOLDER[0]}
- echo "Building $folder"
- echo $SUBNAME
+ echo "Building $folder..."
  docker buildx build "$folder" --push \
  --tag $REPO_NAME/$PROJECT_NAME-$SUBNAME:latest \
  --tag $REPO_NAME/$PROJECT_NAME-$SUBNAME:${{ steps.get_tag.outputs.IMAGE_TAG }} \
- --platform linux/arm64,linux/amd64 
+ --platform "${PLATFORMS:-linux/amd64}"
  done

axis-ptz/camera.py

@@ -619,7 +619,7 @@ def moveCamera(ip, username, password, mqtt_client):
  get_jpeg_request()
  capture_metadata = get_json_request()
  mqtt_client.publish(
- "skyscan/captures/data",
+ publish_topic,
  json.dumps(capture_metadata),
  0,
  False
@@ -816,6 +816,7 @@ def main():
  global cameraConfig
  global flight_topic
  global object_topic
+ global publish_topic
  global logging_directory
  global Active
 
@@ -892,6 +893,12 @@ def main():
  action="store_true", 
  help="Verbose output"
  )
+ parser.add_argument(
+ "--publish-topic",
+ type=str,
+ help="The topic to publish capture information to",
+ default="skyscan/captures/data"
+ )
  parser.add_argument(
  "-f",
  "--flat-file-structure",
@@ -956,6 +963,7 @@ def main():
 
  flight_topic = args.mqtt_flight_topic
  object_topic = args.mqtt_object_topic
+ publish_topic = args.publish_topic
  print(
  "connecting to MQTT broker at "
  + args.mqtt_host

docker-compose.yml

@@ -2,49 +2,62 @@ version: '3.7'
 
 services:
  tracker:
- image: iqtlabs/skyscan-tracker
- command: "./flighttracker.py -m mqtt -H piaware -l ${LAT} -L ${LONG} -a ${ALT} -P skyscan/planes/json -T skyscan/flight/json -M ${MIN_ELEVATION} -c ${CAMERA_LEAD}"
+ image: iqtlabs/skyscan-tracker:${SERVICES_VERSION:-latest}
+ command: "./flighttracker.py -m mqtt -H piaware -l ${LAT} -L ${LONG} -a ${ALT} -P skyscan/planes/json -T skyscan/${HOSTNAME}/flight/tracker/JSON -M ${MIN_ELEVATION} -c ${CAMERA_LEAD}"
  volumes:
  - ./data:/data
- ports:
- - 5000:5000
- depends_on:
- - mqtt
+ env_file:
+ - .env
+ deploy:
+ resources:
+ limits:
+ cpus: '1.0'
  restart: unless-stopped
 
  axis-ptz:
- image: iqtlabs/skyscan-axis-ptz
- command: "./camera.py -m mqtt -t skyscan/flight/json -u ${AXIS_USERNAME} -p ${AXIS_PASSWORD} -a ${AXIS_IP} -z ${CAMERA_ZOOM} -s ${CAMERA_MOVE_SPEED} -d ${CAMERA_DELAY} --lat ${LAT} --lon ${LONG} --alt ${ALT} --roll ${ROLL} --pitch ${PITCH} --yaw ${YAW}"
+ image: iqtlabs/edgetech-axis-ptz-controller
+ command: "./camera.py --publish-topic skyscan/${HOSTNAME}/image/edgetech-axis-ptz/JSON -f -m mqtt -t skyscan/${HOSTNAME}/flight/tracker/JSON -u ${AXIS_USERNAME} -p ${AXIS_PASSWORD} -a ${AXIS_IP} -z ${CAMERA_ZOOM} -s ${CAMERA_MOVE_SPEED} -d ${CAMERA_DELAY} --lat ${LAT} --lon ${LONG} --alt ${ALT} --roll ${ROLL} --pitch ${PITCH} --yaw ${YAW}"
  volumes:
  - /flash/raw:/app/capture
+ - /flash/processed/log:/flash/processed/log
  depends_on:
  - mqtt
- restart: unless-stopped
-
- piaware:
- image: iqtlabs/skyscan-piaware
- tty: true
- container_name: piaware
- devices:
- - /dev/bus/usb:/dev/bus/usb
- ports:
- - 8080:80
- - 30003:30003
- - 30005:30005
- environment:
- - TZ=${TZ}
- - LAT=${LAT}
- - LONG=${LONG}
- - FEEDER_ID=${FEEDER_ID}
- - RECEIVER_TYPE=rtlsdr
- - DUMP1090_DEVICE=${RTL_DEV}
+ env_file:
+ - .env
+ deploy:
+ resources:
+ limits:
+ cpus: '1.0'
  restart: unless-stopped
 
- notebook:
- image: iqtlabs/skyscan-notebook-server
- ports:
- - "8888:8888"
+ coral-detect:
+ image: iqtlabs/skyscan-coral-detect
+ volumes:
+ - /flash/raw:/data/tosort
+ - /flash/coral/plane:/data/plane
+ - /flash/coral/noplane:/data/noplane
+ - /flash/coral/log:/data/log
+ - /flash/weights:/data/weights
+ deploy:
+ resources:
+ limits:
+ cpus: '1.0'
  restart: unless-stopped
+
+ edge-detect:
+ image: iqtlabs/skyscan-yolo-detect
+ volumes:
+ - /flash/coral/plane:/data/tosort
+ - /flash/edge/plane:/data/plane
+ - /flash/edge/log:/data/log
+ - /flash/edge/noplane:/data/noplane
+ - /flash/weights:/data/weights
+ deploy:
+ resources:
+ limits:
+ cpus: '1.0'
+ env_file:
+ - .env
  depends_on:
  - mqtt
 
@@ -53,14 +66,8 @@ services:
  ports:
  - "9001:9001"
  - "1883:1883"
- restart: unless-stopped
-
- egi:
- image: iqtlabs/skyscan-egi
- entrypoint: bash -c "gpsd ${GPS_SERIAL} -F /var/run/gpsd.sock && python3 egi_mqtt.py -m mqtt -l ${LAT} -L ${LONG} -a ${ALT} -r ${ROLL} -p ${PITCH} -y ${YAW}"
- devices:
- - /dev/ttyACM0
- privileged: true
- depends_on:
- - mqtt
- restart: unless-stopped
+ deploy:
+ resources:
+ limits:
+ cpus: '1.0'
+ restart: unless-stopped

install-skyscan.sh

@@ -0,0 +1,38 @@
+# Set environment variables
+export COMPOSE_FILE_URL=https://raw.githubusercontent.com/meadej/SkyScan/main/docker-compose.yml
+export ENV_FILE_URL=https://raw.githubusercontent.com/meadej/SkyScan/main/.env-example
+
+echo "Installing skyscan on ${HOSTNAME} at $(pwd)"
+
+echo "Configuring Mobian base"
+bash <(curl -fsSL https://short.iqt.org/pinephonepro)
+
+
+mkdir skyscan
+cd skyscan
+mkdir data
+cd data
+curl -O https://opensky-network.org/datasets/metadata/aircraftDatabase.csv
+cd ..
+
+# Make necessary folders
+mkdir raw
+mkdir coral
+mkdir coral/plane
+mkdir coral/noplane
+mkdir coral/log
+mkdir weights
+mkdir edge
+mkdir edge/plane
+mkdir edge/noplane
+mkdir edge/log
+mkdir processed
+mkdir processed/log
+
+curl -O $COMPOSE_FILE_URL
+curl -O $ENV_FILE_URL
+
+docker-compose pull
+
+echo "Installation complete. Run `docker-compose up` to start the system"
+echo "Before running, ensure you have replaced the demo weights with your own trained weights"