Py deps upgrade, TF upgrade, TF fixes

cli/config.py

@@ -58,6 +58,7 @@ class BinaryAlertConfig:
  """Wrapper around reading, validating, and updating the terraform.tfvars config file."""
  # Expected configuration value formats.
  VALID_AWS_ACCOUNT_ID_FORMAT = r'\d{12}'
+ VALID_AWS_ACCOUNT_NAME_FORMAT = r'[a-z]+\/?[a-z]+'
  VALID_AWS_REGION_FORMAT = r'[a-z]{2}-[a-z]{2,15}-\d'
  VALID_NAME_PREFIX_FORMAT = r'[a-z][a-z0-9_]{3,50}'
  VALID_CB_API_TOKEN_FORMAT = r'[a-f0-9]{40}' # CarbonBlack API token.
@@ -96,6 +97,19 @@ def aws_account_id(self, value: str) -> None:
  )
  self._config['aws_account_id'] = value
 
+ @property
+ def aws_account_name(self) -> str:
+ return self._config['aws_account_name']
+
+ @aws_account_name.setter
+ def aws_account_name(self, value: str) -> None:
+ if not re.fullmatch(self.VALID_AWS_ACCOUNT_NAME_FORMAT, value, re.ASCII):
+ raise InvalidConfigError(
+ 'aws_account_name "{}" does not match format {}'.format(
+ value, self.VALID_AWS_ACCOUNT_NAME_FORMAT)
+ )
+ self._config['aws_account_name'] = value
+
  @property
  def aws_region(self) -> str:
  return self._config['aws_region']
@@ -260,6 +274,7 @@ def configure(self) -> None:
  Each request will be retried until the answer is in the correct format.
  """
  get_input('AWS Account ID', self.aws_account_id, self, 'aws_account_id')
+ get_input('AWS Account Name', self.aws_account_name, self, 'aws_account_name')
  get_input('AWS Region', self.aws_region, self, 'aws_region')
  get_input('Unique name prefix, e.g. "company_team"', self.name_prefix, self, 'name_prefix')
  enable_downloader = get_input('Enable the CarbonBlack downloader?',
@@ -285,6 +300,7 @@ def validate(self) -> None:
  """
  # Go through the internal setters which have the validation logic.
  self.aws_account_id = self.aws_account_id
+ self.aws_account_name = self.aws_account_name
  self.aws_region = self.aws_region
  self.name_prefix = self.name_prefix
  self.enable_carbon_black_downloader = self.enable_carbon_black_downloader

docs/source/conf.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # BinaryAlert documentation build configuration file, created by

lambda_functions/analyzer/Dockerfile

@@ -0,0 +1,117 @@
+FROM amazonlinux:latest
+
+ENV APP_DIR /var/task
+ENV LAMBDA_DIR ${APP_DIR}/lambda
+ENV PIP_DIR $APP_DIR/pip
+
+ENV CRYPTO_VER 3.3.1
+ENV ASN1CRYPTO_VER 1.4.0
+ENV YARA_VER 4.0.0
+ENV UPX_VER 3.94
+ENV YEXTEND_VER 1.6
+ENV ARCH amd64
+
+VOLUME /var/out
+WORKDIR $APP_DIR
+
+RUN yum -y update \
+ && yum -y install \
+ git \
+ zip \
+ xz \
+ wget \
+ tar \
+ install \
+ autoconf \
+ automake \
+ pkgconfig \
+ bzip2-devel \
+ gcc \
+ gcc-c++ \
+ libarchive-devel \
+ libffi-devel \
+ libtool \
+ libuuid-devel \
+ make \
+ openssl-devel \
+ pcre-devel \
+ poppler-utils \
+ poppler-cpp-devel \
+ zlib-devel \
+ python3 \
+ python3-pip \
+ python3-devel \
+ && yum clean all \
+ && pip3 install nose
+
+# install Yara
+RUN wget -O yara.tar.gz https://github.com/VirusTotal/yara/archive/v${YARA_VER}.tar.gz \
+ && tar -xzf yara.tar.gz \
+ && cd yara-${YARA_VER} \
+ && ./bootstrap.sh \
+ && ./configure \
+ && make \
+ && make check \
+ && make install \
+ && echo "/usr/local/lib" > /etc/ld.so.conf.d/yara.conf \
+ && ldconfig \
+ && yara -v
+
+# Install Python pacakges
+RUN pip3 install \
+ "cryptography==${CRYPTO_VER}" \
+ "yara-python==${YARA_VER}" \
+ "asn1crypto==${ASN1CRYPTO_VER}" \
+ -t $PIP_DIR
+
+# Copy crypto libs
+RUN cd $PIP_DIR \
+ && rm -r *.dist-info *.egg-info \
+ && find . -name __pycache__ | xargs rm -rf \
+ && mv _cffi_backend.cpython-*m-x86_64-linux-gnu.so _cffi_backend.so \
+ && cd cryptography/hazmat/bindings \
+ && mv _openssl.abi3.so _openssl.so \
+ && mv _padding.abi3.so _padding.so
+
+# Gather pip files
+RUN mkdir ${LAMBDA_DIR} \
+ && cp -r ${PIP_DIR}/* ${LAMBDA_DIR}
+
+# Compile yextend
+RUN mkdir ${LAMBDA_DIR}/libs \
+ && git clone https://github.com/BayshoreNetworks/yextend.git \
+ && cd yextend \
+ && git checkout yara-${YARA_VER} \
+ && mv configure.ac configure.ac.orig \
+ && sed 's/python \-c/python3 -c/g' configure.ac.orig > configure.ac \
+ && ./build.sh \
+ && make unittests \
+ && cp yextend ${LAMBDA_DIR} \
+ && cp libs/*.o ${LAMBDA_DIR}/libs \
+ && cp libs/*.yara ${LAMBDA_DIR}/libs
+
+# UPX
+RUN wget https://github.com/upx/upx/releases/download/v${UPX_VER}/upx-${UPX_VER}-${ARCH}_linux.tar.xz \
+ && tar -xf upx-${UPX_VER}-${ARCH}_linux.tar.xz \
+ && cp upx-${UPX_VER}-${ARCH}_linux/upx ${LAMBDA_DIR}
+
+## Gather compiled libraries
+RUN cp /usr/bin/pdftotext ${LAMBDA_DIR} \
+ && mkdir tmplib \
+ && find \
+ /usr/lib64/lib* \
+ -depth \
+ -type f \
+ -o \
+ -type d \
+ -name 'lib*.so.*' | \
+ grep -E 'lib(archive|fontconfig|jbig|jpeg|lcms|lzma|lzo2|openjpeg|pcrecpp|png|poppler|stdc|tiff|xml)' | \
+ cpio -pamVd tmplib \
+ && mv tmplib/usr/lib64/* ${LAMBDA_DIR} \
+ && cp /usr/local/lib/libyara.so.* ${LAMBDA_DIR}
+
+RUN cd ${LAMBDA_DIR} \
+ && zip -r dependencies.zip * \
+ && mv ${LAMBDA_DIR}/dependencies.zip ${APP_DIR}/dependencies.zip
+
+CMD cp ${APP_DIR}/dependencies.zip /var/out/

lambda_functions/analyzer/Makefile

@@ -0,0 +1,3 @@
+all:
+ docker build -t binaryalert-lambda .
+ docker run --rm -it -v ${PWD}:/var/out binaryalert-lambda

lambda_functions/analyzer/README.rst

@@ -1,111 +1,20 @@
 YARA Analyzer
 =============
-This Lambda function is the core of BinaryAlert. Each invocation downloads one or more binaries from
-S3, scans them against all available YARA rules, and forwards any matches to Dynamo and SNS.
+This Lambda function is the core of BinaryAlert. Each invocation downloads one
+or more binaries from S3, scans them against all available YARA rules, and
+forwards any matches to Dynamo and SNS.
 
 
 Updating YARA Binaries
 ----------------------
-Many libraries used by BinaryAlert are natively compiled, and must therefore be pre-built on an
-Amazon Linux AMI in order to run in Lambda. This has already been done for you:
-``dependencies.zip`` contains the following pre-built libraries:
+Many libraries used by BinaryAlert are natively compiled, and must therefore be
+pre-built on an Amazon Linux AMI in order to run in Lambda. This has already
+been done for you in the `dependencies.zip` file that ships with the repo, but you
+can rebuild it yourself via Docker.
 
-- `cryptography <https://cryptography.io>`_ (v2.3)
-- `UPX <https://github.com/upx/upx>`_ (v3.94)
-- `yara-python <https://github.com/VirusTotal/yara-python>`_ (v3.8.0)
- - `yara <https://github.com/VirusTotal/yara>`_ (v3.8.0)
-- `yextend <https://github.com/BayshoreNetworks/yextend>`_ (v1.6)
- - `pdftotext <https://poppler.freedesktop.org/>`_ (v0.26.5)
-
-If, however, you need to update or re-create the zipfile, SSH to an EC2 instance running the
-`AWS Lambda AMI <https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html>`_
-and install the dependencies as follows:
+If you need to update or re-create the ZIP file, do it before deployment.
 
 .. code-block:: bash
+ $ make
 
- # Install requirements
- sudo yum update
- sudo yum install autoconf automake bzip2-devel gcc64 gcc64-c++ libarchive-devel libffi-devel \
- libtool libuuid-devel openssl-devel pcre-devel poppler-utils python36 python36-devel zlib-devel
- sudo pip install nose
-
- # Compile YARA
- wget https://github.com/VirusTotal/yara/archive/v3.8.0.tar.gz
- tar -xzf v3.8.0.tar.gz
- cd yara-3.8.0
- ./bootstrap.sh
- ./configure
- make
- make check # Run unit tests
- sudo make install
-
- # Install cryptography and yara-python
- cd ~
- mkdir pip
- pip-3.6 install cryptography yara-python -t pip
-
- # Compile yextend
- wget https://github.com/BayshoreNetworks/yextend/archive/1.6.tar.gz
- tar -xvzf 1.6.tar.gz
- cd yextend-1.6
- # Manually: modify main.cpp, line 473 to hardcode the yara version to 3.8
- ./build.sh
- make unittests # Run unit tests
-
- # Clean cryptography files
- cd ~/pip
- rm -r *.dist-info *.egg-info
- find . -name __pycache__ | xargs rm -r
- mv _cffi_backend.cpython-36m-x86_64-linux-gnu.so _cffi_backend.so
- cd cryptography/hazmat/bindings
- mv _constant_time.abi3.so _constant_time.so
- mv _openssl.abi3.so _openssl.so
- mv _padding.abi3.so _padding.so
-
- # Gather pip files
- cd ~
- mkdir lambda
- cp pip/.libs_cffi_backend/* lambda
- cp -r pip/* lambda
- mv lambda/yara.cpython-36m-x86_64-linux-gnu.so lambda/yara.so
- wget https://raw.githubusercontent.com/VirusTotal/yara/master/COPYING -O lambda/YARA_LICENSE
- wget https://raw.githubusercontent.com/VirusTotal/yara-python/master/LICENSE -O lambda/YARA_PYTHON_LICENSE
-
- # Gather Yextend files
- cp yextend-1.6/yextend lambda
- cp yextend-1.6/LICENSE lambda/YEXTEND_LICENSE
- mkdir lambda/libs
- cp yextend-1.6/libs/*.o lambda/libs
- cp yextend-1.6/libs/*.yara lambda/libs
-
- # Download UPX
- wget https://github.com/upx/upx/releases/download/v3.94/upx-3.94-amd64_linux.tar.xz
- tar -xf upx-3.94-amd64_linux.tar.xz
- cp upx-3.94-amd64_linux/upx lambda
- cp upx-3.94-amd64_linux/COPYING lambda/UPX_LICENSE
-
- # Gather compiled libraries
- cp /usr/bin/pdftotext lambda
- cp /usr/lib64/libarchive.so.13 lambda
- cp /usr/lib64/libfontconfig.so.1 lambda
- cp /usr/lib64/libfreetype.so.6 lambda
- cp /usr/lib64/libjbig.so.2.0 lambda
- cp /usr/lib64/libjpeg.so.62 lambda
- cp /usr/lib64/liblcms2.so.2 lambda
- cp /usr/lib64/liblzma.so.5 lambda
- cp /usr/lib64/liblzo2.so.2 lambda
- cp /usr/lib64/libopenjpeg.so.2 lambda
- cp /usr/lib64/libpcrecpp.so.0 lambda
- cp /usr/lib64/libpng12.so.0 lambda
- cp /usr/lib64/libpoppler.so.46 lambda
- cp /usr/lib64/libstdc++.so.6 lambda
- cp /usr/lib64/libtiff.so.5 lambda
- cp /usr/lib64/libxml2.so.2 lambda
- cp /usr/local/lib/libyara.so.3 lambda
-
- # Build Zipfile
- cd lambda
- zip -r dependencies.zip *
-
-
-Then ``scp`` the ``dependencies.zip`` package to replace the one in the repo.
+And you'll find the new `dependencies.zip` file in this folder.

manage.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 """Command-line tool for easily managing BinaryAlert."""
 import argparse
 import os