Create pure AWS version of the thumbnailer example. (pulumi#210)

aws-ts-thumbnailer/Pulumi.yaml

@@ -0,0 +1,8 @@
+name: video-thumbnailer
+runtime: nodejs
+description: A video thumbnail extractor using serverless functions and containers
+template:
+ config:
+ aws:region:
+ description: The AWS region to deploy into
+ default: us-west-2

aws-ts-thumbnailer/README.md

@@ -0,0 +1,104 @@
+[![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new)
+
+# Video Thumbnailer
+
+A video thumbnail extractor using serverless functions and containers.
+
+Loosely derived from the example at https://serverless.com/blog/serverless-application-for-long-running-process-fargate-lambda/.
+
+![When a new video is uploaded, extract a thumbnail](thumbnailer-diagram.png)
+
+## Prerequisites
+
+To run this example, make sure [Docker](https://docs.docker.com/engine/installation/) is installed and running.
+
+## Running the App
+
+Note: some values in this example will be different from run to run. These values are indicated
+with `***`.
+
+1. Create a new stack:
+
+ ```
+ $ pulumi stack init thumbnailer-testing
+ ```
+
+1. Configure Pulumi to use an AWS region where Fargate is supported, which is currently only available in `us-east-1`, `us-east-2`, `us-west-2`, and `eu-west-1`:
+
+ ```
+ $ pulumi config set aws:region us-west-2
+ ```
+
+1. Restore NPM modules via `npm install` or `yarn install`.
+
+1. Preview and deploy the app via `pulumi up`. The preview will take some time, as it builds a Docker container. A total of 32 resources are created.
+
+ ```
+ $ pulumi up
+ Previewing update of stack 'thumbnailer-testing'
+ Previewing changes:
+
+ Type Name Plan Info
+ * global global no change 1 info message. info: Building container image 'pulum-
+ + pulumi:pulumi:Stack video-thumbnailer-thumbnailer-testing create... 1 info message. info: Successfully tagged pulum-
+ ...
+
+ Do you want to perform this update? yes
+ Updating stack 'thumbnailer-testing'
+ Performing changes:
+
+ Type Name Status Info
+ * global global unchanged 1 info message. info: Building container image 'pulum-
+ + pulumi:pulumi:Stack video-thumbnailer-thumbnailer-testing created 1 info message. info: 081c66fa4b0c: Pushed
+ + ...
+ ...
+
+ info: 32 changes performed:
+ + 32 resources created
+ Update duration: ***
+
+ Permalink: https://app.pulumi.com/***
+ ```
+
+1. View the stack outputs:
+
+ ```
+ $ pulumi stack output
+ Current stack outputs (1):
+ OUTPUT VALUE
+ bucketName ***
+ ```
+
+1. Upload a video, embedding the timestamp in the filename:
+
+ ```
+ $ aws s3 cp ./sample/cat.mp4 s3:https://$(pulumi stack output bucketName)/cat_00-01.mp4
+ upload: sample/cat.mp4 to s3:https://***/cat_00-01.mp4
+ ```
+
+1. View the logs from both the Lambda function and the ECS task:
+
+ ```
+ $ pulumi logs -f
+ Collecting logs for stack thumbnailer-testing since ***
+
+ 2018-05-25T12:57:26.326-07:00[ onNewVideo] *** New video: file cat_00-01.mp4 was uploaded at 2018-05-25T19:57:25.507Z.
+ 2018-05-25T12:57:30.705-07:00[ onNewVideo] Running thumbnailer task.
+ 2018-05-25T12:58:34.960-07:00[ ffmpegThumbTask] Starting ffmpeg task...
+ 2018-05-25T12:58:34.960-07:00[ ffmpegThumbTask] Copying video from S3 bucket-5ea6b28/cat_00-01.mp4 to cat_00-01.mp4...
+ 2018-05-25T12:58:37.267-07:00[ ffmpegThumbTask] Completed 256.0 KiB/666.5 KiB (2.5 MiB/s) with 1 fildownload: s3:https://bucket-5ea6b28/cat_00-01.mp4 to ./cat_00-01.mp4
+ 2018-05-25T12:58:40.306-07:00[ ffmpegThumbTask] Copying cat.jpg to S3 at bucket-5ea6b28/cat.jpg ...
+ 2018-05-25T12:58:43.034-07:00[ ffmpegThumbTask] Completed 86.6 KiB/86.6 KiB (619.7 KiB/s) with 1 filupload: ./cat.jpg to s3:https://bucket-5ea6b28/cat.jpg
+ 2018-05-25T12:58:43.758-07:00[ onNewThumbnail] *** New thumbnail: file cat.jpg was saved at 2018-05-25T19:58:43.028Z.
+ ```
+
+1. Download the key frame:
+
+ ```
+ $ aws s3 cp s3:https://$(pulumi stack output bucketName)/cat.jpg .
+ download: s3:https://***/cat.jpg to ./cat.jpg
+ ```
+
+## Clean up
+
+To clean up resources, run `pulumi destroy` and answer the confirmation question at the prompt.

aws-ts-thumbnailer/docker-ffmpeg-thumb/Dockerfile

@@ -0,0 +1,17 @@
+FROM jrottenberg/ffmpeg
+
+RUN apt-get update && \
+ apt-get install python-dev python-pip -y && \
+ apt-get clean
+
+RUN pip install awscli
+
+WORKDIR /tmp/workdir
+
+ENTRYPOINT \
+ echo "Starting ffmpeg task..." && \
+ echo "Copying video from s3:https://${S3_BUCKET}/${INPUT_VIDEO} to ${INPUT_VIDEO}..." && \
+ aws s3 cp s3:https://${S3_BUCKET}/${INPUT_VIDEO} ./${INPUT_VIDEO} && \
+ ffmpeg -v error -i ./${INPUT_VIDEO} -ss ${TIME_OFFSET} -vframes 1 -f image2 -an -y ${OUTPUT_FILE} && \
+ echo "Copying thumbnail to S3:https://${S3_BUCKET}/${OUTPUT_FILE} ..." && \
+ aws s3 cp ./${OUTPUT_FILE} s3:https://${S3_BUCKET}/${OUTPUT_FILE}

aws-ts-thumbnailer/index.ts

@@ -0,0 +1,74 @@
+// Copyright 2016-2018, Pulumi Corporation. All rights reserved.
+
+import * as aws from "@pulumi/aws";
+import * as awsx from "@pulumi/aws-infra";
+
+// A simple cluster to run our tasks in.
+const cluster = awsx.ecs.Cluster.getDefault();
+
+// A bucket to store videos and thumbnails.
+const bucket = new aws.s3.Bucket("bucket");
+
+// Export the bucket name.
+export const bucketName = bucket.id;
+
+// A task which runs a containerized FFMPEG job to extract a thumbnail image.
+const ffmpegThumbnailTask = new awsx.ecs.FargateTaskDefinition("ffmpegThumbTask", {
+ container: {
+ image: awsx.ecs.Image.fromPath("./docker-ffmpeg-thumb"),
+ memoryReservation: 512,
+ },
+});
+
+// When a new video is uploaded, run the FFMPEG task on the video file.
+// Use the time index specified in the filename (e.g. cat_00-01.mp4 uses timestamp 00:01)
+bucket.onObjectCreated("onNewVideo", new aws.lambda.CallbackFunction<aws.s3.BucketEvent, void>("onNewVideo", {
+ // Specify appropriate policies so that this AWS lambda can run EC2 tasks.
+ policies: [
+ aws.iam.AWSLambdaFullAccess, // Provides wide access to "serverless" services (Dynamo, S3, etc.)
+ aws.iam.AmazonEC2ContainerServiceFullAccess, // Required for lambda compute to be able to run Tasks
+ ],
+ callback: async bucketArgs => {
+ console.log("onNewVideo called");
+ if (!bucketArgs.Records) {
+ return;
+ }
+
+ for (const record of bucketArgs.Records) {
+ console.log(`*** New video: file ${record.s3.object.key} was uploaded at ${record.eventTime}.`);
+ const file = record.s3.object.key;
+
+ const thumbnailFile = file.substring(0, file.indexOf('_')) + '.jpg';
+ const framePos = file.substring(file.indexOf('_')+1, file.indexOf('.')).replace('-',':');
+
+ await ffmpegThumbnailTask.run({
+ cluster,
+ overrides: {
+ containerOverrides: [{
+ name: "container",
+ environment: [
+ { name: "S3_BUCKET", value: bucketName.get() },
+ { name: "INPUT_VIDEO", value: file },
+ { name: "TIME_OFFSET", value: framePos },
+ { name: "OUTPUT_FILE", value: thumbnailFile },
+ ],
+ }],
+ },
+ });
+
+ console.log(`Running thumbnailer task.`);
+ }
+ },
+}), { filterSuffix: ".mp4" });
+
+// When a new thumbnail is created, log a message.
+bucket.onObjectCreated("onNewThumbnail", async bucketArgs => {
+ console.log("onNewThumbnail called");
+ if (!bucketArgs.Records) {
+ return;
+ }
+
+ for (const record of bucketArgs.Records) {
+ console.log(`*** New thumbnail: file ${record.s3.object.key} was saved at ${record.eventTime}.`);
+ }
+}, { filterSuffix: ".jpg" });

aws-ts-thumbnailer/package.json

@@ -0,0 +1,10 @@
+{
+ "name": "video-thumbnailer",
+ "version": "0.1.0",
+ "main": "index.js",
+ "dependencies": {
+ "@pulumi/pulumi": "^0.16.9",
+ "@pulumi/aws": "^0.16.5",
+ "@pulumi/aws-infra": "dev"
+ }
+}