Add AWS Rekognition to thumbnailer (pulumi#54)

* New version of thumbnailer that uses AWS Rekognition

README.md

@@ -59,6 +59,8 @@ This example features an end-to-end pipeline for generating keyframe thumbnails
 containerized [FFmpeg](https://www.ffmpeg.org/). It combines containers, serverless functions, and cloud storage into
 a single 40-line application using `@pulumi/cloud-aws`.
 
+[An extension of this sample](cloud-js-thumbnailer-machine-learning/) uses AWS Rekognition to find the timestamp with the highest confidence for a particular label.
+
 #### [Raw AWS Serverless](aws-ts-serverless-raw/)
 
 This example deploys a complete serverless C# application using raw `aws.apigateway.RestAPI`, `aws.lambda.Function` and

cloud-js-thumbnailer-machine-learning/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: video-thumbnailer-rekognition
+runtime: nodejs
+description: A video thumbnail extractor using serverless functions, containers, and AWS Rekognition

cloud-js-thumbnailer-machine-learning/README.md

@@ -0,0 +1,138 @@
+# Video Thumbnailer with AWS Rekognition
+
+A video thumbnail extractor using serverless functions, containers, and [AWS Rekognition](https://aws.amazon.com/rekognition/). This is an extension of the sample [cloud-js-thumbnailer](../cloud-js-thumbnailer). When a new video is uploaded to S3, this sample calls AWS Rekognition to find a frame with the highest confidence for the label "cat" and extracts a jpg of this frame, by running ffmpeg in an AWS Fargate container.
+
+![When a new video is uploaded, extract a thumbnail using AWS Rekognition](thumbnailer-rekognition-diagram.png)
+
+## Prerequisites
+
+To use this example, make sure [Docker](https://docs.docker.com/engine/installation/) is installed and running.
+
+## Running the App
+
+1. Create a new stack:
+
+ ```
+ $ pulumi stack init thumbnailer-rekognition
+ ```
+
+1. Configure Pulumi to use AWS Fargate, which is currently only available in `us-east-1`, `us-west-2`, and `eu-west-1`:
+
+ ```
+ $ pulumi config set aws:region us-west-2
+ $ pulumi config set cloud-aws:useFargate true
+ ```
+
+1. Configure the Lambda function role so that it can access Rekognition:
+
+ ```
+ $ pulumi config set cloud-aws:computeIAMRolePolicyARNs arn:aws:iam::aws:policy/AWSLambdaFullAccess,arn:aws:iam::aws:policy/AmazonEC2ContainerServiceFullAccess,arn:aws:iam::aws:policy/AmazonRekognitionFullAccess
+ ```
+
+1. Restore NPM modules via `npm install`.
+
+1. Preview and deploy the app via `pulumi update`. The preview will take some time, as it builds a Docker container. A total of 44 resources are created.
+
+ ```
+ $ pulumi update
+ Previewing update of stack 'donna-thumbnailer-rekognition'
+ ...
+
+ Performing changes:
+
+ Type Name Status Info
+ * global global unchanged 1 info message. info: Building container image 'p
+ + pulumi:pulumi:Stack video-thumbnailer-rekognition created 1 info message. info: 88888b9b1b5b: Pushed
+ + ├─ aws-infra:network:Network default-vpc created
+ + ├─ aws-infra:network:Network default-vpc created
+ + ├─ cloud:global:infrastructure global-infrastructure created
+ + ├─ cloud:global:infrastructure global-infrastructure created
+ + │ ├─ aws:iam:Role pulumi-donna-t-execution created
+ + ├─ cloud:global:infrastructure global-infrastructure created
+ + ├─ cloud:global:infrastructure global-infrastructure created
+ + ├─ cloud:global:infrastructure global-infrastructure created
+ + ├─ cloud:bucket:Bucket bucket created
+ + │ ├─ cloud:function:Function onNewVideo created
+ + │ │ └─ aws:serverless:Function onNewVideo created
+ + │ │ └─ aws:serverless:Function onNewVideo created
+ + │ │ └─ aws:serverless:Function onNewVideo created
+ + │ │ └─ aws:serverless:Function onNewVideo created
+ + │ │ └─ aws:serverless:Function onNewVideo created
+ + │ ├─ cloud:function:Function onNewThumbnail created
+ + │ │ └─ aws:serverless:Function onNewThumbnail created
+ + │ │ └─ aws:serverless:Function onNewThumbnail created
+ + │ │ ├─ aws:iam:RolePolicyAttachment onNewThumbnail-32be53a2 created
+ + │ │ ├─ aws:iam:RolePolicyAttachment onNewThumbnail-fd1a00e5 created
+ + │ │ └─ aws:lambda:Function onNewThumbnail created
+ + │ ├─ aws:s3:Bucket bucket created
+ + │ ├─ aws:lambda:Permission onNewVideo created
+ + │ ├─ aws:lambda:Permission onNewThumbnail created
+ + │ └─ aws:s3:BucketNotification bucket created
+ + ├─ cloud:topic:Topic AmazonRekognitionTopic created
+ + │ └─ aws:sns:Topic AmazonRekognitionTopic created
+ + ├─ aws:iam:Role rekognition-role created
+ + ├─ cloud:function:Function AmazonRekognitionTopic_labelResults created
+ + │ └─ aws:serverless:Function AmazonRekognitionTopic_labelResults created
+ + │ ├─ aws:iam:Role AmazonRekognitionTopic_labelResults created
+ + │ ├─ aws:iam:RolePolicyAttachment AmazonRekognitionTopic_labelResults-32be53a2 created
+ + │ ├─ aws:iam:RolePolicyAttachment AmazonRekognitionTopic_labelResults-fd1a00e5 created
+ + │ └─ aws:lambda:Function AmazonRekognitionTopic_labelResults created
+ + ├─ cloud:task:Task ffmpegThumbTask created
+ + │ ├─ aws:cloudwatch:LogGroup ffmpegThumbTask created
+ + │ └─ aws:ecs:TaskDefinition ffmpegThumbTask created
+ + ├─ aws-infra:cluster:Cluster pulumi-donna-thum-global created
+ + │ ├─ aws:ecs:Cluster pulumi-donna-thum-global created
+ + │ └─ aws:ec2:SecurityGroup pulumi-donna-thum-global created
+ + ├─ aws:iam:RolePolicyAttachment rekognition-access created
+ + ├─ aws:lambda:Permission AmazonRekognitionTopic_labelResults created
+ + └─ aws:sns:TopicSubscription AmazonRekognitionTopic_labelResults created
+
+ ...
+ ---outputs:---
+ bucketName: "bucket-d6c6339"
+
+ info: 44 changes performed:
+ + 44 resources created
+ Update duration: 2m27.112988339s
+
+ Permalink: https://pulumi.com/pulumi/donna-thumbnailer-rekognition/updates/1
+ ```
+
+1. Upload a video:
+
+ ```
+ $ aws s3 cp ./sample/cat.mp4 s3:https://$(pulumi stack output bucketName)
+ upload: sample/cat.mp4 to s3:https://bucket-c647dfb/cat.mp4
+ ```
+
+1. View the logs from both the Lambda function and the ECS task:
+
+ ```
+ $ pulumi logs -f
+ Collecting logs for stack pulumi/donna-thumbnailer-rekognition since 2018-05-21T18:57:11.000-07:00.
+ 2018-05-21T19:57:35.968-07:00[ onNewVideo] *** New video: file cat.mp4 was uploaded at 2018-05-22T02:57:35.431Z.
+ 2018-05-21T19:57:36.376-07:00[ onNewVideo] *** Submitted Rekognition job for cat.mp4
+ 2018-05-21T19:57:45.848-07:00[AmazonRekognitionTopic_labelRe] *** Rekognition job complete
+ 2018-05-21T19:57:50.690-07:00[AmazonRekognitionTopic_labelRe] Raw label results: 
+ ...
+ 2018-05-21T19:57:50.746-07:00[AmazonRekognitionTopic_labelRe] *** Found object Cat at position 1568. Confidence = 50.56669616699219
+ 2018-05-21T19:57:50.746-07:00[AmazonRekognitionTopic_labelRe] *** Rekognition processing complete for bucket-d6c6339/cat.mp4 at timestamp 1.568
+ 2018-05-21T19:57:51.762-07:00[AmazonRekognitionTopic_labelRe] *** Launched thumbnailer task.
+ 2018-05-21T19:58:55.197-07:00[ ffmpegThumbTask] Starting ffmpeg task...
+ 2018-05-21T19:58:55.216-07:00[ ffmpegThumbTask] Copying from S3 bucket-d6c6339/cat.mp4 to cat.mp4 ...
+ download: s3:https://bucket-d6c6339/cat.mp4 to ./cat.mp4 pleted 256.0 KiB/756.1 KiB (2.4 MiB/s) with 1 file(s) remaining
+ 2018-05-21T19:59:02.244-07:00[ ffmpegThumbTask] Copying .jpg to S3 at bucket-d6c6339/.jpg ...
+ upload: ./.jpg to s3:https://bucket-d6c6339/output/.jpg pleted 87.3 KiB/87.3 KiB (428.8 KiB/s) with 1 file(s) remaining
+ 2018-05-21T19:59:05.778-07:00[ onNewThumbnail] *** New thumbnail: file cat.jpg was saved at 2018-05-22T02:59:04.858Z.
+ ```
+
+1. Download the key frame:
+
+ ```
+ $ aws s3 cp s3:https://$(pulumi stack output bucketName)/cat.jpg .
+ download: s3:https://bucket-0e25c2d/cat.jpg to ./cat.jpg 
+ ```
+
+## Clean up
+
+To clean up resources, run `pulumi destroy` and answer the confirmation question at the prompt.

cloud-js-thumbnailer-machine-learning/docker-ffmpeg-thumb/Dockerfile

@@ -0,0 +1,17 @@
+FROM jrottenberg/ffmpeg
+
+RUN apt-get update && \
+ apt-get install python-dev python-pip -y && \
+ apt-get clean
+
+RUN pip install awscli
+
+WORKDIR /tmp/workdir
+
+COPY copy_thumb.sh /tmp/workdir
+COPY copy_video.sh /tmp/workdir
+
+ENTRYPOINT echo "Starting ffmpeg task..." && \
+ ./copy_video.sh && \
+ ffmpeg -v error -i ./${INPUT_VIDEO_FILE_NAME} -ss ${POSITION_TIME_DURATION} -vframes 1 -f image2 -an -y ${OUTPUT_THUMBS_FILE_NAME} && \
+ ./copy_thumb.sh

cloud-js-thumbnailer-machine-learning/docker-ffmpeg-thumb/copy_thumb.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+echo "Copying ${OUTPUT_THUMBS_FILE_NAME} to S3 at ${S3_BUCKET}/${OUTPUT_THUMBS_FILE_NAME} ..."
+aws s3 cp ./${OUTPUT_THUMBS_FILE_NAME} s3:https://${S3_BUCKET}/${OUTPUT_THUMBS_FILE_NAME}

cloud-js-thumbnailer-machine-learning/docker-ffmpeg-thumb/copy_video.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+echo "Copying from S3 ${S3_BUCKET}/${INPUT_VIDEO_FILE_NAME} to ${INPUT_VIDEO_FILE_NAME} ..."
+aws s3 cp s3:https://${S3_BUCKET}/${INPUT_VIDEO_FILE_NAME} ./${INPUT_VIDEO_FILE_NAME}

cloud-js-thumbnailer-machine-learning/index.js

@@ -0,0 +1,51 @@
+// Copyright 2016-2018, Pulumi Corporation. All rights reserved.
+
+"use strict";
+
+const cloud = require("@pulumi/cloud-aws");
+const video = require("./video-label-processor");
+
+// A bucket to store videos and thumbnails.
+const bucket = new cloud.Bucket("bucket");
+const bucketName = bucket.bucket.id;
+
+// A task which runs a containerized FFMPEG job to extract a thumbnail image.
+const ffmpegThumbnailTask = new cloud.Task("ffmpegThumbTask", {
+ build: "./docker-ffmpeg-thumb",
+ memoryReservation: 128,
+});
+
+// Use module for processing video through Rekognition
+const videoProcessor = new video.VideoLabelProcessor();
+
+// When a new video is uploaded, start Rekognition label detection
+bucket.onPut("onNewVideo", async (bucketArgs) => { 
+ console.log(`*** New video: file ${bucketArgs.key} was uploaded at ${bucketArgs.eventTime}.`);
+ videoProcessor.startRekognitionJob(bucketName.get(), bucketArgs.key);
+}, { keySuffix: ".mp4" }); // run this Lambda only on .mp4 files
+
+// When Rekognition processing is complete, run the FFMPEG task on the video file
+// Use the timestamp with the highest confidence for the label "cat"
+videoProcessor.onLabelResult("cat", async (file, framePos) => {
+ console.log(`*** Rekognition processing complete for ${bucketName.get()}/${file} at timestamp ${framePos}`);
+ const thumbnailFile = file.substring(0, file.lastIndexOf('.')) + '.jpg';
+
+ // launch ffmpeg in a container, use environment variables to connect resources together
+ await ffmpegThumbnailTask.run({
+ environment: {
+ "S3_BUCKET": bucketName.get(),
+ "INPUT_VIDEO_FILE_NAME": file,
+ "POSITION_TIME_DURATION": framePos,
+ "OUTPUT_THUMBS_FILE_NAME": thumbnailFile,
+ },
+ });
+ console.log("*** Launched thumbnailer task.");
+});
+
+// When a new thumbnail is created, log a message.
+bucket.onPut("onNewThumbnail", async (bucketArgs) => {
+ console.log(`*** New thumbnail: file ${bucketArgs.key} was saved at ${bucketArgs.eventTime}.`);
+}, { keySuffix: ".jpg" });
+
+// Export the bucket name.
+exports.bucketName = bucketName;

cloud-js-thumbnailer-machine-learning/package.json

@@ -0,0 +1,10 @@
+{
+ "name": "video-thumbnailer-rekognition",
+ "version": "0.1.0",
+ "main": "index.js",
+ "dependencies": {
+ "@pulumi/cloud-aws": "^0.13.0",
+ "@pulumi/aws": "^0.13.0",
+ "aws-sdk": "^2.238.1"
+ }
+}

cloud-js-thumbnailer-machine-learning/video-label-processor.js

@@ -0,0 +1,108 @@
+"use strict";
+
+const cloud = require("@pulumi/cloud-aws");
+const aws = require("@pulumi/aws");
+
+class VideoLabelProcessor {
+
+ constructor() {
+ let topic = new cloud.Topic("AmazonRekognitionTopic");
+ let topicArn = topic.topic.arn;
+ let role = createRekognitionRole();
+
+ this.startRekognitionJob = (bucketName, filename) => { 
+ var aws = require('aws-sdk');
+ var rekognition = new aws.Rekognition();
+
+ var params = {
+ Video: { 
+ S3Object: {
+ Bucket: bucketName,
+ Name: filename,
+ }
+ },
+ NotificationChannel: {
+ RoleArn: role.arn.get(),
+ SNSTopicArn: topicArn.get(),
+ }
+ };
+
+ rekognition.startLabelDetection(params, (err, data) => {
+ if (!err) {
+ console.log(`*** Submitted Rekognition job for ${filename}`);
+ }
+ else console.log(err, err.stack); 
+ });
+ }
+
+ this.onLabelResult = (searchLabel, action) => {
+ topic.subscribe("labelResults", (jobStatus) => { 
+ console.log("*** Rekognition job complete");
+
+ if (jobStatus.Status == 'SUCCEEDED' && jobStatus.API == 'StartLabelDetection') {
+ var aws = require('aws-sdk');
+ var rekognition = new aws.Rekognition();
+
+ rekognition.getLabelDetection( { JobId: jobStatus.JobId }, 
+ function (err, data) {
+ if (!err) {
+ let timestamp = getTimestampForLabel(data.Labels, searchLabel).toString();
+
+ // call callback to process the video at a timestamp
+ action(jobStatus.Video.S3ObjectName, timestamp);
+ }
+ else console.log(err, err.stack); 
+ }
+ );
+ } 
+ });
+ }
+ }
+}
+
+function getTimestampForLabel(labels, filterName) {
+ console.log(`Raw label results: ${JSON.stringify(labels)}`);
+
+ let bestTimestamp = 0;
+ let highestConfidence = 0;
+
+ labels.forEach(element => {
+ if (element.Label.Name.toLowerCase() == filterName.toLowerCase() && 
+ element.Label.Confidence > highestConfidence) {
+ highestConfidence = element.Label.Confidence;
+ bestTimestamp = element.Timestamp;
+ console.log(` *** Found object ${element.Label.Name} at position ${bestTimestamp}. Confidence = ${highestConfidence}`);
+ }
+ });
+
+ return bestTimestamp / 1000; // convert to milliseconds 
+}
+
+function createRekognitionRole() {
+ let policy = {
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Principal": {
+ "Service": "rekognition.amazonaws.com",
+ },
+ "Effect": "Allow",
+ "Sid": "",
+ },
+ ],
+ };
+
+ let role = new aws.iam.Role("rekognition-role", {
+ assumeRolePolicy: JSON.stringify(policy),
+ });
+
+ let serviceRoleAccess = new aws.iam.RolePolicyAttachment("rekognition-access", {
+ role: role,
+ policyArn: "arn:aws:iam::aws:policy/service-role/AmazonRekognitionServiceRole", // use managed AWS policy
+ });
+
+ return role;
+}
+
+module.exports.VideoLabelProcessor = VideoLabelProcessor;

cloud-js-thumbnailer/README.md

@@ -122,7 +122,7 @@ To run this example, make sure [Docker](https://docs.docker.com/engine/installat
 
  ```
  $ aws s3 cp s3:https://$(pulumi stack output bucketName)/cat.jpg .
- download: s3:https://bucket-0e25c2d/cat.png to ./cat.jpg 
+ download: s3:https://bucket-0e25c2d/cat.jpg to ./cat.jpg 
  ```
 
 ## Clean up