From 59e1bdf32ea1a4a9be287906bd856eeddc8dd0f3 Mon Sep 17 00:00:00 2001 From: gsuess Date: Sun, 18 Jan 2015 15:22:34 +0100 Subject: [PATCH] #5 - Added JSON API integration for Google Cloud. There are some remaining CORS issues though. --- .jshintrc | 8 ++- CHANGELOG.md | 3 + README.md | 1 + lib/directive.js | 14 ++-- lib/oauth2.js | 107 +++++++++++++++++++++++++++++ lib/storage-policy.js | 1 - lib/upload.js | 10 ++- lib/utils.js | 17 +++++ package.js | 4 ++ services/aws-s3.js | 90 ++++++++++++++---------- services/google-cloud-resumable.js | 80 +++++++++++++++++++++ services/google-cloud.js | 6 +- services/rackspace.js | 3 +- versions.json | 8 +++ 14 files changed, 296 insertions(+), 56 deletions(-) create mode 100644 lib/oauth2.js create mode 100644 lib/utils.js create mode 100644 services/google-cloud-resumable.js diff --git a/.jshintrc b/.jshintrc index 765e3d1..edb35e1 100644 --- a/.jshintrc +++ b/.jshintrc @@ -74,7 +74,7 @@ "dojo" : false, // Dojo Toolkit "jquery" : false, // jQuery "mootools" : false, // MooTools - "node" : false, // Node.js + "node" : true, // Node.js "nonstandard" : false, // Widely adopted globals (escape, unescape, etc) "prototypejs" : false, // Prototype and Scriptaculous "rhino" : false, // Rhino @@ -124,8 +124,10 @@ "$": false, //Internals - "S3Policy": false, "Slingshot": true, - "matchAllowedFileTypes": false + "oauth": false, + "matchAllowedFileTypes": false, + "rsaSha256": false, + "hmac256": false } } diff --git a/CHANGELOG.md b/CHANGELOG.md index b2ed527..e376e9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ Slingshot Changelog * Added region parameters to S3. The default is `us-east-1`. This fixes bucketUrl problems [#33](https://github.com/CulturalMe/meteor-slingshot/issues/33). * Upgrade to `AWS4-HMAC-256` for S3 policy signing to make slingshot compatible with new AWS datacenters, such as Frankfurt. [#33](https://github.com/CulturalMe/meteor-slingshot/issues/33) * Added Rackspace Cloud Files support [#17](https://github.com/CulturalMe/meteor-slingshot/issues/17). + * Added dependency to the core HTTP package on the server side. + * Added resumable upload support for Google Cloud Storage. + * Upload instructions returned by upload services now must include an upload method (`PUT` or `POST`) ## Version 0.3.0 diff --git a/README.md b/README.md index 0dff534..f628ead 100644 --- a/README.md +++ b/README.md @@ -420,6 +420,7 @@ Meteor core packages: * tracker * reactive-var * check + * http ## API Reference diff --git a/lib/directive.js b/lib/directive.js index 5f26459..f5bc451 100644 --- a/lib/directive.js +++ b/lib/directive.js @@ -153,11 +153,15 @@ _.extend(Slingshot.Directive.prototype, { check(instructions, { upload: String, download: String, - postData: [{ + postData: Match.Optional([{ name: String, value: Match.OneOf(String, Number, null) - }], - headers: Match.Optional(Object) + }]), + headers: Match.Optional(Object), + method: Match.Where(function (method) { + check(method, String); + return method === "POST" || method === "PUT"; + }) }); return instructions; @@ -232,7 +236,3 @@ Meteor.methods({ return directive.getInstructions(this, file, meta); } }); - -function quoteString(string, quotes) { - return quotes + string.replace(quotes, '\\' + quotes) + quotes; -} diff --git a/lib/oauth2.js b/lib/oauth2.js new file mode 100644 index 0000000..1fbd862 --- /dev/null +++ b/lib/oauth2.js @@ -0,0 +1,107 @@ +/* global oauth: true */ + + +oauth = { + + Jwt: function (claim, scope) { + _.defaults(claim, { + aud: "https://www.googleapis.com/oauth2/v3/token" + }); + + check(claim, { + "iss": String, + "aud": String, + "sub": Match.Optional(String) + }); + + check(scope, [String]); + + claim.scope = scope.join(" "); + + _.extend(this, { + _claim: claim, + _header: this.encode({ + "alg": "RS256", + "typ": "JWT" + }) + }); + }, + + TokenGenerator: function (jwt, key) { + check(jwt, oauth.Jwt); + check(key, String); + + this.endpoint = "https://www.googleapis.com/oauth2/v3/token"; + this.jwt = jwt; + this.key = key; + this.grantType = "urn:ietf:params:oauth:grant-type:jwt-bearer"; + } +}; + +_.extend(oauth.Jwt.prototype, { + + encode: function (object) { + return new Buffer(JSON.stringify(object)).toString("base64"); + }, + + sign: rsaSha256, + + makeClaim: function (iat, exp) { + return this.encode(_.extend({ + exp: Math.round(exp.getTime() / 1000), + iat: Math.round(iat.getTime() / 1000) + }, this._claim)); + }, + + compile: function (key, iat, exp) { + var jwt = this._header + "." + this.makeClaim(iat, exp); + return jwt + "." + this.sign(jwt, key); + } +}); + +_.extend(oauth.TokenGenerator.prototype, { + + timeout: function () { + if (!this._expires) + return 0; + + return Math.max(this._expires.getTime() - Date.now(), 0); + }, + + getToken: Meteor.wrapAsync(function (callback) { + if (!this._token || this.timeout() < 60000) + this.requestNewToken(callback); + else + callback(null, this._token); + }), + + requestNewToken: Meteor.wrapAsync(function (callback) { + var self = this, + querystring = Npm.require("querystring"), + iat = new Date(), + exp = new Date(iat.getTime() + 3600 * 1000); + + delete this._token; + delete this._expires; + + HTTP.post(this.endpoint, { + headers: { + "Content-Type": "application/x-www-form-urlencoded" + }, + + content: querystring.stringify({ + grant_type: this.grantType, + assertion: this.jwt.compile(this.key, iat, exp) + }) + }, function (error, response) { + if (!error && response) { + self._expires = new Date(iat.getTime() + + response.data.expires_in * 1000); + self._token = response.data.access_token; + } + + callback(error, self._token); + }); + }) + +}); diff --git a/lib/storage-policy.js b/lib/storage-policy.js index f7a1c3e..6dde8ff 100644 --- a/lib/storage-policy.js +++ b/lib/storage-policy.js @@ -100,7 +100,6 @@ Slingshot.StoragePolicy = function () { */ stringify: function (encoding) { - /* global Buffer: false */ return Buffer(JSON.stringify(policy), "utf-8") .toString(encoding || "base64"); } diff --git a/lib/upload.js b/lib/upload.js index 9edde12..2c14c17 100644 --- a/lib/upload.js +++ b/lib/upload.js @@ -188,13 +188,19 @@ Slingshot.Upload = function (directive, metaData) { "The upload has been aborted by the user")); }); - xhr.open("POST", self.instructions.upload, true); + xhr.open(self.instructions.method, self.instructions.upload, true); _.each(self.instructions.headers, function (value, key) { xhr.setRequestHeader(key, value); }); - xhr.send(buildFormData()); + switch (self.instructions.method) { + case "POST": + xhr.send(buildFormData()); + break; + case "PUT": + xhr.send(self.file); + } return self; }, diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 0000000..e82191c --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,17 @@ +var crypto = Npm.require("crypto"); + +/* global rsaSha256: true */ +rsaSha256 = function (data, key, encoding) { + return crypto + .createSign('RSA-SHA256') + .update(data) + .sign(key, encoding || "base64"); +}; + +/* global hmac256: true */ +hmac256 = function (key, data, encoding) { + return crypto + .createHmac("sha256", key) + .update(new Buffer(data, "utf-8")) + .digest(encoding); +}; diff --git a/package.js b/package.js index 874d63f..928ce5c 100644 --- a/package.js +++ b/package.js @@ -8,6 +8,7 @@ Package.describe({ Package.on_use(function (api) { api.versionsFrom('METEOR@1.0'); + api.use("http", "server"); api.use(["underscore", "check"]); api.use(["tracker", "reactive-var"], "client"); @@ -19,10 +20,13 @@ Package.on_use(function (api) { api.add_files("lib/upload.js", "client"); api.add_files([ + "lib/utils.js", "lib/directive.js", "lib/storage-policy.js", + "lib/oauth2.js", "services/aws-s3.js", "services/google-cloud.js", + "services/google-cloud-resumable.js", "services/rackspace.js" ], "server"); diff --git a/services/aws-s3.js b/services/aws-s3.js index 9a05acd..25f7d64 100644 --- a/services/aws-s3.js +++ b/services/aws-s3.js @@ -65,39 +65,16 @@ Slingshot.S3Storage = { */ upload: function (method, directive, file, meta) { - var url = Npm.require("url"), - - policy = new Slingshot.StoragePolicy() - .expireIn(directive.expire) - .contentLength(0, Math.min(file.size, directive.maxSize || Infinity)), - - payload = { - key: _.isFunction(directive.key) ? - directive.key.call(method, file, meta) : directive.key, - - bucket: directive.bucket, - - "Content-Type": file.type, - "acl": directive.acl, - - "Cache-Control": directive.cacheControl, - "Content-Disposition": directive.contentDisposition || file.name && - "inline; filename=" + quoteString(file.name, '"') - }, - - bucketUrl = _.isFunction(directive.bucketUrl) ? - directive.bucketUrl(directive.bucket, directive.region) : - directive.bucketUrl, - - download = _.extend(url.parse(directive.cdn || bucketUrl), { - pathname: payload.key - }); + var policy = this.createPolicy(directive, file), + payload = this.getPayload(method, directive, file, meta), + bucketUrl = this.bucketUrl(directive), + download = this.downloadUrl(directive, payload.key); this.applySignature(payload, policy, directive); return { upload: bucketUrl, - download: url.format(download), + download: download, postData: [{ name: "key", value: payload.key @@ -106,10 +83,56 @@ Slingshot.S3Storage = { name: name, value: value }; - }).compact().value()) + }).compact().value()), + method: "POST" }; }, + createPolicy: function (directive, file) { + return new Slingshot.StoragePolicy() + .expireIn(directive.expire) + .contentLength(0, Math.min(file.size, directive.maxSize || Infinity)); + }, + + getPayload: function (method, directive, file, meta) { + return { + key: this.objectName(method, directive, file, meta), + + bucket: directive.bucket, + + "Content-Type": file.type, + "acl": directive.acl, + + "Cache-Control": directive.cacheControl, + "Content-Disposition": this.contentDisposition(directive, file) + }; + }, + + objectName: function (method, directive, file, meta) { + return _.isFunction(directive.key) ? + directive.key.call(method, file, meta) : directive.key; + }, + + contentDisposition: function (directive, file) { + return directive.contentDisposition || file.name && + "inline; filename=" + quoteString(file.name, '"'); + }, + + downloadUrl: function (directive, key) { + var url = Npm.require("url"), + bucketUrl = this.bucketUrl(directive); + + return url.format(_.extend(url.parse(directive.cdn || bucketUrl), { + pathname: key + })); + }, + + bucketUrl: function (directive) { + return _.isFunction(directive.bucketUrl) ? + directive.bucketUrl(directive.bucket, directive.region) : + directive.bucketUrl; + }, + /** Applies signature an upload payload * * @param {Object} payload - Data to be upload along with file @@ -170,12 +193,3 @@ function formatNumber(num, digits) { return Array(digits - string.length + 1).join("0").concat(string); } -var crypto = Npm.require("crypto"); - -function hmac256(key, data, encoding) { - /* global Buffer: false */ - return crypto - .createHmac("sha256", key) - .update(new Buffer(data, "utf-8")) - .digest(encoding); -} diff --git a/services/google-cloud-resumable.js b/services/google-cloud-resumable.js new file mode 100644 index 0000000..6e2892e --- /dev/null +++ b/services/google-cloud-resumable.js @@ -0,0 +1,80 @@ +var GoogleCloud = Slingshot.GoogleCloud; + +Slingshot.GoogleCloudResumable = _.defaults({ + + credentials: {}, + + maxSize: Infinity, + + directiveMatch: _.defaults({ + acl: Match.Optional(Match.Where(function (acl) { + check(acl, String); + + return [ + "authenticatedread", + "bucketownerfullcontrol", + "bucketownerread", + "private", + "projectprivate", + "publicread" + ].indexOf(acl) >= 0; + })) + }, GoogleCloud.directiveMatch), + + upload: function (method, directive, file, meta) { + var url = Npm.require("url"), + endpoint = { + protocol: "https", + host: "www.googleapis.com", + pathname: "/upload/storage/v1/b/" + directive.bucket + "/o" + }, + key = this.objectName(method, directive, file, meta); + + method.unblock(); + + var response = HTTP.post(url.format(endpoint), { + headers: { + "Authorization": "Bearer " + this.getCredentials(directive), + "Origin": Meteor.absoluteUrl(), + "X-Upload-Content-Type": file.type, + "X-Upload-Content-Length": file.length + }, + params: { + predefinedAcl: directive.acl, + uploadType: "resumable" + }, + data: { + name: key, + contentDisposition: this.contentDisposition(directive, file), + cacheControl: directive.cacheControl + } + }); + + return { + upload: response.headers.location, + download: this.downloadUrl(directive, key), + headers: { + "X-Upload-Content-Type": file.type, + "X-Upload-Content-Length": file.length + }, + method: "PUT" + }; + }, + + getCredentials: function (directive) { + var accessId = directive[this.accessId], + tokenGenerator = this.credentials[accessId]; + + if (!tokenGenerator) { + var jwt = new oauth.Jwt({iss: accessId}, [ + "https://www.googleapis.com/auth/devstorage.full_control" + ]); + + tokenGenerator = (this.credentials[accessId] = + new oauth.TokenGenerator(jwt, directive[this.secretKey])); + } + + return tokenGenerator.getToken(); + } + +}, GoogleCloud); diff --git a/services/google-cloud.js b/services/google-cloud.js index 53fd32e..f594a0d 100644 --- a/services/google-cloud.js +++ b/services/google-cloud.js @@ -50,9 +50,7 @@ Slingshot.GoogleCloud = _.defaults({ */ sign: function (secretKey, policy) { - return Npm.require("crypto") - .createSign('RSA-SHA256') - .update(policy) - .sign(secretKey, "base64"); + /* global rsaSha256: false*/ + return rsaSha256(policy, secretKey); } }, Slingshot.S3Storage); diff --git a/services/rackspace.js b/services/rackspace.js index 460e8d3..ea953a2 100644 --- a/services/rackspace.js +++ b/services/rackspace.js @@ -96,7 +96,8 @@ Slingshot.RackspaceFiles = { return { upload: url, download: (cdn && cdn + "/" + pathPrefix || host + path) + file.name, - postData: data + postData: data, + method: "POST" }; }, diff --git a/versions.json b/versions.json index ade065e..d789867 100644 --- a/versions.json +++ b/versions.json @@ -12,6 +12,10 @@ "ejson", "1.0.4" ], + [ + "http", + "1.0.8" + ], [ "json", "1.0.1" @@ -31,6 +35,10 @@ [ "underscore", "1.0.1" + ], + [ + "url", + "1.0.2" ] ], "pluginDependencies": [],