From e25a004fe427c146c4bbfcc4a77031457424e602 Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Tue, 2 Aug 2022 09:03:22 -0300 Subject: [PATCH 01/67] 3903 readwise - New Features (#3915) * New Features Actions: - List Highlights - Get Highlight Details Triggers: - New Highlights (polling trigger) * minor updates Co-authored-by: michelle0927 --- components/readwise/.gitignore | 3 - .../get-highlight-details.mjs | 36 +++++ .../list-highlights/list-highlights.mjs | 50 +++++++ components/readwise/app/readwise.app.ts | 13 -- components/readwise/common/utils.mjs | 20 +++ components/readwise/readwise.app.mjs | 138 ++++++++++++++++++ .../sources/new-highlights/new-highlights.mjs | 84 +++++++++++ 7 files changed, 328 insertions(+), 16 deletions(-) delete mode 100644 components/readwise/.gitignore create mode 100644 components/readwise/actions/get-highlight-details/get-highlight-details.mjs create mode 100644 components/readwise/actions/list-highlights/list-highlights.mjs delete mode 100644 components/readwise/app/readwise.app.ts create mode 100644 components/readwise/common/utils.mjs create mode 100644 components/readwise/readwise.app.mjs create mode 100644 components/readwise/sources/new-highlights/new-highlights.mjs diff --git a/components/readwise/.gitignore b/components/readwise/.gitignore deleted file mode 100644 index ec761ccab7595..0000000000000 --- a/components/readwise/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.js -*.mjs -dist \ No newline at end of file diff --git a/components/readwise/actions/get-highlight-details/get-highlight-details.mjs b/components/readwise/actions/get-highlight-details/get-highlight-details.mjs new file mode 100644 index 0000000000000..e6883e320ccc0 --- /dev/null +++ b/components/readwise/actions/get-highlight-details/get-highlight-details.mjs @@ -0,0 +1,36 @@ +import readwise from "../../readwise.app.mjs"; + +export default { + key: "readwise-get-highlight-details", + name: "Get Highlight Details", + description: "Get Highlight´s Details [See the docs here](https://readwise.io/api_deets)", + version: "0.0.1", + type: "action", + props: { + readwise, + bookId: { + propDefinition: [ + readwise, + "bookId", + ], + }, + highlightId: { + propDefinition: [ + readwise, + "highlightId", + (c) => ({ + bookId: c.bookId, + }), + ], + }, + }, + async run({ $ }) { + const highlight = await this.readwise.getHighlight({ + $, + highlightId: this.highlightId, + }); + + $.export("$summary", `Successfully returned highlight (${this.highlightId})`); + return highlight; + }, +}; diff --git a/components/readwise/actions/list-highlights/list-highlights.mjs b/components/readwise/actions/list-highlights/list-highlights.mjs new file mode 100644 index 0000000000000..d48fb61ea2715 --- /dev/null +++ b/components/readwise/actions/list-highlights/list-highlights.mjs @@ -0,0 +1,50 @@ +import readwise from "../../readwise.app.mjs"; + +export default { + key: "readwise-list-highlights", + name: "List Highlights", + description: "A list of highlights with a pagination metadata. The rate limit of this endpoint is restricted to 20 requests per minute. Each request returns 1000 items. [See the docs here](https://readwise.io/api_deets)", + version: "0.0.1", + type: "action", + props: { + readwise, + bookId: { + propDefinition: [ + readwise, + "bookId", + ], + optional: true, + }, + limit: { + propDefinition: [ + readwise, + "limit", + ], + }, + }, + async run({ $ }) { + const params = { + page_size: 1000, + limit: this.limit, + }; + if (this.bookId) params.book_id = this.bookId; + + const items = []; + + const paginator = this.readwise.paginate({ + $, + fn: this.readwise.listHighlights, + params, + }); + for await (const item of paginator) { + items.push(item); + } + + const suffix = items.length === 1 + ? "" + : "s"; + + $.export("$summary", `Successfully returned ${items.length} highlight${suffix}`); + return items; + }, +}; diff --git a/components/readwise/app/readwise.app.ts b/components/readwise/app/readwise.app.ts deleted file mode 100644 index e5239b3cd6315..0000000000000 --- a/components/readwise/app/readwise.app.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { defineApp } from "@pipedream/types"; - -export default defineApp({ - type: "app", - app: "readwise", - propDefinitions: {}, - methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); - }, - }, -}); \ No newline at end of file diff --git a/components/readwise/common/utils.mjs b/components/readwise/common/utils.mjs new file mode 100644 index 0000000000000..8d33f0281c625 --- /dev/null +++ b/components/readwise/common/utils.mjs @@ -0,0 +1,20 @@ +/* eslint-disable no-unused-vars */ +export const clearObj = (obj) => { + return Object.entries(obj) + .filter(([ + _, + v, + ]) => v != null) + .reduce( + (acc, [ + k, + v, + ]) => ({ + ...acc, + [k]: v === Object(v) + ? clearObj(v) + : v, + }), + {}, + ); +}; diff --git a/components/readwise/readwise.app.mjs b/components/readwise/readwise.app.mjs new file mode 100644 index 0000000000000..c99ed4ca08f15 --- /dev/null +++ b/components/readwise/readwise.app.mjs @@ -0,0 +1,138 @@ +import { axios } from "@pipedream/platform"; +import { clearObj } from "./common/utils.mjs"; + +export default { + type: "app", + app: "readwise", + propDefinitions: { + bookId: { + type: "integer", + label: "Book Id", + description: "Id of the book to list highlights", + async options({ page }) { + const { results } = await this.listBooks({ + params: { + page: page + 1, + }, + }); + + return results.map(({ + title, id, + }) => ({ + label: title, + value: id, + })) || []; + }, + }, + limit: { + type: "integer", + label: "Limit", + description: "Specify number of results per page (default is 100, max is 1000)", + default: 100, + }, + highlightId: { + type: "integer", + label: "Highlight Id", + description: "Id of the highlight to list details", + async options({ + page, bookId: book_id, + }) { + const { results } = await this.listHighlights({ + params: { + book_id, + page: page + 1, + }, + }); + + return results.map(({ + text, id, + }) => ({ + label: `(${id}) ${text}`, + value: id, + })) || []; + }, + }, + }, + methods: { + _getBaseUrl() { + return "https://readwise.io/api/v2"; + }, + _getHeaders() { + return { + "Authorization": `Token ${this.$auth.accesss_token}`, + }; + }, + async _makeRequest({ + $, path, ...otherConfig + }) { + const config = { + url: `${this._getBaseUrl()}/${path}`, + headers: this._getHeaders(), + ...otherConfig, + }; + + return axios($ || this, clearObj(config)); + }, + async listBooks({ + $, params, + }) { + return this._makeRequest({ + $, + path: "books", + params, + }); + }, + async listHighlights({ + $, params, + }) { + return this._makeRequest({ + $, + path: "highlights", + params, + }); + }, + async getHighlight({ + $, highlightId, + }) { + return this._makeRequest({ + $, + path: `highlights/${highlightId}`, + }); + }, + async *paginate({ + $, fn, params = {}, page, + }) { + const { limit } = params; + let count = 0; + + do { + const { + results, + next, + } = await fn({ + $, + params: { + ...params, + page, + }, + }); + + for (const d of results) { + yield d; + + if (limit && ++count === limit) { + return count; + } + } + page = null; + + if (next) { + const regex = next.match(/page=([^&]+)/g); + const pageNumber = regex[0].split("="); + page = pageNumber[1]; + } + + } while (page); + }, + }, +}; diff --git a/components/readwise/sources/new-highlights/new-highlights.mjs b/components/readwise/sources/new-highlights/new-highlights.mjs new file mode 100644 index 0000000000000..a15ff99e3294a --- /dev/null +++ b/components/readwise/sources/new-highlights/new-highlights.mjs @@ -0,0 +1,84 @@ +import readwise from "../../readwise.app.mjs"; + +export default { + key: "readwise-new-highlights", + name: "New Highlights", + description: "Emit new Highlight", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + readwise, + db: "$.service.db", + timer: { + label: "Polling interval", + description: "Pipedream will poll the Readwise API on this schedule", + type: "$.interface.timer", + default: { + intervalSeconds: 60 * 15, + }, + }, + bookId: { + propDefinition: [ + readwise, + "bookId", + ], + optional: true, + }, + }, + hooks: { + async activate() { + await this.processHighlighs({ + book_id: this.bookId, + page_size: 25, + }); + }, + }, + methods: { + _getLastHighlightedAt() { + return this.db.get("lastHighlightedAt"); + }, + _setLastHighlightedAt(lastHighlightedAt) { + this.db.set("lastHighlightedAt", lastHighlightedAt); + }, + async retrieveTicket(id) { + return this.readwise.retrieveTicket({ + id, + }); + }, + async processHighlighs(params) { + const { results: events } = await this.readwise.listHighlights({ + params, + }); + for (const event of events) { + this.emitEvent(await this.readwise.getHighlight({ + highlightId: event.id, + })); + } + }, + async processEvent() { + const lastHighlightedAt = this._getLastHighlightedAt(); + await this.processHighlighs({ + book_id: this.bookId, + highlighted_at__gt: lastHighlightedAt, + }); + }, + emitEvent(event, lastHighlightedAt = null) { + lastHighlightedAt = lastHighlightedAt || this._getLastHighlightedAt(); + + if (!lastHighlightedAt || (new Date(event.highlighted_at) > new Date(lastHighlightedAt))) + this._setLastHighlightedAt(event.highlighted_at); + + const ts = Date.parse(event.highlighted_at); + this.$emit(event, { + id: `${event.id}_${ts}`, + ts, + summary: `New highlight: ${event.id}`, + }); + }, + }, + async run() { + console.log("Raw received event:"); + return this.processEvent(); + }, +}; From e8e76b6d7ca0d7965b74d96e5d3000df47edf0a9 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 2 Aug 2022 09:44:46 -0400 Subject: [PATCH 02/67] Discord - new actions & triggers/sources (#3898) * discord actions * send-message-with-file action * new discord sources * update pnpm-lock.yaml * channels prop optional * remove console.log * remove incorrect props --- components/discord/actions/common/common.mjs | 51 ++++++++++++++++++ .../send-message-advanced.mjs | 52 +++++++++++++++++++ .../send-message-with-file.mjs | 51 ++++++++++++++++++ .../actions/send-message/send-message.mjs | 30 +++++++++++ components/discord/discord.app.mjs | 36 +++++++++++++ components/discord/package.json | 19 +++++++ .../message-deleted/message-deleted.mjs | 38 ++++++++++++++ .../new-guild-member/new-guild-member.mjs | 27 ++++++++++ .../sources/reaction-added/reaction-added.mjs | 37 +++++++++++++ pnpm-lock.yaml | 6 +++ 10 files changed, 347 insertions(+) create mode 100644 components/discord/actions/common/common.mjs create mode 100644 components/discord/actions/send-message-advanced/send-message-advanced.mjs create mode 100644 components/discord/actions/send-message-with-file/send-message-with-file.mjs create mode 100644 components/discord/actions/send-message/send-message.mjs create mode 100644 components/discord/discord.app.mjs create mode 100644 components/discord/package.json create mode 100644 components/discord/sources/message-deleted/message-deleted.mjs create mode 100644 components/discord/sources/new-guild-member/new-guild-member.mjs create mode 100644 components/discord/sources/reaction-added/reaction-added.mjs diff --git a/components/discord/actions/common/common.mjs b/components/discord/actions/common/common.mjs new file mode 100644 index 0000000000000..f893105b21955 --- /dev/null +++ b/components/discord/actions/common/common.mjs @@ -0,0 +1,51 @@ +import discord from "../../discord.app.mjs"; + +export default { + props: { + discord, + channel: { + type: "$.discord.channel", + appProp: "discord", + }, + message: { + propDefinition: [ + discord, + "message", + ], + }, + includeSentViaPipedream: { + propDefinition: [ + discord, + "includeSentViaPipedream", + ], + }, + }, + methods: { + getUserInputProps(omit = [ + "discord", + ]) { + return Object.keys(this) + .filter((k) => typeof this[k] !== "function" && !omit.includes(k)) + .reduce((props, key) => { + props[key] = this[key]; + return props; + }, {}); + }, + appendPipedreamText(message) { + let content = message; + if (typeof content !== "string") { + content = JSON.stringify(content); + } + content += `\n\n${this.getSentViaPipedreamText()}`; + return content; + }, + getSentViaPipedreamText() { + const workflowId = process.env.PIPEDREAM_WORKFLOW_ID; + // The link text is a URL without a protocol for consistency with the "Send via link" text in + // Slack messages + const linkText = `pipedream.com/@/${workflowId}?o=a&a=discord`; + const link = `https://${linkText}`; + return `Sent via [${linkText}](<${link}>)`; + }, + }, +}; diff --git a/components/discord/actions/send-message-advanced/send-message-advanced.mjs b/components/discord/actions/send-message-advanced/send-message-advanced.mjs new file mode 100644 index 0000000000000..1bbd2053b9625 --- /dev/null +++ b/components/discord/actions/send-message-advanced/send-message-advanced.mjs @@ -0,0 +1,52 @@ +import common from "../common/common.mjs"; + +export default { + ...common, + key: "discord-send-message-advanced", + name: "Send Message (Advanced)", + description: "Send a simple or structured message (using embeds) to a Discord channel", + version: "0.0.1", + type: "action", + props: { + ...common.props, + message: { + propDefinition: [ + common.props.discord, + "message", + ], + optional: true, + }, + embeds: { + propDefinition: [ + common.props.discord, + "embeds", + ], + }, + }, + async run({ $ }) { + const { + message, + includeSentViaPipedream, + embeds, + } = this; + + if (!message && !embeds) { + throw new Error("This action requires at least 1 message OR embeds object. Please enter one or the other above."); + } + + try { + const resp = await this.discord.createMessage(this.channel, { + embeds, + content: includeSentViaPipedream + ? this.appendPipedreamText(message ?? "") + : message, + }); + $.export("$summary", "Message sent successfully"); + return resp; + } catch (err) { + const unsentMessage = this.getUserInputProps(); + $.export("unsent", unsentMessage); + throw err; + } + }, +}; diff --git a/components/discord/actions/send-message-with-file/send-message-with-file.mjs b/components/discord/actions/send-message-with-file/send-message-with-file.mjs new file mode 100644 index 0000000000000..577afbf96333a --- /dev/null +++ b/components/discord/actions/send-message-with-file/send-message-with-file.mjs @@ -0,0 +1,51 @@ +import common from "../common/common.mjs"; + +export default { + ...common, + key: "discord-send-message-with-file", + name: "Send Message With File", + description: "Post a message with an attached file", + version: "0.0.1", + type: "action", + props: { + ...common.props, + message: { + propDefinition: [ + common.props.discord, + "message", + ], + optional: true, + }, + fileUrl: { + type: "string", + label: "File URL", + description: "The URL of the file to attach", + }, + }, + async run({ $ }) { + const { + message, + fileUrl, + includeSentViaPipedream, + } = this; + + try { + const resp = await this.discord.createMessage(this.channel, { + files: [ + { + attachment: fileUrl, + }, + ], + content: includeSentViaPipedream + ? this.appendPipedreamText(message ?? "") + : message, + }); + $.export("$summary", "Message sent successfully"); + return resp; + } catch (err) { + const unsentMessage = this.getUserInputProps(); + $.export("unsent", unsentMessage); + throw err; + } + }, +}; diff --git a/components/discord/actions/send-message/send-message.mjs b/components/discord/actions/send-message/send-message.mjs new file mode 100644 index 0000000000000..5da74d682581f --- /dev/null +++ b/components/discord/actions/send-message/send-message.mjs @@ -0,0 +1,30 @@ +import common from "../common/common.mjs"; + +export default { + ...common, + key: "discord-send-message", + name: "Send Message", + description: "Send a simple message to a Discord channel", + version: "0.0.1", + type: "action", + async run({ $ }) { + const { + message, + includeSentViaPipedream, + } = this; + + try { + const resp = await this.discord.createMessage(this.channel, { + content: includeSentViaPipedream + ? this.appendPipedreamText(message) + : message, + }); + $.export("$summary", "Message sent successfully"); + return resp; + } catch (err) { + const unsentMessage = this.getUserInputProps(); + $.export("unsent", unsentMessage); + throw err; + } + }, +}; diff --git a/components/discord/discord.app.mjs b/components/discord/discord.app.mjs new file mode 100644 index 0000000000000..abc06b0d440a0 --- /dev/null +++ b/components/discord/discord.app.mjs @@ -0,0 +1,36 @@ +export default { + type: "app", + app: "discord", + propDefinitions: { + message: { + type: "string", + label: "Message", + description: "Enter a simple message up to 2000 characters. This is the most commonly used field. However, it's optional if you pass embed content.", + }, + includeSentViaPipedream: { + type: "boolean", + optional: true, + default: true, + label: "Include link to workflow", + description: "Defaults to `true`, includes a link to this workflow at the end of your Discord message.", + }, + embeds: { + type: "any", + label: "Embeds", + description: "Optionally pass an [array of embed objects](https://birdie0.github.io/discord-webhooks-guide/discord_webhook.html). E.g., ``{{ [{\"description\":\"Use markdown including *Italic* **bold** __underline__ ~~strikeout~~ [hyperlink](https://google.com) `code`\"}] }}``. To pass data from another step, enter a reference using double curly brackets (e.g., `{{steps.mydata.$return_value}}`).\nTip: Construct the `embeds` array in a Node.js code step, return it, and then pass the return value to this step.", + optional: true, + }, + fileUrl: { + type: "string", + label: "File URL", + description: "The URL of the file to attach. Must specify either **File URL** or **File Path**.", + optional: true, + }, + filePath: { + type: "string", + label: "File Path", + description: "The path to the file, e.g. `/tmp/myFile.csv`. Must specify either **File URL** or **File Path**.", + optional: true, + }, + }, +}; diff --git a/components/discord/package.json b/components/discord/package.json new file mode 100644 index 0000000000000..9eeb9891cfebb --- /dev/null +++ b/components/discord/package.json @@ -0,0 +1,19 @@ +{ + "name": "@pipedream/discord", + "version": "0.0.1", + "description": "Pipedream Discord Components", + "main": "discord.app.mjs", + "keywords": [ + "pipedream", + "discord" + ], + "homepage": "https://pipedream.com/apps/discord", + "author": "Pipedream (https://pipedream.com/)", + "license": "MIT", + "dependencies": { + "@pipedream/platform": "^0.10.0" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/components/discord/sources/message-deleted/message-deleted.mjs b/components/discord/sources/message-deleted/message-deleted.mjs new file mode 100644 index 0000000000000..9922c29867564 --- /dev/null +++ b/components/discord/sources/message-deleted/message-deleted.mjs @@ -0,0 +1,38 @@ +import discord from "../../discord.app.mjs"; + +export default { + key: "discord-message-deleted", + name: "Message Deleted (Instant)", + description: "Emit new event for each message deleted", + version: "0.0.1", + dedupe: "unique", + type: "source", + props: { + discord, + channels: { + type: "$.discord.channel[]", + appProp: "discord", + label: "Channels", + description: "Select the channel(s) you'd like to be notified for", + optional: true, + }, + discordApphook: { + type: "$.interface.apphook", + appProp: "discord", + eventNames() { + return this.channels?.length > 0 + ? this.channels.map((channel) => `MESSAGE_DELETE:${channel}`) + : [ + "MESSAGE_DELETE", + ]; + }, + }, + }, + async run(event) { + this.$emit(event, { + id: event.id, + summary: `Message ${event.id} deleted from ${event.channel}`, + ts: Date.now(), + }); + }, +}; diff --git a/components/discord/sources/new-guild-member/new-guild-member.mjs b/components/discord/sources/new-guild-member/new-guild-member.mjs new file mode 100644 index 0000000000000..c57da82de71cf --- /dev/null +++ b/components/discord/sources/new-guild-member/new-guild-member.mjs @@ -0,0 +1,27 @@ +import discord from "../../discord.app.mjs"; + +export default { + key: "discord-guild-member", + name: "New Guild Member (Instant)", + description: "Emit new event for each new member added to a guild", + version: "0.0.1", + dedupe: "unique", + type: "source", + props: { + discord, + discordApphook: { + type: "$.interface.apphook", + appProp: "discord", + eventNames: [ + "GUILD_MEMBER_ADD", + ], + }, + }, + async run(event) { + this.$emit(event, { + id: `${event.userId}${event.guildId}`, + summary: `Member ${event.displayName} added`, + ts: Date.now(), + }); + }, +}; diff --git a/components/discord/sources/reaction-added/reaction-added.mjs b/components/discord/sources/reaction-added/reaction-added.mjs new file mode 100644 index 0000000000000..e240c4285d5ac --- /dev/null +++ b/components/discord/sources/reaction-added/reaction-added.mjs @@ -0,0 +1,37 @@ +import discord from "../../discord.app.mjs"; + +export default { + key: "discord-reaction-added", + name: "Reaction Added (Instant)", + description: "Emit new event for each reaction added to a message", + version: "0.0.1", + type: "source", + props: { + discord, + channels: { + type: "$.discord.channel[]", + appProp: "discord", + label: "Channels", + description: "Select the channel(s) you'd like to be notified for", + optional: true, + }, + discordApphook: { + type: "$.interface.apphook", + appProp: "discord", + eventNames() { + return this.channels?.length > 0 + ? this.channels.map((channel) => `MESSAGE_REACTION_ADD:${channel}`) + : [ + "MESSAGE_REACTION_ADD", + ]; + }, + }, + }, + async run(event) { + this.$emit(event, { + id: event.messageId, + summary: `Reaction added to message ${event.messageId}`, + ts: Date.now(), + }); + }, +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f14e4e5d3b58a..ba2e10f2029cc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -429,6 +429,12 @@ importers: dependencies: do-wrapper: 4.5.1 + components/discord: + specifiers: + '@pipedream/platform': ^0.10.0 + dependencies: + '@pipedream/platform': 0.10.0 + components/discord_bot: specifiers: '@pipedreamhq/platform': ^0.8.1 From a7fcbc6c36f68180332ca8f50bd1b288df8656fe Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Tue, 2 Aug 2022 11:11:54 -0300 Subject: [PATCH 03/67] Fix data stores object parsing (#3895) * return parsedValue whenever possible * create value prop definition * minor refactors * refactor summaries exports * use Function to evaluate js objects * bump patch versions * fix additional props --- .../add-update-multiple-records.mjs | 20 +++------ .../add-update-record/add-update-record.mjs | 15 ++++--- .../delete-single-record.mjs | 9 ++-- .../get-record-or-create.mjs | 29 +++++++------ .../has-key-or-create/has-key-or-create.mjs | 23 +++++----- components/data_stores/data_stores.app.mjs | 43 +++++++++++-------- 6 files changed, 72 insertions(+), 67 deletions(-) diff --git a/components/data_stores/actions/add-update-multiple-records/add-update-multiple-records.mjs b/components/data_stores/actions/add-update-multiple-records/add-update-multiple-records.mjs index 9962c0f4d0a0b..33095b7aebff7 100644 --- a/components/data_stores/actions/add-update-multiple-records/add-update-multiple-records.mjs +++ b/components/data_stores/actions/add-update-multiple-records/add-update-multiple-records.mjs @@ -1,11 +1,10 @@ import app from "../../data_stores.app.mjs"; -import xss from "xss"; export default { key: "data_stores-add-update-multiple-records", name: "Add or update multiple records", description: "Add or update multiple records to your [Pipedream Data Store](https://pipedream.com/data-stores/).", - version: "0.0.4", + version: "0.0.5", type: "action", props: { app, @@ -53,13 +52,7 @@ export default { } } - // Try to evaluate string as javascript, using xss as extra security - // If some problem occurs, return the original string - try { - return eval(`(${xss(value)})`); - } catch { - return value; - } + return this.app.evaluate(value); }, /** * Add all the key-value pairs in the map object to be used in the data store @@ -68,8 +61,7 @@ export default { */ populateHashMapOfData(data, map) { if (!Array.isArray(data) && typeof(data) === "object") { - Object.keys(data) - .forEach((key) => map[key] = this.convertString(data[key])); + Object.keys(data).forEach((key) => map[key] = this.convertString(data[key])); return; } @@ -87,7 +79,7 @@ export default { }, async run({ $ }) { if (typeof this.data === "string") { - this.data = eval(`(${this.data})`); + this.data = this.app.evaluate(this.data); } const map = this.getHashMapOfData(this.data); const keys = Object.keys(map); @@ -96,7 +88,9 @@ export default { if (keys.length === 0) { $.export("$summary", "No data was added to the data store."); } else { - $.export("$summary", `Successfully added or updated ${keys.length} record(s)`); + // eslint-disable-next-line multiline-ternary + $.export("$summary", `Successfully added or updated ${keys.length} record${keys.length === 1 ? "" : "s"}`); } + return map; }, }; diff --git a/components/data_stores/actions/add-update-record/add-update-record.mjs b/components/data_stores/actions/add-update-record/add-update-record.mjs index eb95e55340c71..bcd54ac4cf255 100644 --- a/components/data_stores/actions/add-update-record/add-update-record.mjs +++ b/components/data_stores/actions/add-update-record/add-update-record.mjs @@ -4,7 +4,7 @@ export default { key: "data_stores-add-update-record", name: "Add or update a single record", description: "Add or update a single record in your [Pipedream Data Store](https://pipedream.com/data-stores/).", - version: "0.0.7", + version: "0.0.8", type: "action", props: { app, @@ -25,9 +25,10 @@ export default { description: "Enter a key for the record you'd like to create or select an existing key to update.", }, value: { - label: "Value", - type: "any", - description: "Enter a string, object, or array.", + propDefinition: [ + app, + "value", + ], }, }, async run({ $ }) { @@ -35,14 +36,14 @@ export default { key, value, } = this; - const record = await this.dataStore.get(key); + const exists = await this.dataStore.has(key); const parsedValue = this.app.parseValue(value); await this.dataStore.set(key, parsedValue); // eslint-disable-next-line multiline-ternary - $.export("$summary", `Successfully ${record ? "updated the record for" : "added a new record with the"} key, \`${key}\`.`); + $.export("$summary", `Successfully ${exists ? "updated the record for" : "added a new record with the"} key, \`${key}\`.`); return { key, - value, + value: parsedValue, }; }, }; diff --git a/components/data_stores/actions/delete-single-record/delete-single-record.mjs b/components/data_stores/actions/delete-single-record/delete-single-record.mjs index 0f001894dece2..daabe2a7a3fe2 100644 --- a/components/data_stores/actions/delete-single-record/delete-single-record.mjs +++ b/components/data_stores/actions/delete-single-record/delete-single-record.mjs @@ -4,7 +4,7 @@ export default { key: "data_stores-delete-single-record", name: "Delete a single record", description: "Delete a single record in your [Pipedream Data Store](https://pipedream.com/data-stores/).", - version: "0.0.6", + version: "0.0.7", type: "action", props: { app, @@ -30,9 +30,10 @@ export default { if (record) { await this.dataStore.delete(this.key); - $.export("$summary", "Successfully deleted the record for key, `" + this.key + "`."); - } else { - $.export("$summary", "No record found for key, `" + this.key + "`. No data was deleted."); + $.export("$summary", `Successfully deleted the record for key, \`${this.key}\`.`); + return record; } + + $.export("$summary", `No record found for key, \`${this.key}\`. No data was deleted.`); }, }; diff --git a/components/data_stores/actions/get-record-or-create/get-record-or-create.mjs b/components/data_stores/actions/get-record-or-create/get-record-or-create.mjs index 941ebda62053d..643efddcdbaca 100644 --- a/components/data_stores/actions/get-record-or-create/get-record-or-create.mjs +++ b/components/data_stores/actions/get-record-or-create/get-record-or-create.mjs @@ -4,7 +4,7 @@ export default { key: "data_stores-get-record-or-create", name: "Get record (or create one if not found)", description: "Get a single record in your [Pipedream Data Store](https://pipedream.com/data-stores/) or create one if it doesn't exist.", - version: "0.0.7", + version: "0.0.8", type: "action", props: { app, @@ -32,27 +32,28 @@ export default { }, }, async additionalProps() { + const props = {}; if (this.app.shouldAddRecord(this.addRecordIfNotFound)) { - return this.app.valueProp(); + props.value = app.propDefinitions.value; } - return {}; + return props; }, async run({ $ }) { const record = await this.dataStore.get(this.key); if (record) { - $.export("$summary", "Found data for the key, `" + this.key + "`."); - } else { - if (this.app.shouldAddRecord(this.addRecordIfNotFound)) { - const parsedValue = this.app.parseValue(this.value); - await this.dataStore.set(this.key, parsedValue); - $.export("$summary", "Successfully added a new record with the key, `" + this.key + "`."); - return this.dataStore.get(this.key); - } else { - $.export("$summary", "No data found for key, `" + this.key + "`."); - } + $.export("$summary", `Found data for the key, \`${this.key}\`.`); + return record; } - return record; + if (!this.app.shouldAddRecord(this.addRecordIfNotFound)) { + $.export("$summary", `No data found for key, \`${this.key}\`.`); + return; + } + + const parsedValue = this.app.parseValue(this.value); + await this.dataStore.set(this.key, parsedValue); + $.export("$summary", `Successfully added a new record with the key, \`${this.key}\`.`); + return parsedValue; }, }; diff --git a/components/data_stores/actions/has-key-or-create/has-key-or-create.mjs b/components/data_stores/actions/has-key-or-create/has-key-or-create.mjs index 12c4e40a790a4..72237b9cbb8d0 100644 --- a/components/data_stores/actions/has-key-or-create/has-key-or-create.mjs +++ b/components/data_stores/actions/has-key-or-create/has-key-or-create.mjs @@ -4,7 +4,7 @@ export default { key: "data_stores-has-key-or-create", name: "Check for existence of key", description: "Check if a key exists in your [Pipedream Data Store](https://pipedream.com/data-stores/) or create one if it doesn't exist.", - version: "0.0.3", + version: "0.0.4", type: "action", props: { app, @@ -32,25 +32,26 @@ export default { }, }, async additionalProps() { + const props = {}; if (this.app.shouldAddRecord(this.addRecordIfNotFound)) { - return this.app.valueProp(); + props.value = app.propDefinitions.value; } - return {}; + return props; }, async run ({ $ }) { if (await this.dataStore.has(this.key)) { - $.export("$summary", `Key "${this.key}" exists.`); + $.export("$summary", `Key \`${this.key}\` exists.`); return true; } - if (this.app.shouldAddRecord(this.addRecordIfNotFound)) { - const parsedValue = this.app.parseValue(this.value); - await this.dataStore.set(this.key, parsedValue); - $.export("$summary", `Key "${this.key}" was not found. Successfully added a new record.`); - return this.dataStore.get(this.key); + if (!this.app.shouldAddRecord(this.addRecordIfNotFound)) { + $.export("$summary", `Key \`${this.key}\` does not exist.`); + return false; } - $.export("$summary", `Key "${this.key}" does not exist.`); - return false; + const parsedValue = this.app.parseValue(this.value); + await this.dataStore.set(this.key, parsedValue); + $.export("$summary", `Key \`${this.key}\` was not found. Successfully added a new record.`); + return parsedValue; }, }; diff --git a/components/data_stores/data_stores.app.mjs b/components/data_stores/data_stores.app.mjs index 7a6d54b83abf3..2bf4ed49291d4 100644 --- a/components/data_stores/data_stores.app.mjs +++ b/components/data_stores/data_stores.app.mjs @@ -1,3 +1,10 @@ +import xss from "xss"; + +/** + * Should support the following data types: + * https://pipedream.com/docs/data-stores/#supported-data-types + */ + export default { type: "app", app: "data_stores", @@ -15,6 +22,11 @@ export default { return dataStore.keys(); }, }, + value: { + label: "Value", + type: "any", + description: "Enter a string, object, or array.", + }, addRecordIfNotFound: { label: "Create a new record if the key is not found?", description: "Create a new record if no records are found for the specified key.", @@ -28,36 +40,31 @@ export default { }, }, methods: { + // using Function approach instead of eval: + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/eval#never_use_eval! + evaluate(value) { + try { + return Function(`"use strict"; return (${xss(value)})`)(); + } catch (err) { + return value; + } + }, + parseJSON(value) { + return JSON.parse(JSON.stringify(this.evaluate(value))); + }, shouldAddRecord(option) { return option === "Yes"; }, - valueProp() { - return { - value: { - label: "Value", - type: "any", - description: "Enter a string, object, or array.", - }, - }; - }, parseValue(value) { if (typeof value !== "string") { return value; } try { - return JSON.parse(this.sanitizeJson(value)); + return this.parseJSON(value); } catch (err) { return value; } }, - //Because user might enter a JSON as JS object; - //This method converts a JS object string to a JSON string before parsing it - //e.g. {a:"b", 'c':1} => {"a":"b", "c":1} - //We don't use eval here because of security reasons. - //Using eval may cause something undesired run in the script. - sanitizeJson(str) { - return str.replace(/(['"])?([a-z0-9A-Z_]+)(['"])?:/g, "\"$2\": "); - }, }, }; From 5895ed2bd021ee07a5455bf1655684c9e88b5fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Falc=C3=A3o?= <48412907+GTFalcao@users.noreply.github.com> Date: Tue, 2 Aug 2022 12:56:02 -0300 Subject: [PATCH 04/67] #983 - Infusionsoft Actions & Sources (#3870) * [App:Infusionsoft] Creating TS structure * [App:Infusionsoft] Creating 'get company' And company listing * [App:Infusionsoft] 'Get Company' action * [App:Infusionsoft] Creating 'Get Contact' * [App:Infusionsoft] 'Create Payment' action * [App:Infusionsoft] 'Create order item' + types * [App:Infusionsoft] Type adjustments * [App:Infusionsoft] Organizing type files * [App:Infusionsoft] Create Payment params * [App:Infusionsoft] Fixing types in async options listing * [App:Infusionsoft] Fixing http request * [App:Infusionsoft] Webhook initial implementation TS type correction pending * [App:Infusionsoft] Functional webhook system * [App:Infusionsoft] Creating generic hookObject type * pnpm-lock.yaml * Updating types package to 0.1.4 * [App:Infusionsoft] Common webhook source * [App:Infusionsoft] Source structure additions * [App:Infusionsoft] New appointment and desc changes * [App:Infusionsoft] Order summary * [App:Infusionsoft] Simplifying types * [App:Infusionsoft] Webhook data and summaries * [App:Infusionsoft] Summary changes and doc links * [App:Infusionsoft] Simplifying types * pnpm-lock.yaml * [App:Infusionsoft] Simplifying types * [App:Infusionsoft] Removing status from http response * [App:Infusionsoft] ESLint + PD Axios * [App:Infusionsoft] Naming convention fixes * [App:Infusionsoft] Pull request suggested changes * [App:Infusionsoft] Using PD Axios + $ for all requests * [App:Infusionsoft] Request fix and quality improvements * ESLint * 'Create Payment' making some props mandatory * ESLint * tsconfig.json newline * Removing 'new appointment' source --- components/infusionsoft/.gitignore | 3 + .../create-order-item/create-order-item.ts | 62 +++++ .../actions/create-payment/create-payment.ts | 96 ++++++++ .../actions/get-company/get-company.ts | 36 +++ .../actions/get-contact/get-contact.ts | 38 +++ .../infusionsoft/app/infusionsoft.app.ts | 220 ++++++++++++++++++ components/infusionsoft/infusionsoft.app.mjs | 11 - components/infusionsoft/package.json | 25 ++ components/infusionsoft/sources/common.ts | 96 ++++++++ .../sources/new-invoice/new-invoice.ts | 22 ++ .../sources/new-order/new-order.ts | 22 ++ .../sources/new-payment/new-payment.ts | 24 ++ components/infusionsoft/tsconfig.json | 22 ++ .../infusionsoft/types/requestParams.ts | 58 +++++ .../infusionsoft/types/responseSchemas.ts | 55 +++++ pnpm-lock.yaml | 33 ++- tsconfig.json | 5 +- 17 files changed, 814 insertions(+), 14 deletions(-) create mode 100644 components/infusionsoft/.gitignore create mode 100644 components/infusionsoft/actions/create-order-item/create-order-item.ts create mode 100644 components/infusionsoft/actions/create-payment/create-payment.ts create mode 100644 components/infusionsoft/actions/get-company/get-company.ts create mode 100644 components/infusionsoft/actions/get-contact/get-contact.ts create mode 100644 components/infusionsoft/app/infusionsoft.app.ts delete mode 100644 components/infusionsoft/infusionsoft.app.mjs create mode 100644 components/infusionsoft/package.json create mode 100644 components/infusionsoft/sources/common.ts create mode 100644 components/infusionsoft/sources/new-invoice/new-invoice.ts create mode 100644 components/infusionsoft/sources/new-order/new-order.ts create mode 100644 components/infusionsoft/sources/new-payment/new-payment.ts create mode 100644 components/infusionsoft/tsconfig.json create mode 100644 components/infusionsoft/types/requestParams.ts create mode 100644 components/infusionsoft/types/responseSchemas.ts diff --git a/components/infusionsoft/.gitignore b/components/infusionsoft/.gitignore new file mode 100644 index 0000000000000..ec761ccab7595 --- /dev/null +++ b/components/infusionsoft/.gitignore @@ -0,0 +1,3 @@ +*.js +*.mjs +dist \ No newline at end of file diff --git a/components/infusionsoft/actions/create-order-item/create-order-item.ts b/components/infusionsoft/actions/create-order-item/create-order-item.ts new file mode 100644 index 0000000000000..58c663c5a4247 --- /dev/null +++ b/components/infusionsoft/actions/create-order-item/create-order-item.ts @@ -0,0 +1,62 @@ +import infusionsoft from "../../app/infusionsoft.app"; +import { CreateOrderItemParams } from "../../types/requestParams"; +import { defineAction } from "@pipedream/types"; + +export default defineAction({ + name: "Create Order Item", + description: + "Add an item to an existing order [See docs here](https://developer.infusionsoft.com/docs/rest/#operation/createOrderItemsOnOrderUsingPOST)", + key: "infusionsoft-create-order-item", + version: "0.0.1", + type: "action", + props: { + infusionsoft, + orderId: { + propDefinition: [ + infusionsoft, + "orderId", + ], + }, + productId: { + propDefinition: [ + infusionsoft, + "productId", + ], + }, + quantity: { + type: "integer", + label: "Quantity", + min: 1, + }, + description: { + type: "string", + label: "Description", + optional: true, + }, + price: { + type: "string", + label: "Price", + description: + "Overridable price of the product. If not specified, the default will be used. Must be greater than or equal to 0.", + optional: true, + }, + }, + async run({ $ }): Promise { + const params: CreateOrderItemParams = { + $, + orderId: this.orderId, + data: { + description: this.description, + price: this.price, + product_id: this.productId, + quantity: this.quantity, + }, + }; + + const data: object = await this.infusionsoft.createOrderItem(params); + + $.export("$summary", "Created Order Item successfully"); + + return data; + }, +}); diff --git a/components/infusionsoft/actions/create-payment/create-payment.ts b/components/infusionsoft/actions/create-payment/create-payment.ts new file mode 100644 index 0000000000000..a6ac5d2080eae --- /dev/null +++ b/components/infusionsoft/actions/create-payment/create-payment.ts @@ -0,0 +1,96 @@ +import infusionsoft from "../../app/infusionsoft.app"; +import { defineAction } from "@pipedream/types"; +import { CreatePaymentParams } from "../../types/requestParams"; + +export default defineAction({ + name: "Create Payment", + description: + "Create or add a payment record [See docs here](https://developer.infusionsoft.com/docs/rest/#operation/createPaymentOnOrderUsingPOST)", + key: "infusionsoft-create-payment", + version: "0.0.1", + type: "action", + props: { + infusionsoft, + orderId: { + propDefinition: [ + infusionsoft, + "orderId", + ], + }, + paymentAmount: { + type: "string", + label: "Payment Amount", + }, + paymentMethodType: { + type: "string", + label: "Payment Method", + options: [ + { + label: "Credit Card", + value: "CREDIT_CARD", + }, + { + label: "Cash", + value: "CASH", + }, + { + label: "Check", + value: "CHECK", + }, + ], + }, + applyToCommissions: { + type: "boolean", + label: "Apply to Commissions", + optional: true, + }, + chargeNow: { + type: "boolean", + label: "Charge Now", + optional: true, + }, + creditCardId: { + type: "integer", + label: "Credit Card ID", + optional: true, + }, + date: { + type: "string", + label: "Date", + description: + "Used when `Charge Now` is **false** or if inserting historical data. Must be a date-time string such as `2017-01-01T22:17:59.039Z`", + optional: true, + }, + notes: { + type: "string", + label: "Notes", + optional: true, + }, + paymentGatewayId: { + type: "string", + label: "Payment Gateway ID", + optional: true, + }, + }, + async run({ $ }): Promise { + const params: CreatePaymentParams = { + $, + orderId: this.orderId, + data: { + apply_to_commissions: this.applyToCommissions, + charge_now: this.chargeNow, + credit_card_id: this.creditCardId, + date: this.date, + notes: this.notes, + payment_amount: this.paymentAmount, + payment_gateway_id: this.paymentGatewayId, + payment_method_type: this.paymentMethodType, + }, + }; + const data: object = await this.infusionsoft.createPayment(params); + + $.export("$summary", "Created Payment successfully"); + + return data; + }, +}); diff --git a/components/infusionsoft/actions/get-company/get-company.ts b/components/infusionsoft/actions/get-company/get-company.ts new file mode 100644 index 0000000000000..dfef0cd91eec8 --- /dev/null +++ b/components/infusionsoft/actions/get-company/get-company.ts @@ -0,0 +1,36 @@ +import infusionsoft from "../../app/infusionsoft.app"; +import { defineAction } from "@pipedream/types"; +import { Company } from "../../types/responseSchemas"; +import { GetObjectParams } from "../../types/requestParams"; + +export default defineAction({ + name: "Get Company", + description: + "Retrieve details of a Company [See docs here](https://developer.infusionsoft.com/docs/rest/#operation/getCompanyUsingGET)", + key: "infusionsoft-get-company", + version: "0.0.1", + type: "action", + props: { + infusionsoft, + companyId: { + propDefinition: [ + infusionsoft, + "companyId", + ], + }, + }, + async run({ $ }): Promise { + const params: GetObjectParams = { + $, + id: this.companyId, + }; + const data: Company = await this.infusionsoft.getCompany(params); + + $.export( + "$summary", + `Retrieved Company "${data.company_name}" successfully`, + ); + + return data; + }, +}); diff --git a/components/infusionsoft/actions/get-contact/get-contact.ts b/components/infusionsoft/actions/get-contact/get-contact.ts new file mode 100644 index 0000000000000..9b6a11efaa422 --- /dev/null +++ b/components/infusionsoft/actions/get-contact/get-contact.ts @@ -0,0 +1,38 @@ +import infusionsoft from "../../app/infusionsoft.app"; +import { defineAction } from "@pipedream/types"; +import { Contact } from "../../types/responseSchemas"; +import { GetObjectParams } from "../../types/requestParams"; + +export default defineAction({ + name: "Get Contact", + description: + "Retrieve details of a Contact [See docs here](https://developer.infusionsoft.com/docs/rest/#operation/getContactUsingGET)", + key: "infusionsoft-get-contact", + version: "0.0.1", + type: "action", + props: { + infusionsoft, + contactId: { + propDefinition: [ + infusionsoft, + "contactId", + ], + }, + }, + async run({ $ }): Promise { + const params: GetObjectParams = { + $, + id: this.contactId, + }; + const data: Contact = await this.infusionsoft.getContact(params); + + $.export( + "$summary", + `Retrieved Contact "${ + data.given_name ?? data.id.toString() + }" successfully`, + ); + + return data; + }, +}); diff --git a/components/infusionsoft/app/infusionsoft.app.ts b/components/infusionsoft/app/infusionsoft.app.ts new file mode 100644 index 0000000000000..b89296d776583 --- /dev/null +++ b/components/infusionsoft/app/infusionsoft.app.ts @@ -0,0 +1,220 @@ +import { defineApp } from "@pipedream/types"; +import { axios } from "@pipedream/platform"; +import { + CreateHookParams, + DeleteHookParams, + CreateOrderItemParams, + CreatePaymentParams, + GetObjectParams, + HttpRequestParams, +} from "../types/requestParams"; +import { + Appointment, + Company, + Contact, + Webhook, + Order, + Product, +} from "../types/responseSchemas"; + +export default defineApp({ + type: "app", + app: "infusionsoft", + methods: { + _baseUrl(): string { + return "https://api.infusionsoft.com/crm/rest/v1"; + }, + async _httpRequest({ + $ = this, + endpoint, + url, + ...args + }: HttpRequestParams): Promise { + return axios($, { + url: url ?? this._baseUrl() + endpoint, + headers: { + Authorization: `Bearer ${this.$auth.oauth_access_token}`, + }, + ...args, + }); + }, + async hookResponseRequest(apiUrl: string): Promise { + if (!(apiUrl && apiUrl.startsWith(this._baseUrl()))) { + return { + noUrl: true, + }; + } + + return this._httpRequest({ + url: apiUrl, + }); + }, + async createHook(data: CreateHookParams): Promise { + return this._httpRequest({ + endpoint: "/hooks", + method: "POST", + data, + }); + }, + async deleteHook({ key }: DeleteHookParams): Promise { + return this._httpRequest({ + endpoint: `/hooks/${key}`, + method: "DELETE", + }); + }, + async listCompanies(): Promise { + const response = await this._httpRequest({ + endpoint: "/companies", + }); + + return response.companies; + }, + async getCompany({ + id, ...params + }: GetObjectParams): Promise { + return this._httpRequest({ + endpoint: `/companies/${id}`, + ...params, + }); + }, + async getAppointment({ + id, + ...params + }: GetObjectParams): Promise { + return this._httpRequest({ + endpoint: `/appointments/${id}`, + ...params, + }); + }, + async listContacts(): Promise { + const response = await this._httpRequest({ + endpoint: "/contacts", + }); + + return response.contacts; + }, + async getContact({ + id, ...params + }: GetObjectParams): Promise { + return this._httpRequest({ + endpoint: `/contacts/${id}`, + ...params, + }); + }, + async listOrders(): Promise { + const response = await this._httpRequest({ + endpoint: "/orders", + }); + + return response.orders; + }, + async getOrder({ + id, ...params + }: GetObjectParams): Promise { + return this._httpRequest({ + endpoint: `/orders/${id}`, + ...params, + }); + }, + getOrderSummary({ + contact, order_items, total, + }: Order): string { + return `${order_items.length} items (total $${total}) by ${contact.first_name}`; + }, + async listProducts(): Promise { + const response = await this._httpRequest({ + endpoint: "/products", + }); + + return response.products; + }, + async createOrderItem({ + orderId, + ...params + }: CreateOrderItemParams): Promise { + return this._httpRequest({ + endpoint: `/orders/${orderId}/items`, + method: "POST", + ...params, + }); + }, + async createPayment({ + orderId, + ...params + }: CreatePaymentParams): Promise { + return this._httpRequest({ + endpoint: `/orders/${orderId}/payments`, + method: "POST", + ...params, + }); + }, + }, + propDefinitions: { + companyId: { + type: "integer", + label: "Company", + description: `Select a **Company** from the list. + \\ + Alternatively, you can provide a custom *Company ID*.`, + async options() { + const companies: Company[] = await this.listCompanies(); + + return companies.map(({ + company_name, id, + }) => ({ + label: company_name, + value: id, + })); + }, + }, + contactId: { + type: "integer", + label: "Contact", + description: `Select a **Contact** from the list. + \\ + Alternatively, you can provide a custom *Contact ID*.`, + async options() { + const contacts: Contact[] = await this.listContacts(); + + return contacts.map(({ + given_name, id, + }) => ({ + label: given_name ?? id.toString(), + value: id, + })); + }, + }, + orderId: { + type: "integer", + label: "Order", + description: `Select an **Order** from the list. + \\ + Alternatively, you can provide a custom *Order ID*.`, + async options() { + const orders: Order[] = await this.listOrders(); + + return orders.map((order) => ({ + label: this.getOrderSummary(order), + value: order.id, + })); + }, + }, + productId: { + type: "integer", + label: "Product", + description: `Select a **Product** from the list. + \\ + Alternatively, you can provide a custom *Product ID*.`, + async options() { + const products: Product[] = await this.listProducts(); + + return products.map(({ + product_name, product_price, id, + }) => ({ + label: `${product_name} ($${product_price})`, + value: id, + })); + }, + }, + }, +}); diff --git a/components/infusionsoft/infusionsoft.app.mjs b/components/infusionsoft/infusionsoft.app.mjs deleted file mode 100644 index 72efef6c6025a..0000000000000 --- a/components/infusionsoft/infusionsoft.app.mjs +++ /dev/null @@ -1,11 +0,0 @@ -export default { - type: "app", - app: "infusionsoft", - propDefinitions: {}, - methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); - }, - }, -}; diff --git a/components/infusionsoft/package.json b/components/infusionsoft/package.json new file mode 100644 index 0000000000000..c861fa5251679 --- /dev/null +++ b/components/infusionsoft/package.json @@ -0,0 +1,25 @@ +{ + "name": "@pipedream/infusionsoft", + "version": "0.0.1", + "description": "Pipedream InfusionSoft Components", + "main": "dist/app/infusionsoft.app.mjs", + "keywords": [ + "pipedream", + "infusionsoft" + ], + "files": [ + "dist" + ], + "homepage": "https://pipedream.com/apps/infusionsoft", + "author": "Pipedream (https://pipedream.com/)", + "license": "MIT", + "publishConfig": { + "access": "public" + }, + "devDependencies": { + "@pipedream/types": "^0.1.4" + }, + "dependencies": { + "@pipedream/platform": "^1.1.0" + } +} diff --git a/components/infusionsoft/sources/common.ts b/components/infusionsoft/sources/common.ts new file mode 100644 index 0000000000000..0f164bff6b1ff --- /dev/null +++ b/components/infusionsoft/sources/common.ts @@ -0,0 +1,96 @@ +import infusionsoft from "../app/infusionsoft.app"; +import { SourceHttpRunOptions } from "@pipedream/types"; +import { CreateHookParams } from "../types/requestParams"; +import { + Webhook, WebhookObject, +} from "../types/responseSchemas"; + +export default { + props: { + infusionsoft, + db: "$.service.db", + http: { + type: "$.interface.http", + customResponse: true, + }, + }, + methods: { + getHookType() { + // Available hooks: GET https://api.infusionsoft.com/crm/rest/v1/hooks/event_keys + throw new Error("Hook type not defined for this source"); + }, + getSummary() { + throw new Error("Summary defined for this source"); + }, + getHookSecretName(): string { + return "x-hook-secret"; + }, + }, + hooks: { + async activate() { + const data: CreateHookParams = { + eventKey: this.getHookType(), + hookUrl: this.http.endpoint, + }; + + const { key }: Webhook = await this.infusionsoft.createHook(data); + + this.db.set("hookKey", key); + }, + async deactivate() { + const key: string = this.db.get("hookKey"); + + await this.infusionsoft.deleteHook({ + key, + }); + }, + }, + async run(data: SourceHttpRunOptions) { + const hookSecretName: string = this.getHookSecretName(); + const hookSecret = data.headers[hookSecretName]; + + const httpResponse = { + headers: {}, + status: 200, + }; + + // If this is a hook verification request: + // Do not trigger an event (respond with the secret received) + if (hookSecret && data.method === "POST") { + httpResponse.headers[hookSecretName] = hookSecret; + this.http.respond(httpResponse); + return; + } + + this.http.respond(httpResponse); + + // Actual event trigger + const { object_keys: objectKeys } = data.body; + if (!Array.isArray(objectKeys)) { + throw new Error("Unknown data received from Infusionsoft webhook"); + } + + const promises: Promise<{ + obj: WebhookObject; + response: any; + }>[] = objectKeys.map(async (obj: WebhookObject) => ({ + obj, + response: await this.infusionsoft.hookResponseRequest(obj.apiUrl), + })); + + const result = await Promise.all(promises); + result.forEach(({ + obj, response, + }) => { + const data = response.noUrl + ? obj + : response; + const summary = this.getSummary(data); + this.$emit(data, { + id: obj.id, + summary, + ts: new Date(obj.timestamp).valueOf(), + }); + }); + }, +}; diff --git a/components/infusionsoft/sources/new-invoice/new-invoice.ts b/components/infusionsoft/sources/new-invoice/new-invoice.ts new file mode 100644 index 0000000000000..6c8791fe26627 --- /dev/null +++ b/components/infusionsoft/sources/new-invoice/new-invoice.ts @@ -0,0 +1,22 @@ +import { defineSource } from "@pipedream/types"; +import { WebhookObject } from "../../types/responseSchemas"; +import common from "../common"; + +export default defineSource({ + ...common, + name: "New Invoice", + description: + "Emit new event for each new **invoice** [See docs here](https://developer.infusionsoft.com/docs/rest/#tag/REST-Hooks)", + key: "infusionsoft-new-invoice", + version: "0.0.1", + type: "source", + methods: { + ...common.methods, + getHookType(): string { + return "invoice.add"; + }, + getSummary({ id }: WebhookObject): string { + return `Invoice ID ${id}`; + }, + }, +}); diff --git a/components/infusionsoft/sources/new-order/new-order.ts b/components/infusionsoft/sources/new-order/new-order.ts new file mode 100644 index 0000000000000..be5c6d304875f --- /dev/null +++ b/components/infusionsoft/sources/new-order/new-order.ts @@ -0,0 +1,22 @@ +import { defineSource } from "@pipedream/types"; +import { Order } from "../../types/responseSchemas"; +import common from "../common"; + +export default defineSource({ + ...common, + name: "New Order", + description: + "Emit new event for each new **order** [See docs here](https://developer.infusionsoft.com/docs/rest/#operation/getOrderUsingGET)", + key: "infusionsoft-new-order", + version: "0.0.1", + type: "source", + methods: { + ...common.methods, + getHookType(): string { + return "order.add"; + }, + getSummary(order: Order): string { + return this.infusionsoft.getOrderSummary(order); + }, + }, +}); diff --git a/components/infusionsoft/sources/new-payment/new-payment.ts b/components/infusionsoft/sources/new-payment/new-payment.ts new file mode 100644 index 0000000000000..33095481ff555 --- /dev/null +++ b/components/infusionsoft/sources/new-payment/new-payment.ts @@ -0,0 +1,24 @@ +import { defineSource } from "@pipedream/types"; +import { Transaction } from "../../types/responseSchemas"; +import common from "../common"; + +export default defineSource({ + ...common, + name: "New Payment", + description: + "Emit new event for each new **payment** [See docs here](https://developer.infusionsoft.com/docs/rest/#operation/getTransactionUsingGET)", + key: "infusionsoft-new-payment", + version: "0.0.1", + type: "source", + methods: { + ...common.methods, + getHookType(): string { + return "invoice.payment.add"; + }, + getSummary({ + amount, order_ids, + }: Transaction): string { + return `${amount} for orders ${order_ids}`; + }, + }, +}); diff --git a/components/infusionsoft/tsconfig.json b/components/infusionsoft/tsconfig.json new file mode 100644 index 0000000000000..873d4ff05af64 --- /dev/null +++ b/components/infusionsoft/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "lib": ["es2020"], + "module": "ES2020", + "target": "ES2020", + "moduleResolution": "node", + "listEmittedFiles": true, // Used as a part of the build task, since we need to pass emitted files to our post-build script + "composite": true, + "outDir": "dist", + "allowSyntheticDefaultImports": true, + }, + "allowJs": true, + "include": [ + "app", + "actions", + "sources", + "types" + ], + "exclude": [ + "dist", + ] + } diff --git a/components/infusionsoft/types/requestParams.ts b/components/infusionsoft/types/requestParams.ts new file mode 100644 index 0000000000000..d2f99477393e4 --- /dev/null +++ b/components/infusionsoft/types/requestParams.ts @@ -0,0 +1,58 @@ +import { Pipedream } from "@pipedream/types"; + +interface ActionRequestParams { + $?: Pipedream; +} + +interface HttpRequestParams extends ActionRequestParams { + endpoint?: string; + data?: object; + method?: string; + url?: string; +} + +interface CreateOrderItemParams extends ActionRequestParams { + orderId: number; + data: { + description: string; + price: string; + product_id: number; + quantity: number; + }; +} + +interface CreatePaymentParams extends ActionRequestParams { + orderId: number; + data: { + apply_to_commissions: boolean; + charge_now: boolean; + credit_card_id: number; + date: string; + notes: string; + payment_amount: string; + payment_gateway_id: string; + payment_method_type: string; + }; +} + +interface GetObjectParams extends ActionRequestParams { + id: number; +} + +interface CreateHookParams { + eventKey: string; + hookUrl: string; +} + +interface DeleteHookParams { + key: string; +} + +export { + CreateHookParams, + DeleteHookParams, + CreateOrderItemParams, + CreatePaymentParams, + GetObjectParams, + HttpRequestParams, +}; diff --git a/components/infusionsoft/types/responseSchemas.ts b/components/infusionsoft/types/responseSchemas.ts new file mode 100644 index 0000000000000..5d0e80bebb125 --- /dev/null +++ b/components/infusionsoft/types/responseSchemas.ts @@ -0,0 +1,55 @@ +type Appointment = { + title: string; +}; + +type Company = { + company_name: string; + id: number; +}; + +type Contact = { + given_name: string; + id: number; +}; + +type Order = { + contact: { + first_name: string; + last_name: string; + }; + id: number; + order_items: object[]; + total: number; +}; + +type Product = { + id: number; + product_name: string; + product_price: number; +}; + +type Transaction = { + amount: number; + order_ids: string; +}; + +type Webhook = { + key: string; +}; + +type WebhookObject = { + apiUrl: string; + id: number; + timestamp: string; +}; + +export { + Appointment, + Company, + Contact, + Order, + Product, + Transaction, + Webhook, + WebhookObject, +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ba2e10f2029cc..fd21a58383b62 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -789,6 +789,15 @@ importers: dependencies: '@pipedream/platform': 0.10.0 + components/infusionsoft: + specifiers: + '@pipedream/platform': ^1.1.0 + '@pipedream/types': ^0.1.4 + dependencies: + '@pipedream/platform': 1.1.0 + devDependencies: + '@pipedream/types': 0.1.4 + components/inksprout: specifiers: {} @@ -5967,8 +5976,8 @@ packages: resolution: {integrity: sha512-aARW1ZQZgZ68Qq1134G9ZktwI5tFxh0reE1uOunn4YFv0lGGfbo55zqEf5zbtE3zkbQ9seJ/GMyEDeSrw70PcQ==} dependencies: axios: 0.19.2 - fp-ts: 2.12.1 - io-ts: 2.2.16_fp-ts@2.12.1 + fp-ts: 2.12.2 + io-ts: 2.2.16_fp-ts@2.12.2 transitivePeerDependencies: - supports-color dev: false @@ -5984,6 +5993,12 @@ packages: dependencies: typescript: 4.7.4 + /@pipedream/types/0.1.4: + resolution: {integrity: sha512-0Jhk9ydw9REDJKHgl2GBzpnfK1COR3rXFrmAxzUWv2X1O1RmCB/HoeRcEhkZAZ7+kUSm8Em0Di3obiv4+oVSoA==} + dependencies: + typescript: 4.7.4 + dev: true + /@pipedreamhq/platform/0.8.1: resolution: {integrity: sha512-VMSEi4i5gUXfcbmmXkntTXNCu8uq02lmENh5KHW+PLUHDjX259w5BahdGdD7KWanQSJsyf2BaWXjnEMf9YVEaA==} dependencies: @@ -8574,6 +8589,7 @@ packages: /axios/0.19.2: resolution: {integrity: sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==} + deprecated: Critical security vulnerability fixed in v0.21.1. For more information, see https://github.com/axios/axios/pull/3410 dependencies: follow-redirects: 1.5.10 transitivePeerDependencies: @@ -11242,6 +11258,10 @@ packages: resolution: {integrity: sha512-oxvgqUYR6O9VkKXrxkJ0NOyU0FrE705MeqgBUMEPWyTu6Pwn768cJbHChw2XOBlgFLKfIHxjr2OOBFpv2mUGZw==} dev: false + /fp-ts/2.12.2: + resolution: {integrity: sha512-v8J7ud+nTkP5Zz17GhpCsY19wiRbB9miuj61nBcCJyDpu52zs9Z4O7OLDfYoKFQMJ9EsSZA7W1vRgC1d3jy5qw==} + dev: false + /fragment-cache/0.2.1: resolution: {integrity: sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==} engines: {node: '>=0.10.0'} @@ -12234,6 +12254,14 @@ packages: fp-ts: 2.12.1 dev: false + /io-ts/2.2.16_fp-ts@2.12.2: + resolution: {integrity: sha512-y5TTSa6VP6le0hhmIyN0dqEXkrZeJLeC5KApJq6VLci3UEKF80lZ+KuoUs02RhBxNWlrqSNxzfI7otLX1Euv8Q==} + peerDependencies: + fp-ts: ^2.5.0 + dependencies: + fp-ts: 2.12.2 + dev: false + /ip/1.1.8: resolution: {integrity: sha512-PuExPYUiu6qMBQb4l06ecm6T6ujzhmh+MeJcW9wa89PoAz5pvd4zPgN5WJV104mb6S2T1AwNIAaB70JNrLQWhg==} dev: false @@ -18619,6 +18647,7 @@ packages: /typescript/4.7.4: resolution: {integrity: sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==} engines: {node: '>=4.2.0'} + hasBin: true /typical/4.0.0: resolution: {integrity: sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==} diff --git a/tsconfig.json b/tsconfig.json index 39ef763db1ef0..5b70d630f84a0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -408,6 +408,9 @@ { "path": "components/fedex" }, + { + "path": "components/infusionsoft" + }, { "path": "components/expensify" }, @@ -454,4 +457,4 @@ "path": "components/google_directory" } ] -} \ No newline at end of file +} From 29be4f370cc5712cb298aeead1cde055738960dc Mon Sep 17 00:00:00 2001 From: Lucas Caresia Date: Tue, 2 Aug 2022 15:20:25 -0300 Subject: [PATCH 05/67] [BUG] Datadog - New Monitor Event Trigger async options broken #3459 (#3879) * Fixed monitors list * pnpm * Source versions * Package.json new line on end * A little fix * A little fix * Update from datadog lib to axios * pnpm * Updating versions --- .../post-metric-data/post-metric-data.mjs | 24 +-- components/datadog/datadog.app.mjs | 179 +++++++++++------- components/datadog/package.json | 3 +- .../new-monitor-event/new-monitor-event.mjs | 25 +-- pnpm-lock.yaml | 56 +----- 5 files changed, 136 insertions(+), 151 deletions(-) diff --git a/components/datadog/actions/post-metric-data/post-metric-data.mjs b/components/datadog/actions/post-metric-data/post-metric-data.mjs index 7390e1d743575..19ea6d5543eba 100644 --- a/components/datadog/actions/post-metric-data/post-metric-data.mjs +++ b/components/datadog/actions/post-metric-data/post-metric-data.mjs @@ -4,7 +4,7 @@ export default { key: "datadog-post-metric-data", name: "Post Metric Data", description: "The metrics end-point allows you to post time-series data that can be graphed on Datadog's dashboards. [See docs](https://docs.datadoghq.com/metrics)", - version: "0.0.1", + version: "0.1.0", type: "action", props: { datadog, @@ -29,18 +29,20 @@ export default { }, }, async run({ $ }) { - const metric = this.metric; - const points = this.convertMetricPoints(this.points); - const response = await this.datadog.postMetricData({ - series: [ - { - metric, - points, - }, - ], + $, + data: { + series: [ + { + metric: this.metric, + points: this.convertMetricPoints(this.points), + }, + ], + }, }); - $.export("$summary", `Posted to ${metric} timeseries`); + + $.export("$summary", `Posted to ${this.metric} timeseries`); + return response; }, }; diff --git a/components/datadog/datadog.app.mjs b/components/datadog/datadog.app.mjs index 0d139e2ddb75e..9f18063c5156b 100644 --- a/components/datadog/datadog.app.mjs +++ b/components/datadog/datadog.app.mjs @@ -1,4 +1,4 @@ -import { v1 } from "@datadog/datadog-api-client@1.0.0-beta.8"; +import { axios } from "@pipedream/platform"; import { v4 as uuid } from "uuid"; import constants from "./actions/common/constants.mjs"; @@ -16,7 +16,9 @@ export default { hostList, totalReturned, } = await this.listHosts({ - start, + query: { + start, + }, }); return { options: hostList.map((host) => host.name), @@ -32,8 +34,10 @@ export default { description: "The name of the timeseries", async options({ host }) { const { metrics } = await this.listActiveMetrics({ - from: 1, - host, + query: { + from: 1, + host, + }, }); return metrics; }, @@ -45,7 +49,9 @@ export default { optional: true, async options({ hostName }) { const { tags } = await this.listTags({ - hostName, + query: { + hostName, + }, }); return tags; }, @@ -56,30 +62,47 @@ export default { description: "The type of the metric", options: constants.metricTypes, }, + monitors: { + type: "string[]", + label: "Monitors", + description: "The monitors to observe for notifications", + optional: true, + async options({ page }) { + const monitors = await this.listMonitors({ + query: { + page, + pageSize: 1000, + }, + }); + + return monitors.map((monitor) => ({ + label: monitor.name, + value: monitor.id, + })); + }, + }, }, methods: { - _v1Config() { - return v1.createConfiguration({ - authMethods: { - apiKeyAuth: this.$auth.api_key, - appKeyAuth: this.$auth.application_key, - }, - }); - }, - _webhooksApi() { - return new v1.WebhooksIntegrationApi(this._v1Config()); + _apiKey() { + return this.$auth.api_key; }, - _monitorsApi() { - return new v1.MonitorsApi(this._v1Config()); + _applicationKey() { + return this.$auth.application_key; }, - _hostsApi() { - return new v1.HostsApi(this._v1Config()); + _apiUrl() { + return "https://api.datadoghq.com/api"; }, - _tagsApi() { - return new v1.TagsApi(this._v1Config()); - }, - _metricsApi() { - return new v1.MetricsApi(this._v1Config()); + async _makeRequest({ + $ = this, path, ...args + }) { + return axios($ ?? this, { + url: `${this._apiUrl()}${path}`, + headers: { + "DD-API-KEY": this._apiKey(), + "DD-APPLICATION-KEY": this._applicationKey(), + }, + ...args, + }); }, _webhookSecretKeyHeader() { return "x-webhook-secretkey"; @@ -92,29 +115,37 @@ export default { return headers[this._webhookSecretKeyHeader()] === secretKey; }, async _getMonitor(monitorId) { - return this._monitorsApi().getMonitor({ - monitorId, + return this._makeRequest({ + path: `/v1/monitor/${monitorId}`, }); }, - async _editMonitor(monitorId, monitorChanges) { - await this._monitorsApi().updateMonitor({ - monitorId, - body: monitorChanges, + async _editMonitor({ + monitorId, ...args + }) { + if (!monitorId) return; + + return this._makeRequest({ + path: `/v1/monitor/${monitorId}`, + method: "put", + ...args, }); }, - async *_searchMonitors(query) { + async *_searchMonitors({ ...args }) { let page = 0; let pageCount; let perPage; do { - const params = { - page, - query, - }; const { monitors, metadata, - } = await this._monitorsApi().searchMonitors(params); + } = await this._makeRequest({ + path: "/v1/monitor/search", + query: { + ...args.query, + page, + }, + ...args, + }); for (const monitor of monitors) { yield monitor; } @@ -123,37 +154,34 @@ export default { perPage = metadata.perPage; } while (pageCount === perPage); }, - async listMonitors(page, pageSize) { - return this._monitorsApi().listMonitors({ - page, - pageSize, - }); - }, async createWebhook( url, payloadFormat = null, secretKey = uuid(), ) { const name = `pd-${uuid()}`; - const customHeaders = { - [this._webhookSecretKeyHeader()]: secretKey, - }; - await this._webhooksApi().createWebhooksIntegration({ - body: { - customHeaders: JSON.stringify(customHeaders), + + await this._makeRequest({ + path: "/v1/integration/webhooks/configuration/webhooks", + method: "post", + data: { payload: JSON.stringify(payloadFormat), name, url, }, }); + return { name, secretKey, }; }, async deleteWebhook(webhookName) { - await this._webhooksApi().deleteWebhooksIntegration({ - webhookName, + if (!webhookName) return; + + await this._makeRequest({ + path: `/v1/integration/webhooks/configuration/webhooks/${webhookName}`, + method: "delete", }); }, async addWebhookNotification(webhookName, monitorId) { @@ -161,14 +189,16 @@ export default { const webhookTagPattern = this._webhookTagPattern(webhookName); if (new RegExp(webhookTagPattern).test(message)) { // Monitor is already notifying this webhook + console.log("Monitor is already notifying this webhook"); return; } const newMessage = `${message}\n${webhookTagPattern}`; - const monitorChanges = { - message: newMessage, - }; - await this._editMonitor(monitorId, monitorChanges); + await this._editMonitor(monitorId, { + data: { + message: newMessage, + }, + }); }, async removeWebhookNotifications(webhookName) { // Users could have manually added this webhook in other monitors, or @@ -178,7 +208,11 @@ export default { const webhookTagPattern = new RegExp( `\n?${this._webhookTagPattern(webhookName)}`, ); - const monitorSearchResults = this._searchMonitors(webhookName); + const monitorSearchResults = this._searchMonitors({ + query: { + query: webhookName, + }, + }); for await (const monitorInfo of monitorSearchResults) { const { id: monitorId } = monitorInfo; const { message } = await this._getMonitor(monitorId); @@ -195,18 +229,35 @@ export default { await this._editMonitor(monitorId, monitorChanges); } }, - async listHosts(params) { - return this._hostsApi().listHosts(params); + async listMonitors(args) { + return this._makeRequest({ + path: "/v1/monitor", + ...args, + }); }, - async listTags(params) { - return this._tagsApi().getHostTags(params); + async listHosts(args) { + return this._makeRequest({ + path: "/v1/hosts", + ...args, + }); + }, + async listTags(args) { + return this._makeRequest({ + path: "/v1/tags/hosts", + ...args, + }); }, - async listActiveMetrics(params) { - return this._metricsApi().listActiveMetrics(params); + async listActiveMetrics(args) { + return this._makeRequest({ + path: "/v1/metrics", + ...args, + }); }, - async postMetricData(params) { - return this._metricsApi().submitMetrics({ - body: params, + async postMetricData(args) { + return this._makeRequest({ + path: "/v2/series", + method: "post", + ...args, }); }, }, diff --git a/components/datadog/package.json b/components/datadog/package.json index a4c0428577e59..5d96c2bfb7659 100644 --- a/components/datadog/package.json +++ b/components/datadog/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/datadog", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream Datadog Components", "main": "datadog.app.mjs", "keywords": [ @@ -11,7 +11,6 @@ "author": "Pipedream (https://pipedream.com/)", "license": "MIT", "dependencies": { - "@datadog/datadog-api-client": "^1.0.0-beta.8", "uuid": "^8.3.2" } } diff --git a/components/datadog/sources/new-monitor-event/new-monitor-event.mjs b/components/datadog/sources/new-monitor-event/new-monitor-event.mjs index 964ed99a85057..0a6137c720f5b 100644 --- a/components/datadog/sources/new-monitor-event/new-monitor-event.mjs +++ b/components/datadog/sources/new-monitor-event/new-monitor-event.mjs @@ -3,10 +3,10 @@ import { payloadFormat } from "../common/payload-format.mjs"; export default { key: "datadog-new-monitor-event", - name: "New Monitor Event (Instant)", + name: "New Monitor Event (Instant) [Updated]", description: "Emit new events captured by a Datadog monitor", dedupe: "unique", - version: "0.0.2", + version: "0.1.0", type: "source", props: { datadog, @@ -16,23 +16,10 @@ export default { customResponse: true, }, monitors: { - type: "string[]", - label: "Monitors", - description: "The monitors to observe for notifications", - optional: true, - async options(context) { - const { page } = context; - const pageSize = 10; - const monitors = await this.datadog.listMonitors(page, pageSize); - const options = monitors.map((monitor) => ({ - label: monitor.name, - value: monitor.id, - })); - - return { - options, - }; - }, + propDefinition: [ + datadog, + "monitors", + ], }, }, hooks: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fd21a58383b62..f7736e312263e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -365,10 +365,8 @@ importers: components/datadog: specifiers: - '@datadog/datadog-api-client': ^1.0.0-beta.8 uuid: ^8.3.2 dependencies: - '@datadog/datadog-api-client': 1.1.0 uuid: 8.3.2 components/dear: @@ -682,6 +680,7 @@ importers: '@pipedream/platform': 0.10.0 googleapis: 96.0.0 + components/google_dialogflow: specifiers: '@google-cloud/dialogflow': ^5.1.0 @@ -4760,26 +4759,6 @@ packages: kuler: 2.0.0 dev: false - /@datadog/datadog-api-client/1.1.0: - resolution: {integrity: sha512-aGXnKvSqBG2tZzGpxwrNseJ7nuIIs6bgsqe5KhmgJOnpL3a0jpBhuE66m6UJ9mmgb/Tusc+kh3lPKCJnPq7mSQ==} - engines: {node: '>=12.0.0'} - dependencies: - '@types/buffer-from': 1.1.0 - '@types/node': 18.0.6 - '@types/pako': 1.0.4 - btoa: 1.2.1 - buffer-from: 1.1.2 - cross-fetch: 3.1.5 - durations: 3.4.2 - es6-promise: 4.2.8 - form-data: 3.0.1 - loglevel: 1.8.0 - pako: 2.0.4 - url-parse: 1.5.10 - transitivePeerDependencies: - - encoding - dev: false - /@definitelytyped/header-parser/0.0.121: resolution: {integrity: sha512-78HY1J+QiwZPXFZQWb9gPQPxAMNg6/1e4gl7gL/8dmd17vVWLi3AGrLJoG8Pn1p8d7MF2rlPw/2AE6+r1IHNHA==} dependencies: @@ -7612,12 +7591,6 @@ packages: '@types/node': 18.0.6 dev: false - /@types/buffer-from/1.1.0: - resolution: {integrity: sha512-BLFpLBcN+RPKUsFxqRkMiwqTOOdi+TrKr5OpLJ9qCnUdSxS6S80+QRX/mIhfR66u0Ykc4QTkReaejOM2ILh+9Q==} - dependencies: - '@types/node': 18.0.6 - dev: false - /@types/cacheable-request/6.0.2: resolution: {integrity: sha512-B3xVo+dlKM6nnKTcmm5ZtY/OL8bOAOd2Olee9M1zft65ox50OzjEHW91sDiU9j6cvW8Ejg1/Qkf4xd2kugApUA==} dependencies: @@ -7824,10 +7797,6 @@ packages: resolution: {integrity: sha512-i/rtaJFCsPljrZvP/akBqEwUP2y5cZLOmvO+JaYnz01aPknrQ+hB5MRcO7iqCUsFaYfTG8kGfKUyboA07xeDHQ==} dev: true - /@types/pako/1.0.4: - resolution: {integrity: sha512-Z+5bJSm28EXBSUJEgx29ioWeEEHUh6TiMkZHDhLwjc9wVFH+ressbkmX6waUZc5R3Gobn4Qu5llGxaoflZ+yhA==} - dev: false - /@types/parse-json/4.0.0: resolution: {integrity: sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==} dev: true @@ -9024,11 +8993,6 @@ packages: resolution: {integrity: sha512-gvW7InbIyF8AicrqWoptdW08pUxuhq8BEgowNajy9RhiE86fmGAGl+bLKo6oB8QP0CkqHLowfN0oJdKC/J6LbA==} dev: false - /btoa/1.2.1: - resolution: {integrity: sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==} - engines: {node: '>= 0.4.0'} - dev: false - /buffer-crc32/0.2.13: resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} dev: false @@ -10292,11 +10256,6 @@ packages: stream-shift: 1.0.1 dev: false - /durations/3.4.2: - resolution: {integrity: sha512-V/lf7y33dGaypZZetVI1eu7BmvkbC4dItq12OElLRpKuaU5JxQstV2zHwLv8P7cNbQ+KL1WD80zMCTx5dNC4dg==} - engines: {node: '>=0.10.0'} - dev: false - /eastasianwidth/0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} dev: true @@ -10426,10 +10385,6 @@ packages: resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==} dev: false - /es6-promise/4.2.8: - resolution: {integrity: sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==} - dev: false - /esbuild/0.11.10: resolution: {integrity: sha512-XvGbf+UreVFA24Tlk6sNOqNcvF2z49XAZt4E7A4H80+yqn944QOLTTxaU0lkdYNtZKFiITNea+VxmtrfjvnLPA==} requiresBuild: true @@ -14329,11 +14284,6 @@ packages: triple-beam: 1.3.0 dev: false - /loglevel/1.8.0: - resolution: {integrity: sha512-G6A/nJLRgWOuuwdNuA6koovfEV1YpqqAG4pRUlFaz3jj2QNZ8M4vBqnVA+HBTmU/AMNUtlOsMmSpF6NyOjztbA==} - engines: {node: '>= 0.6.0'} - dev: false - /long/4.0.0: resolution: {integrity: sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==} dev: false @@ -15615,10 +15565,6 @@ packages: resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==} dev: true - /pako/2.0.4: - resolution: {integrity: sha512-v8tweI900AUkZN6heMU/4Uy4cXRc2AYNRggVmTR+dEncawDJgCdLMximOVA2p4qO57WMynangsfGRb5WD6L1Bg==} - dev: false - /parallel-transform/1.2.0: resolution: {integrity: sha512-P2vSmIu38uIlvdcU7fDkyrxj33gTUy/ABO5ZUbGowxNCopBq/OoD42bP4UmMrJoPyk4Uqf0mu3mtWBhHCZD8yg==} dependencies: From 4ae7e0456529385979954af66090bc32788b7441 Mon Sep 17 00:00:00 2001 From: vellames-turing <91911778+vellames-turing@users.noreply.github.com> Date: Wed, 3 Aug 2022 11:05:03 -0300 Subject: [PATCH 06/67] Bugfix - Monday.com (#3874) * fix bug * Change Update ID to string Co-authored-by: vunguyenhung --- components/monday/actions/create-update/create-update.mjs | 2 +- .../monday/actions/update-item-name/update-item-name.mjs | 2 +- components/monday/monday.app.mjs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/monday/actions/create-update/create-update.mjs b/components/monday/actions/create-update/create-update.mjs index a4e83726f37dd..ee5b37ce61865 100644 --- a/components/monday/actions/create-update/create-update.mjs +++ b/components/monday/actions/create-update/create-update.mjs @@ -6,7 +6,7 @@ export default { name: "Create an Update", description: "Creates a new update. [See the docs here](https://api.developer.monday.com/docs/updates-queries#create-an-update)", type: "action", - version: "0.0.2", + version: "0.0.3", props: { monday, updateBody: { diff --git a/components/monday/actions/update-item-name/update-item-name.mjs b/components/monday/actions/update-item-name/update-item-name.mjs index 9494e4b80af16..3a333da21b53d 100644 --- a/components/monday/actions/update-item-name/update-item-name.mjs +++ b/components/monday/actions/update-item-name/update-item-name.mjs @@ -5,7 +5,7 @@ export default { name: "Update Item Name", description: "Update an item's name. [See the docs here](https://api.developer.monday.com/docs/item-name)", type: "action", - version: "0.0.2", + version: "0.0.3", props: { monday, boardId: { diff --git a/components/monday/monday.app.mjs b/components/monday/monday.app.mjs index b830d6cbd2f31..d13e26dd62289 100644 --- a/components/monday/monday.app.mjs +++ b/components/monday/monday.app.mjs @@ -91,7 +91,7 @@ export default { description: "The update text", }, itemId: { - type: "integer", + type: "string", label: "Item ID", description: "The item's unique identifier", optional: true, @@ -102,7 +102,7 @@ export default { }, }, updateId: { - type: "integer", + type: "string", label: "Update ID", description: "The update's unique identifier", optional: true, From d745c6d07ba2b288b17c0b3adb65ab867ee2526f Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Wed, 3 Aug 2022 11:23:32 -0300 Subject: [PATCH 07/67] fix mediaIds prop (#3896) --- components/twitter/actions/create-tweet/create-tweet.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/twitter/actions/create-tweet/create-tweet.mjs b/components/twitter/actions/create-tweet/create-tweet.mjs index 3fedfe328ab38..5ae394a677b3a 100644 --- a/components/twitter/actions/create-tweet/create-tweet.mjs +++ b/components/twitter/actions/create-tweet/create-tweet.mjs @@ -4,7 +4,7 @@ export default { key: "twitter-create-tweet", name: "Create Tweet", description: "Create a new tweet. [See the docs here](https://developer.twitter.com/en/docs/twitter-api/v1/tweets/post-and-engage/api-reference/post-statuses-update)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { twitter, @@ -124,7 +124,7 @@ export default { autoPopulateReplyMetadata, excludeReplyUserIds, attachmentUrl, - mediaIds, + media_ids: mediaIds, possiblySensitive, lat, long, From d0db2af4b4b4c18907231ed490fad422ba731c17 Mon Sep 17 00:00:00 2001 From: vellames-turing <91911778+vellames-turing@users.noreply.github.com> Date: Wed, 3 Aug 2022 11:49:28 -0300 Subject: [PATCH 08/67] [SOURCE] Slack - New Star Added (#3526) * new star added * fix bug --- components/slack/sources/common/constants.mjs | 20 +++++++++++ .../new-star-added-message.mjs | 35 ------------------- .../sources/new-star-added/new-star-added.mjs | 19 ++++++++-- 3 files changed, 37 insertions(+), 37 deletions(-) delete mode 100644 components/slack/sources/new-star-added-message/new-star-added-message.mjs diff --git a/components/slack/sources/common/constants.mjs b/components/slack/sources/common/constants.mjs index 64d3185ced9b7..9c36f312d5863 100644 --- a/components/slack/sources/common/constants.mjs +++ b/components/slack/sources/common/constants.mjs @@ -5,6 +5,26 @@ const events = { channel: "Channel", }; +const eventsOptions = [ + { + label: "User", + value: "im", + }, + { + label: "Message", + value: "message", + }, + { + label: "File", + value: "file", + }, + { + label: "Channel", + value: "channel", + }, +]; + export { events, + eventsOptions, }; diff --git a/components/slack/sources/new-star-added-message/new-star-added-message.mjs b/components/slack/sources/new-star-added-message/new-star-added-message.mjs deleted file mode 100644 index 7a2435bd901ff..0000000000000 --- a/components/slack/sources/new-star-added-message/new-star-added-message.mjs +++ /dev/null @@ -1,35 +0,0 @@ -import common from "../common/base.mjs"; - -export default { - ...common, - key: "slack-new-star-added-message", - name: "New Star Added To Message (Instant)", - version: "0.0.1", - description: "Emit new event when a star is added to a message", - type: "source", - dedupe: "unique", - props: { - ...common.props, - // eslint-disable-next-line pipedream/props-description,pipedream/props-label - slackApphook: { - type: "$.interface.apphook", - appProp: "slack", - async eventNames() { - return [ - "star_added", - ]; - }, - }, - }, - methods: { - ...common.methods, - getSummary() { - return "New star added - Message"; - }, - async processEvent(event) { - if (event.item.type === "message") { - return event; - } - }, - }, -}; diff --git a/components/slack/sources/new-star-added/new-star-added.mjs b/components/slack/sources/new-star-added/new-star-added.mjs index 2de9b274e514a..0d0c11b25df62 100644 --- a/components/slack/sources/new-star-added/new-star-added.mjs +++ b/components/slack/sources/new-star-added/new-star-added.mjs @@ -1,11 +1,14 @@ import common from "../common/base.mjs"; -import { events } from "../common/constants.mjs"; +import { + events, + eventsOptions, +} from "../common/constants.mjs"; export default { ...common, key: "slack-new-star-added", name: "New Star Added (Instant)", - version: "0.0.2", + version: "0.0.3", description: "Emit new event when a star is added to an item", type: "source", dedupe: "unique", @@ -21,11 +24,23 @@ export default { ]; }, }, + eventTypes: { + type: "string[]", + label: "Event Types", + description: "The types of event to emit. If not specified, all events will be emitted.", + options: eventsOptions, + optional: true, + }, }, methods: { ...common.methods, getSummary({ item: { type } }) { return `New star added - ${events[type] ?? type}`; }, + async processEvent(event) { + if (this.eventTypes?.length === 0 || this.eventTypes.includes(event.item.type)) { + return event; + } + }, }, }; From 5ed1c19bc1645e44e8787f9241f0da8d4409de0a Mon Sep 17 00:00:00 2001 From: Dylan Pierce Date: Wed, 3 Aug 2022 11:01:12 -0400 Subject: [PATCH 09/67] Fixing old migration guide docs --- docs/docs/migrate-from-v1/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/migrate-from-v1/README.md b/docs/docs/migrate-from-v1/README.md index 2707009532f27..834534501e60f 100644 --- a/docs/docs/migrate-from-v1/README.md +++ b/docs/docs/migrate-from-v1/README.md @@ -309,7 +309,7 @@ You can still edit a deployed workflow, just like in v1 but automatic version ro In the v2 builder, you can still view individual events that trigger your v2 workflows in the **Inspector** events log. You can delete specific events or all of them in one click as well. -However, at this time it's replaying past events against your deploy v2 workflows is not possible. +To replay past events against your deploy v2 workflows, open the event's menu and click **Replay Event**. This will rerun your workflow with this same event. ## FAQs From 68d31770ddc3cd2b8cd33c11ab22cc3282a6c438 Mon Sep 17 00:00:00 2001 From: Dylan Pierce Date: Wed, 3 Aug 2022 11:42:27 -0400 Subject: [PATCH 10/67] Adding PoC auth and data store docs (#3892) * Adding PoC auth and data store docs * Adding gifs and TOC links * Fixing links * edits * Warning preamble * Fixing alpha link, updating slack channel to use general * Adding python support --- docs/docs/.vuepress/configs/sidebarConfig.js | 7 +- docs/docs/code/python/README.md | 138 ++++++---- docs/docs/code/python/auth/README.md | 107 ++++++++ .../code/python/using-data-stores/README.md | 238 ++++++++++++++++++ docs/docs/data-stores/README.md | 2 +- 5 files changed, 436 insertions(+), 56 deletions(-) create mode 100644 docs/docs/code/python/auth/README.md create mode 100644 docs/docs/code/python/using-data-stores/README.md diff --git a/docs/docs/.vuepress/configs/sidebarConfig.js b/docs/docs/.vuepress/configs/sidebarConfig.js index dad6769c7851d..3bfb4bde9d753 100644 --- a/docs/docs/.vuepress/configs/sidebarConfig.js +++ b/docs/docs/.vuepress/configs/sidebarConfig.js @@ -46,7 +46,12 @@ const docsNav = [ { title: "Python", type: "group", - children: ["/code/python/", "/code/python/import-mappings/",] + children: [ + "/code/python/", + "/code/python/auth/", + "/code/python/using-data-stores/", + "/code/python/import-mappings/", + ] }, "/code/go/", { diff --git a/docs/docs/code/python/README.md b/docs/docs/code/python/README.md index 7c98960c6f4c8..22b9750551294 100644 --- a/docs/docs/code/python/README.md +++ b/docs/docs/code/python/README.md @@ -7,9 +7,9 @@ Pipedream supports [Python v{{$site.themeConfig.PYTHON_VERSION}}](https://www.py ::: warning Python steps are available in a limited alpha release. -You can still run arbitrary Python code, including [sharing data between steps](/code/python/#sharing-data-between-steps) as well as [accessing environment variables](/code/python/#using-environment-variables). +You can still run arbitrary Python code, including [sharing data between steps](/code/python/#sharing-data-between-steps), [send API requests using connected accounts](/code/python/auth/), [use Data Stores](/code/python/using-data-stores/), and [accessing environment variables](/code/python/#using-environment-variables). -However, you can't connect accounts, return HTTP responses, or take advantage of other features available in the [Node.js](/code/nodejs/) environment at this time. If you have any questions please [contact support](https://pipedream.com/support). +However, you can't delay or retry steps, or take advantage of other features available in the [Node.js](/code/nodejs/) environment at this time. If you have any questions please [contact support](https://pipedream.com/support). ::: @@ -19,6 +19,26 @@ However, you can't connect accounts, return HTTP responses, or take advantage of 2. Click **Custom Code** 3. In the new step, select the `python` language runtime in language dropdown +## Python Code Step Structure + +A new Python Code step will have the following structure, with a `handler` method and a `pd` argument passed into it: + +```python + +def handler(pd: "pipedream"): + # Exports a variable called message with contents "Hello, World!" + pd.export("message", "Hello, World!") + +``` + +The `handler` method is called during the step's execution, and the `pd` object contains helper methods to [use Data Stores](/code/python/using-data-stores/) and make [authenticated API requests to apps](/code/python/auth/). + +* [Import data exported from other steps](/code/python/#using-data-from-another-step) +* [Export data to downstream steps](/code/python/#sending-data-downstream-to-other-steps) +* [Retrieve data from a data store](/code/python/using-data-stores/#retrieving-data) +* [Store data into a data store](/code/python/using-data-stores/#saving-data) +* [Access API credentials from connected accounts](/code/python/auth/) + ## Logging and debugging You can use `print` at any time in a Python code step to log information as the script is running. @@ -76,15 +96,16 @@ GET requests typically are for retrieving data from an API. Below is an example. ```python import requests -url = 'https://swapi.dev/api/people/1' +def handler(pd: "pipedream"): + url = 'https://swapi.dev/api/people/1' -r = requests.get(url) + r = requests.get(url) -# The response is logged in your Pipedream step results: -print(r.text) + # The response is logged in your Pipedream step results: + print(r.text) -# The response status code is logged in your Pipedream step results: -print(r.status) + # The response status code is logged in your Pipedream step results: + print(r.status) ``` ### Making a POST request @@ -92,18 +113,19 @@ print(r.status) ```python import requests -# This a POST request to this URL will echo back whatever data we send to it -url = 'https://postman-echo.com/post' +def handler(pd: "pipedream"): + # This a POST request to this URL will echo back whatever data we send to it + url = 'https://postman-echo.com/post' -data = {"name": "Bulbasaur"} + data = {"name": "Bulbasaur"} -r = requests.post(url, data) + r = requests.post(url, data) -# The response is logged in your Pipedream step results: -print(r.text) + # The response is logged in your Pipedream step results: + print(r.text) -# The response status code is logged in your Pipedream step results: -print(r.status) + # The response status code is logged in your Pipedream step results: + print(r.status) ``` ### Sending files @@ -113,10 +135,13 @@ You can also send files within a step. An example of sending a previously stored file in the workflow's `/tmp` directory: ```python -# Retrieving a previously saved file from workflow storage -files = {'image': open('/tmp/python-logo.png', 'rb')} +import requests + +def handler(pd: "pipedream"): + # Retrieving a previously saved file from workflow storage + files = {'image': open('/tmp/python-logo.png', 'rb')} -r = requests.post(url='https://api.imgur.com/3/image', files=files) + r = requests.post(url='https://api.imgur.com/3/image', files=files) ``` ## Sharing data between steps @@ -125,11 +150,12 @@ A step can accept data from other steps in the same workflow, or pass data downs ### Using data from another step -In Python steps, data from the initial workflow trigger and other steps are available in the `pipedream.script_helpers.export` module. +In Python steps, data from the initial workflow trigger and other steps are available in the `pd.steps` object. -In this example, we'll pretend this data is coming into our HTTP trigger via POST request. +In this example, we'll pretend this data is coming into our workflow's HTTP trigger via POST request. ```json +// POST .m.pipedream.net { "id": 1, "name": "Bulbasaur", @@ -137,35 +163,34 @@ In this example, we'll pretend this data is coming into our HTTP trigger via POS } ``` -In our Python step, we can access this data in the `exports` variable from the `pipedream.script_helpers` module. Specifically, this data from the POST request into our workflow is available in the `trigger` dictionary item. +In our Python step, we can access this data in the `exports` variable from the `pd.steps` object passed into the `handler`. Specifically, this data from the POST request into our workflow is available in the `trigger` dictionary item. ```python -from pipedream.script_helpers import (steps, export) - -# retrieve the data points from the HTTP request in the initial workflow trigger -pokemon_name = steps["trigger"]["event"]["name"] -pokemon_type = steps["trigger"]["event"]["type"] +def handler(pd: "pipedream"): + # retrieve the data from the HTTP request in the initial workflow trigger + pokemon_name = pd.steps["trigger"]["event"]["name"] + pokemon_type = pd.steps["trigger"]["event"]["type"] -print(f"{pokemon_name} is a {pokemon_type} type Pokemon") + print(f"{pokemon_name} is a {pokemon_type} type Pokemon") ``` ### Sending data downstream to other steps -To share data created, retrieved, transformed or manipulated by a step to others downstream call the `export` module from `pipedream.script_helpers`. +To share data created, retrieved, transformed or manipulated by a step to others downstream call the `pd.export` method: ```python # This step is named "code" in the workflow -from pipedream.script_helpers import (steps, export) -r = requests.get("https://pokeapi.co/api/v2/pokemon/charizard") -# Store the JSON contents into a variable called "pokemon" -pokemon = r.json() +def handler(pd: "pipedream): + r = requests.get("https://pokeapi.co/api/v2/pokemon/charizard") + # Store the JSON contents into a variable called "pokemon" + pokemon = r.json() -# Expose the pokemon data downstream to others steps in the "pokemon" key from this step -export('pokemon', pokemon) + # Expose the data to other steps in the "pokemon" key from this step + export('pokemon', pokemon) ``` -Now this `pokemon` data is accessible to downstream steps within `steps["code"]["pokemon"]` +Now this `pokemon` data is accessible to downstream steps within `pd.steps["code"]["pokemon"]` ::: warning You can only export JSON-serializable data from steps. Things like: @@ -186,9 +211,10 @@ To access them, use the `os` module. import os import requests -token = os.environ['TWITTER_API_KEY'] +def handler(pd: "pipedream"): + token = os.environ['TWITTER_API_KEY'] -print(token) + print(token) ``` Or an even more useful example, using the stored environment variable to make an authenticated API request. @@ -203,14 +229,15 @@ This proves your identity to the service so you can interact with it: import requests import os -token = os.environ['TWITTER_API_KEY'] +def handler(pd: "pipedream"): + token = os.environ['TWITTER_API_KEY'] -url = 'https://api.twitter.com/2/users/@pipedream/mentions' + url = 'https://api.twitter.com/2/users/@pipedream/mentions' -headers { 'Authorization': f"Bearer {token}"} -r = requests.get(url, headers=headers) + headers { 'Authorization': f"Bearer {token}"} + r = requests.get(url, headers=headers) -print(r.text) + print(r.text) ``` :::tip @@ -247,13 +274,14 @@ You have full access to read and write both files in `/tmp`. ```python import requests -# Download the Python logo -r = requests.get('https://www.python.org/static/img/python-logo@2x.png') +def handler(pd: "pipedream"): + # Download the Python logo + r = requests.get('https://www.python.org/static/img/python-logo@2x.png') -# Create a new file python-logo.png in the /tmp/data directory -with open('/tmp/python-logo.png', 'wb') as f: - # Save the content of the HTTP response into the file - f.write(r.content) + # Create a new file python-logo.png in the /tmp/data directory + with open('/tmp/python-logo.png', 'wb') as f: + # Save the content of the HTTP response into the file + f.write(r.content) ``` Now `/tmp/python-logo.png` holds the official Python logo. @@ -265,9 +293,10 @@ You can also open files you have previously stored in the `/tmp` directory. Let' ```python import os -with open('/tmp/python-logo.png') as f: - # Store the contents of the file into a variable - file_data = f.read() +def handler(pd: "pipedream"): + with open('/tmp/python-logo.png') as f: + # Store the contents of the file into a variable + file_data = f.read() ``` ### Listing files in /tmp @@ -277,8 +306,9 @@ If you need to check what files are currently in `/tmp` you can list them and pr ```python import os -# Prints the files in the tmp directory -print(os.listdir('/tmp')) +def handler(pd: "pipedream"): + # Prints the files in the tmp directory + print(os.listdir('/tmp')) ``` :::warning diff --git a/docs/docs/code/python/auth/README.md b/docs/docs/code/python/auth/README.md new file mode 100644 index 0000000000000..4e10cec9b1b89 --- /dev/null +++ b/docs/docs/code/python/auth/README.md @@ -0,0 +1,107 @@ +--- +short_description: Connect to apps with Python code with ease. +thumbnail: https://res.cloudinary.com/pipedreamin/image/upload/v1646763806/docs/icons/icons8-connected-96_fcbhxc.png +--- + +# Connecting apps in Python + +:::warning + +This is an experimental feature and is available to to enable or disable in the [alpha](https://pipedream.com/alpha). + +There may be changes to this feature while we prepare it for a full release. + +::: + +When you use [prebuilt actions](/components#actions) tied to apps, you don't need to write the code to authorize API requests. Just [connect your account](/connected-accounts/#connecting-accounts) for that app and run your workflow. + +But sometimes you'll need to [write your own code](/code/python/). You can also connect apps to custom code steps, using the auth information to authorize requests to that app. + +For example, you may want to send a Slack message from a step. We use Slack's OAuth integration to authorize sending messages from your workflows. + +Add Slack as an app on the Python step, then connect your Slack account. + +![Add your Slack account to a Python code step by adding it](https://res.cloudinary.com/pipedreamin/image/upload/v1658954165/docs/components/CleanShot_2022-07-27_at_16.35.37_ytofp2.gif) + +Then within the Python code step, `pd.steps["slack"]["$auth"]["oauth_access_token"]` will contain your Slack account OAuth token. + +With that token, you can make authenticated API calls to Slack: + +```python +from slack_sdk import WebClient + +def handler(pd: "pipedream"): + # Your Slack OAuth token is available under pd.inputs + token = pd.inputs["slack"]["$auth"]["oauth_access_token"] + + # Instantiate a new Slack client with your token + client = WebClient(token=token) + + # Use the client to send messages to Slack channels + response = client.chat_postMessage( + channel='#general', + text='Hello from Pipedream!' + ) + + # Export the Slack response payload for use in future steps + pd.export("response", response.data) +``` + + +[[toc]] + +## Accessing connected account data with `pd.inputs[appName]["$auth"]` + +In our Slack example above, we created a Slack `WebClient` using the Slack OAuth access token: + +```python + # Instantiate a new Slack client with your token + client = WebClient(token=token) +``` + +Where did `pd.inputs["slack"]` come from? Good question. It was generated when we connected Slack to our Python step. + +![The Slack app generates the pd.inputs["slack"] data](https://res.cloudinary.com/pipedreamin/image/upload/v1658954351/docs/components/CleanShot_2022-07-27_at_16.38.42_f08ocy.png) + +The Slack access token is generated by Pipedream, and is available to this step in the `pd.inputs[appName]["$auth"]` object: + +```python +from slack_sdk import WebClient + +def handler(pd: "pipedream"): + token = pd.inputs["slack"]["$auth"]["oauth_access_token"] + # Authentication details for all of your apps are accessible under the special pd.inputs["slack"] variable: + console.log(pd.inputs["slack"]["$auth"]) +``` + +`pd.inputs["slack"]["$auth"]` contains named properties for each account you connect to the associated step. Here, we connected Slack, so `this.slack.$auth` contains the Slack auth info (the `oauth_access_token`). + +The names of the properties for each connected account will differ with the account. Pipedream typically exposes OAuth access tokens as `oauth_access_token`, and API keys under the property `api_key`. But if there's a service-specific name for the tokens (for example, if the service calls it `server_token`), we prefer that name, instead. + +To list the `pd.inputs["slack"]["$auth"]` properties available to you for a given app, just print the contents of the `$auth` property: + +```python +print(pd.inputs["slack"]["$auth"]) # Replace "slack" with your app's name +``` + +and run your workflow. You'll see the property names in the logs below your step. + +### Using the code templates tied to apps + +When you write custom code that connects to an app, you can start with a code snippet Pipedream provides for each app. This is called the **test request**. + +When you search for an app in a step: + +1. Click the **+** button below any step. +2. Search for the app you're looking for and select it from the list. +3. Select the option to **Run Python with any [app] API**. + +![Create Python API scaffolding for any app](https://res.cloudinary.com/pipedreamin/image/upload/v1658954462/docs/components/CleanShot_2022-07-27_at_16.40.41_isy1e6.png) + +This code operates as a template you can extend, and comes preconfigured with the connection to the target app and the code for authorizing requests to the API. You can modify this code however you'd like. + +## Custom auth tokens / secrets + +When you want to connect to a 3rd party service that isn't supported by Pipedream, you can store those secrets in [Environment Variables](/environment-variables/). + +