GitHub - Add GET request to certain Instant Webhook-Based Sources (Pi…

…pedreamHQ#3938)

* new-card-in-column historical events

* webhook-event updates

* new-commit historical events

* updates per review

components/github/github.app.mjs

@@ -83,6 +83,20 @@ export default {
  }));
  },
  },
+ branch: {
+ label: "Branch",
+ description: "Branch to monitor for new commits",
+ type: "string",
+ async options({ repoFullname }) {
+ const branches = await this.getBranches({
+ repoFullname,
+ });
+ return branches.map((branch) => ({
+ label: branch.name,
+ value: branch.commit.sha,
+ }));
+ },
+ },
  pullNumber: {
  type: "integer",
  label: "PR Number",
@@ -136,7 +150,7 @@ export default {
  async removeWebhook({
  repoFullname, webhookId,
  }) {
- return this._client().request(`DELETE /webhooks/${repoFullname}/hooks/${webhookId}`, {});
+ return this._client().request(`DELETE /repos/${repoFullname}/hooks/${webhookId}`, {});
  },
  async getOrganizations() {
  const response = await this._client().request("GET /user/orgs", {});
@@ -273,5 +287,29 @@ export default {
 
  return response.data;
  },
+ async getCommits({
+ repoFullname, ...data
+ }) {
+ const { data: commits } = await this._client().request(`GET /repos/${repoFullname}/commits`, {
+ ...data,
+ });
+ return commits;
+ },
+ async getBranches({
+ repoFullname, ...data
+ }) {
+ const { data: branches } = await this._client().request(`GET /repos/${repoFullname}/branches`, {
+ ...data,
+ });
+ return branches;
+ },
+ async getProjectCards({
+ columnId, ...data
+ }) {
+ const { data: cards } = await this._client().request(`GET /projects/columns/${columnId}/cards`, {
+ ...data,
+ });
+ return cards;
+ },
  },
 };

components/github/readme.md

@@ -249,3 +249,9 @@ If an issue _doesn't_ yet exist and you need to create one, please [use the issu
 You can read about our platform security and privacy [here](https://pipedream.com/docs/privacy-and-security/).
 
 If you'd like to report a suspected vulnerability or security issue, or have any questions about the security of the product, please contact our security team at **[email protected]**.
+
+## Troubleshooting
+
+Note: Event Source [New Card in Column](https://github.com/PipedreamHQ/pipedream/blob/master/components/github/sources/new-card-in-column/new-card-in-column.mjs) only supports legacy (classic) projects.
+
+Please [reach out](https://pipedream.com/support/) to the Pipedream team with any technical issues or questions about the Github integration. We're happy to help!

components/github/sources/common/common-webhook.mjs

@@ -25,8 +25,17 @@ export default {
  getWebhookEvents() {
  throw new Error("getWebhookEvents is not implemented");
  },
+ generateMeta() {
+ throw new Error("generateMeta is not implemented");
+ },
+ loadHistoricalEvents() {
+ return true;
+ },
  },
  hooks: {
+ async deploy() {
+ await this.loadHistoricalEvents();
+ },
  async activate() {
  const response = await this.github.createWebhook({
  repoFullname: this.repoFullname,
@@ -39,7 +48,6 @@ export default {
  events: this.getWebhookEvents(),
  },
  });
-
  this._setWebhookId(response.id);
  },
  async deactivate() {

components/github/sources/new-card-in-column/new-card-in-column.mjs

@@ -5,7 +5,7 @@ export default {
  key: "github-new-card-in-column",
  name: "New Card in Column (Instant)",
  description: "Emit new event when a project card is created or moved to a specific column",
- version: "0.0.1",
+ version: "0.1.0",
  type: "source",
  props: {
  ...common.props,
@@ -59,24 +59,39 @@ export default {
  ts: Date.parse(card.updated_at),
  };
  },
+ async loadHistoricalEvents() {
+ const cards = await this.github.getProjectCards({
+ columnId: this.getThisColumnValue(),
+ per_page: 25,
+ });
+ for (const card of cards) {
+ await this.processCard(card);
+ }
+ },
+ async processCard(card) {
+ const meta = this.generateMeta(card);
+ const issue = await this.github.getIssueFromProjectCard({
+ repoFullname: this.repoFullname,
+ cardId: card.id,
+ });
+ this.$emit({
+ card,
+ issue,
+ }, meta);
+ },
  },
  async run(event) {
  const card = event.body.project_card;
+ if (!card) {
+ console.log("No card in event. Skipping event.");
+ return;
+ }
 
  if (!this.isCardInThisColumn(card)) {
  console.log(`Card not in ${this.getThisColumnLabel()}. Skipping...`);
  return;
  }
 
- const meta = this.generateMeta(card);
- const issue = await this.github.getIssueFromProjectCard({
- repoFullName: this.repoFullname,
- cardId: card.id,
- });
-
- this.$emit({
- card,
- issue,
- }, meta);
+ this.processCard(card);
  },
 };

components/github/sources/new-commit/new-commit.mjs

@@ -0,0 +1,81 @@
+import common from "../common/common-webhook.mjs";
+
+export default {
+ ...common,
+ key: "github-new-commit",
+ name: "New Commit (Instant)",
+ description: "Emit new events on new commits to a repo or branch",
+ version: "0.1.0",
+ type: "source",
+ dedupe: "unique",
+ props: {
+ ...common.props,
+ branch: {
+ propDefinition: [
+ common.props.github,
+ "branch",
+ (c) => ({
+ repoFullname: c.repoFullname,
+ }),
+ ],
+ description: "Branch to monitor for new commits. Defaults to master",
+ optional: true,
+ withLabel: true,
+ },
+ },
+ methods: {
+ ...common.methods,
+ getWebhookEvents() {
+ return [
+ "push",
+ ];
+ },
+ isEventForThisBranch(branch) {
+ return !this.branch || branch === this.branch.label;
+ },
+ generateMeta(data) {
+ return {
+ id: data.id,
+ summary: data.message,
+ ts: Date.parse(data.timestamp),
+ };
+ },
+ async loadHistoricalEvents() {
+ const commitInfo = await this.github.getCommits({
+ repoFullname: this.repoFullname,
+ sha: this.branch
+ ? this.branch.value
+ : undefined,
+ per_page: 25,
+ });
+ const commits = commitInfo.map((info) => ({
+ id: info.commit.url.split("/").pop(),
+ timestamp: info.commit.committer.date,
+ ...info.commit,
+ }));
+ this.processCommits(commits);
+ },
+ processCommits(commits) {
+ for (const commit of commits) {
+ const meta = this.generateMeta(commit);
+ this.$emit(commit, meta);
+ }
+ },
+ },
+ async run(event) {
+ const { body } = event;
+
+ // skip initial response from Github
+ if (body?.zen) {
+ console.log(body.zen);
+ return;
+ }
+
+ const branch = body.ref.split("refs/heads/").pop();
+ if (!this.isEventForThisBranch(branch)) {
+ return;
+ }
+
+ this.processCommits(body.commits);
+ },
+};

components/github/sources/webhook-event/webhook-event.mjs

@@ -3,16 +3,16 @@ import constants from "../common/constants.mjs";
 
 export default {
  ...common,
- key: "github-weebhook-vents",
+ key: "github-webhook-events",
  name: "New Webhook Event (Instant)",
- description: "Emit new event for each selected event types",
+ description: "Emit new event for each selected event type",
  type: "source",
- version: "0.0.1",
+ version: "0.0.2",
  props: {
  ...common.props,
  events: {
  label: "Webhook Events",
- description: "The event will be emited",
+ description: "The event types to be emited",
  type: "string[]",
  options: constants.REPOSITORY_WEBHOOK_EVENTS,
  },
@@ -30,6 +30,12 @@ export default {
  body,
  } = event;
 
+ // skip initial response from Github
+ if (body?.zen) {
+ console.log(body.zen);
+ return;
+ }
+
  this.$emit(body, {
  id: headers["x-github-delivery"],
  summary: `New event ${headers["x-github-hook-installation-target-id"]} of type ${headers["x-github-hook-installation-target-type"]}}`,