add github template

src/create.ts

@@ -62,15 +62,45 @@ export async function create(effects: CreateEffects = defaultEffects): Promise<v
  placeholder: inferTitle(rootPath!),
  defaultValue: inferTitle(rootPath!)
  }),
- includeSampleFiles: () =>
+ template: () =>
  clack.select({
  message: "Include sample files to help you get started?",
  options: [
- {value: true, label: "Yes, include sample files", hint: "recommended"},
- {value: false, label: "No, create an empty project"}
+ {value: "default", label: "Yes, include sample files", hint: "recommended"},
+ {value: "empty", label: "No, create an empty project"},
+ {value: "github", label: "Yes, show me my GitHub stats"}
  ],
- initialValue: true
+ initialValue: "default"
  }),
+ GITHUB_TOKEN: ({results: {template}}) => {
+ if (template === "github") {
+ return clack.text({
+ message: "Your GitHub personal access token",
+ placeholder: "ghp_xxxxxxxx"
+ // validate: // TODO async?
+ });
+ }
+ },
+ asyncValidateToken: async ({results: {GITHUB_TOKEN}}) => {
+ if (GITHUB_TOKEN) {
+ const headers = {
+ authorization: `token ${GITHUB_TOKEN}`,
+ accept: "application/vnd.github.v3+json"
+ };
+ const response = await fetch("https://api.github.com/user", {headers});
+ return !response.ok
+ ? clack.log.error("Invalid token! Please review your settings in the .env file.")
+ : clack.log.success(`Token validated for GitHub user ${(await response.json()).login}`);
+ }
+ },
+ GITHUB_ORG_REPOS: async ({results: {GITHUB_TOKEN}}) => {
+ if (GITHUB_TOKEN) {
+ return clack.text({
+ message: "[Optional] The GitHub organization, or list of repos, that you want to visualize",
+ placeholder: "@observablehq or @observablehq/plot, mrdoobs/three"
+ });
+ }
+ },
  packageManager: () =>
  clack.select({
  message: "Install dependencies?",
@@ -85,13 +115,14 @@ export async function create(effects: CreateEffects = defaultEffects): Promise<v
  clack.confirm({
  message: "Initialize git repository?"
  }),
- installing: async ({results: {rootPath, projectTitle, includeSampleFiles, packageManager, initializeGit}}) => {
+ installing: async ({
+ results: {rootPath, projectTitle, template, packageManager, initializeGit, GITHUB_TOKEN, GITHUB_ORG_REPOS}
+ }) => {
  rootPath = untildify(rootPath!);
  let spinning = true;
  const s = clack.spinner();
  s.start("Copying template files");
- const template = includeSampleFiles ? "default" : "empty";
- const templateDir = op.resolve(fileURLToPath(import.meta.url), "..", "..", "templates", template);
+ const templateDir = op.resolve(fileURLToPath(import.meta.url), "..", "..", "templates", template!);
  const runCommand = packageManager === "yarn" ? "yarn" : `${packageManager ?? "npm"} run`;
  const installCommand = `${packageManager ?? "npm"} install`;
  await effects.sleep(1000); // this step is fast; give the spinner a chance to show
@@ -108,6 +139,11 @@ export async function create(effects: CreateEffects = defaultEffects): Promise<v
  },
  effects
  );
+ if (GITHUB_TOKEN) {
+ s.message("Saving your parameters in the .env file");
+ const env = makeGitHubTemplateEnv(GITHUB_TOKEN as string, GITHUB_ORG_REPOS);
+ await effects.writeFile(join(rootPath, ".env"), env);
+ }
  if (packageManager) {
  s.message(`Installing dependencies via ${packageManager}`);
  if (packageManager === "yarn") await writeFile(join(rootPath, "yarn.lock"), "");
@@ -230,3 +266,15 @@ function inferPackageManager(defaultValue: string | null): string | null {
  if (!name || !version) return defaultValue;
  return name;
 }
+
+function makeGitHubTemplateEnv(GITHUB_TOKEN: string, GITHUB_ORG_REPOS: any): string {
+ let env = `GITHUB_TOKEN=${JSON.stringify(GITHUB_TOKEN)}\n`;
+ if (typeof GITHUB_ORG_REPOS === "string" && GITHUB_ORG_REPOS) {
+ if (!GITHUB_ORG_REPOS.includes(",") && !GITHUB_ORG_REPOS.includes("/")) {
+ env += `GITHUB_ORG=${JSON.stringify(GITHUB_ORG_REPOS.trim().replace(/^@/, ""))}\n`;
+ } else {
+ env += `GITHUB_REPOS=${JSON.stringify(GITHUB_ORG_REPOS)}\n`;
+ }
+ }
+ return env;
+}

templates/github/README.md

@@ -0,0 +1,47 @@
+# GitHub Stats
+
+This is an [Observable Framework](https://observablehq.com/framework) project. To start the local preview server, run:
+
+```
+yarn dev
+```
+
+Then visit <http:https://localhost:3000> to preview your project.
+
+For more, see <https://observablehq.com/framework/getting-started>.
+
+## Configuration
+
+_To run this project, please enter the following information_
+
+Your GitHub access token:
+
+<kbd>GITHUB_TOKEN="gh_xxxxx"</kbd>
+
+This personal access token identifies you as a user, and allows the data loaders to make many requests on your behalf on GitHub’s API.
+
+By default, we collect information on the repos you own or have contributed to. If some of the repos you want to analyze are private, make sure to grant access to them when you define your token.
+
+Instead of looking at your own repos, you might want to indicate an organization name:
+
+<kbd>GITHUB_ORG="observablehq"</kbd>
+
+in that case, the data loaders will retrieve information on repos from that organization.
+
+Alternatively, specify an explicit list of repos to visualize, separated by commas:
+
+<kbd>GITHUB_REPOS="observablehq/framework, mrdoob/three.js"</kbd>
+
+<div class="tip">
+
+TIP: This information is stored in the <code>.env</code> file at the root of the project. It is used by data loaders to request information from GitHub. It is _not_ shared anywhere else on the Internet. If you deploy the build as a public project, visitors will see the list of repos and the information extracted from GitHub, but they will not have access to your PAT.
+
+</div>
+
+## Limits
+
+By default, we set some limits, which you can override in the `.env` file:
+
+- `MAX_REPOS` - number of repos to scan for details; defaults to 10
+- `MAX_COMMITS` - number of commits per repo; defaults to 1,000
+- `MAX_ISSUES` - number of issues per repo; defaults to 1,000

templates/github/docs/clone.json.ts

@@ -0,0 +1,105 @@
+import { GITHUB_TOKEN, MAX_COMMITS } from "./config.js";
+
+import type { Walker } from "isomorphic-git";
+import git from "isomorphic-git";
+import http from "isomorphic-git/http/node";
+import { fs } from "memfs";
+import { getRepos } from "./github-repos.js";
+
+const data: any[] = [];
+
+const { repos } = await getRepos();
+
+for (const { nameWithOwner: repo } of repos) {
+ console.warn("analyzing", repo);
+ try {
+ data.push(await analyzeRepo(repo));
+ } catch (error) {
+ console.warn(error);
+ }
+}
+
+console.log(JSON.stringify(data));
+
+// https://isomorphic-git.org/docs/en/clone
+async function analyzeRepo(repo: string): Promise<Object> {
+ console.warn("cloning", `github.com/${repo}`);
+ const dir = `/tmp/${repo}`;
+
+ await git.clone({
+ fs,
+ http,
+ dir,
+ depth: MAX_COMMITS, // shallow clone
+ noCheckout: true,
+ singleBranch: true,
+ noTags: true,
+ onProgress: () => void process.stderr.write("."),
+ url: `https://${GITHUB_TOKEN}@github.com/${repo}`,
+ });
+
+ console.warn("\nreading commits");
+ const commits: Object[] = [];
+
+ for (const { oid: id, commit, payload } of await git.log({
+ fs,
+ dir,
+ depth: MAX_COMMITS,
+ ref: "main",
+ })) {
+ let prev;
+ try {
+ const {
+ message,
+ author: { name: author },
+ parent: [parentid],
+ } = commit;
+ process.stderr.write("+");
+ const current = git.TREE({ ref: id });
+ const parent =
+ prev?.id === parentid ? prev.tree : git.TREE({ ref: parentid });
+ commits.push({
+ id,
+ date: getDateFromPayload(payload),
+ author,
+ message,
+ files: await getFileStateChanges(current, parent, dir),
+ });
+ prev = { id, tree: current };
+ } catch (error) {
+ break; // missing commit, we've reached the end of the shallow clone
+ }
+ }
+ console.warn("!\n");
+
+ return { repo, commits };
+}
+
+// git log --name-status
+async function getFileStateChanges(
+ current: Walker,
+ parent: Walker,
+ dir: string
+) {
+ const files: string[] = [];
+ await git.walk({
+ fs,
+ dir,
+ trees: [current, parent],
+ map: async function (walker, [a1, a2]) {
+ if ((await a1?.type()) === "blob") {
+ const [o1, o2] = await Promise.all([a1?.oid(), a2?.oid()]);
+ if (o1 === o2) return false;
+ files.push(walker);
+ }
+ return true;
+ },
+ });
+
+ return files;
+}
+
+function getDateFromPayload(payload: string): Date {
+ const a = payload.match(/^author .* (\d{10}) [+-]?\d{4}$/m);
+ return new Date(1000 * +(a?.[1] ?? NaN));
+}

templates/github/docs/collaborations.md

@@ -0,0 +1,134 @@
+# Collaborations
+
+The graph below represents the collaborations in the selected repositories.
+
+Nodes describe all the authors of the last ${Number(config.MAX_COMMITS).toLocaleString("en-US")} commits in any of the selected repos, and are sized by their total number of commits across the repos. Their color corresponds to the repo in which they did the largest number of commits.
+
+Edges represent collaborations. How do we detect them? Whenever two authors edit the same subset of files during a given time period (a sliding window of 10 commits), their connection accrues points. The edges of the graph are the top-scored connections.
+
+```js
+// normalization
+const maxValue = d3.max(links, (d) => d.value);
+const maxCommits = d3.max(nodes, (d) => d.commits);
+const nodeRadius = (d, i) =>
+ 2.5 + 20 * Math.pow((1 + (nodes[i].commits ?? 0)) / (1 + maxCommits), 0.33);
+const nodeGroups = d3
+ .groupSort(
+ nodes,
+ (v) => -d3.sum(v, (d) => d.commits),
+ (d) => d.group
+ )
+ .slice(0, 9);
+
+const chart = ForceGraph(
+ { nodes, links },
+ {
+ nodeId: (d) => d.id,
+ nodeGroup: (d) => (nodeGroups.includes(d.group) ? d.group : "other"),
+ nodeTitle: (d) => `${d.id}\n${d.group}`,
+ linkStrokeWidth: (l) => 5 * Math.sqrt(l.value / maxValue),
+ width,
+ nodeRadius,
+ nodeStrength: -20,
+ //linkStrength: 0.2,
+ linkDistance: 2,
+ height: width,
+ nodeGroups,
+ initialize: (simulation) =>
+ simulation
+ .force("x", d3.forceX().strength(0.05))
+ .force("y", d3.forceY().strength(0.05))
+ .force(
+ "collide2",
+ d3
+ .forceCollide()
+ .radius((d, i) => 2 + nodeRadius(d, i))
+ .strength(0.4)
+ ),
+ invalidation, // a promise to stop the simulation when the cell is re-run
+ }
+);
+display(Object.assign(chart, { style: "overflow: visible;" }));
+```
+
+```js
+const color = view(
+ Inputs.radio(["org", "repo"], { value: "repo", label: "color by" })
+);
+```
+
+```js
+import { ForceGraph } from "/components/force-graph.js";
+```
+
+```js
+const clones = FileAttachment("clone.json").json();
+const config = FileAttachment("config.json").json();
+```
+
+```js
+function distance(a, b) {
+ return (
+ a.length &&
+ b.length &&
+ d3.intersection(a, b).size ** 2 / (a.length * b.length)
+ );
+}
+```
+
+```js
+const pairs = new Map();
+const authors = new Map();
+
+for (const { repo, commits } of clones) {
+ for (let i = 0; i < commits.length; ++i) {
+ const a = commits[i];
+ if (!authors.has(a.author))
+ authors.set(a.author, {
+ id: a.author,
+ commits: 0,
+ repos: [],
+ orgs: [],
+ });
+ const author = authors.get(a.author);
+ author.commits++;
+ author.repos.push(repo);
+ author.orgs.push(repo.split("/")[0]);
+
+ for (let j = i + 1; j < commits.length && j < i + 10; ++j) {
+ const b = commits[j];
+ if (a.author === b.author) continue;
+ const pair = [a.author, b.author].sort().join("\t");
+ if (!pairs.has(pair)) pairs.set(pair, 0);
+ pairs.set(pair, pairs.get(pair) + distance(a.files, b.files));
+ }
+ }
+}
+
+const nodes = [...authors].map(([, { commits, id, repos, orgs }]) => ({
+ id,
+ commits,
+ // org: d3.mode(orgs),
+ // repo: d3.mode(repos),
+ group: d3.mode(color === "repo" ? repos : orgs),
+}));
+const links = d3
+ .sort(pairs, ([, value]) => -value)
+ .slice(0, 1000)
+ .map(([pair, value]) => ({
+ source: pair.split("\t")[0],
+ target: pair.split("\t")[1],
+ value,
+ }));
+
+// regroup the stray nodes
+const solos = d3.difference(
+ nodes.map((d) => d.id),
+ links.map((d) => d.source),
+ links.map((d) => d.target)
+);
+for (let i = 0; i < 2; ++i) {
+ for (const [source, target] of d3.pairs(d3.shuffle([...solos])))
+ links.push({ source, target, value: 5 });
+}
+```