Incremental indexing support for api

.env.example

@@ -4,3 +4,5 @@ API_DOCS_PATH=../ember-jsonapi-docs/tmp
 GUIDES_DOCS_PATH=../guides-app/dist/content
 DEBUG=false
 DRIVER=algolia
+AWS_ACCESS_KEY=""
+AWS_SECRET_KEY=""

.gitignore

@@ -3,3 +3,6 @@ node_modules
 build
 drivers-output
 dist
+yarn-error.log
+tmp
+.vscode

.prettierrc

@@ -1,4 +1,5 @@
 {
  "singleQuote": true,
- "semi": false
+ "semi": false,
+ "trailingComma": "es5"
 }

index.js

@@ -9,6 +9,10 @@ program
  .version(version, '-v, --version')
  .description(description)
  .option('-p, --project <project>', 'Project name. Accepts "api" or "guides"')
+ .option(
+ '-c, --clear-index',
+ 'Whether indexes of the project should be cleared while processing'
+ )
 
 program.on('--help', function() {
  console.log(`
@@ -25,7 +29,7 @@ switch (program.project) {
  runGuides()
  break
  case 'api':
- runApi()
+ runApi(program.clearIndex)
  break
  default:
  throw new Error('Invalid --project property')

lib/api-docs-sync.js

@@ -0,0 +1,58 @@
+import { Promise, all as waitForAllPromises } from 'bluebird'
+import S3 from 's3'
+import ora from 'ora'
+import humanSize from 'human-size'
+import http from 'http'
+import https from 'https'
+
+// To increase s3's download & upload dir perf
+http.globalAgent.maxSockets = https.globalAgent.maxSockets = 30
+
+const { AWS_ACCESS_KEY, AWS_SECRET_KEY } = process.env
+
+const client = S3.createClient({
+ s3Options: { accessKeyId: AWS_ACCESS_KEY, secretAccessKey: AWS_SECRET_KEY }
+})
+
+const jsonDocsDirDownloadOptions = {
+ localDir: 'tmp/json-docs',
+ s3Params: { Bucket: 'api-docs.emberjs.com', Prefix: 'json-docs' }
+}
+
+let revDocsDirDownloadOptions = {
+ localDir: 'tmp/rev-index',
+ s3Params: { Bucket: 'api-docs.emberjs.com', Prefix: 'rev-index' }
+}
+
+const syncDir = options => {
+ return new Promise((resolve, reject) => {
+ let sync = client.downloadDir(options)
+ let progressIndicator = ora(
+ `downloading ${options.s3Params.Prefix} docs`
+ ).start()
+
+ sync.on('progress', () => {
+ const { progressAmount, progressTotal } = sync
+ progressIndicator.text = `Downloading json docs (${humanSize(
+ progressAmount
+ )} of ${humanSize(progressTotal)})`
+ })
+
+ sync.on('end', () => {
+ progressIndicator.succeed(`downloaded ${options.s3Params.Prefix} docs`)
+ resolve()
+ })
+
+ sync.on('error', err => {
+ progressIndicator.fail()
+ reject(err)
+ })
+ })
+}
+
+export default function downloadExistingDocsToLocal() {
+ return waitForAllPromises([
+ syncDir(jsonDocsDirDownloadOptions),
+ syncDir(revDocsDirDownloadOptions)
+ ])
+}

lib/api.js

@@ -1,10 +1,14 @@
 require('dotenv').config()
 
-import Bluebird from 'bluebird'
+import { all, resolve } from 'bluebird'
+import { compare as compareSemVers } from 'semver'
+import { difference } from 'lodash'
+
 import logger from './utils/logger'
 import drivers from './drivers'
 import { readTmpFileFactory, readTmpFileAsyncFactory } from './utils/fs'
 import schemas from './schemas'
+import downloadApiDocs from './api-docs-sync'
 
 // Get 'readTmpFile' and 'readTmpFileAsync' bound by 'api'
 const PROJECT_TYPE = 'api'
@@ -15,46 +19,68 @@ const { DRIVER } = process.env
 
 const SelectedDriver = drivers[DRIVER]
 
-export function run() {
- // Initialise drivers
- SelectedDriver.init('modules')
- SelectedDriver.init('classes')
- SelectedDriver.init('methods')
-
- // Load ember.json which includes all available ember versions.
- return (
- readTmpFileAsync('rev-index/ember.json')
- // Extract available versions
- .then(emberJson => emberJson.meta.availableVersions)
- // Clear the driver contents
- .tap(clearDriver)
- // Grab the json file of each ember version
- .map(readEmberIndexFileForVersion)
+const apiIndexes = ['modules', 'classes', 'methods', 'versions']
+
+export async function run(clearIndex = false) {
+ apiIndexes.map(SelectedDriver.init)
+
+ if (clearIndex) {
+ await all(apiIndexes.map(SelectedDriver.clear))
+ }
+
+ await downloadApiDocs()
+
+ await all([processDocs('ember'), processDocs('ember-data')])
+}
+
+async function processDocs(project) {
+ let prevIndexedVersions = await SelectedDriver.getPreviouslyIndexedVersions(
+ project
+ )
+
+ const {
+ meta: { availableVersions }
+ } = await readTmpFileAsync(`rev-index/${project}.json`)
+
+ let versionsToProcess = difference(availableVersions, prevIndexedVersions)
+
+ if (versionsToProcess.length === 0) {
+ console.log(`No new versions to process for ${project}`)
+ return
+ }
+
+ try {
+ console.log(`Processing ${project} for versions: ${versionsToProcess}`)
+
+ await versionsToProcess
+ .map(version => readIndexFileForVersion(version, project))
  // Fetch all public modules and public classes
- .map(fetchPublicModuleClassesForEmberVersion)
+ .map(versionIndexObject =>
+ fetchPublicModuleClassesForVersion(versionIndexObject, project)
+ )
  // Run the schema against all data stored
  .map(mapDataForVersion)
  // Write out to selected driver.
  .map(writeToDriver)
- // Load ember-data.json which includes all available ember-data versions
- .then(() => readTmpFileAsync('rev-index/ember-data.json'))
- .then(emberJson => emberJson.meta.availableVersions)
- .map(readEmberDataIndexFileForVersion)
- .map(fetchPublicModuleClassesForEmberDataVersion)
- .map(mapDataForVersion)
- .map(writeToDriver)
- // Handle script error
- .catch(errorHandler)
- )
-}
 
-function readEmberIndexFileForVersion(version) {
- return readIndexFileForVersion(version, 'ember')
+ await SelectedDriver.write(
+ 'versions',
+ [
+ {
+ id: project,
+ name: project,
+ versions: [...prevIndexedVersions, ...versionsToProcess].sort(
+ compareSemVers
+ )
+ }
+ ],
+ project
+ )
+ } catch (err) {
+ console.log('Error:: ', err)
+ }
 }
 
-function readEmberDataIndexFileForVersion(version) {
- return readIndexFileForVersion(version, 'ember-data')
-}
 /**
  * Read index file for version
  *
@@ -68,14 +94,6 @@ function readIndexFileForVersion(version, libName) {
  return readTmpFile(emberVersionJSONPath)
 }
 
-function fetchPublicModuleClassesForEmberVersion(versionIndexObject) {
- return fetchPublicModuleClassesForVersion(versionIndexObject, 'ember')
-}
-
-function fetchPublicModuleClassesForEmberDataVersion(versionIndexObject) {
- return fetchPublicModuleClassesForVersion(versionIndexObject, 'ember-data')
-}
-
 /**
  * Fetch public modules and classes for version
  *
@@ -143,40 +161,37 @@ function mapDataForVersion(versionObject) {
  * @param versionObject - Object version to write out
  */
 function writeToDriver(versionObject) {
+ const { id } = versionObject.version.data
+
+ let tokens = id.split('-')
+ let version = tokens.pop()
+ let projectName = tokens.join('-')
+
  logger.logGreen(
- `version: ${versionObject.version.data.id}, public classes: ${
+ `version: ${id}, public classes: ${
  versionObject.publicClasses.length
  }, public modules: ${versionObject.publicModules.length}, methods: ${
  versionObject.methods.length
  }`
  )
 
- // Wait for all promises to complete before continuing
- return Bluebird.all([
- SelectedDriver.write('modules', versionObject.publicModules),
- SelectedDriver.write('classes', versionObject.publicClasses),
- SelectedDriver.write('methods', versionObject.methods)
+ return all([
+ SelectedDriver.write(
+ 'modules',
+ versionObject.publicModules,
+ projectName,
+ version
+ ),
+ SelectedDriver.write(
+ 'classes',
+ versionObject.publicClasses,
+ projectName,
+ version
+ ),
+ SelectedDriver.write('methods', versionObject.methods, projectName, version)
  ])
 }
 
-/**
- * Clears the driver indices
- *
- * @returns {Promise} - Promise with all drivers cleared.
- */
-function clearDriver() {
- return Bluebird.all([
- SelectedDriver.clear('modules'),
- SelectedDriver.clear('classes'),
- SelectedDriver.clear('methods')
- ])
-}
-
-// Handle errors
-function errorHandler(err) {
- console.log('Error:: ', err)
-}
-
 /**
  * Takes an array of classes, extracts the methods from each one,
  * and runs the method schema to transform the payload

lib/drivers/algolia.js

@@ -9,7 +9,7 @@ let client = null
 
 const indices = {}
 
-export function init(indexName) {
+function init(indexName) {
  if (!client) client = AlgoliaSearch(ALGOLIA_APP_ID, ALGOLIA_ADMIN_KEY)
  // Create an algolia index
  const index = client.initIndex(indexName)
@@ -19,7 +19,7 @@ export function init(indexName) {
  return index
 }
 
-export function write(indexName, records) {
+function write(indexName, records) {
  const index = indices[indexName]
 
  return Bluebird.resolve(index.addObjects(records))
@@ -36,13 +36,18 @@ export function write(indexName, records) {
  })
 }
 
-export function clear(indexName) {
+function clear(indexName) {
  const index = indices[indexName]
  return Bluebird.resolve(index.clearIndex())
 }
 
-export default {
- init,
- write,
- clear
+async function getPreviouslyIndexedVersions(projectName) {
+ let { hits } = await indices['versions'].search(projectName)
+ if (!hits) {
+ return []
+ }
+ let { versions } = hits.find(hit => hit.name === projectName)
+ return versions
 }
+
+export default { init, write, clear, getPreviouslyIndexedVersions }

lib/drivers/json.js

@@ -1,18 +1,40 @@
-import Bluebird from 'bluebird'
-import fs from 'file-system'
+import { resolve } from 'bluebird'
+import {
+ emptyDirSync,
+ existsSync,
+ outputJsonSync,
+ readJsonSync,
+ rmdirSync,
+} from 'fs-extra'
 
-export function init(fileName) {}
+const outputFolder = './drivers-output'
 
-export function write(fileName, content) {
- return fs.writeFileSync(
- `./drivers-output/json/${fileName}.json`,
- JSON.stringify(content),
- 'utf8'
- )
+function init() {}
+
+function write(indexName, content, projectName, version) {
+ let fileName = `${outputFolder}/json/${projectName}/`
+ fileName += version ? `${version}/${indexName}.json` : `${indexName}.json`
+
+ return outputJsonSync(fileName, content, { spaces: 2 })
+}
+
+function clear() {
+ if (!existsSync(outputFolder)) {
+ return resolve()
+ }
+ emptyDirSync(outputFolder)
+ return resolve(rmdirSync(outputFolder))
 }
 
-export function clear() {
- return Bluebird.resolve()
+function getPreviouslyIndexedVersions(projectName) {
+ let fileName = `${outputFolder}/json/${projectName}/versions.json`
+
+ if (!existsSync(fileName)) {
+ return []
+ }
+
+ let [{ versions }] = readJsonSync(fileName)
+ return versions
 }
 
-export default { init, write, clear }
+export default { init, write, clear, getPreviouslyIndexedVersions }