Generate api data on each push (qmk#10609)

* add new qmk generate-api command, to generate a complete set of API data.

* Generate api data and push it to the keyboard repo

* fix typo

* Apply suggestions from code review

Co-authored-by: Joel Challis <[email protected]>

* fixup api workflow

* remove file-changes-action

* use a more mainstream github action

* fix yaml error

* Apply suggestions from code review

Co-authored-by: Erovia <[email protected]>

* more uniform date handling

* make flake8 happy

* Update lib/python/qmk/decorators.py

Co-authored-by: Erovia <[email protected]>

Co-authored-by: Joel Challis <[email protected]>
Co-authored-by: Erovia <[email protected]>

.github/workflows/api.yml

@@ -0,0 +1,35 @@
+name: Update API Data
+
+on:
+ push:
+ branches:
+ - master
+ paths:
+ - 'keyboards/**'
+ - 'layouts/community/**'
+
+jobs:
+ api_data:
+ runs-on: ubuntu-latest
+ container: qmkfm/base_container
+
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ fetch-depth: 1
+ persist-credentials: false
+
+ - name: Generate API Data
+ run: qmk generate-api
+
+ - name: Upload API Data
+ uses: JamesIves/[email protected]
+ with:
+ ACCESS_TOKEN: ${{ secrets.API_TOKEN_GITHUB }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ BRANCH: main
+ FOLDER: api_data/v1
+ CLEAN: true
+ GIT_CONFIG_EMAIL: [email protected]
+ REPOSITORY_NAME: qmk/qmk_keyboards
+ TARGET_FOLDER: v1

.gitignore

@@ -16,6 +16,7 @@
 *.swp
 tags
 *~
+api_data/v1
 build/
 .build/
 *.bak

api_data/_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-cayman

api_data/readme.md

@@ -0,0 +1,5 @@
+# QMK Keyboard Metadata
+
+This directory contains machine parsable data about keyboards supported by QMK. The latest version is always available online at <https://keyboards.qmk.fm>.
+
+Do not edit anything here by hand. It is generated with the `qmk generate-api` command.

lib/python/qmk/cli/__init__.py

@@ -13,6 +13,7 @@
 from . import docs
 from . import doctor
 from . import flash
+from . import generate
 from . import hello
 from . import info
 from . import json

lib/python/qmk/cli/c2json.py

@@ -44,7 +44,7 @@ def c2json(cli):
 
  # Generate the keymap.json
  try:
- keymap_json = qmk.keymap.generate(keymap_json['keyboard'], keymap_json['layout'], keymap_json['layers'], type='json', keymap=keymap_json['keymap'])
+ keymap_json = qmk.keymap.generate_json(keymap_json['keymap'], keymap_json['keyboard'], keymap_json['layout'], keymap_json['layers'])
  except KeyError:
  cli.log.error('Something went wrong. Try to use --no-cpp.')
  sys.exit(1)

lib/python/qmk/cli/generate/__init__.py

@@ -0,0 +1 @@
+from . import api

lib/python/qmk/cli/generate/api.py

@@ -0,0 +1,58 @@
+"""This script automates the generation of the QMK API data.
+"""
+from pathlib import Path
+from shutil import copyfile
+import json
+
+from milc import cli
+
+from qmk.datetime import current_datetime
+from qmk.info import info_json
+from qmk.keyboard import list_keyboards
+
+
+@cli.subcommand('Creates a new keymap for the keyboard of your choosing', hidden=False if cli.config.user.developer else True)
+def generate_api(cli):
+ """Generates the QMK API data.
+ """
+ api_data_dir = Path('api_data')
+ v1_dir = api_data_dir / 'v1'
+ keyboard_list = v1_dir / 'keyboard_list.json'
+ keyboard_all = v1_dir / 'keyboards.json'
+ usb_file = v1_dir / 'usb.json'
+
+ if not api_data_dir.exists():
+ api_data_dir.mkdir()
+
+ kb_all = {'last_updated': current_datetime(), 'keyboards': {}}
+ usb_list = {'last_updated': current_datetime(), 'devices': {}}
+
+ # Generate and write keyboard specific JSON files
+ for keyboard_name in list_keyboards():
+ kb_all['keyboards'][keyboard_name] = info_json(keyboard_name)
+ keyboard_dir = v1_dir / 'keyboards' / keyboard_name
+ keyboard_info = keyboard_dir / 'info.json'
+ keyboard_readme = keyboard_dir / 'readme.md'
+ keyboard_readme_src = Path('keyboards') / keyboard_name / 'readme.md'
+
+ keyboard_dir.mkdir(parents=True, exist_ok=True)
+ keyboard_info.write_text(json.dumps(kb_all['keyboards'][keyboard_name]))
+
+ if keyboard_readme_src.exists():
+ copyfile(keyboard_readme_src, keyboard_readme)
+
+ if 'usb' in kb_all['keyboards'][keyboard_name]:
+ usb = kb_all['keyboards'][keyboard_name]['usb']
+
+ if usb['vid'] not in usb_list['devices']:
+ usb_list['devices'][usb['vid']] = {}
+
+ if usb['pid'] not in usb_list['devices'][usb['vid']]:
+ usb_list['devices'][usb['vid']][usb['pid']] = {}
+
+ usb_list['devices'][usb['vid']][usb['pid']][keyboard_name] = usb
+
+ # Write the global JSON files
+ keyboard_list.write_text(json.dumps({'last_updated': current_datetime(), 'keyboards': sorted(kb_all['keyboards'])}))
+ keyboard_all.write_text(json.dumps(kb_all))
+ usb_file.write_text(json.dumps(usb_list))

lib/python/qmk/cli/info.py

@@ -16,7 +16,7 @@
 COL_LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijilmnopqrstuvwxyz'
 
 
-def show_keymap(info_json, title_caps=True):
+def show_keymap(kb_info_json, title_caps=True):
  """Render the keymap in ascii art.
  """
  keymap_path = locate_keymap(cli.config.info.keyboard, cli.config.info.keymap)
@@ -36,7 +36,7 @@ def show_keymap(info_json, title_caps=True):
  else:
  cli.echo('{fg_cyan}layer_%s{fg_reset}:', layer_num)
 
- print(render_layout(info_json['layouts'][layout_name]['layout'], layer))
+ print(render_layout(kb_info_json['layouts'][layout_name]['layout'], layer))
 
 
 def show_layouts(kb_info_json, title_caps=True):
@@ -48,10 +48,10 @@ def show_layouts(kb_info_json, title_caps=True):
  print(layout_art) # Avoid passing dirty data to cli.echo()
 
 
-def show_matrix(info_json, title_caps=True):
+def show_matrix(kb_info_json, title_caps=True):
  """Render the layout with matrix labels in ascii art.
  """
- for layout_name, layout in info_json['layouts'].items():
+ for layout_name, layout in kb_info_json['layouts'].items():
  # Build our label list
  labels = []
  for key in layout['layout']:
@@ -69,54 +69,54 @@ def show_matrix(info_json, title_caps=True):
  else:
  cli.echo('{fg_blue}matrix_%s{fg_reset}:', layout_name)
 
- print(render_layout(info_json['layouts'][layout_name]['layout'], labels))
+ print(render_layout(kb_info_json['layouts'][layout_name]['layout'], labels))
 
 
-def print_friendly_output(info_json):
+def print_friendly_output(kb_info_json):
  """Print the info.json in a friendly text format.
  """
- cli.echo('{fg_blue}Keyboard Name{fg_reset}: %s', info_json.get('keyboard_name', 'Unknown'))
- cli.echo('{fg_blue}Manufacturer{fg_reset}: %s', info_json.get('manufacturer', 'Unknown'))
- if 'url' in info_json:
- cli.echo('{fg_blue}Website{fg_reset}: %s', info_json.get('url', ''))
- if info_json.get('maintainer', 'qmk') == 'qmk':
+ cli.echo('{fg_blue}Keyboard Name{fg_reset}: %s', kb_info_json.get('keyboard_name', 'Unknown'))
+ cli.echo('{fg_blue}Manufacturer{fg_reset}: %s', kb_info_json.get('manufacturer', 'Unknown'))
+ if 'url' in kb_info_json:
+ cli.echo('{fg_blue}Website{fg_reset}: %s', kb_info_json.get('url', ''))
+ if kb_info_json.get('maintainer', 'qmk') == 'qmk':
  cli.echo('{fg_blue}Maintainer{fg_reset}: QMK Community')
  else:
- cli.echo('{fg_blue}Maintainer{fg_reset}: %s', info_json['maintainer'])
- cli.echo('{fg_blue}Keyboard Folder{fg_reset}: %s', info_json.get('keyboard_folder', 'Unknown'))
- cli.echo('{fg_blue}Layouts{fg_reset}: %s', ', '.join(sorted(info_json['layouts'].keys())))
- if 'width' in info_json and 'height' in info_json:
- cli.echo('{fg_blue}Size{fg_reset}: %s x %s' % (info_json['width'], info_json['height']))
- cli.echo('{fg_blue}Processor{fg_reset}: %s', info_json.get('processor', 'Unknown'))
- cli.echo('{fg_blue}Bootloader{fg_reset}: %s', info_json.get('bootloader', 'Unknown'))
+ cli.echo('{fg_blue}Maintainer{fg_reset}: %s', kb_info_json['maintainer'])
+ cli.echo('{fg_blue}Keyboard Folder{fg_reset}: %s', kb_info_json.get('keyboard_folder', 'Unknown'))
+ cli.echo('{fg_blue}Layouts{fg_reset}: %s', ', '.join(sorted(kb_info_json['layouts'].keys())))
+ if 'width' in kb_info_json and 'height' in kb_info_json:
+ cli.echo('{fg_blue}Size{fg_reset}: %s x %s' % (kb_info_json['width'], kb_info_json['height']))
+ cli.echo('{fg_blue}Processor{fg_reset}: %s', kb_info_json.get('processor', 'Unknown'))
+ cli.echo('{fg_blue}Bootloader{fg_reset}: %s', kb_info_json.get('bootloader', 'Unknown'))
 
  if cli.config.info.layouts:
- show_layouts(info_json, True)
+ show_layouts(kb_info_json, True)
 
  if cli.config.info.matrix:
- show_matrix(info_json, True)
+ show_matrix(kb_info_json, True)
 
  if cli.config_source.info.keymap and cli.config_source.info.keymap != 'config_file':
- show_keymap(info_json, True)
+ show_keymap(kb_info_json, True)
 
 
-def print_text_output(info_json):
+def print_text_output(kb_info_json):
  """Print the info.json in a plain text format.
  """
- for key in sorted(info_json):
+ for key in sorted(kb_info_json):
  if key == 'layouts':
- cli.echo('{fg_blue}layouts{fg_reset}: %s', ', '.join(sorted(info_json['layouts'].keys())))
+ cli.echo('{fg_blue}layouts{fg_reset}: %s', ', '.join(sorted(kb_info_json['layouts'].keys())))
  else:
- cli.echo('{fg_blue}%s{fg_reset}: %s', key, info_json[key])
+ cli.echo('{fg_blue}%s{fg_reset}: %s', key, kb_info_json[key])
 
  if cli.config.info.layouts:
- show_layouts(info_json, False)
+ show_layouts(kb_info_json, False)
 
  if cli.config.info.matrix:
- show_matrix(info_json, False)
+ show_matrix(kb_info_json, False)
 
  if cli.config_source.info.keymap and cli.config_source.info.keymap != 'config_file':
- show_keymap(info_json, False)
+ show_keymap(kb_info_json, False)
 
 
 @cli.argument('-kb', '--keyboard', help='Keyboard to show info for.')

lib/python/qmk/cli/json2c.py

@@ -38,7 +38,7 @@ def json2c(cli):
  user_keymap = json.load(fd)
 
  # Generate the keymap
- keymap_c = qmk.keymap.generate(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
+ keymap_c = qmk.keymap.generate_c(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
 
  if cli.args.output:
  cli.args.output.parent.mkdir(parents=True, exist_ok=True)

lib/python/qmk/cli/list/keyboards.py

@@ -1,28 +1,13 @@
 """List the keyboards currently defined within QMK
 """
-# We avoid pathlib here because this is performance critical code.
-import os
-import glob
-
 from milc import cli
 
-BASE_PATH = os.path.join(os.getcwd(), "keyboards") + os.path.sep
-KB_WILDCARD = os.path.join(BASE_PATH, "**", "rules.mk")
-
-
-def find_name(path):
- """Determine the keyboard name by stripping off the base_path and rules.mk.
- """
- return path.replace(BASE_PATH, "").replace(os.path.sep + "rules.mk", "")
+import qmk.keyboard
 
 
 @cli.subcommand("List the keyboards currently defined within QMK")
 def list_keyboards(cli):
  """List the keyboards currently defined within QMK
  """
- # find everywhere we have rules.mk where keymaps isn't in the path
- paths = [path for path in glob.iglob(KB_WILDCARD, recursive=True) if 'keymaps' not in path]
-
- # Extract the keyboard name from the path and print it
- for keyboard_name in sorted(map(find_name, paths)):
+ for keyboard_name in qmk.keyboard.list_keyboards():
  print(keyboard_name)

lib/python/qmk/constants.py

@@ -12,3 +12,8 @@
 CHIBIOS_PROCESSORS = 'cortex-m0', 'cortex-m0plus', 'cortex-m3', 'cortex-m4', 'MKL26Z64', 'MK20DX128', 'MK20DX256', 'STM32F042', 'STM32F072', 'STM32F103', 'STM32F303', 'STM32F401', 'STM32F411'
 LUFA_PROCESSORS = 'atmega16u2', 'atmega32u2', 'atmega16u4', 'atmega32u4', 'at90usb646', 'at90usb647', 'at90usb1286', 'at90usb1287', None
 VUSB_PROCESSORS = 'atmega32a', 'atmega328p', 'atmega328', 'attiny85'
+
+# Common format strings
+DATE_FORMAT = '%Y-%m-%d'
+DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
+TIME_FORMAT = '%H:%M:%S'

lib/python/qmk/datetime.py

@@ -0,0 +1,29 @@
+"""Functions to work with dates and times in a uniform way.
+
+The results of these functions are cached for 5 seconds to provide uniform time strings across short running processes. Long running processes that need more precise timekeeping should not use these functions.
+"""
+from time import gmtime, strftime
+
+from qmk.constants import DATE_FORMAT, DATETIME_FORMAT, TIME_FORMAT
+from qmk.decorators import lru_cache
+
+
+@lru_cache(timeout=5)
+def current_date():
+ """Returns the current time in UTZ as a formatted string.
+ """
+ return strftime(DATE_FORMAT, gmtime())
+
+
+@lru_cache(timeout=5)
+def current_datetime():
+ """Returns the current time in UTZ as a formatted string.
+ """
+ return strftime(DATETIME_FORMAT, gmtime())
+
+
+@lru_cache(timeout=5)
+def current_time():
+ """Returns the current time in UTZ as a formatted string.
+ """
+ return strftime(TIME_FORMAT, gmtime())

lib/python/qmk/decorators.py

@@ -2,6 +2,7 @@
 """
 import functools
 from pathlib import Path
+from time import monotonic
 
 from milc import cli
 
@@ -84,3 +85,38 @@ def wrapper(*args, **kwargs):
  return func(*args, **kwargs)
 
  return wrapper
+
+
+def lru_cache(timeout=10, maxsize=128, typed=False):
+ """Least Recently Used Cache- cache the result of a function.
+
+ Args:
+
+ timeout
+ How many seconds to cache results for.
+
+ maxsize
+ The maximum size of the cache in bytes
+
+ typed
+ When `True` argument types will be taken into consideration, for example `3` and `3.0` will be treated as different keys.
+ """
+ def wrapper_cache(func):
+ func = functools.lru_cache(maxsize=maxsize, typed=typed)(func)
+ func.expiration = monotonic() + timeout
+
+ @functools.wraps(func)
+ def wrapped_func(*args, **kwargs):
+ if monotonic() >= func.expiration:
+ func.expiration = monotonic() + timeout
+
+ func.cache_clear()
+
+ return func(*args, **kwargs)
+
+ wrapped_func.cache_info = func.cache_info
+ wrapped_func.cache_clear = func.cache_clear
+
+ return wrapped_func
+
+ return wrapper_cache

lib/python/qmk/info.py

@@ -9,6 +9,7 @@
 from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
 from qmk.c_parse import find_layouts
 from qmk.keyboard import config_h, rules_mk
+from qmk.keymap import list_keymaps
 from qmk.makefile import parse_rules_mk_file
 from qmk.math import compute
 
@@ -25,14 +26,21 @@ def info_json(keyboard):
  info_data = {
  'keyboard_name': str(keyboard),
  'keyboard_folder': str(keyboard),
+ 'keymaps': {},
  'layouts': {},
  'maintainer': 'qmk',
  }
 
+ # Populate the list of JSON keymaps
+ for keymap in list_keymaps(keyboard, c=False, fullpath=True):
+ info_data['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'}
+
+ # Populate layout data
  for layout_name, layout_json in _find_all_layouts(keyboard, rules).items():
  if not layout_name.startswith('LAYOUT_kc'):
  info_data['layouts'][layout_name] = layout_json
 
+ # Merge in the data from info.json, config.h, and rules.mk
  info_data = merge_info_jsons(keyboard, info_data)
  info_data = _extract_config_h(info_data)
  info_data = _extract_rules_mk(info_data)