Community modules (#24848)

This commit is contained in:
Nick Brassel 2025-02-26 22:25:41 +11:00 committed by GitHub
parent 63b095212b
commit 1efc82403b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
37 changed files with 987 additions and 84 deletions

View file

@ -49,6 +49,7 @@ subcommands = [
'qmk.cli.generate.api',
'qmk.cli.generate.autocorrect_data',
'qmk.cli.generate.compilation_database',
'qmk.cli.generate.community_modules',
'qmk.cli.generate.config_h',
'qmk.cli.generate.develop_pr_list',
'qmk.cli.generate.dfu_header',

View file

@ -10,7 +10,7 @@ from qmk.path import normpath
from qmk.c_parse import c_source_files
c_file_suffixes = ('c', 'h', 'cpp', 'hpp')
core_dirs = ('drivers', 'quantum', 'tests', 'tmk_core', 'platforms')
core_dirs = ('drivers', 'quantum', 'tests', 'tmk_core', 'platforms', 'modules')
ignored = ('tmk_core/protocol/usb_hid', 'platforms/chibios/boards')

View file

@ -9,7 +9,7 @@ from milc import cli
from qmk.info import info_json
from qmk.json_schema import json_load, validate
from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder, UserspaceJSONEncoder
from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder, UserspaceJSONEncoder, CommunityModuleJSONEncoder
from qmk.path import normpath
@ -30,6 +30,13 @@ def _detect_json_format(file, json_data):
except ValidationError:
pass
if json_encoder is None:
try:
validate(json_data, 'qmk.community_module.v1')
json_encoder = CommunityModuleJSONEncoder
except ValidationError:
pass
if json_encoder is None:
try:
validate(json_data, 'qmk.keyboard.v1')
@ -54,6 +61,8 @@ def _get_json_encoder(file, json_data):
json_encoder = KeymapJSONEncoder
elif cli.args.format == 'userspace':
json_encoder = UserspaceJSONEncoder
elif cli.args.format == 'community_module':
json_encoder = CommunityModuleJSONEncoder
else:
# This should be impossible
cli.log.error('Unknown format: %s', cli.args.format)
@ -61,7 +70,7 @@ def _get_json_encoder(file, json_data):
@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format')
@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap', 'userspace'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)')
@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap', 'userspace', 'community_module'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)')
@cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file')
@cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ')
@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True)

View file

@ -0,0 +1,263 @@
import contextlib
from argcomplete.completers import FilesCompleter
from pathlib import Path
from milc import cli
import qmk.path
from qmk.info import get_modules
from qmk.keyboard import keyboard_completer, keyboard_folder
from qmk.commands import dump_lines
from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE
from qmk.community_modules import module_api_list, load_module_jsons, find_module_path
@contextlib.contextmanager
def _render_api_guard(lines, api):
if api.guard:
lines.append(f'#if {api.guard}')
yield
if api.guard:
lines.append(f'#endif // {api.guard}')
def _render_api_header(api):
lines = []
if api.header:
lines.append('')
with _render_api_guard(lines, api):
lines.append(f'#include <{api.header}>')
return lines
def _render_keycodes(module_jsons):
lines = []
lines.append('')
lines.append('enum {')
first = True
for module_json in module_jsons:
module_name = Path(module_json['module']).name
keycodes = module_json.get('keycodes', [])
if len(keycodes) > 0:
lines.append(f' // From module: {module_name}')
for keycode in keycodes:
key = keycode.get('key', None)
if first:
lines.append(f' {key} = QK_COMMUNITY_MODULE,')
first = False
else:
lines.append(f' {key},')
for alias in keycode.get('aliases', []):
lines.append(f' {alias} = {key},')
lines.append('')
lines.append(' LAST_COMMUNITY_MODULE_KEY')
lines.append('};')
lines.append('_Static_assert((int)LAST_COMMUNITY_MODULE_KEY <= (int)(QK_COMMUNITY_MODULE_MAX+1), "Too many community module keycodes");')
return lines
def _render_api_declarations(api, module, user_kb=True):
lines = []
lines.append('')
with _render_api_guard(lines, api):
if user_kb:
lines.append(f'{api.ret_type} {api.name}_{module}_user({api.args});')
lines.append(f'{api.ret_type} {api.name}_{module}_kb({api.args});')
lines.append(f'{api.ret_type} {api.name}_{module}({api.args});')
return lines
def _render_api_implementations(api, module):
module_name = Path(module).name
lines = []
lines.append('')
with _render_api_guard(lines, api):
# _user
lines.append(f'__attribute__((weak)) {api.ret_type} {api.name}_{module_name}_user({api.args}) {{')
if api.ret_type == 'bool':
lines.append(' return true;')
else:
pass
lines.append('}')
lines.append('')
# _kb
lines.append(f'__attribute__((weak)) {api.ret_type} {api.name}_{module_name}_kb({api.args}) {{')
if api.ret_type == 'bool':
lines.append(f' if(!{api.name}_{module_name}_user({api.call_params})) {{ return false; }}')
lines.append(' return true;')
else:
lines.append(f' {api.name}_{module_name}_user({api.call_params});')
lines.append('}')
lines.append('')
# module (non-suffixed)
lines.append(f'__attribute__((weak)) {api.ret_type} {api.name}_{module_name}({api.args}) {{')
if api.ret_type == 'bool':
lines.append(f' if(!{api.name}_{module_name}_kb({api.call_params})) {{ return false; }}')
lines.append(' return true;')
else:
lines.append(f' {api.name}_{module_name}_kb({api.call_params});')
lines.append('}')
return lines
def _render_core_implementation(api, modules):
lines = []
lines.append('')
with _render_api_guard(lines, api):
lines.append(f'{api.ret_type} {api.name}_modules({api.args}) {{')
if api.ret_type == 'bool':
lines.append(' return true')
for module in modules:
module_name = Path(module).name
if api.ret_type == 'bool':
lines.append(f' && {api.name}_{module_name}({api.call_params})')
else:
lines.append(f' {api.name}_{module_name}({api.call_params});')
if api.ret_type == 'bool':
lines.append(' ;')
lines.append('}')
return lines
@cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to')
@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate community_modules.h for.')
@cli.argument('filename', nargs='?', type=qmk.path.FileType('r'), arg_only=True, completer=FilesCompleter('.json'), help='Configurator JSON file')
@cli.subcommand('Creates a community_modules.h from a keymap.json file.')
def generate_community_modules_h(cli):
"""Creates a community_modules.h from a keymap.json file
"""
if cli.args.output and cli.args.output.name == '-':
cli.args.output = None
api_list, api_version, ver_major, ver_minor, ver_patch = module_api_list()
lines = [
GPL2_HEADER_C_LIKE,
GENERATED_HEADER_C_LIKE,
'#pragma once',
'#include <stdint.h>',
'#include <stdbool.h>',
'#include <keycodes.h>',
'',
'#define COMMUNITY_MODULES_API_VERSION_BUILDER(ver_major,ver_minor,ver_patch) (((((uint32_t)(ver_major))&0xFF) << 24) | ((((uint32_t)(ver_minor))&0xFF) << 16) | (((uint32_t)(ver_patch))&0xFF))',
f'#define COMMUNITY_MODULES_API_VERSION COMMUNITY_MODULES_API_VERSION_BUILDER({ver_major},{ver_minor},{ver_patch})',
f'#define ASSERT_COMMUNITY_MODULES_MIN_API_VERSION(ver_major,ver_minor,ver_patch) _Static_assert(COMMUNITY_MODULES_API_VERSION_BUILDER(ver_major,ver_minor,ver_patch) <= COMMUNITY_MODULES_API_VERSION, "Community module requires a newer version of QMK modules API -- needs: " #ver_major "." #ver_minor "." #ver_patch ", current: {api_version}.")',
'',
'typedef struct keyrecord_t keyrecord_t; // forward declaration so we don\'t need to include quantum.h',
'',
]
modules = get_modules(cli.args.keyboard, cli.args.filename)
module_jsons = load_module_jsons(modules)
if len(modules) > 0:
lines.extend(_render_keycodes(module_jsons))
for api in api_list:
lines.extend(_render_api_header(api))
for module in modules:
lines.append('')
lines.append(f'// From module: {module}')
for api in api_list:
lines.extend(_render_api_declarations(api, Path(module).name))
lines.append('')
lines.append('// Core wrapper')
for api in api_list:
lines.extend(_render_api_declarations(api, 'modules', user_kb=False))
dump_lines(cli.args.output, lines, cli.args.quiet, remove_repeated_newlines=True)
@cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to')
@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate community_modules.c for.')
@cli.argument('filename', nargs='?', type=qmk.path.FileType('r'), arg_only=True, completer=FilesCompleter('.json'), help='Configurator JSON file')
@cli.subcommand('Creates a community_modules.c from a keymap.json file.')
def generate_community_modules_c(cli):
"""Creates a community_modules.c from a keymap.json file
"""
if cli.args.output and cli.args.output.name == '-':
cli.args.output = None
api_list, _, _, _, _ = module_api_list()
lines = [
GPL2_HEADER_C_LIKE,
GENERATED_HEADER_C_LIKE,
'',
'#include "community_modules.h"',
]
modules = get_modules(cli.args.keyboard, cli.args.filename)
if len(modules) > 0:
for module in modules:
for api in api_list:
lines.extend(_render_api_implementations(api, Path(module).name))
for api in api_list:
lines.extend(_render_core_implementation(api, modules))
dump_lines(cli.args.output, lines, cli.args.quiet, remove_repeated_newlines=True)
@cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to')
@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate community_modules.c for.')
@cli.argument('filename', nargs='?', type=qmk.path.FileType('r'), arg_only=True, completer=FilesCompleter('.json'), help='Configurator JSON file')
@cli.subcommand('Creates a community_modules_introspection.h from a keymap.json file.')
def generate_community_modules_introspection_h(cli):
"""Creates a community_modules_introspection.h from a keymap.json file
"""
if cli.args.output and cli.args.output.name == '-':
cli.args.output = None
lines = [
GPL2_HEADER_C_LIKE,
GENERATED_HEADER_C_LIKE,
'',
]
modules = get_modules(cli.args.keyboard, cli.args.filename)
if len(modules) > 0:
for module in modules:
module_path = find_module_path(module)
lines.append(f'#if __has_include("{module_path}/introspection.h")')
lines.append(f'#include "{module_path}/introspection.h"')
lines.append(f'#endif // __has_include("{module_path}/introspection.h")')
lines.append('')
dump_lines(cli.args.output, lines, cli.args.quiet, remove_repeated_newlines=True)
@cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to')
@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate community_modules.c for.')
@cli.argument('filename', nargs='?', type=qmk.path.FileType('r'), arg_only=True, completer=FilesCompleter('.json'), help='Configurator JSON file')
@cli.subcommand('Creates a community_modules_introspection.c from a keymap.json file.')
def generate_community_modules_introspection_c(cli):
"""Creates a community_modules_introspection.c from a keymap.json file
"""
if cli.args.output and cli.args.output.name == '-':
cli.args.output = None
lines = [
GPL2_HEADER_C_LIKE,
GENERATED_HEADER_C_LIKE,
'',
]
modules = get_modules(cli.args.keyboard, cli.args.filename)
if len(modules) > 0:
for module in modules:
module_path = find_module_path(module)
lines.append(f'#if __has_include("{module_path}/introspection.c")')
lines.append(f'#include "{module_path}/introspection.c"')
lines.append(f'#endif // __has_include("{module_path}/introspection.c")')
lines.append('')
dump_lines(cli.args.output, lines, cli.args.quiet, remove_repeated_newlines=True)

View file

@ -6,12 +6,13 @@ from dotty_dict import dotty
from argcomplete.completers import FilesCompleter
from milc import cli
from qmk.info import info_json
from qmk.info import info_json, get_modules
from qmk.json_schema import json_load
from qmk.keyboard import keyboard_completer, keyboard_folder
from qmk.commands import dump_lines, parse_configurator_json
from qmk.path import normpath, FileType
from qmk.constants import GPL2_HEADER_SH_LIKE, GENERATED_HEADER_SH_LIKE
from qmk.community_modules import find_module_path, load_module_jsons
def generate_rule(rules_key, rules_value):
@ -46,6 +47,42 @@ def process_mapping_rule(kb_info_json, rules_key, info_dict):
return generate_rule(rules_key, rules_value)
def generate_features_rules(features_dict):
lines = []
for feature, enabled in features_dict.items():
feature = feature.upper()
enabled = 'yes' if enabled else 'no'
lines.append(generate_rule(f'{feature}_ENABLE', enabled))
return lines
def generate_modules_rules(keyboard, filename):
lines = []
modules = get_modules(keyboard, filename)
if len(modules) > 0:
lines.append('')
lines.append('OPT_DEFS += -DCOMMUNITY_MODULES_ENABLE=TRUE')
for module in modules:
module_path = find_module_path(module)
if not module_path:
raise FileNotFoundError(f"Module '{module}' not found.")
lines.append('')
lines.append(f'COMMUNITY_MODULES += {module_path.name}') # use module_path here instead of module as it may be a subdirectory
lines.append(f'OPT_DEFS += -DCOMMUNITY_MODULE_{module_path.name.upper()}_ENABLE=TRUE')
lines.append(f'COMMUNITY_MODULE_PATHS += {module_path}')
lines.append(f'VPATH += {module_path}')
lines.append(f'SRC += $(wildcard {module_path}/{module_path.name}.c)')
lines.append(f'-include {module_path}/rules.mk')
module_jsons = load_module_jsons(modules)
for module_json in module_jsons:
if 'features' in module_json:
lines.append('')
lines.append(f'# Module: {module_json["module_name"]}')
lines.extend(generate_features_rules(module_json['features']))
return lines
@cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON to be compiled and flashed or a pre-compiled binary firmware file (bin/hex) to be flashed.')
@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
@ -80,10 +117,7 @@ def generate_rules_mk(cli):
# Iterate through features to enable/disable them
if 'features' in kb_info_json:
for feature, enabled in kb_info_json['features'].items():
feature = feature.upper()
enabled = 'yes' if enabled else 'no'
rules_mk_lines.append(generate_rule(f'{feature}_ENABLE', enabled))
rules_mk_lines.extend(generate_features_rules(kb_info_json['features']))
# Set SPLIT_TRANSPORT, if needed
if kb_info_json.get('split', {}).get('transport', {}).get('protocol') == 'custom':
@ -99,6 +133,8 @@ def generate_rules_mk(cli):
if converter:
rules_mk_lines.append(generate_rule('CONVERT_TO', converter))
rules_mk_lines.extend(generate_modules_rules(cli.args.keyboard, cli.args.filename))
# Show the results
dump_lines(cli.args.output, rules_mk_lines)

View file

@ -52,6 +52,11 @@ def show_keymap(kb_info_json, title_caps=True):
if keymap_path and keymap_path.suffix == '.json':
keymap_data = json.load(keymap_path.open(encoding='utf-8'))
# cater for layout-less keymap.json
if 'layout' not in keymap_data:
return
layout_name = keymap_data['layout']
layout_name = kb_info_json.get('layout_aliases', {}).get(layout_name, layout_name) # Resolve alias names

View file

@ -98,11 +98,14 @@ def in_virtualenv():
return active_prefix != sys.prefix
def dump_lines(output_file, lines, quiet=True):
def dump_lines(output_file, lines, quiet=True, remove_repeated_newlines=False):
"""Handle dumping to stdout or file
Creates parent folders if required
"""
generated = '\n'.join(lines) + '\n'
if remove_repeated_newlines:
while '\n\n\n' in generated:
generated = generated.replace('\n\n\n', '\n\n')
if output_file and output_file.name != '-':
output_file.parent.mkdir(parents=True, exist_ok=True)
if output_file.exists():

View file

@ -0,0 +1,100 @@
import os
from pathlib import Path
from functools import lru_cache
from milc.attrdict import AttrDict
from qmk.json_schema import json_load, validate, merge_ordered_dicts
from qmk.util import truthy
from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE
from qmk.path import under_qmk_firmware, under_qmk_userspace
COMMUNITY_MODULE_JSON_FILENAME = 'qmk_module.json'
class ModuleAPI(AttrDict):
def __init__(self, **kwargs):
super().__init__()
for key, value in kwargs.items():
self[key] = value
@lru_cache(maxsize=1)
def module_api_list():
module_definition_files = sorted(set(QMK_FIRMWARE.glob('data/constants/module_hooks/*.hjson')))
module_definition_jsons = [json_load(f) for f in module_definition_files]
module_definitions = merge_ordered_dicts(module_definition_jsons)
latest_module_version = module_definition_files[-1].stem
latest_module_version_parts = latest_module_version.split('.')
api_list = []
for name, mod in module_definitions.items():
api_list.append(ModuleAPI(
ret_type=mod['ret_type'],
name=name,
args=mod['args'],
call_params=mod.get('call_params', ''),
guard=mod.get('guard', None),
header=mod.get('header', None),
))
return api_list, latest_module_version, latest_module_version_parts[0], latest_module_version_parts[1], latest_module_version_parts[2]
def find_available_module_paths():
"""Find all available modules.
"""
search_dirs = []
if HAS_QMK_USERSPACE:
search_dirs.append(QMK_USERSPACE / 'modules')
search_dirs.append(QMK_FIRMWARE / 'modules')
modules = []
for search_dir in search_dirs:
for module_json_path in search_dir.rglob(COMMUNITY_MODULE_JSON_FILENAME):
modules.append(module_json_path.parent)
return modules
def find_module_path(module):
"""Find a module by name.
"""
for module_path in find_available_module_paths():
# Ensure the module directory is under QMK Firmware or QMK Userspace
relative_path = under_qmk_firmware(module_path)
if not relative_path:
relative_path = under_qmk_userspace(module_path)
if not relative_path:
continue
lhs = str(relative_path.as_posix())[len('modules/'):]
rhs = str(Path(module).as_posix())
if relative_path and lhs == rhs:
return module_path
return None
def load_module_json(module):
"""Load a module JSON file.
"""
module_path = find_module_path(module)
if not module_path:
raise FileNotFoundError(f'Module not found: {module}')
module_json = json_load(module_path / COMMUNITY_MODULE_JSON_FILENAME)
if not truthy(os.environ.get('SKIP_SCHEMA_VALIDATION'), False):
validate(module_json, 'qmk.community_module.v1')
module_json['module'] = module
module_json['module_path'] = module_path
return module_json
def load_module_jsons(modules):
"""Load the module JSON files, matching the specified order.
"""
return list(map(load_module_json, modules))

View file

@ -1059,3 +1059,30 @@ def keymap_json(keyboard, keymap, force_layout=None):
_extract_config_h(kb_info_json, parse_config_h_file(keymap_config))
return kb_info_json
def get_modules(keyboard, keymap_filename):
"""Get the modules for a keyboard/keymap.
"""
modules = []
if keymap_filename:
keymap_json = parse_configurator_json(keymap_filename)
if keymap_json:
kb = keymap_json.get('keyboard', None)
if not kb:
kb = keyboard
if kb:
kb_info_json = info_json(kb)
if kb_info_json:
modules.extend(kb_info_json.get('modules', []))
modules.extend(keymap_json.get('modules', []))
elif keyboard:
kb_info_json = info_json(keyboard)
modules.extend(kb_info_json.get('modules', []))
return list(dict.fromkeys(modules)) # remove dupes

View file

@ -235,3 +235,31 @@ class UserspaceJSONEncoder(QMKJSONEncoder):
return '01build_targets'
return key
class CommunityModuleJSONEncoder(QMKJSONEncoder):
"""Custom encoder to make qmk_module.json's a little nicer to work with.
"""
def sort_dict(self, item):
"""Sorts the hashes in a nice way.
"""
key = item[0]
if self.indentation_level == 1:
if key == 'module_name':
return '00module_name'
if key == 'maintainer':
return '01maintainer'
if key == 'url':
return '02url'
if key == 'features':
return '03features'
if key == 'keycodes':
return '04keycodes'
elif self.indentation_level == 3: # keycodes
if key == 'key':
return '00key'
if key == 'aliases':
return '01aliases'
return key

View file

@ -334,33 +334,6 @@ def write_json(keyboard, keymap, layout, layers, macros=None):
return write_file(keymap_file, keymap_content)
def write(keymap_json):
"""Generate the `keymap.c` and write it to disk.
Returns the filename written to.
`keymap_json` should be a dict with the following keys:
keyboard
The name of the keyboard
keymap
The name of the keymap
layout
The LAYOUT macro this keymap uses.
layers
An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
macros
A list of macros for this keymap.
"""
keymap_content = generate_c(keymap_json)
keymap_file = qmk.path.keymaps(keymap_json['keyboard'])[0] / keymap_json['keymap'] / 'keymap.c'
return write_file(keymap_file, keymap_content)
def locate_keymap(keyboard, keymap, force_layout=None):
"""Returns the path to a keymap for a specific keyboard.
"""