diff options
Diffstat (limited to 'lib')
31 files changed, 659 insertions, 354 deletions
| diff --git a/lib/chibios b/lib/chibios -Subproject 0e9d558b525a8f28285f3bb509fd48a897c4315 +Subproject 0062927e3058a8b5ef587234bbd98d42fb4e595 diff --git a/lib/chibios-contrib b/lib/chibios-contrib -Subproject bb8356fb5a3a9bbc1561826f174a9a631c61454 +Subproject a224be155ae18d38deccf33a6c1d259b9a5ad8d diff --git a/lib/lvgl b/lib/lvgl new file mode 160000 +Subproject e19410f8f8a256609da72cff549598e0df6fa4c diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 1da4d25741..778eccada8 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -34,13 +34,11 @@ subcommands = [      'qmk.cli.bux',      'qmk.cli.c2json',      'qmk.cli.cd', -    'qmk.cli.cformat',      'qmk.cli.chibios.confmigrate',      'qmk.cli.clean',      'qmk.cli.compile',      'qmk.cli.docs',      'qmk.cli.doctor', -    'qmk.cli.fileformat',      'qmk.cli.flash',      'qmk.cli.format.c',      'qmk.cli.format.json', @@ -57,9 +55,11 @@ subcommands = [      'qmk.cli.generate.keyboard_c',      'qmk.cli.generate.keyboard_h',      'qmk.cli.generate.keycodes', +    'qmk.cli.generate.keycodes_tests',      'qmk.cli.generate.rgb_breathe_table',      'qmk.cli.generate.rules_mk',      'qmk.cli.generate.version_h', +    'qmk.cli.git.submodule',      'qmk.cli.hello',      'qmk.cli.import.kbfirmware',      'qmk.cli.import.keyboard', @@ -67,16 +67,15 @@ subcommands = [      'qmk.cli.info',      'qmk.cli.json2c',      'qmk.cli.lint', +    'qmk.cli.kle2json',      'qmk.cli.list.keyboards',      'qmk.cli.list.keymaps',      'qmk.cli.list.layouts', -    'qmk.cli.kle2json',      'qmk.cli.mass_compile', -    'qmk.cli.multibuild', +    'qmk.cli.migrate',      'qmk.cli.new.keyboard',      'qmk.cli.new.keymap',      'qmk.cli.painter', -    'qmk.cli.pyformat',      'qmk.cli.pytest',      'qmk.cli.via2json',  ] diff --git a/lib/python/qmk/cli/cformat.py b/lib/python/qmk/cli/cformat.py deleted file mode 100755 index 9d0ecaeba3..0000000000 --- a/lib/python/qmk/cli/cformat.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Point people to the new command name. -""" -import sys -from pathlib import Path - -from milc import cli - - [email protected]('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.") [email protected]('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.') [email protected]('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.') [email protected]('--core-only', arg_only=True, action='store_true', help='Format core files only.') [email protected]('files', nargs='*', arg_only=True, help='Filename(s) to format.') [email protected]('Pointer to the new command name: qmk format-c.', hidden=True) -def cformat(cli): -    """Pointer to the new command name: qmk format-c. -    """ -    cli.log.warning('"qmk cformat" has been renamed to "qmk format-c". Please use the new command in the future.') -    argv = [sys.executable, *sys.argv] -    argv[argv.index('cformat')] = 'format-c' -    script_path = Path(argv[1]) -    script_path_exe = Path(f'{argv[1]}.exe') - -    if not script_path.exists() and script_path_exe.exists(): -        # For reasons I don't understand ".exe" is stripped from the script name on windows. -        argv[1] = str(script_path_exe) - -    return cli.run(argv, capture_output=False).returncode diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 9e7629906f..f43e5f32de 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -10,7 +10,17 @@ import qmk.path  from qmk.decorators import automagic_keyboard, automagic_keymap  from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment  from qmk.keyboard import keyboard_completer, keyboard_folder -from qmk.keymap import keymap_completer +from qmk.keymap import keymap_completer, locate_keymap + + +def _is_keymap_target(keyboard, keymap): +    if keymap == 'all': +        return True + +    if locate_keymap(keyboard, keymap): +        return True + +    return False  @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile') @@ -43,6 +53,11 @@ def compile(cli):      elif cli.config.compile.keyboard and cli.config.compile.keymap:          # Generate the make command for a specific keyboard/keymap. +        if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap): +            cli.log.error('Invalid keymap argument.') +            cli.print_help() +            return False +          if cli.args.clean:              commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs))          commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs)) diff --git a/lib/python/qmk/cli/doctor/check.py b/lib/python/qmk/cli/doctor/check.py index 8a0422ba72..426876e98a 100644 --- a/lib/python/qmk/cli/doctor/check.py +++ b/lib/python/qmk/cli/doctor/check.py @@ -3,7 +3,7 @@  from enum import Enum  import re  import shutil -from subprocess import DEVNULL +from subprocess import DEVNULL, TimeoutExpired  from milc import cli  from qmk import submodules @@ -41,9 +41,8 @@ def _parse_gcc_version(version):  def _check_arm_gcc_version():      """Returns True if the arm-none-eabi-gcc version is not known to cause problems.      """ -    if 'output' in ESSENTIAL_BINARIES['arm-none-eabi-gcc']: -        version_number = ESSENTIAL_BINARIES['arm-none-eabi-gcc']['output'].strip() -        cli.log.info('Found arm-none-eabi-gcc version %s', version_number) +    version_number = ESSENTIAL_BINARIES['arm-none-eabi-gcc']['output'].strip() +    cli.log.info('Found arm-none-eabi-gcc version %s', version_number)      return CheckStatus.OK  # Right now all known arm versions are ok @@ -51,44 +50,37 @@ def _check_arm_gcc_version():  def _check_avr_gcc_version():      """Returns True if the avr-gcc version is not known to cause problems.      """ -    rc = CheckStatus.ERROR -    if 'output' in ESSENTIAL_BINARIES['avr-gcc']: -        version_number = ESSENTIAL_BINARIES['avr-gcc']['output'].strip() +    version_number = ESSENTIAL_BINARIES['avr-gcc']['output'].strip() +    cli.log.info('Found avr-gcc version %s', version_number) -        cli.log.info('Found avr-gcc version %s', version_number) -        rc = CheckStatus.OK +    parsed_version = _parse_gcc_version(version_number) +    if parsed_version['major'] > 8: +        cli.log.warning('{fg_yellow}We do not recommend avr-gcc newer than 8. Downgrading to 8.x is recommended.') +        return CheckStatus.WARNING -        parsed_version = _parse_gcc_version(version_number) -        if parsed_version['major'] > 8: -            cli.log.warning('{fg_yellow}We do not recommend avr-gcc newer than 8. Downgrading to 8.x is recommended.') -            rc = CheckStatus.WARNING - -    return rc +    return CheckStatus.OK  def _check_avrdude_version(): -    if 'output' in ESSENTIAL_BINARIES['avrdude']: -        last_line = ESSENTIAL_BINARIES['avrdude']['output'].split('\n')[-2] -        version_number = last_line.split()[2][:-1] -        cli.log.info('Found avrdude version %s', version_number) +    last_line = ESSENTIAL_BINARIES['avrdude']['output'].split('\n')[-2] +    version_number = last_line.split()[2][:-1] +    cli.log.info('Found avrdude version %s', version_number)      return CheckStatus.OK  def _check_dfu_util_version(): -    if 'output' in ESSENTIAL_BINARIES['dfu-util']: -        first_line = ESSENTIAL_BINARIES['dfu-util']['output'].split('\n')[0] -        version_number = first_line.split()[1] -        cli.log.info('Found dfu-util version %s', version_number) +    first_line = ESSENTIAL_BINARIES['dfu-util']['output'].split('\n')[0] +    version_number = first_line.split()[1] +    cli.log.info('Found dfu-util version %s', version_number)      return CheckStatus.OK  def _check_dfu_programmer_version(): -    if 'output' in ESSENTIAL_BINARIES['dfu-programmer']: -        first_line = ESSENTIAL_BINARIES['dfu-programmer']['output'].split('\n')[0] -        version_number = first_line.split()[1] -        cli.log.info('Found dfu-programmer version %s', version_number) +    first_line = ESSENTIAL_BINARIES['dfu-programmer']['output'].split('\n')[0] +    version_number = first_line.split()[1] +    cli.log.info('Found dfu-programmer version %s', version_number)      return CheckStatus.OK @@ -96,11 +88,16 @@ def _check_dfu_programmer_version():  def check_binaries():      """Iterates through ESSENTIAL_BINARIES and tests them.      """ -    ok = True +    ok = CheckStatus.OK      for binary in sorted(ESSENTIAL_BINARIES): -        if not is_executable(binary): -            ok = False +        try: +            if not is_executable(binary): +                ok = CheckStatus.ERROR +        except TimeoutExpired: +            cli.log.debug('Timeout checking %s', binary) +            if ok != CheckStatus.ERROR: +                ok = CheckStatus.WARNING      return ok @@ -108,8 +105,22 @@ def check_binaries():  def check_binary_versions():      """Check the versions of ESSENTIAL_BINARIES      """ +    checks = { +        'arm-none-eabi-gcc': _check_arm_gcc_version, +        'avr-gcc': _check_avr_gcc_version, +        'avrdude': _check_avrdude_version, +        'dfu-util': _check_dfu_util_version, +        'dfu-programmer': _check_dfu_programmer_version, +    } +      versions = [] -    for check in (_check_arm_gcc_version, _check_avr_gcc_version, _check_avrdude_version, _check_dfu_util_version, _check_dfu_programmer_version): +    for binary in sorted(ESSENTIAL_BINARIES): +        if 'output' not in ESSENTIAL_BINARIES[binary]: +            cli.log.warning('Unknown version for %s', binary) +            versions.append(CheckStatus.WARNING) +            continue + +        check = checks[binary]          versions.append(check())      return versions @@ -119,10 +130,8 @@ def check_submodules():      """      for submodule in submodules.status().values():          if submodule['status'] is None: -            cli.log.error('Submodule %s has not yet been cloned!', submodule['name'])              return CheckStatus.ERROR          elif not submodule['status']: -            cli.log.warning('Submodule %s is not up to date!', submodule['name'])              return CheckStatus.WARNING      return CheckStatus.OK diff --git a/lib/python/qmk/cli/doctor/linux.py b/lib/python/qmk/cli/doctor/linux.py index a803305c0d..95bafe8c64 100644 --- a/lib/python/qmk/cli/doctor/linux.py +++ b/lib/python/qmk/cli/doctor/linux.py @@ -78,10 +78,13 @@ def check_udev_rules():          # Collect all rules from the config files          for rule_file in udev_rules: -            for line in rule_file.read_text(encoding='utf-8').split('\n'): -                line = line.strip() -                if not line.startswith("#") and len(line): -                    current_rules.add(line) +            try: +                for line in rule_file.read_text(encoding='utf-8').split('\n'): +                    line = line.strip() +                    if not line.startswith("#") and len(line): +                        current_rules.add(line) +            except PermissionError: +                cli.log.debug("Failed to read: %s", rule_file)          # Check if the desired rules are among the currently present rules          for bootloader, rules in desired_rules.items(): diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py index 1600ab8dd4..6a6feb87d1 100755 --- a/lib/python/qmk/cli/doctor/main.py +++ b/lib/python/qmk/cli/doctor/main.py @@ -119,13 +119,15 @@ def doctor(cli):      # Make sure the basic CLI tools we need are available and can be executed.      bin_ok = check_binaries() -    if not bin_ok: +    if bin_ok == CheckStatus.ERROR:          if yesno('Would you like to install dependencies?', default=True):              cli.run(['util/qmk_install.sh', '-y'], stdin=DEVNULL, capture_output=False)              bin_ok = check_binaries() -    if bin_ok: +    if bin_ok == CheckStatus.OK:          cli.log.info('All dependencies are installed.') +    elif bin_ok == CheckStatus.WARNING: +        cli.log.warning('Issues encountered while checking dependencies.')      else:          status = CheckStatus.ERROR @@ -142,7 +144,7 @@ def doctor(cli):      if sub_ok == CheckStatus.OK:          cli.log.info('Submodules are up to date.')      else: -        if yesno('Would you like to clone the submodules?', default=True): +        if git_check_repo() and yesno('Would you like to clone the submodules?', default=True):              submodules.update()              sub_ok = check_submodules() diff --git a/lib/python/qmk/cli/fileformat.py b/lib/python/qmk/cli/fileformat.py deleted file mode 100755 index cee4ba1acd..0000000000 --- a/lib/python/qmk/cli/fileformat.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Point people to the new command name. -""" -import sys -from pathlib import Path - -from milc import cli - - [email protected]('Pointer to the new command name: qmk format-text.', hidden=True) -def fileformat(cli): -    """Pointer to the new command name: qmk format-text. -    """ -    cli.log.warning('"qmk fileformat" has been renamed to "qmk format-text". Please use the new command in the future.') -    argv = [sys.executable, *sys.argv] -    argv[argv.index('fileformat')] = 'format-text' -    script_path = Path(argv[1]) -    script_path_exe = Path(f'{argv[1]}.exe') - -    if not script_path.exists() and script_path_exe.exists(): -        # For reasons I don't understand ".exe" is stripped from the script name on windows. -        argv[1] = str(script_path_exe) - -    return cli.run(argv, capture_output=False).returncode diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index 40bfbdab56..8724f26889 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -11,12 +11,24 @@ import qmk.path  from qmk.decorators import automagic_keyboard, automagic_keymap  from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment  from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.keymap import keymap_completer, locate_keymap  from qmk.flashers import flasher -def print_bootloader_help(): +def _is_keymap_target(keyboard, keymap): +    if keymap == 'all': +        return True + +    if locate_keymap(keyboard, keymap): +        return True + +    return False + + +def _list_bootloaders():      """Prints the available bootloaders listed in docs.qmk.fm.      """ +    cli.print_help()      cli.log.info('Here are the available bootloaders:')      cli.echo('\tavrdude')      cli.echo('\tbootloadhid') @@ -36,14 +48,29 @@ def print_bootloader_help():      cli.echo('\tuf2-split-left')      cli.echo('\tuf2-split-right')      cli.echo('For more info, visit https://docs.qmk.fm/#/flashing') +    return False + + +def _flash_binary(filename, mcu): +    """Try to flash binary firmware +    """ +    cli.echo('Flashing binary firmware...\nPlease reset your keyboard into bootloader mode now!\nPress Ctrl-C to exit.\n') +    try: +        err, msg = flasher(mcu, filename) +        if err: +            cli.log.error(msg) +            return False +    except KeyboardInterrupt: +        cli.log.info('Ctrl-C was pressed, exiting...') +    return True  @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON to be compiled and flashed or a pre-compiled binary firmware file (bin/hex) to be flashed.')  @cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.')  @cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.')  @cli.argument('-m', '--mcu', help='The MCU name. Required for HalfKay, HID, USBAspLoader and ISP flashing.') [email protected]('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') [email protected]('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') [email protected]('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') [email protected]('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')  @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")  @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")  @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @@ -56,30 +83,17 @@ def flash(cli):      If a binary firmware is supplied, try to flash that. -    If a Configurator JSON export is supplied this command will create a new keymap. Keymap and Keyboard arguments -    will be ignored. +    If a Configurator export is supplied this command will create a new keymap, overwriting an existing keymap if one exists. -    If no file is supplied, keymap and keyboard are expected. +    If a keyboard and keymap are provided this command will build a firmware based on that.      If bootloader is omitted the make system will use the configured bootloader for that keyboard.      """ -    if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex']: -        # Try to flash binary firmware -        cli.echo('Flashing binary firmware...\nPlease reset your keyboard into bootloader mode now!\nPress Ctrl-C to exit.\n') -        try: -            err, msg = flasher(cli.args.mcu, cli.args.filename) -            if err: -                cli.log.error(msg) -                return False -        except KeyboardInterrupt: -            cli.log.info('Ctrl-C was pressed, exiting...') -        return True +    if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']: +        return _flash_binary(cli.args.filename, cli.args.mcu)      if cli.args.bootloaders: -        # Provide usage and list bootloaders -        cli.print_help() -        print_bootloader_help() -        return False +        return _list_bootloaders()      # Build the environment vars      envs = build_environment(cli.args.env) @@ -94,6 +108,11 @@ def flash(cli):      elif cli.config.flash.keyboard and cli.config.flash.keymap:          # Generate the make command for a specific keyboard/keymap. +        if not _is_keymap_target(cli.config.flash.keyboard, cli.config.flash.keymap): +            cli.log.error('Invalid keymap argument.') +            cli.print_help() +            return False +          if cli.args.clean:              commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean', **envs))          commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)) diff --git a/lib/python/qmk/cli/generate/api.py b/lib/python/qmk/cli/generate/api.py index 8650a36b84..d662e14e00 100755 --- a/lib/python/qmk/cli/generate/api.py +++ b/lib/python/qmk/cli/generate/api.py @@ -10,8 +10,9 @@ from qmk.datetime import current_datetime  from qmk.info import info_json  from qmk.json_encoders import InfoJSONEncoder  from qmk.json_schema import json_load +from qmk.keymap import list_keymaps  from qmk.keyboard import find_readme, list_keyboards -from qmk.keycodes import load_spec, list_versions +from qmk.keycodes import load_spec, list_versions, list_languages  DATA_PATH = Path('data')  TEMPLATE_PATH = DATA_PATH / 'templates/api/' @@ -44,6 +45,13 @@ def _resolve_keycode_specs(output_folder):          output_file = output_folder / f'constants/keycodes_{version}.json'          output_file.write_text(json.dumps(overall, indent=4), encoding='utf-8') +    for lang in list_languages(): +        for version in list_versions(lang): +            overall = load_spec(version, lang) + +            output_file = output_folder / f'constants/keycodes_{lang}_{version}.json' +            output_file.write_text(json.dumps(overall, indent=4), encoding='utf-8') +      # Purge files consumed by 'load_spec'      shutil.rmtree(output_folder / 'constants/keycodes/') @@ -104,6 +112,11 @@ def generate_api(cli):      # Generate and write keyboard specific JSON files      for keyboard_name in keyboard_list:          kb_all[keyboard_name] = info_json(keyboard_name) + +        # Populate the list of JSON keymaps +        for keymap in list_keymaps(keyboard_name, c=False, fullpath=True): +            kb_all[keyboard_name]['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'} +          keyboard_dir = v1_dir / 'keyboards' / keyboard_name          keyboard_info = keyboard_dir / 'info.json'          keyboard_readme = keyboard_dir / 'readme.md' diff --git a/lib/python/qmk/cli/generate/keycodes.py b/lib/python/qmk/cli/generate/keycodes.py index 29b7db3c80..2ed84cd589 100644 --- a/lib/python/qmk/cli/generate/keycodes.py +++ b/lib/python/qmk/cli/generate/keycodes.py @@ -8,6 +8,24 @@ from qmk.path import normpath  from qmk.keycodes import load_spec +def _render_key(key): +    width = 7 +    if 'S(' in key: +        width += len('S()') +    if 'A(' in key: +        width += len('A()') +    if 'RCTL(' in key: +        width += len('RCTL()') +    if 'ALGR(' in key: +        width += len('ALGR()') +    return key.ljust(width) + + +def _render_label(label): +    label = label.replace("\\", "(backslash)") +    return label + +  def _generate_ranges(lines, keycodes):      lines.append('')      lines.append('enum qk_keycode_ranges {') @@ -67,6 +85,23 @@ def _generate_helpers(lines, keycodes):          lines.append(f'#define IS_{ group.upper() }_KEYCODE(code) ((code) >= {lo} && (code) <= {hi})') +def _generate_aliases(lines, keycodes): +    lines.append('') +    lines.append('// Aliases') +    for key, value in keycodes["aliases"].items(): +        define = _render_key(value.get("key")) +        val = _render_key(key) +        if 'label' in value: +            lines.append(f'#define {define} {val} // {_render_label(value.get("label"))}') +        else: +            lines.append(f'#define {define} {val}') + +    lines.append('') +    for key, value in keycodes["aliases"].items(): +        for alias in value.get("aliases", []): +            lines.append(f'#define {alias} {value.get("key")}') + +  @cli.argument('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.')  @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')  @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") @@ -86,3 +121,23 @@ def generate_keycodes(cli):      # Show the results      dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet) + + [email protected]('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.') [email protected]('-l', '--lang', arg_only=True, required=True, help='Language of keycodes to generate.') [email protected]('-o', '--output', arg_only=True, type=normpath, help='File to write to') [email protected]('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") [email protected]('Used by the make system to generate keymap_{lang}.h from keycodes_{lang}_{version}.json', hidden=True) +def generate_keycode_extras(cli): +    """Generates the header file. +    """ + +    # Build the header file. +    keycodes_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#pragma once', '#include "keymap.h"', '// clang-format off'] + +    keycodes = load_spec(cli.args.version, cli.args.lang) + +    _generate_aliases(keycodes_h_lines, keycodes) + +    # Show the results +    dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet) diff --git a/lib/python/qmk/cli/generate/keycodes_tests.py b/lib/python/qmk/cli/generate/keycodes_tests.py new file mode 100644 index 0000000000..453b4693a7 --- /dev/null +++ b/lib/python/qmk/cli/generate/keycodes_tests.py @@ -0,0 +1,39 @@ +"""Used by the make system to generate a keycode lookup table from keycodes_{version}.json +""" +from milc import cli + +from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE +from qmk.commands import dump_lines +from qmk.path import normpath +from qmk.keycodes import load_spec + + +def _generate_defines(lines, keycodes): +    lines.append('') +    lines.append('std::map<uint16_t, std::string> KEYCODE_ID_TABLE = {') +    for key, value in keycodes["keycodes"].items(): +        lines.append(f'    {{{value.get("key")}, "{value.get("key")}"}},') +    lines.append('};') + + [email protected]('-v', '--version', arg_only=True, required=True, help='Version of keycodes to generate.') [email protected]('-o', '--output', arg_only=True, type=normpath, help='File to write to') [email protected]('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") [email protected]('Used by the make system to generate a keycode lookup table from keycodes_{version}.json', hidden=True) +def generate_keycodes_tests(cli): +    """Generates a keycode to identifier lookup table for unit test output. +    """ + +    # Build the keycodes.h file. +    keycodes_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '// clang-format off'] +    keycodes_h_lines.append('extern "C" {\n#include <keycode.h>\n}') +    keycodes_h_lines.append('#include <map>') +    keycodes_h_lines.append('#include <string>') +    keycodes_h_lines.append('#include <cstdint>') + +    keycodes = load_spec(cli.args.version) + +    _generate_defines(keycodes_h_lines, keycodes) + +    # Show the results +    dump_lines(cli.args.output, keycodes_h_lines, cli.args.quiet) diff --git a/lib/python/qmk/cli/git/__init__.py b/lib/python/qmk/cli/git/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/python/qmk/cli/git/__init__.py diff --git a/lib/python/qmk/cli/git/submodule.py b/lib/python/qmk/cli/git/submodule.py new file mode 100644 index 0000000000..9f354c021e --- /dev/null +++ b/lib/python/qmk/cli/git/submodule.py @@ -0,0 +1,38 @@ +import shutil + +from milc import cli + +from qmk.path import normpath +from qmk import submodules + +REMOVE_DIRS = [ +    'lib/ugfx', +    'lib/pico-sdk', +    'lib/chibios-contrib/ext/mcux-sdk', +    'lib/lvgl', +] + + [email protected]('--check', arg_only=True, action='store_true', help='Check if the submodules are dirty, and display a warning if they are.') [email protected]('--sync', arg_only=True, action='store_true', help='Shallow clone any missing submodules.') [email protected]('Git Submodule actions.') +def git_submodule(cli): +    """Git Submodule actions +    """ +    if cli.args.check: +        return all(item['status'] for item in submodules.status().values()) + +    if cli.args.sync: +        cli.run(['git', 'submodule', 'sync', '--recursive']) +        for name, item in submodules.status().items(): +            if item['status'] is None: +                cli.run(['git', 'submodule', 'update', '--depth=50', '--init', name], capture_output=False) +        return True + +    for folder in REMOVE_DIRS: +        if normpath(folder).is_dir(): +            print(f"Removing '{folder}'") +            shutil.rmtree(folder) + +    cli.run(['git', 'submodule', 'sync', '--recursive'], capture_output=False) +    cli.run(['git', 'submodule', 'update', '--init', '--recursive', '--progress'], capture_output=False) diff --git a/lib/python/qmk/cli/list/keyboards.py b/lib/python/qmk/cli/list/keyboards.py index 8b6c451673..405b9210e4 100644 --- a/lib/python/qmk/cli/list/keyboards.py +++ b/lib/python/qmk/cli/list/keyboards.py @@ -5,9 +5,10 @@ from milc import cli  import qmk.keyboard [email protected]('--no-resolve-defaults', arg_only=True, action='store_false', help='Ignore any "DEFAULT_FOLDER" within keyboards rules.mk')  @cli.subcommand("List the keyboards currently defined within QMK")  def list_keyboards(cli):      """List the keyboards currently defined within QMK      """ -    for keyboard_name in qmk.keyboard.list_keyboards(): +    for keyboard_name in qmk.keyboard.list_keyboards(cli.args.no_resolve_defaults):          print(keyboard_name) diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py index a98f7ad482..d8dda0cac3 100755 --- a/lib/python/qmk/cli/mass_compile.py +++ b/lib/python/qmk/cli/mass_compile.py @@ -134,7 +134,7 @@ all: {keyboard_safe}_{keymap_name}_binary  {keyboard_safe}_{keymap_name}_binary:  	@rm -f "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}.{keymap_name}" || true  	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" -	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" REQUIRE_PLATFORM_KEY= COLOR=true SILENT=false {' '.join(cli.args.env)} \\ +	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {' '.join(cli.args.env)} \\  		>>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" 2>&1 \\  		|| cp "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" "{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"  	@{{ grep '\[ERRORS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\ diff --git a/lib/python/qmk/cli/migrate.py b/lib/python/qmk/cli/migrate.py new file mode 100644 index 0000000000..4164f9c8ad --- /dev/null +++ b/lib/python/qmk/cli/migrate.py @@ -0,0 +1,81 @@ +"""Migrate keyboard configuration to "Data Driven" +""" +import json +from pathlib import Path +from dotty_dict import dotty + +from milc import cli + +from qmk.keyboard import keyboard_completer, keyboard_folder, resolve_keyboard +from qmk.info import info_json, find_info_json +from qmk.json_encoders import InfoJSONEncoder +from qmk.json_schema import json_load + + +def _candidate_files(keyboard): +    kb_dir = Path(resolve_keyboard(keyboard)) + +    cur_dir = Path('keyboards') +    files = [] +    for dir in kb_dir.parts: +        cur_dir = cur_dir / dir +        files.append(cur_dir / 'config.h') +        files.append(cur_dir / 'rules.mk') + +    return [file for file in files if file.exists()] + + [email protected]('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the performed migrations based on the supplied value. Supported format is 'KEY' located from 'data/mappings'. May be passed multiple times.") [email protected]('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='The keyboard\'s name') [email protected]('Migrate keyboard config to "Data Driven".', hidden=True) +def migrate(cli): +    """Migrate keyboard configuration to "Data Driven" +    """ +    # Merge mappings as we do not care to where "KEY" is found just that its removed +    info_config_map = json_load(Path('data/mappings/info_config.hjson')) +    info_rules_map = json_load(Path('data/mappings/info_rules.hjson')) +    info_map = {**info_config_map, **info_rules_map} + +    # Parse target info.json which will receive updates +    target_info = Path(find_info_json(cli.args.keyboard)[0]) +    info_data = dotty(json_load(target_info)) + +    # Already parsed used for updates +    kb_info_json = dotty(info_json(cli.args.keyboard)) + +    # List of candidate files +    files = _candidate_files(cli.args.keyboard) + +    # Filter down keys if requested +    keys = info_map.keys() +    if cli.args.filter: +        keys = list(set(keys) & set(cli.args.filter)) + +    cli.log.info(f'{{fg_green}}Migrating keyboard {{fg_cyan}}{cli.args.keyboard}{{fg_green}}.{{fg_reset}}') + +    # Start migration +    for file in files: +        cli.log.info(f'  Migrating file {file}') +        file_contents = file.read_text(encoding='utf-8').split('\n') +        for key in keys: +            for num, line in enumerate(file_contents): +                if line.startswith(f'{key} =') or line.startswith(f'#define {key} '): +                    cli.log.info(f'    Migrating {key}...') + +                    while line.rstrip().endswith('\\'): +                        file_contents.pop(num) +                        line = file_contents[num] +                    file_contents.pop(num) + +                    update_key = info_map[key]["info_key"] +                    if update_key in kb_info_json: +                        info_data[update_key] = kb_info_json[update_key] + +        file.write_text('\n'.join(file_contents), encoding='utf-8') + +    # Finally write out updated info.json +    cli.log.info(f'  Updating {target_info}') +    target_info.write_text(json.dumps(info_data.to_dict(), cls=InfoJSONEncoder)) + +    cli.log.info(f'{{fg_green}}Migration of keyboard {{fg_cyan}}{cli.args.keyboard}{{fg_green}} complete!{{fg_reset}}') +    cli.log.info(f"Verify build with {{fg_yellow}}qmk compile -kb {cli.args.keyboard} -km default{{fg_reset}}.") diff --git a/lib/python/qmk/cli/multibuild.py b/lib/python/qmk/cli/multibuild.py deleted file mode 100755 index 5e0f0b5188..0000000000 --- a/lib/python/qmk/cli/multibuild.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Compile all keyboards. - -This will compile everything in parallel, for testing purposes. -""" -import os -import re -from pathlib import Path -from subprocess import DEVNULL - -from milc import cli - -from qmk.constants import QMK_FIRMWARE -from qmk.commands import _find_make, get_make_parallel_args -import qmk.keyboard -import qmk.keymap - - -def _make_rules_mk_filter(key, value): -    def _rules_mk_filter(keyboard_name): -        rules_mk = qmk.keyboard.rules_mk(keyboard_name) -        return True if key in rules_mk and rules_mk[key].lower() == str(value).lower() else False - -    return _rules_mk_filter - - -def _is_split(keyboard_name): -    rules_mk = qmk.keyboard.rules_mk(keyboard_name) -    return True if 'SPLIT_KEYBOARD' in rules_mk and rules_mk['SPLIT_KEYBOARD'].lower() == 'yes' else False - - [email protected]('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.") [email protected]('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") [email protected]('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") [email protected]('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on the supplied value in rules.mk. Supported format is 'SPLIT_KEYBOARD=yes'. May be passed multiple times.") [email protected]('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.") [email protected]('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") [email protected]('Compile QMK Firmware for all keyboards.', hidden=False if cli.config.user.developer else True) -def multibuild(cli): -    """Compile QMK Firmware against all keyboards. -    """ - -    make_cmd = _find_make() -    if cli.args.clean: -        cli.run([make_cmd, 'clean'], capture_output=False, stdin=DEVNULL) - -    builddir = Path(QMK_FIRMWARE) / '.build' -    makefile = builddir / 'parallel_kb_builds.mk' - -    keyboard_list = qmk.keyboard.list_keyboards() - -    filter_re = re.compile(r'^(?P<key>[A-Z0-9_]+)\s*=\s*(?P<value>[^#]+)$') -    for filter_txt in cli.args.filter: -        f = filter_re.match(filter_txt) -        if f is not None: -            keyboard_list = filter(_make_rules_mk_filter(f.group('key'), f.group('value')), keyboard_list) - -    keyboard_list = list(sorted(keyboard_list)) - -    if len(keyboard_list) == 0: -        return - -    builddir.mkdir(parents=True, exist_ok=True) -    with open(makefile, "w") as f: -        for keyboard_name in keyboard_list: -            if qmk.keymap.locate_keymap(keyboard_name, cli.args.keymap) is not None: -                keyboard_safe = keyboard_name.replace('/', '_') -                # yapf: disable -                f.write( -                    f"""\ -all: {keyboard_safe}_binary -{keyboard_safe}_binary: -	@rm -f "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}" || true -	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{cli.args.keymap}'..." >>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" -	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{cli.args.keymap}" REQUIRE_PLATFORM_KEY= COLOR=true SILENT=false {' '.join(cli.args.env)} \\ -		>>"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" 2>&1 \\ -		|| cp "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" "{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}" -	@{{ grep '\[ERRORS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{cli.args.keymap}" ; }} \\ -		|| {{ grep '\[WARNINGS\]' "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" >/dev/null 2>&1 && printf "Build %-64s \e[1;33m[WARNINGS]\e[0m\\n" "{keyboard_name}:{cli.args.keymap}" ; }} \\ -		|| printf "Build %-64s \e[1;32m[OK]\e[0m\\n" "{keyboard_name}:{cli.args.keymap}" -	@rm -f "{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}" || true -"""# noqa -                ) -                # yapf: enable - -                if cli.args.no_temp: -                    # yapf: disable -                    f.write( -                        f"""\ -	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.elf" 2>/dev/null || true -	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.map" 2>/dev/null || true -	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.hex" 2>/dev/null || true -	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.bin" 2>/dev/null || true -	@rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{cli.args.keymap}.uf2" 2>/dev/null || true -	@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}" || true -	@rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}_{cli.args.keymap}" || true -"""# noqa -                    ) -                    # yapf: enable -                f.write('\n') - -    cli.run([make_cmd, *get_make_parallel_args(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) - -    # Check for failures -    failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')] -    if len(failures) > 0: -        return False diff --git a/lib/python/qmk/cli/new/keyboard.py b/lib/python/qmk/cli/new/keyboard.py index 251ad919dd..cdd3919168 100644 --- a/lib/python/qmk/cli/new/keyboard.py +++ b/lib/python/qmk/cli/new/keyboard.py @@ -195,11 +195,6 @@ def new_keyboard(cli):      cli.echo('')      kb_name = cli.args.keyboard if cli.args.keyboard else prompt_keyboard() -    user_name = cli.config.new_keyboard.name if cli.config.new_keyboard.name else prompt_user() -    real_name = cli.args.realname or cli.config.new_keyboard.name if cli.args.realname or cli.config.new_keyboard.name else prompt_name(user_name) -    default_layout = cli.args.layout if cli.args.layout else prompt_layout() -    mcu = cli.args.type if cli.args.type else prompt_mcu() -      if not validate_keyboard_name(kb_name):          cli.log.error('Keyboard names must contain only {fg_cyan}lowercase a-z{fg_reset}, {fg_cyan}0-9{fg_reset}, and {fg_cyan}_{fg_reset}! Please choose a different name.')          return 1 @@ -208,6 +203,11 @@ def new_keyboard(cli):          cli.log.error(f'Keyboard {{fg_cyan}}{kb_name}{{fg_reset}} already exists! Please choose a different name.')          return 1 +    user_name = cli.config.new_keyboard.name if cli.config.new_keyboard.name else prompt_user() +    real_name = cli.args.realname or cli.config.new_keyboard.name if cli.args.realname or cli.config.new_keyboard.name else prompt_name(user_name) +    default_layout = cli.args.layout if cli.args.layout else prompt_layout() +    mcu = cli.args.type if cli.args.type else prompt_mcu() +      # Preprocess any development_board presets      if mcu in dev_boards:          defaults_map = json_load(Path('data/mappings/defaults.hjson')) diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py index 60cb743cb6..e7823bc46d 100755 --- a/lib/python/qmk/cli/new/keymap.py +++ b/lib/python/qmk/cli/new/keymap.py @@ -1,12 +1,32 @@  """This script automates the copying of the default keymap into your own keymap.  """  import shutil -from pathlib import Path -import qmk.path +from milc import cli +from milc.questions import question + +from qmk.path import is_keyboard, keymap +from qmk.git import git_get_username  from qmk.decorators import automagic_keyboard, automagic_keymap  from qmk.keyboard import keyboard_completer, keyboard_folder -from milc import cli + + +def prompt_keyboard(): +    prompt = """{fg_yellow}Select Keyboard{style_reset_all} +If you`re unsure you can view a full list of supported keyboards with {fg_yellow}qmk list-keyboards{style_reset_all}. + +Keyboard Name? """ + +    return question(prompt) + + +def prompt_user(): +    prompt = """ +{fg_yellow}Name Your Keymap{style_reset_all} +Used for maintainer, copyright, etc + +Your GitHub Username? """ +    return question(prompt, default=git_get_username())  @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Specify keyboard name. Example: 1upkeyboards/1up60hse') @@ -17,32 +37,34 @@ from milc import cli  def new_keymap(cli):      """Creates a new keymap for the keyboard of your choosing.      """ -    # ask for user input if keyboard or keymap was not provided in the command line -    keyboard = cli.config.new_keymap.keyboard if cli.config.new_keymap.keyboard else input("Keyboard Name: ") -    keymap = cli.config.new_keymap.keymap if cli.config.new_keymap.keymap else input("Keymap Name: ") +    cli.log.info('{style_bright}Generating a new keymap{style_normal}') +    cli.echo('') -    # generate keymap paths -    kb_path = Path('keyboards') / keyboard -    keymap_path = qmk.path.keymap(keyboard) -    keymap_path_default = keymap_path / 'default' -    keymap_path_new = keymap_path / keymap +    # ask for user input if keyboard or keymap was not provided in the command line +    kb_name = cli.config.new_keymap.keyboard if cli.config.new_keymap.keyboard else prompt_keyboard() +    user_name = cli.config.new_keymap.keymap if cli.config.new_keymap.keymap else prompt_user()      # check directories -    if not kb_path.exists(): -        cli.log.error('Keyboard %s does not exist!', kb_path) +    if not is_keyboard(kb_name): +        cli.log.error(f'Keyboard {{fg_cyan}}{kb_name}{{fg_reset}} does not exist! Please choose a valid name.')          return False +    # generate keymap paths +    km_path = keymap(kb_name) +    keymap_path_default = km_path / 'default' +    keymap_path_new = km_path / user_name +      if not keymap_path_default.exists(): -        cli.log.error('Keyboard default %s does not exist!', keymap_path_default) +        cli.log.error(f'Default keymap {{fg_cyan}}{keymap_path_default}{{fg_reset}} does not exist!')          return False      if keymap_path_new.exists(): -        cli.log.error('Keymap %s already exists!', keymap_path_new) +        cli.log.error(f'Keymap {{fg_cyan}}{user_name}{{fg_reset}} already exists! Please choose a different name.')          return False      # create user directory with default keymap files      shutil.copytree(keymap_path_default, keymap_path_new, symlinks=True)      # end message to user -    cli.log.info("%s keymap directory created in: %s", keymap, keymap_path_new) -    cli.log.info("Compile a firmware with your new keymap by typing: \n\n\tqmk compile -kb %s -km %s\n", keyboard, keymap) +    cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}.{{fg_reset}}') +    cli.log.info(f"Compile a firmware with your new keymap by typing: {{fg_yellow}}qmk compile -kb {kb_name} -km {user_name}{{fg_reset}}.") diff --git a/lib/python/qmk/cli/pyformat.py b/lib/python/qmk/cli/pyformat.py deleted file mode 100755 index c624f74aeb..0000000000 --- a/lib/python/qmk/cli/pyformat.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Point people to the new command name. -""" -import sys -from pathlib import Path - -from milc import cli - - [email protected]('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.") [email protected]('Pointer to the new command name: qmk format-python.', hidden=False if cli.config.user.developer else True) -def pyformat(cli): -    """Pointer to the new command name: qmk format-python. -    """ -    cli.log.warning('"qmk pyformat" has been renamed to "qmk format-python". Please use the new command in the future.') -    argv = [sys.executable, *sys.argv] -    argv[argv.index('pyformat')] = 'format-python' -    script_path = Path(argv[1]) -    script_path_exe = Path(f'{argv[1]}.exe') - -    if not script_path.exists() and script_path_exe.exists(): -        # For reasons I don't understand ".exe" is stripped from the script name on windows. -        argv[1] = str(script_path_exe) - -    return cli.run(argv, capture_output=False).returncode diff --git a/lib/python/qmk/flashers.py b/lib/python/qmk/flashers.py index e902e5072f..f83665d9ac 100644 --- a/lib/python/qmk/flashers.py +++ b/lib/python/qmk/flashers.py @@ -71,6 +71,12 @@ def _find_usb_device(vid_hex, pid_hex):              return usb.core.find(idVendor=vid_hex, idProduct=pid_hex) +def _find_uf2_devices(): +    """Delegate to uf2conv.py as VID:PID pairs can potentially fluctuate more than other bootloaders +    """ +    return cli.run(['util/uf2conv.py', '--list']).stdout.splitlines() + +  def _find_bootloader():      # To avoid running forever in the background, only look for bootloaders for 10min      start_time = time.time() @@ -95,6 +101,8 @@ def _find_bootloader():                      else:                          details = None                      return (bl, details) +        if _find_uf2_devices(): +            return ('_uf2_compatible_', None)          time.sleep(0.1)      return (None, None) @@ -184,6 +192,10 @@ def _flash_mdloader(file):      cli.run(['mdloader', '--first', '--download', file, '--restart'], capture_output=False) +def _flash_uf2(file): +    cli.run(['util/uf2conv.py', '--deploy', file], capture_output=False) + +  def flasher(mcu, file):      bl, details = _find_bootloader()      # Add a small sleep to avoid race conditions @@ -208,6 +220,8 @@ def flasher(mcu, file):              return (True, "Specifying the MCU with '-m' is necessary for ISP flashing!")      elif bl == 'md-boot':          _flash_mdloader(file) +    elif bl == '_uf2_compatible_': +        _flash_uf2(file)      else:          return (True, "Known bootloader found but flashing not currently supported!") diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 7e588b5182..e73d825696 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -10,7 +10,6 @@ from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS  from qmk.c_parse import find_layouts, parse_config_h_file, find_led_config  from qmk.json_schema import deep_update, json_load, validate  from qmk.keyboard import config_h, rules_mk -from qmk.keymap import list_keymaps, locate_keymap  from qmk.commands import parse_configurator_json  from qmk.makefile import parse_rules_mk_file  from qmk.math import compute @@ -53,7 +52,7 @@ def _validate(keyboard, info_data):      community_layouts_names = list(map(lambda layout: f'LAYOUT_{layout}', community_layouts))      # Make sure we have at least one layout -    if len(layouts) == 0: +    if len(layouts) == 0 or all(not layout.get('json_layout', False) for layout in layouts.values()):          _log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in info.json.')      # Providing only LAYOUT_all "because I define my layouts in a 3rd party tool" @@ -99,10 +98,6 @@ def info_json(keyboard):          'maintainer': 'qmk',      } -    # Populate the list of JSON keymaps -    for keymap in list_keymaps(keyboard, c=False, fullpath=True): -        info_data['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'} -      # Populate layout data      layouts, aliases = _search_keyboard_h(keyboard) @@ -112,6 +107,7 @@ def info_json(keyboard):      for layout_name, layout_json in layouts.items():          if not layout_name.startswith('LAYOUT_kc'):              layout_json['c_macro'] = True +            layout_json['json_layout'] = False              info_data['layouts'][layout_name] = layout_json      # Merge in the data from info.json, config.h, and rules.mk @@ -561,8 +557,16 @@ def _process_defaults(info_data):      for default_type in defaults_map.keys():          thing_map = defaults_map[default_type]          if default_type in info_data: -            for key, value in thing_map.get(info_data[default_type], {}).items(): -                info_data[key] = value +            merged_count = 0 +            thing_items = thing_map.get(info_data[default_type], {}).items() +            for key, value in thing_items: +                if key not in info_data: +                    info_data[key] = value +                    merged_count += 1 + +            if merged_count == 0 and len(thing_items) > 0: +                _log_warning(info_data, 'All defaults for \'%s\' were skipped, potential redundant config or misconfiguration detected' % (default_type)) +      return info_data @@ -748,6 +752,7 @@ def arm_processor_rules(info_data, rules):      """      info_data['processor_type'] = 'arm'      info_data['protocol'] = 'ChibiOS' +    info_data['platform_key'] = 'chibios'      if 'STM32' in info_data['processor']:          info_data['platform'] = 'STM32' @@ -755,6 +760,7 @@ def arm_processor_rules(info_data, rules):          info_data['platform'] = rules['MCU_SERIES']      elif 'ARM_ATSAM' in rules:          info_data['platform'] = 'ARM_ATSAM' +        info_data['platform_key'] = 'arm_atsam'      return info_data @@ -764,6 +770,7 @@ def avr_processor_rules(info_data, rules):      """      info_data['processor_type'] = 'avr'      info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown' +    info_data['platform_key'] = 'avr'      info_data['protocol'] = 'V-USB' if info_data['processor'] in VUSB_PROCESSORS else 'LUFA'      # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk: @@ -818,6 +825,7 @@ def merge_info_jsons(keyboard, info_data):                      msg = 'Number of keys for %s does not match! info.json specifies %d keys, C macro specifies %d'                      _log_error(info_data, msg % (layout_name, len(layout['layout']), len(info_data['layouts'][layout_name]['layout'])))                  else: +                    info_data['layouts'][layout_name]['json_layout'] = True                      for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']):                          existing_key.update(new_key)              else: @@ -825,6 +833,7 @@ def merge_info_jsons(keyboard, info_data):                      _log_error(info_data, f'Layout "{layout_name}" has no "matrix" definition in either "info.json" or "<keyboard>.h"!')                  else:                      layout['c_macro'] = False +                    layout['json_layout'] = True                      info_data['layouts'][layout_name] = layout          # Update info_data with the new data @@ -864,6 +873,9 @@ def find_info_json(keyboard):  def keymap_json_config(keyboard, keymap):      """Extract keymap level config      """ +    # TODO: resolve keymap.py and info.py circular dependencies +    from qmk.keymap import locate_keymap +      keymap_folder = locate_keymap(keyboard, keymap).parent      km_info_json = parse_configurator_json(keymap_folder / 'keymap.json') @@ -873,6 +885,9 @@ def keymap_json_config(keyboard, keymap):  def keymap_json(keyboard, keymap):      """Generate the info.json data for a specific keymap.      """ +    # TODO: resolve keymap.py and info.py circular dependencies +    from qmk.keymap import locate_keymap +      keymap_folder = locate_keymap(keyboard, keymap).parent      # Files to scan diff --git a/lib/python/qmk/json_schema.py b/lib/python/qmk/json_schema.py index 934e2f841f..c886a0d868 100644 --- a/lib/python/qmk/json_schema.py +++ b/lib/python/qmk/json_schema.py @@ -1,12 +1,13 @@  """Functions that help us generate and use info.json files.  """  import json +import hjson +import jsonschema  from collections.abc import Mapping  from functools import lru_cache +from typing import OrderedDict  from pathlib import Path -import hjson -import jsonschema  from milc import cli @@ -101,3 +102,34 @@ def deep_update(origdict, newdict):              origdict[key] = value      return origdict + + +def merge_ordered_dicts(dicts): +    """Merges nested OrderedDict objects resulting from reading a hjson file. +    Later input dicts overrides earlier dicts for plain values. +    Arrays will be appended. If the first entry of an array is "!reset!", the contents of the array will be cleared and replaced with RHS. +    Dictionaries will be recursively merged. If any entry is "!reset!", the contents of the dictionary will be cleared and replaced with RHS. +    """ +    result = OrderedDict() + +    def add_entry(target, k, v): +        if k in target and isinstance(v, (OrderedDict, dict)): +            if "!reset!" in v: +                target[k] = v +            else: +                target[k] = merge_ordered_dicts([target[k], v]) +            if "!reset!" in target[k]: +                del target[k]["!reset!"] +        elif k in target and isinstance(v, list): +            if v[0] == '!reset!': +                target[k] = v[1:] +            else: +                target[k] = target[k] + v +        else: +            target[k] = v + +    for d in dicts: +        for (k, v) in d.items(): +            add_entry(result, k, v) + +    return result diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py index 6ddbba8fa5..0c980faf2b 100644 --- a/lib/python/qmk/keyboard.py +++ b/lib/python/qmk/keyboard.py @@ -98,14 +98,18 @@ def keyboard_completer(prefix, action, parser, parsed_args):      return list_keyboards() -def list_keyboards(): -    """Returns a list of all keyboards. +def list_keyboards(resolve_defaults=True): +    """Returns a list of all keyboards - optionally processing any DEFAULT_FOLDER.      """      # We avoid pathlib here because this is performance critical code.      kb_wildcard = os.path.join(base_path, "**", "rules.mk")      paths = [path for path in glob(kb_wildcard, recursive=True) if os.path.sep + 'keymaps' + os.path.sep not in path] -    return sorted(set(map(resolve_keyboard, map(_find_name, paths)))) +    found = map(_find_name, paths) +    if resolve_defaults: +        found = map(resolve_keyboard, found) + +    return sorted(set(found))  def resolve_keyboard(keyboard): diff --git a/lib/python/qmk/keycodes.py b/lib/python/qmk/keycodes.py index cf1ee0767a..d2f2492829 100644 --- a/lib/python/qmk/keycodes.py +++ b/lib/python/qmk/keycodes.py @@ -1,8 +1,64 @@  from pathlib import Path -from qmk.json_schema import deep_update, json_load, validate +from qmk.json_schema import merge_ordered_dicts, deep_update, json_load, validate -CONSTANTS_PATH = Path('data/constants/keycodes/') +CONSTANTS_PATH = Path('data/constants/') +KEYCODES_PATH = CONSTANTS_PATH / 'keycodes' +EXTRAS_PATH = KEYCODES_PATH / 'extras' + + +def _find_versions(path, prefix): +    ret = [] +    for file in path.glob(f'{prefix}_[0-9].[0-9].[0-9].hjson'): +        ret.append(file.stem.split('_')[-1]) + +    ret.sort(reverse=True) +    return ret + + +def _potential_search_versions(version, lang=None): +    versions = list_versions(lang) +    versions.reverse() + +    loc = versions.index(version) + 1 + +    return versions[:loc] + + +def _search_path(lang=None): +    return EXTRAS_PATH if lang else KEYCODES_PATH + + +def _search_prefix(lang=None): +    return f'keycodes_{lang}' if lang else 'keycodes' + + +def _locate_files(path, prefix, versions): +    # collate files by fragment "type" +    files = {'_': []} +    for version in versions: +        files['_'].append(path / f'{prefix}_{version}.hjson') + +        for file in path.glob(f'{prefix}_{version}_*.hjson'): +            fragment = file.stem.replace(f'{prefix}_{version}_', '') +            if fragment not in files: +                files[fragment] = [] +            files[fragment].append(file) + +    return files + + +def _process_files(files): +    # allow override within types of fragments - but not globally +    spec = {} +    for category in files.values(): +        specs = [] +        for file in category: +            specs.append(json_load(file)) + +        deep_update(spec, merge_ordered_dicts(specs)) + +    return spec  def _validate(spec): @@ -19,26 +75,21 @@ def _validate(spec):          raise ValueError(f'Keycode spec contains duplicate keycodes! ({",".join(duplicates)})') -def load_spec(version): +def load_spec(version, lang=None):      """Build keycode data from the requested spec file      """      if version == 'latest': -        version = list_versions()[0] +        version = list_versions(lang)[0] -    file = CONSTANTS_PATH / f'keycodes_{version}.hjson' -    if not file.exists(): -        raise ValueError(f'Requested keycode spec ({version}) is invalid!') +    path = _search_path(lang) +    prefix = _search_prefix(lang) +    versions = _potential_search_versions(version, lang) -    # Load base -    spec = json_load(file) - -    # Merge in fragments -    fragments = CONSTANTS_PATH.glob(f'keycodes_{version}_*.hjson') -    for file in fragments: -        deep_update(spec, json_load(file)) +    # Load bases + any fragments +    spec = _process_files(_locate_files(path, prefix, versions))      # Sort? -    spec['keycodes'] = dict(sorted(spec['keycodes'].items())) +    spec['keycodes'] = dict(sorted(spec.get('keycodes', {}).items()))      # Validate?      _validate(spec) @@ -46,12 +97,20 @@ def load_spec(version):      return spec -def list_versions(): +def list_versions(lang=None):      """Return available versions - sorted newest first      """ -    ret = [] -    for file in CONSTANTS_PATH.glob('keycodes_[0-9].[0-9].[0-9].hjson'): -        ret.append(file.stem.split('_')[1]) +    path = _search_path(lang) +    prefix = _search_prefix(lang) + +    return _find_versions(path, prefix) + + +def list_languages(): +    """Return available languages +    """ +    ret = set() +    for file in EXTRAS_PATH.glob('keycodes_*_[0-9].[0-9].[0-9].hjson'): +        ret.add(file.stem.split('_')[1]) -    ret.sort(reverse=True)      return ret diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index 315af35b73..ce518e379d 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -12,8 +12,9 @@ from pygments.token import Token  from pygments import lex  import qmk.path -from qmk.keyboard import find_keyboard_from_dir, rules_mk, keyboard_folder +from qmk.keyboard import find_keyboard_from_dir, keyboard_folder  from qmk.errors import CppError +from qmk.info import info_json  # The `keymap.c` template to use when a keyboard doesn't have its own  DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H @@ -374,11 +375,11 @@ def locate_keymap(keyboard, keymap):          return keymap_path      # Check community layouts as a fallback -    rules = rules_mk(keyboard) +    info = info_json(keyboard) -    if "LAYOUTS" in rules: -        for layout in rules["LAYOUTS"].split(): -            community_layout = Path('layouts/community') / layout / keymap +    for community_parent in Path('layouts').glob('*/'): +        for layout in info.get("community_layouts", []): +            community_layout = community_parent / layout / keymap              if community_layout.exists():                  if (community_layout / 'keymap.json').exists():                      return community_layout / 'keymap.json' @@ -408,37 +409,36 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa      Returns:          a sorted list of valid keymap names.      """ -    # parse all the rules.mk files for the keyboard -    rules = rules_mk(keyboard)      names = set() -    if rules is not None: -        keyboards_dir = Path('keyboards') -        kb_path = keyboards_dir / keyboard +    keyboards_dir = Path('keyboards') +    kb_path = keyboards_dir / keyboard -        # walk up the directory tree until keyboards_dir -        # and collect all directories' name with keymap.c file in it -        while kb_path != keyboards_dir: -            keymaps_dir = kb_path / "keymaps" +    # walk up the directory tree until keyboards_dir +    # and collect all directories' name with keymap.c file in it +    while kb_path != keyboards_dir: +        keymaps_dir = kb_path / "keymaps" -            if keymaps_dir.is_dir(): -                for keymap in keymaps_dir.iterdir(): +        if keymaps_dir.is_dir(): +            for keymap in keymaps_dir.iterdir(): +                if is_keymap_dir(keymap, c, json, additional_files): +                    keymap = keymap if fullpath else keymap.name +                    names.add(keymap) + +        kb_path = kb_path.parent + +    # Check community layouts as a fallback +    info = info_json(keyboard) + +    for community_parent in Path('layouts').glob('*/'): +        for layout in info.get("community_layouts", []): +            cl_path = community_parent / layout +            if cl_path.is_dir(): +                for keymap in cl_path.iterdir():                      if is_keymap_dir(keymap, c, json, additional_files):                          keymap = keymap if fullpath else keymap.name                          names.add(keymap) -            kb_path = kb_path.parent - -        # if community layouts are supported, get them -        if "LAYOUTS" in rules: -            for layout in rules["LAYOUTS"].split(): -                cl_path = Path('layouts/community') / layout -                if cl_path.is_dir(): -                    for keymap in cl_path.iterdir(): -                        if is_keymap_dir(keymap, c, json, additional_files): -                            keymap = keymap if fullpath else keymap.name -                            names.add(keymap) -      return sorted(names) diff --git a/lib/python/qmk/painter.py b/lib/python/qmk/painter.py index d0cc1dddec..7ecdc55404 100644 --- a/lib/python/qmk/painter.py +++ b/lib/python/qmk/painter.py @@ -7,6 +7,20 @@ from PIL import Image, ImageOps  # The list of valid formats Quantum Painter supports  valid_formats = { +    'rgb888': { +        'image_format': 'IMAGE_FORMAT_RGB888', +        'bpp': 24, +        'has_palette': False, +        'num_colors': 16777216, +        'image_format_byte': 0x09,  # see qp_internal_formats.h +    }, +    'rgb565': { +        'image_format': 'IMAGE_FORMAT_RGB565', +        'bpp': 16, +        'has_palette': False, +        'num_colors': 65536, +        'image_format_byte': 0x08,  # see qp_internal_formats.h +    },      'pal256': {          'image_format': 'IMAGE_FORMAT_PALETTE',          'bpp': 8, @@ -144,19 +158,33 @@ def convert_requested_format(im, format):      ncolors = format["num_colors"]      image_format = format["image_format"] -    # Ensure we have a valid number of colors for the palette -    if ncolors <= 0 or ncolors > 256 or (ncolors & (ncolors - 1) != 0): -        raise ValueError("Number of colors must be 2, 4, 16, or 256.") -      # Work out where we're getting the bytes from      if image_format == 'IMAGE_FORMAT_GRAYSCALE': +        # Ensure we have a valid number of colors for the palette +        if ncolors <= 0 or ncolors > 256 or (ncolors & (ncolors - 1) != 0): +            raise ValueError("Number of colors must be 2, 4, 16, or 256.")          # If mono, convert input to grayscale, then to RGB, then grab the raw bytes corresponding to the intensity of the red channel          im = ImageOps.grayscale(im)          im = im.convert("RGB")      elif image_format == 'IMAGE_FORMAT_PALETTE': +        # Ensure we have a valid number of colors for the palette +        if ncolors <= 0 or ncolors > 256 or (ncolors & (ncolors - 1) != 0): +            raise ValueError("Number of colors must be 2, 4, 16, or 256.")          # If color, convert input to RGB, palettize based on the supplied number of colors, then get the raw palette bytes          im = im.convert("RGB")          im = im.convert("P", palette=Image.ADAPTIVE, colors=ncolors) +    elif image_format == 'IMAGE_FORMAT_RGB565': +        # Ensure we have a valid number of colors for the palette +        if ncolors != 65536: +            raise ValueError("Number of colors must be 65536.") +        # If color, convert input to RGB +        im = im.convert("RGB") +    elif image_format == 'IMAGE_FORMAT_RGB888': +        # Ensure we have a valid number of colors for the palette +        if ncolors != 1677216: +            raise ValueError("Number of colors must be 16777216.") +        # If color, convert input to RGB +        im = im.convert("RGB")      return im @@ -170,8 +198,12 @@ def convert_image_bytes(im, format):      image_format = format["image_format"]      shifter = int(math.log2(ncolors))      pixels_per_byte = int(8 / math.log2(ncolors)) +    bytes_per_pixel = math.ceil(math.log2(ncolors) / 8)      (width, height) = im.size -    expected_byte_count = ((width * height) + (pixels_per_byte - 1)) // pixels_per_byte +    if (pixels_per_byte != 0): +        expected_byte_count = ((width * height) + (pixels_per_byte - 1)) // pixels_per_byte +    else: +        expected_byte_count = width * height * bytes_per_pixel      if image_format == 'IMAGE_FORMAT_GRAYSCALE':          # Take the red channel @@ -212,6 +244,44 @@ def convert_image_bytes(im, format):                      byte = byte | ((image_bytes[byte_offset] & (ncolors - 1)) << int(n * shifter))              bytearray.append(byte) +    if image_format == 'IMAGE_FORMAT_RGB565': +        # Take the red, green, and blue channels +        image_bytes_red = im.tobytes("raw", "R") +        image_bytes_green = im.tobytes("raw", "G") +        image_bytes_blue = im.tobytes("raw", "B") +        image_pixels_len = len(image_bytes_red) + +        # No palette +        palette = None + +        bytearray = [] +        for x in range(image_pixels_len): +            # 5 bits of red, 3 MSb of green +            byte = ((image_bytes_red[x] >> 3 & 0x1F) << 3) + (image_bytes_green[x] >> 5 & 0x07) +            bytearray.append(byte) +            # 3 LSb of green, 5 bits of blue +            byte = ((image_bytes_green[x] >> 2 & 0x07) << 5) + (image_bytes_blue[x] >> 3 & 0x1F) +            bytearray.append(byte) + +    if image_format == 'IMAGE_FORMAT_RGB888': +        # Take the red, green, and blue channels +        image_bytes_red = im.tobytes("raw", "R") +        image_bytes_green = im.tobytes("raw", "G") +        image_bytes_blue = im.tobytes("raw", "B") +        image_pixels_len = len(image_bytes_red) + +        # No palette +        palette = None + +        bytearray = [] +        for x in range(image_pixels_len): +            byte = image_bytes_red[x] +            bytearray.append(byte) +            byte = image_bytes_green[x] +            bytearray.append(byte) +            byte = image_bytes_blue[x] +            bytearray.append(byte) +      if len(bytearray) != expected_byte_count:          raise Exception(f"Wrong byte count, was {len(bytearray)}, expected {expected_byte_count}") diff --git a/lib/python/qmk/submodules.py b/lib/python/qmk/submodules.py index 52efa602a0..d0050b371d 100644 --- a/lib/python/qmk/submodules.py +++ b/lib/python/qmk/submodules.py @@ -21,15 +21,17 @@ def status():      status is None when the submodule doesn't exist, False when it's out of date, and True when it's current      """      submodules = {} -    git_cmd = cli.run(['git', 'submodule', 'status'], timeout=30) - -    for line in git_cmd.stdout.split('\n'): -        if not line: -            continue +    gitmodule_config = cli.run(['git', 'config', '-f', '.gitmodules', '-l'], timeout=30) +    for line in gitmodule_config.stdout.splitlines(): +        key, value = line.split('=', maxsplit=2) +        if key.endswith('.path'): +            submodules[value] = {'name': value, 'status': None} +    git_cmd = cli.run(['git', 'submodule', 'status'], timeout=30) +    for line in git_cmd.stdout.splitlines():          status = line[0]          githash, submodule = line[1:].split()[:2] -        submodules[submodule] = {'name': submodule, 'githash': githash} +        submodules[submodule]['githash'] = githash          if status == '-':              submodules[submodule]['status'] = None @@ -40,11 +42,8 @@ def status():          else:              raise ValueError('Unknown `git submodule status` sha-1 prefix character: "%s"' % status) -    submodule_logs = cli.run(['git', 'submodule', '-q', 'foreach', 'git --no-pager log --pretty=format:"$sm_path%x01%h%x01%ad%x01%s%x0A" --date=iso -n1']) -    for log_line in submodule_logs.stdout.split('\n'): -        if not log_line: -            continue - +    submodule_logs = cli.run(['git', 'submodule', '-q', 'foreach', 'git --no-pager log --no-show-signature --pretty=format:"$sm_path%x01%h%x01%ad%x01%s%x0A" --date=iso -n1']) +    for log_line in submodule_logs.stdout.splitlines():          r = log_line.split('\x01')          submodule = r[0]          submodules[submodule]['shorthash'] = r[1] if len(r) > 1 else '' @@ -52,10 +51,7 @@ def status():          submodules[submodule]['last_log_message'] = r[3] if len(r) > 3 else ''      submodule_tags = cli.run(['git', 'submodule', '-q', 'foreach', '\'echo $sm_path `git describe --tags`\'']) -    for log_line in submodule_tags.stdout.split('\n'): -        if not log_line: -            continue - +    for log_line in submodule_tags.stdout.splitlines():          r = log_line.split()          submodule = r[0]          submodules[submodule]['describe'] = r[1] if len(r) > 1 else '' | 
