diff options
| author | Nick Brassel <[email protected]> | 2023-11-28 16:39:48 +1100 | 
|---|---|---|
| committer | Nick Brassel <[email protected]> | 2023-11-28 16:40:47 +1100 | 
| commit | 0115951de12bd4c798a43a380651c34552552e85 (patch) | |
| tree | 62999011ae782d93aa6dc9c889615db2a840cafc /lib/python/qmk | |
| parent | 0379d1f59e58cefa18cdb72ba1b77507d1108ae6 (diff) | |
| parent | 4d99e0a23cff02f4c6e55e093b5de08a9be0df7c (diff) | |
| download | qmk_firmware-0.23.0.tar.gz qmk_firmware-0.23.0.zip | |
`develop` -> `master`, 2023q4 edition0.23.0
Diffstat (limited to 'lib/python/qmk')
33 files changed, 1230 insertions, 489 deletions
| diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py new file mode 100644 index 0000000000..1ab489cec3 --- /dev/null +++ b/lib/python/qmk/build_targets.py @@ -0,0 +1,238 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +import json +import shutil +from typing import List, Union +from pathlib import Path +from dotty_dict import dotty, Dotty +from milc import cli +from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX +from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json +from qmk.keyboard import keyboard_folder +from qmk.info import keymap_json +from qmk.keymap import locate_keymap +from qmk.path import is_under_qmk_firmware, is_under_qmk_userspace + + +class BuildTarget: +    def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None): +        self._keyboard = keyboard_folder(keyboard) +        self._keyboard_safe = self._keyboard.replace('/', '_') +        self._keymap = keymap +        self._parallel = 1 +        self._clean = False +        self._compiledb = False +        self._target = f'{self._keyboard_safe}_{self.keymap}' +        self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}') +        self._generated_files_path = self._intermediate_output / 'src' +        self._json = json.to_dict() if isinstance(json, Dotty) else json + +    def __str__(self): +        return f'{self.keyboard}:{self.keymap}' + +    def __repr__(self): +        return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap})' + +    def __eq__(self, __value: object) -> bool: +        if not isinstance(__value, BuildTarget): +            return False +        return self.__repr__() == __value.__repr__() + +    def __ne__(self, __value: object) -> bool: +        return not self.__eq__(__value) + +    def __hash__(self) -> int: +        return self.__repr__().__hash__() + +    def configure(self, parallel: int = None, clean: bool = None, compiledb: bool = None) -> None: +        if parallel is not None: +            self._parallel = parallel +        if clean is not None: +            self._clean = clean +        if compiledb is not None: +            self._compiledb = compiledb + +    @property +    def keyboard(self) -> str: +        return self._keyboard + +    @property +    def keymap(self) -> str: +        return self._keymap + +    @property +    def json(self) -> dict: +        if not self._json: +            self._load_json() +        if not self._json: +            return {} +        return self._json + +    @property +    def dotty(self) -> Dotty: +        return dotty(self.json) + +    def _common_make_args(self, dry_run: bool = False, build_target: str = None): +        compile_args = [ +            find_make(), +            *get_make_parallel_args(self._parallel), +            '-r', +            '-R', +            '-f', +            'builddefs/build_keyboard.mk', +        ] + +        if not cli.config.general.verbose: +            compile_args.append('-s') + +        verbose = 'true' if cli.config.general.verbose else 'false' +        color = 'true' if cli.config.general.color else 'false' + +        if dry_run: +            compile_args.append('-n') + +        if build_target: +            compile_args.append(build_target) + +        compile_args.extend([ +            f'KEYBOARD={self.keyboard}', +            f'KEYMAP={self.keymap}', +            f'KEYBOARD_FILESAFE={self._keyboard_safe}', +            f'TARGET={self._target}', +            f'INTERMEDIATE_OUTPUT={self._intermediate_output}', +            f'VERBOSE={verbose}', +            f'COLOR={color}', +            'SILENT=false', +            'QMK_BIN="qmk"', +        ]) + +        return compile_args + +    def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: +        raise NotImplementedError("prepare_build() not implemented in base class") + +    def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]: +        raise NotImplementedError("compile_command() not implemented in base class") + +    def generate_compilation_database(self, build_target: str = None, skip_clean: bool = False, **env_vars) -> None: +        self.prepare_build(build_target=build_target, **env_vars) +        command = self.compile_command(build_target=build_target, dry_run=True, **env_vars) +        from qmk.cli.generate.compilation_database import write_compilation_database  # Lazy load due to circular references +        write_compilation_database(command=command, output_path=QMK_FIRMWARE / 'compile_commands.json', skip_clean=skip_clean, **env_vars) + +    def compile(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: +        if self._clean or self._compiledb: +            command = [find_make(), "clean"] +            if dry_run: +                command.append('-n') +            cli.log.info('Cleaning with {fg_cyan}%s', ' '.join(command)) +            cli.run(command, capture_output=False) + +        if self._compiledb and not dry_run: +            self.generate_compilation_database(build_target=build_target, skip_clean=True, **env_vars) + +        self.prepare_build(build_target=build_target, dry_run=dry_run, **env_vars) +        command = self.compile_command(build_target=build_target, **env_vars) +        cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) +        if not dry_run: +            cli.echo('\n') +            ret = cli.run(command, capture_output=False) +            if ret.returncode: +                return ret.returncode + + +class KeyboardKeymapBuildTarget(BuildTarget): +    def __init__(self, keyboard: str, keymap: str, json: dict = None): +        super().__init__(keyboard=keyboard, keymap=keymap, json=json) + +    def __repr__(self): +        return f'KeyboardKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap})' + +    def _load_json(self): +        self._json = keymap_json(self.keyboard, self.keymap) + +    def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: +        pass + +    def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]: +        compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target) + +        for key, value in env_vars.items(): +            compile_args.append(f'{key}={value}') + +        # Need to override the keymap path if the keymap is a userspace directory. +        # This also ensures keyboard aliases as per `keyboard_aliases.hjson` still work if the userspace has the keymap +        # in an equivalent historical location. +        keymap_location = locate_keymap(self.keyboard, self.keymap) +        if is_under_qmk_userspace(keymap_location) and not is_under_qmk_firmware(keymap_location): +            keymap_directory = keymap_location.parent +            compile_args.extend([ +                f'MAIN_KEYMAP_PATH_1={keymap_directory}', +                f'MAIN_KEYMAP_PATH_2={keymap_directory}', +                f'MAIN_KEYMAP_PATH_3={keymap_directory}', +                f'MAIN_KEYMAP_PATH_4={keymap_directory}', +                f'MAIN_KEYMAP_PATH_5={keymap_directory}', +            ]) + +        return compile_args + + +class JsonKeymapBuildTarget(BuildTarget): +    def __init__(self, json_path): +        if isinstance(json_path, Path): +            self.json_path = json_path +        else: +            self.json_path = None + +        json = parse_configurator_json(json_path)  # Will load from stdin if provided + +        # In case the user passes a keymap.json from a keymap directory directly to the CLI. +        # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json +        json["keymap"] = json.get("keymap", "default_json") + +        super().__init__(keyboard=json['keyboard'], keymap=json['keymap'], json=json) + +        self._keymap_json = self._generated_files_path / 'keymap.json' + +    def __repr__(self): +        return f'JsonKeymapTarget(keyboard={self.keyboard}, keymap={self.keymap}, path={self.json_path})' + +    def _load_json(self): +        pass  # Already loaded in constructor + +    def prepare_build(self, build_target: str = None, dry_run: bool = False, **env_vars) -> None: +        if self._clean: +            if self._intermediate_output.exists(): +                shutil.rmtree(self._intermediate_output) + +        # begin with making the deepest folder in the tree +        self._generated_files_path.mkdir(exist_ok=True, parents=True) + +        # Compare minified to ensure consistent comparison +        new_content = json.dumps(self.json, separators=(',', ':')) +        if self._keymap_json.exists(): +            old_content = json.dumps(json.loads(self._keymap_json.read_text(encoding='utf-8')), separators=(',', ':')) +            if old_content == new_content: +                new_content = None + +        # Write the keymap.json file if different so timestamps are only updated +        # if the content changes -- running `make` won't treat it as modified. +        if new_content: +            self._keymap_json.write_text(new_content, encoding='utf-8') + +    def compile_command(self, build_target: str = None, dry_run: bool = False, **env_vars) -> List[str]: +        compile_args = self._common_make_args(dry_run=dry_run, build_target=build_target) +        compile_args.extend([ +            f'MAIN_KEYMAP_PATH_1={self._intermediate_output}', +            f'MAIN_KEYMAP_PATH_2={self._intermediate_output}', +            f'MAIN_KEYMAP_PATH_3={self._intermediate_output}', +            f'MAIN_KEYMAP_PATH_4={self._intermediate_output}', +            f'MAIN_KEYMAP_PATH_5={self._intermediate_output}', +            f'KEYMAP_JSON={self._keymap_json}', +            f'KEYMAP_PATH={self._generated_files_path}', +        ]) + +        for key, value in env_vars.items(): +            compile_args.append(f'{key}={value}') + +        return compile_args diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 695a180066..cf60903687 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -81,6 +81,11 @@ subcommands = [      'qmk.cli.new.keymap',      'qmk.cli.painter',      'qmk.cli.pytest', +    'qmk.cli.userspace.add', +    'qmk.cli.userspace.compile', +    'qmk.cli.userspace.doctor', +    'qmk.cli.userspace.list', +    'qmk.cli.userspace.remove',      'qmk.cli.via2json',  ] diff --git a/lib/python/qmk/cli/c2json.py b/lib/python/qmk/cli/c2json.py index 7f6aca070a..f7f1f2ffba 100644 --- a/lib/python/qmk/cli/c2json.py +++ b/lib/python/qmk/cli/c2json.py @@ -1,45 +1,57 @@  """Generate a keymap.json from a keymap.c file.  """ +import re  import json  from argcomplete.completers import FilesCompleter  from milc import cli -import qmk.keymap  import qmk.path  from qmk.json_encoders import InfoJSONEncoder +from qmk.decorators import automagic_keyboard, automagic_keymap  from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.keymap import locate_keymap, find_keymap_from_dir, generate_json, c2json as c2json_impl  from qmk.errors import CppError +from qmk.commands import dump_lines  @cli.argument('--no-cpp', arg_only=True, action='store_false', help='Do not use \'cpp\' on keymap.c')  @cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to')  @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") [email protected]('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='The keyboard\'s name') [email protected]('-km', '--keymap', arg_only=True, required=True, help='The keymap\'s name') [email protected]('filename', arg_only=True, completer=FilesCompleter('.c'), help='keymap.c file') [email protected]('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard\'s name') [email protected]('-km', '--keymap', help='The keymap\'s name') [email protected]('filename', nargs='?', type=qmk.path.FileType('r'), arg_only=True, completer=FilesCompleter('.c'), help='keymap.c file')  @cli.subcommand('Creates a keymap.json from a keymap.c file.') +@automagic_keyboard +@automagic_keymap  def c2json(cli):      """Generate a keymap.json from a keymap.c file.      This command uses the `qmk.keymap` module to generate a keymap.json from a keymap.c file. The generated keymap is written to stdout, or to a file if -o is provided.      """ -    if cli.args.filename != '-': -        cli.args.filename = qmk.path.normpath(cli.args.filename) +    filename = cli.args.filename +    keyboard = cli.config.c2json.keyboard +    keymap = cli.config.c2json.keymap -        # Error checking -        if not cli.args.filename.exists(): -            cli.log.error('C file does not exist!') -            cli.print_usage() -            return False +    if filename: +        if not keyboard and not keymap: +            # fallback to inferring keyboard/keymap from path +            (keymap, found_type) = find_keymap_from_dir(filename) +            if found_type == 'keymap_directory': +                keyboard = re.search(fr"keyboards/(.+)/keymaps/{keymap}/.*", filename.as_posix()).group(1) -    # Environment processing -    if cli.args.output == ('-'): -        cli.args.output = None +    elif keyboard and keymap: +        if not filename: +            # fallback to inferring keyboard/keymap from path +            filename = locate_keymap(keyboard, keymap) + +    if not all((filename, keyboard, keymap)): +        cli.log.error('You must supply keyboard and keymap, a path to a keymap.c within qmk_firmware, or absolute filename and keyboard and keymap') +        cli.print_help() +        return False -    # Parse the keymap.c      try: -        keymap_json = qmk.keymap.c2json(cli.args.keyboard, cli.args.keymap, cli.args.filename, use_cpp=cli.args.no_cpp) +        keymap_json = c2json_impl(keyboard, keymap, filename, use_cpp=cli.args.no_cpp)      except CppError as e:          if cli.config.general.verbose:              cli.log.debug('The C pre-processor ran into a fatal error: %s', e) @@ -48,19 +60,14 @@ def c2json(cli):      # Generate the keymap.json      try: -        keymap_json = qmk.keymap.generate_json(keymap_json['keymap'], keymap_json['keyboard'], keymap_json['layout'], keymap_json['layers']) +        keymap_json = generate_json(keymap_json['keymap'], keymap_json['keyboard'], keymap_json['layout'], keymap_json['layers'])      except KeyError:          cli.log.error('Something went wrong. Try to use --no-cpp.')          return False      if cli.args.output: -        cli.args.output.parent.mkdir(parents=True, exist_ok=True) -        if cli.args.output.exists(): -            cli.args.output.replace(cli.args.output.parent / (cli.args.output.name + '.bak')) -        cli.args.output.write_text(json.dumps(keymap_json, cls=InfoJSONEncoder, sort_keys=True)) - -        if not cli.args.quiet: -            cli.log.info('Wrote keymap to %s.', cli.args.output) - +        keymap_lines = [json.dumps(keymap_json, cls=InfoJSONEncoder, sort_keys=True)]      else: -        print(json.dumps(keymap_json)) +        keymap_lines = [json.dumps(keymap_json)] + +    dump_lines(cli.args.output, keymap_lines, cli.args.quiet) diff --git a/lib/python/qmk/cli/ci/validate_aliases.py b/lib/python/qmk/cli/ci/validate_aliases.py index a205d03cff..7f781d4397 100644 --- a/lib/python/qmk/cli/ci/validate_aliases.py +++ b/lib/python/qmk/cli/ci/validate_aliases.py @@ -1,11 +1,8 @@  """Validates the list of keyboard aliases.  """ -from pathlib import Path -  from milc import cli -from qmk.json_schema import json_load -from qmk.keyboard import resolve_keyboard, keyboard_folder +from qmk.keyboard import resolve_keyboard, keyboard_folder, keyboard_alias_definitions  def _safe_keyboard_folder(target): @@ -34,7 +31,7 @@ def _target_keyboard_exists(target):  @cli.subcommand('Validates the list of keyboard aliases.', hidden=True)  def ci_validate_aliases(cli): -    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) +    aliases = keyboard_alias_definitions()      success = True      for alias in aliases.keys(): diff --git a/lib/python/qmk/cli/clean.py b/lib/python/qmk/cli/clean.py index 72b7ffe810..bdec01e4b6 100644 --- a/lib/python/qmk/cli/clean.py +++ b/lib/python/qmk/cli/clean.py @@ -2,7 +2,7 @@  """  from subprocess import DEVNULL -from qmk.commands import create_make_target +from qmk.commands import find_make  from milc import cli @@ -11,4 +11,4 @@ from milc import cli  def clean(cli):      """Runs `make clean` (or `make distclean` if --all is passed)      """ -    cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL) +    cli.run([find_make(), 'distclean' if cli.args.all else 'clean'], capture_output=False, stdin=DEVNULL) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 2478ea1fed..3c8f3664ea 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -7,22 +7,11 @@ from argcomplete.completers import FilesCompleter  from milc import cli  import qmk.path -from qmk.constants import QMK_FIRMWARE  from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment +from qmk.commands import build_environment  from qmk.keyboard import keyboard_completer, keyboard_folder_or_all, is_all_keyboards  from qmk.keymap import keymap_completer, locate_keymap -from qmk.cli.generate.compilation_database import write_compilation_database - - -def _is_keymap_target(keyboard, keymap): -    if keymap == 'all': -        return True - -    if locate_keymap(keyboard, keymap): -        return True - -    return False +from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget  @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile') @@ -32,6 +21,7 @@ def _is_keymap_target(keyboard, keymap):  @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")  @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")  @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") [email protected]('-t', '--target', type=str, default=None, help="Intended alternative build target, such as `production` in `make planck/rev4:default:production`.")  @cli.argument('--compiledb', arg_only=True, action='store_true', help="Generates the clang compile_commands.json file during build. Implies --clean.")  @cli.subcommand('Compile a QMK Firmware.')  @automagic_keyboard @@ -47,53 +37,35 @@ def compile(cli):          from .mass_compile import mass_compile          cli.args.builds = []          cli.args.filter = [] -        cli.args.no_temp = False +        cli.config.mass_compile.keymap = cli.config.compile.keymap +        cli.config.mass_compile.parallel = cli.config.compile.parallel +        cli.config.mass_compile.no_temp = False          return mass_compile(cli)      # Build the environment vars      envs = build_environment(cli.args.env) -    # Determine the compile command -    commands = [] - -    current_keyboard = None -    current_keymap = None +    # Handler for the build target +    target = None      if cli.args.filename: -        # If a configurator JSON was provided generate a keymap and compile it -        user_keymap = parse_configurator_json(cli.args.filename) -        commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, clean=cli.args.clean, **envs)] +        # if we were given a filename, assume we have a json build target +        target = JsonKeymapBuildTarget(cli.args.filename)      elif cli.config.compile.keyboard and cli.config.compile.keymap: -        # Generate the make command for a specific keyboard/keymap. -        if not _is_keymap_target(cli.config.compile.keyboard, cli.config.compile.keymap): +        # if we got a keyboard and keymap, attempt to find it +        if not locate_keymap(cli.config.compile.keyboard, cli.config.compile.keymap):              cli.log.error('Invalid keymap argument.')              cli.print_help()              return False -        if cli.args.clean: -            commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean', **envs)) -        commands.append(create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs)) +        # If we got here, then we have a valid keyboard and keymap for a build target +        target = KeyboardKeymapBuildTarget(cli.config.compile.keyboard, cli.config.compile.keymap) -        current_keyboard = cli.config.compile.keyboard -        current_keymap = cli.config.compile.keymap - -    if not commands: +    if not target:          cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')          cli.print_help()          return False -    if cli.args.compiledb: -        if current_keyboard is None or current_keymap is None: -            cli.log.error('You must supply both `--keyboard` and `--keymap` or be in a directory with a keymap to generate a compile_commands.json file.') -            cli.print_help() -            return False -        write_compilation_database(current_keyboard, current_keymap, QMK_FIRMWARE / 'compile_commands.json') - -    cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1])) -    if not cli.args.dry_run: -        cli.echo('\n') -        for command in commands: -            ret = cli.run(command, capture_output=False) -            if ret.returncode: -                return ret.returncode +    target.configure(parallel=cli.config.compile.parallel, clean=cli.args.clean, compiledb=cli.args.compiledb) +    target.compile(cli.args.target, dry_run=cli.args.dry_run, **envs) diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py index 6a6feb87d1..dd8b58b2c7 100755 --- a/lib/python/qmk/cli/doctor/main.py +++ b/lib/python/qmk/cli/doctor/main.py @@ -9,10 +9,11 @@ from milc import cli  from milc.questions import yesno  from qmk import submodules -from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM +from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM, QMK_USERSPACE, HAS_QMK_USERSPACE  from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules  from qmk.git import git_check_repo, git_get_branch, git_get_tag, git_get_last_log_entry, git_get_common_ancestor, git_is_dirty, git_get_remotes, git_check_deviation  from qmk.commands import in_virtualenv +from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate, UserspaceValidationError  def os_tests(): @@ -92,6 +93,25 @@ def output_submodule_status():                  cli.log.error(f'- {sub_name}: <<< missing or unknown >>>') +def userspace_tests(qmk_firmware): +    if qmk_firmware: +        cli.log.info(f'QMK home: {{fg_cyan}}{qmk_firmware}') + +    for path in qmk_userspace_paths(): +        try: +            qmk_userspace_validate(path) +            cli.log.info(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`') +        except FileNotFoundError: +            cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Missing `qmk.json`') +        except UserspaceValidationError as err: +            cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`') +            cli.log.warn(f' -- {{fg_cyan}}{path}/qmk.json{{fg_reset}} validation error: {err}') + +    if QMK_USERSPACE is not None: +        cli.log.info(f'QMK userspace: {{fg_cyan}}{QMK_USERSPACE}') +    cli.log.info(f'Userspace enabled: {{fg_cyan}}{HAS_QMK_USERSPACE}') + +  @cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.')  @cli.argument('-n', '--no', action='store_true', arg_only=True, help='Answer no to all questions.')  @cli.subcommand('Basic QMK environment checks') @@ -108,6 +128,9 @@ def doctor(cli):      cli.log.info('QMK home: {fg_cyan}%s', QMK_FIRMWARE)      status = os_status = os_tests() + +    userspace_tests(None) +      git_status = git_tests()      if git_status == CheckStatus.ERROR or (os_status == CheckStatus.OK and git_status == CheckStatus.WARNING): diff --git a/lib/python/qmk/cli/find.py b/lib/python/qmk/cli/find.py index f2135bbc16..55a0530092 100644 --- a/lib/python/qmk/cli/find.py +++ b/lib/python/qmk/cli/find.py @@ -19,13 +19,9 @@ from qmk.search import search_keymap_targets  def find(cli):      """Search through all keyboards and keymaps for a given search criteria.      """ +    targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter) +    for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)): +        print(f'{target}') -    if len(cli.args.filter) == 0 and len(cli.args.print) > 0: -        cli.log.warning('No filters supplied -- keymaps not parsed, unable to print requested values.') - -    targets = search_keymap_targets([('all', cli.config.find.keymap)], cli.args.filter, cli.args.print) -    for keyboard, keymap, print_vals in targets: -        print(f'{keyboard}:{keymap}') - -        for key, val in print_vals: -            print(f'    {key}={val}') +        for key in cli.args.print: +            print(f'    {key}={target.dotty.get(key, None)}') diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index 8724f26889..5df31e4781 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -4,25 +4,17 @@ You can compile a keymap already in the repo or using a QMK Configurator export.  A bootloader must be specified.  """  from argcomplete.completers import FilesCompleter +from pathlib import Path  from milc import cli  import qmk.path  from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json, build_environment +from qmk.commands import build_environment  from qmk.keyboard import keyboard_completer, keyboard_folder  from qmk.keymap import keymap_completer, locate_keymap  from qmk.flashers import flasher - - -def _is_keymap_target(keyboard, keymap): -    if keymap == 'all': -        return True - -    if locate_keymap(keyboard, keymap): -        return True - -    return False +from qmk.build_targets import KeyboardKeymapBuildTarget, JsonKeymapBuildTarget  def _list_bootloaders(): @@ -89,7 +81,7 @@ def flash(cli):      If bootloader is omitted the make system will use the configured bootloader for that keyboard.      """ -    if cli.args.filename and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']: +    if cli.args.filename and isinstance(cli.args.filename, Path) and cli.args.filename.suffix in ['.bin', '.hex', '.uf2']:          return _flash_binary(cli.args.filename, cli.args.mcu)      if cli.args.bootloaders: @@ -98,34 +90,27 @@ def flash(cli):      # Build the environment vars      envs = build_environment(cli.args.env) -    # Determine the compile command -    commands = [] +    # Handler for the build target +    target = None      if cli.args.filename: -        # If a configurator JSON was provided generate a keymap and compile it -        user_keymap = parse_configurator_json(cli.args.filename) -        commands = [compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, clean=cli.args.clean, **envs)] +        # if we were given a filename, assume we have a json build target +        target = JsonKeymapBuildTarget(cli.args.filename)      elif cli.config.flash.keyboard and cli.config.flash.keymap: -        # Generate the make command for a specific keyboard/keymap. -        if not _is_keymap_target(cli.config.flash.keyboard, cli.config.flash.keymap): +        # if we got a keyboard and keymap, attempt to find it +        if not locate_keymap(cli.config.flash.keyboard, cli.config.flash.keymap):              cli.log.error('Invalid keymap argument.')              cli.print_help()              return False -        if cli.args.clean: -            commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean', **envs)) -        commands.append(create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)) +        # If we got here, then we have a valid keyboard and keymap for a build target +        target = KeyboardKeymapBuildTarget(cli.config.flash.keyboard, cli.config.flash.keymap) -    if not commands: +    if not target:          cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.')          cli.print_help()          return False -    cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(commands[-1])) -    if not cli.args.dry_run: -        cli.echo('\n') -        for command in commands: -            ret = cli.run(command, capture_output=False) -            if ret.returncode: -                return ret.returncode +    target.configure(parallel=cli.config.flash.parallel, clean=cli.args.clean) +    target.compile(cli.args.bootloader, dry_run=cli.args.dry_run, **envs) diff --git a/lib/python/qmk/cli/format/json.py b/lib/python/qmk/cli/format/json.py index 3299a0d807..283513254c 100755 --- a/lib/python/qmk/cli/format/json.py +++ b/lib/python/qmk/cli/format/json.py @@ -9,48 +9,74 @@ from milc import cli  from qmk.info import info_json  from qmk.json_schema import json_load, validate -from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder +from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder, UserspaceJSONEncoder  from qmk.path import normpath [email protected]('json_file', arg_only=True, type=normpath, help='JSON file to format') [email protected]('-f', '--format', choices=['auto', 'keyboard', 'keymap'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)') [email protected]('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file') [email protected]('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ') [email protected]('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True) -def format_json(cli): -    """Format a json file. +def _detect_json_format(file, json_data): +    """Detect the format of a json file.      """ -    json_file = json_load(cli.args.json_file) - -    if cli.args.format == 'auto': +    json_encoder = None +    try: +        validate(json_data, 'qmk.user_repo.v1') +        json_encoder = UserspaceJSONEncoder +    except ValidationError: +        pass + +    if json_encoder is None:          try: -            validate(json_file, 'qmk.keyboard.v1') +            validate(json_data, 'qmk.keyboard.v1')              json_encoder = InfoJSONEncoder -          except ValidationError as e: -            cli.log.warning('File %s did not validate as a keyboard:\n\t%s', cli.args.json_file, e) -            cli.log.info('Treating %s as a keymap file.', cli.args.json_file) +            cli.log.warning('File %s did not validate as a keyboard info.json or userspace qmk.json:\n\t%s', file, e) +            cli.log.info('Treating %s as a keymap file.', file)              json_encoder = KeymapJSONEncoder + +    return json_encoder + + +def _get_json_encoder(file, json_data): +    """Get the json encoder for a file. +    """ +    json_encoder = None +    if cli.args.format == 'auto': +        json_encoder = _detect_json_format(file, json_data)      elif cli.args.format == 'keyboard':          json_encoder = InfoJSONEncoder      elif cli.args.format == 'keymap':          json_encoder = KeymapJSONEncoder +    elif cli.args.format == 'userspace': +        json_encoder = UserspaceJSONEncoder      else:          # This should be impossible          cli.log.error('Unknown format: %s', cli.args.format) +    return json_encoder + + [email protected]('json_file', arg_only=True, type=normpath, help='JSON file to format') [email protected]('-f', '--format', choices=['auto', 'keyboard', 'keymap', 'userspace'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)') [email protected]('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file') [email protected]('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ') [email protected]('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True) +def format_json(cli): +    """Format a json file. +    """ +    json_data = json_load(cli.args.json_file) + +    json_encoder = _get_json_encoder(cli.args.json_file, json_data) +    if json_encoder is None:          return False -    if json_encoder == KeymapJSONEncoder and 'layout' in json_file: +    if json_encoder == KeymapJSONEncoder and 'layout' in json_data:          # Attempt to format the keycodes. -        layout = json_file['layout'] -        info_data = info_json(json_file['keyboard']) +        layout = json_data['layout'] +        info_data = info_json(json_data['keyboard'])          if layout in info_data.get('layout_aliases', {}): -            layout = json_file['layout'] = info_data['layout_aliases'][layout] +            layout = json_data['layout'] = info_data['layout_aliases'][layout]          if layout in info_data.get('layouts'): -            for layer_num, layer in enumerate(json_file['layers']): +            for layer_num, layer in enumerate(json_data['layers']):                  current_layer = []                  last_row = 0 @@ -61,9 +87,9 @@ def format_json(cli):                      current_layer.append(keymap_key) -                json_file['layers'][layer_num] = current_layer +                json_data['layers'][layer_num] = current_layer -    output = json.dumps(json_file, cls=json_encoder, sort_keys=True) +    output = json.dumps(json_data, cls=json_encoder, sort_keys=True)      if cli.args.inplace:          with open(cli.args.json_file, 'w+', encoding='utf-8') as outfile: diff --git a/lib/python/qmk/cli/generate/api.py b/lib/python/qmk/cli/generate/api.py index 61a2f9f732..1948b483a1 100755 --- a/lib/python/qmk/cli/generate/api.py +++ b/lib/python/qmk/cli/generate/api.py @@ -10,7 +10,7 @@ from qmk.datetime import current_datetime  from qmk.info import info_json  from qmk.json_schema import json_load  from qmk.keymap import list_keymaps -from qmk.keyboard import find_readme, list_keyboards +from qmk.keyboard import find_readme, list_keyboards, keyboard_alias_definitions  from qmk.keycodes import load_spec, list_versions, list_languages  DATA_PATH = Path('data') @@ -166,7 +166,7 @@ def generate_api(cli):      # Generate data for the global files      keyboard_list = sorted(kb_all) -    keyboard_aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) +    keyboard_aliases = keyboard_alias_definitions()      keyboard_metadata = {          'last_updated': current_datetime(),          'keyboards': keyboard_list, diff --git a/lib/python/qmk/cli/generate/compilation_database.py b/lib/python/qmk/cli/generate/compilation_database.py index 9e5c266516..5100d2b6d2 100755 --- a/lib/python/qmk/cli/generate/compilation_database.py +++ b/lib/python/qmk/cli/generate/compilation_database.py @@ -12,7 +12,7 @@ from typing import Dict, Iterator, List, Union  from milc import cli, MILC -from qmk.commands import create_make_command +from qmk.commands import find_make  from qmk.constants import QMK_FIRMWARE  from qmk.decorators import automagic_keyboard, automagic_keymap  from qmk.keyboard import keyboard_completer, keyboard_folder @@ -76,9 +76,12 @@ def parse_make_n(f: Iterator[str]) -> List[Dict[str, str]]:      return records -def write_compilation_database(keyboard: str, keymap: str, output_path: Path) -> bool: +def write_compilation_database(keyboard: str = None, keymap: str = None, output_path: Path = QMK_FIRMWARE / 'compile_commands.json', skip_clean: bool = False, command: List[str] = None, **env_vars) -> bool:      # Generate the make command for a specific keyboard/keymap. -    command = create_make_command(keyboard, keymap, dry_run=True) +    if not command: +        from qmk.build_targets import KeyboardKeymapBuildTarget  # Lazy load due to circular references +        target = KeyboardKeymapBuildTarget(keyboard, keymap) +        command = target.compile_command(dry_run=True, **env_vars)      if not command:          cli.log.error('You must supply both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') @@ -90,9 +93,10 @@ def write_compilation_database(keyboard: str, keymap: str, output_path: Path) ->      env.pop("MAKEFLAGS", None)      # re-use same executable as the main make invocation (might be gmake) -    clean_command = [command[0], 'clean'] -    cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command)) -    cli.run(clean_command, capture_output=False, check=True, env=env) +    if not skip_clean: +        clean_command = [find_make(), "clean"] +        cli.log.info('Making clean with {fg_cyan}%s', ' '.join(clean_command)) +        cli.run(clean_command, capture_output=False, check=True, env=env)      cli.log.info('Gathering build instructions from {fg_cyan}%s', ' '.join(command)) diff --git a/lib/python/qmk/cli/generate/config_h.py b/lib/python/qmk/cli/generate/config_h.py index 64d4db6ffe..00fb1d9585 100755 --- a/lib/python/qmk/cli/generate/config_h.py +++ b/lib/python/qmk/cli/generate/config_h.py @@ -15,7 +15,12 @@ from qmk.constants import GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE  def generate_define(define, value=None): +    is_keymap = cli.args.filename      value = f' {value}' if value is not None else '' +    if is_keymap: +        return f""" +#undef {define} +#define {define}{value}"""      return f"""  #ifndef {define}  #    define {define}{value} @@ -67,6 +72,19 @@ def generate_matrix_size(kb_info_json, config_h_lines):          config_h_lines.append(generate_define('MATRIX_ROWS', kb_info_json['matrix_size']['rows'])) +def generate_matrix_masked(kb_info_json, config_h_lines): +    """"Enable matrix mask if required""" +    mask_required = False + +    if 'matrix_grid' in kb_info_json.get('dip_switch', {}): +        mask_required = True +    if 'matrix_grid' in kb_info_json.get('split', {}).get('handedness', {}): +        mask_required = True + +    if mask_required: +        config_h_lines.append(generate_define('MATRIX_MASKED')) + +  def generate_config_items(kb_info_json, config_h_lines):      """Iterate through the info_config map to generate basic config values.      """ @@ -75,9 +93,9 @@ def generate_config_items(kb_info_json, config_h_lines):      for config_key, info_dict in info_config_map.items():          info_key = info_dict['info_key']          key_type = info_dict.get('value_type', 'raw') -        to_config = info_dict.get('to_config', True) +        to_c = info_dict.get('to_c', True) -        if not to_config: +        if not to_c:              continue          try: @@ -130,23 +148,11 @@ def generate_encoder_config(encoder_json, config_h_lines, postfix=''):  def generate_split_config(kb_info_json, config_h_lines):      """Generate the config.h lines for split boards.""" -    if 'primary' in kb_info_json['split']: -        if kb_info_json['split']['primary'] in ('left', 'right'): -            config_h_lines.append('') -            config_h_lines.append('#ifndef MASTER_LEFT') -            config_h_lines.append('#    ifndef MASTER_RIGHT') -            if kb_info_json['split']['primary'] == 'left': -                config_h_lines.append('#        define MASTER_LEFT') -            elif kb_info_json['split']['primary'] == 'right': -                config_h_lines.append('#        define MASTER_RIGHT') -            config_h_lines.append('#    endif // MASTER_RIGHT') -            config_h_lines.append('#endif // MASTER_LEFT') -        elif kb_info_json['split']['primary'] == 'pin': -            config_h_lines.append(generate_define('SPLIT_HAND_PIN')) -        elif kb_info_json['split']['primary'] == 'matrix_grid': -            config_h_lines.append(generate_define('SPLIT_HAND_MATRIX_GRID', f'{{ {",".join(kb_info_json["split"]["matrix_grid"])} }}')) -        elif kb_info_json['split']['primary'] == 'eeprom': -            config_h_lines.append(generate_define('EE_HANDS')) +    if 'handedness' in kb_info_json['split']: +        # TODO: change SPLIT_HAND_MATRIX_GRID to require brackets +        handedness = kb_info_json['split']['handedness'] +        if 'matrix_grid' in handedness: +            config_h_lines.append(generate_define('SPLIT_HAND_MATRIX_GRID', ', '.join(handedness['matrix_grid'])))      if 'protocol' in kb_info_json['split'].get('transport', {}):          if kb_info_json['split']['transport']['protocol'] == 'i2c': @@ -159,10 +165,13 @@ def generate_split_config(kb_info_json, config_h_lines):          generate_encoder_config(kb_info_json['split']['encoder']['right'], config_h_lines, '_RIGHT') -def generate_led_animations_config(led_feature_json, config_h_lines, prefix): +def generate_led_animations_config(feature, led_feature_json, config_h_lines, enable_prefix, animation_prefix): +    if 'animation' in led_feature_json.get('default', {}): +        config_h_lines.append(generate_define(f'{feature.upper()}_DEFAULT_MODE', f'{animation_prefix}{led_feature_json["default"]["animation"].upper()}')) +      for animation in led_feature_json.get('animations', {}):          if led_feature_json['animations'][animation]: -            config_h_lines.append(generate_define(f'{prefix}{animation.upper()}')) +            config_h_lines.append(generate_define(f'{enable_prefix}{animation.upper()}'))  @cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON to be compiled and flashed or a pre-compiled binary firmware file (bin/hex) to be flashed.') @@ -191,6 +200,8 @@ def generate_config_h(cli):      generate_matrix_size(kb_info_json, config_h_lines) +    generate_matrix_masked(kb_info_json, config_h_lines) +      if 'matrix_pins' in kb_info_json:          config_h_lines.append(matrix_pins(kb_info_json['matrix_pins'])) @@ -201,13 +212,13 @@ def generate_config_h(cli):          generate_split_config(kb_info_json, config_h_lines)      if 'led_matrix' in kb_info_json: -        generate_led_animations_config(kb_info_json['led_matrix'], config_h_lines, 'ENABLE_LED_MATRIX_') +        generate_led_animations_config('led_matrix', kb_info_json['led_matrix'], config_h_lines, 'ENABLE_LED_MATRIX_', 'LED_MATRIX_')      if 'rgb_matrix' in kb_info_json: -        generate_led_animations_config(kb_info_json['rgb_matrix'], config_h_lines, 'ENABLE_RGB_MATRIX_') +        generate_led_animations_config('rgb_matrix', kb_info_json['rgb_matrix'], config_h_lines, 'ENABLE_RGB_MATRIX_', 'RGB_MATRIX_')      if 'rgblight' in kb_info_json: -        generate_led_animations_config(kb_info_json['rgblight'], config_h_lines, 'RGBLIGHT_EFFECT_') +        generate_led_animations_config('rgblight', kb_info_json['rgblight'], config_h_lines, 'RGBLIGHT_EFFECT_', 'RGBLIGHT_MODE_')      # Show the results      dump_lines(cli.args.output, config_h_lines, cli.args.quiet) diff --git a/lib/python/qmk/cli/generate/keyboard_c.py b/lib/python/qmk/cli/generate/keyboard_c.py index 9004b41abb..f8a2372cf3 100755 --- a/lib/python/qmk/cli/generate/keyboard_c.py +++ b/lib/python/qmk/cli/generate/keyboard_c.py @@ -57,6 +57,32 @@ def _gen_led_config(info_data):      return lines +def _gen_matrix_mask(info_data): +    """Convert info.json content to matrix_mask +    """ +    cols = info_data['matrix_size']['cols'] +    rows = info_data['matrix_size']['rows'] + +    # Default mask to everything disabled +    mask = [['0'] * cols for i in range(rows)] + +    # Mirror layout macros squashed on top of each other +    for layout_data in info_data['layouts'].values(): +        for key_data in layout_data['layout']: +            row, col = key_data['matrix'] +            mask[row][col] = '1' + +    lines = [] +    lines.append('#ifdef MATRIX_MASKED') +    lines.append('__attribute__((weak)) const matrix_row_t matrix_mask[] = {') +    for i in range(rows): +        lines.append(f'    0b{"".join(reversed(mask[i]))},') +    lines.append('};') +    lines.append('#endif') + +    return lines + +  @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')  @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")  @cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate keyboard.c for.') @@ -70,6 +96,7 @@ def generate_keyboard_c(cli):      keyboard_h_lines = [GPL2_HEADER_C_LIKE, GENERATED_HEADER_C_LIKE, '#include QMK_KEYBOARD_H', '']      keyboard_h_lines.extend(_gen_led_config(kb_info_json)) +    keyboard_h_lines.extend(_gen_matrix_mask(kb_info_json))      # Show the results      dump_lines(cli.args.output, keyboard_h_lines, cli.args.quiet) diff --git a/lib/python/qmk/cli/generate/keycodes.py b/lib/python/qmk/cli/generate/keycodes.py index ed8b6827bd..719fced5d5 100644 --- a/lib/python/qmk/cli/generate/keycodes.py +++ b/lib/python/qmk/cli/generate/keycodes.py @@ -94,6 +94,14 @@ def _generate_helpers(lines, keycodes):          hi = keycodes["keycodes"][f'0x{codes[1]:04X}']['key']          lines.append(f'#define IS_{ _translate_group(group).upper() }_KEYCODE(code) ((code) >= {lo} && (code) <= {hi})') +    lines.append('') +    lines.append('// Switch statement Helpers') +    for group, codes in temp.items(): +        lo = keycodes["keycodes"][f'0x{codes[0]:04X}']['key'] +        hi = keycodes["keycodes"][f'0x{codes[1]:04X}']['key'] +        name = f'{ _translate_group(group).upper() }_KEYCODE_RANGE' +        lines.append(f'#define { name.ljust(35) } {lo} ... {hi}') +  def _generate_aliases(lines, keycodes):      # Work around ChibiOS ch.h include guard diff --git a/lib/python/qmk/cli/generate/rules_mk.py b/lib/python/qmk/cli/generate/rules_mk.py index fc272da6c6..5291556109 100755 --- a/lib/python/qmk/cli/generate/rules_mk.py +++ b/lib/python/qmk/cli/generate/rules_mk.py @@ -14,6 +14,12 @@ from qmk.path import normpath, FileType  from qmk.constants import GPL2_HEADER_SH_LIKE, GENERATED_HEADER_SH_LIKE +def generate_rule(rules_key, rules_value): +    is_keymap = cli.args.filename +    rule_assignment_operator = '=' if is_keymap else '?=' +    return f'{rules_key} {rule_assignment_operator} {rules_value}' + +  def process_mapping_rule(kb_info_json, rules_key, info_dict):      """Return the rules.mk line(s) for a mapping rule.      """ @@ -29,15 +35,15 @@ def process_mapping_rule(kb_info_json, rules_key, info_dict):          return None      if key_type in ['array', 'list']: -        return f'{rules_key} ?= {" ".join(rules_value)}' +        return generate_rule(rules_key, " ".join(rules_value))      elif key_type == 'bool': -        return f'{rules_key} ?= {"yes" if rules_value else "no"}' +        return generate_rule(rules_key, "yes" if rules_value else "no")      elif key_type == 'mapping': -        return '\n'.join([f'{key} ?= {value}' for key, value in rules_value.items()]) +        return '\n'.join([generate_rule(key, value) for key, value in rules_value.items()])      elif key_type == 'str': -        return f'{rules_key} ?= "{rules_value}"' +        return generate_rule(rules_key, f'"{rules_value}"') -    return f'{rules_key} ?= {rules_value}' +    return generate_rule(rules_key, rules_value)  @cli.argument('filename', nargs='?', arg_only=True, type=FileType('r'), completer=FilesCompleter('.json'), help='A configurator export JSON to be compiled and flashed or a pre-compiled binary firmware file (bin/hex) to be flashed.') @@ -77,21 +83,21 @@ def generate_rules_mk(cli):          for feature, enabled in kb_info_json['features'].items():              feature = feature.upper()              enabled = 'yes' if enabled else 'no' -            rules_mk_lines.append(f'{feature}_ENABLE ?= {enabled}') +            rules_mk_lines.append(generate_rule(f'{feature}_ENABLE', enabled))      # Set SPLIT_TRANSPORT, if needed      if kb_info_json.get('split', {}).get('transport', {}).get('protocol') == 'custom': -        rules_mk_lines.append('SPLIT_TRANSPORT ?= custom') +        rules_mk_lines.append(generate_rule('SPLIT_TRANSPORT', 'custom'))      # Set CUSTOM_MATRIX, if needed      if kb_info_json.get('matrix_pins', {}).get('custom'):          if kb_info_json.get('matrix_pins', {}).get('custom_lite'): -            rules_mk_lines.append('CUSTOM_MATRIX ?= lite') +            rules_mk_lines.append(generate_rule('CUSTOM_MATRIX', 'lite'))          else: -            rules_mk_lines.append('CUSTOM_MATRIX ?= yes') +            rules_mk_lines.append(generate_rule('CUSTOM_MATRIX', 'yes'))      if converter: -        rules_mk_lines.append(f'CONVERT_TO ?= {converter}') +        rules_mk_lines.append(generate_rule('CONVERT_TO', converter))      # Show the results      dump_lines(cli.args.output, rules_mk_lines) diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py index 06e6e411a7..b025f85701 100755 --- a/lib/python/qmk/cli/mass_compile.py +++ b/lib/python/qmk/cli/mass_compile.py @@ -3,26 +3,28 @@  This will compile everything in parallel, for testing purposes.  """  import os +from typing import List  from pathlib import Path  from subprocess import DEVNULL  from milc import cli  from qmk.constants import QMK_FIRMWARE -from qmk.commands import _find_make, get_make_parallel_args +from qmk.commands import find_make, get_make_parallel_args, build_environment  from qmk.search import search_keymap_targets, search_make_targets +from qmk.build_targets import BuildTarget, JsonKeymapBuildTarget -def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env): +def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, no_temp: bool, parallel: int, **env):      if len(targets) == 0:          return -    make_cmd = _find_make() +    make_cmd = find_make()      builddir = Path(QMK_FIRMWARE) / '.build'      makefile = builddir / 'parallel_kb_builds.mk'      if dry_run:          cli.log.info('Compilation targets:') -        for target in sorted(targets): +        for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):              cli.log.info(f"{{fg_cyan}}qmk compile -kb {target[0]} -km {target[1]}{{fg_reset}}")      else:          if clean: @@ -30,9 +32,13 @@ def mass_compile_targets(targets, clean, dry_run, no_temp, parallel, env):          builddir.mkdir(parents=True, exist_ok=True)          with open(makefile, "w") as f: -            for target in sorted(targets): -                keyboard_name = target[0] -                keymap_name = target[1] +            for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)): +                keyboard_name = target.keyboard +                keymap_name = target.keymap +                target.configure(parallel=1)  # We ignore parallelism on a per-build basis as we defer to the parent make invocation +                target.prepare_build(**env)  # If we've got json targets, allow them to write out any extra info to .build before we kick off `make` +                command = target.compile_command(**env) +                command[0] = '+@$(MAKE)'  # Override the make so that we can use jobserver to handle parallelism                  keyboard_safe = keyboard_name.replace('/', '_')                  build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"                  failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}" @@ -43,7 +49,7 @@ all: {keyboard_safe}_{keymap_name}_binary  {keyboard_safe}_{keymap_name}_binary:  	@rm -f "{build_log}" || true  	@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}" -	+@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/builddefs/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="{keymap_name}" COLOR=true SILENT=false {' '.join(env)} \\ +	{' '.join(command)} \\  		>>"{build_log}" 2>&1 \\  		|| cp "{build_log}" "{failed_log}"  	@{{ grep '\[ERRORS\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\ @@ -66,7 +72,7 @@ all: {keyboard_safe}_{keymap_name}_binary                      # yapf: enable                  f.write('\n') -        cli.run([make_cmd, *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) +        cli.run([find_make(), *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)          # Check for failures          failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')] @@ -95,8 +101,11 @@ def mass_compile(cli):      """Compile QMK Firmware against all keyboards.      """      if len(cli.args.builds) > 0: -        targets = search_make_targets(cli.args.builds, cli.args.filter) +        json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)]) +        make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds)) +        targets = search_make_targets(make_like_targets) +        targets.extend([JsonKeymapBuildTarget(e) for e in json_like_targets])      else:          targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter) -    return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, cli.args.env) +    return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.config.mass_compile.no_temp, cli.config.mass_compile.parallel, **build_environment(cli.args.env)) diff --git a/lib/python/qmk/cli/new/keymap.py b/lib/python/qmk/cli/new/keymap.py index 9b0ac221a4..d4339bc9ef 100755 --- a/lib/python/qmk/cli/new/keymap.py +++ b/lib/python/qmk/cli/new/keymap.py @@ -5,10 +5,12 @@ import shutil  from milc import cli  from milc.questions import question +from qmk.constants import HAS_QMK_USERSPACE, QMK_USERSPACE  from qmk.path import is_keyboard, keymaps, keymap  from qmk.git import git_get_username  from qmk.decorators import automagic_keyboard, automagic_keymap  from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.userspace import UserspaceDefs  def prompt_keyboard(): @@ -68,3 +70,9 @@ def new_keymap(cli):      # end message to user      cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}.{{fg_reset}}')      cli.log.info(f"Compile a firmware with your new keymap by typing: {{fg_yellow}}qmk compile -kb {kb_name} -km {user_name}{{fg_reset}}.") + +    # Add to userspace compile if we have userspace available +    if HAS_QMK_USERSPACE: +        userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') +        userspace.add_target(keyboard=kb_name, keymap=user_name, do_print=False) +        return userspace.save() diff --git a/lib/python/qmk/cli/userspace/__init__.py b/lib/python/qmk/cli/userspace/__init__.py new file mode 100644 index 0000000000..5757d3a4c9 --- /dev/null +++ b/lib/python/qmk/cli/userspace/__init__.py @@ -0,0 +1,5 @@ +from . import doctor +from . import add +from . import remove +from . import list +from . import compile diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py new file mode 100644 index 0000000000..8993d54dba --- /dev/null +++ b/lib/python/qmk/cli/userspace/add.py @@ -0,0 +1,51 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from pathlib import Path +from milc import cli + +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.keyboard import keyboard_completer, keyboard_folder_or_all +from qmk.keymap import keymap_completer, is_keymap_target +from qmk.userspace import UserspaceDefs + + [email protected]('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.") [email protected]('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') [email protected]('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') [email protected]('Adds a build target to userspace `qmk.json`.') +def userspace_add(cli): +    if not HAS_QMK_USERSPACE: +        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') +        return False + +    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') + +    if len(cli.args.builds) > 0: +        json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)]) +        make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds)) + +        for e in json_like_targets: +            userspace.add_target(json_path=e) + +        for e in make_like_targets: +            s = e.split(':') +            userspace.add_target(keyboard=s[0], keymap=s[1]) + +    else: +        failed = False +        try: +            if not is_keymap_target(cli.args.keyboard, cli.args.keymap): +                failed = True +        except KeyError: +            failed = True + +        if failed: +            from qmk.cli.new.keymap import new_keymap +            cli.config.new_keymap.keyboard = cli.args.keyboard +            cli.config.new_keymap.keymap = cli.args.keymap +            if new_keymap(cli) is not False: +                userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap) +        else: +            userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap) + +    return userspace.save() diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py new file mode 100644 index 0000000000..0a42dd5bf5 --- /dev/null +++ b/lib/python/qmk/cli/userspace/compile.py @@ -0,0 +1,38 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from pathlib import Path +from milc import cli + +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.commands import build_environment +from qmk.userspace import UserspaceDefs +from qmk.build_targets import JsonKeymapBuildTarget +from qmk.search import search_keymap_targets +from qmk.cli.mass_compile import mass_compile_targets + + [email protected]('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.") [email protected]('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") [email protected]('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") [email protected]('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.") [email protected]('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") [email protected]('Compiles the build targets specified in userspace `qmk.json`.') +def userspace_compile(cli): +    if not HAS_QMK_USERSPACE: +        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') +        return False + +    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') + +    build_targets = [] +    keyboard_keymap_targets = [] +    for e in userspace.build_targets: +        if isinstance(e, Path): +            build_targets.append(JsonKeymapBuildTarget(e)) +        elif isinstance(e, dict): +            keyboard_keymap_targets.append((e['keyboard'], e['keymap'])) + +    if len(keyboard_keymap_targets) > 0: +        build_targets.extend(search_keymap_targets(keyboard_keymap_targets)) + +    mass_compile_targets(list(set(build_targets)), cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, **build_environment(cli.args.env)) diff --git a/lib/python/qmk/cli/userspace/doctor.py b/lib/python/qmk/cli/userspace/doctor.py new file mode 100644 index 0000000000..2b7e29aa7e --- /dev/null +++ b/lib/python/qmk/cli/userspace/doctor.py @@ -0,0 +1,11 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from milc import cli + +from qmk.constants import QMK_FIRMWARE +from qmk.cli.doctor.main import userspace_tests + + [email protected]('Checks userspace configuration.') +def userspace_doctor(cli): +    userspace_tests(QMK_FIRMWARE) diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py new file mode 100644 index 0000000000..a63f669dd7 --- /dev/null +++ b/lib/python/qmk/cli/userspace/list.py @@ -0,0 +1,51 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from pathlib import Path +from dotty_dict import Dotty +from milc import cli + +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.userspace import UserspaceDefs +from qmk.build_targets import BuildTarget +from qmk.keyboard import is_all_keyboards, keyboard_folder +from qmk.keymap import is_keymap_target +from qmk.search import search_keymap_targets + + [email protected]('-e', '--expand', arg_only=True, action='store_true', help="Expands any use of `all` for either keyboard or keymap.") [email protected]('Lists the build targets specified in userspace `qmk.json`.') +def userspace_list(cli): +    if not HAS_QMK_USERSPACE: +        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') +        return False + +    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') + +    if cli.args.expand: +        build_targets = [] +        for e in userspace.build_targets: +            if isinstance(e, Path): +                build_targets.append(e) +            elif isinstance(e, dict) or isinstance(e, Dotty): +                build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])])) +    else: +        build_targets = userspace.build_targets + +    for e in build_targets: +        if isinstance(e, Path): +            # JSON keymap from userspace +            cli.log.info(f'JSON keymap: {{fg_cyan}}{e}{{fg_reset}}') +            continue +        elif isinstance(e, dict) or isinstance(e, Dotty): +            # keyboard/keymap dict from userspace +            keyboard = e['keyboard'] +            keymap = e['keymap'] +        elif isinstance(e, BuildTarget): +            # BuildTarget from search_keymap_targets() +            keyboard = e.keyboard +            keymap = e.keymap + +        if is_all_keyboards(keyboard) or is_keymap_target(keyboard_folder(keyboard), keymap): +            cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}') +        else: +            cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}} -- not found!') diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py new file mode 100644 index 0000000000..c7d180bfd1 --- /dev/null +++ b/lib/python/qmk/cli/userspace/remove.py @@ -0,0 +1,37 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from pathlib import Path +from milc import cli + +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.keyboard import keyboard_completer, keyboard_folder_or_all +from qmk.keymap import keymap_completer +from qmk.userspace import UserspaceDefs + + [email protected]('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.") [email protected]('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') [email protected]('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') [email protected]('Removes a build target from userspace `qmk.json`.') +def userspace_remove(cli): +    if not HAS_QMK_USERSPACE: +        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.') +        return False + +    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json') + +    if len(cli.args.builds) > 0: +        json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)]) +        make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds)) + +        for e in json_like_targets: +            userspace.remove_target(json_path=e) + +        for e in make_like_targets: +            s = e.split(':') +            userspace.remove_target(keyboard=s[0], keymap=s[1]) + +    else: +        userspace.remove_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap) + +    return userspace.save() diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 34696e3793..d95ff5f923 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -2,18 +2,18 @@  """  import os  import sys -import json  import shutil  from pathlib import Path  from milc import cli  import jsonschema -from qmk.constants import INTERMEDIATE_OUTPUT_PREFIX +from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE  from qmk.json_schema import json_load, validate +from qmk.keyboard import keyboard_alias_definitions -def _find_make(): +def find_make():      """Returns the correct make command for this environment.      """      make_cmd = os.environ.get('MAKE') @@ -24,74 +24,6 @@ def _find_make():      return make_cmd -def create_make_target(target, dry_run=False, parallel=1, **env_vars): -    """Create a make command - -    Args: - -        target -            Usually a make rule, such as 'clean' or 'all'. - -        dry_run -            make -n -- don't actually build - -        parallel -            The number of make jobs to run in parallel - -        **env_vars -            Environment variables to be passed to make. - -    Returns: - -        A command that can be run to make the specified keyboard and keymap -    """ -    env = [] -    make_cmd = _find_make() - -    for key, value in env_vars.items(): -        env.append(f'{key}={value}') - -    if cli.config.general.verbose: -        env.append('VERBOSE=true') - -    return [make_cmd, *(['-n'] if dry_run else []), *get_make_parallel_args(parallel), *env, target] - - -def create_make_command(keyboard, keymap, target=None, dry_run=False, parallel=1, **env_vars): -    """Create a make compile command - -    Args: - -        keyboard -            The path of the keyboard, for example 'plank' - -        keymap -            The name of the keymap, for example 'algernon' - -        target -            Usually a bootloader. - -        dry_run -            make -n -- don't actually build - -        parallel -            The number of make jobs to run in parallel - -        **env_vars -            Environment variables to be passed to make. - -    Returns: - -        A command that can be run to make the specified keyboard and keymap -    """ -    make_args = [keyboard, keymap] - -    if target: -        make_args.append(target) - -    return create_make_target(':'.join(make_args), dry_run=dry_run, parallel=parallel, **env_vars) - -  def get_make_parallel_args(parallel=1):      """Returns the arguments for running the specified number of parallel jobs.      """ @@ -100,7 +32,7 @@ def get_make_parallel_args(parallel=1):      if int(parallel) <= 0:          # 0 or -1 means -j without argument (unlimited jobs)          parallel_args.append('--jobs') -    else: +    elif int(parallel) > 1:          parallel_args.append('--jobs=' + str(parallel))      if int(parallel) != 1: @@ -110,96 +42,6 @@ def get_make_parallel_args(parallel=1):      return parallel_args -def compile_configurator_json(user_keymap, bootloader=None, parallel=1, clean=False, **env_vars): -    """Convert a configurator export JSON file into a C file and then compile it. - -    Args: - -        user_keymap -            A deserialized keymap export - -        bootloader -            A bootloader to flash - -        parallel -            The number of make jobs to run in parallel - -    Returns: - -        A command to run to compile and flash the C file. -    """ -    # In case the user passes a keymap.json from a keymap directory directly to the CLI. -    # e.g.: qmk compile - < keyboards/clueboard/california/keymaps/default/keymap.json -    user_keymap["keymap"] = user_keymap.get("keymap", "default_json") - -    keyboard_filesafe = user_keymap['keyboard'].replace('/', '_') -    target = f'{keyboard_filesafe}_{user_keymap["keymap"]}' -    intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{keyboard_filesafe}_{user_keymap["keymap"]}') -    keymap_dir = intermediate_output / 'src' -    keymap_json = keymap_dir / 'keymap.json' - -    if clean: -        if intermediate_output.exists(): -            shutil.rmtree(intermediate_output) - -    # begin with making the deepest folder in the tree -    keymap_dir.mkdir(exist_ok=True, parents=True) - -    # Compare minified to ensure consistent comparison -    new_content = json.dumps(user_keymap, separators=(',', ':')) -    if keymap_json.exists(): -        old_content = json.dumps(json.loads(keymap_json.read_text(encoding='utf-8')), separators=(',', ':')) -        if old_content == new_content: -            new_content = None - -    # Write the keymap.json file if different -    if new_content: -        keymap_json.write_text(new_content, encoding='utf-8') - -    # Return a command that can be run to make the keymap and flash if given -    verbose = 'true' if cli.config.general.verbose else 'false' -    color = 'true' if cli.config.general.color else 'false' -    make_command = [_find_make()] - -    if not cli.config.general.verbose: -        make_command.append('-s') - -    make_command.extend([ -        *get_make_parallel_args(parallel), -        '-r', -        '-R', -        '-f', -        'builddefs/build_keyboard.mk', -    ]) - -    if bootloader: -        make_command.append(bootloader) - -    make_command.extend([ -        f'KEYBOARD={user_keymap["keyboard"]}', -        f'KEYMAP={user_keymap["keymap"]}', -        f'KEYBOARD_FILESAFE={keyboard_filesafe}', -        f'TARGET={target}', -        f'INTERMEDIATE_OUTPUT={intermediate_output}', -        f'MAIN_KEYMAP_PATH_1={intermediate_output}', -        f'MAIN_KEYMAP_PATH_2={intermediate_output}', -        f'MAIN_KEYMAP_PATH_3={intermediate_output}', -        f'MAIN_KEYMAP_PATH_4={intermediate_output}', -        f'MAIN_KEYMAP_PATH_5={intermediate_output}', -        f'KEYMAP_JSON={keymap_json}', -        f'KEYMAP_PATH={keymap_dir}', -        f'VERBOSE={verbose}', -        f'COLOR={color}', -        'SILENT=false', -        'QMK_BIN="qmk"', -    ]) - -    for key, value in env_vars.items(): -        make_command.append(f'{key}={value}') - -    return make_command - -  def parse_configurator_json(configurator_file):      """Open and parse a configurator json export      """ @@ -213,7 +55,7 @@ def parse_configurator_json(configurator_file):          exit(1)      keyboard = user_keymap['keyboard'] -    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) +    aliases = keyboard_alias_definitions()      while keyboard in aliases:          last_keyboard = keyboard @@ -235,6 +77,10 @@ def build_environment(args):              envs[key] = value          else:              cli.log.warning('Invalid environment variable: %s', env) + +    if HAS_QMK_USERSPACE: +        envs['QMK_USERSPACE'] = Path(QMK_USERSPACE).resolve() +      return envs diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py index 1967441fc8..90e4452f2b 100644 --- a/lib/python/qmk/constants.py +++ b/lib/python/qmk/constants.py @@ -4,9 +4,17 @@ from os import environ  from datetime import date  from pathlib import Path +from qmk.userspace import detect_qmk_userspace +  # The root of the qmk_firmware tree.  QMK_FIRMWARE = Path.cwd() +# The detected userspace tree +QMK_USERSPACE = detect_qmk_userspace() + +# Whether or not we have a separate userspace directory +HAS_QMK_USERSPACE = True if QMK_USERSPACE is not None else False +  # Upstream repo url  QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware' diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 9c8521a2a3..3efd34555c 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -55,6 +55,29 @@ def _get_key_left_position(key):      return key['x'] - 0.25 if key.get('h', 1) == 2 and key.get('w', 1) == 1.25 else key['x'] +def _find_invalid_encoder_index(info_data): +    """Perform additional validation of encoders +    """ +    enc_count = len(info_data.get('encoder', {}).get('rotary', [])) +    enc_count += len(info_data.get('split', {}).get('encoder', {}).get('right', {}).get('rotary', [])) + +    ret = [] +    layouts = info_data.get('layouts', {}) +    for layout_name, layout_data in layouts.items(): +        found = set() +        for key in layout_data['layout']: +            if 'encoder' in key: +                if enc_count == 0: +                    ret.append((layout_name, key['encoder'], 'non-configured')) +                elif key['encoder'] >= enc_count: +                    ret.append((layout_name, key['encoder'], 'out of bounds')) +                elif key['encoder'] in found: +                    ret.append((layout_name, key['encoder'], 'duplicate')) +                found.add(key['encoder']) + +    return ret + +  def _additional_validation(keyboard, info_data):      """Non schema checks      """ @@ -105,6 +128,11 @@ def _additional_validation(keyboard, info_data):              if not decl.get("aliases", []):                  _log_error(info_data, f'Keycode {decl["key"]} has no short form alias') +    # encoder IDs in layouts must be in range and not duplicated +    found = _find_invalid_encoder_index(info_data) +    for layout_name, encoder_index, reason in found: +        _log_error(info_data, f'Layout "{layout_name}" contains {reason} encoder index {encoder_index}.') +  def _validate(keyboard, info_data):      """Perform various validation on the provided info.json data @@ -352,55 +380,12 @@ def _extract_secure_unlock(info_data, config_c):          info_data['secure']['unlock_sequence'] = unlock_array -def _extract_split_main(info_data, config_c): -    """Populate data about the split configuration -    """ -    # Figure out how the main half is determined -    if config_c.get('SPLIT_HAND_PIN') is True: -        if 'split' not in info_data: -            info_data['split'] = {} - -        if 'main' in info_data['split']: -            _log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_PIN) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) - -        info_data['split']['main'] = 'pin' - -    if config_c.get('SPLIT_HAND_MATRIX_GRID'): -        if 'split' not in info_data: -            info_data['split'] = {} - -        if 'main' in info_data['split']: -            _log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_MATRIX_GRID) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) - -        info_data['split']['main'] = 'matrix_grid' -        info_data['split']['matrix_grid'] = _extract_pins(config_c['SPLIT_HAND_MATRIX_GRID']) - -    if config_c.get('EE_HANDS') is True: -        if 'split' not in info_data: -            info_data['split'] = {} - -        if 'main' in info_data['split']: -            _log_warning(info_data, 'Split main hand is specified in both config.h (EE_HANDS) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) - -        info_data['split']['main'] = 'eeprom' - -    if config_c.get('MASTER_RIGHT') is True: -        if 'split' not in info_data: -            info_data['split'] = {} - -        if 'main' in info_data['split']: -            _log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_RIGHT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) - -        info_data['split']['main'] = 'right' - -    if config_c.get('MASTER_LEFT') is True: -        if 'split' not in info_data: -            info_data['split'] = {} - -        if 'main' in info_data['split']: -            _log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_LEFT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) - -        info_data['split']['main'] = 'left' +def _extract_split_handedness(info_data, config_c): +    # Migrate +    split = info_data.get('split', {}) +    if 'matrix_grid' in split: +        split['handedness'] = split.get('handedness', {}) +        split['handedness']['matrix_grid'] = split.pop('matrix_grid')  def _extract_split_transport(info_data, config_c): @@ -428,6 +413,15 @@ def _extract_split_transport(info_data, config_c):          if 'protocol' not in info_data['split']['transport']:              info_data['split']['transport']['protocol'] = 'serial' +    # Migrate +    transport = info_data.get('split', {}).get('transport', {}) +    if 'sync_matrix_state' in transport: +        transport['sync'] = transport.get('sync', {}) +        transport['sync']['matrix_state'] = transport.pop('sync_matrix_state') +    if 'sync_modifiers' in transport: +        transport['sync'] = transport.get('sync', {}) +        transport['sync']['modifiers'] = transport.pop('sync_modifiers') +  def _extract_split_right_pins(info_data, config_c):      # Figure out the right half matrix pins @@ -520,6 +514,8 @@ def _config_to_json(key_type, config_value):              return list(map(str.strip, config_value.split(',')))      elif key_type == 'bool': +        if isinstance(config_value, bool): +            return config_value          return config_value in true_values      elif key_type == 'hex': @@ -583,7 +579,7 @@ def _extract_config_h(info_data, config_c):      _extract_matrix_info(info_data, config_c)      _extract_audio(info_data, config_c)      _extract_secure_unlock(info_data, config_c) -    _extract_split_main(info_data, config_c) +    _extract_split_handedness(info_data, config_c)      _extract_split_transport(info_data, config_c)      _extract_split_right_pins(info_data, config_c)      _extract_encoders(info_data, config_c) diff --git a/lib/python/qmk/json_encoders.py b/lib/python/qmk/json_encoders.py index 1e90f6a288..0e4ad1d220 100755 --- a/lib/python/qmk/json_encoders.py +++ b/lib/python/qmk/json_encoders.py @@ -217,3 +217,21 @@ class KeymapJSONEncoder(QMKJSONEncoder):                  return '50' + str(key)          return key + + +class UserspaceJSONEncoder(QMKJSONEncoder): +    """Custom encoder to make userspace qmk.json's a little nicer to work with. +    """ +    def sort_dict(self, item): +        """Sorts the hashes in a nice way. +        """ +        key = item[0] + +        if self.indentation_level == 1: +            if key == 'userspace_version': +                return '00userspace_version' + +            if key == 'build_targets': +                return '01build_targets' + +        return key diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py index 9826f3f887..b56505d649 100644 --- a/lib/python/qmk/keyboard.py +++ b/lib/python/qmk/keyboard.py @@ -30,6 +30,28 @@ BOX_DRAWING_CHARACTERS = {          "h": "_",      },  } +ENC_DRAWING_CHARACTERS = { +    "unicode": { +        "tl": "╭", +        "tr": "╮", +        "bl": "╰", +        "br": "╯", +        "vl": "▲", +        "vr": "▼", +        "v": "│", +        "h": "─", +    }, +    "ascii": { +        "tl": " ", +        "tr": " ", +        "bl": "\\", +        "br": "/", +        "v": "|", +        "vl": "/", +        "vr": "\\", +        "h": "_", +    }, +}  class AllKeyboards: @@ -48,16 +70,25 @@ class AllKeyboards:  base_path = os.path.join(os.getcwd(), "keyboards") + os.path.sep +@lru_cache(maxsize=1) +def keyboard_alias_definitions(): +    return json_load(Path('data/mappings/keyboard_aliases.hjson')) + +  def is_all_keyboards(keyboard):      """Returns True if the keyboard is an AllKeyboards object.      """ +    if isinstance(keyboard, str): +        return (keyboard == 'all')      return isinstance(keyboard, AllKeyboards)  def find_keyboard_from_dir():      """Returns a keyboard name based on the user's current directory.      """ -    relative_cwd = qmk.path.under_qmk_firmware() +    relative_cwd = qmk.path.under_qmk_userspace() +    if not relative_cwd: +        relative_cwd = qmk.path.under_qmk_firmware()      if relative_cwd and len(relative_cwd.parts) > 1 and relative_cwd.parts[0] == 'keyboards':          # Attempt to extract the keyboard name from the current directory @@ -90,7 +121,7 @@ def keyboard_folder(keyboard):      This checks aliases and DEFAULT_FOLDER to resolve the actual path for a keyboard.      """ -    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) +    aliases = keyboard_alias_definitions()      while keyboard in aliases:          last_keyboard = keyboard @@ -98,11 +129,7 @@ def keyboard_folder(keyboard):          if keyboard == last_keyboard:              break -    rules_mk_file = Path(base_path, keyboard, 'rules.mk') - -    if rules_mk_file.exists(): -        rules_mk = parse_rules_mk_file(rules_mk_file) -        keyboard = rules_mk.get('DEFAULT_FOLDER', keyboard) +    keyboard = resolve_keyboard(keyboard)      if not qmk.path.is_keyboard(keyboard):          raise ValueError(f'Invalid keyboard: {keyboard}') @@ -110,6 +137,22 @@ def keyboard_folder(keyboard):      return keyboard +def keyboard_aliases(keyboard): +    """Returns the list of aliases for the supplied keyboard. + +    Includes the keyboard itself. +    """ +    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) + +    if keyboard in aliases: +        keyboard = aliases[keyboard].get('target', keyboard) + +    keyboards = set(filter(lambda k: aliases[k].get('target', '') == keyboard, aliases.keys())) +    keyboards.add(keyboard) +    keyboards = list(sorted(keyboards)) +    return keyboards + +  def keyboard_folder_or_all(keyboard):      """Returns the actual keyboard folder. @@ -217,7 +260,9 @@ def render_layout(layout_data, render_ascii, key_labels=None):          else:              label = key.get('label', '') -        if x >= 0.25 and w == 1.25 and h == 2: +        if 'encoder' in key: +            render_encoder(textpad, x, y, w, h, label, style) +        elif x >= 0.25 and w == 1.25 and h == 2:              render_key_isoenter(textpad, x, y, w, h, label, style)          elif w == 1.5 and h == 2:              render_key_baenter(textpad, x, y, w, h, label, style) @@ -335,3 +380,32 @@ def render_key_baenter(textpad, x, y, w, h, label, style):      textpad[y + 3][x - 3:x + w] = crn_line      textpad[y + 4][x - 3:x + w] = lab_line      textpad[y + 5][x - 3:x + w] = bot_line + + +def render_encoder(textpad, x, y, w, h, label, style): +    box_chars = ENC_DRAWING_CHARACTERS[style] +    x = ceil(x * 4) +    y = ceil(y * 3) +    w = ceil(w * 4) +    h = ceil(h * 3) + +    label_len = w - 2 +    label_leftover = label_len - len(label) + +    if len(label) > label_len: +        label = label[:label_len] + +    label_blank = ' ' * label_len +    label_border = box_chars['h'] * label_len +    label_middle = label + ' ' * label_leftover + +    top_line = array('u', box_chars['tl'] + label_border + box_chars['tr']) +    lab_line = array('u', box_chars['vl'] + label_middle + box_chars['vr']) +    mid_line = array('u', box_chars['v'] + label_blank + box_chars['v']) +    bot_line = array('u', box_chars['bl'] + label_border + box_chars['br']) + +    textpad[y][x:x + w] = top_line +    textpad[y + 1][x:x + w] = lab_line +    for i in range(h - 3): +        textpad[y + i + 2][x:x + w] = mid_line +    textpad[y + h - 1][x:x + w] = bot_line diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index 11e8d39dad..b7bf897377 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -12,7 +12,8 @@ from pygments.token import Token  from pygments import lex  import qmk.path -from qmk.keyboard import find_keyboard_from_dir, keyboard_folder +from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE +from qmk.keyboard import find_keyboard_from_dir, keyboard_folder, keyboard_aliases  from qmk.errors import CppError  from qmk.info import info_json @@ -191,33 +192,41 @@ def _strip_any(keycode):      return keycode -def find_keymap_from_dir(): -    """Returns `(keymap_name, source)` for the directory we're currently in. - +def find_keymap_from_dir(*args): +    """Returns `(keymap_name, source)` for the directory provided (or cwd if not specified).      """ -    relative_cwd = qmk.path.under_qmk_firmware() +    def _impl_find_keymap_from_dir(relative_path): +        if relative_path and len(relative_path.parts) > 1: +            # If we're in `qmk_firmware/keyboards` and `keymaps` is in our path, try to find the keyboard name. +            if relative_path.parts[0] == 'keyboards' and 'keymaps' in relative_path.parts: +                current_path = Path('/'.join(relative_path.parts[1:]))  # Strip 'keyboards' from the front + +                if 'keymaps' in current_path.parts and current_path.name != 'keymaps': +                    while current_path.parent.name != 'keymaps': +                        current_path = current_path.parent -    if relative_cwd and len(relative_cwd.parts) > 1: -        # If we're in `qmk_firmware/keyboards` and `keymaps` is in our path, try to find the keyboard name. -        if relative_cwd.parts[0] == 'keyboards' and 'keymaps' in relative_cwd.parts: -            current_path = Path('/'.join(relative_cwd.parts[1:]))  # Strip 'keyboards' from the front +                    return current_path.name, 'keymap_directory' -            if 'keymaps' in current_path.parts and current_path.name != 'keymaps': -                while current_path.parent.name != 'keymaps': -                    current_path = current_path.parent +            # If we're in `qmk_firmware/layouts` guess the name from the community keymap they're in +            elif relative_path.parts[0] == 'layouts' and is_keymap_dir(relative_path): +                return relative_path.name, 'layouts_directory' -                return current_path.name, 'keymap_directory' +            # If we're in `qmk_firmware/users` guess the name from the userspace they're in +            elif relative_path.parts[0] == 'users': +                # Guess the keymap name based on which userspace they're in +                return relative_path.parts[1], 'users_directory' +        return None, None -        # If we're in `qmk_firmware/layouts` guess the name from the community keymap they're in -        elif relative_cwd.parts[0] == 'layouts' and is_keymap_dir(relative_cwd): -            return relative_cwd.name, 'layouts_directory' +    if HAS_QMK_USERSPACE: +        name, source = _impl_find_keymap_from_dir(qmk.path.under_qmk_userspace(*args)) +        if name and source: +            return name, source -        # If we're in `qmk_firmware/users` guess the name from the userspace they're in -        elif relative_cwd.parts[0] == 'users': -            # Guess the keymap name based on which userspace they're in -            return relative_cwd.parts[1], 'users_directory' +    name, source = _impl_find_keymap_from_dir(qmk.path.under_qmk_firmware(*args)) +    if name and source: +        return name, source -    return None, None +    return (None, None)  def keymap_completer(prefix, action, parser, parsed_args): @@ -418,29 +427,45 @@ def locate_keymap(keyboard, keymap):          raise KeyError('Invalid keyboard: ' + repr(keyboard))      # Check the keyboard folder first, last match wins -    checked_dirs = ''      keymap_path = '' -    for dir in keyboard_folder(keyboard).split('/'): -        if checked_dirs: -            checked_dirs = '/'.join((checked_dirs, dir)) -        else: -            checked_dirs = dir +    search_dirs = [QMK_FIRMWARE] +    keyboard_dirs = [keyboard_folder(keyboard)] +    if HAS_QMK_USERSPACE: +        # When we've got userspace, check there _last_ as we want them to override anything in the main repo. +        search_dirs.append(QMK_USERSPACE) +        # We also want to search for any aliases as QMK's folder structure may have changed, with an alias, but the user +        # hasn't updated their keymap location yet. +        keyboard_dirs.extend(keyboard_aliases(keyboard)) +        keyboard_dirs = list(set(keyboard_dirs)) + +    for search_dir in search_dirs: +        for keyboard_dir in keyboard_dirs: +            checked_dirs = '' +            for dir in keyboard_dir.split('/'): +                if checked_dirs: +                    checked_dirs = '/'.join((checked_dirs, dir)) +                else: +                    checked_dirs = dir -        keymap_dir = Path('keyboards') / checked_dirs / 'keymaps' +                keymap_dir = Path(search_dir) / Path('keyboards') / checked_dirs / 'keymaps' -        if (keymap_dir / keymap / 'keymap.c').exists(): -            keymap_path = keymap_dir / keymap / 'keymap.c' -        if (keymap_dir / keymap / 'keymap.json').exists(): -            keymap_path = keymap_dir / keymap / 'keymap.json' +                if (keymap_dir / keymap / 'keymap.c').exists(): +                    keymap_path = keymap_dir / keymap / 'keymap.c' +                if (keymap_dir / keymap / 'keymap.json').exists(): +                    keymap_path = keymap_dir / keymap / 'keymap.json' -    if keymap_path: -        return keymap_path +        if keymap_path: +            return keymap_path      # Check community layouts as a fallback      info = info_json(keyboard) -    for community_parent in Path('layouts').glob('*/'): +    community_parents = list(Path('layouts').glob('*/')) +    if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists(): +        community_parents.append(Path(QMK_USERSPACE) / "layouts") + +    for community_parent in community_parents:          for layout in info.get("community_layouts", []):              community_layout = community_parent / layout / keymap              if community_layout.exists(): @@ -450,6 +475,16 @@ def locate_keymap(keyboard, keymap):                      return community_layout / 'keymap.c' +def is_keymap_target(keyboard, keymap): +    if keymap == 'all': +        return True + +    if locate_keymap(keyboard, keymap): +        return True + +    return False + +  def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=False):      """List the available keymaps for a keyboard. @@ -474,26 +509,30 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa      """      names = set() -    keyboards_dir = Path('keyboards') -    kb_path = keyboards_dir / keyboard -      # walk up the directory tree until keyboards_dir      # and collect all directories' name with keymap.c file in it -    while kb_path != keyboards_dir: -        keymaps_dir = kb_path / "keymaps" - -        if keymaps_dir.is_dir(): -            for keymap in keymaps_dir.iterdir(): -                if is_keymap_dir(keymap, c, json, additional_files): -                    keymap = keymap if fullpath else keymap.name -                    names.add(keymap) +    for search_dir in [QMK_FIRMWARE, QMK_USERSPACE] if HAS_QMK_USERSPACE else [QMK_FIRMWARE]: +        keyboards_dir = search_dir / Path('keyboards') +        kb_path = keyboards_dir / keyboard + +        while kb_path != keyboards_dir: +            keymaps_dir = kb_path / "keymaps" +            if keymaps_dir.is_dir(): +                for keymap in keymaps_dir.iterdir(): +                    if is_keymap_dir(keymap, c, json, additional_files): +                        keymap = keymap if fullpath else keymap.name +                        names.add(keymap) -        kb_path = kb_path.parent +            kb_path = kb_path.parent      # Check community layouts as a fallback      info = info_json(keyboard) -    for community_parent in Path('layouts').glob('*/'): +    community_parents = list(Path('layouts').glob('*/')) +    if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists(): +        community_parents.append(Path(QMK_USERSPACE) / "layouts") + +    for community_parent in community_parents:          for layout in info.get("community_layouts", []):              cl_path = community_parent / layout              if cl_path.is_dir(): @@ -657,7 +696,7 @@ def parse_keymap_c(keymap_file, use_cpp=True):      Returns:          a dictionary containing the parsed keymap      """ -    if keymap_file == '-': +    if not isinstance(keymap_file, (Path, str)) or keymap_file == '-':          if use_cpp:              keymap_file = _c_preprocess(None, sys.stdin)          else: diff --git a/lib/python/qmk/path.py b/lib/python/qmk/path.py index 9d248451b8..74364ee04b 100644 --- a/lib/python/qmk/path.py +++ b/lib/python/qmk/path.py @@ -5,7 +5,7 @@ import os  import argparse  from pathlib import Path -from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE +from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE  from qmk.errors import NoSuchKeyboardError @@ -19,17 +19,49 @@ def is_keyboard(keyboard_name):          return rules_mk.exists() -def under_qmk_firmware(): +def under_qmk_firmware(path=Path(os.environ['ORIG_CWD'])):      """Returns a Path object representing the relative path under qmk_firmware, or None.      """ -    cwd = Path(os.environ['ORIG_CWD']) -      try: -        return cwd.relative_to(QMK_FIRMWARE) +        return path.relative_to(QMK_FIRMWARE)      except ValueError:          return None +def under_qmk_userspace(path=Path(os.environ['ORIG_CWD'])): +    """Returns a Path object representing the relative path under $QMK_USERSPACE, or None. +    """ +    try: +        if HAS_QMK_USERSPACE: +            return path.relative_to(QMK_USERSPACE) +    except ValueError: +        pass +    return None + + +def is_under_qmk_firmware(path=Path(os.environ['ORIG_CWD'])): +    """Returns a boolean if the input path is a child under qmk_firmware. +    """ +    if path is None: +        return False +    try: +        return Path(os.path.commonpath([Path(path), QMK_FIRMWARE])) == QMK_FIRMWARE +    except ValueError: +        return False + + +def is_under_qmk_userspace(path=Path(os.environ['ORIG_CWD'])): +    """Returns a boolean if the input path is a child under $QMK_USERSPACE. +    """ +    if path is None: +        return False +    try: +        if HAS_QMK_USERSPACE: +            return Path(os.path.commonpath([Path(path), QMK_USERSPACE])) == QMK_USERSPACE +    except ValueError: +        return False + +  def keyboard(keyboard_name):      """Returns the path to a keyboard's directory relative to the qmk root.      """ @@ -47,11 +79,28 @@ def keymaps(keyboard_name):      keyboard_folder = keyboard(keyboard_name)      found_dirs = [] +    if HAS_QMK_USERSPACE: +        this_keyboard_folder = Path(QMK_USERSPACE) / keyboard_folder +        for _ in range(MAX_KEYBOARD_SUBFOLDERS): +            if (this_keyboard_folder / 'keymaps').exists(): +                found_dirs.append((this_keyboard_folder / 'keymaps').resolve()) + +            this_keyboard_folder = this_keyboard_folder.parent +            if this_keyboard_folder.resolve() == QMK_USERSPACE.resolve(): +                break + +        # We don't have any relevant keymap directories in userspace, so we'll use the fully-qualified path instead. +        if len(found_dirs) == 0: +            found_dirs.append((QMK_USERSPACE / keyboard_folder / 'keymaps').resolve()) + +    this_keyboard_folder = QMK_FIRMWARE / keyboard_folder      for _ in range(MAX_KEYBOARD_SUBFOLDERS): -        if (keyboard_folder / 'keymaps').exists(): -            found_dirs.append((keyboard_folder / 'keymaps').resolve()) +        if (this_keyboard_folder / 'keymaps').exists(): +            found_dirs.append((this_keyboard_folder / 'keymaps').resolve()) -        keyboard_folder = keyboard_folder.parent +        this_keyboard_folder = this_keyboard_folder.parent +        if this_keyboard_folder.resolve() == QMK_FIRMWARE.resolve(): +            break      if len(found_dirs) > 0:          return found_dirs diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py index a74450ca87..84cf6cbe32 100644 --- a/lib/python/qmk/search.py +++ b/lib/python/qmk/search.py @@ -6,13 +6,14 @@ import fnmatch  import logging  import re  from typing import List, Tuple -from dotty_dict import dotty +from dotty_dict import dotty, Dotty  from milc import cli  from qmk.util import parallel_map  from qmk.info import keymap_json -import qmk.keyboard -import qmk.keymap +from qmk.keyboard import list_keyboards, keyboard_folder +from qmk.keymap import list_keymaps, locate_keymap +from qmk.build_targets import KeyboardKeymapBuildTarget, BuildTarget  def _set_log_level(level): @@ -36,15 +37,15 @@ def _all_keymaps(keyboard):      """Returns a list of tuples of (keyboard, keymap) for all keymaps for the given keyboard.      """      with ignore_logging(): -        keyboard = qmk.keyboard.resolve_keyboard(keyboard) -        return [(keyboard, keymap) for keymap in qmk.keymap.list_keymaps(keyboard)] +        keyboard = keyboard_folder(keyboard) +        return [(keyboard, keymap) for keymap in list_keymaps(keyboard)]  def _keymap_exists(keyboard, keymap):      """Returns the keyboard name if the keyboard+keymap combination exists, otherwise None.      """      with ignore_logging(): -        return keyboard if qmk.keymap.locate_keymap(keyboard, keymap) is not None else None +        return keyboard if locate_keymap(keyboard, keymap) is not None else None  def _load_keymap_info(kb_km): @@ -75,7 +76,7 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =      Caters for 'all' in either keyboard or keymap, or both.      """      if all_keyboards is None: -        all_keyboards = qmk.keyboard.list_keyboards() +        all_keyboards = list_keyboards()      if keyboard == 'all':          if keymap == 'all': @@ -90,30 +91,38 @@ def _expand_keymap_target(keyboard: str, keymap: str, all_keyboards: List[str] =              return [(kb, keymap) for kb in filter(lambda e: e is not None, parallel_map(keyboard_filter, all_keyboards))]      else:          if keymap == 'all': -            keyboard = qmk.keyboard.resolve_keyboard(keyboard)              cli.log.info(f'Retrieving list of keymaps for keyboard "{keyboard}"...')              return _all_keymaps(keyboard)          else: -            return [(qmk.keyboard.resolve_keyboard(keyboard), keymap)] +            return [(keyboard, keymap)]  def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str]]:      """Expand a list of (keyboard, keymap) tuples inclusive of 'all', into a list of explicit (keyboard, keymap) tuples.      """      overall_targets = [] -    all_keyboards = qmk.keyboard.list_keyboards() +    all_keyboards = list_keyboards()      for target in targets:          overall_targets.extend(_expand_keymap_target(target[0], target[1], all_keyboards))      return list(sorted(set(overall_targets))) -def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]: +def _construct_build_target_kb_km(e): +    return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1]) + + +def _construct_build_target_kb_km_json(e): +    return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2]) + + +def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]:      """Filter a list of (keyboard, keymap) tuples based on the supplied filters.      Optionally includes the values of the queried info.json keys.      """ -    if len(filters) == 0 and len(print_vals) == 0: -        targets = [(kb, km, {}) for kb, km in target_list] +    if len(filters) == 0: +        cli.log.info('Preparing target list...') +        targets = list(set(parallel_map(_construct_build_target_kb_km, target_list)))      else:          cli.log.info('Parsing data for all matching keyboard/keymap combinations...')          valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)] @@ -172,18 +181,20 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str                  cli.log.warning(f'Unrecognized filter expression: {filter_expr}')                  continue -            targets = [(e[0], e[1], [(p, e[2].get(p)) for p in print_vals]) for e in valid_keymaps] +        cli.log.info('Preparing target list...') +        valid_keymaps = [(e[0], e[1], e[2].to_dict() if isinstance(e[2], Dotty) else e[2]) for e in valid_keymaps]  # need to convert dotty_dict back to dict because it doesn't survive parallelisation +        targets = list(set(parallel_map(_construct_build_target_kb_km_json, list(valid_keymaps))))      return targets -def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]: +def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]:      """Search for build targets matching the supplied criteria.      """ -    return list(sorted(_filter_keymap_targets(expand_keymap_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1]))) +    return _filter_keymap_targets(expand_keymap_targets(targets), filters) -def search_make_targets(targets: List[str], filters: List[str] = [], print_vals: List[str] = []) -> List[Tuple[str, str, List[Tuple[str, str]]]]: +def search_make_targets(targets: List[str], filters: List[str] = []) -> List[BuildTarget]:      """Search for build targets matching the supplied criteria.      """ -    return list(sorted(_filter_keymap_targets(expand_make_targets(targets), filters, print_vals), key=lambda e: (e[0], e[1]))) +    return _filter_keymap_targets(expand_make_targets(targets), filters) diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py new file mode 100644 index 0000000000..3783568006 --- /dev/null +++ b/lib/python/qmk/userspace.py @@ -0,0 +1,185 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later +from os import environ +from pathlib import Path +import json +import jsonschema + +from milc import cli + +from qmk.json_schema import validate, json_load +from qmk.json_encoders import UserspaceJSONEncoder + + +def qmk_userspace_paths(): +    test_dirs = [] + +    # If we're already in a directory with a qmk.json and a keyboards or layouts directory, interpret it as userspace +    current_dir = Path(environ['ORIG_CWD']) +    while len(current_dir.parts) > 1: +        if (current_dir / 'qmk.json').is_file(): +            test_dirs.append(current_dir) +        current_dir = current_dir.parent + +    # If we have a QMK_USERSPACE environment variable, use that +    if environ.get('QMK_USERSPACE') is not None: +        current_dir = Path(environ.get('QMK_USERSPACE')) +        if current_dir.is_dir(): +            test_dirs.append(current_dir) + +    # If someone has configured a directory, use that +    if cli.config.user.overlay_dir is not None: +        current_dir = Path(cli.config.user.overlay_dir) +        if current_dir.is_dir(): +            test_dirs.append(current_dir) + +    return test_dirs + + +def qmk_userspace_validate(path): +    # Construct a UserspaceDefs object to ensure it validates correctly +    if (path / 'qmk.json').is_file(): +        UserspaceDefs(path / 'qmk.json') +        return + +    # No qmk.json file found +    raise FileNotFoundError('No qmk.json file found.') + + +def detect_qmk_userspace(): +    # Iterate through all the detected userspace paths and return the first one that validates correctly +    test_dirs = qmk_userspace_paths() +    for test_dir in test_dirs: +        try: +            qmk_userspace_validate(test_dir) +            return test_dir +        except FileNotFoundError: +            continue +        except UserspaceValidationError: +            continue +    return None + + +class UserspaceDefs: +    def __init__(self, userspace_json: Path): +        self.path = userspace_json +        self.build_targets = [] +        json = json_load(userspace_json) + +        exception = UserspaceValidationError() +        success = False + +        try: +            validate(json, 'qmk.user_repo.v0')  # `qmk.json` must have a userspace_version at minimum +        except jsonschema.ValidationError as err: +            exception.add('qmk.user_repo.v0', err) +            raise exception + +        # Iterate through each version of the schema, starting with the latest and decreasing to v1 +        try: +            validate(json, 'qmk.user_repo.v1') +            self.__load_v1(json) +            success = True +        except jsonschema.ValidationError as err: +            exception.add('qmk.user_repo.v1', err) + +        if not success: +            raise exception + +    def save(self): +        target_json = { +            "userspace_version": "1.0",  # Needs to match latest version +            "build_targets": [] +        } + +        for e in self.build_targets: +            if isinstance(e, dict): +                target_json['build_targets'].append([e['keyboard'], e['keymap']]) +            elif isinstance(e, Path): +                target_json['build_targets'].append(str(e.relative_to(self.path.parent))) + +        try: +            # Ensure what we're writing validates against the latest version of the schema +            validate(target_json, 'qmk.user_repo.v1') +        except jsonschema.ValidationError as err: +            cli.log.error(f'Could not save userspace file: {err}') +            return False + +        # Only actually write out data if it changed +        old_data = json.dumps(json.loads(self.path.read_text()), cls=UserspaceJSONEncoder, sort_keys=True) +        new_data = json.dumps(target_json, cls=UserspaceJSONEncoder, sort_keys=True) +        if old_data != new_data: +            self.path.write_text(new_data) +            cli.log.info(f'Saved userspace file to {self.path}.') +        return True + +    def add_target(self, keyboard=None, keymap=None, json_path=None, do_print=True): +        if json_path is not None: +            # Assume we're adding a json filename/path +            json_path = Path(json_path) +            if json_path not in self.build_targets: +                self.build_targets.append(json_path) +                if do_print: +                    cli.log.info(f'Added {json_path} to userspace build targets.') +            else: +                cli.log.info(f'{json_path} is already a userspace build target.') + +        elif keyboard is not None and keymap is not None: +            # Both keyboard/keymap specified +            e = {"keyboard": keyboard, "keymap": keymap} +            if e not in self.build_targets: +                self.build_targets.append(e) +                if do_print: +                    cli.log.info(f'Added {keyboard}:{keymap} to userspace build targets.') +            else: +                if do_print: +                    cli.log.info(f'{keyboard}:{keymap} is already a userspace build target.') + +    def remove_target(self, keyboard=None, keymap=None, json_path=None, do_print=True): +        if json_path is not None: +            # Assume we're removing a json filename/path +            json_path = Path(json_path) +            if json_path in self.build_targets: +                self.build_targets.remove(json_path) +                if do_print: +                    cli.log.info(f'Removed {json_path} from userspace build targets.') +            else: +                cli.log.info(f'{json_path} is not a userspace build target.') + +        elif keyboard is not None and keymap is not None: +            # Both keyboard/keymap specified +            e = {"keyboard": keyboard, "keymap": keymap} +            if e in self.build_targets: +                self.build_targets.remove(e) +                if do_print: +                    cli.log.info(f'Removed {keyboard}:{keymap} from userspace build targets.') +            else: +                if do_print: +                    cli.log.info(f'{keyboard}:{keymap} is not a userspace build target.') + +    def __load_v1(self, json): +        for e in json['build_targets']: +            if isinstance(e, list) and len(e) == 2: +                self.add_target(keyboard=e[0], keymap=e[1], do_print=False) +            if isinstance(e, str): +                p = self.path.parent / e +                if p.exists() and p.suffix == '.json': +                    self.add_target(json_path=p, do_print=False) + + +class UserspaceValidationError(Exception): +    def __init__(self, *args, **kwargs): +        super().__init__(*args, **kwargs) +        self.__exceptions = [] + +    def __str__(self): +        return self.message + +    @property +    def exceptions(self): +        return self.__exceptions + +    def add(self, schema, exception): +        self.__exceptions.append((schema, exception)) +        errorlist = "\n\n".join([f"{schema}: {exception}" for schema, exception in self.__exceptions]) +        self.message = f'Could not validate against any version of the userspace schema. Errors:\n\n{errorlist}' | 
