From b0069c5c05dac2c910d51ef7f3bf4133721a9c49 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sun, 7 Mar 2021 19:10:03 -0800 Subject: Begin the process of deprecating bin/qmk in favor of the global cli (#12109) * Begin the process of deprecating bin/qmk in favor of the global cli * Correctly set the qmk bin --- lib/python/qmk/commands.py | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/python/qmk/commands.py') diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 3c6f0d001d..233d1034b4 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -180,6 +180,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va f'VERBOSE={verbose}', f'COLOR={color}', 'SILENT=false', + 'QMK_BIN=qmk', ]) return make_command -- cgit v1.2.3 From f2715a05939b749771d9826f9ad23b4cab280e82 Mon Sep 17 00:00:00 2001 From: Zach White Date: Thu, 18 Mar 2021 16:10:40 -0700 Subject: Consistently use bin/qmk when that script is called (#12286) * Pass QMK_BIN down to build_keyboard.mk * choose the correct qmk script --- Makefile | 4 ++-- bin/qmk | 1 + lib/python/qmk/commands.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) (limited to 'lib/python/qmk/commands.py') diff --git a/Makefile b/Makefile index de0a2d4153..80e1a90a19 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ endif endif # Determine which qmk cli to use -ifeq (, $(shell which qmk)) +ifeq (,$(shell which qmk)) QMK_BIN = bin/qmk else QMK_BIN = qmk @@ -391,7 +391,7 @@ define PARSE_KEYMAP # Format it in bold KB_SP := $(BOLD)$$(KB_SP)$(NO_COLOR) # Specify the variables that we are passing forward to submake - MAKE_VARS := KEYBOARD=$$(CURRENT_KB) KEYMAP=$$(CURRENT_KM) REQUIRE_PLATFORM_KEY=$$(REQUIRE_PLATFORM_KEY) + MAKE_VARS := KEYBOARD=$$(CURRENT_KB) KEYMAP=$$(CURRENT_KM) REQUIRE_PLATFORM_KEY=$$(REQUIRE_PLATFORM_KEY) QMK_BIN=$$(QMK_BIN) # And the first part of the make command MAKE_CMD := $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET) # The message to display diff --git a/bin/qmk b/bin/qmk index 28486026f9..a2af2951c9 100755 --- a/bin/qmk +++ b/bin/qmk @@ -73,6 +73,7 @@ def main(): """ # Change to the root of our checkout os.environ['ORIG_CWD'] = os.getcwd() + os.environ['DEPRECATED_BIN_QMK'] = '1' os.chdir(qmk_dir) print('Warning: The bin/qmk script is being deprecated. Please install the QMK CLI: python3 -m pip install qmk', file=sys.stderr) diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index ac98376958..4809365af3 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -180,7 +180,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va f'VERBOSE={verbose}', f'COLOR={color}', 'SILENT=false', - 'QMK_BIN=qmk', + f'QMK_BIN={"bin/qmk" if "DEPRECATED_BIN_QMK" in os.environ else "qmk"}', ]) return make_command -- cgit v1.2.3 From bbe43a91ebf193bbc8c09ba59209b0524367e68c Mon Sep 17 00:00:00 2001 From: Ryan Date: Sun, 27 Jun 2021 02:29:02 +1000 Subject: CLI: Add subcommand to generate version.h (#13151) --- Makefile | 27 +++----------- lib/python/qmk/cli/__init__.py | 1 + lib/python/qmk/cli/generate/version_h.py | 28 ++++++++++++++ lib/python/qmk/commands.py | 62 +++++++++++++++++++------------ lib/python/qmk/tests/test_cli_commands.py | 6 +++ 5 files changed, 79 insertions(+), 45 deletions(-) create mode 100644 lib/python/qmk/cli/generate/version_h.py (limited to 'lib/python/qmk/commands.py') diff --git a/Makefile b/Makefile index e007ae3679..866bcaaf3f 100644 --- a/Makefile +++ b/Makefile @@ -548,29 +548,14 @@ git-submodule: git submodule sync --recursive git submodule update --init --recursive --progress -ifdef SKIP_VERSION -SKIP_GIT := yes -endif - # Generate the version.h file -ifndef SKIP_GIT - GIT_VERSION := $(shell git describe --abbrev=6 --dirty --always --tags 2>/dev/null || date +"%Y-%m-%d-%H:%M:%S") - CHIBIOS_VERSION := $(shell cd lib/chibios && git describe --abbrev=6 --dirty --always --tags 2>/dev/null || date +"%Y-%m-%d-%H:%M:%S") - CHIBIOS_CONTRIB_VERSION := $(shell cd lib/chibios-contrib && git describe --abbrev=6 --dirty --always --tags 2>/dev/null || date +"%Y-%m-%d-%H:%M:%S") -else - GIT_VERSION := NA - CHIBIOS_VERSION := NA - CHIBIOS_CONTRIB_VERSION := NA +ifdef SKIP_GIT +VERSION_H_FLAGS := --skip-git endif -ifndef SKIP_VERSION -BUILD_DATE := $(shell date +"%Y-%m-%d-%H:%M:%S") -else -BUILD_DATE := 2020-01-01-00:00:00 +ifdef SKIP_VERSION +VERSION_H_FLAGS := --skip-all +SKIP_GIT := yes endif - -$(shell echo '#define QMK_VERSION "$(GIT_VERSION)"' > $(ROOT_DIR)/quantum/version.h) -$(shell echo '#define QMK_BUILDDATE "$(BUILD_DATE)"' >> $(ROOT_DIR)/quantum/version.h) -$(shell echo '#define CHIBIOS_VERSION "$(CHIBIOS_VERSION)"' >> $(ROOT_DIR)/quantum/version.h) -$(shell echo '#define CHIBIOS_CONTRIB_VERSION "$(CHIBIOS_CONTRIB_VERSION)"' >> $(ROOT_DIR)/quantum/version.h) +$(shell $(QMK_BIN) generate-version-h $(VERSION_H_FLAGS) -q -o quantum/version.h) include $(ROOT_DIR)/testlist.mk diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 7f5e0a1fa6..2c3c9c4211 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -49,6 +49,7 @@ subcommands = [ 'qmk.cli.generate.layouts', 'qmk.cli.generate.rgb_breathe_table', 'qmk.cli.generate.rules_mk', + 'qmk.cli.generate.version_h', 'qmk.cli.hello', 'qmk.cli.info', 'qmk.cli.json2c', diff --git a/lib/python/qmk/cli/generate/version_h.py b/lib/python/qmk/cli/generate/version_h.py new file mode 100644 index 0000000000..b8e52588c4 --- /dev/null +++ b/lib/python/qmk/cli/generate/version_h.py @@ -0,0 +1,28 @@ +"""Used by the make system to generate version.h for use in code. +""" +from milc import cli + +from qmk.commands import create_version_h +from qmk.path import normpath + + +@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to') +@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages") +@cli.argument('--skip-git', arg_only=True, action='store_true', help='Skip Git operations') +@cli.argument('--skip-all', arg_only=True, action='store_true', help='Use placeholder values for all defines (implies --skip-git)') +@cli.subcommand('Used by the make system to generate version.h for use in code', hidden=True) +def generate_version_h(cli): + """Generates the version.h file. + """ + if cli.args.skip_all: + cli.args.skip_git = True + + version_h = create_version_h(cli.args.skip_git, cli.args.skip_all) + + if cli.args.output: + cli.args.output.write_text(version_h) + + if not cli.args.quiet: + cli.log.info('Wrote version.h to %s.', cli.args.output) + else: + print(version_h) diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 3a35c11031..104c875457 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -86,11 +86,17 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars): return create_make_target(':'.join(make_args), parallel, **env_vars) -def get_git_version(repo_dir='.', check_dir='.'): +def get_git_version(current_time, repo_dir='.', check_dir='.'): """Returns the current git version for a repo, or the current time. """ git_describe_cmd = ['git', 'describe', '--abbrev=6', '--dirty', '--always', '--tags'] + if repo_dir != '.': + repo_dir = Path('lib') / repo_dir + + if check_dir != '.': + check_dir = repo_dir / check_dir + if Path(check_dir).exists(): git_describe = cli.run(git_describe_cmd, stdin=DEVNULL, cwd=repo_dir) @@ -100,23 +106,40 @@ def get_git_version(repo_dir='.', check_dir='.'): else: cli.log.warn(f'"{" ".join(git_describe_cmd)}" returned error code {git_describe.returncode}') print(git_describe.stderr) - return strftime(time_fmt) + return current_time - return strftime(time_fmt) + return current_time -def write_version_h(git_version, build_date, chibios_version, chibios_contrib_version): - """Generate and write quantum/version.h +def create_version_h(skip_git=False, skip_all=False): + """Generate version.h contents """ - version_h = [ - f'#define QMK_VERSION "{git_version}"', - f'#define QMK_BUILDDATE "{build_date}"', - f'#define CHIBIOS_VERSION "{chibios_version}"', - f'#define CHIBIOS_CONTRIB_VERSION "{chibios_contrib_version}"', - ] + if skip_all: + current_time = "1970-01-01-00:00:00" + else: + current_time = strftime(time_fmt) + + if skip_git: + git_version = "NA" + chibios_version = "NA" + chibios_contrib_version = "NA" + else: + git_version = get_git_version(current_time) + chibios_version = get_git_version(current_time, "chibios", "os") + chibios_contrib_version = get_git_version(current_time, "chibios-contrib", "os") + + version_h_lines = f"""/* This file was automatically generated. Do not edit or copy. + */ + +#pragma once + +#define QMK_VERSION "{git_version}" +#define QMK_BUILDDATE "{current_time}" +#define CHIBIOS_VERSION "{chibios_version}" +#define CHIBIOS_CONTRIB_VERSION "{chibios_contrib_version}" +""" - version_h_file = Path('quantum/version.h') - version_h_file.write_text('\n'.join(version_h)) + return version_h_lines def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_vars): @@ -149,13 +172,8 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va keymap_dir.mkdir(exist_ok=True, parents=True) keymap_c.write_text(c_text) - # Write the version.h file - git_version = get_git_version() - build_date = strftime('%Y-%m-%d-%H:%M:%S') - chibios_version = get_git_version("lib/chibios", "lib/chibios/os") - chibios_contrib_version = get_git_version("lib/chibios-contrib", "lib/chibios-contrib/os") - - write_version_h(git_version, build_date, chibios_version, chibios_contrib_version) + version_h = Path('quantum/version.h') + version_h.write_text(create_version_h()) # Return a command that can be run to make the keymap and flash if given verbose = 'true' if cli.config.general.verbose else 'false' @@ -181,10 +199,6 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va make_command.append(f'{key}={value}') make_command.extend([ - f'GIT_VERSION={git_version}', - f'BUILD_DATE={build_date}', - f'CHIBIOS_VERSION={chibios_version}', - f'CHIBIOS_CONTRIB_VERSION={chibios_contrib_version}', f'KEYBOARD={user_keymap["keyboard"]}', f'KEYMAP={user_keymap["keymap"]}', f'KEYBOARD_FILESAFE={keyboard_filesafe}', diff --git a/lib/python/qmk/tests/test_cli_commands.py b/lib/python/qmk/tests/test_cli_commands.py index afdbc81429..b341e1c912 100644 --- a/lib/python/qmk/tests/test_cli_commands.py +++ b/lib/python/qmk/tests/test_cli_commands.py @@ -258,6 +258,12 @@ def test_generate_rules_mk(): assert 'MCU ?= atmega32u4' in result.stdout +def test_generate_version_h(): + result = check_subcommand('generate-version-h') + check_returncode(result) + assert '#define QMK_VERSION' in result.stdout + + def test_generate_layouts(): result = check_subcommand('generate-layouts', '-kb', 'handwired/pytest/basic') check_returncode(result) -- cgit v1.2.3 From e05f9c4a08cef384989cd20ebe27b66622912055 Mon Sep 17 00:00:00 2001 From: Erovia Date: Sat, 10 Jul 2021 16:04:50 +0100 Subject: CLI: Add git and venv info to doctor's output (#13405) Most of the checks are saved from zvecr's retired 'up/status' subcommand PR. --- lib/python/qmk/cli/doctor/main.py | 45 +++++++++++++++++++++---- lib/python/qmk/commands.py | 71 ++++++++++++++++++++++++++++++++++++++- lib/python/qmk/constants.py | 3 ++ 3 files changed, 111 insertions(+), 8 deletions(-) (limited to 'lib/python/qmk/commands.py') diff --git a/lib/python/qmk/cli/doctor/main.py b/lib/python/qmk/cli/doctor/main.py index 5e93fad367..6a31ccdfdd 100755 --- a/lib/python/qmk/cli/doctor/main.py +++ b/lib/python/qmk/cli/doctor/main.py @@ -9,8 +9,9 @@ from milc import cli from milc.questions import yesno from qmk import submodules -from qmk.constants import QMK_FIRMWARE -from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules, check_git_repo +from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM +from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules +from qmk.commands import git_check_repo, git_get_branch, git_is_dirty, git_get_remotes, git_check_deviation, in_virtualenv def os_tests(): @@ -32,6 +33,37 @@ def os_tests(): return CheckStatus.WARNING +def git_tests(): + """Run Git-related checks + """ + status = CheckStatus.OK + + # Make sure our QMK home is a Git repo + git_ok = git_check_repo() + if not git_ok: + cli.log.warning("{fg_yellow}QMK home does not appear to be a Git repository! (no .git folder)") + status = CheckStatus.WARNING + else: + git_branch = git_get_branch() + if git_branch: + cli.log.info('Git branch: %s', git_branch) + git_dirty = git_is_dirty() + if git_dirty: + cli.log.warning('{fg_yellow}Git has unstashed/uncommitted changes.') + status = CheckStatus.WARNING + git_remotes = git_get_remotes() + if 'upstream' not in git_remotes.keys() or QMK_FIRMWARE_UPSTREAM not in git_remotes['upstream'].get('url', ''): + cli.log.warning('{fg_yellow}The official repository does not seem to be configured as git remote "upstream".') + status = CheckStatus.WARNING + else: + git_deviation = git_check_deviation(git_branch) + if git_branch in ['master', 'develop'] and git_deviation: + cli.log.warning('{fg_yellow}The local "%s" branch contains commits not found in the upstream branch.', git_branch) + status = CheckStatus.WARNING + + return status + + @cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.') @cli.argument('-n', '--no', action='store_true', arg_only=True, help='Answer no to all questions.') @cli.subcommand('Basic QMK environment checks') @@ -49,12 +81,11 @@ def doctor(cli): status = os_tests() - # Make sure our QMK home is a Git repo - git_ok = check_git_repo() + status = git_tests() - if git_ok == CheckStatus.WARNING: - cli.log.warning("QMK home does not appear to be a Git repository! (no .git folder)") - status = CheckStatus.WARNING + venv = in_virtualenv() + if venv: + cli.log.info('CLI installed in virtualenv.') # Make sure the basic CLI tools we need are available and can be executed. bin_ok = check_binaries() diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 104c875457..8ff8501bf6 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -2,6 +2,7 @@ """ import json import os +import sys import shutil from pathlib import Path from subprocess import DEVNULL @@ -10,7 +11,7 @@ from time import strftime from milc import cli import qmk.keymap -from qmk.constants import KEYBOARD_OUTPUT_PREFIX +from qmk.constants import QMK_FIRMWARE, KEYBOARD_OUTPUT_PREFIX from qmk.json_schema import json_load time_fmt = '%Y-%m-%d-%H:%M:%S' @@ -237,3 +238,71 @@ def parse_configurator_json(configurator_file): user_keymap['layout'] = aliases[orig_keyboard]['layouts'][user_keymap['layout']] return user_keymap + + +def git_check_repo(): + """Checks that the .git directory exists inside QMK_HOME. + + This is a decent enough indicator that the qmk_firmware directory is a + proper Git repository, rather than a .zip download from GitHub. + """ + dot_git_dir = QMK_FIRMWARE / '.git' + + return dot_git_dir.is_dir() + + +def git_get_branch(): + """Returns the current branch for a repo, or None. + """ + git_branch = cli.run(['git', 'branch', '--show-current']) + if not git_branch.returncode != 0 or not git_branch.stdout: + # Workaround for Git pre-2.22 + git_branch = cli.run(['git', 'rev-parse', '--abbrev-ref', 'HEAD']) + + if git_branch.returncode == 0: + return git_branch.stdout.strip() + + +def git_is_dirty(): + """Returns 1 if repo is dirty, or 0 if clean + """ + git_diff_staged_cmd = ['git', 'diff', '--quiet'] + git_diff_unstaged_cmd = [*git_diff_staged_cmd, '--cached'] + + unstaged = cli.run(git_diff_staged_cmd) + staged = cli.run(git_diff_unstaged_cmd) + + return unstaged.returncode != 0 or staged.returncode != 0 + + +def git_get_remotes(): + """Returns the current remotes for a repo. + """ + remotes = {} + + git_remote_show_cmd = ['git', 'remote', 'show'] + git_remote_get_cmd = ['git', 'remote', 'get-url'] + + git_remote_show = cli.run(git_remote_show_cmd) + if git_remote_show.returncode == 0: + for name in git_remote_show.stdout.splitlines(): + git_remote_name = cli.run([*git_remote_get_cmd, name]) + remotes[name.strip()] = {"url": git_remote_name.stdout.strip()} + + return remotes + + +def git_check_deviation(active_branch): + """Return True if branch has custom commits + """ + cli.run(['git', 'fetch', 'upstream', active_branch]) + deviations = cli.run(['git', '--no-pager', 'log', f'upstream/{active_branch}...{active_branch}']) + return bool(deviations.returncode) + + +def in_virtualenv(): + """Check if running inside a virtualenv. + Based on https://stackoverflow.com/a/1883251 + """ + active_prefix = getattr(sys, "base_prefix", None) or getattr(sys, "real_prefix", None) or sys.prefix + return active_prefix != sys.prefix diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py index be5ce5d4ed..1078f4ad5e 100644 --- a/lib/python/qmk/constants.py +++ b/lib/python/qmk/constants.py @@ -6,6 +6,9 @@ from pathlib import Path # The root of the qmk_firmware tree. QMK_FIRMWARE = Path.cwd() +# Upstream repo url +QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware' + # This is the number of directories under `qmk_firmware/keyboards` that will be traversed. This is currently a limitation of our make system. MAX_KEYBOARD_SUBFOLDERS = 5 -- cgit v1.2.3 From b021c2f2c5890df5335d3dd163167c6fe6213e0d Mon Sep 17 00:00:00 2001 From: Ryan Date: Sat, 31 Jul 2021 06:57:40 +1000 Subject: Port new_keyboard.sh to CLI (#13706) Co-authored-by: Erovia --- docs/cli_commands.md | 4 +- docs/es/hardware_avr.md | 38 +++++----- docs/hardware_avr.md | 38 +++++----- docs/ja/hardware_avr.md | 38 +++++----- lib/python/qmk/cli/new/keyboard.py | 141 +++++++++++++++++++++++++++++++++++-- lib/python/qmk/commands.py | 9 +++ util/new_keyboard.sh | 6 ++ 7 files changed, 213 insertions(+), 61 deletions(-) (limited to 'lib/python/qmk/commands.py') diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 314e7448d7..c7468eb5e7 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -280,12 +280,12 @@ qmk list-keymaps -kb planck/ez This command creates a new keyboard based on available templates. -This command will prompt for input to guide you though the generation process. +Any arguments that are not provided will prompt for input. If `-u` is not passed and `user.name` is set in .gitconfig, it will be used as the default username in the prompt. **Usage**: ``` -qmk new-keyboard +qmk new-keyboard [-kb KEYBOARD] [-t {avr,ps2avrgb}] -u USERNAME ``` ## `qmk new-keymap` diff --git a/docs/es/hardware_avr.md b/docs/es/hardware_avr.md index f8c426381f..ac6a715658 100644 --- a/docs/es/hardware_avr.md +++ b/docs/es/hardware_avr.md @@ -6,26 +6,28 @@ Si aún no lo has hecho, debes leer las [Pautas de teclados](hardware_keyboard_g ## Añadir tu Teclado AVR a QMK -QMK tiene varias características para simplificar el trabajo con teclados AVR. Para la mayoría de los teclados no tienes que escribir ni una sola línea de código. Para empezar, ejecuta el archivo `util/new_keyboard.sh`: +QMK tiene varias características para simplificar el trabajo con teclados AVR. Para la mayoría de los teclados no tienes que escribir ni una sola línea de código. Para empezar, ejecuta `qmk new-keyboard`: ``` -$ ./util/new_keyboard.sh -Generating a new QMK keyboard directory - -Keyboard Name: mycoolkb -Keyboard Type [avr]: -Your Name [John Smith]: - -Copying base template files... done -Copying avr template files... done -Renaming keyboard files... done -Replacing %KEYBOARD% with mycoolkb... done -Replacing %YOUR_NAME% with John Smith... done - -Created a new keyboard called mycoolkb. - -To start working on things, cd into keyboards/mycoolkb, -or open the directory in your favourite text editor. +$ qmk new-keyboard +Ψ Generating a new QMK keyboard directory + +Keyboard Name: mycoolkeeb +Keyboard Type: + 1. avr + 2. ps2avrgb +Please enter your choice: [1] +Your Name: [John Smith] +Ψ Copying base template files... +Ψ Copying avr template files... +Ψ Renaming keyboard.[ch] to mycoolkeeb.[ch]... +Ψ Replacing %YEAR% with 2021... +Ψ Replacing %KEYBOARD% with mycoolkeeb... +Ψ Replacing %YOUR_NAME% with John Smith... + +Ψ Created a new keyboard called mycoolkeeb. +Ψ To start working on things, `cd` into keyboards/mycoolkeeb, +Ψ or open the directory in your preferred text editor. ``` Esto creará todos los archivos necesarios para tu nuevo teclado, y rellenará la configuración con valores predeterminados. Ahora sólo tienes que personalizarlo para tu teclado. diff --git a/docs/hardware_avr.md b/docs/hardware_avr.md index eb536ca961..3d58cdc055 100644 --- a/docs/hardware_avr.md +++ b/docs/hardware_avr.md @@ -6,26 +6,28 @@ If you have not yet you should read the [Keyboard Guidelines](hardware_keyboard_ ## Adding Your AVR Keyboard to QMK -QMK has a number of features to simplify working with AVR keyboards. For most keyboards you don't have to write a single line of code. To get started, run the `util/new_keyboard.sh` script: +QMK has a number of features to simplify working with AVR keyboards. For most keyboards you don't have to write a single line of code. To get started, run `qmk new-keyboard`: ``` -$ ./util/new_keyboard.sh -Generating a new QMK keyboard directory - -Keyboard Name: mycoolkb -Keyboard Type [avr]: -Your Name [John Smith]: - -Copying base template files... done -Copying avr template files... done -Renaming keyboard files... done -Replacing %KEYBOARD% with mycoolkb... done -Replacing %YOUR_NAME% with John Smith... done - -Created a new keyboard called mycoolkb. - -To start working on things, cd into keyboards/mycoolkb, -or open the directory in your favourite text editor. +$ qmk new-keyboard +Ψ Generating a new QMK keyboard directory + +Keyboard Name: mycoolkeeb +Keyboard Type: + 1. avr + 2. ps2avrgb +Please enter your choice: [1] +Your Name: [John Smith] +Ψ Copying base template files... +Ψ Copying avr template files... +Ψ Renaming keyboard.[ch] to mycoolkeeb.[ch]... +Ψ Replacing %YEAR% with 2021... +Ψ Replacing %KEYBOARD% with mycoolkeeb... +Ψ Replacing %YOUR_NAME% with John Smith... + +Ψ Created a new keyboard called mycoolkeeb. +Ψ To start working on things, `cd` into keyboards/mycoolkeeb, +Ψ or open the directory in your preferred text editor. ``` This will create all the files needed to support your new keyboard, and populate the settings with default values. Now you just need to customize it for your keyboard. diff --git a/docs/ja/hardware_avr.md b/docs/ja/hardware_avr.md index 66be2f71c9..cdc5f8cb86 100644 --- a/docs/ja/hardware_avr.md +++ b/docs/ja/hardware_avr.md @@ -12,26 +12,28 @@ ## AVR を使用したキーボードを QMK に追加する -QMK には AVR を使ったキーボードでの作業を簡略化するための機能が多数あります。大体のキーボードでは1行もコードを書く必要がありません。まずはじめに、`util/new_keyboard.sh` スクリプトを実行します。 +QMK には AVR を使ったキーボードでの作業を簡略化するための機能が多数あります。大体のキーボードでは1行もコードを書く必要がありません。まずはじめに、`qmk new-keyboard` を実行します。 ``` -$ ./util/new_keyboard.sh -Generating a new QMK keyboard directory - -Keyboard Name: mycoolkb -Keyboard Type [avr]: -Your Name [John Smith]: - -Copying base template files... done -Copying avr template files... done -Renaming keyboard files... done -Replacing %KEYBOARD% with mycoolkb... done -Replacing %YOUR_NAME% with John Smith... done - -Created a new keyboard called mycoolkb. - -To start working on things, cd into keyboards/mycoolkb, -or open the directory in your favourite text editor. +$ qmk new-keyboard +Ψ Generating a new QMK keyboard directory + +Keyboard Name: mycoolkeeb +Keyboard Type: + 1. avr + 2. ps2avrgb +Please enter your choice: [1] +Your Name: [John Smith] +Ψ Copying base template files... +Ψ Copying avr template files... +Ψ Renaming keyboard.[ch] to mycoolkeeb.[ch]... +Ψ Replacing %YEAR% with 2021... +Ψ Replacing %KEYBOARD% with mycoolkeeb... +Ψ Replacing %YOUR_NAME% with John Smith... + +Ψ Created a new keyboard called mycoolkeeb. +Ψ To start working on things, `cd` into keyboards/mycoolkeeb, +Ψ or open the directory in your preferred text editor. ``` これにより、新しいキーボードをサポートするために必要なすべてのファイルが作成され、デフォルト値で設定が入力されます。あとはあなたのキーボード用にカスタマイズするだけです。 diff --git a/lib/python/qmk/cli/new/keyboard.py b/lib/python/qmk/cli/new/keyboard.py index ae4445ca48..9e4232679d 100644 --- a/lib/python/qmk/cli/new/keyboard.py +++ b/lib/python/qmk/cli/new/keyboard.py @@ -1,11 +1,142 @@ -"""This script automates the creation of keyboards. +"""This script automates the creation of new keyboard directories using a starter template. """ +from datetime import date +import fileinput +from pathlib import Path +import re +import shutil + +from qmk.commands import git_get_username +import qmk.path from milc import cli +from milc.questions import choice, question + +KEYBOARD_TYPES = ['avr', 'ps2avrgb'] + + +def keyboard_name(name): + """Callable for argparse validation. + """ + if not validate_keyboard_name(name): + raise ValueError + return name -@cli.subcommand('Creates a new keyboard') +def validate_keyboard_name(name): + """Returns True if the given keyboard name contains only lowercase a-z, 0-9 and underscore characters. + """ + regex = re.compile(r'^[a-z0-9][a-z0-9/_]+$') + return bool(regex.match(name)) + + +@cli.argument('-kb', '--keyboard', help='Specify the name for the new keyboard directory', arg_only=True, type=keyboard_name) +@cli.argument('-t', '--type', help='Specify the keyboard type', arg_only=True, choices=KEYBOARD_TYPES) +@cli.argument('-u', '--username', help='Specify your username (default from Git config)', arg_only=True) +@cli.subcommand('Creates a new keyboard directory') def new_keyboard(cli): - """Creates a new keyboard + """Creates a new keyboard. """ - # TODO: replace this bodge to the existing script - cli.run(['util/new_keyboard.sh'], stdin=None, capture_output=False) + cli.log.info('{style_bright}Generating a new QMK keyboard directory{style_normal}') + cli.echo('') + + # Get keyboard name + new_keyboard_name = None + while not new_keyboard_name: + new_keyboard_name = cli.args.keyboard if cli.args.keyboard else question('Keyboard Name:') + if not validate_keyboard_name(new_keyboard_name): + cli.log.error('Keyboard names must contain only {fg_cyan}lowercase a-z{fg_reset}, {fg_cyan}0-9{fg_reset}, and {fg_cyan}_{fg_reset}! Please choose a different name.') + + # Exit if passed by arg + if cli.args.keyboard: + return False + + new_keyboard_name = None + continue + + keyboard_path = qmk.path.keyboard(new_keyboard_name) + if keyboard_path.exists(): + cli.log.error(f'Keyboard {{fg_cyan}}{new_keyboard_name}{{fg_reset}} already exists! Please choose a different name.') + + # Exit if passed by arg + if cli.args.keyboard: + return False + + new_keyboard_name = None + + # Get keyboard type + keyboard_type = cli.args.type if cli.args.type else choice('Keyboard Type:', KEYBOARD_TYPES, default=0) + + # Get username + user_name = None + while not user_name: + user_name = question('Your Name:', default=find_user_name()) + + if not user_name: + cli.log.error('You didn\'t provide a username, and we couldn\'t find one set in your QMK or Git configs. Please try again.') + + # Exit if passed by arg + if cli.args.username: + return False + + # Copy all the files + copy_templates(keyboard_type, keyboard_path) + + # Replace all the placeholders + keyboard_basename = keyboard_path.name + replacements = [ + ('%YEAR%', str(date.today().year)), + ('%KEYBOARD%', keyboard_basename), + ('%YOUR_NAME%', user_name), + ] + filenames = [ + keyboard_path / 'config.h', + keyboard_path / 'info.json', + keyboard_path / 'readme.md', + keyboard_path / f'{keyboard_basename}.c', + keyboard_path / f'{keyboard_basename}.h', + keyboard_path / 'keymaps/default/readme.md', + keyboard_path / 'keymaps/default/keymap.c', + ] + replace_placeholders(replacements, filenames) + + cli.echo('') + cli.log.info(f'{{fg_green}}Created a new keyboard called {{fg_cyan}}{new_keyboard_name}{{fg_green}}.{{fg_reset}}') + cli.log.info(f'To start working on things, `cd` into {{fg_cyan}}{keyboard_path}{{fg_reset}},') + cli.log.info('or open the directory in your preferred text editor.') + + +def find_user_name(): + if cli.args.username: + return cli.args.username + elif cli.config.user.name: + return cli.config.user.name + else: + return git_get_username() + + +def copy_templates(keyboard_type, keyboard_path): + """Copies the template files from quantum/template to the new keyboard directory. + """ + template_base_path = Path('quantum/template') + keyboard_basename = keyboard_path.name + + cli.log.info('Copying base template files...') + shutil.copytree(template_base_path / 'base', keyboard_path) + + cli.log.info(f'Copying {{fg_cyan}}{keyboard_type}{{fg_reset}} template files...') + shutil.copytree(template_base_path / keyboard_type, keyboard_path, dirs_exist_ok=True) + + cli.log.info(f'Renaming {{fg_cyan}}keyboard.[ch]{{fg_reset}} to {{fg_cyan}}{keyboard_basename}.[ch]{{fg_reset}}...') + shutil.move(keyboard_path / 'keyboard.c', keyboard_path / f'{keyboard_basename}.c') + shutil.move(keyboard_path / 'keyboard.h', keyboard_path / f'{keyboard_basename}.h') + + +def replace_placeholders(replacements, filenames): + """Replaces the given placeholders in each template file. + """ + for replacement in replacements: + cli.log.info(f'Replacing {{fg_cyan}}{replacement[0]}{{fg_reset}} with {{fg_cyan}}{replacement[1]}{{fg_reset}}...') + + with fileinput.input(files=filenames, inplace=True) as file: + for line in file: + print(line.replace(replacement[0], replacement[1]), end='') diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 8ff8501bf6..8c66228b2b 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -240,6 +240,15 @@ def parse_configurator_json(configurator_file): return user_keymap +def git_get_username(): + """Retrieves user's username from Git config, if set. + """ + git_username = cli.run(['git', 'config', '--get', 'user.name']) + + if git_username.returncode == 0 and git_username.stdout: + return git_username.stdout.strip() + + def git_check_repo(): """Checks that the .git directory exists inside QMK_HOME. diff --git a/util/new_keyboard.sh b/util/new_keyboard.sh index d0db23bb42..87b7cde0c9 100755 --- a/util/new_keyboard.sh +++ b/util/new_keyboard.sh @@ -129,6 +129,12 @@ if [ ! -d "quantum" ]; then exit 1 fi +echo_bold "########################################" +echo_bold "# NOTICE #" +echo_bold "# This script has been deprecated. #" +echo_bold "# Please use qmk new-keyboard instead. #" +echo_bold "########################################" +echo echo_bold "Generating a new QMK keyboard directory" echo -- cgit v1.2.3 From 3b28178deb7a42a2df8c19b9bf190bb5da732b33 Mon Sep 17 00:00:00 2001 From: ruro Date: Wed, 18 Aug 2021 01:46:59 +0300 Subject: `--parallel` improvements (#13800) * improve make parallel jobs support * document the --parallel option * disable the output-sync for interactive targets --- docs/cli_commands.md | 17 ++++++++++++++--- lib/python/qmk/cli/compile.py | 2 +- lib/python/qmk/cli/flash.py | 2 +- lib/python/qmk/cli/multibuild.py | 6 +++--- lib/python/qmk/commands.py | 23 ++++++++++++++++++++--- message.mk | 4 ++++ platforms/arm_atsam/flash.mk | 2 +- platforms/avr/flash.mk | 14 +++++++------- platforms/chibios/flash.mk | 10 +++++----- 9 files changed, 56 insertions(+), 24 deletions(-) (limited to 'lib/python/qmk/commands.py') diff --git a/docs/cli_commands.md b/docs/cli_commands.md index c7468eb5e7..8fa7ad41dc 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -17,7 +17,7 @@ qmk compile [-c] **Usage for Keymaps**: ``` -qmk compile [-c] [-e =] -kb -km +qmk compile [-c] [-e =] [-j ] -kb -km ``` **Usage in Keyboard Directory**: @@ -73,6 +73,17 @@ $ qmk compile -kb dz60 ... ``` +**Parallel Compilation**: + +It is possible to speed up compilation by adding the `-j`/`--parallel` flag. +``` +qmk compile -j -kb +``` +The `num_jobs` argument determines the maximum number of jobs that can be used. Setting it to zero will enable parallel compilation without limiting the maximum number of jobs. +``` +qmk compile -j 0 -kb +``` + ## `qmk flash` This command is similar to `qmk compile`, but can also target a bootloader. The bootloader is optional, and is set to `:flash` by default. To specify a different bootloader, use `-bl `. Visit the [Flashing Firmware](flashing.md) guide for more details of the available bootloaders. @@ -82,13 +93,13 @@ This command is directory aware. It will automatically fill in KEYBOARD and/or K **Usage for Configurator Exports**: ``` -qmk flash [-bl ] [-c] [-e =] +qmk flash [-bl ] [-c] [-e =] [-j ] ``` **Usage for Keymaps**: ``` -qmk flash -kb -km [-bl ] [-c] [-e =] +qmk flash -kb -km [-bl ] [-c] [-e =] [-j ] ``` **Listing the Bootloaders** diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 7a45e77214..acbd778649 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -18,7 +18,7 @@ from qmk.keymap import keymap_completer @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.subcommand('Compile a QMK Firmware.') diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index 1b2932a5b2..c2d9e09c69 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -38,7 +38,7 @@ def print_bootloader_help(): @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.subcommand('QMK Flash.') diff --git a/lib/python/qmk/cli/multibuild.py b/lib/python/qmk/cli/multibuild.py index bdb0b493c8..85ed0fa1e9 100755 --- a/lib/python/qmk/cli/multibuild.py +++ b/lib/python/qmk/cli/multibuild.py @@ -10,7 +10,7 @@ from subprocess import DEVNULL from milc import cli from qmk.constants import QMK_FIRMWARE -from qmk.commands import _find_make +from qmk.commands import _find_make, get_make_parallel_args import qmk.keyboard import qmk.keymap @@ -28,7 +28,7 @@ def _is_split(keyboard_name): return True if 'SPLIT_KEYBOARD' in rules_mk and rules_mk['SPLIT_KEYBOARD'].lower() == 'yes' else False -@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on the supplied value in rules.mk. Supported format is 'SPLIT_KEYBOARD=yes'. May be passed multiple times.") @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.") @@ -80,7 +80,7 @@ all: {keyboard_safe}_binary ) # yapf: enable - cli.run([make_cmd, '-j', str(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) + cli.run([make_cmd, *get_make_parallel_args(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL) # Check for failures failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')] diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 8c66228b2b..421453d837 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -52,7 +52,7 @@ def create_make_target(target, parallel=1, **env_vars): for key, value in env_vars.items(): env.append(f'{key}={value}') - return [make_cmd, '-j', str(parallel), *env, target] + return [make_cmd, *get_make_parallel_args(parallel), *env, target] def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars): @@ -112,6 +112,24 @@ def get_git_version(current_time, repo_dir='.', check_dir='.'): return current_time +def get_make_parallel_args(parallel=1): + """Returns the arguments for running the specified number of parallel jobs. + """ + parallel_args = [] + + if int(parallel) <= 0: + # 0 or -1 means -j without argument (unlimited jobs) + parallel_args.append('--jobs') + else: + parallel_args.append('--jobs=' + str(parallel)) + + if int(parallel) != 1: + # If more than 1 job is used, synchronize parallel output by target + parallel_args.append('--output-sync=target') + + return parallel_args + + def create_version_h(skip_git=False, skip_all=False): """Generate version.h contents """ @@ -185,8 +203,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va make_command.append('-s') make_command.extend([ - '-j', - str(parallel), + *get_make_parallel_args(parallel), '-r', '-R', '-f', diff --git a/message.mk b/message.mk index dcd47e5501..419e3f5f31 100644 --- a/message.mk +++ b/message.mk @@ -36,6 +36,10 @@ PRINT_OK = $(SILENT) || printf " $(OK_STRING)" | $(AWK_STATUS) BUILD_CMD = LOG=$$($(CMD) 2>&1) ; if [ $$? -gt 0 ]; then $(PRINT_ERROR); elif [ "$$LOG" != "" ] ; then $(PRINT_WARNING); else $(PRINT_OK); fi; MAKE_MSG_FORMAT = $(AWK) '{ printf "%-118s", $$0;}' +# The UNSYNC_OUTPUT_CMD command disables the `--output-sync` for the current command, if the `--output-sync` granularity is `target` or lower. +# This is achieved by telling make to treat the current command as if it invokes a recursive make subcommand (as if by calling `$(MAKE)`). +UNSYNC_OUTPUT_CMD = +true + # Define Messages # English MSG_ERRORS_NONE = Errors: none diff --git a/platforms/arm_atsam/flash.mk b/platforms/arm_atsam/flash.mk index 444ba77f53..f31d4b4d95 100644 --- a/platforms/arm_atsam/flash.mk +++ b/platforms/arm_atsam/flash.mk @@ -5,7 +5,7 @@ flash: bin ifneq ($(strip $(PROGRAM_CMD)),) - $(PROGRAM_CMD) + $(UNSYNC_OUTPUT_CMD) && $(PROGRAM_CMD) else $(PRINT_OK); $(SILENT) || printf "$(MSG_FLASH_ARCH)" endif diff --git a/platforms/avr/flash.mk b/platforms/avr/flash.mk index e679bf0ace..8eef5fbb03 100644 --- a/platforms/avr/flash.mk +++ b/platforms/avr/flash.mk @@ -161,19 +161,19 @@ hid_bootloader: $(BUILD_DIR)/$(TARGET).hex check-size cpfirmware flash: $(BUILD_DIR)/$(TARGET).hex check-size cpfirmware ifneq ($(strip $(PROGRAM_CMD)),) - $(PROGRAM_CMD) + $(UNSYNC_OUTPUT_CMD) && $(PROGRAM_CMD) else ifeq ($(strip $(BOOTLOADER)), caterina) - $(call EXEC_AVRDUDE) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_AVRDUDE) else ifeq ($(strip $(BOOTLOADER)), halfkay) - $(call EXEC_TEENSY) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_TEENSY) else ifeq (dfu,$(findstring dfu,$(BOOTLOADER))) - $(call EXEC_DFU) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_DFU) else ifeq ($(strip $(BOOTLOADER)), USBasp) - $(call EXEC_USBASP) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_USBASP) else ifeq ($(strip $(BOOTLOADER)), bootloadHID) - $(call EXEC_BOOTLOADHID) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_BOOTLOADHID) else ifeq ($(strip $(BOOTLOADER)), qmk-hid) - $(call EXEC_HID_LUFA) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_HID_LUFA) else $(PRINT_OK); $(SILENT) || printf "$(MSG_FLASH_BOOTLOADER)" endif diff --git a/platforms/chibios/flash.mk b/platforms/chibios/flash.mk index 46aff6e4b0..c0b32c2f2b 100644 --- a/platforms/chibios/flash.mk +++ b/platforms/chibios/flash.mk @@ -73,15 +73,15 @@ teensy: $(BUILD_DIR)/$(TARGET).hex cpfirmware sizeafter flash: $(BUILD_DIR)/$(TARGET).bin cpfirmware sizeafter ifneq ($(strip $(PROGRAM_CMD)),) - $(PROGRAM_CMD) + $(UNSYNC_OUTPUT_CMD) && $(PROGRAM_CMD) else ifeq ($(strip $(BOOTLOADER)),kiibohd) - $(call EXEC_DFU_UTIL) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_DFU_UTIL) else ifeq ($(strip $(MCU_FAMILY)),KINETIS) - $(call EXEC_TEENSY) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_TEENSY) else ifeq ($(strip $(MCU_FAMILY)),MIMXRT1062) - $(call EXEC_TEENSY) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_TEENSY) else ifeq ($(strip $(MCU_FAMILY)),STM32) - $(call EXEC_DFU_UTIL) + $(UNSYNC_OUTPUT_CMD) && $(call EXEC_DFU_UTIL) else $(PRINT_OK); $(SILENT) || printf "$(MSG_FLASH_BOOTLOADER)" endif -- cgit v1.2.3