1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
"""Compile all keyboards.
This will compile everything in parallel, for testing purposes.
"""
import os
from typing import List
from pathlib import Path
from subprocess import DEVNULL
from milc import cli
import shlex
from qmk.constants import QMK_FIRMWARE
from qmk.commands import find_make, get_make_parallel_args, build_environment
from qmk.search import search_keymap_targets, search_make_targets
from qmk.build_targets import BuildTarget, JsonKeymapBuildTarget
from qmk.util import maybe_exit_config
def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool, no_temp: bool, parallel: int, print_failures: bool, **env):
if len(targets) == 0:
return
os.environ.setdefault('SKIP_SCHEMA_VALIDATION', '1')
make_cmd = find_make()
builddir = Path(QMK_FIRMWARE) / '.build'
makefile = builddir / 'parallel_kb_builds.mk'
if dry_run:
cli.log.info('Compilation targets:')
for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
extra_args = ' '.join([f"-e {shlex.quote(f'{k}={v}')}" for k, v in target.extra_args.items()])
cli.log.info(f"{{fg_cyan}}qmk compile -kb {target.keyboard} -km {target.keymap} {extra_args}{{fg_reset}}")
else:
if clean:
cli.run([make_cmd, 'clean'], capture_output=False, stdin=DEVNULL)
builddir.mkdir(parents=True, exist_ok=True)
with open(makefile, "w") as f:
# yapf: disable
f.write(
f"""\
# This file is auto-generated by qmk mass-compile
# Do not edit this file directly.
all: print_failures
.PHONY: all_targets print_failures
print_failures: all_targets
"""# noqa
)
if print_failures:
f.write(
f"""\
@for f in $$(ls .build/failed.log.{os.getpid()}.* 2>/dev/null | sort); do \\
echo; \\
echo "======================================================================================"; \\
echo "Failed build log: $$f"; \\
echo "------------------------------------------------------"; \\
cat $$f; \\
echo "------------------------------------------------------"; \\
done
"""# noqa
)
# yapf: enable
for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
keyboard_name = target.keyboard
keymap_name = target.keymap
keyboard_safe = keyboard_name.replace('/', '_')
target_filename = target.target_name(**env)
target.configure(parallel=1) # We ignore parallelism on a per-build basis as we defer to the parent make invocation
target.prepare_build(**env) # If we've got json targets, allow them to write out any extra info to .build before we kick off `make`
command = target.compile_command(**env)
command[0] = '+@$(MAKE)' # Override the make so that we can use jobserver to handle parallelism
extra_args = '_'.join([f"{k}_{v}" for k, v in target.extra_args.items()])
build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
target_suffix = ''
if len(extra_args) > 0:
build_log += f".{extra_args}"
failed_log += f".{extra_args}"
target_suffix = f"_{extra_args}"
# yapf: disable
f.write(
f"""\
.PHONY: {target_filename}{target_suffix}_binary
all_targets: {target_filename}{target_suffix}_binary
{target_filename}{target_suffix}_binary:
@rm -f "{build_log}" || true
@echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}"
{' '.join(command)} \\
>>"{build_log}" 2>&1 \\
|| cp "{build_log}" "{failed_log}"
@{{ grep '\\[ERRORS\\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \\e[1;31m[ERRORS]\\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
|| {{ grep '\\[WARNINGS\\]' "{build_log}" >/dev/null 2>&1 && printf "Build %-64s \\e[1;33m[WARNINGS]\\e[0m\\n" "{keyboard_name}:{keymap_name}" ; }} \\
|| printf "Build %-64s \\e[1;32m[OK]\\e[0m\\n" "{keyboard_name}:{keymap_name}"
@rm -f "{build_log}" || true
"""# noqa
)
# yapf: enable
if no_temp:
# yapf: disable
f.write(
f"""\
@rm -rf "{QMK_FIRMWARE}/.build/{target_filename}.elf" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/{target_filename}.map" 2>/dev/null || true
@rm -rf "{QMK_FIRMWARE}/.build/obj_{target_filename}" || true
"""# noqa
)
# yapf: enable
f.write('\n')
cli.run([find_make(), *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
# Check for failures
failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')]
if len(failures) > 0:
return False
@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap> to compile in parallel. Specifying this overrides all other target search options.")
@cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.")
@cli.argument('-p', '--print-failures', arg_only=True, action='store_true', help="Print failed builds.")
@cli.argument(
'-f',
'--filter',
arg_only=True,
action='append',
default=[],
help= # noqa: `format-python` and `pytest` don't agree here.
"Filter the list of keyboards based on the supplied value in rules.mk. Matches info.json structure, and accepts the formats 'features.rgblight=true' or 'exists(matrix_pins.direct)'. May be passed multiple times, all filters need to match. Value may include wildcards such as '*' and '?'." # noqa: `format-python` and `pytest` don't agree here.
)
@cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@cli.subcommand('Compile QMK Firmware for all keyboards.', hidden=False if cli.config.user.developer else True)
def mass_compile(cli):
"""Compile QMK Firmware against all keyboards.
"""
maybe_exit_config(should_exit=False, should_reraise=True)
if len(cli.args.builds) > 0:
json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
targets = search_make_targets(make_like_targets)
targets.extend([JsonKeymapBuildTarget(e) for e in json_like_targets])
else:
targets = search_keymap_targets([('all', cli.config.mass_compile.keymap)], cli.args.filter)
return mass_compile_targets(targets, cli.args.clean, cli.args.dry_run, cli.args.no_temp, cli.config.mass_compile.parallel, cli.args.print_failures, **build_environment(cli.args.env))
|