`--parallel` improvements (#13800)
* improve make parallel jobs support * document the --parallel option * disable the output-sync for interactive targets
This commit is contained in:
ruro
GitHub
parent
999391f3ae
commit
3b28178deb
|
|
@ -17,7 +17,7 @@ qmk compile [-c] <configuratorExport.json>
|
|||
**Usage for Keymaps**:
|
||||
|
||||
```
|
||||
qmk compile [-c] [-e <var>=<value>] -kb <keyboard_name> -km <keymap_name>
|
||||
qmk compile [-c] [-e <var>=<value>] [-j <num_jobs>] -kb <keyboard_name> -km <keymap_name>
|
||||
```
|
||||
|
||||
**Usage in Keyboard Directory**:
|
||||
|
|
@ -73,6 +73,17 @@ $ qmk compile -kb dz60
|
|||
...
|
||||
```
|
||||
|
||||
**Parallel Compilation**:
|
||||
|
||||
It is possible to speed up compilation by adding the `-j`/`--parallel` flag.
|
||||
```
|
||||
qmk compile -j <num_jobs> -kb <keyboard_name>
|
||||
```
|
||||
The `num_jobs` argument determines the maximum number of jobs that can be used. Setting it to zero will enable parallel compilation without limiting the maximum number of jobs.
|
||||
```
|
||||
qmk compile -j 0 -kb <keyboard_name>
|
||||
```
|
||||
|
||||
## `qmk flash`
|
||||
|
||||
This command is similar to `qmk compile`, but can also target a bootloader. The bootloader is optional, and is set to `:flash` by default. To specify a different bootloader, use `-bl <bootloader>`. Visit the [Flashing Firmware](flashing.md) guide for more details of the available bootloaders.
|
||||
|
|
@ -82,13 +93,13 @@ This command is directory aware. It will automatically fill in KEYBOARD and/or K
|
|||
**Usage for Configurator Exports**:
|
||||
|
||||
```
|
||||
qmk flash [-bl <bootloader>] [-c] [-e <var>=<value>] <configuratorExport.json>
|
||||
qmk flash [-bl <bootloader>] [-c] [-e <var>=<value>] [-j <num_jobs>] <configuratorExport.json>
|
||||
```
|
||||
|
||||
**Usage for Keymaps**:
|
||||
|
||||
```
|
||||
qmk flash -kb <keyboard_name> -km <keymap_name> [-bl <bootloader>] [-c] [-e <var>=<value>]
|
||||
qmk flash -kb <keyboard_name> -km <keymap_name> [-bl <bootloader>] [-c] [-e <var>=<value>] [-j <num_jobs>]
|
||||
```
|
||||
|
||||
**Listing the Bootloaders**
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from qmk.keymap import keymap_completer
|
|||
@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
|
||||
@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
|
||||
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
|
||||
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
|
||||
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
|
||||
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
|
||||
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
|
||||
@cli.subcommand('Compile a QMK Firmware.')
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ def print_bootloader_help():
|
|||
@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
|
||||
@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
|
||||
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
|
||||
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
|
||||
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
|
||||
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
|
||||
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
|
||||
@cli.subcommand('QMK Flash.')
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from subprocess import DEVNULL
|
|||
from milc import cli
|
||||
|
||||
from qmk.constants import QMK_FIRMWARE
|
||||
from qmk.commands import _find_make
|
||||
from qmk.commands import _find_make, get_make_parallel_args
|
||||
import qmk.keyboard
|
||||
import qmk.keymap
|
||||
|
||||
|
|
@ -28,7 +28,7 @@ def _is_split(keyboard_name):
|
|||
return True if 'SPLIT_KEYBOARD' in rules_mk and rules_mk['SPLIT_KEYBOARD'].lower() == 'yes' else False
|
||||
|
||||
|
||||
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
|
||||
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
|
||||
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
|
||||
@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on the supplied value in rules.mk. Supported format is 'SPLIT_KEYBOARD=yes'. May be passed multiple times.")
|
||||
@cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.")
|
||||
|
|
@ -80,7 +80,7 @@ all: {keyboard_safe}_binary
|
|||
)
|
||||
# yapf: enable
|
||||
|
||||
cli.run([make_cmd, '-j', str(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
|
||||
cli.run([make_cmd, *get_make_parallel_args(cli.args.parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
|
||||
|
||||
# Check for failures
|
||||
failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')]
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ def create_make_target(target, parallel=1, **env_vars):
|
|||
for key, value in env_vars.items():
|
||||
env.append(f'{key}={value}')
|
||||
|
||||
return [make_cmd, '-j', str(parallel), *env, target]
|
||||
return [make_cmd, *get_make_parallel_args(parallel), *env, target]
|
||||
|
||||
|
||||
def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
|
||||
|
|
@ -112,6 +112,24 @@ def get_git_version(current_time, repo_dir='.', check_dir='.'):
|
|||
return current_time
|
||||
|
||||
|
||||
def get_make_parallel_args(parallel=1):
|
||||
"""Returns the arguments for running the specified number of parallel jobs.
|
||||
"""
|
||||
parallel_args = []
|
||||
|
||||
if int(parallel) <= 0:
|
||||
# 0 or -1 means -j without argument (unlimited jobs)
|
||||
parallel_args.append('--jobs')
|
||||
else:
|
||||
parallel_args.append('--jobs=' + str(parallel))
|
||||
|
||||
if int(parallel) != 1:
|
||||
# If more than 1 job is used, synchronize parallel output by target
|
||||
parallel_args.append('--output-sync=target')
|
||||
|
||||
return parallel_args
|
||||
|
||||
|
||||
def create_version_h(skip_git=False, skip_all=False):
|
||||
"""Generate version.h contents
|
||||
"""
|
||||
|
|
@ -185,8 +203,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va
|
|||
make_command.append('-s')
|
||||
|
||||
make_command.extend([
|
||||
'-j',
|
||||
str(parallel),
|
||||
*get_make_parallel_args(parallel),
|
||||
'-r',
|
||||
'-R',
|
||||
'-f',
|
||||
|
|
|
|||
|
|
@ -36,6 +36,10 @@ PRINT_OK = $(SILENT) || printf " $(OK_STRING)" | $(AWK_STATUS)
|
|||
BUILD_CMD = LOG=$$($(CMD) 2>&1) ; if [ $$? -gt 0 ]; then $(PRINT_ERROR); elif [ "$$LOG" != "" ] ; then $(PRINT_WARNING); else $(PRINT_OK); fi;
|
||||
MAKE_MSG_FORMAT = $(AWK) '{ printf "%-118s", $$0;}'
|
||||
|
||||
# The UNSYNC_OUTPUT_CMD command disables the `--output-sync` for the current command, if the `--output-sync` granularity is `target` or lower.
|
||||
# This is achieved by telling make to treat the current command as if it invokes a recursive make subcommand (as if by calling `$(MAKE)`).
|
||||
UNSYNC_OUTPUT_CMD = +true
|
||||
|
||||
# Define Messages
|
||||
# English
|
||||
MSG_ERRORS_NONE = Errors: none
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
flash: bin
|
||||
ifneq ($(strip $(PROGRAM_CMD)),)
|
||||
$(PROGRAM_CMD)
|
||||
$(UNSYNC_OUTPUT_CMD) && $(PROGRAM_CMD)
|
||||
else
|
||||
$(PRINT_OK); $(SILENT) || printf "$(MSG_FLASH_ARCH)"
|
||||
endif
|
||||
|
|
|
|||
|
|
@ -161,19 +161,19 @@ hid_bootloader: $(BUILD_DIR)/$(TARGET).hex check-size cpfirmware
|
|||
|
||||
flash: $(BUILD_DIR)/$(TARGET).hex check-size cpfirmware
|
||||
ifneq ($(strip $(PROGRAM_CMD)),)
|
||||
$(PROGRAM_CMD)
|
||||
$(UNSYNC_OUTPUT_CMD) && $(PROGRAM_CMD)
|
||||
else ifeq ($(strip $(BOOTLOADER)), caterina)
|
||||
$(call EXEC_AVRDUDE)
|
||||
$(UNSYNC_OUTPUT_CMD) && $(call EXEC_AVRDUDE)
|
||||
else ifeq ($(strip $(BOOTLOADER)), halfkay)
|
||||
$(call EXEC_TEENSY)
|
||||
$(UNSYNC_OUTPUT_CMD) && $(call EXEC_TEENSY)
|
||||
else ifeq (dfu,$(findstring dfu,$(BOOTLOADER)))
|
||||
$(call EXEC_DFU)
|
||||
$(UNSYNC_OUTPUT_CMD) && $(call EXEC_DFU)
|
||||
else ifeq ($(strip $(BOOTLOADER)), USBasp)
|
||||
$(call EXEC_USBASP)
|
||||
$(UNSYNC_OUTPUT_CMD) && $(call EXEC_USBASP)
|
||||
else ifeq ($(strip $(BOOTLOADER)), bootloadHID)
|
||||
$(call EXEC_BOOTLOADHID)
|
||||
$(UNSYNC_OUTPUT_CMD) && $(call EXEC_BOOTLOADHID)
|
||||
else ifeq ($(strip $(BOOTLOADER)), qmk-hid)
|
||||
$(call EXEC_HID_LUFA)
|
||||
$(UNSYNC_OUTPUT_CMD) && $(call EXEC_HID_LUFA)
|
||||
else
|
||||
$(PRINT_OK); $(SILENT) || printf "$(MSG_FLASH_BOOTLOADER)"
|
||||
endif
|
||||
|
|
|
|||
|
|
@ -73,15 +73,15 @@ teensy: $(BUILD_DIR)/$(TARGET).hex cpfirmware sizeafter
|
|||
|
||||
flash: $(BUILD_DIR)/$(TARGET).bin cpfirmware sizeafter
|
||||
ifneq ($(strip $(PROGRAM_CMD)),)
|
||||
$(PROGRAM_CMD)
|
||||
$(UNSYNC_OUTPUT_CMD) && $(PROGRAM_CMD)
|
||||
else ifeq ($(strip $(BOOTLOADER)),kiibohd)
|
||||
$(call EXEC_DFU_UTIL)
|
||||
$(UNSYNC_OUTPUT_CMD) && $(call EXEC_DFU_UTIL)
|
||||
else ifeq ($(strip $(MCU_FAMILY)),KINETIS)
|
||||
$(call EXEC_TEENSY)
|
||||
$(UNSYNC_OUTPUT_CMD) && $(call EXEC_TEENSY)
|
||||
else ifeq ($(strip $(MCU_FAMILY)),MIMXRT1062)
|
||||
$(call EXEC_TEENSY)
|
||||
$(UNSYNC_OUTPUT_CMD) && $(call EXEC_TEENSY)
|
||||
else ifeq ($(strip $(MCU_FAMILY)),STM32)
|
||||
$(call EXEC_DFU_UTIL)
|
||||
$(UNSYNC_OUTPUT_CMD) && $(call EXEC_DFU_UTIL)
|
||||
else
|
||||
$(PRINT_OK); $(SILENT) || printf "$(MSG_FLASH_BOOTLOADER)"
|
||||
endif
|
||||
|
|
|
|||
Loading…
Reference in New Issue