During the porting of the layer for my SoM from the pyro branch to the dunfell one, I've encountered some problems related to the COMPATIBLE_MACHINE list in my recipes (BitBake says that there is no recipe compatible with my machine).
In order to further investigate this issue, I tried to read the actual value of the MACHINEOVERRIDES variable using the bitbake -e command. However, I don't think this list is complete, because watching the bitbake -e output I can see that there are other variable expansions which are used to form the value of the general OVERRIDES variable.
This is the output of the bitbake -e | grep OVERRIDES command run from my build environment:
# $DISTROOVERRIDES [3 operations]
DISTROOVERRIDES="fslc"
# $FILESOVERRIDES [2 operations]
# "${TRANSLATED_TARGET_ARCH}:${MACHINEOVERRIDES}:${DISTROOVERRIDES}"
# [doc] "A subset of OVERRIDES used by the OpenEmbedded build system for creating FILESPATH."
# "${TRANSLATED_TARGET_ARCH}:${MACHINEOVERRIDES}:${DISTROOVERRIDES}"
FILESOVERRIDES="arm:isiot:armv7ve:use-mainline-bsp:isiot-geamx6ul:fslc"
# $MACHINEOVERRIDES [14 operations]
# "PRISTINE_MACHINEOVERRIDES"
MACHINEOVERRIDES="isiot:armv7ve:use-mainline-bsp:isiot-geamx6ul"
# $MACHINEOVERRIDES_EXTENDER_FILTER_OUT
MACHINEOVERRIDES_EXTENDER_FILTER_OUT=" imx mx6 mx6q mx6dl mx6sx mx6sl mx6sll mx6ul mx6ull mx7 mx7d mx7ulp mx8 mx8qm mx8mm mx8mn mx8mq mx8qxp "
# $MACHINEOVERRIDES_EXTENDER_FILTER_OUT_use-mainline-bsp
MACHINEOVERRIDES_EXTENDER_FILTER_OUT_use-mainline-bsp=" imx mx6 mx6q mx6dl mx6sx mx6sl mx6sll mx6ul mx6ull mx7 mx7d mx7ulp mx8 mx8qm mx8mm mx8mn mx8mq mx8qxp "
# $MACHINEOVERRIDES_EXTENDER_mx25
MACHINEOVERRIDES_EXTENDER_mx25="use-mainline-bsp"
# $MACHINEOVERRIDES_EXTENDER_mx6dl
MACHINEOVERRIDES_EXTENDER_mx6dl="imxfbdev:imxpxp:imxipu:imxvpu:imxgpu:imxgpu2d:imxgpu3d:imxepdc"
# $MACHINEOVERRIDES_EXTENDER_mx6q
MACHINEOVERRIDES_EXTENDER_mx6q="imxfbdev:imxipu:imxvpu:imxgpu:imxgpu2d:imxgpu3d"
# $MACHINEOVERRIDES_EXTENDER_mx6sl
MACHINEOVERRIDES_EXTENDER_mx6sl="imxfbdev:imxpxp:imxgpu:imxgpu2d:imxepdc"
# $MACHINEOVERRIDES_EXTENDER_mx6sll
MACHINEOVERRIDES_EXTENDER_mx6sll="imxfbdev:imxpxp:imxepdc"
# $MACHINEOVERRIDES_EXTENDER_mx6sx
MACHINEOVERRIDES_EXTENDER_mx6sx="imxfbdev:imxpxp:imxgpu:imxgpu2d:imxgpu3d"
# $MACHINEOVERRIDES_EXTENDER_mx6ul
MACHINEOVERRIDES_EXTENDER_mx6ul="imxfbdev:imxpxp"
# $MACHINEOVERRIDES_EXTENDER_mx6ull
MACHINEOVERRIDES_EXTENDER_mx6ull="imxfbdev:imxpxp:imxepdc"
# $MACHINEOVERRIDES_EXTENDER_mx7d
MACHINEOVERRIDES_EXTENDER_mx7d="imxfbdev:imxpxp:imxepdc"
# $MACHINEOVERRIDES_EXTENDER_mx7ulp
MACHINEOVERRIDES_EXTENDER_mx7ulp="imxfbdev:imxpxp:imxgpu:imxgpu2d:imxgpu3d"
# $MACHINEOVERRIDES_EXTENDER_mx8mm
MACHINEOVERRIDES_EXTENDER_mx8mm="imxdrm:imxvpu:imxgpu:imxgpu2d:imxgpu3d"
# $MACHINEOVERRIDES_EXTENDER_mx8mn
MACHINEOVERRIDES_EXTENDER_mx8mn="imxdrm:imxgpu:imxgpu3d"
# $MACHINEOVERRIDES_EXTENDER_mx8mq
MACHINEOVERRIDES_EXTENDER_mx8mq="imxdrm:imxvpu:imxgpu:imxgpu3d"
# $MACHINEOVERRIDES_EXTENDER_mx8qm
MACHINEOVERRIDES_EXTENDER_mx8qm="imxdrm:imxdpu:imxgpu:imxgpu2d:imxgpu3d"
# $MACHINEOVERRIDES_EXTENDER_mx8qxp
MACHINEOVERRIDES_EXTENDER_mx8qxp="imxdrm:imxdpu:imxgpu:imxgpu2d:imxgpu3d"
# $OVERRIDES [2 operations]
# "${TARGET_OS}:${TRANSLATED_TARGET_ARCH}:pn-${PN}:${MACHINEOVERRIDES}:${DISTROOVERRIDES}:${CLASSOVERRIDE}${LIBCOVERRIDE}:forcevariable"
# [doc] "BitBake uses OVERRIDES to control what variables are overridden after BitBake parses recipes and configuration files."
# "${TARGET_OS}:${TRANSLATED_TARGET_ARCH}:pn-${PN}:${MACHINEOVERRIDES}:${DISTROOVERRIDES}:${CLASSOVERRIDE}${LIBCOVERRIDE}:forcevariable"
OVERRIDES="linux-gnueabi:arm:pn-defaultpkgname:isiot:armv7ve:use-mainline-bsp:isiot-geamx6ul:fslc:class-target:libc-glibc:forcevariable"
# $PRISTINE_MACHINEOVERRIDES [13 operations]
# rename from MACHINEOVERRIDES machine-overrides-extender.bbclass:49 [machine_overrides_extender_handler]
PRISTINE_MACHINEOVERRIDES="mx6:mx6ul:isiot:armv7ve:imx:use-mainline-bsp:isiot-geamx6ul"
# $SRC_URI_OVERRIDES_PACKAGE_ARCH
overrides = d.getVar('OVERRIDES').split(':')
msg = 'Recipe %s has PN of "%s" which is in OVERRIDES, this can result in unexpected behaviour.' % (d.getVar("FILE"), pn)
compat_machines = (d.getVar('MACHINEOVERRIDES') or "").split(":")
# unless the package sets SRC_URI_OVERRIDES_PACKAGE_ARCH=0
override = d.getVar('SRC_URI_OVERRIDES_PACKAGE_ARCH')
overrides = (":" + (d.getVar("FILESOVERRIDES") or "")).split(":")
overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + multilib
localdata.setVar("OVERRIDES", overrides)
overrides = d.getVar("OVERRIDES").split(":")
machine_overrides = (d.getVar('PRISTINE_MACHINEOVERRIDES') or '').split(':')
machine_overrides_filter_out += (d.getVar('MACHINEOVERRIDES_EXTENDER_FILTER_OUT_%s' % override) or '').split()
extender = d.getVar('MACHINEOVERRIDES_EXTENDER_%s' % override)
# so we can reprocess OVERRIDES if/as/when needed.
d.renameVar("MACHINEOVERRIDES", "PRISTINE_MACHINEOVERRIDES")
d.setVar("MACHINEOVERRIDES", "${#machine_overrides_extender(d)}")
localdata.setVar('OVERRIDES', pkg)
localdata.setVar('OVERRIDES', localdata.getVar('OVERRIDES') + ':' + pkg)
overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + item
localdata.setVar("OVERRIDES", overrides)
localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + pkg)
Is there a way to get the value of MACHINEOVERRIDES and OVERRIDES variables after those operations occur?
# so we can reprocess OVERRIDES if/as/when needed.
d.renameVar("MACHINEOVERRIDES", "PRISTINE_MACHINEOVERRIDES")
d.setVar("MACHINEOVERRIDES", "${#machine_overrides_extender(d)}")
localdata.setVar('OVERRIDES', pkg)
localdata.setVar('OVERRIDES', localdata.getVar('OVERRIDES') + ':' + pkg)
overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + item
localdata.setVar("OVERRIDES", overrides)
localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + pkg)
Thanks.
bitbake -e already shows you the variables after the operations.
MACHINEOVERRIDES="isiot:armv7ve:use-mainline-bsp:isiot-geamx6ul" and
OVERRIDES="linux-gnueabi:arm:pn-defaultpkgname:isiot:armv7ve:use-mainline-bsp:isiot-geamx6ul:fslc:class-target:libc-glibc:forcevariable"
bitbake -e first shows you how many operations act on a variable e.g. # $MACHINEOVERRIDES [14 operations] and where they are coming from and after that the expanded value.
You could use bitbake -e | less to browse the output instead of grepping it.
You can simply use :
bitbake -e <package_name> | grep ^MACHINEOVERRIDES=
bitbake -e <package_name> | grep ^OVERRIDES=
Related
I am noticing that all my rules request memory twice, one at a lower maximum than what I requested (mem_mb) and then what I actually requested (mem_gb). If I run the rules as localrules they do run faster. How can I make sure the default settings do not interfere?
resources: mem_mb=100, disk_mb=8620, tmpdir=/tmp/pop071.54835, partition=h24, qos=normal, mem_gb=100, time=120:00:00
The rules are as follows:
rule bwa_mem2_mem:
input:
R1 = "data/results/qc/{species}.{population}.{individual}_1.fq.gz",
R2 = "data/results/qc/{species}.{population}.{individual}_2.fq.gz",
R1_unp = "data/results/qc/{species}.{population}.{individual}_1_unp.fq.gz",
R2_unp = "data/results/qc/{species}.{population}.{individual}_2_unp.fq.gz",
idx= "data/results/genome/genome",
ref = "data/results/genome/genome.fa"
output:
bam = "data/results/mapped_reads/{species}.{population}.{individual}.bam",
log:
bwa ="logs/bwa_mem2/{species}.{population}.{individual}.log",
sam ="logs/samtools_view/{species}.{population}.{individual}.log",
benchmark:
"benchmark/bwa_mem2_mem/{species}.{population}.{individual}.tsv",
resources:
time = parameters["bwa_mem2"]["time"],
mem_gb = parameters["bwa_mem2"]["mem_gb"],
params:
extra = parameters["bwa_mem2"]["extra"],
tag = compose_rg_tag,
threads:
parameters["bwa_mem2"]["threads"],
shell:
"bwa-mem2 mem -t {threads} -R '{params.tag}' {params.extra} {input.idx} {input.R1} {input.R2} | "
"samtools sort -l 9 -o {output.bam} --reference {input.ref} --output-fmt CRAM -# {threads} /dev/stdin 2> {log.sam}"
and the config is:
cluster:
mkdir -p logs/{rule} && # change the log file to logs/slurm/{rule}
sbatch
--partition={resources.partition}
--time={resources.time}
--qos={resources.qos}
--cpus-per-task={threads}
--mem={resources.mem_gb}
--job-name=smk-{rule}-{wildcards}
--output=logs/{rule}/{rule}-{wildcards}-%j.out
--parsable # Required to pass job IDs to scancel
default-resources:
- partition=h24
- qos=normal
- mem_gb=100
- time="04:00:00"
restart-times: 3
max-jobs-per-second: 10
max-status-checks-per-second: 1
local-cores: 1
latency-wait: 60
jobs: 100
keep-going: True
rerun-incomplete: True
printshellcmds: True
scheduler: greedy
use-conda: True # Required to run with local conda enviroment
cluster-status: status-sacct.sh # Required to monitor the status of the submitted jobs
cluster-cancel: scancel # Required to cancel the jobs with Ctrl + C
cluster-cancel-nargs: 50
Cheers,
Angel
Right now there are two separate memory resource requirements:
mem_mb
mem_gb
From the perspective of snakemake these are different, so both will be passed to the cluster. A quick fix is to use the same units, e.g. if the resource really requires only 100 mb, then the default resource should be changed to:
default-resources:
- partition=h24
- qos=normal
- mem_mb=100
I want to add poetry to the existing project
to add a dependency from requirements.txt I ran the following command
poetry add $( cat requirements.txt )
In either case, I get the same error - shown below
.venv) bash-3.2$ poetry add $( cat requirements.txt )
Stack trace:
11 ~/.poetry/lib/poetry/_vendor/py3.8/clikit/console_application.py:123 in run
io = io_factory(
10 ~/.poetry/lib/poetry/console/config/application_config.py:221 in create_io
resolved_command = application.resolve_command(args)
9 ~/.poetry/lib/poetry/_vendor/py3.8/clikit/console_application.py:110 in resolve_command
return self._config.command_resolver.resolve(args, self)
8 ~/.poetry/lib/poetry/_vendor/py3.8/clikit/resolver/default_resolver.py:34 in resolve
return self.create_resolved_command(result)
7 ~/.poetry/lib/poetry/_vendor/py3.8/clikit/resolver/default_resolver.py:166 in create_resolved_command
if not result.is_parsable():
6 ~/.poetry/lib/poetry/_vendor/py3.8/clikit/resolver/resolve_result.py:43 in is_parsable
self._parse()
5 ~/.poetry/lib/poetry/_vendor/py3.8/clikit/resolver/resolve_result.py:49 in _parse
self._parsed_args = self._command.parse(self._raw_args)
4 ~/.poetry/lib/poetry/_vendor/py3.8/clikit/api/command/command.py:113 in parse
return self._config.args_parser.parse(args, self._args_format, lenient)
3 ~/.poetry/lib/poetry/_vendor/py3.8/clikit/args/default_args_parser.py:53 in parse
self._parse(args, _fmt, lenient)
2 ~/.poetry/lib/poetry/_vendor/py3.8/clikit/args/default_args_parser.py:103 in _parse
self._parse_short_option(token, tokens, fmt, lenient)
1 ~/.poetry/lib/poetry/_vendor/py3.8/clikit/args/default_args_parser.py:272 in _parse_short_option
self._add_short_option(name, None, tokens, fmt, lenient)
NoSuchOptionException
The "-r" option does not exist.
at ~/.poetry/lib/poetry/_vendor/py3.8/clikit/args/default_args_parser.py:349 in _add_short_option
345│ def _add_short_option(
346│ self, name, value, tokens, fmt, lenient
347│ ): # type: (str, Optional[str], List[str], ArgsFormat, bool) -> None
348│ if not fmt.has_option(name):
→ 349│ raise NoSuchOptionException(name)
350│
351│ self._add_long_option(
352│ fmt.get_option(name).long_name, value, tokens, fmt, lenient
353│ )
for reference here is my requirement.txt
# This file is autogenerated by pip-compile with python 3.8
# To update, run:
#
# pip-compile requirements.in
#
click==8.0.3
# via -r requirements.in
gevent==21.8.0
# via grequests
greenlet==1.1.2
# via gevent
grequests==0.6.0
# via -r requirements.in
idna==3.3
# via requests
iniconfig==1.1.1
# via pytest
Looks like It was reading comments in requirements.txt
simple just ignore the comments run the following
cat requirements.txt|grep -v '#'|xargs poetry add
I am building core-image-minimal for warrior branch. My device has atom processor so I have changed nehalem to atom in tune-corei7.inc file. My machine is set to intel-corei7-64. While generating core-image-minimal, I am facing following error:
NOTE: Installing complementary packages ...
NOTE: Running ['oe-pkgdata-util', '-p', '/home/panther2/warrior/build_panther1/tmp/pkgdata/panther1', 'glob', '/tmp/installed-pkgs03hhi936', '']
NOTE: Running intercept scripts:
NOTE: > Executing update_gio_module_cache intercept ...
NOTE: Exit code 1. Output:
+ [ True = False -a qemuwrapper-cross != nativesdk-qemuwrapper-cross ]
+ qemu-x86_64 -r 3.2.0 -cpu atom,check=false -E LD_LIBRARY_PATH=/home/panther2/warrior/build_panther1/tmp/work/panther1-poky-linux/core-image-minimal/1.0-r0/rootfs/usr/lib:/home/panther2/warrior/build_panther1/tmp/work/panther1-poky-linux/core-image-minimal/1.0-r0/rootfs/lib -L /home/panther2/warrior/build_panther1/tmp/work/panther1-poky-linux/core-image-minimal/1.0-r0/rootfs /home/panther2/warrior/build_panther1/tmp/work/panther1-poky-linux/core-image-minimal/1.0-r0/rootfs/usr/libexec/gio-querymodules /home/panther2/warrior/build_panther1/tmp/work/panther1-poky-linux/core-image-minimal/1.0-r0/rootfs/usr/lib/gio/modules/
unable to find CPU model 'atom'
ERROR: The postinstall intercept hook 'update_gio_module_cache' failed, details in /home/panther2/warrior/build_panther1/tmp/work/panther1-poky-linux/core-image-minimal/1.0-r0/temp/log.do_rootfs
ERROR:
DEBUG: Python function do_rootfs finished
ERROR: Function failed: do_rootfs
Any help here?
Thanks in advance..!
Edit : Attaching "tune-corei7.inc" file
# Settings for the GCC(1) cpu-type "atom":
#
# Intel atom CPU with 64-bit extensions, MMX, SSE, SSE2, SSE3, SSSE3, SSE4.1,
# SSE4.2 and POPCNT instruction set support.
#
# This tune is recommended for Intel atom and Silvermont (e.g. Bay Trail) CPUs
# (and beyond).
#
DEFAULTTUNE ?= "corei7-64"
# Include the previous tune to pull in PACKAGE_EXTRA_ARCHS
require conf/machine/include/tune-atom.inc
# Extra tune features
TUNEVALID[corei7] = "Enable corei7 specific processor optimizations"
TUNE_CCARGS .= "${#bb.utils.contains('TUNE_FEATURES', 'corei7', ' -march=atom -mtune=generic -mfpmath=sse -msse4.2', '', d)}"
# Extra tune selections
AVAILTUNES += "corei7-32"
TUNE_FEATURES_tune-corei7-32 = "${TUNE_FEATURES_tune-x86} corei7"
BASE_LIB_tune-corei7-32 = "lib"
TUNE_PKGARCH_tune-corei7-32 = "corei7-32"
PACKAGE_EXTRA_ARCHS_tune-corei7-32 = "${PACKAGE_EXTRA_ARCHS_tune-atom-32} corei7-32"
QEMU_EXTRAOPTIONS_corei7-32 = " -cpu nehalem,check=false"
AVAILTUNES += "corei7-64"
TUNE_FEATURES_tune-corei7-64 = "${TUNE_FEATURES_tune-x86-64} corei7"
BASE_LIB_tune-corei7-64 = "lib64"
TUNE_PKGARCH_tune-corei7-64 = "corei7-64"
PACKAGE_EXTRA_ARCHS_tune-corei7-64 = "${PACKAGE_EXTRA_ARCHS_tune-atom-64} corei7-64"
QEMU_EXTRAOPTIONS_corei7-64 = " -cpu nehalem,check=false"
AVAILTUNES += "corei7-64-x32"
TUNE_FEATURES_tune-corei7-64-x32 = "${TUNE_FEATURES_tune-x86-64-x32} corei7"
BASE_LIB_tune-corei7-64-x32 = "libx32"
TUNE_PKGARCH_tune-corei7-64-x32 = "corei7-64-x32"
PACKAGE_EXTRA_ARCHS_tune-corei7-64-x32 = "${PACKAGE_EXTRA_ARCHS_tune-atom-64-x32} corei7-64-x32"
QEMU_EXTRAOPTIONS_corei7-64-x32 = " -cpu nehalem,check=false"
I am working on snakemake to make a pipeline for my work.
My input files are (Example fastq files)
/project/ateeq/PROJECT/snakemake-example/raw_data/H667-1_R1.fastq.gz
/project/ateeq/PROJECT/snakemake-example/raw_data/H667-1_R2.fastq.gz
/project/ateeq/PROJECT/snakemake-example/raw_data/H667-2_R1.fastq.gz
/project/ateeq/PROJECT/snakemake-example/raw_data/H667-2_R2.fastq.gz
I have written the following code for trimming the data using FastP
"""
Author: Dave Amir
Affiliation: St Lukes
Aim: A simple Snakemake workflow to process paired-end stranded RNA-Seq.
Date: 11 June 2015
Run: snakemake -s snakefile
Latest modification:
- todo
"""
# This should be placed in the Snakefile.
##-----------------------------------------------##
## Working directory ##
## Adapt to your needs ##
##-----------------------------------------------##
BASE_DIR = "/project/ateeq/PROJECT"
WDIR = BASE_DIR + "/snakemake-example"
workdir: WDIR
#message("The current working directory is " + WDIR)
##--------------------------------------------------------------------------------------##
## Variables declaration
## Declaring some variables used by topHat and other tools...
## (GTF file, INDEX, chromosome length)
##--------------------------------------------------------------------------------------##
INDEX = BASE_DIR + "/ref_files/hg19/assembly/"
GTF = BASE_DIR + "/hg19/Hg19_CTAT_resource_lib/ref_annot.gtf"
CHR = BASE_DIR + "/static/humanhg19.annot.csv"
FASTA = BASE_DIR + "/ref_files/hg19/assembly/hg19.fasta"
##--------------------------------------------------------------------------------------##
## The list of samples to be processed
##--------------------------------------------------------------------------------------##
SAMPLES, = glob_wildcards("/project/ateeq/PROJECT/snakemake-example/raw_data/{smp}_R1.fastq.gz")
NB_SAMPLES = len(SAMPLES)
for smp in SAMPLES:
message:("Sample " + smp + " will be processed")
##--------------------------------------------------------------------------------------##
## Our First rule - sample trimming
##--------------------------------------------------------------------------------------##
rule final:
input: expand("/project/ateeq/PROJECT/snakemake-example/trimmed/{smp}_R1_trimmed.fastq", smp=SAMPLES)
rule trimming:
input: fwd="/project/ateeq/PROJECT/snakemake-example/raw_data/{smp}_R1.fastq.gz",rev="/project/ateeq/PROJECT/snakemake-example/raw_data/{smp}_R2.fastq.gz"
output: fwd="/project/ateeq/PROJECT/snakemake-example/trimmed/{smp}_R1_trimmed.fastq", rev="/project/ateeq/PROJECT/snakemake-example/trimmed/{smp}_R2_trimmed.fastq", rep="/project/ateeq/PROJECT/snakemake-example/report/{smp}.html"
threads: 30
message: """--- Trimming."""
shell: """
fastp -i {input.fwd} -I {input.rev} -o {output.fwd} -O {output.rev} --detect_adapter_for_pe --disable_length_filtering --correction --qualified_quality_phred 30 --thread 16 --html {output.rep} --report_title "Fastq Quality Control Report" &>>{input.fwd}.log
"""
when I am running the pipeline it shows the following error
MissingOutputException in line 51 of /project/ateeq/PROJECT/snakemake-example/snakefile:
Missing files after 5 seconds:
/project/ateeq/PROJECT/snakemake-example/trimmed/H667-1_R1_trimmed.fastq
/project/ateeq/PROJECT/snakemake-example/trimmed/H667-1_R2_trimmed.fastq
/project/ateeq/PROJECT/snakemake-example/report/H667-1.html
This might be due to filesystem latency. If that is the case, consider to increase the wait time with --latency-wait.
Though I have increased the latency wait period for 2000 sec it still ends up throwing an error
snakemake -s snakefile -j 30 --latency-wait 2000
I am using snakemake version 5.4.5 and Python 3.6.8. Please let me know where I am going wrong, it would be a great help to me
Thanks for kind Help,
Sincerely,
Dave
I don't know if this is due to the copy & paste but the indentation is wrong:
rule trimming:
input: fwd="/project/ateeq/PROJECT/snakemake-example/raw_data/{smp}_R1.fastq.gz",rev="/project/ateeq/PROJECT/snakemake-example/raw_data/{smp}_R2.fastq.gz"
output: fwd="/project/ateeq/PROJECT/snakemake-example/trimmed/{smp}_R1_trimmed.fastq", rev="/project/ateeq/PROJECT/snakemake-example/trimmed/{smp}_R2_trimmed.fastq", rep="/project/ateeq/PROJECT/snakemake-example/report/{smp}.html"
threads: 30
message: """--- Trimming."""
shell: """
fastp -i {input.fwd} -I {input.rev} -o {output.fwd} -O {output.rev} --detect_adapter_for_pe --disable_length_filtering --correction --qualified_quality_phred 30 --thread 16 --html {output.rep} --report_title "Fastq Quality Control Report" &>>{input.fwd}.log
"""
Shouldn't it be:
rule trimming:
input:
fwd="/project/ateeq/PROJECT/snakemake-example/raw_data/{smp}_R1.fastq.gz",
rev="/project/ateeq/PROJECT/snakemake-example/raw_data/{smp}_R2.fastq.gz"
output:
fwd="/project/ateeq/PROJECT/snakemake-example/trimmed/{smp}_R1_trimmed.fastq",
rev="/project/ateeq/PROJECT/snakemake-example/trimmed/{smp}_R2_trimmed.fastq",
rep="/project/ateeq/PROJECT/snakemake-example/report/{smp}.html"
threads: 30
message: """--- Trimming."""
shell: """
fastp -i {input.fwd} -I {input.rev} -o {output.fwd} -O {output.rev} --detect_adapter_for_pe --disable_length_filtering --correction --qualified_quality_phred 30 --thread 16 --html {output.rep} --report_title "Fastq Quality Control Report" &>>{input.fwd}.log
"""
I wish to generate a lot of --enable-*/--disable-* options by something like:
COMPONENTS([a b c], [yes])
where the second argument is the default value of the automatic enable_* variable. My first attempt was to write an AC_ARG_ENABLE(...) within an m4_foreach_w, but so far, I'm only getting the first component to appear in the ./configure --help output.
If I add hand-written AC_ARG_ENABLEs, they work as usual.
Regardless, the --enable-*/--disable-* options work as they should, just the help text is missing.
Here's the full code to reproduce the problem:
AC_INIT([foo], 1.0)
AM_INIT_AUTOMAKE([foreign])
AC_DEFUN([COMPONENTS],
[
m4_foreach_w([component], [$1], [
AS_ECHO(["Processing [component] component with default enable=$2"])
AC_ARG_ENABLE([component],
[AS_HELP_STRING([--enable-[]component], [component] component)],
,
[enable_[]AS_TR_SH([component])=$2]
)
])
AC_ARG_ENABLE([x],
[AS_HELP_STRING([--enable-[]x], [component x])],
,
[enable_[]AS_TR_SH([x])=$2]
)
AC_ARG_ENABLE([y],
[AS_HELP_STRING([--enable-[]y], [component y])],
,
[enable_[]AS_TR_SH([y])=$2]
)
])
COMPONENTS([a b c], [yes])
for var in a b c x y; do
echo -n "\$enable_$var="
eval echo "\$enable_$var"
done
AC_CONFIG_FILES(Makefile)
AC_OUTPUT
And an empty Makefile.am. To verify that the options work:
$ ./configure --disable-a --disable-b --disable-d --disable-x
configure: WARNING: unrecognized options: --disable-d
...
Processing component a with default enable=yes
Processing component b with default enable=yes
Processing component c with default enable=yes
$enable_a=no
$enable_b=no
$enable_c=yes
$enable_x=no
$enable_y=yes
After I poked around in autoconf sources, I figured out this has to do with the m4_divert_once call in the implementation of AC_ARG_ENABLE:
# AC_ARG_ENABLE(FEATURE, HELP-STRING, [ACTION-IF-TRUE], [ACTION-IF-FALSE])
# ------------------------------------------------------------------------
AC_DEFUN([AC_ARG_ENABLE],
[AC_PROVIDE_IFELSE([AC_PRESERVE_HELP_ORDER],
[],
[m4_divert_once([HELP_ENABLE], [[
Optional Features:
--disable-option-checking ignore unrecognized --enable/--with options
--disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no)
--enable-FEATURE[=ARG] include FEATURE [ARG=yes]]])])dnl
m4_divert_once([HELP_ENABLE], [$2])dnl
_AC_ENABLE_IF([enable], [$1], [$3], [$4])dnl
])# AC_ARG_ENABLE
# m4_divert_once(DIVERSION-NAME, CONTENT)
# ---------------------------------------
# Output CONTENT into DIVERSION-NAME once, if not already there.
# An end of line is appended for free to CONTENT.
m4_define([m4_divert_once],
[m4_expand_once([m4_divert_text([$1], [$2])])])
I'm guessing that the HELP-STRING argument is remembered in it's unexpanded form, so it is added just once for all components. Manually expanding the AC_HELP_STRING does what I want:
AC_DEFUN([COMPONENTS],
[
m4_foreach_w([comp], [$1], [
AS_ECHO(["Processing component 'comp' with default enable=$2"])
AC_ARG_ENABLE([comp],
m4_expand([AS_HELP_STRING([--enable-comp], enable component comp)]),
,
[enable_[]AS_TR_SH([comp])=$2]
)
])
])
COMPONENTS([a b c x y], [yes])
I couldn't find a way to properly quote components so that it appears as a string, after being used as the loop variable in m4_foreach_w, so I just renamed it to spare me the trouble.