Skip to content

Commit

Permalink
add update-meta-yml option
Browse files Browse the repository at this point in the history
  • Loading branch information
@mirpedrol
mirpedrol committed Jun 18, 2024
1 parent 1de47ae commit 7f8a35d
Show file tree
Hide file tree
Showing 5 changed files with 226 additions and 60 deletions.
8 changes: 7 additions & 1 deletion nf_core/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1288,7 +1288,10 @@ def test_module(ctx, tool, dir, no_prompts, update, once, profile):
is_flag=True,
help="Fix the module version if a newer version is available",
)
def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version):
@click.option(
"--update-meta-yml", is_flag=True, help="Update the meta.yml file with the correct format of input and outputs"
)
def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version, update_meta_yml):
"""
Lint one or more modules in a directory.
Expand All @@ -1305,6 +1308,7 @@ def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed,
module_lint = ModuleLint(
dir,
fail_warned=fail_warned,
update_meta_yml=update_meta_yml,
registry=ctx.params["registry"],
remote_url=ctx.obj["modules_repo_url"],
branch=ctx.obj["modules_repo_branch"],
Expand All @@ -1325,9 +1329,11 @@ def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed,
if len(module_lint.failed) > 0:
sys.exit(1)
except LintExceptionError as e:
raise
log.error(e)
sys.exit(1)
except (UserWarning, LookupError) as e:
raise
log.critical(e)
sys.exit(1)

Expand Down
2 changes: 2 additions & 0 deletions nf_core/components/lint/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ def __init__(
component_type,
dir,
fail_warned=False,
update_meta_yml=False,
remote_url=None,
branch=None,
no_pull=False,
Expand All @@ -72,6 +73,7 @@ def __init__(
)

self.fail_warned = fail_warned
self.update_meta_yml = update_meta_yml
self.passed = []
self.warned = []
self.failed = []
Expand Down
4 changes: 2 additions & 2 deletions nf_core/components/nfcore_component.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ def get_inputs_from_main_nf(self):
elif match.group(4):
input_val = match.group(4).split(",")[0] # handle `files, stageAs: "inputs/*"` cases
if input_type and input_val:
channel_elements.append({input_val: {"type": input_type}})
channel_elements.append({input_val: {"qualifier": input_type}})
if len(channel_elements) > 0:
inputs.append(channel_elements)
log.debug(f"Found {len(inputs)} inputs in {self.main_nf}")
Expand Down Expand Up @@ -215,7 +215,7 @@ def get_outputs_from_main_nf(self):
output_val = match_element.group(4)
if output_type and output_val:
output_val = output_val.strip("'").strip('"') # remove quotes
output_channel[match_emit.group(1)].append({output_val: {"type": output_type}})
output_channel[match_emit.group(1)].append({output_val: {"qualifier": output_type}})
outputs.append(output_channel)
log.debug(f"Found {len(outputs)} outputs in {self.main_nf}")
self.outputs = outputs
105 changes: 104 additions & 1 deletion nf_core/modules/lint/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import questionary
import rich
import yaml

import nf_core.modules.modules_utils
import nf_core.utils
Expand All @@ -29,7 +30,12 @@ class ModuleLint(ComponentLint):
# Import lint functions
from .environment_yml import environment_yml # type: ignore[misc]
from .main_nf import main_nf # type: ignore[misc]
from .meta_yml import meta_yml # type: ignore[misc]
from .meta_yml import ( # type: ignore[misc]
meta_yml,
obtain_correct_and_specified_inputs,
obtain_correct_and_specified_outputs,
read_meta_yml,
)
from .module_changes import module_changes # type: ignore[misc]
from .module_deprecations import module_deprecations # type: ignore[misc]
from .module_patch import module_patch # type: ignore[misc]
Expand All @@ -41,6 +47,7 @@ def __init__(
self,
dir,
fail_warned=False,
update_meta_yml=False,
remote_url=None,
branch=None,
no_pull=False,
Expand All @@ -51,6 +58,7 @@ def __init__(
component_type="modules",
dir=dir,
fail_warned=fail_warned,
update_meta_yml=update_meta_yml,
remote_url=remote_url,
branch=branch,
no_pull=no_pull,
Expand Down Expand Up @@ -213,6 +221,12 @@ def lint_module(self, mod, progress_bar, registry, local=False, fix_version=Fals

# Otherwise run all the lint tests
else:
mod.get_inputs_from_main_nf()
mod.get_outputs_from_main_nf()
# Update meta.yml file if requested
if self.update_meta_yml:
self.update_meta_yml_file(mod)

if self.repo_type == "pipeline" and self.modules_json:
# Set correct sha
version = self.modules_json.get_module_version(mod.component_name, mod.repo_url, mod.org)
Expand All @@ -232,3 +246,92 @@ def lint_module(self, mod, progress_bar, registry, local=False, fix_version=Fals
self.failed += warned

self.failed += [LintResult(mod, *m) for m in mod.failed]

def update_meta_yml_file(self, mod):
"""
Update the meta.yml file with the correct inputs and outputs
"""
meta_yml = self.read_meta_yml(mod)
corrected_meta_yml = meta_yml.copy()

# Obtain inputs and outputs from main.nf and meta.yml
# Used to compare only the structure of channels and elements
# Do not compare features to allow for custom features in meta.yml (i.e. pattern)
if "input" in meta_yml:
correct_inputs, meta_inputs = self.obtain_correct_and_specified_inputs(mod, meta_yml)
if "output" in meta_yml:
correct_outputs, meta_outputs = self.obtain_correct_and_specified_outputs(mod, meta_yml)

if correct_inputs != meta_inputs:
log.debug(
f"Correct inputs: '{correct_inputs}' differ from current inputs: '{meta_inputs}' in '{mod.meta_yml}'"
)
corrected_meta_yml["input"] = mod.inputs.copy() # list of lists (channels) of dicts (elements)
for i, channel in enumerate(corrected_meta_yml["input"]):
for j, element in enumerate(channel):
element_name = list(element.keys())[0]
for k, meta_element in enumerate(meta_yml["input"]):
try:
# Handle old format of meta.yml: list of dicts (channels)
if element_name in meta_element.keys():
# Copy current features of that input element form meta.yml
for feature in meta_element[element_name].keys():
if feature not in element[element_name].keys():
corrected_meta_yml["input"][i][j][element_name][feature] = meta_element[
element_name
][feature]
break
except AttributeError:
# Handle new format of meta.yml: list of lists (channels) of elements (dicts)
for x, meta_ch_element in enumerate(meta_element):
if element_name in meta_ch_element.keys():
# Copy current features of that input element form meta.yml
for feature in meta_element[x][element_name].keys():
if feature not in element[element_name].keys():
corrected_meta_yml["input"][i][j][element_name][feature] = meta_element[x][
element_name
][feature]
break

if correct_outputs != meta_outputs:
log.debug(
f"Correct outputs: '{correct_outputs}' differ from current outputs: '{meta_outputs}' in '{mod.meta_yml}'"
)
corrected_meta_yml["output"] = mod.outputs.copy() # list of dicts (channels) with list of dicts (elements)
for i, channel in enumerate(corrected_meta_yml["output"]):
ch_name = list(channel.keys())[0]
for j, element in enumerate(channel[ch_name]):
element_name = list(element.keys())[0]
for k, meta_element in enumerate(meta_yml["output"]):
if element_name in meta_element.keys():
# Copy current features of that output element form meta.yml
for feature in meta_element[element_name].keys():
if feature not in element[element_name].keys():
corrected_meta_yml["output"][i][ch_name][j][element_name][feature] = meta_element[
element_name
][feature]
break
elif ch_name in meta_element.keys():
# When the previous output element was using the name of the channel
# Copy current features of that output element form meta.yml
try:
# Handle old format of meta.yml
for feature in meta_element[ch_name].keys():
if feature not in element[element_name].keys():
corrected_meta_yml["output"][i][ch_name][j][element_name][feature] = (
meta_element[ch_name][feature]
)
except AttributeError:
# Handle new format of meta.yml
for x, meta_ch_element in enumerate(meta_element[ch_name]):
for meta_ch_element_name in meta_ch_element.keys():
for feature in meta_ch_element[meta_ch_element_name].keys():
if feature not in element[element_name].keys():
corrected_meta_yml["output"][i][ch_name][j][element_name][feature] = (
meta_ch_element[meta_ch_element_name][feature]
)
break

with open(mod.meta_yml, "w") as fh:
log.info(f"Updating {mod.meta_yml}")
yaml.dump(corrected_meta_yml, fh, sort_keys=False, Dumper=nf_core.utils.custom_yaml_dumper())
Loading

0 comments on commit 7f8a35d

Please sign in to comment.