From 222f0a86d44f83d3c3a51ca8181e9ef4a8886ae1 Mon Sep 17 00:00:00 2001 From: peter Date: Sun, 28 Aug 2022 12:09:33 -0700 Subject: [PATCH 1/2] enable tests (this will not work) use tempdir, closes #427 . temp file names better context handling extra level of escaping in eidos escape logfile cache bump . --- stdpopsim/cli.py | 81 ++++++++++++++++++--------------------- stdpopsim/slim_engine.py | 43 +++++++++++++-------- tests/test_cli.py | 3 -- tests/test_dfes.py | 6 --- tests/test_masking.py | 4 -- tests/test_slim_engine.py | 32 ---------------- 6 files changed, 64 insertions(+), 105 deletions(-) diff --git a/stdpopsim/cli.py b/stdpopsim/cli.py index 8bc92df24..14265fa84 100644 --- a/stdpopsim/cli.py +++ b/stdpopsim/cli.py @@ -35,8 +35,6 @@ pass -IS_WINDOWS = sys.platform.startswith("win") - logger = logging.getLogger(__name__) @@ -1040,47 +1038,44 @@ def time_or_model( "This option may provided multiple times.", ) - # SLiM is not available for windows. - if not IS_WINDOWS: - - def slim_exec(path): - # Hack to set the SLIM environment variable at parse time, - # before get_version() can be called. - os.environ["SLIM"] = path - return path - - slim_parser = top_parser.add_argument_group("SLiM specific parameters") - slim_parser.add_argument( - "--slim-path", - metavar="PATH", - type=slim_exec, - default=None, - help="Full path to `slim' executable.", - ) - slim_parser.add_argument( - "--slim-script", - action="store_true", - default=False, - help="Write script to stdout and exit without running SLiM.", - ) - slim_parser.add_argument( - "--slim-scaling-factor", - metavar="Q", - default=1, - type=float, - help="Rescale model parameters by Q to speed up simulation. " - "See SLiM manual: `5.5 Rescaling population sizes to " - "improve simulation performance`. " - "[default=%(default)s].", - ) - slim_parser.add_argument( - "--slim-burn-in", - metavar="X", - default=10, - type=float, - help="Length of the burn-in phase, in units of N generations " - "[default=%(default)s].", - ) + def slim_exec(path): + # Hack to set the SLIM environment variable at parse time, + # before get_version() can be called. + os.environ["SLIM"] = path + return path + + slim_parser = top_parser.add_argument_group("SLiM specific parameters") + slim_parser.add_argument( + "--slim-path", + metavar="PATH", + type=slim_exec, + default=None, + help="Full path to `slim' executable.", + ) + slim_parser.add_argument( + "--slim-script", + action="store_true", + default=False, + help="Write script to stdout and exit without running SLiM.", + ) + slim_parser.add_argument( + "--slim-scaling-factor", + metavar="Q", + default=1, + type=float, + help="Rescale model parameters by Q to speed up simulation. " + "See SLiM manual: `5.5 Rescaling population sizes to " + "improve simulation performance`. " + "[default=%(default)s].", + ) + slim_parser.add_argument( + "--slim-burn-in", + metavar="X", + default=10, + type=float, + help="Length of the burn-in phase, in units of N generations " + "[default=%(default)s].", + ) subparsers = top_parser.add_subparsers(dest="subcommand") subparsers.required = True diff --git a/stdpopsim/slim_engine.py b/stdpopsim/slim_engine.py index 69b134d3e..40ff2bcea 100644 --- a/stdpopsim/slim_engine.py +++ b/stdpopsim/slim_engine.py @@ -40,13 +40,13 @@ import os import sys +import contextlib import copy import string import tempfile import subprocess import functools import itertools -import contextlib import random import textwrap import logging @@ -62,6 +62,12 @@ logger = logging.getLogger(__name__) + +def _escape_eidos(s): + # this is for Windows paths passed as strings in Eidos + return "\\\\".join(s.split("\\")) + + _slim_upper = """ initialize() { if (!exists("dry_run")) @@ -1135,7 +1141,7 @@ def fix_time(event): recombination_rates=recomb_rates_str, recombination_ends=recomb_ends_str, generation_time=demographic_model.generation_time, - trees_file=trees_file, + trees_file=_escape_eidos(trees_file), pop_names=f"c({pop_names_str})", ) ) @@ -1410,7 +1416,7 @@ def matrix2str( if logfile is not None: printsc( string.Template(_slim_logfile).substitute( - logfile=logfile, + logfile=_escape_eidos(str(logfile)), loginterval=logfile_interval, ) ) @@ -1552,21 +1558,27 @@ def simulate( run_slim = not slim_script - mktemp = functools.partial(tempfile.NamedTemporaryFile, mode="w") + tempdir = tempfile.TemporaryDirectory(prefix="stdpopsim_") + ts_filename = os.path.join(tempdir.name, f"{os.urandom(3).hex()}.trees") @contextlib.contextmanager def script_file_f(): - f = mktemp(suffix=".slim") if not slim_script else sys.stdout - yield f + if run_slim: + fname = os.path.join(tempdir.name, f"{os.urandom(3).hex()}.slim") + f = open(fname, "w") + else: + fname = "stdout" + f = sys.stdout + yield f, fname # Don't close sys.stdout. - if not slim_script: + if run_slim: f.close() - with script_file_f() as script_file, mktemp(suffix=".ts") as ts_file: - + with script_file_f() as sf: + script_file, script_filename = sf recap_epoch = slim_makescript( script_file, - ts_file.name, + ts_filename, demographic_model, contig, sample_sets, @@ -1584,7 +1596,7 @@ def script_file_f(): return None self._run_slim( - script_file.name, + script_filename, slim_path=slim_path, seed=seed, dry_run=dry_run, @@ -1594,7 +1606,7 @@ def script_file_f(): if dry_run: return None - ts = tskit.load(ts_file.name) + ts = tskit.load(ts_filename) ts = _add_dfes_to_metadata(ts, contig) if _recap_and_rescale: @@ -1634,8 +1646,7 @@ def _run_slim( if slim_path is None: slim_path = self.slim_path() - # SLiM v3.6 sends `stop()` output to stderr, which we rely upon. - self._assert_min_version("3.6", slim_path) + self._assert_min_version("4.0", slim_path) slim_cmd = [slim_path] if seed is not None: @@ -1921,6 +1932,4 @@ def recap_and_rescale( return ts -# SLiM does not currently work on Windows. -if sys.platform != "win32": - stdpopsim.register_engine(_SLiMEngine()) +stdpopsim.register_engine(_SLiMEngine()) diff --git a/tests/test_cli.py b/tests/test_cli.py index 596aef32e..9b918de73 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -543,9 +543,6 @@ def test_gutenkunst_three_pop_ooa(self): def test_browning_america(self): self.verify_bad_samples("HomSap -d AmericanAdmixture_4B11 2 3 4 5 6") - IS_WINDOWS = sys.platform.startswith("win") - - @pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") def test_browning_america_dfe(self): self.verify_bad_samples( "HomSap -d AmericanAdmixture_4B11 --dfe Gamma_K17 2 3 4 5 6" diff --git a/tests/test_dfes.py b/tests/test_dfes.py index 54b16b305..43d206958 100644 --- a/tests/test_dfes.py +++ b/tests/test_dfes.py @@ -9,8 +9,6 @@ from stdpopsim import dfe from stdpopsim import utils -IS_WINDOWS = sys.platform.startswith("win") - class TestCreateMutationType: """ @@ -554,7 +552,6 @@ def test_dfe_is_neutral(self): ) assert d.is_neutral is (neutral and dist == "f") - @pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") def test_no_msprime_dfe(self): # test we cannot simulate a non-neutral DFE with msprime m1 = dfe.MutationType( @@ -644,7 +641,6 @@ def test_bad_qc_dfe(self): ) -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class DFETestMixin: """ Mixin for testing specific DFEs. Subclass should extend @@ -694,7 +690,6 @@ def test_simulation_runs(self): assert num_nonneutral > 0 # nonneutral mutations -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class CatalogDFETestMixin(DFETestMixin): """ Mixin for DFEs in the catalog. @@ -704,7 +699,6 @@ def test_id_valid(self): assert utils.is_valid_dfe_id(self.dfe.id) -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class QcdCatalogDFETestMixin(CatalogDFETestMixin): """ Extends the tests to also check that the qc DFE is equal to diff --git a/tests/test_masking.py b/tests/test_masking.py index 3453ff03d..14aba3c85 100644 --- a/tests/test_masking.py +++ b/tests/test_masking.py @@ -1,7 +1,6 @@ """ Tests for the genetic maps management. """ -import sys import numpy as np import msprime @@ -9,8 +8,6 @@ import stdpopsim.utils -IS_WINDOWS = sys.platform.startswith("win") - class TestMasking: @pytest.mark.usefixtures("tmp_path") @@ -119,7 +116,6 @@ def test_mask_tree_sequence(self): assert np.all(np.logical_and(100 <= positions, positions < 200)) -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestSimulate: @pytest.mark.filterwarnings("ignore::msprime.IncompletePopulationMetadataWarning") def test_simulate_with_mask(self): diff --git a/tests/test_slim_engine.py b/tests/test_slim_engine.py index d0e30a562..0af8cf5a3 100644 --- a/tests/test_slim_engine.py +++ b/tests/test_slim_engine.py @@ -3,7 +3,6 @@ """ import os import io -import sys import tempfile import math from unittest import mock @@ -21,7 +20,6 @@ import stdpopsim.cli from .test_cli import capture_output -IS_WINDOWS = sys.platform.startswith("win") slim_path = os.environ.get("SLIM", "slim") @@ -33,7 +31,6 @@ def count_mut_types(ts): return [num_neutral, abs(len(selection_coeffs) - num_neutral)] -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestAPI: def test_bad_params(self): engine = stdpopsim.get_engine("slim") @@ -404,7 +401,6 @@ def test_allele_codes(self): assert all([x.isnumeric() for x in alleles]) -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestCLI: def docmd(self, _cmd): cmd = (f"-q -e slim --slim-burn-in 0 {_cmd} -l 0.001 -c chr1 -s 1234").split() @@ -734,7 +730,6 @@ def test_bad_slim_path(self): os.environ["SLIM"] = saved_slim_env -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestWarningsAndErrors: """ Checks that warning messages are printed when appropriate. @@ -1019,23 +1014,6 @@ def test_warning_when_scaling(self, scaling_factor): ) -class TestSlimAvailable: - """ - Checks whether SLiM is available or not on platforms that support it. - """ - - def test_parser_has_options(self): - parser = stdpopsim.cli.stdpopsim_cli_parser() - with mock.patch("sys.exit", autospec=True): - _, stderr = capture_output(parser.parse_args, ["--help"]) - # On windows we should have no "slim" options - assert IS_WINDOWS == ("slim" not in stderr) - - def test_engine_available(self): - all_engines = [engine.id for engine in stdpopsim.all_engines()] - assert IS_WINDOWS == ("slim" not in all_engines) - - def get_test_contig( spp="HomSap", chrom="chr22", length_multiplier=0.001, left=None, right=None ): @@ -1081,7 +1059,6 @@ def allele_frequency(self, ts): return af -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestRecombinationMap(PiecewiseConstantSizeMixin): def verify_recombination_map(self, contig, ts): Q = ts.metadata["SLiM"]["user_metadata"]["Q"] @@ -1147,7 +1124,6 @@ def test_off_by_one(self): assert list(ts.breakpoints()) == [0.0, midpoint, contig.length] -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestGenomicElementTypes(PiecewiseConstantSizeMixin): mut_params = { @@ -1631,7 +1607,6 @@ def test_chromosomal_segment(self): assert int(ts.sequence_length) == right - left -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestLogfile(PiecewiseConstantSizeMixin): # tmp_path is a pytest fixture def test_logfile(self, tmp_path): @@ -1655,7 +1630,6 @@ def test_logfile(self, tmp_path): assert np.all(data[:, 2] == 0.0) -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestDrawMutation(PiecewiseConstantSizeMixin): def test_draw_mutation(self): contig = get_test_contig() @@ -1960,7 +1934,6 @@ def test_bad_time(self): ) -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestAlleleFrequencyConditioning(PiecewiseConstantSizeMixin): @pytest.mark.filterwarnings("ignore::stdpopsim.SLiMScalingFactorWarning") def test_drawn_mutation_not_lost(self): @@ -2170,7 +2143,6 @@ def test_no_drawn_mutation(self): ) -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestChangeMutationFitness(PiecewiseConstantSizeMixin): # Testing stdpopsim.ext.ChangeMutationFitness is challenging, because # the side-effects are not deterministic. But if we condition on fixation @@ -2348,7 +2320,6 @@ def test_bad_GenerationAfter_times(self): ) -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestExtendedEvents(PiecewiseConstantSizeMixin): def test_bad_extended_events(self): engine = stdpopsim.get_engine("slim") @@ -2368,7 +2339,6 @@ def test_bad_extended_events(self): ) -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestSelectiveSweep(PiecewiseConstantSizeMixin): @staticmethod def _get_island_model(Ne=1000, migration_rate=0.01): @@ -2814,7 +2784,6 @@ def test_sweep_with_background_selection(self, tmp_path): assert np.all(p1_outside_sweep <= 1) -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestSelectionCoeffFromMutation: species = stdpopsim.get_species("HomSap") @@ -2888,7 +2857,6 @@ def test_errors(self): stdpopsim.ext.selection_coeff_from_mutation(ts, "bar") -@pytest.mark.skipif(IS_WINDOWS, reason="SLiM not available on windows") class TestPloidy: """ Test that population sizes used in SLiM engine are scaled correctly From be9ff1cd937e83e8a783d28138fc4eaf22401548 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 21 Mar 2023 15:12:49 -0700 Subject: [PATCH 2/2] bump slim version in conda --- requirements/CI/conda.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/CI/conda.txt b/requirements/CI/conda.txt index 6954e1f5e..43faa07a8 100644 --- a/requirements/CI/conda.txt +++ b/requirements/CI/conda.txt @@ -1,2 +1,2 @@ msprime==1.2.0 -slim==4.0 +slim==4.0.1