passing tests

Author: toliwaga

.pylintrc

@@ -0,0 +1,16 @@
+[MESSAGES CONTROL]
+disable=locally-disabled,
+ # False positive for type annotations with typing module
+ # invalid-sequence-index,
+ # False positive for OK test methods names and few other places
+ invalid-name,
+ # False positive for test file classes and methods
+ missing-docstring
+
+[REPORTS]
+# Simplify pylint reports
+reports=no
+
+[SIMILARITIES]
+min-similarity-lines=10
+ignore-docstrings=yes

.travis.yml

@@ -15,7 +15,7 @@ install:
 - |
   conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION cytoolz numpy pandas pip pytables pyyaml toolz psutil
 - source activate test-environment
-- pip install orca openmatrix zbox
+- pip install openmatrix zbox future
 - pip install pytest pytest-cov coveralls pycodestyle
 - pip install sphinx numpydoc sphinx_rtd_theme
 - pip install .

activitysim/abm/models/util/test/test_vectorize_tour_scheduling.py

@@ -5,7 +5,6 @@
 import pytest
 import pandas as pd
 import numpy as np
-import orca
 
 import pandas.util.testing as pdt
 
@@ -17,15 +16,15 @@
 
 def test_vts():
 
-    orca.add_injectable("settings", {})
+    inject.add_injectable("settings", {})
 
     # note: need 0 duration tour on one end of day to guarantee at least one available tour
     alts = pd.DataFrame({
         "start": [1, 1, 2, 3],
         "end": [1, 4, 5, 6]
     })
     alts['duration'] = alts.end - alts.start
-    orca.add_injectable("tdd_alts", alts)
+    inject.add_injectable("tdd_alts", alts)
 
     current_tour_person_ids = pd.Series(['b', 'c'],
                                         index=['d', 'e'])
@@ -54,13 +53,13 @@ def test_vts():
     persons = pd.DataFrame({
         "income": [20, 30, 25]
     }, index=[1, 2, 3])
-    orca.add_table('persons', persons)
+    inject.add_table('persons', persons)
 
     spec = pd.DataFrame({"Coefficient": [1.2]},
                         index=["income"])
     spec.index.name = "Expression"
 
-    orca.add_injectable("check_for_variability", True)
+    inject.add_injectable("check_for_variability", True)
 
     tdd_choices = vectorize_tour_scheduling(tours, persons, alts, spec)
 

activitysim/abm/test/test_misc.py

@@ -5,46 +5,40 @@
 import tempfile
 
 import numpy as np
-import orca
 import pytest
 import yaml
 
-# orca injectables complicate matters because the decorators are executed at module load time
-# and since py.test collects modules and loads them at the start of a run
-# if a test method does something that has a lasting side-effect, then that side effect
-# will carry over not just to subsequent test functions, but to subsequently called modules
-# for instance, columns added with add_column will remain attached to orca tables
-# pytest-xdist allows us to run py.test with the --boxed option which runs every function
-# with a brand new python interpreter
+from activitysim.core import inject
 
-# Also note that the following import statement has the side-effect of registering injectables:
+
+# The following import statement has the side-effect of registering injectables:
 from .. import __init__
 
 
 def test_misc():
 
-    orca.clear_cache()
+    inject.clear_cache()
 
     with pytest.raises(RuntimeError) as excinfo:
-        orca.get_injectable("configs_dir")
+        inject.get_injectable("configs_dir")
     assert "directory does not exist" in str(excinfo.value)
 
     with pytest.raises(RuntimeError) as excinfo:
-        orca.get_injectable("data_dir")
+        inject.get_injectable("data_dir")
     assert "directory does not exist" in str(excinfo.value)
 
     with pytest.raises(RuntimeError) as excinfo:
-        orca.get_injectable("output_dir")
+        inject.get_injectable("output_dir")
     assert "directory does not exist" in str(excinfo.value)
 
     configs_dir = os.path.join(os.path.dirname(__file__), 'configs_test_misc')
-    orca.add_injectable("configs_dir", configs_dir)
+    inject.add_injectable("configs_dir", configs_dir)
 
-    settings = orca.get_injectable("settings")
+    settings = inject.get_injectable("settings")
     assert isinstance(settings, dict)
 
     data_dir = os.path.join(os.path.dirname(__file__), 'data')
-    orca.add_injectable("data_dir", data_dir)
+    inject.add_injectable("data_dir", data_dir)
 
     # default values if not specified in settings
-    assert orca.get_injectable("chunk_size") == 0
+    assert inject.get_injectable("chunk_size") == 0

activitysim/abm/test/test_pipeline.py

@@ -6,7 +6,6 @@
 import logging
 
 import numpy as np
-import orca
 import pandas as pd
 import pandas.util.testing as pdt
 import pytest
@@ -35,7 +34,7 @@
 
 
 def teardown_function(func):
-    orca.clear_cache()
+    inject.clear_cache()
     inject.reinject_decorated_tables()
 
 
@@ -64,25 +63,25 @@ def inject_settings(configs_dir, households_sample_size, chunk_size=None,
         if check_for_variability is not None:
             settings['check_for_variability'] = check_for_variability
 
-    orca.add_injectable("settings", settings)
+        inject.add_injectable("settings", settings)
 
     return settings
 
 
 def test_rng_access():
 
     configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
-    orca.add_injectable("configs_dir", configs_dir)
+    inject.add_injectable("configs_dir", configs_dir)
 
     output_dir = os.path.join(os.path.dirname(__file__), 'output')
-    orca.add_injectable("output_dir", output_dir)
+    inject.add_injectable("output_dir", output_dir)
 
     data_dir = os.path.join(os.path.dirname(__file__), 'data')
-    orca.add_injectable("data_dir", data_dir)
+    inject.add_injectable("data_dir", data_dir)
 
     inject_settings(configs_dir, households_sample_size=HOUSEHOLDS_SAMPLE_SIZE)
 
-    orca.clear_cache()
+    inject.clear_cache()
 
     inject.add_injectable('rng_base_seed', 0)
 
@@ -91,7 +90,7 @@ def test_rng_access():
     rng = pipeline.get_rn_generator()
 
     pipeline.close_pipeline()
-    orca.clear_cache()
+    inject.clear_cache()
 
 
 def regress_mini_auto():
@@ -149,22 +148,20 @@ def regress_mini_mtf():
 def test_mini_pipeline_run():
 
     configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
-    orca.add_injectable("configs_dir", configs_dir)
+    inject.add_injectable("configs_dir", configs_dir)
 
     output_dir = os.path.join(os.path.dirname(__file__), 'output')
-    orca.add_injectable("output_dir", output_dir)
+    inject.add_injectable("output_dir", output_dir)
 
     data_dir = os.path.join(os.path.dirname(__file__), 'data')
-    orca.add_injectable("data_dir", data_dir)
+    inject.add_injectable("data_dir", data_dir)
 
     inject_settings(configs_dir, households_sample_size=HOUSEHOLDS_SAMPLE_SIZE)
 
-    orca.clear_cache()
+    inject.clear_cache()
 
     tracing.config_logger()
 
-    # assert len(orca.get_table("households").index) == HOUSEHOLDS_SAMPLE_SIZE
-
     _MODELS = [
         'initialize_landuse',
         'compute_accessibility',
@@ -198,7 +195,7 @@ def test_mini_pipeline_run():
     assert "not in checkpoints" in str(excinfo.value)
 
     pipeline.close_pipeline()
-    orca.clear_cache()
+    inject.clear_cache()
 
     close_handlers()
 
@@ -210,17 +207,17 @@ def test_mini_pipeline_run2():
     # when we restart pipeline
 
     configs_dir = os.path.join(os.path.dirname(__file__), 'configs')
-    orca.add_injectable("configs_dir", configs_dir)
+    inject.add_injectable("configs_dir", configs_dir)
 
     output_dir = os.path.join(os.path.dirname(__file__), 'output')
-    orca.add_injectable("output_dir", output_dir)
+    inject.add_injectable("output_dir", output_dir)
 
     data_dir = os.path.join(os.path.dirname(__file__), 'data')
-    orca.add_injectable("data_dir", data_dir)
+    inject.add_injectable("data_dir", data_dir)
 
     inject_settings(configs_dir, households_sample_size=HOUSEHOLDS_SAMPLE_SIZE)
 
-    orca.clear_cache()
+    inject.clear_cache()
 
     # should be able to get this BEFORE pipeline is opened
     checkpoints_df = pipeline.get_checkpoints()
@@ -249,21 +246,21 @@ def test_mini_pipeline_run2():
     assert len(checkpoints_df.index) == prev_checkpoint_count
 
     pipeline.close_pipeline()
-    orca.clear_cache()
+    inject.clear_cache()
 
 
 def full_run(resume_after=None, chunk_size=0,
              households_sample_size=HOUSEHOLDS_SAMPLE_SIZE,
              trace_hh_id=None, trace_od=None, check_for_variability=None):
 
     configs_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'example', 'configs')
-    orca.add_injectable("configs_dir", configs_dir)
+    inject.add_injectable("configs_dir", configs_dir)
 
     data_dir = os.path.join(os.path.dirname(__file__), 'data')
-    orca.add_injectable("data_dir", data_dir)
+    inject.add_injectable("data_dir", data_dir)
 
     output_dir = os.path.join(os.path.dirname(__file__), 'output')
-    orca.add_injectable("output_dir", output_dir)
+    inject.add_injectable("output_dir", output_dir)
 
     settings = inject_settings(
         configs_dir,
@@ -273,7 +270,7 @@ def full_run(resume_after=None, chunk_size=0,
         trace_od=trace_od,
         check_for_variability=check_for_variability)
 
-    orca.clear_cache()
+    inject.clear_cache()
 
     tracing.config_logger()
 
@@ -284,10 +281,6 @@ def full_run(resume_after=None, chunk_size=0,
     tours = pipeline.get_table('tours')
     tour_count = len(tours.index)
 
-    # pipeline.close_pipeline()
-    #
-    # orca.clear_cache()
-
     return tour_count
 
 

activitysim/core/inject.py

@@ -1,3 +1,6 @@
+# ActivitySim
+# See full license in LICENSE.txt.
+
 import logging
 
 import pandas as pd
@@ -84,6 +87,7 @@ def add_table(table_name, table, cache=False):
     return orca.add_table(table_name, table, cache=cache)
 
 
+#fixme remove?
 def add_column(table_name, column_name, column, cache=False):
     return orca.add_column(table_name, column_name, column, cache=cache)
 
@@ -114,6 +118,13 @@ def get_injectable(name, default=_NO_DEFAULT):
         return default
 
 
+def remove_injectable(name):
+
+    #fixme
+    #del orca.orca._INJECTABLES[name]
+    orca.orca._INJECTABLES.pop(name, None)
+
+
 def reinject_decorated_tables():
     """
     reinject the decorated tables (and columns)
@@ -141,6 +152,10 @@ def reinject_decorated_tables():
         orca.add_injectable(name, args['func'], cache=args['cache'])
 
 
+def clear_cache():
+    return orca.clear_cache()
+
+
 def set_step_args(args=None):
 
     assert isinstance(args, dict) or args is None
@@ -156,3 +171,9 @@ def get_step_arg(arg_name, default=_NO_DEFAULT):
         raise "step arg '%s' not found and no default" % arg_name
 
     return args.get(arg_name, default)
+
+
+def dump_state():
+
+    print "_DECORATED_STEPS", _DECORATED_STEPS.keys()
+    print "orca._STEPS", orca.orca._STEPS.keys()

activitysim/core/orca/orca.py

@@ -24,7 +24,7 @@
 from collections import namedtuple
 
 warnings.filterwarnings('ignore', category=tables.NaturalNameWarning)
-logger = logging.getLogger('orca')
+logger = logging.getLogger(__name__)
 
 _TABLES = {}
 _COLUMNS = {}
@@ -1978,7 +1978,6 @@ def run(steps, iter_vars=None, data_out=None, out_interval=1,
                 'running iteration {} with iteration value {!r}'.format(
                     i, var))
 
-        t1 = time.time()
         for j, step_name in enumerate(steps):
             add_injectable('iter_step', iter_step(j, step_name))
             with log_start_finish(

activitysim/core/pipeline.py

@@ -1,3 +1,6 @@
+# ActivitySim
+# See full license in LICENSE.txt.
+
 import os
 import datetime as dt
 
@@ -224,6 +227,7 @@ def rewrap(table_name, df=None):
 
         for column_name in orca.list_columns_for_table(table_name):
             # logger.debug("pop %s.%s: %s" % (table_name, column_name, t.column_type(column_name)))
+            #fixme
             orca.orca._COLUMNS.pop((table_name, column_name), None)
 
         # remove from orca's table list
@@ -501,7 +505,6 @@ def last_checkpoint():
         name of last checkpoint
     """
 
-    #fixme
     if not _PIPELINE.is_open:
         raise RuntimeError("Pipeline is not open!")
 
@@ -551,10 +554,11 @@ def run(models, resume_after=None):
 
     if resume_after == '_':
         resume_after = _PIPELINE.last_checkpoint[CHECKPOINT_NAME]
-        logger.info("Setting resume_after to %s" % (resume_after, ))
+
+    if resume_after:
+        logger.info('resume_after %s' % resume_after)
         if resume_after in models:
             models = models[models.index(resume_after) + 1:]
-        #bug
 
     # preload any bulky injectables (e.g. skims) not in pipeline
     if orca.is_injectable('preload_injectables'):