Optional cached imports

docs/functionality.rst

@@ -89,7 +89,10 @@ containing those import statements at ``config.imports``, as a list of
 To get the imported objects, these imports must be resolved, which can be done through
 the :func:`ymmsl.v0_2.resolve` function. This takes the name of the module corresponding
 to a ``Configuration`` (which is the file name without the ``.ymmsl`` extension, wrapped
-in a :class:`ymmsl.v0_2.Reference`) and the configuration itself.
+in a :class:`ymmsl.v0_2.Reference`), the configuration itself, and an optional
+``reuse_cached_imports`` keyword argument. By default, imports are cached between calls to
+avoid re-reading previously loaded modules. Set ``reuse_cached_imports=False`` when the
+search path is changed (``YMMSL_PATH`` or ``sys.path``) to avoid stale module imports.
 
 It updates that configuration in place, renaming any local models and programs to their
 absolute name by prefixing them with the given module name, then imports the

ymmsl/v0_2/resolver.py

@@ -89,7 +89,9 @@ def search_paths(self, rel_path: Path, indent_depth: int) -> str:
         )
 
 
-def resolve(module: Reference, config: Configuration) -> None:
+def resolve(
+    module: Reference, config: Configuration, reuse_cached_imports: bool = True
+) -> None:
     """Resolve imports for the given configuration.
 
     This updates the given configuration in place, removing all import statements and
@@ -100,11 +102,16 @@ def resolve(module: Reference, config: Configuration) -> None:
     Args:
         module: The module corresponding to this configuration
         config: The configuration to resolve
+        reuse_cached_imports: If cached imports should be reused from previous calls of
+            this method. Disable if YMMSL_PATH or sys.path changes.
 
     Raises:
         RuntimeError: if an error occurs due to an invalid configuration. This will
             leave config in a broken state, so reload it if you want to try again.
     """
+    if not reuse_cached_imports:
+        ymmsl_cache.clear()
+
     overwritten = do_resolve(Path("<main>"), module, config, ResolutionContext())
     remove_overwritten_implementations(config, overwritten)
 

ymmsl/v0_2/tests/test_resolver.py

@@ -1,6 +1,6 @@
 import logging
 import os
-from collections.abc import Generator
+from collections.abc import Callable, Generator
 from importlib.metadata import EntryPoint, EntryPoints
 from pathlib import Path
 from unittest.mock import Mock, patch
@@ -10,9 +10,25 @@
 from ymmsl.io import load
 from ymmsl.v0_2.configuration import Configuration
 from ymmsl.v0_2.identity import Reference
-from ymmsl.v0_2.resolver import resolve
+from ymmsl.v0_2.resolver import resolve as resolve_impl, ymmsl_cache
 
 Ref = Reference
+Resolve = Callable[[Reference, Configuration], None]
+
+
+@pytest.fixture(autouse=True)
+def clear_resolver_cache() -> Generator[None, None, None]:
+    ymmsl_cache.clear()
+    yield
+    ymmsl_cache.clear()
+
+
+@pytest.fixture(params=[False, True])
+def resolve(request: pytest.FixtureRequest) -> Resolve:
+    def resolve_config(module: Reference, config: Configuration) -> None:
+        resolve_impl(module, config, reuse_cached_imports=request.param)
+
+    return resolve_config
 
 
 @pytest.fixture
@@ -29,7 +45,7 @@ def env_ymmsl_path() -> Generator[None, None, None]:
     del os.environ["YMMSL_PATH"]
 
 
-def test_resolve_imports(env_ymmsl_path: None) -> None:
+def test_resolve_imports(env_ymmsl_path: None, resolve: Resolve) -> None:
     ymmsl = (
         "ymmsl_version: v0.2\n"
         "description: Testing resolving imports\n"
@@ -51,7 +67,9 @@ def test_resolve_imports(env_ymmsl_path: None) -> None:
     assert config.programs[Reference("a.b.c.macro")].executable == Path("my_program")
 
 
-def test_apply_custom_implementations_simple(env_ymmsl_path: None) -> None:
+def test_apply_custom_implementations_simple(
+    env_ymmsl_path: None, resolve: Resolve
+) -> None:
     # should import a model, and substitute a local program into it
     # then check that we have a single root model
     ymmsl = (
@@ -77,7 +95,7 @@ def test_apply_custom_implementations_simple(env_ymmsl_path: None) -> None:
     assert c.implementation == "a.b.c.macro"
 
 
-def test_apply_custom_implementations(env_ymmsl_path: None) -> None:
+def test_apply_custom_implementations(env_ymmsl_path: None, resolve: Resolve) -> None:
     ymmsl = (
         "ymmsl_version: v0.2\n"
         "description: Testing resolving imports with custom_implementations\n"
@@ -119,7 +137,9 @@ def test_apply_custom_implementations(env_ymmsl_path: None) -> None:
     assert config.programs[Reference("a.g.macro2")].args == ["/home/user/macro2.py"]
 
 
-def test_apply_custom_implementations_double_use(env_ymmsl_path: None) -> None:
+def test_apply_custom_implementations_double_use(
+    env_ymmsl_path: None, resolve: Resolve
+) -> None:
     ymmsl = (
         "ymmsl_version: v0.2\n"
         "description: |\n"
@@ -171,7 +191,9 @@ def test_apply_custom_implementations_double_use(env_ymmsl_path: None) -> None:
     assert config.programs[Reference("a.h.macro2")].args == ["/home/user/macro2.py"]
 
 
-def test_apply_custom_implementations_set_none(env_ymmsl_path: None) -> None:
+def test_apply_custom_implementations_set_none(
+    env_ymmsl_path: None, resolve: Resolve
+) -> None:
     ymmsl = (
         "ymmsl_version: v0.2\n"
         "description: Testing resolving imports with custom_implementations\n"
@@ -203,7 +225,7 @@ def test_apply_custom_implementations_set_none(env_ymmsl_path: None) -> None:
     assert model.components[Reference("micro")].implementation is None
 
 
-def test_apply_custom_implementations_no_hidden_copies() -> None:
+def test_apply_custom_implementations_no_hidden_copies(resolve: Resolve) -> None:
     ymmsl = (
         "ymmsl_version: v0.2\n"
         "description: |\n"
@@ -276,7 +298,9 @@ def test_apply_custom_implementations_no_hidden_copies() -> None:
     assert a.components[Reference("c1")].implementation == "no_copies2.p"
 
 
-def test_apply_custom_implementations_everything_localised() -> None:
+def test_apply_custom_implementations_everything_localised(
+    env_ymmsl_path: None, resolve: Resolve
+) -> None:
     ymmsl = (
         "ymmsl_version: v0.2\n"
         "description: |\n"
@@ -333,7 +357,9 @@ def test_apply_custom_implementations_everything_localised() -> None:
     assert b.components[Ref("macro")].implementation == "el.p"
 
 
-def test_apply_custom_implementations_cut_branch(env_ymmsl_path: None) -> None:
+def test_apply_custom_implementations_cut_branch(
+    env_ymmsl_path: None, resolve: Resolve
+) -> None:
     ymmsl = (
         "ymmsl_version: v0.2\n"
         "description: |\n"
@@ -369,7 +395,9 @@ def test_apply_custom_implementations_cut_branch(env_ymmsl_path: None) -> None:
     assert len(config.models) == 1
 
 
-def test_apply_custom_implementations_errors(env_ymmsl_path: None) -> None:
+def test_apply_custom_implementations_errors(
+    env_ymmsl_path: None, resolve: Resolve
+) -> None:
     ymmsl = (
         "ymmsl_version: v0.2\n"
         "description: |\n"
@@ -445,7 +473,9 @@ def test_apply_custom_implementations_errors(env_ymmsl_path: None) -> None:
         resolve(Reference("test_resolve_imports"), config)
 
 
-def test_resolve_imports_module_not_found(env_ymmsl_path: None) -> None:
+def test_resolve_imports_module_not_found(
+    env_ymmsl_path: None, resolve: Resolve
+) -> None:
     ymmsl = (
         "ymmsl_version: v0.2\n"
         "description: Testing missing modules\n"
@@ -463,7 +493,7 @@ def test_resolve_imports_module_not_found(env_ymmsl_path: None) -> None:
     assert len(config.imports) == 1
 
 
-def test_resolve_imports_broken_module(env_ymmsl_path: None) -> None:
+def test_resolve_imports_broken_module(env_ymmsl_path: None, resolve: Resolve) -> None:
     ymmsl = (
         "ymmsl_version: v0.2\n"
         "description: Testing missing modules\n"
@@ -481,7 +511,9 @@ def test_resolve_imports_broken_module(env_ymmsl_path: None) -> None:
     assert len(config.imports) == 1
 
 
-def test_resolve_imports_implementation_not_found(env_ymmsl_path: None) -> None:
+def test_resolve_imports_implementation_not_found(
+    env_ymmsl_path: None, resolve: Resolve
+) -> None:
     ymmsl = (
         "ymmsl_version: v0.2\n"
         "description: Testing missing modules\n"
@@ -499,7 +531,7 @@ def test_resolve_imports_implementation_not_found(env_ymmsl_path: None) -> None:
     assert len(config.imports) == 1
 
 
-def test_resolve_imports_no_shadowing(env_ymmsl_path: None) -> None:
+def test_resolve_imports_no_shadowing(env_ymmsl_path: None, resolve: Resolve) -> None:
     ymmsl = (
         "ymmsl_version: v0.2\n"
         "description: Testing resolving imports\n"
@@ -551,7 +583,7 @@ def mock_entry_points() -> Generator[Mock]:
         yield mock_entry_points
 
 
-def test_resolve_entrypoints(mock_entry_points: Mock) -> None:
+def test_resolve_entrypoints(mock_entry_points: Mock, resolve: Resolve) -> None:
     config = load("""
         ymmsl_version: v0.2
         description: Test
@@ -568,8 +600,9 @@ def test_resolve_entrypoints(mock_entry_points: Mock) -> None:
 
 
 def test_resolve_entrypoints_duplicate_name(
-    mock_entry_points: Mock, caplog: pytest.LogCaptureFixture
+    mock_entry_points: Mock, caplog: pytest.LogCaptureFixture, resolve: Resolve
 ) -> None:
+
     config = load("""
         ymmsl_version: v0.2
         description: Test
@@ -593,7 +626,9 @@ def test_resolve_entrypoints_duplicate_name(
     assert config.programs[test_importing].args == ["test_program.py"]
 
 
-def test_resolve_entrypoints_loading_error(mock_entry_points: Mock) -> None:
+def test_resolve_entrypoints_loading_error(
+    mock_entry_points: Mock, resolve: Resolve
+) -> None:
     config = load("""
         ymmsl_version: v0.2
         description: Test
@@ -603,3 +638,48 @@ def test_resolve_entrypoints_loading_error(mock_entry_points: Mock) -> None:
     assert isinstance(config, Configuration)
     with pytest.raises(RuntimeError, match="Error while loading the entrypoint"):
         resolve(Reference("test_importing"), config)
+
+
+def test_resolve_cache_after_ymmsl_path_change() -> None:
+    cur_dir = Path(__file__).parent
+    ymmsl1 = cur_dir / "ymmsl1"
+    ymmsl_other = cur_dir / "ymmsl_other"
+
+    ymmsl = (
+        "ymmsl_version: v0.2\n"
+        "description: Testing cache after changing YMMSL_PATH\n"
+        "imports:\n"
+        "- from a.d import implementation test_importing\n"
+    )
+
+    os.environ["YMMSL_PATH"] = str(ymmsl1)
+
+    config = load(ymmsl)
+    assert isinstance(config, Configuration)
+    resolve_impl(Reference("test_importing"), config)
+
+    module_path = Path("a/d.ymmsl")
+    assert module_path in ymmsl_cache
+    assert ymmsl_cache[module_path][1] == ymmsl1 / module_path
+
+    os.environ["YMMSL_PATH"] = str(ymmsl_other)
+
+    config = load(ymmsl)
+    assert isinstance(config, Configuration)
+    resolve_impl(
+        Reference("test_importing"),
+        config,
+        reuse_cached_imports=True,
+    )
+
+    assert ymmsl_cache[module_path][1] == ymmsl1 / module_path
+
+    config = load(ymmsl)
+    assert isinstance(config, Configuration)
+    resolve_impl(
+        Reference("test_importing"),
+        config,
+        reuse_cached_imports=False,
+    )
+
+    assert ymmsl_cache[module_path][1] == ymmsl_other / module_path

ymmsl/v0_2/tests/ymmsl_other/a/d.ymmsl

@@ -0,0 +1,41 @@
+ymmsl_version: v0.2
+
+description: A model to be imported, but in ymmsl_other dir
+
+imports:
+
+
+models:
+  test_importing:
+    ports:
+      f_init: in
+      o_f: out
+    components:
+      macro1:
+        ports:
+          f_init: init
+          o_i: out1
+          s: in1
+          o_f: final
+        description: The macro1 component
+      micro1:
+        ports:
+          f_init: init
+          o_f: final
+        description: The micro1 component
+        implementation: micro
+
+    conduits:
+      in: macro1.init
+      macro1.out1: micro1.init
+      micro1.final: macro1.in1
+      macro1.final: out
+
+programs:
+  micro:
+    ports:
+      f_init: init
+      o_f: final
+    description: The micro program
+    executable: python3
+    args: /home/user/script.py