[msm] use Ipython's ShimModule approach to warn upon access of a moved module. (#805)

* [msm] use IPython's ShimModule approach to warn upon access of a moved module.

* This addresses #550. Remove those packages again in version 2.3
* remove io->dtraj alias (has already been deprecated ages ago)
* fix #806

Author: marscher

pyemma/msm/__init__.py

@@ -84,14 +84,21 @@
    msm.flux
 
 """
-from __future__ import absolute_import, print_function
+from __future__ import absolute_import as _
 
 #####################################################
 # Low-level MSM functions (imported from msmtools)
 # backward compatibility to PyEMMA 1.2.x
-from msmtools import analysis, estimation, generation, dtraj, flux
-from msmtools.analysis.dense.pcca import PCCA
+# TODO: finally remove this stuff...
+import warnings as _warnings
+from pyemma.util.exceptions import PyEMMA_DeprecationWarning as _dep_warning
+with _warnings.catch_warnings():
+    _warnings.filterwarnings('ignore', category=_dep_warning)
+    from . import analysis, estimation, generation, dtraj, flux
 io = dtraj
+del _warnings, _dep_warning
+######################################################
+from msmtools.analysis.dense.pcca import PCCA
 
 #####################################################
 # Estimators and models

pyemma/msm/analysis/__init__.py

@@ -0,0 +1,12 @@
+import sys
+import warnings
+
+from pyemma.util._ext.shimmodule import ShimModule
+from pyemma.util.exceptions import PyEMMA_DeprecationWarning
+
+warnings.warn("The pyemma.msm.analysis module has been deprecated. "
+              "You should import msmtools.analysis now.", PyEMMA_DeprecationWarning)
+
+sys.modules['pyemma.msm.analysis'] = ShimModule(src='pyemma.msm.analysis', mirror='msmtools.analysis')
+
+from msmtools.analysis import *

pyemma/msm/dtraj/__init__.py

@@ -0,0 +1,12 @@
+import sys
+import warnings
+
+from pyemma.util._ext.shimmodule import ShimModule
+from pyemma.util.exceptions import PyEMMA_DeprecationWarning
+
+warnings.warn("The pyemma.msm.dtraj module has been deprecated. "
+              "You should import msmtools.dtraj now.", PyEMMA_DeprecationWarning)
+
+sys.modules['pyemma.msm.dtraj'] = ShimModule(src='pyemma.msm.dtraj', mirror='msmtools.dtraj')
+#sys.modules['pyemma.msm.io'] = sys.modules['pyemma.msm.dtraj']
+

pyemma/msm/estimation/__init__.py

@@ -0,0 +1,11 @@
+import sys
+import warnings
+
+from pyemma.util._ext.shimmodule import ShimModule
+from pyemma.util.exceptions import PyEMMA_DeprecationWarning
+
+warnings.warn("The pyemma.msm.estimation module has been deprecated. "
+              "You should import msmtools.estimation now.", PyEMMA_DeprecationWarning)
+
+sys.modules['pyemma.msm.estimation'] = ShimModule(src='pyemma.msm.estimation', mirror='msmtools.estimation')
+

pyemma/msm/flux/__init__.py

@@ -0,0 +1,11 @@
+import sys
+import warnings
+
+from pyemma.util._ext.shimmodule import ShimModule
+from pyemma.util.exceptions import PyEMMA_DeprecationWarning
+
+warnings.warn("The pyemma.msm.flux module has been deprecated. "
+              "You should import msmtools.flux now.", PyEMMA_DeprecationWarning)
+
+sys.modules['pyemma.msm.flux'] = ShimModule(src='pyemma.msm.flux', mirror='msmtools.flux')
+

pyemma/msm/generation/__init__.py

@@ -0,0 +1,11 @@
+import sys
+import warnings
+
+from pyemma.util._ext.shimmodule import ShimModule
+from pyemma.util.exceptions import PyEMMA_DeprecationWarning
+
+warnings.warn("The pyemma.msm.generation module has been deprecated. "
+              "You should import msmtools.generation now.", PyEMMA_DeprecationWarning)
+
+sys.modules['pyemma.msm.generation'] = ShimModule(src='pyemma.msm.generation', mirror='msmtools.generation')
+

pyemma/msm/tests/test_msm_lowlevel_deprecation.py

@@ -0,0 +1,142 @@
+import sys
+import unittest
+import warnings
+
+import mock
+
+import pyemma
+from pyemma.util.exceptions import PyEMMA_DeprecationWarning
+
+
+@unittest.skipIf(sys.version_info.major == 2, "disabled on py2 for nosetest stupidness")
+class TestShowDeprecationWarningOnLowLevelAPIUsage(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.old_filters = warnings.filters[:]
+        if sys.version_info.major == 2:
+            warnings.filters = []
+
+    @classmethod
+    def tearDownClass(cls):
+        warnings.filters = cls.old_filters
+
+    def test_analysis(self):
+        with warnings.catch_warnings(record=True) as cm:
+            warnings.simplefilter("always")
+            from pyemma.msm import analysis
+            analysis.is_transition_matrix
+
+        self.assertEqual(len(cm), 1)
+        self.assertIn('analysis', cm[0].message.args[0])
+
+        with warnings.catch_warnings(record=True) as cm:
+            warnings.simplefilter("always")
+            pyemma.msm.analysis.is_transition_matrix
+        self.assertEqual(len(cm), 1)
+        self.assertIsInstance(cm[0].message, PyEMMA_DeprecationWarning)
+        self.assertIn('analysis', cm[0].message.args[0])
+
+    @unittest.skipIf(sys.version_info.major == 2, "not on py2")
+    def test_warn_was_called(self):
+        shim_mod = sys.modules['pyemma.msm.analysis']
+        with mock.patch.object(shim_mod, '_warn') as m:
+            from pyemma.msm import analysis
+            analysis.is_transition_matrix
+
+        m.assert_called_once()
+
+    def test_estimation(self):
+        with warnings.catch_warnings(record=True) as cm:
+            warnings.simplefilter("always")
+            from pyemma.msm import estimation
+            estimation.count_matrix
+
+        self.assertEqual(len(cm), 1)
+        self.assertIsInstance(cm[0].message, PyEMMA_DeprecationWarning)
+
+        self.assertIn('estimation', cm[0].message.args[0])
+
+        with warnings.catch_warnings(record=True) as cm:
+            warnings.simplefilter("always")
+            pyemma.msm.estimation.count_matrix
+        self.assertEqual(len(cm), 1)
+        self.assertIsInstance(cm[0].message, PyEMMA_DeprecationWarning)
+
+        self.assertIn('estimation', cm[0].message.args[0])
+
+    def test_generation(self):
+        with warnings.catch_warnings(record=True) as cm:
+            warnings.simplefilter("always")
+            from pyemma.msm import generation
+            generation.generate_traj
+    
+        self.assertEqual(len(cm), 1)
+        self.assertIsInstance(cm[0].message, PyEMMA_DeprecationWarning)
+
+        self.assertIn('generation', cm[0].message.args[0])
+    
+        with warnings.catch_warnings(record=True) as cm:
+            warnings.simplefilter("always")
+            pyemma.msm.generation.generate_traj
+        self.assertEqual(len(cm), 1)
+        self.assertIsInstance(cm[0].message, PyEMMA_DeprecationWarning)
+
+        self.assertIn('generation', cm[0].message.args[0])
+
+    def test_dtraj(self):
+        with warnings.catch_warnings(record=True) as cm:
+            warnings.simplefilter("always")
+            from pyemma.msm import dtraj
+            dtraj.load_discrete_trajectory
+    
+        self.assertEqual(len(cm), 1)
+        self.assertIsInstance(cm[0].message, PyEMMA_DeprecationWarning)
+
+        self.assertIn('dtraj', cm[0].message.args[0])
+    
+        with warnings.catch_warnings(record=True) as cm:
+            warnings.simplefilter("always")
+            pyemma.msm.dtraj.load_discrete_trajectory
+        self.assertEqual(len(cm), 1)
+        self.assertIsInstance(cm[0].message, PyEMMA_DeprecationWarning)
+
+        self.assertIn('dtraj', cm[0].message.args[0])
+
+    def test_io(self):
+        with warnings.catch_warnings(record=True) as cm:
+            warnings.simplefilter("always")
+            from pyemma.msm import io as dtraj
+            dtraj.load_discrete_trajectory
+
+        self.assertEqual(len(cm), 1)
+        self.assertIsInstance(cm[0].message, PyEMMA_DeprecationWarning)
+
+        self.assertIn('dtraj', cm[0].message.args[0])
+
+        with warnings.catch_warnings(record=True) as cm:
+            warnings.simplefilter("always")
+            pyemma.msm.dtraj.load_discrete_trajectory
+        self.assertEqual(len(cm), 1)
+        self.assertIsInstance(cm[0].message, PyEMMA_DeprecationWarning)
+
+        self.assertIn('dtraj', cm[0].message.args[0])
+
+    def test_flux(self):
+        with warnings.catch_warnings(record=True) as cm:
+            warnings.simplefilter("always")
+            from pyemma.msm import flux
+            flux.total_flux
+    
+        self.assertEqual(len(cm), 1)
+        self.assertIsInstance(cm[0].message, PyEMMA_DeprecationWarning)
+
+        self.assertIn('flux', cm[0].message.args[0])
+    
+        with warnings.catch_warnings(record=True) as cm:
+            warnings.simplefilter("always")
+            pyemma.msm.flux.total_flux
+        self.assertEqual(len(cm), 1)
+        self.assertIsInstance(cm[0].message, PyEMMA_DeprecationWarning)
+
+        self.assertIn('flux', cm[0].message.args[0])

pyemma/util/_ext/__init__.py

@@ -0,0 +1,131 @@
+"""A shim module for deprecated imports
+"""
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import sys
+import types
+import warnings
+
+
+def import_item(name):
+    """Import and return ``bar`` given the string ``foo.bar``.
+    Calling ``bar = import_item("foo.bar")`` is the functional equivalent of
+    executing the code ``from foo import bar``.
+    Parameters
+    ----------
+    name : string
+      The fully qualified name of the module/package being imported.
+    Returns
+    -------
+    mod : module object
+       The module that was imported.
+    """
+
+    parts = name.rsplit('.', 1)
+    if len(parts) == 2:
+        # called with 'foo.bar....'
+        package, obj = parts
+        module = __import__(package, fromlist=[obj])
+        try:
+            pak = getattr(module, obj)
+        except AttributeError:
+            raise ImportError('No module named %s' % obj)
+        return pak
+    else:
+        # called with un-dotted string
+        return __import__(parts[0])
+
+
+class ShimImporter(object):
+    """Import hook for a shim.
+
+    This ensures that submodule imports return the real target module,
+    not a clone that will confuse `is` and `isinstance` checks.
+    """
+
+    def __init__(self, src, mirror):
+        self.src = src
+        self.mirror = mirror
+
+    def _mirror_name(self, fullname):
+        """get the name of the mirrored module"""
+
+        return self.mirror + fullname[len(self.src):]
+
+    def find_module(self, fullname, path=None):
+        """Return self if we should be used to import the module."""
+        if fullname.startswith(self.src + '.'):
+            mirror_name = self._mirror_name(fullname)
+            try:
+                mod = import_item(mirror_name)
+            except ImportError:
+                return
+            else:
+                if not isinstance(mod, types.ModuleType):
+                    # not a module
+                    return None
+                return self
+
+    def load_module(self, fullname):
+        """Import the mirrored module, and insert it into sys.modules"""
+        mirror_name = self._mirror_name(fullname)
+        mod = import_item(mirror_name)
+        sys.modules[fullname] = mod
+        return mod
+
+
+class ShimModule(types.ModuleType):
+    def __init__(self, *args, **kwargs):
+        self._mirror = kwargs.pop("mirror")
+        src = kwargs.pop("src", None)
+        if src:
+            kwargs['name'] = src.rsplit('.', 1)[-1]
+        super(ShimModule, self).__init__(*args, **kwargs)
+        # add import hook for descendent modules
+        if src:
+            sys.meta_path.append(
+                ShimImporter(src=src, mirror=self._mirror)
+            )
+        self.msg = kwargs.pop("msg", None)
+        self.default_msg = "Access to a moved module '%s' detected!" \
+                           " Please use '%s' in the future." % (src, self._mirror)
+
+    @property
+    def __path__(self):
+        return []
+
+    @property
+    def __spec__(self):
+        """Don't produce __spec__ until requested"""
+        self._warn()
+        return __import__(self._mirror).__spec__
+
+    def __dir__(self):
+        self._warn()
+        return dir(__import__(self._mirror))
+
+    @property
+    def __all__(self):
+        """Ensure __all__ is always defined"""
+        self._warn()
+        mod = __import__(self._mirror)
+        try:
+            return mod.__all__
+        except AttributeError:
+            return [name for name in dir(mod) if not name.startswith('_')]
+
+    def __getattr__(self, key):
+        # Use the equivalent of import_item(name), see below
+        name = "%s.%s" % (self._mirror, key)
+        try:
+            item = import_item(name)
+            self._warn()
+            return item
+        except ImportError:
+            raise AttributeError(key)
+
+    def _warn(self):
+        from pyemma.util.exceptions import PyEMMA_DeprecationWarning
+        warnings.warn(self.msg if self.msg else self.default_msg,
+                      category=PyEMMA_DeprecationWarning)

pyemma/util/_ext/shimmodule.py

@@ -39,6 +39,8 @@ def foo(self):
 from decorator import decorator, decorate
 from inspect import stack
 
+from pyemma.util.exceptions import PyEMMA_DeprecationWarning
+
 __all__ = ['alias',
            'aliased',
            'deprecated',
@@ -213,7 +215,7 @@ def _deprecated(func, *args, **kw):
 
         warnings.warn_explicit(
             user_msg,
-            category=DeprecationWarning,
+            category=PyEMMA_DeprecationWarning,
             filename=filename,
             lineno=lineno
         )