Merge pull request #444 from sparkprime/expandybird_imports

Fix expandybird Python importing to be lazy

expansion/sandbox_loader.py

@@ -21,127 +21,39 @@ import os.path
 import sys
 
 
-class AllowedImportsLoader(object):
-    # Dictionary with modules loaded from user provided imports
-    user_modules = {}
+_IMPORTS = {}
 
-    @staticmethod
-    def get_filename(name):
-        return '%s.py' % name.replace('.', '/')
+
+class AllowedImportsLoader(object):
 
     def load_module(self, name, etc=None):  # pylint: disable=unused-argument
         """Implements loader.load_module()
         for loading user provided imports."""
 
-        if name in AllowedImportsLoader.user_modules:
-            return AllowedImportsLoader.user_modules[name]
-
         module = imp.new_module(name)
+        content = _IMPORTS[name]
 
-        try:
-            data = \
-                FileAccessRedirector.allowed_imports[self.get_filename(name)]
-        except Exception:  # pylint: disable=broad-except
-            return None
-
-        # Run the module code.
-        exec data in module.__dict__    # pylint: disable=exec-used
-
-        AllowedImportsLoader.user_modules[name] = module
+        if content is None:
+            module.__path__ = [name.replace('.', '/')]
+        else:
+            # Run the module code.
+            exec content in module.__dict__   # pylint: disable=exec-used
 
-        # We need to register the module in module registry, since new_module
-        # doesn't do this, but we need it for hierarchical references.
+        # Register the module so Python code will find it.
         sys.modules[name] = module
-
-        # If this module has children load them recursively.
-        if name in FileAccessRedirector.parents:
-            for child in FileAccessRedirector.parents[name]:
-                full_name = name + '.' + child
-                self.load_module(full_name)
-                # If we have helpers/common.py package,
-                # then for it to be successfully resolved helpers.common name
-                #  must resolvable, hence, once we load
-                # child package we attach it to parent module immeadiately.
-                module.__dict__[child] = \
-                    AllowedImportsLoader.user_modules[full_name]
         return module
 
 
 class AllowedImportsHandler(object):
 
     def find_module(self, name, path=None):  # pylint: disable=unused-argument
-        filename = AllowedImportsLoader.get_filename(name)
-
-        if filename in FileAccessRedirector.allowed_imports:
+        if name in _IMPORTS:
             return AllowedImportsLoader()
         else:
-            return None
-
-
-def process_imports(imports):
-    """Processes the imports by copying them and adding necessary parent packages.
-
-        Copies the imports and then for all the hierarchical packages creates
-        dummy entries for those parent packages, so that hierarchical imports
-        can be resolved. In the process parent child relationship map is built.
-        For example: helpers/extra/common.py will generate helpers,
-        helpers.extra and helpers.extra.common packages
-        along with related .py files.
-
-    Args:
-        imports: map of files to their relative paths.
-    Returns:
-        dictionary of imports to their contents and parent-child pacakge
-        relationship map.
-    """
-    # First clone all the existing ones.
-    ret = {}
-    parents = {}
-    for k in imports:
-        ret[k] = imports[k]
-
-    # Now build the hierarchical modules.
-    for k in imports.keys():
-        path = imports[k]['path']
-        if path.endswith('.jinja'):
-            continue
-        # Normalize paths and trim .py extension, if any.
-        normalized = os.path.splitext(os.path.normpath(path))[0]
-        # If this is actually a path and not an absolute name,
-        # split it and process the hierarchical packages.
-        if sep in normalized:
-            parts = normalized.split(sep)
-            # Create dummy file entries for package levels and also retain
-            # parent-child relationships.
-            for i in xrange(0, len(parts)-1):
-                # Generate the partial package path.
-                path = os.path.join(parts[0], *parts[1:i+1])
-                # __init__.py file might have been provided and
-                # non-empty by the user.
-                if path not in ret:
-                    # exec requires at least new line to be present
-                    # to successfully compile the file.
-                    ret[path + '.py'] = '\n'
-                else:
-                    # To simplify our code, we'll store both versions
-                    # in that case, since loader code expects files
-                    # with .py extension.
-                    ret[path + '.py'] = ret[path]
-                # Generate fully qualified package name.
-                fqpn = '.'.join(parts[0:i+1])
-                if fqpn in parents:
-                    parents[fqpn].append(parts[i+1])
-                else:
-                    parents[fqpn] = [parts[i+1]]
-    return ret, parents
+            return None  # Delegate to system handlers.
 
 
 class FileAccessRedirector(object):
-    # Dictionary with user provided imports.
-    allowed_imports = {}
-    # Dictionary that shows parent child relationships,
-    # key is the parent, value is the list of child packages.
-    parents = {}
 
     @staticmethod
     def redirect(imports):
@@ -150,13 +62,27 @@ class FileAccessRedirector(object):
         Imports already available in sys.modules will continue to be available.
 
         Args:
-            imports: map from string to string, the map of imported files names
-                    and contents.
+            imports: map from string to dict, the map of files from names.
         """
         if imports is not None:
-            imps, parents = process_imports(imports)
-            FileAccessRedirector.allowed_imports = imps
-            FileAccessRedirector.parents = parents
+            # Build map of fully qualified module names to either the content
+            # of that module (if it is a file within a package) or just None if
+            # the module is a package (i.e. a directory).
+            for name, entry in imports.iteritems():
+                path = entry['path']
+                content = entry['content']
+                prefix, ext = os.path.splitext(os.path.normpath(path))
+                if ext not in {'.py', '.pyc'}:
+                    continue
+                if '.' in prefix:
+                    # Python modules cannot contain '.', ignore these files.
+                    continue
+                parts = prefix.split(sep)
+                dirs = ('.'.join(parts[0:i]) for i in xrange(0, len(parts)))
+                for d in dirs:
+                    if d not in _IMPORTS:
+                        _IMPORTS[d] = None
+                _IMPORTS['.'.join(parts)] = content
 
             # Prepend our module handler before standard ones.
             sys.meta_path = [AllowedImportsHandler()] + sys.meta_path