Skip to content

H
helm3
Commit 69f31b9e authored by Dave Cunningham's avatar Dave Cunningham
Browse files
Options

Merge pull request #444 from sparkprime/expandybird_imports 

Fix expandybird Python importing to be lazy
parents 8ec10d6e f553dc80
Hide whitespace changes
Inline Side-by-side
Showing with 31 additions and 105 deletions
expansion/sandbox_loader.py
View file @ 69f31b9e
......@@ -21,127 +21,39 @@ import os.path
import sys
class AllowedImportsLoader(object):
# Dictionary with modules loaded from user provided imports
user_modules = {}
_IMPORTS = {}
@staticmethod
def get_filename(name):
return '%s.py' % name.replace('.', '/')
class AllowedImportsLoader(object):
def load_module(self, name, etc=None): # pylint: disable=unused-argument
"""Implements loader.load_module()
for loading user provided imports."""
if name in AllowedImportsLoader.user_modules:
return AllowedImportsLoader.user_modules[name]
module = imp.new_module(name)
content = _IMPORTS[name]
try:
data = \
FileAccessRedirector.allowed_imports[self.get_filename(name)]
except Exception: # pylint: disable=broad-except
return None
# Run the module code.
exec data in module.__dict__ # pylint: disable=exec-used
AllowedImportsLoader.user_modules[name] = module
if content is None:
module.__path__ = [name.replace('.', '/')]
else:
# Run the module code.
exec content in module.__dict__ # pylint: disable=exec-used
# We need to register the module in module registry, since new_module
# doesn't do this, but we need it for hierarchical references.
# Register the module so Python code will find it.
sys.modules[name] = module
# If this module has children load them recursively.
if name in FileAccessRedirector.parents:
for child in FileAccessRedirector.parents[name]:
full_name = name + '.' + child
self.load_module(full_name)
# If we have helpers/common.py package,
# then for it to be successfully resolved helpers.common name
# must resolvable, hence, once we load
# child package we attach it to parent module immeadiately.
module.__dict__[child] = \
AllowedImportsLoader.user_modules[full_name]
return module
class AllowedImportsHandler(object):
def find_module(self, name, path=None): # pylint: disable=unused-argument
filename = AllowedImportsLoader.get_filename(name)
if filename in FileAccessRedirector.allowed_imports:
if name in _IMPORTS:
return AllowedImportsLoader()
else:
return None
def process_imports(imports):
"""Processes the imports by copying them and adding necessary parent packages.
Copies the imports and then for all the hierarchical packages creates
dummy entries for those parent packages, so that hierarchical imports
can be resolved. In the process parent child relationship map is built.
For example: helpers/extra/common.py will generate helpers,
helpers.extra and helpers.extra.common packages
along with related .py files.
Args:
imports: map of files to their relative paths.
Returns:
dictionary of imports to their contents and parent-child pacakge
relationship map.
"""
# First clone all the existing ones.
ret = {}
parents = {}
for k in imports:
ret[k] = imports[k]
# Now build the hierarchical modules.
for k in imports.keys():
path = imports[k]['path']
if path.endswith('.jinja'):
continue
# Normalize paths and trim .py extension, if any.
normalized = os.path.splitext(os.path.normpath(path))[0]
# If this is actually a path and not an absolute name,
# split it and process the hierarchical packages.
if sep in normalized:
parts = normalized.split(sep)
# Create dummy file entries for package levels and also retain
# parent-child relationships.
for i in xrange(0, len(parts)-1):
# Generate the partial package path.
path = os.path.join(parts[0], *parts[1:i+1])
# __init__.py file might have been provided and
# non-empty by the user.
if path not in ret:
# exec requires at least new line to be present
# to successfully compile the file.
ret[path + '.py'] = '\n'
else:
# To simplify our code, we'll store both versions
# in that case, since loader code expects files
# with .py extension.
ret[path + '.py'] = ret[path]
# Generate fully qualified package name.
fqpn = '.'.join(parts[0:i+1])
if fqpn in parents:
parents[fqpn].append(parts[i+1])
else:
parents[fqpn] = [parts[i+1]]
return ret, parents
return None # Delegate to system handlers.
class FileAccessRedirector(object):
# Dictionary with user provided imports.
allowed_imports = {}
# Dictionary that shows parent child relationships,
# key is the parent, value is the list of child packages.
parents = {}
@staticmethod
def redirect(imports):
......@@ -150,13 +62,27 @@ class FileAccessRedirector(object):
Imports already available in sys.modules will continue to be available.
Args:
imports: map from string to string, the map of imported files names
and contents.
imports: map from string to dict, the map of files from names.
"""
if imports is not None:
imps, parents = process_imports(imports)
FileAccessRedirector.allowed_imports = imps
FileAccessRedirector.parents = parents
# Build map of fully qualified module names to either the content
# of that module (if it is a file within a package) or just None if
# the module is a package (i.e. a directory).
for name, entry in imports.iteritems():
path = entry['path']
content = entry['content']
prefix, ext = os.path.splitext(os.path.normpath(path))
if ext not in {'.py', '.pyc'}:
continue
if '.' in prefix:
# Python modules cannot contain '.', ignore these files.
continue
parts = prefix.split(sep)
dirs = ('.'.join(parts[0:i]) for i in xrange(0, len(parts)))
for d in dirs:
if d not in _IMPORTS:
_IMPORTS[d] = None
_IMPORTS['.'.join(parts)] = content
# Prepend our module handler before standard ones.
sys.meta_path = [AllowedImportsHandler()] + sys.meta_path
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment