Allow non-toplevel modules in python environment (fission#1042)

environments/python/server.py

@@ -1,12 +1,24 @@
 #!/usr/bin/env python
-
+import importlib
 import logging
-import sys
-import imp
 import os
-import bjoern
-from gevent.pywsgi import WSGIServer
+import sys
+
 from flask import Flask, request, abort, g
+from gevent.pywsgi import WSGIServer
+import bjoern
+
+
+IS_PY2 = (sys.version_info.major == 2)
+
+
+def import_src(path):
+ if IS_PY2:
+ import imp
+ return imp.load_source('mod', path)
+ else:
+ # the imp module is deprecated in Python3. use importlib instead.
+ return importlib.machinery.SourceFileLoader('mod', path).load_module()
 
 
 class FuncApp(Flask):
@@ -34,48 +46,46 @@ def __init__(self, name, loglevel=logging.DEBUG):
  #
  @self.route('/specialize', methods=['POST'])
  def load():
+ self.logger.info('/specialize called')
  # load user function from codepath
- codepath = '/userfunc/user'
- self.userfunc = (imp.load_source('user', codepath)).main
+ self.userfunc = import_src('/userfunc/user').main
  return ""
 
  @self.route('/v2/specialize', methods=['POST'])
  def loadv2():
  body = request.get_json()
  filepath = body['filepath']
  handler = body['functionName']
-
- # The value of "functionName" is consist of
- # `<module-name>.<function-name>`.
- moduleName, funcName = handler.split(".")
-
+ self.logger.info('/v2/specialize called with filepath = "{}" handler = "{}"'.format(filepath, handler))
+
+ # handler looks like `path.to.module.function`
+ parts = handler.rsplit(".", 1)
+ if len(handler) == 0:
+ # default to main.main if entrypoint wasn't provided
+ moduleName = 'main'
+ funcName = 'main'
+ elif len(parts) == 1:
+ moduleName = 'main'
+ funcName = parts[0]
+ else:
+ moduleName = parts[0]
+ funcName = parts[1]
+ self.logger.debug('moduleName = "{}" funcName = "{}"'.format(moduleName, funcName))
+
  # check whether the destination is a directory or a file
  if os.path.isdir(filepath):
  # add package directory path into module search path
  sys.path.append(filepath)
 
- # find module from package path we append previously.
- # Python will try to find module from the same name
- # file under the package directory. If search is
- # successful, the return value is a 3-element tuple;
- # otherwise, an exception "ImportError" is raised.
- # Second parameter of find_module enforces python to
- # find same name module from the given list of
- # directories to prevent name confliction with
- # built-in modules.
- f, path, desc = imp.find_module(moduleName, [filepath])
-
- # load module
- # Return module object is the load is successful;
- # otherwise, an exception is raised.
- try:
- mod = imp.load_module(moduleName, f, path, desc)
- finally:
- if f:
- f.close()
+ self.logger.debug('__package__ = "{}"'.format(__package__))
+ if __package__:
+ mod = importlib.import_module(moduleName, __package__)
+ else:
+ mod = importlib.import_module(moduleName)
+
  else:
  # load source from destination python file
- mod = imp.load_source(moduleName, filepath)
+ mod = import_src(filepath)
 
  # load user function from module
  self.userfunc = getattr(mod, funcName)

test/test_utils.sh

@@ -463,6 +463,7 @@ run_all_tests() {
  $ROOT/test/tests/test_buildermgr.sh \
  $ROOT/test/tests/test_canary.sh \
  $ROOT/test/tests/test_env_vars.sh \
+ $ROOT/test/tests/test_environments/test_python_env.sh \
  $ROOT/test/tests/test_fn_update/test_idle_objects_reaper.sh \
  $ROOT/test/tests/test_function_test/test_fn_test.sh \
  $ROOT/test/tests/test_function_update.sh \

test/tests/test_environments/python_src/foo.py

@@ -0,0 +1,2 @@
+def bar():
+ return 'THIS_IS_FOO_BAR'

test/tests/test_environments/python_src/main.py

@@ -0,0 +1,5 @@
+def main():
+ return 'THIS_IS_MAIN_MAIN'
+
+def func():
+ return 'THIS_IS_MAIN_FUNC'

test/tests/test_environments/python_src/sub_mod/altmain.py

@@ -0,0 +1,2 @@
+def entrypoint():
+ return 'THIS_IS_ALTMAIN_ENTRYPOINT'

test/tests/test_environments/test_python_env.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -euo pipefail
+source $(dirname $0)/../../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
+
+ROOT=$(dirname $0)/../../..
+
+cleanup() {
+ clean_resource_by_id $TEST_ID
+ rm -rf $tmp_dir
+}
+
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+ trap cleanup EXIT
+else
+ log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+env_v1api=python-v1-$TEST_ID
+env_v2api=python-v2-$TEST_ID
+fn1=test-python-env-1-$TEST_ID
+fn2=test-python-env-2-$TEST_ID
+fn3=test-python-env-3-$TEST_ID
+fn4=test-python-env-4-$TEST_ID
+fn5=test-python-env-5-$TEST_ID
+pkg=
+
+
+log "Creating v1api environment ..."
+log "PYTHON_RUNTIME_IMAGE = $PYTHON_RUNTIME_IMAGE"
+fission env create \
+ --name $env_v1api \
+ --image $PYTHON_RUNTIME_IMAGE \
+
+log "Creating v2api environment ..."
+log "PYTHON_RUNTIME_IMAGE = $PYTHON_RUNTIME_IMAGE PYTHON_BUILDER_IMAGE = $PYTHON_BUILDER_IMAGE"
+fission env create \
+ --name $env_v2api \
+ --image $PYTHON_RUNTIME_IMAGE \
+ --builder $PYTHON_BUILDER_IMAGE
+timeout 180s bash -c "wait_for_builder $env_v2api"
+
+log "Creating package ..."
+pushd $ROOT/test/tests/test_environments/python_src/
+zip -r $tmp_dir/src-pkg.zip *
+popd
+pkg=$(fission package create --src $tmp_dir/src-pkg.zip --env $env_v2api | cut -f2 -d' '| tr -d \')
+timeout 60s bash -c "waitBuild $pkg"
+
+
+log "===== 1. test env with v1 api ====="
+fission fn create --name $fn1 --env $env_v1api --code $ROOT/examples/python/hello.py
+fission route create --function $fn1 --url /$fn1 --method GET
+sleep 3 # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn1 'Hello, world!'"
+
+
+log "===== 2. test entrypoint = '' ====="
+fission fn create --name $fn2 --env $env_v2api --pkg $pkg
+fission route create --function $fn2 --url /$fn2 --method GET
+sleep 3 # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn2 'THIS_IS_MAIN_MAIN'"
+
+
+log "===== 3. test entrypoint = func ====="
+fission fn create --name $fn3 --env $env_v2api --pkg $pkg --entrypoint func
+fission route create --function $fn3 --url /$fn3 --method GET
+sleep 3 # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn3 'THIS_IS_MAIN_FUNC'"
+
+
+log "===== 4. test entrypoint = foo.bar ====="
+fission fn create --name $fn4 --env $env_v2api --pkg $pkg --entrypoint foo.bar
+fission route create --function $fn4 --url /$fn4 --method GET
+sleep 3 # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn4 'THIS_IS_FOO_BAR'"
+
+
+log "===== 5. test entrypoint = sub_mod.altmain.entrypoint ====="
+fission fn create --name $fn5 --env $env_v2api --pkg $pkg --entrypoint sub_mod.altmain.entrypoint
+fission route create --function $fn5 --url /$fn5 --method GET
+sleep 3 # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn5 'THIS_IS_ALTMAIN_ENTRYPOINT'"
+
+
+log "Test PASSED"