Lambci

cmd/bench.go

@@ -34,7 +34,7 @@ functions and configured the provider.`,
 			benchCmdConfig.functionArgs,
 			benchCmdConfig.benchParams,
 			benchCmdConfig.trackingUrl,
-			benchCmdConfig.benchName + ".out",
+			benchCmdConfig.logFile,
 		}
 
 		switch benchCmdConfig.benchName {
@@ -96,7 +96,7 @@ func init() {
 	benchCmd.Flags().StringVarP(&benchCmdConfig.functionArgs, "function-args", "a", "{}", "Arguments to the function")
 	benchCmd.Flags().StringVarP(&benchCmdConfig.benchParams, "params", "p", "{}", "Parameters for the benchmark")
 	benchCmd.Flags().StringVarP(&benchCmdConfig.trackingUrl, "trackingUrl", "u", "", "URL for posting responses")
-	benchCmd.Flags().StringVarP(&benchCmdConfig.logFile, "output", "o", "log.txt", "Output File")
+	benchCmd.Flags().StringVarP(&benchCmdConfig.logFile, "output", "o", "", "Output File")
 }
 
 func getLocalIp() string {

cmd/clean.go

@@ -3,7 +3,6 @@
 package cmd
 
 import (
-	"os"
 	"path/filepath"
 
 	"github.com/pkg/errors"
@@ -25,15 +24,8 @@ var cleanCmd = &cobra.Command{
 			cleanGlob = filepath.Join(srkManager.Cfg.GetString("buildDir"), "functions", "*")
 		}
 
-		matches, err := filepath.Glob(cleanGlob)
-		if err != nil {
-			return errors.Wrap(err, "Failed to clean build directory")
-		}
-
-		for _, path := range matches {
-			if err := os.RemoveAll(path); err != nil {
-				return errors.Wrapf(err, "Failed to remove build directory: "+path)
-			}
+		if err := srkManager.CleanDirectory(cleanGlob); err != nil {
+			return errors.Wrap(err, "Failed to clean function")
 		}
 
 		srkManager.Logger.Info("Successfully cleaned function")

cmd/create.go

@@ -13,8 +13,11 @@ import (
 
 var createCmdConfig struct {
 	source  string
-	include string
+	include []string
+	files   []string
 	name    string
+	env     map[string]string
+	runtime string
 }
 
 var createCmd = &cobra.Command{
@@ -38,10 +41,9 @@ function package" and "srk function install".`,
 			return err
 		}
 
-		includes := strings.Split(createCmdConfig.include, ",")
 		rawDir := srkManager.GetRawPath(funcName)
 
-		if err := srkManager.CreateRaw(createCmdConfig.source, funcName, includes); err != nil {
+		if err := srkManager.CreateRaw(createCmdConfig.source, funcName, createCmdConfig.include, createCmdConfig.files); err != nil {
 			return errors.Wrap(err, "Create command failed")
 		}
 		srkManager.Logger.Info("Created raw function: " + rawDir)
@@ -52,7 +54,7 @@ function package" and "srk function install".`,
 		}
 		srkManager.Logger.Info("Created FaaS Package: " + pkgPath)
 
-		if err := srkManager.Provider.Faas.Install(rawDir); err != nil {
+		if err := srkManager.Provider.Faas.Install(rawDir, createCmdConfig.env, createCmdConfig.runtime); err != nil {
 			return errors.Wrap(err, "Installation failed")
 		}
 		srkManager.Logger.Info("Successfully installed function")
@@ -65,9 +67,12 @@ func init() {
 
 	// Define the command line arguments for this subcommand
 	createCmd.Flags().StringVarP(&createCmdConfig.source, "source", "s", "", "source directory or file")
-	createCmd.Flags().StringVarP(&createCmdConfig.include, "include", "i", "", "what to include, e.g., bench")
+	createCmd.Flags().StringSliceVarP(&createCmdConfig.include, "include", "i", []string{}, "what to include, e.g., bench")
+	createCmd.Flags().StringSliceVarP(&createCmdConfig.files, "files", "f", []string{}, "additional files to include")
+	createCmd.Flags().StringToStringVarP(&createCmdConfig.env, "env", "e", make(map[string]string), "list of environment vars to set for function execution: var1=value1,var2=value2")
+	createCmd.Flags().StringVarP(&createCmdConfig.runtime, "runtime", "r", "", "runtime to use for function execution")
 	// The actual default is derived from the source option, so we set it
 	// something that will be clear in the help output until we have all the
 	// options parsed
-	createCmd.Flags().StringVarP(&createCmdConfig.name, "function-name", "n", "source", "Optional name for this function, if different than the source name")
+	createCmd.Flags().StringVarP(&createCmdConfig.name, "function-name", "n", "source", "optional name for this function, if different than the source name")
 }

cmd/function.go

@@ -10,8 +10,8 @@ import (
 // functionCmd represents the function command
 var functionCmd = &cobra.Command{
 	Use:   "function",
-	Short: "FaaS interaction",
-	Long:  `Commands for dealing with your configured FaaS provider.`,
+	Short: "Manage FaaS function",
+	Long:  `Commands for dealing with functions of your configured FaaS provider.`,
 }
 
 func init() {

cmd/install.go

@@ -7,17 +7,24 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var installName string
+var installCmdConfig struct {
+	name    string
+	env     map[string]string
+	runtime string
+}
 
 // installCmd represents the install command
 var installCmd = &cobra.Command{
 	Use:   "install",
 	Short: "Install a pre-packaged function to the configured FaaS service",
 	Long:  `Install a function to the FaaS service. It is assumed that you have already packaged this function (using the 'package' command).`,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		rawDir := srkManager.GetRawPath(installName)
 
-		if err := srkManager.Provider.Faas.Install(rawDir); err != nil {
+		env := installCmdConfig.env
+		runtime := installCmdConfig.runtime
+		rawDir := srkManager.GetRawPath(installCmdConfig.name)
+
+		if err := srkManager.Provider.Faas.Install(rawDir, env, runtime); err != nil {
 			return errors.Wrap(err, "Installation failed")
 		}
 		srkManager.Logger.Info("Successfully installed function")
@@ -28,5 +35,7 @@ var installCmd = &cobra.Command{
 func init() {
 	functionCmd.AddCommand(installCmd)
 
-	installCmd.Flags().StringVarP(&installName, "function-name", "n", "", "The function to install")
+	installCmd.Flags().StringVarP(&installCmdConfig.name, "function-name", "n", "", "The function to install")
+	installCmd.Flags().StringToStringVarP(&installCmdConfig.env, "env", "e", make(map[string]string), "list of environment vars to set for function execution: var1=value1,var2=value2")
+	installCmd.Flags().StringVarP(&installCmdConfig.runtime, "runtime", "r", "", "runtime to use for function execution")
 }

cmd/package.go

@@ -12,7 +12,8 @@ import (
 
 var packageCmdConfig struct {
 	source  string
-	include string
+	include []string
+	files   []string
 	name    string
 }
 
@@ -35,17 +36,16 @@ can manually inspect or modify it.`,
 			return err
 		}
 
-		includes := strings.Split(packageCmdConfig.include, ",")
 		rawDir := srkManager.GetRawPath(packageCmdConfig.name)
 
-		if err := srkManager.CreateRaw(packageCmdConfig.source, packageCmdConfig.name, includes); err != nil {
+		if err := srkManager.CreateRaw(packageCmdConfig.source, packageCmdConfig.name, packageCmdConfig.include, packageCmdConfig.files); err != nil {
 			return errors.Wrap(err, "Packaging function failed")
 		}
 		srkManager.Logger.Info("Created raw function: " + rawDir)
 
 		pkgPath, err := srkManager.Provider.Faas.Package(rawDir)
 		if err != nil {
-			return errors.Wrap(err, "Packaing failed")
+			return errors.Wrap(err, "Packaging failed")
 		}
 		srkManager.Logger.Info("Package created at: " + pkgPath)
 		return nil
@@ -57,10 +57,10 @@ func init() {
 
 	// Define the command line arguments for this subcommand
 	packageCmd.Flags().StringVarP(&packageCmdConfig.source, "source", "s", "", "source directory or file")
-	packageCmd.Flags().StringVarP(&packageCmdConfig.include, "include", "i", "", "SRK-provided libraries to include")
+	packageCmd.Flags().StringSliceVarP(&packageCmdConfig.include, "include", "i", []string{}, "SRK-provided libraries to include")
+	packageCmd.Flags().StringSliceVarP(&packageCmdConfig.files, "files", "f", []string{}, "additional files to include")
 	// The actual default is derived from the source option, so we set it
 	// something that will be clear in the help output until we have all the
 	// options parsed
-	packageCmd.Flags().StringVarP(&packageCmdConfig.name, "function-name", "n", "source", "Optional name for this function, if different than the source name")
-
+	packageCmd.Flags().StringVarP(&packageCmdConfig.name, "function-name", "n", "source", "optional name for this function, if different than the source name")
 }

cmd/remove.go

@@ -15,7 +15,7 @@ var removeCmd = &cobra.Command{
 	Long:  `Remove will delete a function from the configured provider so that is no longer visible or using resources. If you have installed the function to multiple services, you will need to call "remove" on each service separately. Remove is the inverse of "install", it does not affect packages.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		if err := srkManager.Provider.Faas.Remove(removeName); err != nil {
-			return errors.Wrap(err, "Service removal failed")
+			return errors.Wrap(err, "Function removal failed")
 		}
 
 		srkManager.Logger.Info("Successfully removed function")

cmd/root.go

@@ -4,10 +4,11 @@
 package cmd
 
 import (
-	"fmt"
 	"os"
 
 	"github.com/serverlessresearch/srk/pkg/srkmgr"
+
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -29,8 +30,7 @@ var rootCmd = &cobra.Command{
 		var err error
 		srkManager, err = srkmgr.NewManager(mgrArgs)
 		if err != nil {
-			fmt.Println("Failed to initialize srk manager: %v\n", err)
-			os.Exit(1)
+			log.Fatalf("Failed to initialize srk manager: %v\n", err)
 		}
 	},
 	PersistentPostRun: func(cmd *cobra.Command, args []string) {
@@ -43,7 +43,7 @@ var rootCmd = &cobra.Command{
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
 		if srkManager == nil || srkManager.Logger == nil {
-			fmt.Printf("%v\n", err)
+			log.Error(err)
 		} else {
 			srkManager.Logger.Error(err)
 		}

configs/.gitignore

@@ -1 +1 @@
-srk.yaml
+*srk.yaml

docs/source/Configuration.rst

@@ -87,6 +87,91 @@ If you would like to use a custom vpc for your functions, you can configure
 that here. If you don't know what this is, you can leave it as null and SRK
 will use Amazon's default behavior.
 
+runtimes
+"""""""""""""""""""""
+You can set up a list of runtimes for your functions here. A runtime consists
+of an AWS provided ``base`` (`<https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html>`_)
+and optional additional ``layers`` (`<https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html>`_).
+Layers have to be uploaded to AWS lambda beforehand and then specified in the
+configuration by their ARN.
+
+::
+
+      # Optional custom runtime and layer configuration
+      runtimes :
+        # example custom runtime definition
+        cffs-python :
+          # AWS runtime as base, use 'provided' for custom runtime
+          base : provided
+          # list of additional layers
+          layers :
+            # e.g. - 'arn:aws:lambda:eu-central-1:123459789012:layer:runtime-python37:3'
+
+default-runtime
+"""""""""""""""""""""
+This specifies the runtime to use if it is not given as CLI parameter.
+
+lambciLambda
+^^^^^^^^^^^^^^^
+`LambCI lambda <https://hub.docker.com/r/lambci/lambda/>`_ provides a
+Docker-based sandbox environment that mimics AWS lambda. This can be used to
+test AWS lambda functions on a local machine which is easier to debug and does
+not create AWS costs. See ``Examples / How to use LambCI lambda`` for a setup
+guide.
+
+address
+"""""""""""""""""""""
+The server address of the Lambci lambda invocation API, usually
+``<hostname>:9001``.
+
+directory
+"""""""""""""""""""""
+The path to the Lambci lambda work directory.
+
+runtimes
+"""""""""""""""""""""
+As the LambCI lambda runtimes is included in the used Docker image, the runtimes
+section does configure additional layers only. Here, layers are directories inside
+the layers directory of the Lambci lambda work directory that need to be created
+manually beforehand.
+
+::
+
+      # runtime configuration
+      runtimes :
+        # example runtime definition
+        cffs-python :
+          # list of directories that make up the runtime
+          layers :
+            - 'runtime-python37-1'
+
+default-runtime
+"""""""""""""""""""""
+This specifies the runtime to use if it is not given as CLI parameter.
+
+remote
+"""""""""""""""""""""
+This section must be configured only if LambCI lambda is running on a remote
+machine. It contains the information necessary to execute shell commands via 
+SSH.
+
+::
+
+      # optional remote configuration
+      # if set the directory value below is bound to the specified host
+      remote:
+        # path to local scp command if not in path
+        scp : '/usr/bin/scp'
+        # path to local ssh command if not in path
+        ssh : '/usr/bin/ssh'
+        # IP or hostname of server running the lambci/lambda docker image
+        host : 'ec2-instance'
+        # user for scp + ssh
+        user : 'ubuntu'
+        # key file for scp + ssh
+        pem : '~/.aws/AWS.pem'
+
+
 global
 ^^^^^^^^^^^^^^^^^^^^
 This section provides global behaviors for all FaaS implementations. Note that

docs/source/Examples/advanced.rst

@@ -0,0 +1,50 @@
+.. _example_advanced:
+
+===============================================================================
+Advanced Function Setup
+===============================================================================
+
+This example shows how to setup a function using additional files, environment
+variables and a configurable runtime.
+
+*******************************************************************************
+Adding Additional Files
+*******************************************************************************
+
+Sometimes it is necessary to add additional files from outside the source
+directory to the function package. This can be done via the ``--files``
+parameter.
+
+::
+
+	$ ./srk function create -s <source-dir> -f <path-to-file1>,<path-to-file2>
+
+
+*******************************************************************************
+Setting Environment Variables
+*******************************************************************************
+Environment_ variables can provide additional information to the function and
+the runtime environment. They can be set by adding the ``--env`` parameter at
+function creation.
+
+::
+
+	$ ./srk function create -s <source-dir> -e VAR1=VALUE1,VAR2=VALUE2
+
+
+*******************************************************************************
+Setting the Runtime
+*******************************************************************************
+If a function does not use the configured default runtime it can be specified
+via the ``--runtime`` parameter.
+
+::
+
+	$ ./srk function create -s <source-dir> -r <runtime>
+
+The runtime_ can either be provided by the FaaS provider or defined as a set of
+layers_ configured in the configuration.
+
+.. _Runtime: https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html
+.. _Layers: https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html
+.. _Environment: https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html

docs/source/Examples/index.rst

@@ -7,6 +7,8 @@ learn the system.
    :maxdepth: 2
    :caption: Contents:
 
+   advanced.rst
+   lambci.rst
    GPU.rst
 
 .. Indices and tables