Lambci

cmd/bench.go

@@ -34,7 +34,7 @@ functions and configured the provider.`,
 			benchCmdConfig.functionArgs,
 			benchCmdConfig.benchParams,
 			benchCmdConfig.trackingUrl,
-			benchCmdConfig.benchName + ".out",
+			benchCmdConfig.logFile,
 		}
 
 		switch benchCmdConfig.benchName {
@@ -96,7 +96,7 @@ func init() {
 	benchCmd.Flags().StringVarP(&benchCmdConfig.functionArgs, "function-args", "a", "{}", "Arguments to the function")
 	benchCmd.Flags().StringVarP(&benchCmdConfig.benchParams, "params", "p", "{}", "Parameters for the benchmark")
 	benchCmd.Flags().StringVarP(&benchCmdConfig.trackingUrl, "trackingUrl", "u", "", "URL for posting responses")
-	benchCmd.Flags().StringVarP(&benchCmdConfig.logFile, "output", "o", "log.txt", "Output File")
+	benchCmd.Flags().StringVarP(&benchCmdConfig.logFile, "output", "o", "", "Output File")
 }
 
 func getLocalIp() string {

cmd/clean.go

@@ -3,7 +3,6 @@
 package cmd
 
 import (
-	"os"
 	"path/filepath"
 
 	"github.com/pkg/errors"
@@ -25,15 +24,8 @@ var cleanCmd = &cobra.Command{
 			cleanGlob = filepath.Join(srkManager.Cfg.GetString("buildDir"), "functions", "*")
 		}
 
-		matches, err := filepath.Glob(cleanGlob)
-		if err != nil {
-			return errors.Wrap(err, "Failed to clean build directory")
-		}
-
-		for _, path := range matches {
-			if err := os.RemoveAll(path); err != nil {
-				return errors.Wrapf(err, "Failed to remove build directory: "+path)
-			}
+		if err := srkManager.CleanDirectory(cleanGlob); err != nil {
+			return errors.Wrap(err, "Failed to clean function")
 		}
 
 		srkManager.Logger.Info("Successfully cleaned function")

cmd/create.go

@@ -13,7 +13,10 @@ import (
 var createCmdConfig struct {
 	source  string
 	include string
+	files   string
 	name    string
+	env     string
+	runtime string
 }
 
 var createCmd = &cobra.Command{
@@ -32,10 +35,13 @@ function package" and "srk function install".`,
 		}
 		srkManager.Logger.Info("Function name: " + funcName)
 
-		includes := strings.Split(createCmdConfig.include, ",")
+		includes := parseList(createCmdConfig.include)
+		files := parseList(createCmdConfig.files)
+		env := parseKeyValue(createCmdConfig.env)
+		runtime := createCmdConfig.runtime
 		rawDir := srkManager.GetRawPath(funcName)
 
-		if err := srkManager.CreateRaw(createCmdConfig.source, funcName, includes); err != nil {
+		if err := srkManager.CreateRaw(createCmdConfig.source, funcName, includes, files); err != nil {
 			return errors.Wrap(err, "Create command failed")
 		}
 		srkManager.Logger.Info("Created raw function: " + rawDir)
@@ -46,7 +52,7 @@ function package" and "srk function install".`,
 		}
 		srkManager.Logger.Info("Created FaaS Package: " + pkgPath)
 
-		if err := srkManager.Provider.Faas.Install(rawDir); err != nil {
+		if err := srkManager.Provider.Faas.Install(rawDir, env, runtime); err != nil {
 			return errors.Wrap(err, "Installation failed")
 		}
 		srkManager.Logger.Info("Successfully installed function")
@@ -60,8 +66,11 @@ func init() {
 	// Define the command line arguments for this subcommand
 	createCmd.Flags().StringVarP(&createCmdConfig.source, "source", "s", "", "source directory or file")
 	createCmd.Flags().StringVarP(&createCmdConfig.include, "include", "i", "", "what to include, e.g., bench")
+	createCmd.Flags().StringVarP(&createCmdConfig.files, "files", "f", "", "additional files to include")
+	createCmd.Flags().StringVarP(&createCmdConfig.env, "env", "e", "", "list of environment vars: var1=value1,var2=value2")
+	createCmd.Flags().StringVarP(&createCmdConfig.runtime, "runtime", "r", "", "runtime to use for function execution")
 	// The actual default is derived from the source option, so we set it
 	// something that will be clear in the help output until we have all the
 	// options parsed
-	createCmd.Flags().StringVarP(&createCmdConfig.name, "function-name", "n", "source", "Optional name for this function, if different than the source name")
+	createCmd.Flags().StringVarP(&createCmdConfig.name, "function-name", "n", "source", "optional name for this function, if different than the source name")
 }

cmd/function.go

@@ -10,8 +10,8 @@ import (
 // functionCmd represents the function command
 var functionCmd = &cobra.Command{
 	Use:   "function",
-	Short: "FaaS interaction",
-	Long:  `Commands for dealing with your configured FaaS provider.`,
+	Short: "Manage FaaS function",
+	Long:  `Commands for dealing with functions of your configured FaaS provider.`,
 }
 
 func init() {

cmd/install.go

@@ -7,17 +7,24 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var installName string
+var installCmdConfig struct {
+	name    string
+	env     string
+	runtime string
+}
 
 // installCmd represents the install command
 var installCmd = &cobra.Command{
 	Use:   "install",
 	Short: "Install a pre-packaged function to the configured FaaS service",
 	Long:  `Install a function to the FaaS service. It is assumed that you have already packaged this function (using the 'package' command).`,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		rawDir := srkManager.GetRawPath(installName)
 
-		if err := srkManager.Provider.Faas.Install(rawDir); err != nil {
+		env := parseKeyValue(installCmdConfig.env)
+		runtime := installCmdConfig.runtime
+		rawDir := srkManager.GetRawPath(installCmdConfig.name)
+
+		if err := srkManager.Provider.Faas.Install(rawDir, env, runtime); err != nil {
 			return errors.Wrap(err, "Installation failed")
 		}
 		srkManager.Logger.Info("Successfully installed function")
@@ -28,5 +35,7 @@ var installCmd = &cobra.Command{
 func init() {
 	functionCmd.AddCommand(installCmd)
 
-	installCmd.Flags().StringVarP(&installName, "function-name", "n", "", "The function to install")
+	installCmd.Flags().StringVarP(&installCmdConfig.name, "function-name", "n", "", "The function to install")
+	installCmd.Flags().StringVarP(&installCmdConfig.env, "env", "e", "", "list of environment vars: var1=value1,var2=value2")
+	installCmd.Flags().StringVarP(&installCmdConfig.runtime, "runtime", "r", "", "runtime to use for function execution")
 }

cmd/package.go

@@ -12,6 +12,7 @@ import (
 var packageCmdConfig struct {
 	source  string
 	include string
+	files   string
 	name    string
 }
 
@@ -29,17 +30,18 @@ can manually inspect or modify it.`,
 			packageCmdConfig.name = strings.TrimSuffix(path.Base(packageCmdConfig.source), path.Ext(packageCmdConfig.source))
 		}
 
-		includes := strings.Split(packageCmdConfig.include, ",")
+		includes := parseList(packageCmdConfig.include)
+		files := parseList(packageCmdConfig.files)
 		rawDir := srkManager.GetRawPath(packageCmdConfig.name)
 
-		if err := srkManager.CreateRaw(packageCmdConfig.source, packageCmdConfig.name, includes); err != nil {
+		if err := srkManager.CreateRaw(packageCmdConfig.source, packageCmdConfig.name, includes, files); err != nil {
 			return errors.Wrap(err, "Packaging function failed")
 		}
 		srkManager.Logger.Info("Created raw function: " + rawDir)
 
 		pkgPath, err := srkManager.Provider.Faas.Package(rawDir)
 		if err != nil {
-			return errors.Wrap(err, "Packaing failed")
+			return errors.Wrap(err, "Packaging failed")
 		}
 		srkManager.Logger.Info("Package created at: " + pkgPath)
 		return nil
@@ -52,9 +54,9 @@ func init() {
 	// Define the command line arguments for this subcommand
 	packageCmd.Flags().StringVarP(&packageCmdConfig.source, "source", "s", "", "source directory or file")
 	packageCmd.Flags().StringVarP(&packageCmdConfig.include, "include", "i", "", "SRK-provided libraries to include")
+	packageCmd.Flags().StringVarP(&packageCmdConfig.files, "files", "f", "", "additional files to include")
 	// The actual default is derived from the source option, so we set it
 	// something that will be clear in the help output until we have all the
 	// options parsed
-	packageCmd.Flags().StringVarP(&packageCmdConfig.name, "function-name", "n", "source", "Optional name for this function, if different than the source name")
-
+	packageCmd.Flags().StringVarP(&packageCmdConfig.name, "function-name", "n", "source", "optional name for this function, if different than the source name")
 }

cmd/remove.go

@@ -15,7 +15,7 @@ var removeCmd = &cobra.Command{
 	Long:  `Remove will delete a function from the configured provider so that is no longer visible or using resources. If you have installed the function to multiple services, you will need to call "remove" on each service separately. Remove is the inverse of "install", it does not affect packages.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		if err := srkManager.Provider.Faas.Remove(removeName); err != nil {
-			return errors.Wrap(err, "Service removal failed")
+			return errors.Wrap(err, "Function removal failed")
 		}
 
 		srkManager.Logger.Info("Successfully removed function")

cmd/root.go

@@ -4,10 +4,12 @@
 package cmd
 
 import (
-	"fmt"
 	"os"
+	"strings"
 
 	"github.com/serverlessresearch/srk/pkg/srkmgr"
+
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -29,8 +31,7 @@ var rootCmd = &cobra.Command{
 		var err error
 		srkManager, err = srkmgr.NewManager(mgrArgs)
 		if err != nil {
-			fmt.Println("Failed to initialize srk manager: %v\n", err)
-			os.Exit(1)
+			log.Fatalf("Failed to initialize srk manager: %v\n", err)
 		}
 	},
 	PersistentPostRun: func(cmd *cobra.Command, args []string) {
@@ -43,14 +44,41 @@ var rootCmd = &cobra.Command{
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
 		if srkManager == nil || srkManager.Logger == nil {
-			fmt.Printf("%v\n", err)
+			log.Error(err)
 		} else {
 			srkManager.Logger.Error(err)
 		}
 		os.Exit(1)
 	}
 }
 
+func parseList(s string) []string {
+
+	if s == "" {
+		return nil
+	}
+
+	return strings.Split(s, ",")
+}
+
+func parseKeyValue(s string) map[string]string {
+
+	if s == "" {
+		return nil
+	}
+
+	result := make(map[string]string)
+	for _, pair := range strings.Split(s, ",") {
+		keyValue := strings.Split(pair, "=")
+		if len(keyValue) == 2 {
+			result[keyValue[0]] = keyValue[1]
+		}
+	}
+
+	return result
+}
+
 func init() {
+	log.SetLevel(log.DebugLevel)
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is configs/srk.yaml)")
 }

configs/.gitignore

@@ -1 +1 @@
-srk.yaml
+*srk.yaml

configs/example-srk.yaml

@@ -5,6 +5,8 @@ default-provider : "aws"
 # provider will have one implementation of each service category (although it
 # is possible to omit service categories of they are not needed)
 providers :
+  lambci :
+    faas : "lambciLambda"
   aws :
     faas : "awsLambda"
     objStore : "s3"
@@ -34,4 +36,51 @@ service :
       # Optional vpc/security-group setup to use.
       # e.g.: "vpc-123456789abcdef,sg-123456789abcdef"
       vpc-config : null
+      # Optional custom runtime and layer configuration
+      runtimes :
+        # example custom runtime definition
+        cffs-python :
+          # AWS runtime as base, use 'provided' for none
+          base : provided
+          # list of additional layers
+          layers :
+            - 'arn:aws:lambda:eu-central-1:669282166790:layer:runtime-python37:3'
+      # Optional default runtime if runtime is not provided by CLI
+      default-runtime : 'python3.7'
+    lambciLambda:
+      # optional remote configuration
+      # if set the directory value below is bound to the specified host
+      remote:
+        # path to scp command
+        scp : '/usr/bin/scp'
+        # path to ssh command
+        ssh : '/usr/bin/ssh'
+        # IP or hostname of server running the lambci/lambda docker image
+        host : 'ec2-instance'
+        # user for scp + ssh
+        user : 'ubuntu'
+        # key file for scp + ssh
+        pem : '~/.aws/AWS.pem'
+      # address of lambci server API
+      address : 'localhost:9001'
+      # path to the lambci directory - the following sub directories will be used:
+      # * task    directory of lambda function
+      # * runtime directory of the lambda runtime
+      # * layers  directory of layer pool with each layer a sub directory
+      # * env     environment file for lambci docker container
+      directory : '~/lambci'
+      # runtime configuration
+      runtimes :
+        # example runtime definition
+        python-with-request :
+          # list of additional layers 
+          layers :
+            - 'request'
+        # example runtime definition
+        cffs-python :
+          # list of layers that make up the runtime
+          layers :
+            - 'runtime-python37-1'
+      # optional default runtime if runtime is not provided by CLI
+      default-runtime : 'cffs-python'
     global:

docs/source/Examples/advanced.rst

@@ -0,0 +1,50 @@
+.. _example_advanced:
+
+===============================================================================
+Advanced Function Setup
+===============================================================================
+
+This example shows how to setup a function using additional files, environment
+variables and a configurable runtime.
+
+*******************************************************************************
+Adding Additional Files
+*******************************************************************************
+
+Sometimes it is necessary to add additional files from outside the source
+directory to the function package. This can be done via the ``--files``
+parameter.
+
+::
+
+	$ ./srk function create -s <source-dir> -f <path-to-file1>,<path-to-file2>
+
+
+*******************************************************************************
+Setting Environment Variables
+*******************************************************************************
+Environment_ variables can provide additional information to the function and
+the runtime environment. They can be set by adding the ``--env`` parameter at
+function creation.
+
+::
+
+	$ ./srk function create -s <source-dir> -e VAR1=VALUE1,VAR2=VALUE2
+
+
+*******************************************************************************
+Setting the Runtime
+*******************************************************************************
+If a function does not use the configured default runtime it can be specified
+via the ``--runtime`` parameter.
+
+::
+
+	$ ./srk function create -s <source-dir> -r <runtime>
+
+The runtime_ can either be provided by the FaaS provider or defined as a set of
+layers_ configured in the configuration.
+
+.. _Runtime: https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html
+.. _Layers: https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html
+.. _Environment: https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html

docs/source/Examples/index.rst

@@ -7,6 +7,8 @@ learn the system.
    :maxdepth: 2
    :caption: Contents:
 
+   advanced.rst
+   lambci.rst
    GPU.rst
 
 .. Indices and tables