Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,16 @@ Lambda’s orchestrator, or security and authentication configurations. You can

## Installing

Instructions for installing AWS Lambda Runtime Interface Emulator for your platform
The following commands download the RIE binary for your platform. Note that while you can download the binary on any platform, the RIE can only be executed in a Linux environment (typically within a Docker container).

| Platform | Command to install |
| Platform (for downloading) | Command to download |
|---------|---------
| macOS/Linux x86\_64 | `mkdir -p ~/.aws-lambda-rie && curl -Lo ~/.aws-lambda-rie/aws-lambda-rie https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie && chmod +x ~/.aws-lambda-rie/aws-lambda-rie` |
| macOS/Linux arm64 | `mkdir -p ~/.aws-lambda-rie && curl -Lo ~/.aws-lambda-rie/aws-lambda-rie https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie-arm64 && chmod +x ~/.aws-lambda-rie/aws-lambda-rie` |
| Windows x86\_64 | `Invoke-WebRequest -OutFile 'C:\Program Files\aws lambda\aws-lambda-rie' https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie` |
| Windows arm64 | `Invoke-WebRequest -OutFile 'C:\Program Files\aws lambda\aws-lambda-rie' https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie-arm64` |

After downloading, the RIE binary must be used within a Linux environment, typically as part of a Docker container setup. See the Docker configuration instructions below for proper implementation.

## Getting started

Expand Down
617 changes: 519 additions & 98 deletions THIRD-PARTY-LICENSES.md

Large diffs are not rendered by default.

148 changes: 7 additions & 141 deletions cmd/aws-lambda-rie/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,151 +4,17 @@
package main

import (
"context"
"fmt"
"net"
"os"
"runtime/debug"

"github.com/jessevdk/go-flags"
"go.amzn.com/lambda/interop"
"go.amzn.com/lambda/rapidcore"

log "github.com/sirupsen/logrus"
)

const (
optBootstrap = "/opt/bootstrap"
runtimeBootstrap = "/var/runtime/bootstrap"
"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda-managed-instances/aws-lambda-rie/run"
"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda-managed-instances/rapidcore/env"
"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/rie"
)

type options struct {
LogLevel string `long:"log-level" description:"The level of AWS Lambda Runtime Interface Emulator logs to display. Can also be set by the environment variable 'LOG_LEVEL'. Defaults to the value 'info'."`
InitCachingEnabled bool `long:"enable-init-caching" description:"Enable support for Init Caching"`
// Do not have a default value so we do not need to keep it in sync with the default value in lambda/rapidcore/sandbox_builder.go
RuntimeAPIAddress string `long:"runtime-api-address" description:"The address of the AWS Lambda Runtime API to communicate with the Lambda execution environment."`
RuntimeInterfaceEmulatorAddress string `long:"runtime-interface-emulator-address" default:"0.0.0.0:8080" description:"The address for the AWS Lambda Runtime Interface Emulator to accept HTTP request upon."`
}

func main() {
// More frequent GC reduces the tail latencies, equivalent to export GOGC=33
debug.SetGCPercent(33)

opts, args := getCLIArgs()

logLevel := "info"

// If you specify an option by using a parameter on the CLI command line, it overrides any value from either the corresponding environment variable.
//
// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
if opts.LogLevel != "" {
logLevel = opts.LogLevel
} else if envLogLevel, envLogLevelSet := os.LookupEnv("LOG_LEVEL"); envLogLevelSet {
logLevel = envLogLevel
}

rapidcore.SetLogLevel(logLevel)

if opts.RuntimeAPIAddress != "" {
_, _, err := net.SplitHostPort(opts.RuntimeAPIAddress)

if err != nil {
log.WithError(err).Fatalf("The command line value for \"--runtime-api-address\" is not a valid network address %q.", opts.RuntimeAPIAddress)
}
}

_, _, err := net.SplitHostPort(opts.RuntimeInterfaceEmulatorAddress)

if err != nil {
log.WithError(err).Fatalf("The command line value for \"--runtime-interface-emulator-address\" is not a valid network address %q.", opts.RuntimeInterfaceEmulatorAddress)
}

bootstrap, handler := getBootstrap(args, opts)
sandbox := rapidcore.
NewSandboxBuilder().
AddShutdownFunc(context.CancelFunc(func() { os.Exit(0) })).
SetExtensionsFlag(true).
SetInitCachingFlag(opts.InitCachingEnabled)

if len(handler) > 0 {
sandbox.SetHandler(handler)
}

if opts.RuntimeAPIAddress != "" {
sandbox.SetRuntimeAPIAddress(opts.RuntimeAPIAddress)
}

sandboxContext, internalStateFn := sandbox.Create()
// Since we have not specified a custom interop server for standalone, we can
// directly reference the default interop server, which is a concrete type
sandbox.DefaultInteropServer().SetSandboxContext(sandboxContext)
sandbox.DefaultInteropServer().SetInternalStateGetter(internalStateFn)

startHTTPServer(opts.RuntimeInterfaceEmulatorAddress, sandbox, bootstrap)
}

func getCLIArgs() (options, []string) {
var opts options
parser := flags.NewParser(&opts, flags.IgnoreUnknown)
args, err := parser.ParseArgs(os.Args)

if err != nil {
log.WithError(err).Fatal("Failed to parse command line arguments:", os.Args)
if _, ok := os.LookupEnv(env.AWS_LAMBDA_MAX_CONCURRENCY); ok {
run.Run()
return
}

return opts, args
}

func isBootstrapFileExist(filePath string) bool {
file, err := os.Stat(filePath)
return !os.IsNotExist(err) && !file.IsDir()
}

func getBootstrap(args []string, opts options) (interop.Bootstrap, string) {
var bootstrapLookupCmd []string
var handler string
currentWorkingDir := "/var/task" // default value

if len(args) <= 1 {
// set default value to /var/task/bootstrap, but switch to the other options if it doesn't exist
bootstrapLookupCmd = []string{
fmt.Sprintf("%s/bootstrap", currentWorkingDir),
}

if !isBootstrapFileExist(bootstrapLookupCmd[0]) {
var bootstrapCmdCandidates = []string{
optBootstrap,
runtimeBootstrap,
}

for i, bootstrapCandidate := range bootstrapCmdCandidates {
if isBootstrapFileExist(bootstrapCandidate) {
bootstrapLookupCmd = []string{bootstrapCmdCandidates[i]}
break
}
}
}

// handler is used later to set an env var for Lambda Image support
handler = ""
} else if len(args) > 1 {

bootstrapLookupCmd = args[1:]

if cwd, err := os.Getwd(); err == nil {
currentWorkingDir = cwd
}

if len(args) > 2 {
// Assume last arg is the handler
handler = args[len(args)-1]
}

log.Infof("exec '%s' (cwd=%s, handler=%s)", args[1], currentWorkingDir, handler)

} else {
log.Panic("insufficient arguments: bootstrap not provided")
}

return NewSimpleBootstrap(bootstrapLookupCmd, currentWorkingDir), handler
rie.Run()
}
9 changes: 5 additions & 4 deletions cmd/localstack/awsutil.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,6 @@ package main
import (
"context"
"fmt"
log "github.com/sirupsen/logrus"
"go.amzn.com/lambda/interop"
"go.amzn.com/lambda/rapidcore/env"
"golang.org/x/sys/unix"
"io"
"io/fs"
"math"
Expand All @@ -21,6 +17,11 @@ import (
"path/filepath"
"strings"
"time"

"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/interop"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch 👍

So we basically need to change the imports to our fork instead of using upstream via go.amzn.com. I guess these imports might be a frequent area of conflict, so it's worth documenting the rationale in our LocalStack readme.

"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/rapidcore/env"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
)

const (
Expand Down
12 changes: 6 additions & 6 deletions cmd/localstack/custom_interop.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ import (
"strings"
"time"

"github.com/go-chi/chi"
"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/core/statejson"
"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/interop"
"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/rapidcore"
"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/rapidcore/standalone"
"github.com/go-chi/chi/v5"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❓ Was this the necessary HTTP router dependency fix? Any context worth adding why we differ here?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need to apply this change ourselves, given that AWS has already done that upstream? https://github.com/aws/aws-lambda-runtime-interface-emulator/pull/149/files

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is our custom interop server, not the AWS defined one.

So it wouldn't be affected by the upstream changes and requires we manually change it around 🙂

log "github.com/sirupsen/logrus"
"go.amzn.com/lambda/core/statejson"
"go.amzn.com/lambda/interop"
"go.amzn.com/lambda/rapidcore"
"go.amzn.com/lambda/rapidcore/standalone"
)

type CustomInteropServer struct {
Expand Down Expand Up @@ -52,7 +52,7 @@ func (l *LocalStackAdapter) SendStatus(status LocalStackStatus, payload []byte)

// The InvokeRequest is sent by LocalStack to trigger an invocation
type InvokeRequest struct {
InvokeId string `json:"invoke-id"`
InvokeId string `json:"request-id"`
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like that we fixed this inconsistent naming.

❓ Are we safe to ship this without breaking K8 images, which are not directly tied to LocalStack versions?
/cc @dfangl

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll make a corresponding change in LocalStack for this once the latest RIE changes are shipped. Then, in a single PR, we switch over RIE in addition to changing the param on invoke request.

InvokedFunctionArn string `json:"invoked-function-arn"`
Payload string `json:"payload"`
TraceId string `json:"trace-id"`
Expand Down
7 changes: 4 additions & 3 deletions cmd/localstack/hotreloading.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
package main

import (
"github.com/fsnotify/fsnotify"
log "github.com/sirupsen/logrus"
"go.amzn.com/cmd/localstack/filenotify"
"os"
"time"

"github.com/aws/aws-lambda-runtime-interface-emulator/cmd/localstack/filenotify"
"github.com/fsnotify/fsnotify"
log "github.com/sirupsen/logrus"
)

type ChangeListener struct {
Expand Down
7 changes: 4 additions & 3 deletions cmd/localstack/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@ import (
"strings"
"time"

"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/interop"
"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/rapidcore"
"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/rie"
log "github.com/sirupsen/logrus"
"go.amzn.com/lambda/interop"
"go.amzn.com/lambda/rapidcore"
)

type LsOpts struct {
Expand Down Expand Up @@ -244,7 +245,7 @@ func main() {
// start runtime init. It is important to start `InitHandler` synchronously because we need to ensure the
// notification channels and status fields are properly initialized before `AwaitInitialized`
log.Debugln("Starting runtime init.")
InitHandler(sandbox.LambdaInvokeAPI(), GetEnvOrDie("AWS_LAMBDA_FUNCTION_VERSION"), int64(invokeTimeoutSeconds), bootstrap, lsOpts.AccountId) // TODO: replace this with a custom init
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❓ Why is lsOpts.AccountId not necessary anymore?

rie.InitHandler(sandbox.LambdaInvokeAPI(), GetEnvOrDie("AWS_LAMBDA_FUNCTION_VERSION"), int64(invokeTimeoutSeconds), bootstrap) // TODO: replace this with a custom init

log.Debugln("Awaiting initialization of runtime init.")
if err := interopServer.delegate.AwaitInitialized(); err != nil {
Expand Down
6 changes: 3 additions & 3 deletions cmd/localstack/simple_bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ import (
"os"
"path/filepath"

"go.amzn.com/lambda/fatalerror"
"go.amzn.com/lambda/interop"
"go.amzn.com/lambda/rapidcore/env"
"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/fatalerror"
"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/interop"
"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/rapidcore/env"
)

// the type implement a simpler version of the Bootstrap
Expand Down
5 changes: 3 additions & 2 deletions cmd/localstack/tracer.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@ package main
import (
"context"
"encoding/json"
"go.amzn.com/lambda/appctx"
"go.amzn.com/lambda/interop"

"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/appctx"
"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/interop"
)

type LocalStackTracer struct {
Expand Down
5 changes: 4 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
module go.amzn.com
module github.com/aws/aws-lambda-runtime-interface-emulator

go 1.25

Expand All @@ -9,8 +9,11 @@ require (
github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575
github.com/fsnotify/fsnotify v1.6.0
github.com/go-chi/chi v1.5.5
github.com/go-chi/chi/v5 v5.2.3
github.com/google/uuid v1.6.0
github.com/jessevdk/go-flags v1.5.0
github.com/orcaman/concurrent-map v1.0.0
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
github.com/shirou/gopsutil v2.19.10+incompatible
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.9.0
Expand Down
8 changes: 6 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/aws/aws-lambda-go v1.46.0 h1:UWVnvh2h2gecOlFhHQfIPQcD8pL/f7pVCutmFl+oXU8=
github.com/aws/aws-lambda-go v1.46.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
github.com/aws/aws-lambda-go v1.50.0 h1:0GzY18vT4EsCvIyk3kn3ZH5Jg30NRlgYaai1w0aGPMU=
github.com/aws/aws-lambda-go v1.50.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
github.com/aws/aws-sdk-go v1.44.298 h1:5qTxdubgV7PptZJmp/2qDwD2JL187ePL7VOxsSh1i3g=
Expand All @@ -17,6 +15,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/go-chi/chi v1.5.5 h1:vOB/HbEMt9QqBqErz07QehcOKHaWFtuj87tTDVz2qXE=
github.com/go-chi/chi v1.5.5/go.mod h1:C9JqLr3tIYjDOZpzn+BCuxY8z8vmca43EeMgyZt7irw=
github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
Expand All @@ -27,9 +27,13 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY=
github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
github.com/shirou/gopsutil v2.19.10+incompatible h1:lA4Pi29JEVIQIgATSeftHSY0rMGI9CLrl2ZvDLiahto=
github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
Expand Down
50 changes: 50 additions & 0 deletions internal/lambda-managed-instances/agents/agent.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package agents

import (
"log/slog"
"path"
"path/filepath"

"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda-managed-instances/utils"
)

const (
ExtensionsDir = "/opt/extensions"
)

func ListExternalAgentPaths(fileutils utils.FileUtil, dir string, root string) []string {
var agentPaths []string
if !isCanonical(dir) || !isCanonical(root) {
slog.Warn("Agents base paths are not absolute and in canonical form", "dir", dir, "root", root)
return agentPaths
}
fullDir := path.Join(root, dir)
files, err := fileutils.ReadDirectory(fullDir)
if err != nil {
if fileutils.IsNotExist(err) {
slog.Info("The extension's directory does not exist, assuming no extensions to be loaded", "fullDir", fullDir)
} else {

slog.Error("Cannot list external agents", "err", err)
}

return agentPaths
}

for _, file := range files {
if !file.IsDir() {

p := path.Join("/", dir, file.Name())
agentPaths = append(agentPaths, p)
}
}
return agentPaths
}

func isCanonical(path string) bool {
absPath, err := filepath.Abs(path)
return err == nil && absPath == path
}
Loading