Add support for packed modules to keep deployments <250mb

powershell-runtime/source/bootstrap

@@ -18,6 +18,21 @@ Import-Module '/opt/modules/pwsh-runtime.psd1'
 if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host '[RUNTIME-bootstrap]Importing .NET class from .cs file to support script properties and method' }
 Add-Type -TypeDefinition ([System.IO.File]::ReadAllText('/opt/PowerShellLambdaContext.cs'))
 
+# Unpack compressed modules, if present
+$ResolvedModules = Find-RuntimePackedModule
+
+If ($ResolvedModules.Combined) {
+    if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') {Write-Host '[RUNTIME-bootstrap]Unpacking combined module archives'}
+    Import-ModuleArchive -ArchivePath $ResolvedModules.Combined
+    if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') {Write-Host '[RUNTIME-bootstrap]Finished unpacking archives'}
+}
+If ($ResolvedModules.NuPkg) {
+    if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') {Write-Host '[RUNTIME-bootstrap]Unpacking module NuGet packages'}
+    Import-ModulePackage -PackagePath $ResolvedModules.NuPkg
+    if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') {Write-Host '[RUNTIME-bootstrap]Finished unpacking NuGet packages'}
+}
+
+
 # Modify $env:PSModulePath to support Lambda paths
 if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') {Write-Host '[RUNTIME-bootstrap]Modify PSModulePath to support Lambda paths'}
 Set-PSModulePath

powershell-runtime/source/modules/Private/Find-RuntimePackedModule.ps1

@@ -0,0 +1,57 @@
+function private:Find-RuntimePackedModule {
+    <#
+    .SYNOPSIS
+        Searches runtime environment filesystem for compressed module packages (combined .zip or per-module .nupkg)
+    .DESCRIPTION
+        Searches the current runtime environment's filesystem for compressed module packages (combined .zip or per-module .nupkg). Any resolved paths are returned in a dictionary. If nothing is found, no object is returned.
+    .NOTES
+        Looks for module packages in two locations:
+            * /opt/ (Combined Lambda layer directory)
+            * $Env:LAMBDA_TASK_ROOT (Lambda Function package directory)
+
+        Module packages can take two forms:
+            * A single, combined module archive, named "modules.zip".
+              The contents of this archive should match the format of a folder in $Env:PSModulePath.
+              (Module names as top-level directories, optional version subdirectory, corresponding module root)
+            * Individual module archives, as .nupkg files, inside a subdirectory named "module-nupkgs"
+              These files should match:
+                * The naming convention used by PSResourceGet. (e.g. <module-name>.<version>.nupkg)
+                * The Nupkg archive spec (module root at archive root, NuGet [Content_Types].xml/_rels, etc.)
+
+        The following file locations should all be detected (assume $Env:LAMBDA_TASK_ROOT = /var/lambda/)
+        * /opt/modules.zip
+        * /var/lambda/modules.zip
+        * /opt/module-nupkgs/AWS.Tools.Common.4.1.833.nupkg
+        * /var/lambda/module-nupkgs/AWS.Tools.Common.4.1.833.nupkg
+    .EXAMPLE
+        PS> Find-RuntimePackedModule
+        Name                           Value
+        ----                           -----
+        Combined                       {/opt/modules.zip}
+        NuPkg                          {/var/lambda/module-nupkgs/AWS.Tools.Common.4.1.833.nupkg, /var/lambda/module-nupkgs/AWS.Tools.S3.4.1.833.nupkg}
+    #>
+
+    [CmdletBinding()]
+    param(
+    )
+
+    if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host "[RUNTIME-Test-RuntimePackedModule]Searching for packed modules" }
+
+    $ResolvedModules = @{
+        Combined = $(
+            $Script:ModulePaths.Packed.Combined.Values | Get-Item -ErrorAction SilentlyContinue
+        )
+        NuPkg = $(
+            $Script:ModulePaths.Packed.NuPkg.Values | Get-Item -ErrorAction SilentlyContinue
+        )
+    }
+
+
+    if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host "[RUNTIME-Test-RuntimePackedModule]Found $($ResolvedModules.Combined | Measure-Object | ForEach-Object Count) combined module archive(s)" }
+    if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host "[RUNTIME-Test-RuntimePackedModule]Found $($ResolvedModules.NuPkg | Measure-Object | ForEach-Object Count) individual module package(s)" }
+
+    # Only return a value if we found either combined or NuPkg module packages
+    If ($ResolvedModules.Combined -or $ResolvedModules.NuPkg) {
+        return $ResolvedModules
+    }
+}

powershell-runtime/source/modules/Private/Import-ModuleArchive.ps1

@@ -0,0 +1,44 @@
+function private:Import-ModuleArchive {
+    <#
+    .SYNOPSIS
+        Unpacks compressed PowerShell modules from .zip archives (modules.zip)
+    .DESCRIPTION
+        Unpacks compressed PowerShell modules from .zip archives (modules.zip) into a subdirectory of /tmp.
+
+        This folder is later added to $env:PSModulePath, before user code runs, if module archives existed.
+    .NOTES
+        The contents of this archive should match the format of a folder in $Env:PSModulePath. More specifically:
+        * Module names should be top-level directories.
+        * One or more versions of the same module may be hosted in their own subdirectories, with respective version numbers.
+        * The module root (.psd1/.psm1 files, etc.) is contained within either the module-named or module-versioned directory.
+
+        Module packages are imported from two locations, from lowest to highest precedence:
+        * /opt/ (Combined Lambda layer directory)
+        * $Env:LAMBDA_TASK_ROOT (Lambda Function Package deployment directory)
+
+        If archives are detected at both locations, they will be extracted over the top of each-other.
+    #>
+    [CmdletBinding()]
+    param(
+        [ValidatePattern(".zip$")]
+        [ValidateNotNullOrEmpty()]
+        [Parameter(
+            Mandatory,
+            Position = 0
+        )]
+        [System.IO.FileInfo[]]$ArchivePath
+    )
+
+    Begin {
+        if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host '[RUNTIME-Import-ModuleArchive]Creating unpack directory for combined module archives' }
+        $UnpackDirectory = [System.IO.Directory]::CreateDirectory($Script:ModulePaths.Unpacked.Combined)
+    }
+
+    Process {
+        $ArchivePath | ForEach-Object {
+            if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host "[RUNTIME-Import-ModuleArchive]Unpacking $_ to $UnpackDirectory" }
+            Expand-Archive -LiteralPath $_ -DestinationPath $UnpackDirectory -Force
+        }
+        if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host '[RUNTIME-Import-ModuleArchive]Archive unpack complete' }
+    }
+}

powershell-runtime/source/modules/Private/Import-ModulePackage.ps1

@@ -0,0 +1,63 @@
+function private:Import-ModulePackage {
+    <#
+    .SYNOPSIS
+        Installs compressed PowerShell modules from NuGet packages (*.nupkg)
+    .DESCRIPTION
+        Installs compressed PowerShell modules from NuGet packages (*.nupkg) into a subdirectory of /tmp.
+
+        This folder is later added to $env:PSModulePath, before user code runs, if module packages existed.
+    .NOTES
+        These packages should match the NuPkg format used by PSResourceGet or PowerShellGet.
+
+        Packages can be exported either by:
+        * Downloading the .nupkg files directly from an upstream source (e.g. PowerShell Gallery)
+        * Using the -AsNuPkg parameter on Save-PSResource in the Microsoft.PowerShell.PSResourceGet module.
+
+        Module packages are imported from two locations, from lowest to highest precedence:
+        * /opt/module-nupkgs/ (Combined Lambda layer directory)
+        * $Env:LAMBDA_TASK_ROOT/module-nupkgs/ (Lambda Function Package deployment directory)
+    #>
+    [CmdletBinding()]
+    param(
+        [ValidatePattern(".nupkg$")]
+        [ValidateNotNullOrEmpty()]
+        [Parameter(
+            Mandatory,
+            Position = 0
+        )]
+        [System.IO.FileInfo[]]$PackagePath
+    )
+
+    Begin {
+        if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host '[RUNTIME-Import-ModulePackage]Creating unpack directory for individual module packages' }
+        $UnpackDirectory = [System.IO.Directory]::CreateDirectory($Script:ModulePaths.Unpacked.NuPkg)
+    }
+
+    Process {
+        $PackagePath | Group-Object -Property Directory | ForEach-Object {
+
+            # The group key should be the directory for the folder containing the nupkgs.
+            $PackageDirectory = $_.Name
+            if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host "[RUNTIME-Import-ModulePackage]Importing module packages from $PackageDirectory" }
+
+            # We split-path that directory to strip off "module-nupkgs".
+            $RepositoryName = "Lambda-Local-$($_.Group | Split-Path -Parent)"
+
+            # Attach a PSResourceGet repository to the directory holding the packages.
+            if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host "[RUNTIME-Import-ModulePackage]Registering local package repository $RepositoryName" }
+            Register-PSResourceRepository -Name $RepositoryName -Uri $PackageDirectory -Trusted -Priority 1
+
+            # Then, enumerate all the packages in that repository (again, just a directory) and "save" (install/unpack them) into /tmp.
+            if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host "[RUNTIME-Import-ModulePackage]Enumerating packages in $PackageDirectory (PSResource repository $RepositoryName)" }
+            Find-PSResource -Name * -Repository $RepositoryName | ForEach-Object -Parallel {
+                if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host "[RUNTIME-Import-ModulePackage]Saving package $($_.Name) version $($_.Version) (PSResource repository $($using:RepositoryName))" }
+                $_ | Save-PSResource -SkipDependencyCheck -Path $using:UnpackDirectory -Quiet -AcceptLicense -Confirm:$false
+            }
+
+            # Clean up the local repository config. This doesn't uninstall anything (just edits some XML files in PSResourceGet)
+            if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host "[RUNTIME-Import-ModulePackage]Registering local package repository $RepositoryName" }
+            Unregister-PSResourceRepository -Name $RepositoryName -Confirm:$false
+        }
+        if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host '[RUNTIME-Import-ModulePackage]Archive unpack complete' }
+    }
+}

powershell-runtime/source/modules/Private/Set-PSModulePath.ps1

@@ -14,6 +14,7 @@ function private:Set-PSModulePath {
                 1: Modules supplied with pwsh
                 2: User supplied modules as part of Lambda Layers
                 3: User supplied modules as part of function package
+                4: Compressed modules
     #>
     if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host '[RUNTIME-Set-PSModulePath]Start: Set-PSModulePath' }
     if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host '[RUNTIME-Set-PSModulePath]Setting PSModulePath environment variable' }
@@ -22,5 +23,19 @@ function private:Set-PSModulePath {
         '/opt/modules', # User supplied modules as part of Lambda Layers
         [System.IO.Path]::Combine($env:LAMBDA_TASK_ROOT, 'modules') # User supplied modules as part of function package
     ) -join ':'
+
+    # Iterate over both packed module directories (.Combined for modules.zip, .NuPkg for nupkgs) and... add their unpack dirs to PSModulePath if present
+    $Script:ModulePaths.Packed.GetEnumerator() | ForEach-Object {
+
+        # If the unpack directory exists...
+        If (Test-Path -LiteralPath $_.Value -ErrorAction SilentlyContinue) {
+
+            # Add it to PSModulePath.
+            if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host "[RUNTIME-Set-PSModulePath]$($_.Key) module package detected, adding unpack directory to PSModulePath" }
+            $env:PSModulePath += (':' + $_.Value)
+
+        }
+    }
+
     if ($env:POWERSHELL_RUNTIME_VERBOSE -eq 'TRUE') { Write-Host "[RUNTIME-Set-PSModulePath]PSModulePath environment variable set to: $($env:PSModulePath)" }
 }

powershell-runtime/source/modules/pwsh-runtime.psm1

@@ -3,6 +3,23 @@
 
 Set-PSDebug -Strict
 
+$Script:ModulePaths = @{
+    Packed = @{
+        Combined = @{
+            Layer = "/opt/modules.zip"
+            Root = "$Env:LAMBDA_TASK_ROOT/modules.zip"
+        }
+        NuPkg = @{
+            Layer = "/opt/module-nupkgs/*.nupkg"
+            Root = "$Env:LAMBDA_TASK_ROOT/module-nupkgs/*.nupkg"
+        }
+    }
+    Unpacked = @{
+        Combined = '/tmp/powershell-custom-runtime-unpacked-modules/combined'
+        NuPkg = '/tmp/powershell-custom-runtime-unpacked-modules/nupkg'
+    }
+}
+
 ##### All code below this comment is excluded from the build process
 
 # All Private modules merged into this file during build process to speed up module loading.