Enable adding and using different types of Data Extension clients

src/Sage.Engine.Data/DependencyInjection/ConfigureSageInMemoryDataOption.cs

@@ -0,0 +1,32 @@
+// Copyright (c) 2023, salesforce.com, inc.
+// All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+// For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/Apache-2.0
+
+using Microsoft.Extensions.Options;
+using Sage.Engine.Extensions;
+
+namespace Sage.Engine.Data.Sqlite
+{
+    public class ConfigureSageInMemoryDataOption : IConfigureOptions<SageInMemoryDataOption>
+    {
+        private readonly IOptions<SageOptions> _sageOptions;
+
+        public ConfigureSageInMemoryDataOption(
+            IOptions<SageOptions> sageOptions)
+        {
+            _sageOptions = sageOptions;
+        }
+
+        public void Configure(SageInMemoryDataOption options)
+        {
+            if (_sageOptions.Value.WorkingPath != null)
+            {
+                options.DataExtensionDirectory = _sageOptions.Value.WorkingPath.AppendDirectory("DataExtensions");
+                return;
+            }
+
+            options.DataExtensionDirectory = new DirectoryInfo(AppDomain.CurrentDomain.BaseDirectory).AppendDirectory("DataExtensions");
+        }
+    }
+}

src/Sage.Engine.Data/DependencyInjection/SageDataDependencyInjectionExtensions.cs

@@ -4,6 +4,7 @@
 // For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/Apache-2.0
 
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
 using Sage.Engine.Data.Sqlite;
 
 namespace Sage.Engine.Data.DependencyInjection
@@ -19,6 +20,8 @@ public static void AddInMemoryDataExtensions(
                 inMemoryOptions.ConnectionString = "Data Source=:memory:";
             });
 
+            services.AddSingleton<IConfigureOptions<SageInMemoryDataOption>, ConfigureSageInMemoryDataOption>();
+
             if (options != null)
             {
                 services.Configure(options);
@@ -27,4 +30,4 @@ public static void AddInMemoryDataExtensions(
             services.AddScoped<IDataExtensionClient, SqliteDataExtensionClient>();
         }
     }
-}
+}

src/Sage.Engine.Data/Sage.Engine.Data.csproj → src/Sage.Engine.Data/Sage.Engine.InMemoryData.csproj

@@ -12,4 +12,8 @@
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
   </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Sage.Engine\Sage.Engine.csproj" />
+  </ItemGroup>
 </Project>

src/Sage.Engine.Tests/DataTests.cs

@@ -1,8 +1,18 @@
-// Copyright (c) 2022, salesforce.com, inc.
+// Copyright (c) 2022, salesforce.com, inc.
 // All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 // For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/Apache-2.0
 
+using System.Data;
+using Microsoft.Extensions.DependencyInjection;
+using Sage.Engine.Compiler;
+using Sage.Engine.Data;
+using Sage.Engine.Data.DependencyInjection;
+using Sage.Engine.DependencyInjection;
+using Sage.Engine.Extensions;
+using Sage.Engine.Runtime;
+using Sage.Engine.Tests.Compatibility;
+
 namespace Sage.Engine.Tests
 {
     using NUnit.Framework;
@@ -21,4 +31,73 @@ public EngineTestResult TestDataFunctions(CorpusData test)
             return result;
         }
     }
+
+    public class DataExtensionClientTests
+    {
+        /// <summary>
+        /// Whatever the request, returns a table with a single column and single value
+        /// </summary>
+        class MockDataExtensionClient : IDataExtensionClient
+        {
+            public void Dispose()
+            {
+            }
+
+            private DataTable CreateTestTable()
+            {
+                DataTable table = new DataTable();
+                table.Columns.Add("TestRow", typeof(string));
+                DataRow row = table.NewRow();
+                row["TestRow"] = "TestValue";
+                table.Rows.Add(row);
+                return table;
+            }
+
+            public Task<DataTable> LookupAsync(LookupRequest lookup)
+            {
+                return Task.FromResult(CreateTestTable());
+            }
+
+            public Task InsertAsync(InsertRequest insert)
+            {
+                return Task.FromResult(CreateTestTable());
+            }
+
+            public Task<DataTable> GetSchemaAsync(string dataExtension)
+            {
+                return Task.FromResult(CreateTestTable());
+            }
+
+            public Task ConnectAsync()
+            {
+                return Task.CompletedTask;
+            }
+        }
+
+        /// <summary>
+        /// Demonstrates how a simple DE client can be created and tested
+        /// </summary>
+        [Test]
+        public void CanOverrideDataExtensionClient()
+        {
+            var collection = new ServiceCollection();
+            collection.AddSage();
+            collection.AddScoped<IDataExtensionClient, MockDataExtensionClient>();
+            ServiceProvider provider = collection.BuildServiceProvider();
+
+
+            CompilationOptions options = new CompilerOptionsBuilder()
+                .WithContent(new EmbeddedContent(
+                    "%%=Lookup('Test','TestRow','TestRow', 'TestValue')=%%",
+                    "TEST",
+                    "TEST", 
+                    1, 
+                    ContentType.AMPscript))
+                .Build();
+
+            string result = provider.GetService<Renderer>()!.Render(options, null);
+
+            Assert.That(result, Is.EqualTo("TestValue"));
+        }
+    }
 }

src/Sage.Engine.Tests/Sage.Engine.Tests.csproj

@@ -24,7 +24,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\Sage.Engine.Data\Sage.Engine.Data.csproj" />
+    <ProjectReference Include="..\Sage.Engine.Data\Sage.Engine.InMemoryData.csproj" />
     <ProjectReference Include="..\Sage.Engine\Sage.Engine.csproj" />
   </ItemGroup>
 

src/Sage.Engine.Tests/SageTest.cs

@@ -14,6 +14,7 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Sage.Engine.Compiler;
+using Sage.Engine.Data.DependencyInjection;
 using Sage.Engine.DependencyInjection;
 using Sage.Engine.Extensions;
 using Sage.Engine.Tests.Compatibility;
@@ -36,6 +37,7 @@ public void SetupSageTestContainer()
                 sageOptions.WorkingPath = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
                 sageOptions.OutputRootPath = sageOptions.WorkingPath.AppendDirectory("SageOutput");
             }));
+            collection.AddInMemoryDataExtensions();
 
             collection.AddMarketingCloudRenderingService();
 

src/Sage.Engine/DependencyInjection/SageDependencyInjectionExtensions.cs

@@ -7,10 +7,8 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
 using Sage.Engine.Compiler;
-using Sage.Engine.Data.DependencyInjection;
-using Sage.Engine.Data.Sqlite;
-using Sage.Engine.Extensions;
 using Sage.Engine.Handlebars;
+using Sage.Engine.Runtime;
 
 namespace Sage.Engine.DependencyInjection
 {
@@ -50,16 +48,8 @@ public static void AddSage(
                     compilationOptions.OutputDirectory = sageOptions.Value.OutputRootPath;
                 }
             });
-
-            services.AddInMemoryDataExtensions();
-            services.AddOptions<SageInMemoryDataOption>().Configure<IOptions<SageOptions>>((dataOptions, sageOptions) =>
-            {
-                if (sageOptions.Value.WorkingPath != null)
-                {
-                    dataOptions.DataExtensionDirectory = sageOptions.Value.WorkingPath.AppendDirectory("DataExtensions");
-                }
-            });
             services.AddLocalDiskContentClient();
+            services.AddScoped<RuntimeContext>();
             services.AddScoped<Renderer>();
             services.AddScoped<ICompiler, AmpscriptCompiler>();
             services.AddScoped<ICompiler, HandlebarsCompiler>();

src/Sage.Engine/Sage.Engine.csproj

@@ -27,9 +27,4 @@
       <Package>Sage.Engine.Parser</Package>
     </Antlr4>
   </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\Sage.Engine.Data\Sage.Engine.Data.csproj" />
-  </ItemGroup>
-
 </Project>

src/Sage.Webhost/DependencyInjection/SageWebDependencyInjectionExtensions.cs

@@ -7,6 +7,7 @@
 using Microsoft.Extensions.Options;
 using Sage.Engine;
 using Sage.Engine.Compiler;
+using Sage.Engine.Data.DependencyInjection;
 using Sage.Engine.Data.Sqlite;
 using Sage.Engine.DependencyInjection;
 using Sage.Engine.Extensions;
@@ -120,6 +121,9 @@ public static void AddSageWebhost(
             IServiceCollection commandLineCollection = new ServiceCollection();
             services.AddSagePackageManagerCommandLine(commandLineCollection);
             services.AddSage();
+
+            services.AddInMemoryDataExtensions();
+
             services.AddSageLaunchCommand(commandLineCollection);
             ServiceProvider provider = commandLineCollection.BuildServiceProvider();
             provider.GetRequiredService<LaunchCommand>().Invoke(args);

src/Sage.Webhost/Sage.Webhost.csproj

@@ -11,6 +11,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\Sage.Engine.Data\Sage.Engine.InMemoryData.csproj" />
     <ProjectReference Include="..\Sage.Engine\Sage.Engine.csproj" />
     <ProjectReference Include="..\Sage.PackageManager\Sage.PackageManager.csproj" />
   </ItemGroup>

src/Sage.sln

@@ -16,13 +16,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sage.Webhost", "Sage.Webhos
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GenerateComparisonCorpusTesting", "GenerateComparisonCorpusTesting\GenerateComparisonCorpusTesting.csproj", "{901F09B5-E27E-4E94-9CC3-560784D73E53}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sage.Engine.Data", "Sage.Engine.Data\Sage.Engine.Data.csproj", "{BEDD5FF0-4677-4EC0-A984-ECD19B9FB63B}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sage.Engine.InMemoryData", "Sage.Engine.Data\Sage.Engine.InMemoryData.csproj", "{BEDD5FF0-4677-4EC0-A984-ECD19B9FB63B}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sage.PackageManager", "Sage.PackageManager\Sage.PackageManager.csproj", "{19767037-EBDC-46F2-B017-8B5EC6A04802}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Sage.PackageManager.Tests", "Sage.PackageManager.Tests\Sage.PackageManager.Tests.csproj", "{DC2F94A6-8CA2-4DE6-860D-3E711FBAC769}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VSCode", "vscode\vscode.proj", "{48F56A6B-A285-4B40-9E96-044F7AA2C532}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "vscode", "vscode\vscode.proj", "{48F56A6B-A285-4B40-9E96-044F7AA2C532}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution