|
| 1 | +module ArcpSamples.LiteLLM |
| 2 | + |
| 3 | +open System |
| 4 | +open System.Linq |
| 5 | +open System.Net.Http |
| 6 | +open System.Net.Http.Headers |
| 7 | +open System.Text |
| 8 | +open System.Text.Json |
| 9 | +open System.Threading |
| 10 | +open System.Threading.Tasks |
| 11 | +open ARCP.Core |
| 12 | +open ARCP.Runtime |
| 13 | +open ArcpSamples.SampleHarness |
| 14 | + |
| 15 | +type LiteLLMProvisioner(baseUrl: Uri, adminKey: string, http: HttpClient) = |
| 16 | + let modelPatterns (lease: LeaseGrant) = |
| 17 | + Map.tryFind Capabilities.ModelUse lease.Capabilities |
| 18 | + |> Option.defaultValue [] |
| 19 | + |
| 20 | + let maxBudget (lease: LeaseGrant) = |
| 21 | + Map.tryFind Capabilities.CostBudget lease.Capabilities |
| 22 | + |> Option.bind ( |
| 23 | + List.tryPick (fun amount -> |
| 24 | + match Lease.parseBudgetAmount amount with |
| 25 | + | Ok (_, value) -> Some value |
| 26 | + | Error _ -> None)) |
| 27 | + |
| 28 | + let postJsonAsync (path: string) (body: obj) (ct: CancellationToken) = |
| 29 | + task { |
| 30 | + use request = new HttpRequestMessage(HttpMethod.Post, Uri(baseUrl, path)) |
| 31 | + request.Headers.Authorization <- AuthenticationHeaderValue("Bearer", adminKey) |
| 32 | + request.Content <- |
| 33 | + new StringContent( |
| 34 | + JsonSerializer.Serialize body, |
| 35 | + Encoding.UTF8, |
| 36 | + "application/json") |
| 37 | + let! response = http.SendAsync(request, ct) |
| 38 | + response.EnsureSuccessStatusCode() |> ignore |
| 39 | + return! response.Content.ReadAsStringAsync ct |
| 40 | + } |
| 41 | + |
| 42 | + interface ICredentialProvisioner with |
| 43 | + member _.IssueAsync(ctx, ct) = |
| 44 | + task { |
| 45 | + let body = |
| 46 | + {| key_alias = ctx.JobId.Value |
| 47 | + duration = |
| 48 | + ctx.LeaseConstraints |
| 49 | + |> Option.map (fun c -> c.ExpiresAt - DateTimeOffset.UtcNow) |
| 50 | + |> Option.map (fun span -> max 1 (int span.TotalSeconds)) |
| 51 | + |> Option.defaultValue 300 |
| 52 | + models = modelPatterns ctx.Lease |> List.toArray |
| 53 | + max_budget = maxBudget ctx.Lease |> Option.toNullable |} |
| 54 | + :> obj |
| 55 | + let! raw = postJsonAsync "/key/generate" body ct |
| 56 | + use doc = JsonDocument.Parse raw |
| 57 | + let root = doc.RootElement |
| 58 | + let value = |
| 59 | + match root.TryGetProperty("key") with |
| 60 | + | true, p -> p.GetString() |
| 61 | + | _ -> |
| 62 | + match root.TryGetProperty("token") with |
| 63 | + | true, p -> p.GetString() |
| 64 | + | _ -> null |
| 65 | + if String.IsNullOrWhiteSpace value then |
| 66 | + return raise (InvalidOperationException "LiteLLM response did not include key or token") |
| 67 | + else |
| 68 | + let credential: Credential = { |
| 69 | + Id = CredentialId.newId () |
| 70 | + Scheme = "bearer" |
| 71 | + Value = value |
| 72 | + Endpoint = baseUrl.ToString().TrimEnd('/') |
| 73 | + Profile = Some "litellm" |
| 74 | + Constraints = |
| 75 | + Some { |
| 76 | + CostBudget = Map.tryFind Capabilities.CostBudget ctx.Lease.Capabilities |
| 77 | + ModelUse = Map.tryFind Capabilities.ModelUse ctx.Lease.Capabilities |
| 78 | + ExpiresAt = ctx.LeaseConstraints |> Option.map (fun c -> c.ExpiresAt) |
| 79 | + } |
| 80 | + } |
| 81 | + return [ credential ] |
| 82 | + } |
| 83 | + |
| 84 | + member _.RevokeAsync(credentialId, ct) = |
| 85 | + task { |
| 86 | + let body = {| key = credentialId |} :> obj |
| 87 | + let! _ = postJsonAsync "/key/delete" body ct |
| 88 | + return true |
| 89 | + } |
| 90 | + |
| 91 | +[<EntryPoint>] |
| 92 | +let main _argv = |
| 93 | + runAsync (fun () -> |
| 94 | + task { |
| 95 | + let baseUrl = Environment.GetEnvironmentVariable "LITELLM_BASE_URL" |
| 96 | + let adminKey = Environment.GetEnvironmentVariable "LITELLM_ADMIN_KEY" |
| 97 | + if String.IsNullOrWhiteSpace baseUrl || String.IsNullOrWhiteSpace adminKey then |
| 98 | + writeLine "Set LITELLM_BASE_URL and LITELLM_ADMIN_KEY to run this sample." |
| 99 | + return 0 |
| 100 | + else |
| 101 | + use http = new HttpClient() |
| 102 | + let provisioner = |
| 103 | + LiteLLMProvisioner(Uri(baseUrl), adminKey, http) :> ICredentialProvisioner |
| 104 | + let withLiteLLM (options: ArcpServerOptions) = |
| 105 | + { options with |
| 106 | + Provisioner = Some provisioner |
| 107 | + CredentialStore = Some (InMemoryCredentialStore() :> ICredentialStore) } |
| 108 | + let features = |
| 109 | + Set.ofList [ |
| 110 | + Features.ProvisionedCredentials |
| 111 | + Features.ModelUse |
| 112 | + Features.LeaseExpiresAt |
| 113 | + ] |
| 114 | + let! p = |
| 115 | + connectWithOptions |
| 116 | + withLiteLLM |
| 117 | + (fun s -> |
| 118 | + s.RegisterAgent("llm", fun ctx -> |
| 119 | + task { |
| 120 | + do! ctx.ValidateOpAsync( |
| 121 | + Capabilities.ModelUse, |
| 122 | + "gpt-4o-mini", |
| 123 | + ctx.CancellationToken) |
| 124 | + return jsonString "LiteLLM credential issued" |
| 125 | + })) |
| 126 | + features |
| 127 | + let lease = |
| 128 | + Lease.empty |
| 129 | + |> Lease.withCapability Capabilities.ModelUse [ "gpt-4o-mini" ] |
| 130 | + |> Lease.withCapability Capabilities.CostBudget [ "USD:1.00" ] |
| 131 | + let! handle = |
| 132 | + p.Client.SubmitAsync( |
| 133 | + { Agent = "llm" |
| 134 | + Input = jsonInt 0 |
| 135 | + LeaseRequest = Some lease |
| 136 | + LeaseConstraints = Some { ExpiresAt = DateTimeOffset.UtcNow.AddMinutes 10.0 } |
| 137 | + IdempotencyKey = None |
| 138 | + MaxRuntimeSec = None }, |
| 139 | + CancellationToken.None) |
| 140 | + writeLine (sprintf "issued %d LiteLLM credential(s)" handle.Credentials.Length) |
| 141 | + let! _ = handle.Result |
| 142 | + do! teardown p |
| 143 | + return 0 |
| 144 | + }) |
0 commit comments