tests/vrf: init

tests/vrf/README.md

@@ -0,0 +1,100 @@
+# vrf
+
+This NixOS test implements a multi-tenant routing topology using Linux VRFs and FRRouting (FRR).
+Each tenant is isolated in its own VRF, while a shared public VRF provides upstream Internet connectivity.
+
+The test models a small service-provider edge router that:
+- Operates its own Autonomous System
+- Connects to an upstream provider
+- Hosts multiple tenants with independent routing domains
+- Supports both BYOIP and provider-assigned address space
+
+The primary goal is to validate VRF isolation, routing correctness, and controlled route leaking.
+
+## Network diagram
+
+```mermaid
+flowchart TD
+  %% our router with the isolated VRF's
+  subgraph router["**Router (AS65550)**"]
+    rVRFt1["VRF tenant1"]
+    rVRFus["VRF public"]
+    rVRFds["VRF downstream1"]
+  end
+
+  %% upstream provider
+  us1r["BGP **Upstream**
+    (AS64497)
+    203.0.113.0/24
+    3fff:ffff::/32"]
+
+  us1cloud["203.0.113.0/30
+  3fff:ffff:1515:200::/64"]@{ shape: cloud }
+
+  %% downstream provider
+  ds1r["BGP **Downstream** (ds1r)
+  (AS65536)
+    198.51.100.0/24
+    2001:db8:beef::/48"]
+
+  ds1cloud["fe80::/64"]@{ shape: cloud }
+  ds1cloud2["198.51.100.64/26
+  2001:db8:beef:20::/64"]@{ shape: cloud }
+  ds1c
+
+  %% tenant1
+  t1cloud["10.0.10.0/24
+  2001:db8:10::/64"]@{ shape: cloud }
+
+  %% connections
+  us1r -- eth1 --- us1cloud -- eth2 --- rVRFus
+  rVRFds -- eth3 --- ds1cloud -- eth1 --- ds1r -- eth2 --- ds1cloud2 -- eth1 --- ds1c
+
+  rVRFt1 -- eth4 --- t1cloud
+  t1cloud -- eth1 --- t1a
+  t1cloud -- eth1 --- t1b
+```
+
+Notes:
+- The upstream provider supplies a full table to the router.
+- The downstream receives a full table (default free zone) for ipv4 and a default originate route for ipv6 (just to cover both cases).
+- You delegate tenant1 a ipv6 prefix, ipv4 traffic from rfc1918 source addresses should be natted on your router.
+
+### Resources
+| Owner       | Systems    | Resources                                   | Description             |
+|-------------|------------|---------------------------------------------|-------------------------|
+| ISP         | upstream   | AS64497, 203.0.113.0/24, 3fff:ffff::/32     | transit provider        |
+| You         | router     | AS65550, 192.0.2.0/24, 2001:db8::/40        |                         |
+| downstream1 | ds1r, ds1c | AS65536, 198.51.100.0/24, 2001:db8:beef:/48 | bgp downstream customer |
+| tenant1     | t2a, t2b   |                                             | non bgp customer        |
+
+## Design
+Each VRF has:
+- Its own routing table
+- Independent BGP address families
+- No implicit route leakage
+
+### Routing Model
+#### Public VRF
+- Establishes eBGP sessions with upstream providers
+- Serves as the shared exit point for all traffic
+
+#### Tenant VRFs
+- Maintain strict L3 isolation
+- May run BGP (Downstream 1) or static routing (Tenant 1)
+- Use explicit route leaking to access the public VRF
+
+#### Route Leaking
+- Route leaking is intentionally explicit and directional:
+  - Downstream VRFs (RD: `65550:20`, RT: `65550:1020`) may import routes from the public VRF (RD: `65550:10`, RT: `65550:1010`)
+  - Tenant routes are not visible to other tenants
+  - No transitive leakage between tenant VRFs
+
+## TODO
+- NAT for Tenant1 is being applied after VRF public (on eth2), this results in wrong dest address for reply. Is it possible to move NAT between VRF tenant1/public?
+- ds1r has ra default ipv6 routes, besides bird??? -> remove these!
+- Think about how to add default route in vrf tenant1, currently via main
+  > staticd[697]: [PNYPZ-BCP8Y] Static Route using public interface not installed because the interface does not exist in specified vrf
+
+### Extend (beyond first idea)
+- test default originate from upstream provider when route leaking is setup within frr (linux ip rule doesn't support this)

tests/vrf/default.nix

@@ -0,0 +1,75 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+{
+  name = "vrf";
+
+  defaults = {
+    networking = {
+      useDHCP = false;
+      firewall.enable = false;
+    };
+  };
+
+  nodes = {
+    router = import ./router;
+
+    upstream = import ./upstream.nix;
+
+    ds1r = import ./t1r.nix;
+    ds1c = import ./tenantClient.nix {
+      vlan = 5;
+      ipv4 = "198.51.100.99";
+      ipv4gw = "198.51.100.65";
+      ipv4cidr = 26;
+      ipv6 = "2001:db8:beef:20::c";
+    };
+    t1a = import ./tenantClient.nix {
+      vlan = 4;
+      ipv4 = "10.0.10.10";
+      ipv4gw = "10.0.10.1";
+      ipv6 = "2001:db8:10::a";
+    };
+    t1b = import ./tenantClient.nix {
+      vlan = 4;
+      ipv4 = "10.0.10.11";
+      ipv4gw = "10.0.10.1";
+      ipv6 = "2001:db8:10::b";
+    };
+  };
+
+  interactive.nodes = lib.listToAttrs (
+    map
+      (name: {
+        inherit name;
+        value.environment.systemPackages = with pkgs; [
+          nftables
+          tcpdump
+        ];
+      })
+      [
+        "router"
+        "upstream"
+        "ds1r"
+        "ds1c"
+        "t1a"
+        "t1b"
+      ]
+  );
+  # think about ping4/ping6 jobs every three seconds on tXY when in interactive mode
+
+  testScript = ''
+    start_all()
+
+    router.wait_for_unit("network.target")
+    router.wait_for_unit("frr.service")
+    upstream.wait_for_unit("bird.service")
+
+    with subtest("try to ping internet from clients in tenant vrf's"):
+      for m in [ ds1c, t1a, t1b ]:
+        m.succeed("ping -c 1 203.0.113.100")
+        m.succeed("ping -c 1 3fff:ffff::100")
+  '';
+}

tests/vrf/router/default.nix

@@ -0,0 +1,25 @@
+{
+  imports = [
+    ./interfaces
+    ./nat.nix
+  ];
+
+  services.frr = {
+    bgpd.enable = true;
+    # The default bgp instance MUST exist for vrf route leaks to work properly
+    config = ''
+      router bgp 65550
+      exit
+    '';
+  };
+
+  boot.kernel.sysctl = {
+    # enable ip forwarding
+    "net.ipv4.ip_forward" = 1;
+    "net.ipv6.conf.all.forwarding" = 1;
+
+    # bind sockets to all vrf's
+    "net.ipv4.tcp_l3mdev_accept" = 1;
+    "net.ipv4.udp_l3mdev_accept" = 1;
+  };
+}

tests/vrf/router/interfaces/default.nix

@@ -0,0 +1,16 @@
+{
+  imports = [
+    ./loopback.nix
+
+    ./eth1-lan.nix
+
+    ./vrf-public.nix
+    ./vrf-downstream1.nix
+    ./vrf-tenant1.nix
+  ];
+
+  networking = {
+    iproute2.enable = true;
+    ifstate.enable = true;
+  };
+}

tests/vrf/router/interfaces/eth1-lan.nix

@@ -0,0 +1,16 @@
+{
+  virtualisation.interfaces.eth1 = {
+    vlan = 1;
+    assignIP = false;
+  };
+
+  networking.ifstate.settings = {
+    interfaces.eth1 = {
+      addresses = [ ];
+      link = {
+        state = "up";
+        kind = "physical";
+      };
+    };
+  };
+}

tests/vrf/router/interfaces/loopback.nix

@@ -0,0 +1,12 @@
+{
+  networking.ifstate.settings.interfaces.loopback = {
+    addresses = [
+      "192.0.2.1/32"
+      "2001:db8::1/128"
+    ];
+    link = {
+      state = "up";
+      kind = "dummy";
+    };
+  };
+}

tests/vrf/router/interfaces/vrf-downstream1.nix

@@ -0,0 +1,88 @@
+let
+  name = "downstream1";
+in
+{
+  virtualisation.interfaces.eth3 = {
+    vlan = 3;
+    assignIP = false;
+  };
+
+  networking = {
+    iproute2 = {
+      rttablesExtraConfig = ''
+        20 ${name}
+      '';
+    };
+    ifstate.settings.interfaces = {
+      eth3 = {
+        addresses = [
+          "fe80::1/64"
+        ];
+        link = {
+          state = "up";
+          kind = "physical";
+          master = name;
+        };
+      };
+      "${name}" = {
+        link = {
+          state = "up";
+          kind = "vrf";
+          vrf_table = name;
+        };
+      };
+    };
+  };
+
+  services.frr.config = ''
+    ip prefix-list downstream1 seq 20 permit 198.51.100.0/24
+    ipv6 prefix-list downstream1-6 seq 20 permit 2001:db8:beef::/48
+
+    ! remove e.g. </24 prefixes from announcements to downstream
+    ! in reality we would also add stuff like rfc1918 networks here
+    ip prefix-list too-small seq 10 permit 0.0.0.0/0 ge 25
+    route-map downstream-out deny 10
+      match ip address prefix-list too-small
+    exit
+    route-map downstream-out permit 100
+    exit
+
+    router bgp 65550 vrf ${name}
+      no bgp ebgp-requires-policy
+      no bgp default ipv4-unicast
+      bgp router-id 192.0.2.1
+
+      neighbor fe80::2 remote-as 65536
+      neighbor fe80::2 capability extended-nexthop
+      neighbor fe80::2 interface eth3
+
+      address-family ipv4 unicast
+        neighbor fe80::2 activate
+        neighbor fe80::2 soft-reconfiguration inbound
+        neighbor fe80::2 prefix-list downstream1 in
+        neighbor fe80::2 route-map downstream-out out
+
+        rd vpn export 65550:20
+        rt vpn export 65550:1020
+        rt vpn import 65550:1020 65550:1010
+
+        export vpn
+        import vpn
+      exit-address-family
+
+      ! ipv4 example shows how to work with full table, this shows do
+      address-family ipv6 unicast
+        neighbor fe80::2 activate
+        neighbor fe80::2 soft-reconfiguration inbound
+        neighbor fe80::2 default-originate
+        neighbor fe80::2 prefix-list downstream1-6 in
+
+        rd vpn export 65550:20
+        rt vpn both 65550:1020
+
+        export vpn
+        import vpn
+      exit-address-family
+    exit
+  '';
+}