Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions internal/provisioning/hostagent/networkmanager/network_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,28 @@ func (nm *NetworkManager) processNetworkRequest(nr NetworkRequest) error {
return nil
}
operations := []networkOperation{
{
name: "EnsureDriverBoundP0",
f: func(nr NetworkRequest) error {
return hostutil.NewPCIHelper(nr.PCIAddress).PF(0).BindDriver("mlx5_core")
},
},
{
name: "EnsureDriverBoundP1",
f: func(nr NetworkRequest) error {
pf := hostutil.NewPCIHelper(nr.PCIAddress).PF(1)
isDPU, err := pf.IsDPU()
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("failed to check if device is DPU: %w", err)
} else if !isDPU {
return nil
}
return pf.BindDriver("mlx5_core")
},
},
{
name: "CreateP0VF",
f: func(nr NetworkRequest) error {
Expand Down
32 changes: 32 additions & 0 deletions internal/provisioning/hostagent/util/pci.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,38 @@ func (h *PCIHelper) SetNumOfVFs(num int) error {
return os.WriteFile(numvfsPath, []byte(fmt.Sprintf("%d", num)), 0644)
}

// IsDriverBound returns true if a driver is currently bound to the device.
func (h *PCIHelper) IsDriverBound() (bool, error) {
if _, err := os.Lstat(filepath.Join(h.Path(), "driver")); err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, fmt.Errorf("failed to stat driver symlink: %w", err)
}
return true, nil
}

// BindDriver writes the device's BDF to /sys/bus/pci/drivers/<driverName>/bind,
// causing the kernel to attempt to bind the named driver to the device.
// No-op if a driver is already bound. May block while the kernel runs the
// driver's probe routine; if the device's firmware is in pre-init, the bind
// can return -ETIMEDOUT after the kernel's wait_fw_init timeout.
func (h *PCIHelper) BindDriver(driverName string) error {
bound, err := h.IsDriverBound()
if err != nil {
return err
}
if bound {
return nil
}
bdf := filepath.Base(h.Path())
bindPath := filepath.Join(h.sysFSRoot, "bus/pci/drivers", driverName, "bind")
if err := os.WriteFile(bindPath, []byte(bdf), 0644); err != nil {
return fmt.Errorf("failed to bind %s to %s: %w", bdf, driverName, err)
}
return nil
}

// GetMTU returns the current MTU of the PF interface
func (h *PCIHelper) GetMTU() (int, error) {
interfaceName, err := h.InterfaceName()
Expand Down
80 changes: 80 additions & 0 deletions internal/provisioning/hostagent/util/pci_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -522,6 +522,86 @@ var _ = Describe("PCI", func() {
})
})

Context("PCIHelper.IsDriverBound", Label("PCIHelper", "IsDriverBound"), func() {
It("should return false when device has no driver symlink", func() {
mock := createMockSysfs("0000:b1:00.0", "0xa2dc\n", "", nil, "")
defer mock.Cleanup()

helper := NewPCIHelper("0000:b1:00.0").SetSysFS(mock.TempSysfsDir())
bound, err := helper.IsDriverBound()
Expect(err).NotTo(HaveOccurred())
Expect(bound).To(BeFalse())
})

It("should return true when driver symlink exists", func() {
mock := createMockSysfs("0000:b1:00.0", "0xa2dc\n", "", nil, "")
defer mock.Cleanup()

// Create a fake driver symlink to mimic the kernel binding the device
driverTarget := filepath.Join(mock.TempSysfsDir(), "bus/pci/drivers/mlx5_core")
Expect(os.MkdirAll(driverTarget, 0755)).To(Succeed())
driverSymlink := filepath.Join(mock.PCIDevicesDir(), "0000:b1:00.0", "driver")
Expect(os.Symlink(driverTarget, driverSymlink)).To(Succeed())

helper := NewPCIHelper("0000:b1:00.0").SetSysFS(mock.TempSysfsDir())
bound, err := helper.IsDriverBound()
Expect(err).NotTo(HaveOccurred())
Expect(bound).To(BeTrue())
})
})

Context("PCIHelper.BindDriver", Label("PCIHelper", "BindDriver"), func() {
It("should write the BDF to the driver's bind file when no driver is bound", func() {
mock := createMockSysfs("0000:b1:00.0", "0xa2dc\n", "", nil, "")
defer mock.Cleanup()

// Create the driver's bind file (kernel-managed in real sysfs)
bindDir := filepath.Join(mock.TempSysfsDir(), "bus/pci/drivers/mlx5_core")
Expect(os.MkdirAll(bindDir, 0755)).To(Succeed())
bindPath := filepath.Join(bindDir, "bind")
Expect(os.WriteFile(bindPath, []byte{}, 0644)).To(Succeed())

helper := NewPCIHelper("0000:b1:00.0").SetSysFS(mock.TempSysfsDir())
Expect(helper.BindDriver("mlx5_core")).To(Succeed())

content, err := os.ReadFile(bindPath)
Expect(err).NotTo(HaveOccurred())
Expect(string(content)).To(Equal("0000:b1:00.0"))
})

It("should be a no-op when a driver is already bound", func() {
mock := createMockSysfs("0000:b1:00.0", "0xa2dc\n", "", nil, "")
defer mock.Cleanup()

// Pre-bind by creating the driver symlink
driverTarget := filepath.Join(mock.TempSysfsDir(), "bus/pci/drivers/mlx5_core")
Expect(os.MkdirAll(driverTarget, 0755)).To(Succeed())
driverSymlink := filepath.Join(mock.PCIDevicesDir(), "0000:b1:00.0", "driver")
Expect(os.Symlink(driverTarget, driverSymlink)).To(Succeed())

// Create the bind file too, but pre-populated to detect any unwanted writes
bindPath := filepath.Join(driverTarget, "bind")
Expect(os.WriteFile(bindPath, []byte("untouched"), 0644)).To(Succeed())

helper := NewPCIHelper("0000:b1:00.0").SetSysFS(mock.TempSysfsDir())
Expect(helper.BindDriver("mlx5_core")).To(Succeed())

content, err := os.ReadFile(bindPath)
Expect(err).NotTo(HaveOccurred())
Expect(string(content)).To(Equal("untouched"))
})

It("should return error when bind file does not exist", func() {
mock := createMockSysfs("0000:b1:00.0", "0xa2dc\n", "", nil, "")
defer mock.Cleanup()

helper := NewPCIHelper("0000:b1:00.0").SetSysFS(mock.TempSysfsDir())
err := helper.BindDriver("mlx5_core")
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("failed to bind 0000:b1:00.0 to mlx5_core"))
})
})

Context("DiscoverDPUs", Label("DiscoverDPUs"), func() {
It("should return error when PCI devices directory does not exist", func() {
// Create mock to get cleanup() for path restoration, then point to non-existent path
Expand Down