feat: audiocapture support for real iOS devices with device kit

cli/audiocapture.go

@@ -0,0 +1,89 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/mobile-next/mobilecli/commands"
+	"github.com/mobile-next/mobilecli/devices"
+	"github.com/mobile-next/mobilecli/utils"
+	"github.com/spf13/cobra"
+)
+
+var audiocaptureCmd = &cobra.Command{
+	Use:   "audiocapture",
+	Short: "Stream audio capture from a connected device",
+	Long:  "Streams audio capture from a specified device to stdout. Supports Opus (real iOS devices only).",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if audiocaptureFormat != "opus+rtp" && audiocaptureFormat != "opus+ogg" {
+			response := commands.NewErrorResponse(fmt.Errorf("format must be 'opus+rtp' or 'opus+ogg' for audio capture"))
+			printJson(response)
+			return fmt.Errorf("%s", response.Error)
+		}
+
+		targetDevice, err := commands.FindDeviceOrAutoSelect(deviceId)
+		if err != nil {
+			response := commands.NewErrorResponse(fmt.Errorf("error finding device: %v", err))
+			printJson(response)
+			return fmt.Errorf("%s", response.Error)
+		}
+
+		if targetDevice.Platform() != "ios" || targetDevice.DeviceType() != "real" {
+			response := commands.NewErrorResponse(fmt.Errorf("audio capture is only supported on real iOS devices"))
+			printJson(response)
+			return fmt.Errorf("%s", response.Error)
+		}
+
+		var parser *utils.OpusFrameParser
+		var oggWriter *utils.OggOpusWriter
+		if audiocaptureFormat == "opus+ogg" {
+			var err error
+			oggWriter, err = utils.NewOggOpusWriter(os.Stdout)
+			if err != nil {
+				response := commands.NewErrorResponse(fmt.Errorf("failed to initialize ogg writer: %v", err))
+				printJson(response)
+				return fmt.Errorf("%s", response.Error)
+			}
+			parser = utils.NewOpusFrameParser(func(packet []byte) error {
+				return oggWriter.WritePacket(packet)
+			})
+		}
+
+		err = targetDevice.StartAudioCapture(devices.AudioCaptureConfig{
+			Format: audiocaptureFormat,
+			OnProgress: func(message string) {
+				utils.Verbose(message)
+			},
+			OnData: func(data []byte) bool {
+				if parser != nil {
+					if err := parser.Write(data); err != nil {
+						fmt.Fprintf(os.Stderr, "Error writing Ogg Opus data: %v\n", err)
+						return false
+					}
+				} else {
+					_, writeErr := os.Stdout.Write(data)
+					if writeErr != nil {
+						fmt.Fprintf(os.Stderr, "Error writing data: %v\n", writeErr)
+						return false
+					}
+				}
+				return true
+			},
+		})
+
+		if err != nil {
+			response := commands.NewErrorResponse(fmt.Errorf("error starting audio capture: %v", err))
+			printJson(response)
+			return fmt.Errorf("%s", response.Error)
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(audiocaptureCmd)
+
+	audiocaptureCmd.Flags().StringVar(&deviceId, "device", "", "ID of the device to capture from")
+	audiocaptureCmd.Flags().StringVarP(&audiocaptureFormat, "format", "f", "opus+rtp", "Output format for audio capture (opus+rtp, opus+ogg)")
+}

cli/flags.go

@@ -13,6 +13,8 @@ var (
 
 	// for screencapture command
 	screencaptureFormat string
+	// for audiocapture command
+	audiocaptureFormat string
 
 	// for devices command
 	platform   string

commands/audiocapture.go

@@ -0,0 +1,6 @@
+package commands
+
+type AudioCaptureRequest struct {
+	DeviceID string `json:"deviceId"`
+	Format   string `json:"format"`
+}

devices/android.go

@@ -628,7 +628,7 @@ func isAscii(text string) bool {
 // escapeShellText escapes shell special characters
 func escapeShellText(text string) string {
 	// escape all shell special characters that could be used for injection
-	specialChars := `\'"`+ "`" + `
+	specialChars := `\'"` + "`" + `
 |&;()<>{}[]$*?`
 	result := ""
 	for _, char := range text {
@@ -849,6 +849,10 @@ func (d *AndroidDevice) StartScreenCapture(config ScreenCaptureConfig) error {
 	return nil
 }
 
+func (d *AndroidDevice) StartAudioCapture(config AudioCaptureConfig) error {
+	return fmt.Errorf("audio capture is only supported on real iOS devices")
+}
+
 func (d *AndroidDevice) installPackage(apkPath string) error {
 	output, err := d.runAdbCommand("install", apkPath)
 	if err != nil {

devices/common.go

@@ -28,6 +28,13 @@ type ScreenCaptureConfig struct {
 	OnData     func([]byte) bool    // data callback - return false to stop
 }
 
+// AudioCaptureConfig contains configuration for audio capture operations
+type AudioCaptureConfig struct {
+	Format     string
+	OnProgress func(message string) // optional progress callback
+	OnData     func([]byte) bool    // data callback - return false to stop
+}
+
 // StartAgentConfig contains configuration for agent startup operations
 type StartAgentConfig struct {
 	OnProgress func(message string) // optional progress callback
@@ -65,6 +72,7 @@ type ControllableDevice interface {
 	UninstallApp(packageName string) (*InstalledAppInfo, error)
 	Info() (*FullDeviceInfo, error)
 	StartScreenCapture(config ScreenCaptureConfig) error
+	StartAudioCapture(config AudioCaptureConfig) error
 	DumpSource() ([]ScreenElement, error)
 	DumpSourceRaw() (interface{}, error)
 	GetOrientation() (string, error)

devices/ios.go

@@ -32,6 +32,7 @@
 	portRangeEnd              = 8299
 	deviceKitHTTPPort         = 12004 // device-side HTTP server port
 	deviceKitStreamPort       = 12005 // device-side H.264 TCP stream port
+	deviceKitAudioPort        = 12006 // device-side Opus audio TCP stream port
 	deviceKitAppLaunchTimeout = 5 * time.Second
 	deviceKitBroadcastTimeout = 5 * time.Second
 )
@@ -722,7 +723,7 @@
 
 		// stream data in a goroutine
 		go func() {
 			defer conn.Close()
 			buffer := make([]byte, 65536)
 			for {
 				n, err := conn.Read(buffer)
@@ -748,7 +749,7 @@
 		// wait for either signal or stream completion
 		select {
 		case <-sigChan:
 			conn.Close()
 			utils.Verbose("stream closed by user")
 			return nil
 		case err := <-done:
@@ -775,6 +776,78 @@
 	return d.mjpegClient.StartScreenCapture(config.Format, config.OnData)
 }
 
+func (d IOSDevice) StartAudioCapture(config AudioCaptureConfig) error {
+	if config.Format != "opus+rtp" && config.Format != "opus+ogg" {
+		return fmt.Errorf("format must be 'opus+rtp' or 'opus+ogg' for audio capture")
+	}
+
+	if d.Platform() != "ios" || d.DeviceType() != "real" {
+		return fmt.Errorf("audio capture is only supported on real iOS devices")
+	}
+
+	if config.OnProgress != nil {
+		config.OnProgress("Starting port forwarding for Opus audio stream")
+	}
+
+	localAudioPort, err := findAvailablePortInRange(portRangeStart, portRangeEnd)
+	if err != nil {
+		return fmt.Errorf("failed to find available port for audio: %w", err)
+	}
+
+	audioForwarder := ios.NewPortForwarder(d.ID())
+	err = audioForwarder.Forward(localAudioPort, deviceKitAudioPort)
+	if err != nil {
+		return fmt.Errorf("failed to forward audio port: %w", err)
+	}
+	defer func() { _ = audioForwarder.Stop() }()
+
+	if config.OnProgress != nil {
+		config.OnProgress(fmt.Sprintf("Connecting to Opus stream on localhost:%d", localAudioPort))
+	}
+
+	conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", localAudioPort))
+	if err != nil {
+		return fmt.Errorf("failed to connect to audio stream port: %w", err)
+	}
+
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+	done := make(chan error, 1)
+	go func() {
+		defer conn.Close()
+		buffer := make([]byte, 65536)
+		for {
+			n, err := conn.Read(buffer)
+			if err != nil {
+				if err != io.EOF {
+					done <- fmt.Errorf("error reading from audio stream: %w", err)
+				} else {
+					done <- nil
+				}
+				return
+			}
+
+			if n > 0 {
+				if !config.OnData(buffer[:n]) {
+					done <- nil
+					return
+				}
+			}
+		}
+	}()
+
+	select {
+	case <-sigChan:
+		conn.Close()
+		utils.Verbose("audio stream closed by user")
+		return nil
+	case err := <-done:
+		utils.Verbose("audio stream ended")
+		return err
+	}
+}
+
 func (d IOSDevice) DumpSource() ([]ScreenElement, error) {
 	return d.wdaClient.GetSourceElements()
 }
@@ -857,6 +930,7 @@
 type DeviceKitInfo struct {
 	HTTPPort   int `json:"httpPort"`
 	StreamPort int `json:"streamPort"`
+	AudioPort  int `json:"audioPort"`
 }
 
 // clickStartBroadcastButton polls for the "BroadcastUploadExtension" button, taps it,
@@ -1052,13 +1126,31 @@
 	}
 	utils.Verbose("Port forwarding started: localhost:%d -> device:%d (H.264 stream)", localStreamPort, deviceKitStreamPort)
 
+	// Find available local port for audio forwarding.
+	localAudioPort, err := findAvailablePortInRange(portRangeStart, portRangeEnd)
+	if err != nil {
+		_ = httpForwarder.Stop()
+		_ = streamForwarder.Stop()
+		return nil, fmt.Errorf("failed to find available port for audio: %w", err)
+	}
+
+	audioForwarder := ios.NewPortForwarder(d.ID())
+	err = audioForwarder.Forward(localAudioPort, deviceKitAudioPort)
+	if err != nil {
+		_ = httpForwarder.Stop()
+		_ = streamForwarder.Stop()
+		return nil, fmt.Errorf("failed to forward audio port: %w", err)
+	}
+	utils.Verbose("Port forwarding started: localhost:%d -> device:%d (Opus stream)", localAudioPort, deviceKitAudioPort)
+
 	// Launch the main DeviceKit app
 	utils.Verbose("Launching DeviceKit app: %s", devicekitMainAppBundleId)
 	err = d.LaunchApp(devicekitMainAppBundleId)
 	if err != nil {
 		// clean up port forwarders on failure
 		_ = httpForwarder.Stop()
 		_ = streamForwarder.Stop()
+		_ = audioForwarder.Stop()
 		return nil, fmt.Errorf("failed to launch DeviceKit app: %w", err)
 	}
 
@@ -1076,6 +1168,7 @@
 		// clean up port forwarders on failure
 		_ = httpForwarder.Stop()
 		_ = streamForwarder.Stop()
+		_ = audioForwarder.Stop()
 		return nil, fmt.Errorf("failed to start agent: %w", err)
 	}
 
@@ -1085,6 +1178,7 @@
 		// clean up port forwarders on failure
 		_ = httpForwarder.Stop()
 		_ = streamForwarder.Stop()
+		_ = audioForwarder.Stop()
 		return nil, fmt.Errorf("failed to click Start Broadcast button: %w", err)
 	}
 
@@ -1097,6 +1191,7 @@
 	return &DeviceKitInfo{
 		HTTPPort:   localHTTPPort,
 		StreamPort: localStreamPort,
+		AudioPort:  localAudioPort,
 	}, nil
 }
 

devices/simulator.go

@@ -723,6 +723,10 @@ func (s *SimulatorDevice) StartScreenCapture(config ScreenCaptureConfig) error {
 	return mjpegClient.StartScreenCapture(config.Format, config.OnData)
 }
 
+func (s *SimulatorDevice) StartAudioCapture(config AudioCaptureConfig) error {
+	return fmt.Errorf("audio capture is only supported on real iOS devices")
+}
+
 type ProcessInfo struct {
 	PID     int
 	Command string

go.mod

@@ -4,6 +4,7 @@ go 1.25.0
 
 require (
 	github.com/danielpaulus/go-ios v1.0.182
+	github.com/gorilla/websocket v1.5.3
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.9.1
 	github.com/stretchr/testify v1.10.0

go.sum

@@ -28,6 +28,8 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE=
 github.com/grandcat/zeroconf v1.0.0/go.mod h1:lTKmG1zh86XyCoUeIHSA4FJMBwCJiQmGfcP2PdzytEs=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=