Expand SoundCloud downloading functionality

README.md

@@ -19,7 +19,7 @@ any device in podcast client.
 
 ## ✨ Features
 
-- Works with YouTube and Vimeo.
+- Works with YouTube, Vimeo, Twitch, and SoundCloud.
 - Supports feeds configuration: video/audio, high/low quality, max video height, etc.
 - mp3 encoding
 - Update scheduler supports cron expressions
@@ -66,6 +66,10 @@ In order to query YouTube or Vimeo API you have to obtain an API token first.
 - [How to get YouTube API key](https://elfsight.com/blog/2016/12/how-to-get-youtube-api-key-tutorial/)
 - [Generate an access token for Vimeo](https://developer.vimeo.com/api/guides/start#generate-access-token)
 
+SoundCloud does not require a permanent API token. Podsync uses an internal SoundCloud API wrapper that can
+automatically scrape a working `client_id` as needed. You may optionally provide your own `client_id` to reduce
+breakage if SoundCloud changes their public site.
+
 ## ⚙️ Configuration
 
 You need to create a configuration file (for instance `config.toml`) and specify the list of feeds that you're going to host.
@@ -104,17 +108,25 @@ hostname = "https://my.test.host:4443"
 
 Server will be accessible from `http://localhost:8080`, but episode links will point to `https://my.test.host:4443/ID1/...`
 
+### 🎵 SoundCloud URL formats
+
+Podsync supports the following SoundCloud URL formats:
+
+- **Playlists:** `https://soundcloud.com/<username>/sets/<playlist>`
+- **User profiles (uploads feed):** `https://soundcloud.com/<username>` (or `https://soundcloud.com/<username>/tracks`)
+
 ### 🌍 Environment Variables
 
 Podsync supports the following environment variables for configuration and API keys:
 
-| Variable Name                | Description                                                                               | Example Value(s)                              |
-|------------------------------|-------------------------------------------------------------------------------------------|-----------------------------------------------|
-| `PODSYNC_CONFIG_PATH`        | Path to the configuration file (overrides `--config` CLI flag)                            | `/app/config.toml`                            |
-| `PODSYNC_YOUTUBE_API_KEY`    | YouTube API key(s), space-separated for rotation                                          | `key1` or `key1 key2 key3` |
-| `PODSYNC_VIMEO_API_KEY`      | Vimeo API key(s), space-separated for rotation                                            | `key1` or `key1 key2`        |
-| `PODSYNC_SOUNDCLOUD_API_KEY` | SoundCloud API key(s), space-separated for rotation                                       | `soundcloud_key1 soundcloud_key2`             |
-| `PODSYNC_TWITCH_API_KEY`     | Twitch API credentials in the format `CLIENT_ID:CLIENT_SECRET`, space-separated for multi | `id1:secret1 id2:secret2`                     |
+| Variable Name                   | Description                                                                                | Example Value(s)                              |
+|--------------------------------|--------------------------------------------------------------------------------------------|-----------------------------------------------|
+| `PODSYNC_CONFIG_PATH`          | Path to the configuration file (overrides `--config` CLI flag)                             | `/app/config.toml`                            |
+| `PODSYNC_YOUTUBE_API_KEY`      | YouTube API key(s), space-separated for rotation                                           | `key1` or `key1 key2 key3`                    |
+| `PODSYNC_VIMEO_API_KEY`        | Vimeo API key(s), space-separated for rotation                                             | `key1` or `key1 key2`                         |
+| `PODSYNC_SOUNDCLOUD_CLIENT_ID` | SoundCloud client_id override (optional). If unset, Podsync auto-scrapes a working client_id | `client_id1 client_id2`                     |
+| `PODSYNC_SOUNDCLOUD_API_KEY`   | (Deprecated) Alias for `PODSYNC_SOUNDCLOUD_CLIENT_ID` for backward compatibility           | `client_id1 client_id2`                       |
+| `PODSYNC_TWITCH_API_KEY`       | Twitch API credentials in the format `CLIENT_ID:CLIENT_SECRET`, space-separated for multi | `id1:secret1 id2:secret2`                     |
 
 ## 🚀 How to run
 
@@ -139,12 +151,7 @@ Use the editor [Visual Studio Code](https://code.visualstudio.com/) and install
 
 ```
 $ docker pull ghcr.io/mxpv/podsync:latest
-$ docker run \
-    -p 8080:8080 \
-    -v $(pwd)/data:/app/data/ \
-    -v $(pwd)/db:/app/db/ \
-    -v $(pwd)/config.toml:/app/config.toml \
-    ghcr.io/mxpv/podsync:latest
+$ docker run     -p 8080:8080     -v $(pwd)/data:/app/data/     -v $(pwd)/db:/app/db/     -v $(pwd)/config.toml:/app/config.toml     ghcr.io/mxpv/podsync:latest
 ```
 
 ### 🐳 Run via Docker Compose:

cmd/podsync/config.go

@@ -195,13 +195,23 @@ func (c *Config) applyEnv() {
 	envVars := map[model.Provider]string{
 		model.ProviderYoutube:    "PODSYNC_YOUTUBE_API_KEY",
 		model.ProviderVimeo:      "PODSYNC_VIMEO_API_KEY",
-		model.ProviderSoundcloud: "PODSYNC_SOUNDCLOUD_API_KEY",
+		model.ProviderSoundcloud: "PODSYNC_SOUNDCLOUD_CLIENT_ID",
 		model.ProviderTwitch:     "PODSYNC_TWITCH_API_KEY",
 	}
 
 	// Replace API keys from config with environment variables
 	for provider, envVar := range envVars {
 		val, ok := os.LookupEnv(envVar)
+
+		// Backward compatibility for SoundCloud:
+		// Older versions used PODSYNC_SOUNDCLOUD_API_KEY, which actually held a SoundCloud client_id.
+		if !ok && provider == model.ProviderSoundcloud {
+			val, ok = os.LookupEnv("PODSYNC_SOUNDCLOUD_API_KEY")
+			if ok {
+				envVar = "PODSYNC_SOUNDCLOUD_API_KEY"
+			}
+		}
+
 		if ok {
 			log.Infof("Found %s environment variable, replacing config token with it", envVar)
 			// If no tokens are provided in the config.toml, we need to create a new map

config.toml.example

@@ -50,7 +50,8 @@ debug_endpoints = false
 # Alternatively, you can set the following environment variables:
 #   PODSYNC_YOUTUBE_API_KEY for YouTube
 #   PODSYNC_VIMEO_API_KEY for Vimeo
-#   PODSYNC_SOUNDCLOUD_API_KEY for Soundcloud
+#   PODSYNC_SOUNDCLOUD_CLIENT_ID for Soundcloud (optional client_id override)
+#   PODSYNC_SOUNDCLOUD_API_KEY for Soundcloud (deprecated alias of PODSYNC_SOUNDCLOUD_CLIENT_ID)
 #   PODSYNC_TWITCH_API_KEY for Twitch (format: CLIENT_ID:CLIENT_SECRET)
 # Environment variables support multiple keys separated by spaces for API key rotation:
 #   export PODSYNC_YOUTUBE_API_KEY="key1 key2 key3"
@@ -61,6 +62,10 @@ vimeo = [ # Multiple keys will be rotated.
   "VIMEO_API_KEY_2"
 ]
 
+# Optional. SoundCloud client_id override.
+# If unset, Podsync will auto-scrape a working client_id.
+# soundcloud = "SOUNDCLOUD_CLIENT_ID"
+
 # The list of data sources to be hosted by Podsync.
 # These are channels, users, playlists, etc.
 [feeds]

pkg/builder/builder.go

@@ -20,7 +20,8 @@ func New(ctx context.Context, provider model.Provider, key string, downloader Do
 	case model.ProviderVimeo:
 		return NewVimeoBuilder(ctx, key)
 	case model.ProviderSoundcloud:
-		return NewSoundcloudBuilder()
+		// key is optional for SoundCloud. If empty, the SoundCloud client will scrape a valid client_id.
+		return NewSoundcloudBuilder(key)
 	case model.ProviderTwitch:
 		return NewTwitchBuilder(key)
 	default:

pkg/builder/soundcloud.go

@@ -2,6 +2,9 @@ package builder
 
 import (
 	"context"
+	"encoding/json"
+	"net/http"
+	"net/url"
 	"strconv"
 	"time"
 
@@ -13,10 +16,19 @@ import (
 )
 
 type SoundCloudBuilder struct {
-	client *soundcloudapi.API
+	client     *soundcloudapi.API
+	httpClient *http.Client
 }
 
-func (s *SoundCloudBuilder) Build(_ctx context.Context, cfg *feed.Config) (*model.Feed, error) {
+// Build implements Builder for SoundCloud.
+//
+// Supported URL formats (see url.go parsing):
+//   - Playlist: https://soundcloud.com/<user>/sets/<playlist>
+//   - User:     https://soundcloud.com/<user>  (or /tracks)
+//
+// Podsync’s downloader uses Episode.VideoURL; for SoundCloud we can safely set this
+// to the public track permalink URL and let yt-dlp resolve the actual audio.
+func (s *SoundCloudBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) {
 	info, err := ParseURL(cfg.URL)
 	if err != nil {
 		return nil, err
@@ -30,67 +42,170 @@ func (s *SoundCloudBuilder) Build(_ctx context.Context, cfg *feed.Config) (*mode
 		Quality:   cfg.Quality,
 		PageSize:  cfg.PageSize,
 		UpdatedAt: time.Now().UTC(),
+		ItemURL:   cfg.URL,
 	}
 
-	if info.LinkType == model.TypePlaylist {
-		if soundcloudapi.IsPlaylistURL(cfg.URL) {
-			scplaylist, err := s.client.GetPlaylistInfo(cfg.URL)
-			if err != nil {
-				return nil, err
-			}
-
-			_feed.Title = scplaylist.Title
-			_feed.Description = scplaylist.Description
-			_feed.ItemURL = cfg.URL
-
-			date, err := time.Parse(time.RFC3339, scplaylist.CreatedAt)
-			if err == nil {
-				_feed.PubDate = date
-			}
-			_feed.Author = scplaylist.User.Username
-			_feed.CoverArt = scplaylist.ArtworkURL
-
-			var added = 0
-			for _, track := range scplaylist.Tracks {
-				pubDate, _ := time.Parse(time.RFC3339, track.CreatedAt)
-				var (
-					videoID   = strconv.FormatInt(track.ID, 10)
-					duration  = track.DurationMS / 1000
-					mediaURL  = track.PermalinkURL
-					trackSize = track.DurationMS * 15 // very rough estimate
-				)
-
-				_feed.Episodes = append(_feed.Episodes, &model.Episode{
-					ID:          videoID,
-					Title:       track.Title,
-					Description: track.Description,
-					Duration:    duration,
-					Size:        trackSize,
-					VideoURL:    mediaURL,
-					PubDate:     pubDate,
-					Thumbnail:   track.ArtworkURL,
-					Status:      model.EpisodeNew,
-				})
-
-				added++
-
-				if added >= _feed.PageSize {
-					return _feed, nil
-				}
-			}
-
-			return _feed, nil
+	switch info.LinkType {
+	case model.TypePlaylist:
+		return s.buildPlaylist(ctx, cfg, _feed)
+	case model.TypeUser:
+		return s.buildUser(ctx, cfg, _feed)
+	default:
+		return nil, errors.New("unsupported soundcloud feed type")
+	}
+}
+
+func (s *SoundCloudBuilder) buildPlaylist(_ctx context.Context, cfg *feed.Config, _feed *model.Feed) (*model.Feed, error) {
+	if !soundcloudapi.IsPlaylistURL(cfg.URL) {
+		return nil, errors.New("invalid soundcloud playlist url")
+	}
+
+	scplaylist, err := s.client.GetPlaylistInfo(cfg.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	_feed.Title = scplaylist.Title
+	_feed.Description = scplaylist.Description
+	_feed.Author = scplaylist.User.Username
+	_feed.CoverArt = scplaylist.ArtworkURL
+
+	if date, err := time.Parse(time.RFC3339, scplaylist.CreatedAt); err == nil {
+		_feed.PubDate = date
+	}
+
+	added := 0
+	for _, track := range scplaylist.Tracks {
+		_feed.Episodes = append(_feed.Episodes, trackToEpisode(track))
+		added++
+
+		// PageSize <= 0 means "no limit"
+		if _feed.PageSize > 0 && added >= _feed.PageSize {
+			break
 		}
 	}
 
-	return nil, errors.New(("unsupported soundcloud feed type"))
+	return _feed, nil
 }
 
-func NewSoundcloudBuilder() (*SoundCloudBuilder, error) {
+func (s *SoundCloudBuilder) buildUser(ctx context.Context, cfg *feed.Config, _feed *model.Feed) (*model.Feed, error) {
+	// Resolve profile URL to numeric user ID.
+	user, err := s.client.GetUser(soundcloudapi.GetUserOptions{
+		ProfileURL: cfg.URL,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to resolve soundcloud user profile")
+	}
+
+	_feed.Title = user.Username
+	_feed.Author = user.Username
+	_feed.Description = user.Description
+	_feed.CoverArt = user.AvatarURL
+
+	limit := cfg.PageSize
+	if limit <= 0 {
+		// Keep a sane default; the feed can still be "unlimited" by setting PageSize high.
+		limit = 20
+	}
+
+	tracks, err := s.fetchUserTracks(ctx, user.ID, limit)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, track := range tracks {
+		_feed.Episodes = append(_feed.Episodes, trackToEpisode(track))
+	}
+
+	return _feed, nil
+}
+
+func trackToEpisode(track soundcloudapi.Track) *model.Episode {
+	pubDate, _ := time.Parse(time.RFC3339, track.CreatedAt)
+
+	videoID := strconv.FormatInt(track.ID, 10)
+	duration := track.DurationMS / 1000
+	mediaURL := track.PermalinkURL
+	trackSize := track.DurationMS * 15 // very rough estimate
+
+	return &model.Episode{
+		ID:          videoID,
+		Title:       track.Title,
+		Description: track.Description,
+		Duration:    duration,
+		Size:        trackSize,
+		VideoURL:    mediaURL,
+		PubDate:     pubDate,
+		Thumbnail:   track.ArtworkURL,
+		Status:      model.EpisodeNew,
+	}
+}
+
+// fetchUserTracks fetches the most recent public uploads for a SoundCloud user via api-v2.
+//
+// We keep this call isolated because SoundCloud’s private API is subject to change.
+// The soundcloud-api library is used for client_id scraping and resolving profile URL -> user ID.
+func (s *SoundCloudBuilder) fetchUserTracks(ctx context.Context, userID int64, limit int) ([]soundcloudapi.Track, error) {
+	clientID := s.client.ClientID()
+	if clientID == "" {
+		cid, err := soundcloudapi.FetchClientID()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to fetch soundcloud client_id")
+		}
+		s.client.SetClientID(cid)
+		clientID = cid
+	}
+
+	endpoint, err := url.Parse("https://api-v2.soundcloud.com/users/" + strconv.FormatInt(userID, 10) + "/tracks")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build soundcloud api url")
+	}
+
+	q := endpoint.Query()
+	q.Set("client_id", clientID)
+	q.Set("limit", strconv.Itoa(limit))
+	endpoint.RawQuery = q.Encode()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to build soundcloud api request")
+	}
+	req.Header.Set("Accept", "application/json")
+
+	resp, err := s.httpClient.Do(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "soundcloud user tracks request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, errors.Errorf("soundcloud user tracks request returned http %d", resp.StatusCode)
+	}
+
+	var tracks []soundcloudapi.Track
+	if err := json.NewDecoder(resp.Body).Decode(&tracks); err != nil {
+		return nil, errors.Wrap(err, "failed to decode soundcloud user tracks response")
+	}
+
+	return tracks, nil
+}
+
+// NewSoundcloudBuilder creates a SoundCloud builder.
+//
+// The key parameter is optional and is interpreted as a SoundCloud client_id override.
+// If empty, the underlying library will still be able to scrape a client_id when needed.
+func NewSoundcloudBuilder(key string) (*SoundCloudBuilder, error) {
 	sc, err := soundcloudapi.New(soundcloudapi.APIOptions{})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create soundcloud client")
 	}
 
-	return &SoundCloudBuilder{client: sc}, nil
+	if key != "" {
+		sc.SetClientID(key)
+	}
+
+	return &SoundCloudBuilder{
+		client:     sc,
+		httpClient: &http.Client{Timeout: 20 * time.Second},
+	}, nil
 }