Spotify rate limit improvements

components/spotify/actions/add-items-to-playlist/add-items-to-playlist.mjs

@@ -4,7 +4,7 @@ export default {
   name: "Add Items to a Playlist",
   description: "Add one or more items to a user’s playlist. [See the docs here](https://developer.spotify.com/documentation/web-api/reference/#/operations/add-tracks-to-playlist).",
   key: "spotify-add-items-to-playlist",
-  version: "0.1.6",
+  version: "0.1.7",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,

components/spotify/actions/create-playlist/create-playlist.mjs

@@ -4,7 +4,7 @@ export default {
   name: "Create a Playlist",
   description: "Create a playlist for a Spotify user. The playlist will be empty until you add tracks. [See the docs here](https://developer.spotify.com/documentation/web-api/reference/#/operations/create-playlist).",
   key: "spotify-create-playlist",
-  version: "0.1.6",
+  version: "0.1.7",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,

components/spotify/actions/get-album-tracks/get-album-tracks.mjs

@@ -8,7 +8,7 @@ export default {
   name: "Get Album Tracks",
   description: "Get all tracks in an album. [See the docs here](https://developer.spotify.com/documentation/web-api/reference/get-an-albums-tracks)",
   key: "spotify-get-album-tracks",
-  version: "0.0.7",
+  version: "0.0.8",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,

components/spotify/actions/get-all-tracks-by-artist/get-all-tracks-by-artist.mjs

@@ -4,7 +4,7 @@ export default {
   name: "Get All Tracks by Artist",
   description: "Get Spotify tracks information related with an artist's. [see docs here](https://developer.spotify.com/documentation/web-api/reference/#/operations/get-multiple-albums).",
   key: "spotify-get-all-tracks-by-artist",
-  version: "0.1.6",
+  version: "0.1.7",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,
@@ -32,15 +32,15 @@ export default {
       market,
     } = this;
 
-    const chunksOfAlbumIds = await this.spotify.fetchChunksOfAlbumsIds({
+    const albums = await this.spotify.getArtistAlbums({
       $,
       artistId,
       market,
     });
 
-    const tracks = await this.spotify.getAllTracksByChunksOfAlbumIds({
+    const tracks = await this.spotify.getAllTracksByAlbumIds({
       $,
-      chunksOfAlbumIds,
+      albumIds: albums.map((album) => album.id),
       market,
     });
 

components/spotify/actions/get-artist-top-tracks/get-artist-top-tracks.mjs

@@ -4,7 +4,7 @@ export default {
   name: "Get an Artist's Top Tracks",
   description: "Get Spotify catalog information about an artist's top tracks by country. [See the docs here](https://developer.spotify.com/documentation/web-api/reference/#/operations/get-an-artists-top-tracks).",
   key: "spotify-get-artist-top-tracks",
-  version: "0.1.6",
+  version: "0.1.7",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,

components/spotify/actions/get-audio-features-for-a-track/get-audio-features-for-a-track.mjs

@@ -4,7 +4,7 @@ export default {
   name: "Get Audio Features for a Track",
   description: "Get audio feature information for a single track identified by its unique Spotify ID. [See the docs here](https://developer.spotify.com/documentation/web-api/reference/#/operations/get-audio-features).",
   key: "spotify-get-audio-features-for-a-track",
-  version: "0.1.6",
+  version: "0.1.7",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,

components/spotify/actions/get-categorys-playlist/get-categorys-playlist.mjs

@@ -4,7 +4,7 @@ export default {
   name: "Get a Category's Playlists",
   description: "Get a list of Spotify playlists tagged with a particular category. [See the docs here](https://developer.spotify.com/documentation/web-api/reference/#/operations/get-a-categories-playlists).",
   key: "spotify-get-categorys-playlist",
-  version: "0.1.6",
+  version: "0.1.7",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,

components/spotify/actions/get-currently-playing-track/get-currently-playing-track.mjs

@@ -6,7 +6,7 @@ export default {
   description:
     "Get the object currently being played on the user's Spotify account. [See the documentation](https://developer.spotify.com/documentation/web-api/reference/get-the-users-currently-playing-track)",
   key: "spotify-get-currently-playing-track",
-  version: "0.0.7",
+  version: "0.0.8",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,

components/spotify/actions/get-playlist-items/get-playlist-items.mjs

@@ -4,7 +4,7 @@ export default {
   name: "Get a Playlist's Items",
   description: "Get full details of the items of a playlist owned by a Spotify user. [See the docs here](https://developer.spotify.com/documentation/web-api/reference/#/operations/get-playlists-tracks).",
   key: "spotify-get-playlist-items",
-  version: "0.1.6",
+  version: "0.1.7",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,

components/spotify/actions/get-playlist/get-playlist.mjs

@@ -4,7 +4,7 @@ export default {
   name: "Get a Playlist",
   description: "Get a playlist owned by a Spotify user. [See the documentation](https://developer.spotify.com/documentation/web-api/reference/get-playlist).",
   key: "spotify-get-playlist",
-  version: "0.0.7",
+  version: "0.0.8",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,

components/spotify/actions/get-recommendations/get-recommendations.mjs

@@ -5,7 +5,7 @@ export default {
   name: "Get Recommendations",
   description: "Create a list of recommendations based on the available information for a given seed entity and matched against similar artists and tracks. If there is sufficient information about the provided seeds, a list of tracks will be returned together with pool size details. [See the docs here](https://developer.spotify.com/documentation/web-api/reference/#/operations/get-recommendations).",
   key: "spotify-get-recommendations",
-  version: "0.1.6",
+  version: "0.1.7",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,

components/spotify/actions/get-track/get-track.mjs

@@ -4,7 +4,7 @@ export default {
   name: "Get a Track",
   description: "Get a track by its name or ID. [See the docs here](https://developer.spotify.com/documentation/web-api/reference/#/operations/search)",
   key: "spotify-get-track",
-  version: "0.1.6",
+  version: "0.1.7",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,

components/spotify/actions/list-category-id-options/list-category-id-options.mjs

@@ -4,7 +4,7 @@ export default {
   key: "spotify-list-category-id-options",
   name: "List Category ID Options",
   description: "Retrieves available options for the Category ID field.",
-  version: "0.0.2",
+  version: "0.0.3",
   type: "action",
   annotations: {
     destructiveHint: false,

components/spotify/actions/list-genres-options/list-genres-options.mjs

@@ -4,7 +4,7 @@ export default {
   key: "spotify-list-genres-options",
   name: "List Seed Genres Options",
   description: "Retrieves available options for the Seed Genres field.",
-  version: "0.0.2",
+  version: "0.0.3",
   type: "action",
   annotations: {
     destructiveHint: false,

components/spotify/actions/list-playlist-id-options/list-playlist-id-options.mjs

@@ -4,7 +4,7 @@ export default {
   key: "spotify-list-playlist-id-options",
   name: "List Playlist ID Options",
   description: "Retrieves available options for the Playlist ID field.",
-  version: "0.0.2",
+  version: "0.0.3",
   type: "action",
   annotations: {
     destructiveHint: false,

components/spotify/actions/list-saved-user-tracks-id-options/list-saved-user-tracks-id-options.mjs

@@ -4,7 +4,7 @@ export default {
   key: "spotify-list-saved-user-tracks-id-options",
   name: "List Track ID Options",
   description: "Retrieves available options for the Track ID field.",
-  version: "0.0.2",
+  version: "0.0.3",
   type: "action",
   annotations: {
     destructiveHint: false,

components/spotify/actions/remove-items-from-playlist/remove-items-from-playlist.mjs

@@ -4,7 +4,7 @@ export default {
   name: "Remove Items from a Playlist",
   description: "Remove one or more items from a user's playlist. [See the docs here](https://developer.spotify.com/documentation/web-api/reference/#/operations/remove-tracks-playlist)",
   key: "spotify-remove-items-from-playlist",
-  version: "0.1.6",
+  version: "0.1.7",
   annotations: {
     destructiveHint: true,
     openWorldHint: true,

components/spotify/actions/remove-user-saved-tracks/remove-user-saved-tracks.mjs

@@ -4,7 +4,7 @@ export default {
   name: "Remove User's Saved Tracks",
   description: "Remove one or more tracks from the current user's 'Your Music' library. [See the docs here](https://developer.spotify.com/documentation/web-api/reference/#/operations/remove-tracks-user)",
   key: "spotify-remove-user-saved-tracks",
-  version: "0.1.6",
+  version: "0.1.7",
   annotations: {
     destructiveHint: true,
     openWorldHint: true,

components/spotify/actions/save-track/save-track.mjs

@@ -4,7 +4,7 @@ export default {
   name: "Save Tracks for User",
   description: "Save one or more tracks to the current user's \"Your Music\" library. [See the docs here](https://developer.spotify.com/documentation/web-api/reference/#/operations/save-tracks-user).",
   key: "spotify-save-track",
-  version: "0.1.6",
+  version: "0.1.7",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,

components/spotify/actions/search/search.mjs

@@ -10,7 +10,7 @@ export default {
   key: "spotify-search",
   name: "Search",
   description: "Search for items on Spotify (tracks, albums, artists, playlists, shows, or episodes). [See the docs here](https://developer.spotify.com/documentation/web-api/reference/search)",
-  version: "0.0.4",
+  version: "0.0.5",
   type: "action",
   annotations: {
     destructiveHint: false,

components/spotify/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/spotify",
-  "version": "0.9.1",
+  "version": "0.9.2",
   "description": "Pipedream Spotify Components",
   "main": "spotify.app.mjs",
   "keywords": [

components/spotify/sources/new-playlist/new-playlist.mjs

@@ -6,7 +6,7 @@ export default {
   key: "spotify-new-playlist",
   name: "New Playlist",
   description: "Emit new event when a new playlist is created or followed by the current Spotify user.",
-  version: "0.1.5",
+  version: "0.1.6",
   props: {
     ...common.props,
   },

components/spotify/sources/new-saved-track/new-saved-track.mjs

@@ -6,7 +6,7 @@ export default {
   key: "spotify-new-saved-track",
   name: "New Saved Track",
   description: "Emit new event for each new track saved to the current Spotify user's Music Library.",
-  version: "0.1.5",
+  version: "0.1.6",
   props: {
     ...common.props,
     db: "$.service.db",

components/spotify/sources/new-track-by-artist/new-track-by-artist.mjs

@@ -1,13 +1,19 @@
 import spotify from "../../spotify.app.mjs";
 import common from "../common.mjs";
 
+// New albums and albums whose track count changed are fetched immediately. To
+// also catch same-count edits (a track swapped/replaced, which leaves
+// `total_tracks` unchanged), every album is re-scanned once per this
+// interval (default: once per day).
+const MIN_TRACK_RECHECK_INTERVAL_MS = 24 * 60 * 60 * 1000;
+
 export default {
   dedupe: "unique",
   type: "source",
   key: "spotify-new-track-by-artist",
   name: "New Track by Artist",
-  description: "Emit new event for each new Spotify track related with an artist. [see docs here](https://developer.spotify.com/documentation/web-api/reference/#/operations/get-multiple-albums)",
-  version: "0.1.5",
+  description: "Emit new event for each new Spotify track related with an artist. [See the documentation](https://developer.spotify.com/documentation/web-api/reference/#/operations/get-multiple-albums)",
+  version: "0.1.6",
   props: {
     ...common.props,
     db: "$.service.db",
@@ -37,30 +43,72 @@ export default {
         ts: Date.now(),
       };
     },
+    _getLastCheckedAt() {
+      return this.db.get("lastCheckedAt") ?? 0;
+    },
+    _setLastCheckedAt(lastCheckedAt) {
+      this.db.set("lastCheckedAt", lastCheckedAt);
+    },
+    _getAlbumTrackCounts() {
+      return this.db.get("albumTrackCounts") ?? {};
+    },
+    _setAlbumTrackCounts(albumTrackCounts) {
+      this.db.set("albumTrackCounts", albumTrackCounts);
+    },
   },
   async run() {
     const {
       artistId,
       market,
     } = this;
 
-    const chunksOfAlbumIds = await this.spotify.fetchChunksOfAlbumsIds({
+    const now = Date.now();
+    const lastCheckedAt = this._getLastCheckedAt();
+    const trackCounts = this._getAlbumTrackCounts();
+
+    // The album listing is cheap, so fetch it every run to detect new albums and
+    // track-count changes immediately.
+    const albums = await this.spotify.getArtistAlbums({
       artistId,
       market,
     });
 
-    const tracks = await this.spotify.getAllTracksByChunksOfAlbumIds({
-      chunksOfAlbumIds,
-      market,
-    });
+    // Re-scan every album once the interval has elapsed; between sweeps, only
+    // fetch tracks for albums that are new or whose track count changed.
+    const dueForFullSweep = (now - lastCheckedAt) >= MIN_TRACK_RECHECK_INTERVAL_MS;
+    const staleAlbumIds = albums
+      .filter((album) => dueForFullSweep || trackCounts[album.id] !== album.total_tracks)
+      .map((album) => album.id);
+
+    if (staleAlbumIds.length) {
+      const tracks = await this.spotify.getAllTracksByAlbumIds({
+        albumIds: staleAlbumIds,
+        market,
+      });
 
-    for (const track of tracks) {
-      this.$emit(
-        track,
-        this.getMeta({
+      for (const track of tracks) {
+        this.$emit(
           track,
-        }),
-      );
+          this.getMeta({
+            track,
+          }),
+        );
+      }
+    }
+
+    // Rebuild from the current listing so the store stays bounded and drops
+    // albums that are no longer in the artist's catalog.
+    const updatedTrackCounts = Object.fromEntries(
+      albums.map((album) => [
+        album.id,
+        album.total_tracks,
+      ]),
+    );
+    this._setAlbumTrackCounts(updatedTrackCounts);
+
+    // Only advance the sweep timer when a full sweep actually ran.
+    if (dueForFullSweep) {
+      this._setLastCheckedAt(now);
     }
   },
 };

components/spotify/sources/new-track-in-playlist/new-track-in-playlist.mjs

@@ -7,7 +7,7 @@ export default {
   key: "spotify-new-track-in-playlist",
   name: "New Track in Playlist",
   description: "Emit new event for each new Spotify track added to a playlist",
-  version: "0.1.5",
+  version: "0.1.6",
   props: {
     ...common.props,
     db: "$.service.db",