mpris-receiver: Send album art

Implementation of sending album art from phone to PC.

Complementary MR for the PC-side: https://invent.kde.org/network/kdeconnect-kde/-/merge_requests/541

Fixes https://bugs.kde.org/show_bug.cgi?id=422136

src/org/kde/kdeconnect/Plugins/MprisPlugin/AlbumArtCache.kt

@@ -9,6 +9,7 @@ import android.content.Context
 import android.graphics.Bitmap
 import android.graphics.BitmapFactory
 import android.net.ConnectivityManager
+import android.net.Uri
 import android.util.Log
 import androidx.collection.LruCache
 import androidx.core.content.getSystemService
@@ -24,7 +25,6 @@ import java.io.File
 import java.io.IOException
 import java.io.InputStream
 import java.net.HttpURLConnection
-import java.net.MalformedURLException
 import java.net.URL
 import java.net.URLDecoder
 import java.security.MessageDigest
@@ -53,12 +53,12 @@ internal object AlbumArtCache {
     /**
      * A list of urls yet to be fetched.
      */
-    private val fetchUrlList = ArrayList<URL>()
+    private val fetchUrlList = ArrayList<Uri>()
 
     /**
      * A list of urls currently being fetched
      */
-    private val isFetchingList = ArrayList<URL>()
+    private val isFetchingList = ArrayList<Uri>()
 
     /**
      * A integer indicating how many fetches are in progress.
@@ -70,6 +70,15 @@ internal object AlbumArtCache {
      */
     private val registeredPlugins = CopyOnWriteArrayList<MprisPlugin>()
 
+    @JvmStatic
+    val ALLOWED_SCHEMES = listOf("http", "https", "file", "kdeconnect")
+
+    /**
+     * A list of art url schemes that require a fetch from remote side.
+     */
+    @JvmStatic
+    private val REMOTE_FETCH_SCHEMES = listOf("file", "kdeconnect")
+
     /**
      * Initializes the disk cache. Needs to be called at least once before trying to use the cache
      *
@@ -121,16 +130,10 @@ internal object AlbumArtCache {
         if (albumUrl.isNullOrEmpty()) {
             return null
         }
-        val url = try {
+        val url = Uri.parse(albumUrl)
-            URL(albumUrl)
-        } catch (e: MalformedURLException) {
-            //Invalid url, so just return "no album art"
-            //Shouldn't happen (checked on receival of the url), but just to be sure
-            return null
-        }
 
-        //We currently only support http(s) and file urls
+        //We currently only support http(s), file, and kdeconnect urls
-        if (url.protocol !in arrayOf("http", "https", "file")) {
+        if (url.scheme !in ALLOWED_SCHEMES) {
             return null
         }
 
@@ -175,8 +178,8 @@ internal object AlbumArtCache {
 
         /* If not found, we have not tried fetching it (recently), or a fetch is in-progress.
            Either way, just add it to the fetch queue and starting fetching it if no fetch is running. */
-        if ("file" == url.protocol) {
+        if (url.scheme in REMOTE_FETCH_SCHEMES) {
-            //Special-case file, since we need to fetch it from the remote
+            //Special-case file or kdeconnect, since we need to fetch it from the remote
             if (url in isFetchingList) return null
             if (!plugin.askTransferAlbumArt(albumUrl, player)) {
                 //It doesn't support transferring the art, so mark it as failed in the memory cache
@@ -193,7 +196,7 @@ internal object AlbumArtCache {
      *
      * @param url The url
      */
-    private fun fetchUrl(url: URL) {
+    private fun fetchUrl(url: Uri) {
         //We need the disk cache for this
         if (!this::diskCache.isInitialized) {
             Log.e("KDE/Mpris/AlbumArtCache", "The disk cache is not intialized!")
@@ -218,7 +221,7 @@ internal object AlbumArtCache {
      * Does the actual fetching and makes sure only not too many fetches are running at the same time
      */
     private fun initiateFetch() {
-        var url : URL;
+        var url : Uri;
         synchronized(fetchUrlList) {
             if (numFetching >= 2 || fetchUrlList.isEmpty()) return
             //Fetch the last-requested url first, it will probably be needed first
@@ -226,8 +229,8 @@ internal object AlbumArtCache {
             //Remove the url from the to-fetch list
             fetchUrlList.remove(url)
         }
-        if ("file" == url.protocol) {
+        if (url.scheme in REMOTE_FETCH_SCHEMES) {
-            throw AssertionError("Not file urls should be possible here!")
+            throw AssertionError("Only http(s) urls should be possible here!")
         }
 
         //Download the album art ourselves
@@ -266,7 +269,7 @@ internal object AlbumArtCache {
     /**
      * Transfer an asked-for album art payload to the disk cache.
      *
-     * @param albumUrl The url of the album art (should be a file:// url)
+     * @param albumUrl The url of the album art (must be one of the [REMOTE_FETCH_SCHEMES])
      * @param payload  The payload input stream
      */
     @JvmStatic
@@ -280,15 +283,10 @@ internal object AlbumArtCache {
             payload.close()
             return
         }
-        val url = try {
+        val url = Uri.parse(albumUrl)
-            URL(albumUrl)
+        if (url.scheme !in REMOTE_FETCH_SCHEMES) {
-        } catch (e: MalformedURLException) {
             //Shouldn't happen (checked on receival of the url), but just to be sure
-            payload.close()
+            Log.e("KDE/Mpris/AlbumArtCache", "Got invalid art url with payload: $albumUrl")
-            return
-        }
-        if ("file" != url.protocol) {
-            //Shouldn't happen (otherwise we wouldn't have asked for the payload), but just to be sure
             payload.close()
             return
         }
@@ -341,7 +339,7 @@ internal object AlbumArtCache {
      * @param payload      A NetworkPacket Payload (if from the connected device). null if fetched from http(s)
      * @param cacheItem    The disk cache item to edit
      */
-    private suspend fun fetchURL(url: URL, payload: Payload?, cacheItem: DiskLruCache.Editor) {
+    private suspend fun fetchURL(url: Uri, payload: Payload?, cacheItem: DiskLruCache.Editor) {
         var success = withContext(Dispatchers.IO) {
             //See if we need to open a http(s) connection here, or if we use a payload input stream
             val output = cacheItem.newOutputStream(0)
@@ -405,12 +403,13 @@ internal object AlbumArtCache {
      *
      * @return True if succeeded
      */
-    private fun openHttp(url: URL): InputStream? {
+    private fun openHttp(url: Uri): InputStream? {
         //Default android behaviour does not follow https -> http urls, so do this manually
-        if (url.protocol !in arrayOf("http", "https")) {
+        if (url.scheme !in arrayOf("http", "https")) {
             throw AssertionError("Invalid url: not http(s) in background album art fetch")
         }
-        var currentUrl = url
+        // TODO: Should use contentResolver from android instead of opening our own connection
+        var currentUrl = URL(url.toString())
         var connection: HttpURLConnection
         loop@ for (i in 0..4) {
             connection = currentUrl.openConnection() as HttpURLConnection

src/org/kde/kdeconnect/Plugins/MprisPlugin/MprisPlugin.kt

@@ -32,7 +32,6 @@ import org.kde.kdeconnect.Plugins.PluginFactory.LoadablePlugin
 import org.kde.kdeconnect.UserInterface.PluginSettingsFragment
 import org.kde.kdeconnect_tp.R
 import java.net.MalformedURLException
-import java.net.URL
 import java.util.concurrent.ConcurrentHashMap
 
 @LoadablePlugin
@@ -278,11 +277,10 @@ class MprisPlugin : Plugin() {
                 playerStatus.isGoPreviousAllowed = np.getBoolean("canGoPrevious", playerStatus.isGoPreviousAllowed)
                 playerStatus.seekAllowed = np.getBoolean("canSeek", playerStatus.seekAllowed)
                 val newAlbumArtUrlString = np.getString("albumArtUrl", playerStatus.albumArtUrl)
-                try {
+                val newAlbumArtUrl = Uri.parse(newAlbumArtUrlString)
-                    // Turn the url into canonical form (and check its validity)
+                if (newAlbumArtUrl.scheme in AlbumArtCache.ALLOWED_SCHEMES) {
-                    val newAlbumArtUrl = URL(newAlbumArtUrlString)
                     playerStatus.albumArtUrl = newAlbumArtUrl.toString()
-                } catch (ignored: MalformedURLException) {
+                } else {
                     Log.w("MprisControl", "Invalid album art URL: $newAlbumArtUrlString")
                     playerStatus.albumArtUrl = ""
                 }

src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverCallback.java

@@ -6,26 +6,135 @@
 
 package org.kde.kdeconnect.Plugins.MprisReceiverPlugin;
 
+import android.graphics.Bitmap;
 import android.media.MediaMetadata;
 import android.media.session.MediaController;
 import android.media.session.PlaybackState;
+import android.net.Uri;
 import android.os.Build;
+import android.util.Pair;
 
 import androidx.annotation.Nullable;
 import androidx.annotation.RequiresApi;
 
+import java.io.ByteArrayOutputStream;
+import java.util.Arrays;
+import java.util.Objects;
 
 @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP_MR1)
 class MprisReceiverCallback extends MediaController.Callback {
-
-    private static final String TAG = "MprisReceiver";
-
     private final MprisReceiverPlayer player;
     private final MprisReceiverPlugin plugin;
 
+    private Long artHash = null;
+    private Bitmap displayArt = null;
+    private String artUrl = null;
+    private String album = null;
+    private String artist = null;
+
+    private static final String[] PREFERRED_BITMAP_ORDER = {
+            MediaMetadata.METADATA_KEY_DISPLAY_ICON,
+            MediaMetadata.METADATA_KEY_ART,
+            MediaMetadata.METADATA_KEY_ALBUM_ART
+    };
+
+    private static final String[] PREFERRED_URI_ORDER = {
+            MediaMetadata.METADATA_KEY_DISPLAY_ICON_URI,
+            MediaMetadata.METADATA_KEY_ART_URI,
+            MediaMetadata.METADATA_KEY_ALBUM_ART_URI,
+            // Fall back to album name if none of the above is set
+            MediaMetadata.METADATA_KEY_ALBUM,
+            // Youtube doesn't normally provide album info
+            MediaMetadata.METADATA_KEY_TITLE,
+            // Last option, use artist
+            MediaMetadata.METADATA_KEY_ALBUM_ARTIST,
+            MediaMetadata.METADATA_KEY_ARTIST,
+    };
+
+    static String encodeAsUri(String kind, String data) {
+        // there's probably a better way to do this, but meh
+        // TODO: do we want to include the player name?
+        return new Uri.Builder()
+                .scheme("kdeconnect")
+                .path("/artUri")
+                .appendQueryParameter(kind, data)
+                .build().toString();
+    }
+
+    /**
+     * Extract the art bitmap and corresponding uri from the media metadata.
+     *
+     * @return Pair of art,artUrl. May be null if either was not found.
+     */
+    static Pair<Bitmap, String> getArtAndUri(MediaMetadata metadata) {
+        if (metadata == null) return null;
+        String uri = null;
+        Bitmap art = null;
+        for (String s : PREFERRED_BITMAP_ORDER) {
+            Bitmap next = metadata.getBitmap(s);
+            if (next != null) {
+                art = next;
+                break;
+            }
+        }
+        for (String s : PREFERRED_URI_ORDER) {
+            String next = metadata.getString(s);
+            if (next != null && !next.isEmpty()) {
+                String kind;
+                switch (s) {
+                    case MediaMetadata.METADATA_KEY_ALBUM:
+                        kind = "album";
+                        break;
+                    case MediaMetadata.METADATA_KEY_TITLE:
+                        kind = "title";
+                        break;
+                    case MediaMetadata.METADATA_KEY_ARTIST:
+                    case MediaMetadata.METADATA_KEY_ALBUM_ARTIST:
+                        kind = "artist";
+                        break;
+                    default:
+                        kind = "orig";
+                        break;
+                }
+                uri = encodeAsUri(kind, next);
+                break;
+            }
+        }
+
+        if (art == null || uri == null) return null;
+        return new Pair<>(art, uri);
+    }
+
+    private static long hashBitmap(Bitmap bitmap) {
+        int[] buffer = new int[bitmap.getWidth() * bitmap.getHeight()];
+        bitmap.getPixels(buffer, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(), bitmap.getHeight());
+        return Arrays.hashCode(buffer);
+    }
+
+    private String makeArtUrl(long artHash, String artUrl) {
+        // we include the hash in the URL to handle the case when the player changes the bitmap
+        // without changing the url- the PC side won't know the art was modified if we don't do this
+        // also useful when the input url contains only the artist name (eg: Youtube)
+        return Uri.parse(artUrl)
+                .buildUpon()
+                .appendQueryParameter("kdeArtHash", String.valueOf(artHash))
+                .build()
+                .toString();
+    }
+
     MprisReceiverCallback(MprisReceiverPlugin plugin, MprisReceiverPlayer player) {
         this.player = player;
         this.plugin = plugin;
+        // fetch the initial art, when player is already running and we start kdeconnect
+        Pair<Bitmap, String> artAndUri = getArtAndUri(player.getMetadata());
+        if (artAndUri != null) {
+            Bitmap bitmap = artAndUri.first;
+            artHash = hashBitmap(bitmap);
+            artUrl = makeArtUrl(artHash, artAndUri.second);
+            displayArt = bitmap;
+            album = player.getAlbum();
+            artist = player.getArtist();
+        }
     }
 
     @Override
@@ -35,6 +144,43 @@ class MprisReceiverCallback extends MediaController.Callback {
 
     @Override
     public void onMetadataChanged(@Nullable MediaMetadata metadata) {
+        if (metadata == null) {
+            artHash = null;
+            displayArt = null;
+            artUrl = null;
+            artist = null;
+            album = null;
+        } else {
+            // We could check hasRequestedAlbumArt to avoid hashing art for clients that don't support it
+            //  But upon running the profiler, looks like hashBitmap is a minuscule (<1%) part so no
+            //  need to optimize prematurely.
+            Pair<Bitmap, String> artAndUri = getArtAndUri(metadata);
+            String newAlbum = metadata.getString(MediaMetadata.METADATA_KEY_ALBUM);
+            String newArtist = metadata.getString(MediaMetadata.METADATA_KEY_ARTIST);
+            if (artAndUri == null) {
+                // check if the album+artist is still the same- some players don't send art every time
+                if (!Objects.equals(newAlbum, album) || !Objects.equals(newArtist, artist)) {
+                    // there really is no new art
+                    artHash = null;
+                    displayArt = null;
+                    artUrl = null;
+                    album = null;
+                    artist = null;
+                }
+            } else {
+                Long newHash = hashBitmap(artAndUri.first);
+                // In case the hashes are equal, we do a full comparison to protect against collisions
+                if ((!newHash.equals(artHash) || !artAndUri.first.sameAs(displayArt))) {
+                    artHash = newHash;
+                    displayArt = artAndUri.first;
+                    artUrl = makeArtUrl(artHash, artAndUri.second);
+                    artist = newArtist;
+                    album = newAlbum;
+                }
+            }
+
+        }
+
         plugin.sendMetadata(player);
     }
 
@@ -43,4 +189,22 @@ class MprisReceiverCallback extends MediaController.Callback {
         //Note: not called by all media players
         plugin.sendMetadata(player);
     }
+
+    public String getArtUrl() {
+        return artUrl;
+    }
+
+    /**
+     * Get the JPG art of the current track as a bytearray.
+     *
+     * @return null if no art is available, otherwise a PNG image serialized into a bytearray
+     */
+    public byte[] getArtAsArray() {
+        if (displayArt == null) {
+            return null;
+        }
+        ByteArrayOutputStream stream = new ByteArrayOutputStream();
+        displayArt.compress(Bitmap.CompressFormat.JPEG, 90, stream);
+        return stream.toByteArray();
+    }
 }

src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverPlayer.java

@@ -163,4 +163,8 @@ class MprisReceiverPlayer {
 
         return metadata.getLong(MediaMetadata.METADATA_KEY_DURATION);
     }
+
+    MediaMetadata getMetadata() {
+        return controller.getMetadata();
+    }
 }

src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverPlugin.java

@@ -23,6 +23,7 @@ import androidx.fragment.app.DialogFragment;
 
 import org.apache.commons.lang3.StringUtils;
 import org.kde.kdeconnect.Helpers.AppsHelper;
+import org.kde.kdeconnect.Helpers.ThreadHelper;
 import org.kde.kdeconnect.NetworkPacket;
 import org.kde.kdeconnect.Plugins.NotificationsPlugin.NotificationReceiver;
 import org.kde.kdeconnect.Plugins.Plugin;
@@ -49,12 +50,15 @@ public class MprisReceiverPlugin extends Plugin {
     private HashMap<String, MprisReceiverCallback> playerCbs;
     private MediaSessionChangeListener mediaSessionChangeListener;
 
+    public @NonNull String getDeviceId() {
+        return device.getDeviceId();
+    }
+
     @Override
     public boolean onCreate() {
 
         if (!hasPermission())
             return false;
-
         players = new HashMap<>();
         playerCbs = new HashMap<>();
         try {
@@ -103,7 +107,6 @@ public class MprisReceiverPlugin extends Plugin {
 
     @Override
     public boolean onPacketReceived(@NonNull NetworkPacket np) {
-
         if (np.getBoolean("requestPlayerList")) {
             sendPlayerList();
             return true;
@@ -117,6 +120,18 @@ public class MprisReceiverPlugin extends Plugin {
         if (null == player) {
             return false;
         }
+        String artUrl = np.getString("albumArtUrl", "");
+        if (!artUrl.isEmpty()) {
+            String playerName = player.getName();
+            MprisReceiverCallback cb = playerCbs.get(playerName);
+            if (cb == null) {
+                Log.e(TAG, "no callback for " + playerName + " (player likely stopped)");
+                return false;
+            }
+            // run it on a different thread to avoid blocking
+            ThreadHelper.execute(() -> sendAlbumArt(playerName, cb, artUrl));
+            return true;
+        }
 
         if (np.getBoolean("requestNowPlaying", false)) {
             sendMetadata(player);
@@ -181,7 +196,7 @@ public class MprisReceiverPlugin extends Plugin {
                 return;
             }
             for (MprisReceiverPlayer p : players.values()) {
-                p.getController().unregisterCallback(playerCbs.get(p.getName()));
+                p.getController().unregisterCallback(Objects.requireNonNull(playerCbs.get(p.getName())));
             }
             playerCbs.clear();
             players.clear();
@@ -206,6 +221,7 @@ public class MprisReceiverPlugin extends Plugin {
     private void sendPlayerList() {
         NetworkPacket np = new NetworkPacket(PACKET_TYPE_MPRIS);
         np.set("playerList", players.keySet());
+        np.set("supportAlbumArtPayload", true);
         getDevice().sendPacket(np);
     }
 
@@ -214,6 +230,37 @@ public class MprisReceiverPlugin extends Plugin {
         return Build.VERSION_CODES.LOLLIPOP_MR1;
     }
 
+    void sendAlbumArt(String playerName, @NonNull MprisReceiverCallback cb, @Nullable String requestedUrl) {
+        // NOTE: It is possible that the player gets killed in the middle of this method.
+        // The proper thing to do this case would be to abort the send - but that gets into the
+        //   territory of async cancellation or putting a lock.
+        // For now, we just continue to send the art- cb stores the bitmap, so it will be valid.
+        //   cb will get GC'd after this method completes.
+        String localArtUrl = cb.getArtUrl();
+        if (localArtUrl == null) {
+            Log.w(TAG, "art not found!");
+            return;
+        }
+        String artUrl = requestedUrl == null ? localArtUrl : requestedUrl;
+        if (requestedUrl != null && !requestedUrl.contentEquals(localArtUrl)) {
+            Log.w(TAG, "sendAlbumArt: Doesn't match current url");
+            Log.d(TAG, "current:   " + localArtUrl);
+            Log.d(TAG, "requested: " + requestedUrl);
+            return;
+        }
+        byte[] p = cb.getArtAsArray();
+        if (p == null) {
+            Log.w(TAG, "sendAlbumArt: Failed to get art stream");
+            return;
+        }
+        NetworkPacket np = new NetworkPacket(PACKET_TYPE_MPRIS);
+        np.setPayload(new NetworkPacket.Payload(p));
+        np.set("player", playerName);
+        np.set("transferringAlbumArt", true);
+        np.set("albumArtUrl", artUrl);
+        getDevice().sendPacket(np);
+    }
+
     void sendMetadata(MprisReceiverPlayer player) {
         NetworkPacket np = new NetworkPacket(MprisReceiverPlugin.PACKET_TYPE_MPRIS);
         np.set("player", player.getName());
@@ -232,6 +279,15 @@ public class MprisReceiverPlugin extends Plugin {
         np.set("canGoNext", player.canGoNext());
         np.set("canSeek", player.canSeek());
         np.set("volume", player.getVolume());
+        MprisReceiverCallback cb = playerCbs.get(player.getName());
+        assert cb != null;
+        String artUrl = cb.getArtUrl();
+        if (artUrl != null) {
+            np.set("albumArtUrl", artUrl);
+            Log.v(TAG, "Sending metadata with url " + artUrl);
+        } else {
+            Log.v(TAG, "Sending metadata without url ");
+        }
         getDevice().sendPacket(np);
     }