diff --git a/src/org/kde/kdeconnect/Plugins/MprisPlugin/AlbumArtCache.kt b/src/org/kde/kdeconnect/Plugins/MprisPlugin/AlbumArtCache.kt index d8316d46..1e6602db 100644 --- a/src/org/kde/kdeconnect/Plugins/MprisPlugin/AlbumArtCache.kt +++ b/src/org/kde/kdeconnect/Plugins/MprisPlugin/AlbumArtCache.kt @@ -9,6 +9,7 @@ import android.content.Context import android.graphics.Bitmap import android.graphics.BitmapFactory import android.net.ConnectivityManager +import android.net.Uri import android.util.Log import androidx.collection.LruCache import androidx.core.content.getSystemService @@ -24,7 +25,6 @@ import java.io.File import java.io.IOException import java.io.InputStream import java.net.HttpURLConnection -import java.net.MalformedURLException import java.net.URL import java.net.URLDecoder import java.security.MessageDigest @@ -53,12 +53,12 @@ internal object AlbumArtCache { /** * A list of urls yet to be fetched. */ - private val fetchUrlList = ArrayList() + private val fetchUrlList = ArrayList() /** * A list of urls currently being fetched */ - private val isFetchingList = ArrayList() + private val isFetchingList = ArrayList() /** * A integer indicating how many fetches are in progress. @@ -70,6 +70,15 @@ internal object AlbumArtCache { */ private val registeredPlugins = CopyOnWriteArrayList() + @JvmStatic + val ALLOWED_SCHEMES = listOf("http", "https", "file", "kdeconnect") + + /** + * A list of art url schemes that require a fetch from remote side. + */ + @JvmStatic + private val REMOTE_FETCH_SCHEMES = listOf("file", "kdeconnect") + /** * Initializes the disk cache. Needs to be called at least once before trying to use the cache * @@ -121,16 +130,10 @@ internal object AlbumArtCache { if (albumUrl.isNullOrEmpty()) { return null } - val url = try { - URL(albumUrl) - } catch (e: MalformedURLException) { - //Invalid url, so just return "no album art" - //Shouldn't happen (checked on receival of the url), but just to be sure - return null - } + val url = Uri.parse(albumUrl) - //We currently only support http(s) and file urls - if (url.protocol !in arrayOf("http", "https", "file")) { + //We currently only support http(s), file, and kdeconnect urls + if (url.scheme !in ALLOWED_SCHEMES) { return null } @@ -175,8 +178,8 @@ internal object AlbumArtCache { /* If not found, we have not tried fetching it (recently), or a fetch is in-progress. Either way, just add it to the fetch queue and starting fetching it if no fetch is running. */ - if ("file" == url.protocol) { - //Special-case file, since we need to fetch it from the remote + if (url.scheme in REMOTE_FETCH_SCHEMES) { + //Special-case file or kdeconnect, since we need to fetch it from the remote if (url in isFetchingList) return null if (!plugin.askTransferAlbumArt(albumUrl, player)) { //It doesn't support transferring the art, so mark it as failed in the memory cache @@ -193,7 +196,7 @@ internal object AlbumArtCache { * * @param url The url */ - private fun fetchUrl(url: URL) { + private fun fetchUrl(url: Uri) { //We need the disk cache for this if (!this::diskCache.isInitialized) { Log.e("KDE/Mpris/AlbumArtCache", "The disk cache is not intialized!") @@ -218,7 +221,7 @@ internal object AlbumArtCache { * Does the actual fetching and makes sure only not too many fetches are running at the same time */ private fun initiateFetch() { - var url : URL; + var url : Uri; synchronized(fetchUrlList) { if (numFetching >= 2 || fetchUrlList.isEmpty()) return //Fetch the last-requested url first, it will probably be needed first @@ -226,8 +229,8 @@ internal object AlbumArtCache { //Remove the url from the to-fetch list fetchUrlList.remove(url) } - if ("file" == url.protocol) { - throw AssertionError("Not file urls should be possible here!") + if (url.scheme in REMOTE_FETCH_SCHEMES) { + throw AssertionError("Only http(s) urls should be possible here!") } //Download the album art ourselves @@ -266,7 +269,7 @@ internal object AlbumArtCache { /** * Transfer an asked-for album art payload to the disk cache. * - * @param albumUrl The url of the album art (should be a file:// url) + * @param albumUrl The url of the album art (must be one of the [REMOTE_FETCH_SCHEMES]) * @param payload The payload input stream */ @JvmStatic @@ -280,15 +283,10 @@ internal object AlbumArtCache { payload.close() return } - val url = try { - URL(albumUrl) - } catch (e: MalformedURLException) { + val url = Uri.parse(albumUrl) + if (url.scheme !in REMOTE_FETCH_SCHEMES) { //Shouldn't happen (checked on receival of the url), but just to be sure - payload.close() - return - } - if ("file" != url.protocol) { - //Shouldn't happen (otherwise we wouldn't have asked for the payload), but just to be sure + Log.e("KDE/Mpris/AlbumArtCache", "Got invalid art url with payload: $albumUrl") payload.close() return } @@ -341,7 +339,7 @@ internal object AlbumArtCache { * @param payload A NetworkPacket Payload (if from the connected device). null if fetched from http(s) * @param cacheItem The disk cache item to edit */ - private suspend fun fetchURL(url: URL, payload: Payload?, cacheItem: DiskLruCache.Editor) { + private suspend fun fetchURL(url: Uri, payload: Payload?, cacheItem: DiskLruCache.Editor) { var success = withContext(Dispatchers.IO) { //See if we need to open a http(s) connection here, or if we use a payload input stream val output = cacheItem.newOutputStream(0) @@ -405,12 +403,13 @@ internal object AlbumArtCache { * * @return True if succeeded */ - private fun openHttp(url: URL): InputStream? { + private fun openHttp(url: Uri): InputStream? { //Default android behaviour does not follow https -> http urls, so do this manually - if (url.protocol !in arrayOf("http", "https")) { + if (url.scheme !in arrayOf("http", "https")) { throw AssertionError("Invalid url: not http(s) in background album art fetch") } - var currentUrl = url + // TODO: Should use contentResolver from android instead of opening our own connection + var currentUrl = URL(url.toString()) var connection: HttpURLConnection loop@ for (i in 0..4) { connection = currentUrl.openConnection() as HttpURLConnection diff --git a/src/org/kde/kdeconnect/Plugins/MprisPlugin/MprisPlugin.kt b/src/org/kde/kdeconnect/Plugins/MprisPlugin/MprisPlugin.kt index 0587a8fb..28f19758 100644 --- a/src/org/kde/kdeconnect/Plugins/MprisPlugin/MprisPlugin.kt +++ b/src/org/kde/kdeconnect/Plugins/MprisPlugin/MprisPlugin.kt @@ -32,7 +32,6 @@ import org.kde.kdeconnect.Plugins.PluginFactory.LoadablePlugin import org.kde.kdeconnect.UserInterface.PluginSettingsFragment import org.kde.kdeconnect_tp.R import java.net.MalformedURLException -import java.net.URL import java.util.concurrent.ConcurrentHashMap @LoadablePlugin @@ -278,11 +277,10 @@ class MprisPlugin : Plugin() { playerStatus.isGoPreviousAllowed = np.getBoolean("canGoPrevious", playerStatus.isGoPreviousAllowed) playerStatus.seekAllowed = np.getBoolean("canSeek", playerStatus.seekAllowed) val newAlbumArtUrlString = np.getString("albumArtUrl", playerStatus.albumArtUrl) - try { - // Turn the url into canonical form (and check its validity) - val newAlbumArtUrl = URL(newAlbumArtUrlString) + val newAlbumArtUrl = Uri.parse(newAlbumArtUrlString) + if (newAlbumArtUrl.scheme in AlbumArtCache.ALLOWED_SCHEMES) { playerStatus.albumArtUrl = newAlbumArtUrl.toString() - } catch (ignored: MalformedURLException) { + } else { Log.w("MprisControl", "Invalid album art URL: $newAlbumArtUrlString") playerStatus.albumArtUrl = "" } diff --git a/src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverCallback.java b/src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverCallback.java index ed409ee2..e4a382de 100644 --- a/src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverCallback.java +++ b/src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverCallback.java @@ -6,26 +6,135 @@ package org.kde.kdeconnect.Plugins.MprisReceiverPlugin; +import android.graphics.Bitmap; import android.media.MediaMetadata; import android.media.session.MediaController; import android.media.session.PlaybackState; +import android.net.Uri; import android.os.Build; +import android.util.Pair; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; +import java.io.ByteArrayOutputStream; +import java.util.Arrays; +import java.util.Objects; @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP_MR1) class MprisReceiverCallback extends MediaController.Callback { - - private static final String TAG = "MprisReceiver"; - private final MprisReceiverPlayer player; private final MprisReceiverPlugin plugin; + private Long artHash = null; + private Bitmap displayArt = null; + private String artUrl = null; + private String album = null; + private String artist = null; + + private static final String[] PREFERRED_BITMAP_ORDER = { + MediaMetadata.METADATA_KEY_DISPLAY_ICON, + MediaMetadata.METADATA_KEY_ART, + MediaMetadata.METADATA_KEY_ALBUM_ART + }; + + private static final String[] PREFERRED_URI_ORDER = { + MediaMetadata.METADATA_KEY_DISPLAY_ICON_URI, + MediaMetadata.METADATA_KEY_ART_URI, + MediaMetadata.METADATA_KEY_ALBUM_ART_URI, + // Fall back to album name if none of the above is set + MediaMetadata.METADATA_KEY_ALBUM, + // Youtube doesn't normally provide album info + MediaMetadata.METADATA_KEY_TITLE, + // Last option, use artist + MediaMetadata.METADATA_KEY_ALBUM_ARTIST, + MediaMetadata.METADATA_KEY_ARTIST, + }; + + static String encodeAsUri(String kind, String data) { + // there's probably a better way to do this, but meh + // TODO: do we want to include the player name? + return new Uri.Builder() + .scheme("kdeconnect") + .path("/artUri") + .appendQueryParameter(kind, data) + .build().toString(); + } + + /** + * Extract the art bitmap and corresponding uri from the media metadata. + * + * @return Pair of art,artUrl. May be null if either was not found. + */ + static Pair getArtAndUri(MediaMetadata metadata) { + if (metadata == null) return null; + String uri = null; + Bitmap art = null; + for (String s : PREFERRED_BITMAP_ORDER) { + Bitmap next = metadata.getBitmap(s); + if (next != null) { + art = next; + break; + } + } + for (String s : PREFERRED_URI_ORDER) { + String next = metadata.getString(s); + if (next != null && !next.isEmpty()) { + String kind; + switch (s) { + case MediaMetadata.METADATA_KEY_ALBUM: + kind = "album"; + break; + case MediaMetadata.METADATA_KEY_TITLE: + kind = "title"; + break; + case MediaMetadata.METADATA_KEY_ARTIST: + case MediaMetadata.METADATA_KEY_ALBUM_ARTIST: + kind = "artist"; + break; + default: + kind = "orig"; + break; + } + uri = encodeAsUri(kind, next); + break; + } + } + + if (art == null || uri == null) return null; + return new Pair<>(art, uri); + } + + private static long hashBitmap(Bitmap bitmap) { + int[] buffer = new int[bitmap.getWidth() * bitmap.getHeight()]; + bitmap.getPixels(buffer, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(), bitmap.getHeight()); + return Arrays.hashCode(buffer); + } + + private String makeArtUrl(long artHash, String artUrl) { + // we include the hash in the URL to handle the case when the player changes the bitmap + // without changing the url- the PC side won't know the art was modified if we don't do this + // also useful when the input url contains only the artist name (eg: Youtube) + return Uri.parse(artUrl) + .buildUpon() + .appendQueryParameter("kdeArtHash", String.valueOf(artHash)) + .build() + .toString(); + } + MprisReceiverCallback(MprisReceiverPlugin plugin, MprisReceiverPlayer player) { this.player = player; this.plugin = plugin; + // fetch the initial art, when player is already running and we start kdeconnect + Pair artAndUri = getArtAndUri(player.getMetadata()); + if (artAndUri != null) { + Bitmap bitmap = artAndUri.first; + artHash = hashBitmap(bitmap); + artUrl = makeArtUrl(artHash, artAndUri.second); + displayArt = bitmap; + album = player.getAlbum(); + artist = player.getArtist(); + } } @Override @@ -35,6 +144,43 @@ class MprisReceiverCallback extends MediaController.Callback { @Override public void onMetadataChanged(@Nullable MediaMetadata metadata) { + if (metadata == null) { + artHash = null; + displayArt = null; + artUrl = null; + artist = null; + album = null; + } else { + // We could check hasRequestedAlbumArt to avoid hashing art for clients that don't support it + // But upon running the profiler, looks like hashBitmap is a minuscule (<1%) part so no + // need to optimize prematurely. + Pair artAndUri = getArtAndUri(metadata); + String newAlbum = metadata.getString(MediaMetadata.METADATA_KEY_ALBUM); + String newArtist = metadata.getString(MediaMetadata.METADATA_KEY_ARTIST); + if (artAndUri == null) { + // check if the album+artist is still the same- some players don't send art every time + if (!Objects.equals(newAlbum, album) || !Objects.equals(newArtist, artist)) { + // there really is no new art + artHash = null; + displayArt = null; + artUrl = null; + album = null; + artist = null; + } + } else { + Long newHash = hashBitmap(artAndUri.first); + // In case the hashes are equal, we do a full comparison to protect against collisions + if ((!newHash.equals(artHash) || !artAndUri.first.sameAs(displayArt))) { + artHash = newHash; + displayArt = artAndUri.first; + artUrl = makeArtUrl(artHash, artAndUri.second); + artist = newArtist; + album = newAlbum; + } + } + + } + plugin.sendMetadata(player); } @@ -43,4 +189,22 @@ class MprisReceiverCallback extends MediaController.Callback { //Note: not called by all media players plugin.sendMetadata(player); } + + public String getArtUrl() { + return artUrl; + } + + /** + * Get the JPG art of the current track as a bytearray. + * + * @return null if no art is available, otherwise a PNG image serialized into a bytearray + */ + public byte[] getArtAsArray() { + if (displayArt == null) { + return null; + } + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + displayArt.compress(Bitmap.CompressFormat.JPEG, 90, stream); + return stream.toByteArray(); + } } diff --git a/src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverPlayer.java b/src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverPlayer.java index 578995ff..da1ff39d 100644 --- a/src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverPlayer.java +++ b/src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverPlayer.java @@ -163,4 +163,8 @@ class MprisReceiverPlayer { return metadata.getLong(MediaMetadata.METADATA_KEY_DURATION); } + + MediaMetadata getMetadata() { + return controller.getMetadata(); + } } diff --git a/src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverPlugin.java b/src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverPlugin.java index 5116bb99..bfbd6d7a 100644 --- a/src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverPlugin.java +++ b/src/org/kde/kdeconnect/Plugins/MprisReceiverPlugin/MprisReceiverPlugin.java @@ -23,6 +23,7 @@ import androidx.fragment.app.DialogFragment; import org.apache.commons.lang3.StringUtils; import org.kde.kdeconnect.Helpers.AppsHelper; +import org.kde.kdeconnect.Helpers.ThreadHelper; import org.kde.kdeconnect.NetworkPacket; import org.kde.kdeconnect.Plugins.NotificationsPlugin.NotificationReceiver; import org.kde.kdeconnect.Plugins.Plugin; @@ -49,12 +50,15 @@ public class MprisReceiverPlugin extends Plugin { private HashMap playerCbs; private MediaSessionChangeListener mediaSessionChangeListener; + public @NonNull String getDeviceId() { + return device.getDeviceId(); + } + @Override public boolean onCreate() { if (!hasPermission()) return false; - players = new HashMap<>(); playerCbs = new HashMap<>(); try { @@ -103,7 +107,6 @@ public class MprisReceiverPlugin extends Plugin { @Override public boolean onPacketReceived(@NonNull NetworkPacket np) { - if (np.getBoolean("requestPlayerList")) { sendPlayerList(); return true; @@ -117,6 +120,18 @@ public class MprisReceiverPlugin extends Plugin { if (null == player) { return false; } + String artUrl = np.getString("albumArtUrl", ""); + if (!artUrl.isEmpty()) { + String playerName = player.getName(); + MprisReceiverCallback cb = playerCbs.get(playerName); + if (cb == null) { + Log.e(TAG, "no callback for " + playerName + " (player likely stopped)"); + return false; + } + // run it on a different thread to avoid blocking + ThreadHelper.execute(() -> sendAlbumArt(playerName, cb, artUrl)); + return true; + } if (np.getBoolean("requestNowPlaying", false)) { sendMetadata(player); @@ -181,7 +196,7 @@ public class MprisReceiverPlugin extends Plugin { return; } for (MprisReceiverPlayer p : players.values()) { - p.getController().unregisterCallback(playerCbs.get(p.getName())); + p.getController().unregisterCallback(Objects.requireNonNull(playerCbs.get(p.getName()))); } playerCbs.clear(); players.clear(); @@ -206,6 +221,7 @@ public class MprisReceiverPlugin extends Plugin { private void sendPlayerList() { NetworkPacket np = new NetworkPacket(PACKET_TYPE_MPRIS); np.set("playerList", players.keySet()); + np.set("supportAlbumArtPayload", true); getDevice().sendPacket(np); } @@ -214,6 +230,37 @@ public class MprisReceiverPlugin extends Plugin { return Build.VERSION_CODES.LOLLIPOP_MR1; } + void sendAlbumArt(String playerName, @NonNull MprisReceiverCallback cb, @Nullable String requestedUrl) { + // NOTE: It is possible that the player gets killed in the middle of this method. + // The proper thing to do this case would be to abort the send - but that gets into the + // territory of async cancellation or putting a lock. + // For now, we just continue to send the art- cb stores the bitmap, so it will be valid. + // cb will get GC'd after this method completes. + String localArtUrl = cb.getArtUrl(); + if (localArtUrl == null) { + Log.w(TAG, "art not found!"); + return; + } + String artUrl = requestedUrl == null ? localArtUrl : requestedUrl; + if (requestedUrl != null && !requestedUrl.contentEquals(localArtUrl)) { + Log.w(TAG, "sendAlbumArt: Doesn't match current url"); + Log.d(TAG, "current: " + localArtUrl); + Log.d(TAG, "requested: " + requestedUrl); + return; + } + byte[] p = cb.getArtAsArray(); + if (p == null) { + Log.w(TAG, "sendAlbumArt: Failed to get art stream"); + return; + } + NetworkPacket np = new NetworkPacket(PACKET_TYPE_MPRIS); + np.setPayload(new NetworkPacket.Payload(p)); + np.set("player", playerName); + np.set("transferringAlbumArt", true); + np.set("albumArtUrl", artUrl); + getDevice().sendPacket(np); + } + void sendMetadata(MprisReceiverPlayer player) { NetworkPacket np = new NetworkPacket(MprisReceiverPlugin.PACKET_TYPE_MPRIS); np.set("player", player.getName()); @@ -232,6 +279,15 @@ public class MprisReceiverPlugin extends Plugin { np.set("canGoNext", player.canGoNext()); np.set("canSeek", player.canSeek()); np.set("volume", player.getVolume()); + MprisReceiverCallback cb = playerCbs.get(player.getName()); + assert cb != null; + String artUrl = cb.getArtUrl(); + if (artUrl != null) { + np.set("albumArtUrl", artUrl); + Log.v(TAG, "Sending metadata with url " + artUrl); + } else { + Log.v(TAG, "Sending metadata without url "); + } getDevice().sendPacket(np); }