From b5ce768917ffddb7260662cf974828a44bcc83fc Mon Sep 17 00:00:00 2001 From: Nick80835 Date: Sun, 3 May 2020 20:45:35 -0400 Subject: [PATCH] clean up scrapers a bit --- ubot/modules/4chan.py | 5 +---- ubot/modules/danbooru.py | 9 ++++----- ubot/modules/gelbooru.py | 11 +++++------ ubot/modules/sankaku.py | 9 ++++----- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/ubot/modules/4chan.py b/ubot/modules/4chan.py index 604715f..5beb0aa 100644 --- a/ubot/modules/4chan.py +++ b/ubot/modules/4chan.py @@ -16,10 +16,7 @@ VALID_ENDS = (".mp4", ".jpg", ".jpeg", ".png", ".gif") @ldr.add("4c(f|)") async def fourchan(event): - if event.pattern_match.group(1): - as_file = True - else: - as_file = False + as_file = bool(event.pattern_match.group(1)) session = ClientSession() diff --git a/ubot/modules/danbooru.py b/ubot/modules/danbooru.py index 1d3d8fd..dcd3a18 100644 --- a/ubot/modules/danbooru.py +++ b/ubot/modules/danbooru.py @@ -31,22 +31,21 @@ async def danbooru(event): async with session.get(DAN_URL, params=params) as response: if response.status == 200: response = await response.json() + await session.close() else: await event.reply(f"`An error occurred, response code: `**{response.status}**") await session.close() return - await session.close() - if not response: await event.reply(f"`No results for query: `**{event.args}**") return valid_urls = [] - for url in ['file_url', 'large_file_url', 'source']: - if url in response[0].keys(): - valid_urls.append(response[0][url]) + for post in response: + if 'file_url' in post.keys(): + valid_urls.append(post['file_url']) if not valid_urls: await event.reply(f"`Failed to find URLs for query: `**{event.args}**") diff --git a/ubot/modules/gelbooru.py b/ubot/modules/gelbooru.py index fb053dc..6f8aefc 100644 --- a/ubot/modules/gelbooru.py +++ b/ubot/modules/gelbooru.py @@ -34,23 +34,22 @@ async def gelbooru(event): async with session.get(GEL_URL, params=params) as response: if response.status == 200: - response = (await response.json())[0] + response = await response.json() + await session.close() else: await event.reply(f"`An error occurred, response code: `**{response.status}**") await session.close() return - await session.close() - if not response: await event.reply(f"`No results for query: `**{event.args}**") return valid_urls = [] - for url in ['file_url', 'large_file_url', 'source']: - if url in response.keys(): - valid_urls.append(response[url]) + for post in response: + if 'file_url' in post.keys(): + valid_urls.append(post['file_url']) if not valid_urls: await event.reply(f"`Failed to find URLs for query: `**{event.args}**") diff --git a/ubot/modules/sankaku.py b/ubot/modules/sankaku.py index 5c5d691..53e2529 100644 --- a/ubot/modules/sankaku.py +++ b/ubot/modules/sankaku.py @@ -31,22 +31,21 @@ async def sankaku(event): async with session.get(SAN_URL, params=params) as response: if response.status == 200: response = await response.json() + await session.close() else: await event.reply(f"`An error occurred, response code: `**{response.status}**") await session.close() return - await session.close() - if not response: await event.reply(f"`No results for query: `**{event.args}**") return valid_urls = [] - for item in response: - if 'file_url' in item.keys(): - valid_urls.append(item['file_url']) + for post in response: + if 'file_url' in post.keys(): + valid_urls.append(post['file_url']) if not valid_urls: await event.reply(f"`Failed to find URLs for query: `**{event.args}**")