mir/microbot
2
0
Fork 0
mirror of https://github.com/Nick80835/microbot synced 2025-08-30 14:08:31 +00:00
Code Issues Releases Wiki Activity

clean up scrapers a bit

Browse Source
This commit is contained in:
Nick80835 2020-05-03 20:45:35 -04:00
parent 09f585d69f
commit b5ce768917
4 changed files with 14 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ubot/modules/4chan.py
View File

@ -16,10 +16,7 @@ VALID_ENDS = (".mp4", ".jpg", ".jpeg", ".png", ".gif")
@ldr.add("4c(f|)") @ldr.add("4c(f|)")
async def fourchan(event): async def fourchan(event):
if event.pattern_match.group(1): as_file = bool(event.pattern_match.group(1))
as_file = True
else:
as_file = False
session = ClientSession() session = ClientSession()

9
ubot/modules/danbooru.py
View File

@ -31,22 +31,21 @@ async def danbooru(event):
async with session.get(DAN_URL, params=params) as response: async with session.get(DAN_URL, params=params) as response:
if response.status == 200: if response.status == 200:
response = await response.json() response = await response.json()
await session.close()
else: else:
await event.reply(f"`An error occurred, response code: `**{response.status}**") await event.reply(f"`An error occurred, response code: `**{response.status}**")
await session.close() await session.close()
return return
await session.close()
if not response: if not response:
await event.reply(f"`No results for query: `**{event.args}**") await event.reply(f"`No results for query: `**{event.args}**")
return return
valid_urls = [] valid_urls = []
for url in ['file_url', 'large_file_url', 'source']: for post in response:
if url in response[0].keys(): if 'file_url' in post.keys():
valid_urls.append(response[0][url]) valid_urls.append(post['file_url'])
if not valid_urls: if not valid_urls:
await event.reply(f"`Failed to find URLs for query: `**{event.args}**") await event.reply(f"`Failed to find URLs for query: `**{event.args}**")

11
ubot/modules/gelbooru.py
View File

@ -34,23 +34,22 @@ async def gelbooru(event):
async with session.get(GEL_URL, params=params) as response: async with session.get(GEL_URL, params=params) as response:
if response.status == 200: if response.status == 200:
response = (await response.json())[0] response = await response.json()
await session.close()
else: else:
await event.reply(f"`An error occurred, response code: `**{response.status}**") await event.reply(f"`An error occurred, response code: `**{response.status}**")
await session.close() await session.close()
return return
await session.close()
if not response: if not response:
await event.reply(f"`No results for query: `**{event.args}**") await event.reply(f"`No results for query: `**{event.args}**")
return return
valid_urls = [] valid_urls = []
for url in ['file_url', 'large_file_url', 'source']: for post in response:
if url in response.keys(): if 'file_url' in post.keys():
valid_urls.append(response[url]) valid_urls.append(post['file_url'])
if not valid_urls: if not valid_urls:
await event.reply(f"`Failed to find URLs for query: `**{event.args}**") await event.reply(f"`Failed to find URLs for query: `**{event.args}**")

9
ubot/modules/sankaku.py
View File

@ -31,22 +31,21 @@ async def sankaku(event):
async with session.get(SAN_URL, params=params) as response: async with session.get(SAN_URL, params=params) as response:
if response.status == 200: if response.status == 200:
response = await response.json() response = await response.json()
await session.close()
else: else:
await event.reply(f"`An error occurred, response code: `**{response.status}**") await event.reply(f"`An error occurred, response code: `**{response.status}**")
await session.close() await session.close()
return return
await session.close()
if not response: if not response:
await event.reply(f"`No results for query: `**{event.args}**") await event.reply(f"`No results for query: `**{event.args}**")
return return
valid_urls = [] valid_urls = []
for item in response: for post in response:
if 'file_url' in item.keys(): if 'file_url' in post.keys():
valid_urls.append(item['file_url']) valid_urls.append(post['file_url'])
if not valid_urls: if not valid_urls:
await event.reply(f"`Failed to find URLs for query: `**{event.args}**") await event.reply(f"`Failed to find URLs for query: `**{event.args}**")