mir/microbot
2
0
Fork 0
mirror of https://github.com/Nick80835/microbot synced 2025-08-28 21:18:34 +00:00
Code Issues Releases Wiki Activity

parallel download cleanup

Browse Source
This commit is contained in:
Nick80835 2023-10-16 19:25:50 -04:00
parent d58c0454e9
commit ceb29e8583
1 changed files with 3 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
ubot/fixes/parallel_download.py
View File

@ -15,12 +15,7 @@ class ParallelDownload:
async def download_chunk(self, chunk_start: int, chunk_end: int, chunk_number: int) -> str: async def download_chunk(self, chunk_start: int, chunk_end: int, chunk_number: int) -> str:
async with self.aioclient.get(self.url, headers={"Range": f"bytes={chunk_start}-{chunk_end}"}) as response: async with self.aioclient.get(self.url, headers={"Range": f"bytes={chunk_start}-{chunk_end}"}) as response:
async with aiofiles.open(f"ubot/cache/{self.file_name}.part{chunk_number}", mode="wb") as cache_file: async with aiofiles.open(f"ubot/cache/{self.file_name}.part{chunk_number}", mode="wb") as cache_file:
while True: while chunk := await response.content.read(4096):
chunk = await response.content.read(4096)
if not chunk:
break
await cache_file.write(chunk) await cache_file.write(chunk)
await cache_file.flush() await cache_file.flush()
@ -44,7 +39,7 @@ class ParallelDownload:
return chunk_coros return chunk_coros
async def get_download_info(self, threads) -> (int, int, str): async def get_download_info(self, threads) -> tuple[int, int, str|None]:
async with self.aioclient.get(self.url) as response: async with self.aioclient.get(self.url) as response:
content_length = int(response.headers["content-length"]) content_length = int(response.headers["content-length"])
file_extension = mimetypes.guess_extension(response.headers["content-type"]) file_extension = mimetypes.guess_extension(response.headers["content-type"])
@ -65,7 +60,7 @@ class ParallelDownload:
return content_length, chunk_size, file_extension return content_length, chunk_size, file_extension
async def download(url: str, file_name: str, aioclient: ClientSession = ClientSession(), threads: int = None) -> str: async def download(url: str, file_name: str, aioclient: ClientSession = ClientSession(), threads: int|None = None) -> str:
downloader = ParallelDownload(url, aioclient, file_name) downloader = ParallelDownload(url, aioclient, file_name)
content_length, chunk_size, file_extension = await downloader.get_download_info(threads) content_length, chunk_size, file_extension = await downloader.get_download_info(threads)
chunk_coros = await downloader.generate_chunk_coros(content_length, chunk_size) chunk_coros = await downloader.generate_chunk_coros(content_length, chunk_size)