Run Multithread and Asyncio together - python-3.x

Question

i am trying to make a python program that makes http requests and parses the data from the response with asyncio and aiohttp. The program takes as input a list of urls that can reach even more than 300/400 elements and should make requests as fast as possible. Since asyncio runs on only one thread, I was thinking of splitting the list into sublists and starting a thread for each sublist with asyncio and aiohttp, but adding the ThreadPoolExecutor I get coroutine and futures errors. How can I use Thread and asyncio together? I leave the code below.
async def fetch(url, session):
async with session.get(url, ssl=False) as response:
# if response.status == 200:
html_body = await response.json()
url = url.split('/')[-2]
file_name = url if url != 'services' else 'live'
async with aiofiles.open(f'{output_dir}/{file_name}.json', 'w') as f:
await f.write(json.dumps(html_body, indent=4))
return html_body
async def fetch_with_sem(sem, session, url):
async with sem:
return await fetch(url, session)
async def run(url, session, sem):
tasks = [asyncio.create_task(fetch_with_sem(sem, session, url)) for url in url]
page_content = await asyncio.gather(*tasks, return_exceptions=True)
return page_content
async def main(urls):
number = len(urls) // 10 + 1
sem = asyncio.Semaphore(50)
loop = asyncio.get_event_loop()
connector = aiohttp.TCPConnector(limit_per_host=30, limit=50, ttl_dns_cache=100)
headers = {
'user-agent': get_user_agent(),
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': "macOS",
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'x-eb-accept-language': 'it_IT',
'x-eb-marketid': '5',
'x-eb-platformid': '1',
}
async with ClientSession(loop=loop, connector=connector, headers=headers) as session:
if isinstance(urls, list):
with ThreadPoolExecutor(max_workers=25) as executor:
page_content = [executor.submit(
run, urls[number * i : number * (i + 1)], session, sem
).result() for i in range(10)
]
else:
tasks = [asyncio.create_task(fetch_with_sem(sem, session, urls))]
page_content = await asyncio.gather(*tasks, return_exceptions=True)
return page_content
the problem is given from executor.submit and got this error:
RuntimeWarning: coroutine 'wait' was never awaited
page_content = [executor.submit(asyncio.wait(run), urls[number * i : number * (i + 1)], session, sem).result() for i in range(10)]
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
Traceback (most recent call last):
File "/Users/federikowsky/Desktop/Python/Scraping/SureBet/prova.py", line 123, in <module>
x = asyncio.run(main(link))
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/Users/federikowsky/Desktop/Python/Scraping/SureBet/prova.py", line 110, in main
page_content = [executor.submit(asyncio.wait(run), urls[number * i : number * (i + 1)], session, sem).result() for i in range(10)]
File "/Users/federikowsky/Desktop/Python/Scraping/SureBet/prova.py", line 110, in <listcomp>
page_content = [executor.submit(asyncio.wait(run), urls[number * i : number * (i + 1)], session, sem).result() for i in range(10)]
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/concurrent/futures/_base.py", line 437, in result
return self.__get_result()
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
raise self._exception
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
TypeError: 'coroutine' object is not callable