youtube API get all playlist id from a channel : python - python-3.x

I'm trying to have a list of all playlists from a specific channel in youtube API ... in python
I'd like to have a list of playlist_id in a array
all_playlist_item = []
if I launch
https://www.googleapis.com/youtube/v3/playlists?part=snippet&channelId=UC1udnO-W6gpR9qzleJ5SDKw&key=xxxxxx
here's the response from this request
{
"kind": "youtube#playlistListResponse",
"etag": "PMwo-9BIp7p_L2ynH9sFOGIOido",
"nextPageToken": "CAUQAA",
"pageInfo": {
"totalResults": 17,
"resultsPerPage": 5
},
"items": [
{
"kind": "youtube#playlist",
"etag": "-V67IpyB9a1JGDGJ4pVQnEoMRy4",
"id": "PLWi7PxnyAMeN1tb-ldDzJcnJy2yd5wYrO",
"snippet": {
...
I thing i have to used nextPageToken, but I don't know how to code this function for playlist items
Here's my code (extract in excel all videos just from a specific playlist)
channel_name="UC1udnO-W6gpR9qzleJ5SDKw"
playlist_name="PLWi7PxnyAMeOJmVLv8Z_N3edNyipsnHbo"
api_key = "xxxxxx"
from apiclient.discovery import build
import pandas as pd
youtube = build('youtube', 'v3', developerKey=api_key)
def get_channel_videos(channel_id):
# get Uploads playlist id
res = youtube.channels().list(id=channel_id,
part='contentDetails').execute()
playlist_id = res['items'][0]['contentDetails']['relatedPlaylists']['uploads']
videos = []
next_page_token = None
while 1:
res = youtube.playlistItems().list(playlistId=playlist_name,
part='snippet',
maxResults=50,
pageToken=next_page_token).execute()
videos += res['items']
next_page_token = res.get('nextPageToken')
if next_page_token is None:
break
return videos
videos = get_channel_videos(channel_name)
def get_videos_stats(video_ids):
stats = []
for i in range(0, len(video_ids), 50):
res = youtube.videos().list(id=','.join(video_ids[i:i+50]),
part='statistics').execute()
stats += res['items']
return stats
video_ids = list(map(lambda x:x['snippet']['resourceId']['videoId'], videos))
stats = get_videos_stats(video_ids)
d = []
if len(stats)!=len(videos):
i=1
j=0
else:
i=0
j=0
len_video = len(videos)
len_stats = len(stats)
for video in videos:
if i >= len_video:
break
Url_video='https://www.youtube.com/watch?v='+videos[i]['snippet']['resourceId']['videoId']+'&list='+playlist_name
d.append((videos[i]['snippet']['title'],
videos[i]['snippet']['resourceId']['videoId'],
Url_video,
stats[j]['statistics']['viewCount'],
stats[j]['statistics']['likeCount'],
stats[j]['statistics']['dislikeCount']
))
i+=1
j+=1
df=pd.DataFrame(d, columns=('Titre_video', 'ID_video', 'Url_video','vues','like','dislike'))
df['vues'] = df['vues'].astype(int)
df['like'] = df['like'].astype(int)
df['dislike'] = df['dislike'].astype(int)
df.index+=1
df.to_excel("youtube-playlist.xlsx")

From the pagination doc, you can use :
youtube.playlistItems().list_next(request, response) for iterating playlist item response
youtube.playlists().list_next(request, response) for iterating channel response
Get all videos from Playlist
import googleapiclient.discovery
playlist_id = "PLWi7PxnyAMeOJmVLv8Z_N3edNyipsnHbo"
youtube = googleapiclient.discovery.build("youtube", "v3", developerKey = "YOUR_API_KEY")
request = youtube.playlistItems().list(
part = "snippet",
playlistId = playlist_id,
maxResults = 50
)
response = request.execute()
playlist_items = []
while request is not None:
response = request.execute()
playlist_items += response["items"]
request = youtube.playlistItems().list_next(request, response)
print(f"total: {len(playlist_items)}")
print(playlist_items)
Get all playlists from channel
import googleapiclient.discovery
channel_id = "UC1udnO-W6gpR9qzleJ5SDKw"
youtube = googleapiclient.discovery.build("youtube", "v3", developerKey = "YOUR_API_KEY")
request = youtube.playlists().list(
part = "snippet",
channelId = channel_id,
maxResults = 50
)
response = request.execute()
playlists = []
while request is not None:
response = request.execute()
playlists += response["items"]
request = youtube.playlists().list_next(request, response)
print(f"total: {len(playlists)}")
print(playlists)

Related

How to get all data certain website with Scrapy?

I'm currently learn scraping using scrapy, so I want to scrape data from this: https://www.espn.com/nba/stats/player or https://www.espn.com/nba/stats/player/_/season/2023/seasontype/2
if you go to the link you will see the show more button at the bottom data so im confuse about this because if i scrape that right now im only get 50 of data that, not what i want, so i decided to look up the show more button but its only a href=#
[UPDATE] USING Scrapy+playwright
def start_requests(self):
yield scrapy.Request(
url='https://www.espn.com/nba/stats/player/_/season/2023/seasontype/2',
meta= dict(
playwright = True,
playwright_include_page=True,
playwright_page_coroutines = [
PageMethod('wait_for_selector','//a[#class="AnchorLink loadMore__link"]'),
PageMethod('click','//a[#class="AnchorLink loadMore__link"]'),
]
),
callback=self.parse,
)
async def parse(self, response):
page = response.meta['playwright_page']
button = response.meta['playwright_page_coroutines'][0]
if button:
await button.click()
resp = response.body
player_list = sel.xpath(
"//table[#class='Table Table--align-right Table--fixed Table--fixed-left']//tbody//tr")
stats_list = sel.xpath(
"//div[#class='Table__ScrollerWrapper relative overflow-hidden']/div[#class='Table__Scroller']/table/tbody/tr")
await page.wait_for_selector(player_list)
sel = Selector(text=resp)
for player, stat in zip(player_list, stats_list):
player_name = player.xpath(".//a/text()").get()
position = stat.xpath(".//td/div/text()").get()
team_name = player.xpath(".//span/text()").get()
game_played = stat.xpath(".//td[2]/text()").get()
minutes_per_minute = stat.xpath(".//td[3]/text()").get()
points_per_game = stat.xpath(".//td[4]/text()").get()
fields_goal_made = stat.xpath(".//td[5]/text()").get()
fields_goal_attempted = stat.xpath(".//td[6]/text()").get()
field_goal_percentage = stat.xpath(".//td[7]/text()").get()
three_point_goal_made = stat.xpath(".//td[8]/text()").get()
yield {
"player_name": player_name,
"player_position": position,
"team_name": team_name,
"game_played": game_played,
"minutes_per_minute": minutes_per_minute,
"points_per_game": points_per_game,
"fields_goal_made": fields_goal_made,
"fields_goal_attempted": fields_goal_attempted,
"field_goal_percentage": field_goal_percentage,
"three_point_goal_made": three_point_goal_made,
}
When only using scrapy
def start_requests(self):
yield scrapy.Request(
url='https://www.espn.com/nba/stats/player/_/season/2023/seasontype/2',
callback=self.parse,
)
def parse(self, response):
sel = Selector(text=response.body)
player_list = sel.xpath("//table[#class='Table Table--align-right Table--fixed Table--fixed-left']//tbody//tr")
stats_list = sel.xpath("//div[#class='Table__ScrollerWrapper relative overflow-hidden']/div[#class='Table__Scroller']/table/tbody/tr")
for player,stat in zip(player_list,stats_list):
player_name = player.xpath(".//a/text()").get()
position = stat.xpath(".//td/div/text()").get()
team_name = player.xpath(".//span/text()").get()
game_played = stat.xpath(".//td[2]/text()").get()
minutes_per_minute= stat.xpath(".//td[3]/text()").get()
points_per_game = stat.xpath(".//td[4]/text()").get()
fields_goal_made = stat.xpath(".//td[5]/text()").get()
fields_goal_attempted = stat.xpath(".//td[6]/text()").get()
field_goal_percentage = stat.xpath(".//td[7]/text()").get()
three_point_goal_made = stat.xpath(".//td[8]/text()").get()
yield {
"player_name": player_name,
"player_position":position,
"team_name": team_name,
"game_played": game_played,
"minutes_per_minute": minutes_per_minute,
"points_per_game": points_per_game,
"fields_goal_made":fields_goal_made,
"fields_goal_attempted":fields_goal_attempted,
"field_goal_percentage":field_goal_percentage,
"three_point_goal_made":three_point_goal_made,
}
Am i doing it wrong here? also if you click show more it will show the api like down bellow, i can scrape from that api but for now i want it from html with xpath it self.

bithumb Api error -{'status': '5100', 'message': 'Bad Request.(Auth Data)'}

I wanted to make a program where I can check the order details of my order at Bithumb Exchange.
So I looked at the docs(https://api.bithumb.com/info/orders) and made it, but the same error code kept coming out, so I don't know what to do.
import time
import math
import base64
import hmac, hashlib
import urllib.parse
import requests
class XCoinAPI:
api_url = "https://api.bithumb.com";
api_key = "";
api_secret = "";
def __init__(self, api_key, api_secret):
self.api_key = api_key;
self.api_secret = api_secret;
def body_callback(self, buf):
self.contents = buf;
def microtime(self, get_as_float = False):
if get_as_float:
return time.time()
else:
return '%f %d' % math.modf(time.time())
def usecTime(self) :
mt = self.microtime(False)
mt_array = mt.split(" ")[:2];
return mt_array[1] + mt_array[0][2:5];
def xcoinApiCall(self, endpoint, rgParams):
# 1. Api-Sign and Api-Nonce information generation.
# 2. Request related information from the Bithumb API server.
#
# - nonce: it is an arbitrary number that may only be used once.
# - api_sign: API signature information created in various combinations values.
endpoint_item_array = {
"endpoint" : endpoint
}
uri_array = dict(endpoint_item_array, **rgParams) # Concatenate the two arrays.
str_data = urllib.parse.urlencode(uri_array)
nonce = self.usecTime()
data = endpoint + chr(1) + str_data + chr(1) + nonce
utf8_data = data.encode('utf-8')
key = self.api_secret
utf8_key = key.encode('utf-8')
h = hmac.new(bytes(utf8_key), utf8_data, hashlib.sha512)
hex_output = h.hexdigest()
utf8_hex_output = hex_output.encode('utf-8')
api_sign = base64.b64encode(utf8_hex_output)
utf8_api_sign = api_sign.decode('utf-8')
headers = {
"Accept": "application/json",
"Content-Type": "application/x-www-form-urlencoded",
"Api-Key": self.api_key,
"Api-Nonce": nonce,
"Api-Sign": utf8_api_sign
}
url = self.api_url + endpoint
r = requests.post(url, headers=headers, data=uri_array)
return r.json()
a = XCoinAPI(api_key="MYKEY1c", api_secret="MYKEY2")
aa= a.xcoinApiCall("/info/orders",{"order_currency":"LN","payment_currency":"BTC"})
print(aa)
{'status': '5100', 'message': 'Bad Request.(Auth Data)'}
Process finished with exit code 0
The error code 5100, bad request keeps coming up(https://apidocs.bithumb.com/docs/api-%EC%A3%BC%EC%9A%94-%EC%97%90%EB%9F%AC-%EC%BD%94%EB%93%9C)
I really don't know which code to modify.
I think it's a parameter problem with XcoinApiCall, but I don't know.

Can I use urlencode in python to insert a string of characters into this Spotify API request?

I am working with Spotify API to request song data through a URL. I imported urlencode to run the song's ID as a function parameter into the url. Essentially, I need the ID portion of the url request to be the ID by itself, not "id=<id string>"
I tried assigning the id string to a parameter of my 'search' function. The search function takes a user input song ID from spotify, inserts it into the proper position in the URL request, and sends it to the spotify database to retrieve that songs data analysis. The program successfully sends out the request, but the id portion I am filling in puts "id=<song ID>" instead of the song ID by itself.
import requests
import datetime
from urllib.parse import urlencode
# In[3]:
import base64
# In[4]:
client_id = 'fb5af83351d4402fa82904fc04f7fc9e'
client_secret = 'b5057eb39b024180b61b02eb45fb97a6'
# In[5]:
class SpotifyAPI(object):
access_token = None
access_token_expires = datetime.datetime.now()
access_token_did_expire = True
client_id = None
client_secret = None
token_url = "https://accounts.spotify.com/api/token"
def __init__(self, client_id, client_secret, *args, **kwargs):
super().__init__(*args, **kwargs)
self.client_id = client_id
self.client_secret = client_secret
def get_client_credentials(self):
client_id = self.client_id
client_secret = self.client_secret
if client_secret == None or client_id == None:
raise Exception("You must set client_id and client_secret")
client_creds = f"{client_id}:{client_secret}"
client_creds_b64 = base64.b64encode(client_creds.encode())
return client_creds_b64.decode()
def get_token_headers(self):
client_creds_b64 = self.get_client_credentials()
return {
"Authorization": f"Basic {client_creds_b64}"
}
def get_token_data(self):
return {
"grant_type": "client_credentials"
}
def perform_authorization(self):
token_url = self.token_url
token_data = self.get_token_data()
token_headers = self.get_token_headers()
r = requests.post(token_url, data=token_data, headers=token_headers)
print(r.json())
if r.status_code not in range(200,299):
raise Exception("Could not authenticate client")
#return False
data = r.json()
now = datetime.datetime.now()
access_token = data['access_token']
expires_in = data['expires_in'] #seconds
expires = now + datetime.timedelta(seconds=expires_in)
self.access_token = access_token
self.access_token_expires = expires
self.access_token_did_expire = expires < now
return True
def get_access_token(self):
token = self.access_token
expires = self.access_token_expires
now = datetime.datetime.now()
if expires < now:
self.perform_authorization()
return self.get_access_token()
elif token == None:
self.perform_authorization()
return self.get_access_token()
return token
def search(self, id):
access_token = self.get_access_token()
headers = {
"Authorization": f"Bearer {access_token}"
}
end_point = "https://api.spotify.com/v1/audio-analysis/"
data = urlencode({"id":id})
print(data)
lookup_url = f"{end_point}{data}"
print(lookup_url)
r = requests.get(lookup_url, headers = headers)
if r.status_code not in range(200, 299):
return r
return r.json()
it returns
{'access_token': 'BQCLoKT_b2PF7KPSbscosa1dCpE5rzd_RBkswOvwklVdlAL4AeEGCDn0iYuqac5o86BTqCIz0m95u3olLp4', 'token_type': 'Bearer', 'expires_in': 3600}
id=1UGD3lW3tDmgZfAVDh6w7r
https://api.spotify.com/v1/audio-analysis/id=1UGD3lW3tDmgZfAVDh6w7r

python 3 discord bot

lib.py:
def getPlayerId(playername):
params = {"action": "autocompleteList",
"iso": iso,
"server": server
}
rq1 = requests.post(url_base, params)
server_data = rq1.json()
player_data = server_data["player"]
for player in player_data:
if playername in player["pseudo"]:
return player["id"]
return 0
print(getPlayerId('JD')) # print exactly user ID
bot.py:
import discord
from discord.ext import commands
from lib import *
#bot.command(pass_context=True)
async def find(ctx, *args):
...
player_id = getPlayerId('JD')` # return 0 even though the user exists
...
My problem is that when I call getPlayerId in bot command it always get 0, although getPlayerId run properly (return ID of player) if I call it at lib.py
I don't know what is the problem with my code since it doesn't show any error. Any suggestion is appreciated.
def getPlayerId(playername):
player_id = 0;
params = {"action": "autocompleteList",
"iso": iso,
"server": server
}
rq1 = requests.post(url_base, params)
server_data = rq1.json()
player_data = server_data["player"]
for player in player_data:
if playername in player["pseudo"]:
player_id = player["id"]
return player_id

Python Telegram API ReplyKeyboardMarkup

I am new to python. I am trying to send a response showing a custom keyboard with two buttons to the user in Telegram app but it gives me an error about encode.
I would like to know where is my mistake.
import json
from urllib.request import urlopen
from urllib.parse import quote, unquote
import time
def Decode(telegram_response):
decoded=''
for line in telegram_response:
decoded=decoded+line.decode('utf-8')
return decoded
TOKEN = 'mytoken'
URL = 'https://api.telegram.org/bot{}/'.format(TOKEN)
cmd = 'getme'
telegram_response = urlopen(URL + cmd)
decoded = Decode(telegram_response)
gtm = json.loads(decoded)
status = True
while status:
cmd = 'getUpdates'
telegram_response = urlopen(URL + cmd)
decoded = Decode(telegram_response)
upds = json.loads(decoded)
new_message = len(upds['result'])
if new_message !=0:
msg = upds['result'][0]['message']
chat_id = str(msg['chat']['id'])
reply_markup = {'keyboard': [[{'text':'first button'}],[{'text':'second button'}]], 'resize_keyboard': True, 'one_time_keyboard': True}
reply_markup = json.dumps(reply_markup)
params = ({'chat_id': chat_id, 'reply_markup': reply_markup, 'disable_web_page_preview': 'true'})
myresponse =urlopen(URL + 'sendMessage' + quote((params).encode('utf-8')))
Easy way to build powerful bots is to use python-telegram-bot library.
I re-writed your code with few major fixes and features. Hope it help you to learn bots more deeper.
my version of bot:
###############################################################################
!#/usr/bin/python3
from sys import exc_info as error
from urllib.request import urlopen
from urllib.parse import urlencode
import json
TOKEN = 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
URL = 'https://api.telegram.org/bot{}'.format(TOKEN)
STATUS = True
OFFSET = 0
def call_api_method(method='getMe', data=None):
# Call API method with data.
data = urlencode(data).encode("utf-8") if data else data
response = urlopen('{}/{}'.format(URL, method), data)
return json.loads(response.read())
def get_me():
# Get bot info.
bot = call_api_method()
return type('Bot', (), dict(bot['result']))
def get_updates():
# Get new updates from Telegram.
data = {'offset': OFFSET, 'limit': 0, 'timeout': 0}
return type('Updates', (), call_api_method('getUpdates', data))
def handle(update):
# Make usefull objects.
message = type('Message', (object,), dict(update['message']))
user = type('User', (), dict(update['message']['from']))
chat = type('Chat', (), dict(update['message']['chat']))
return message, user, chat
def send_message(chat_id, message):
# Send message to specific chat.
data = {'text': message,
'chat_id': chat_id,
'parse_mode': 'Markdown',
'disable_web_page_preview': True}
call_api_method('sendMessage', data)
def send_keyboard(chat_id, message, keyboard):
# Send message and keyboard to specific chat.
data = {'text': message,
'chat_id': chat_id,
'parse_mode': 'Markdown',
'reply_markup': reply_markup(keyboard),
'disable_web_page_preview': 'true'}
call_api_method('sendMessage', data)
def reply_markup(keyboard):
# Serialize keyboard data to JSON.
return json.dumps({'keyboard': keyboard,
'resize_keyboard': True,
'one_time_keyboard': True,
'selective': True})
def main_keyboard():
# Main menu.
return [first_button(), second_button()]
def one_line_keyboard():
# Menu with buttons in one line.
return [two_buttons()]
def first_button():
# Single keyboard button.
return ['first button']
def second_button():
# Single keyboard button.
return ['second button']
def two_buttons():
# Two buttons on one line.
return ['left button', 'right button']
while STATUS:
# Get updates forever. Except if get Error.
try:
if not OFFSET:
OFFSET = 1
# Print bot info on the start.
bot = get_me()
print('Bot #{} is running...'.format(bot.username))
updates = get_updates()
for update in updates.result:
# Handle last update.
OFFSET = update['update_id'] + 1
message, user, chat = handle(update)
# Greeting user by full name.
greeting = 'Hello, {} {}!'.format(user.first_name, user.last_name)
#send_message(chat.id, greeting)
send_keyboard(chat.id, greeting, one_line_keyboard())
except:
STATUS = False
print('\nERROR:\t', error()[1])
###############################################################################
you can this code: I hope useful for you .
i change :
params = ({'chat_id': chat_id, 'reply_markup': reply_markup, 'disable_web_page_preview': 'true'})
myresponse =urlopen(URL + 'sendMessage' + quote((params).encode('utf-8')))
with:
params = ({'text': 'ss', 'chat_id': chat_id, 'reply_markup': reply_markup, 'disable_web_page_preview': 'true'})
data = urllib.parse.urlencode(params).encode("utf-8")
myresponse = urlopen(URL + 'sendMessage', data)
complate code :
import json
import urllib
from urllib.parse import quote
from urllib.request import urlopen
def Decode(telegram_response):
decoded = ''
for line in telegram_response:
decoded = decoded + line.decode('utf-8')
return decoded
TOKEN = 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
URL = 'https://api.telegram.org/bot{}/'.format(TOKEN)
cmd = 'getme'
telegram_response = urlopen(URL + cmd)
decoded = Decode(telegram_response)
gtm = json.loads(decoded)
status = True
while status:
cmd = 'getUpdates'
telegram_response = urlopen(URL + cmd)
decoded = Decode(telegram_response)
upds = json.loads(decoded)
new_message = len(upds['result'])
if new_message != 0:
msg = upds['result'][0]['message']
chat_id = str(msg['chat']['id'])
reply_markup = {'keyboard': [[{'text': 'first button'}], [{'text': 'second button'}]], 'resize_keyboard': True,
'one_time_keyboard': True}
reply_markup = json.dumps(reply_markup)
params = ({'text': 'ss', 'chat_id': chat_id, 'reply_markup': reply_markup, 'disable_web_page_preview': 'true'})
data = urllib.parse.urlencode(params).encode("utf-8")
myresponse = urlopen(URL + 'sendMessage', data)

Resources