BeautifulSoup - trying to parse website but unsure how to parse json script? - python-3.x

Question

As a fun side project I have been trying to parse a website for a random fact of the day.
I decided to try my hand at this today with BeautifulSoup4 and urllib3. Sadly however I am unsure how to dive any deeper into a script element than I already have gotten.
This is my current level of output:
{
"#context": "http://schema.org",
"#type": "Article",
"headline": "Fact of the Day: 51 Facts Of the Day for 3/19/2019 ←FACTSlides→",
"image": "https://www.FACTSlides.com/imgs/ishots/8224.png",
"author": "Luke Reiner",
"genre": "facts",
"publisher": {
"#type": "Organization",
"name": "FACTSlides",
"logo": {
"#type": "ImageObject",
"url": "https:\/\/www.factslides.com\/imgs\/logo.png"
}
},
"url": "https://www.factslides.com/s-Fact-Of-The-Day",
"mainEntityOfPage": "https://www.factslides.com/s-Fact-Of-The-Day",
"datePublished": "2019-03-19",
"dateCreated": "2019-03-19",
"dateModified": "2019-03-19",
"description": "Description.",
"articleBody": "Article clutter here."
}
The facts themselves are stored under the articleBody and are not delimited, I was going to use '. ' as the Delimiter if I got that far.
This is the code I have so far:
""" Get a random fact. """
import argparse
import json
import urllib3
from bs4 import BeautifulSoup
PARAMETERS = {
"u": ["url", "passes in a url.", "1"],
}
PARSER = argparse.ArgumentParser(
description="Arguments to parse a url."
)
HTTP = urllib3.PoolManager()
def __load_args(parser, cfg_list):
""" Loads the passed arguments. """
for cfg_key in cfg_list:
if len(cfg_list[cfg_key]) > 3:
parser.add_argument(
"-" + cfg_key,
"--" + cfg_list[cfg_key][0],
help=cfg_list[cfg_key][1],
action=cfg_list[cfg_key][2],
nargs=cfg_list[cfg_key][3],
)
else:
parser.add_argument(
"-" + cfg_key,
"--" + cfg_list[cfg_key][0],
default=None,
help=cfg_list[cfg_key][1],
)
def parse_args(parser, section_list=[]):
""" Parses the loaded arguments. """
for section in section_list:
__load_args(parser, section)
return parser.parse_args()
ARGS = parse_args(PARSER, [PARAMETERS])
RESPONSE = HTTP.request('GET', ARGS.url)
SOUP = BeautifulSoup(RESPONSE.data, features="html.parser")
SOUP_SCRIPT = SOUP.find_all("script")
JS_TEXT = SOUP.find('script', type='application/ld+json').text
print(JS_TEXT)
Any help would be appreciated.
NOTE: The url I was parsing for the facts is here.

Answer 1

As long as your json is text/string, you can use json.loads() to read that in:
import json
JS_TEXT = '''{
"#context": "http://schema.org",
"#type": "Article",
"headline": "Fact of the Day: 51 Facts Of the Day for 3/19/2019 ←FACTSlides→",
"image": "https://www.FACTSlides.com/imgs/ishots/8224.png",
"author": "Luke Reiner",
"genre": "facts",
"publisher": {
"#type": "Organization",
"name": "FACTSlides",
"logo": {
"#type": "ImageObject",
"url": "https:\/\/www.factslides.com\/imgs\/logo.png"
}
},
"url": "https://www.factslides.com/s-Fact-Of-The-Day",
"mainEntityOfPage": "https://www.factslides.com/s-Fact-Of-The-Day",
"datePublished": "2019-03-19",
"dateCreated": "2019-03-19",
"dateModified": "2019-03-19",
"description": "Description.",
"articleBody": "Article clutter here."
}'''
jsonObj = json.loads(JS_TEXT)

Answer 2

I solved the issue with the help of #chitown88.
Here is the functioning code here:
""" Get a random fact. """
import argparse
import random
import json
from bs4 import BeautifulSoup
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
PARAMETERS = {
"u": ["url", "passes in a url.", "1"],
}
PARSER = argparse.ArgumentParser(
description="Arguments to parse a url."
)
HTTP = urllib3.PoolManager()
def __load_args(parser, cfg_list):
""" Loads the passed arguments. """
for cfg_key in cfg_list:
if len(cfg_list[cfg_key]) > 3:
parser.add_argument(
"-" + cfg_key,
"--" + cfg_list[cfg_key][0],
help=cfg_list[cfg_key][1],
action=cfg_list[cfg_key][2],
nargs=cfg_list[cfg_key][3],
)
else:
parser.add_argument(
"-" + cfg_key,
"--" + cfg_list[cfg_key][0],
default=None,
help=cfg_list[cfg_key][1],
)
def parse_args(parser, section_list=[]):
""" Parses the loaded arguments. """
for section in section_list:
__load_args(parser, section)
return parser.parse_args()
ARGS = parse_args(PARSER, [PARAMETERS])
RESPONSE = HTTP.request('GET', ARGS.url)
SOUP = BeautifulSoup(RESPONSE.data, features="html.parser")
SOUP_SCRIPT = SOUP.find_all("script")
JS_TEXT = SOUP.find('script', type='application/ld+json').text
JSON_OBJ = json.loads(JS_TEXT)
LIST_TEST = []
# print(JSON_OBJ['articleBody'])
for item in JSON_OBJ['articleBody'].split('. '):
LIST_TEST.append(item.strip())
print(random.choice(LIST_TEST) + ".")
I would like to note that my delimeter is not the best, as some of the 'facts' are spread across two sentences.