I am trying to directly get the data responses from Qualtrics directly into a pandas dataframe python. Is there a way of doing so?
import shutil
import os
import requests
import zipfile
import json
import io
# Setting user Parameters
# apiToken = "myKey"
# surveyId = "mySurveyID"
# fileFormat = "csv"
# dataCenter = "az1"
apiToken = "HfDjOn******"
surveyId = "SV_868******"
fileFormat = "csv"
dataCenter = 'uebs.eu'
# Setting static parameters
requestCheckProgress = 0
progressStatus = "in progress"
baseUrl = "https://{0}.qualtrics.com/API/v3/responseexports/".format(dataCenter)
headers = {
"content-type": "application/json",
"x-api-token": apiToken,
}
Then for # Step 1: Creating Data Export
downloadRequestUrl = baseUrl
then when i try to access the url from my chrom it gives me the following
{"meta":{"httpStatus":"404 - Not Found","error":{"errorMessage":"The requested resource does not exist."}}}
Which I believe the main reason why after running this code
# Step 1: Creating Data Export
downloadRequestUrl = baseUrl
downloadRequestPayload = '{"format":"' + fileFormat + '","surveyId":"' + surveyId + '"}'
downloadRequestResponse = requests.request("POST", downloadRequestUrl, data=downloadRequestPayload, headers=headers)
progressId = downloadRequestResponse.json()["result"]["id"]
print(downloadRequestResponse.text)
It gives me this error
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
<ipython-input-38-cd611e49879c> in <module>
3 downloadRequestPayload = '{"format":"' + fileFormat + '","surveyId":"' + surveyId + '"}'
4 downloadRequestResponse = requests.request("POST", downloadRequestUrl, data=downloadRequestPayload, headers=headers)
----> 5 progressId = downloadRequestResponse.json()["result"]["id"]
6 print(downloadRequestResponse.text)
KeyError: 'result
I am somehow new to Qualtrics/python interface may someone share why I am having this difficulty is it because of the dataCenter?
Thank you
Related
I have deployed a machine learning model as a pickle file in azure machine learning. The endpoint is created. Now, I am trying to consume the endpoint through the following codes:
import requests
import numpy as np
# send a random row from the test set to score
random_index = np.random.randint(0, len(X_test) - 1)
input_data = '{"data": [' + str(list(X_test[random_index])) + "]}"
headers = {"Content-Type": "application/json"}
resp = requests.post(service.scoring_uri, input_data, headers=headers)
print("POST to url", service.scoring_uri)
print("prediction:", resp.text)
It's giving error with following message:
prediction: {"data": "Expecting value: line 1 column 12 (char 11)", "message": "Failed to predict"}
The data looks like:
X_test => array([[[0. ], [0.274710], [0.403273]]])
'{"data": [' + str(list(X_test[random_index])) + "]}"
convert it to
'{"data": [[array([0.]), array([0.274710]), array([0.403273])]]}'
In the current code mentioned, it was mentioned as POST method. But to consume the endpoints, it was suggested to use the GET method.
import requests
import os
import base64
import json
personal_access_token = ":"+os.environ["AZ_ACCESS_TOKEN"]
headers = {}
headers['Content-type'] = "application/json"
headers['Authorization'] = b'Basic ' + base64.b64encode(personal_access_token.encode('utf-8'))
#Get a list of agent pools.
instance = "dev.azure.com/name"
propVals = "{name=Default,isHosted=false}"
api_version = "version _number”
uri = ("complete uri”)
r = requests.get(uri, headers=headers)
To get the complete URI, use the GET method with poolName embedded into the syntax
https://dev.azure.com/{organization}/_apis/distributedtask/pools?poolName={poolName}&api-version=5.1
If the case is like using the POST method itself, change the below line in the 6th line of the code.
input_data = '{"data": [“ + str(list(X_test[random_index])) + "]}’
The single and double quotations are misplaced.
import json
import requests
from os import makedirs
from os.path import join, exists
from datetime import date, timedelta
ARTICLES_DIR = join('tempdata', 'articles')
makedirs(ARTICLES_DIR, exist_ok=True)
API_ENDPOINT = 'http://content.guardianapis.com/search'
my_params = {
'q': 'coronavirus,stock,covid',
'sectionID': 'business',
'from-date': "2019-01-01",
'to-date': "2020-09-30",
'order-by': "newest",
'show-fields': 'all',
'page-size': 300,
'api-key': '### my cryptic key ###'
}
# day iteration from here:
# http://stackoverflow.com/questions/7274267/print-all-day-dates-between-two-dates
start_date = date(2019, 1, 1)
end_date = date(2020,9, 30)
dayrange = range((end_date - start_date).days + 1)
for daycount in dayrange:
dt = start_date + timedelta(days=daycount)
datestr = dt.strftime('%Y-%m-%d')
fname = join(ARTICLES_DIR, datestr + '.json')
if not exists(fname):
# then let's download it
print("Downloading", datestr)
all_results = []
my_params['from-date'] = datestr
my_params['to-date'] = datestr
current_page = 1
total_pages = 1
while current_page <= total_pages:
print("...page", current_page)
my_params['page'] = current_page
resp = requests.get(API_ENDPOINT, my_params)
data = resp.json()
all_results.extend(data['response']['results'])
# if there is more than one page
current_page += 1
total_pages = data['response']['pages']
with open(fname, 'w') as f:
print("Writing to", fname)
# re-serialize it for pretty indentation
f.write(json.dumps(all_results, indent=2))
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
<ipython-input-18-f04b4f0fe9ed> in <module>
49 resp = requests.get(API_ENDPOINT, my_params)
50 data = resp.json()
---> 51 all_results.extend(data['response']['results'])
52 # if there is more than one page
53 current_page += 1
KeyError: 'results'
Same error occurs for 'pages'
At first there was no issues and was able to run it. Download crashed after 2020-03-24. Since then can't get the code running again.
I'm referring to Line 51 and 54. At least at this point the codes crashes.
Not sure how to get rid of the issue. Any ideas?
Understanding the error message would be the first step - it compains about a missing key. Check if data['response']['results'] is present (hint: it is not) and check what exactly the structure of your data['response'] is.
Fortunately one can use the api parameter 'test' so we can help using that key:
my_params = {
'q': 'coronavirus,stock,covid',
'sectionID': 'business',
'from-date': "2019-01-01",
'to-date': "2020-09-30",
'order-by': "newest",
'show-fields': 'all',
'page-size': 300,
'api-key': 'test' # test key for that API
}
On running, I get the same exception, inspect data['response'] and get:
Lets see what parameters are given, shall we?
my_params = {
'q': 'coronavirus,stock,covid',
'sectionID': 'business',
'from-date': "2019-01-01",
'to-date': "2020-09-30",
'order-by': "newest",
'show-fields': 'all',
'page-size': 300, # TOO BIG
'api-key': 'test'
}
Fix that to 200 and you'll get
Downloading 2019-01-01
...page 1
Writing to tempdata\articles\2019-01-01.json
Downloading 2019-01-02
...page 1
Writing to tempdata\articles\2019-01-02.json
Downloading 2019-01-03
...page 1
Writing to tempdata\articles\2019-01-03.json
Downloading 2019-01-04
...page 1
Writing to tempdata\articles\2019-01-04.json
Downloading 2019-01-05
[snipp]
I am trying to use selenium with python to get latitude and longitude from this site. I am also using win32lipboard. But whenever I run my code, randomly it throws me this error pywintypes.error: (5, 'OpenClipboard', 'Access is denied.').
This is my code:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import csv
import win32clipboard
csvreader = csv.reader(open("master_data.csv", 'r'))
csvwriter = csv.writer(open('final_master_data.csv', 'w', newline=''))
headers = next(csvreader)
headers.append("latitude")
headers.append("longitude")
csvwriter.writerow(headers)
locations = list(csvreader)
chromedriver = 'C:\\Users\\UserName\\Downloads\\chromedriver.exe'
driver = webdriver.Chrome(chromedriver)
driver.get('http://www.whatsmygps.com')
for places in locations:
place = places[6] + ", " + places[4] + ", " + places[2]
location = driver.find_element_by_id("address")
location.send_keys(Keys.CONTROL, 'a')
location.send_keys(place)
location.submit()
time.sleep(3)
lat_input = driver.find_element_by_id("latitude")
lat_input.send_keys(Keys.CONTROL, 'a')
lat_input.send_keys(Keys.CONTROL, 'c')
win32clipboard.OpenClipboard()
lat = win32clipboard.GetClipboardData()
places.append(lat)
win32clipboard.CloseClipboard()
lon_input = driver.find_element_by_id("longitude")
lon_input.send_keys(Keys.CONTROL, 'a')
lon_input.send_keys(Keys.CONTROL, 'c')
win32clipboard.OpenClipboard()
lon = win32clipboard.GetClipboardData()
win32clipboard.CloseClipboard()
places.append(lon)
print(places)
csvwriter.writerow(places)
driver.close()
so, whenever I run this code, it starts with no issues, it reads csv file and enter location names into this sites and starts to copy latitude and longitude from the site and insert those into another csv file. But after some time, randomly, it throws error pywintypes.error: (5, 'OpenClipboard', 'Access is denied.'). I am unable to find the solution about this since yesterday.
UPDATE: I am using Anaconda and I am running anaconda shell as an administrator, so there is no issue with the access permission.
Access denied error may occur if clipboard is locked by another process. To avoid python messages, you can use WinAPI version of clipboard as described in this SO link: https://stackoverflow.com/a/23285159/4603670
As an alternative, use BingMap which requires API key. As of this writing, you can register a developer account at https://www.bingmapsportal.com for the free API key (I am not sure about the quota).
import pythoncom
import win32com.client
import json
pythoncom.CoInitialize()
winhttp = win32com.client.Dispatch('WinHTTP.WinHTTPRequest.5.1')
def bing_find_gps(addressLine, postalCode, country):
q = 'http://dev.virtualearth.net/REST/v1/Locations?key='
q = q + 'my_api_key'
if country: q = q + '&countryRegion=' + country
if postalCode: q = q + '&postalCode=' + postalCode
if addressLine: q = q + '&addressLine=' + addressLine
try:
winhttp.Open('GET', q, False)
winhttp.Send()
if not winhttp.responseText:
return 0
list = json.loads(winhttp.responseText)
if list['statusCode'] != 200:
return 0
gps = list['resourceSets'][0]['resources'][0]['point']['coordinates']
if gps:
return (1, gps[0], gps[1])
except:
return 0
res = bing_find_gps('One Microsoft Way, Redmond, WA, 98052-6399', '0', 'United States')
if res:
print("lat/long %s, %s" % (res[1], res[2]))
res = bing_find_gps(0, '98052-6399', 'United States')
if res:
print("lat/long %s, %s" % (res[1], res[2]))
Or use openstreetmap.org:
address = "98052-6399" #Testing with Microsoft zip code
url = "https://nominatim.openstreetmap.org/search?format=json&q=" + address
winhttp.Open('GET', url, False)
winhttp.Send()
list = json.loads(winhttp.responseText)
print(list[0].get('lat'))
print(list[0].get('lon'))
Expected output:
Latitude: 47.670119
Longitude: -122.118237
Or you may also wish to avoid copying the element altogether, use get_attribute('value') to read the value in latitude and longitude. Example:
chromedriver = 'C:\\Users\\UserName\\Downloads\\chromedriver.exe'
driver = webdriver.Chrome(chromedriver)
driver.get('http://www.whatsmygps.com')
element = driver.find_element_by_id("address")
element.send_keys(Keys.CONTROL, 'a')
#enter Microsoft's zip code
element.send_keys('98052-6399')
element.submit()
time.sleep(3)
lat_input = driver.find_element_by_id("latitude")
print('latitude: ')
print(lat_input.get_attribute('value'))
lon_input = driver.find_element_by_id("longitude")
print('longitude: ')
print(lon_input.get_attribute('value'))
driver.close()
Ik there are many questions like this but the answers are all specific and can only fix the solution for the persons specific script.
I am currently trying to print a bunch of info from supremenewyork.com
from the uk website. This script can succsesfully print all the info I want from supreme us but when I added the proxy script I starte to get alot of errors.
I know the prxy script works becuase I tested it on a small scipt and It was able to pull info that was on supreme uk and didnt exist on supreme us
Here is my script.
import requests
from bs4 import BeautifulSoup
UK_Proxy1 = raw_input('UK http Proxy1: ')
UK_Proxy2 = raw_input('UK http Proxy2: ')
proxies = {
'http': 'http://' + UK_Proxy1 + '',
'https': 'http://' + UK_Proxy2 + '',
}
categorys = ['jackets','shirts','tops_sweaters','sweatshirts','pants','shorts','t- shirts','hats','hats','bags','accessories','shoes','skate']
catNumb = 0
altArray = []
nameArray = []
styleArray = []
for cat in categorys:
catStr = str(categorys[catNumb])
cUrl = 'http://www.supremenewyork.com/shop/all/' + catStr
proxy_script = requests.get((cUrl.text), proxies=proxies)
bSoup = BeautifulSoup(proxy_script, 'lxml')
print('\n*******************"'+ catStr.upper() + '"*******************\n')
catNumb += 1
for item in bSoup.find_all('div', class_='inner-article'):
url = item.a['href']
alt = item.find('img')['alt']
req = requests.get('http://www.supremenewyork.com' + url)
item_soup = BeautifulSoup(req.text, 'lxml')
name = item_soup.find('h1', itemprop='name').text
style = item_soup.find('p', itemprop='model').text
print alt +(' --- ')+ name +(' --- ')+ style
altArray.append(alt)
nameArray.append(name)
styleArray.append(style)
print altArray
print nameArray
print styleArray
I am getting this error when I execute the script
AttributeError: 'str' object has no attribute 'text' with the error pointing towards the
proxy_script = requests.get((cUrl.text), proxies=proxies)
i recently added this to the script which sorta fixed it... It was able to print the category's but no info between them. Which (I NEED) it just printed ****************jackets**************, ****shirts******, etc.... here is what I changed
import requests
from bs4 import BeautifulSoup
# make sure proxy is http and port 8080
UK_Proxy1 = raw_input('UK http Proxy1: ')
UK_Proxy2 = raw_input('UK http Proxy2: ')
proxies = {
'http': 'http://' + UK_Proxy1 + '',
'https': 'http://' + UK_Proxy2 + '',
}
categorys = ['jackets','shirts','tops_sweaters','sweatshirts','pants','shorts','t-shirts','hats','bags','accessories','shoes','skate']
catNumb = 0
altArray = []
nameArray = []
styleArray = []
for cat in categorys:
catStr = str(categorys[catNumb])
cUrl = 'http://www.supremenewyork.com/shop/all/' + catStr
proxy_script = requests.get(cUrl, proxies=proxies).text
bSoup = BeautifulSoup(proxy_script, 'lxml')
print('\n*******************"'+ catStr.upper() + '"*******************\n')
catNumb += 1
for item in bSoup.find_all('div', class_='inner-article'):
url = item.a['href']
alt = item.find('img')['alt']
req = requests.get('http://www.supremenewyork.com' + url)
item_soup = BeautifulSoup(req.text, 'lxml')
name = item_soup.find('h1', itemprop='name').text
style = item_soup.find('p', itemprop='model').text
print alt +(' --- ')+ name +(' --- ')+ style
altArray.append(alt)
nameArray.append(name)
styleArray.append(style)
print altArray
print nameArray
print styleArray
I put .text at the end and it worked sorta.... How do i fix it so it prints the info I want???
I think you miss smt. Your cUrl is a string type, not request type. I guess you want:
proxy_script = requests.get(cUrl, proxies=proxies).text
I am trying to learn how to automatically fetch urls from a page. In the following code I am trying to get the port co-ordinates of the webpage from different links:
import urllib.request
import re
a = input("What country is your port in?: ")
b = input("What is the name of the port?: ")
url = "http://ports.com/"
totalurl = "http://ports.com/" + a + "/" + b + "/"
htmlfile = request.urlopen(url)
htmltext = htmlfile.read()
regex = '<span class="small'+ a + "/" + b + "/" '">...</span>'
pattern = re.compile(regex)
with urllib.request.urlopen(url) as response:
html = response.read().decode()
num = re.findall(pattern, html)
print(num)
This is the error message I receive:
What country is your port in?: greece
What is the name of the port?: port-of-eleusis
Traceback (most recent call last):
File "/Users/kieronblemings/Desktop/PROGRAMS PYTHON/ports extraction.py", line 13, in <module>
htmlfile = request.urlopen(url)
NameError: name 'request' is not defined