the code can run and debug
but it shows"unresolved import 'requests'Python(unresolved-import)"
is there any other settings what i missed?
import requests
import os
url = 'https://www.google.com'
response = requests.get(url)
response.encoding = 'utf-8'
html = response.text
path = os.path.abspath(os.path.dirname(__file__))+'\\test.txt'
with open(path, 'w', encoding = 'gbk') as f:
f.write(html)
Related
I managed to get data from the target (see code below), but I want to continuously download as the log file on the server is appended to.
from importlib.resources import path
import requests
import json
import shutil
import logging
logging.captureWarnings(True)
url = "https://<IP address>/wga/reverseproxy_logging/instance/wga/request.log"
headers = {'Accept': 'application/json'}
r = requests.get(url, stream=True, headers=headers, verify=False, auth=('user','password'))
with open('request.log', 'wb') as out_file:
shutil.copyfileobj(r.raw, out_file)
I want to Crawl a Confluence Forum that requieres Authentication aswell as Authorization. After long research i somehow did manage to make a script that does exactly that but i´m kinda lost in how to implement Auth in front of the CrawlSpider including the Header with Auth-Token so the Spider is logged in and is able to Crawl all accessable Subsites.
Here´s my double Auth Script:
from urllib import response
from atlassian import Confluence
import subprocess
from urllib.parse import urlparse
import atlassian
from bs4 import BeautifulSoup
import getpass
import requests
import logging
from PyQt6.QtGui import *
from PyQt6.QtCore import *
from lxml import html
from selenium import webdriver
from selenium.webdriver.common.by import By
logging.basicConfig(filename='conf_connect.log', filemode='w', level=logging.DEBUG)
global url
url='http://MYCONFL.atlassian.net/wiki/'
confluence = Confluence(
url='http://MYCONFL.atlassian.net/wiki/',
username='USERNAME',
password='PASSWORD')
with requests.session() as s:
auth = ('USERNAME', getpass.getpass())
session = requests.session()
session.verify = True;
session.auth = auth;
headers = {
'Authorization': 'Basic base64encodedauthtoken',
'Content-Type': 'application/json'}
#storing response
response = requests.get(url)
if response.status_code != 200:
print("Error - Something went wrong!")
else:
print("all good, Code 200")
Here is the CrawlSpider´s Code:
import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from bs4 import BeautifulSoup
from urllib.parse import urlparse
import tldextract
url_list = [
'https://www.MYCONFL.atlassian.com/wiki/'
]
class CrawlySpider(CrawlSpider):
name = 'crawly'
allowed_domains = ['MYCONFL.atlassian.com']
start_urls = url_list
rules = (
# Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
Rule(LinkExtractor(), callback='parse_page', follow=True),
)
def parse_page(self, response):
html = response.body
soup = BeautifulSoup(html, 'lxml')
text = soup.get_text()
domain = tldextract.extract(response.request.url)[1]
path = urlparse(response.request.url)[2].replace("/", "")
with open(f'{domain}.txt', 'a') as fp:
fp.write(text)
with open(f'{domain} {path}.html', 'wb') as fp:
fp.write(response.body)
Thanks in Advance for Advice :D
I am trying to achieve this through a Python program. I am able to pull logs from API to a local drive.
However I am struggling to copy them to AWS S3. I appreciate your help on this.
I am using the code below to copy files to a local drive
''''''
import requests
import requests.auth
from bs4 import BeautifulSoup
import os.path
from pathlib import Path
import os
import boto3
CLIENT_ID = "xxxxxxxx"
CLIENT_SECRET = "xxxxxxxx"
TOKEN_URL = "https://xxxxxxxx/dwsso/oauth2/access_token"
REDIRECT_URI = "https://xxxxxxxx/dwsso/oauth2/callback"
def get_token(code=None):
client_auth = requests.auth.HTTPBasicAuth(CLIENT_ID, CLIENT_SECRET)
post_data = {"grant_type": "client_credentials",
"code": code,
"redirect_uri": REDIRECT_URI}
response = requests.post(TOKEN_URL,
auth=client_auth,
data=post_data)
token_json = response.json()
return token_json["access_token"]
my_token=get_token()
headersAPI = {
'accept': 'application/json',
'Authorization': 'Bearer '+ my_token,
}
params = (
('offset', '0'),
('limit', '20'),
)
def download_files(urls):
for a in soup.find_all('a', href=True):
url = urls
a = a['href']
logs_url = url + a
r = requests.get(logs_url, headers=headersAPI, params=params, verify=True, stream = True)
save_path = "download" + ((url.split('/')[2]).split('.')[0])
Path(save_path).mkdir(parents=True, exist_ok=True)
lname = (logs_url.split('/')[-1])
completeName= os.path.join(save_path, lname)
print("Downloding file from %s domain: %s" % ( save_path, lname )
open(completeName, 'wb').write(r.content)
url_lst = [ "https://xxxxxxxx/webdav/Sites/Logs/",
"https://xxxxxxxx/webdav/Sites/Logs/",
"https://xxxxxxxx/Sites/Logs/" ]
for i in url_lst:
response = requests.get(myUrl, headers=headersAPI, params=params, verify=True, stream = True)
soup = BeautifulSoup(response.content, 'html.parser')
f_url = ((i.split('/')[2]))
url = "https://" + f_url
download_files(url)
''''''
You can upload to S3 buckets using Boto3:
import boto3
s3 = boto3.resource('s3')
s3.Bucket('mybucket').upload_file('/tmp/hello.txt', 'hello.txt')
See Upload to S3 Bucket documentation here.
And here for installation and config of Boto3.
I am trying to send a compressed numpy array (compressed with zlib) to the flask server with post request, but the compressed bytes object is getting changed in the server end. How to properly send the bytes object with requests post request so that I can decompress on the server end?
server.py
from flask import Flask
from flask_restful import Resource, Api, reqparse
import json
import numpy as np
import base64
# compression
import zlib
app = Flask(__name__)
api = Api(app)
parser = reqparse.RequestParser()
parser.add_argument('imgb64')
class Predict(Resource):
def post(self):
data = parser.parse_args()
if data['imgb64'] == "":
return {
'data':'',
'message':'No file found',
'status':'error'
}
img = data['imgb64']
print('rec')
# decompress
print(type(img))
print(img)
dec = zlib.decompress(img) # this gives me error
if img:
pass
return json.dumps({
'data': 'done',
'message':'darknet processed',
'status':'success'
})
return {
'data':'',
'message':'Something when wrong',
'status':'error'
}
api.add_resource(Predict,'/predict')
if __name__ == '__main__':
app.run(debug=True, host = '0.0.0.0', port = 5000, threaded=True)
client.py
import numpy as np
import base64
import zlib
import requests
frame = np.random.randint(0,255,(5,5,3)) # dummy rgb image
# compress
data = zlib.compress(frame)
print('b64 encoded')
print(data)
print(len(data))
print(type(data))
r = requests.post("http://127.0.0.1:5000/predict", data={'imgb64' : data}) # sending compressed numpy array
This gives me the following error:
TypeError: a bytes-like object is required, not 'str'
So, I tried to convert the string to bytes object:
dec = zlib.decompress(img.encode()) # this gives me error
But, this one also gives me an error:
zlib.error: Error -3 while decompressing data: incorrect header check
I tried with other encodings, they also failed.
One thing I noticed is, when I print the compressed bytes in the client end, it reads:
b'x\x9c-\xcf?J\x82q\x00\x06\xe0\x0ftrVP\\C\xf4\x06\x9268\t\xcdR\x8ej.F\xa0\xe0\xd0\xa6\xa3\xe0V\x07\x10\x1cL\xc8\xd1\x03\xd4\xe4\t\x0c\x12\x84\xb6D\x0c#\xbc\x80O\xf0\x1b\x9e\xf5\xfdS\x89\xa2h\xcf\x9a\x03\xef\xc4\xf8cF\x92\r\xbf4i\x11g\xc83\x0f\x8c\xb9\xa2#\x8e\x1bn\xd91g\xc0\x91%\xd7\xdc\xf3M\x83<i:L\xa8\xf1\x19\xfa\xffw\xfd\xf0\xc5\x94:O\x9cH\x85\xcc6#\x1e\xc3\xf6\x05\xe5\xa0\xc7\x96\x04]J\\\x90\xa1\x1f~Ty\xe1\x8d\x15w|P\xe4\x95K\xb2!\xe3\x0cw)%I'
But on the server end, the received string is completely different:
�4ig�3���#�n�1g��%���M�<i:L����w��Ŕ:O�H��6#���ǖ]J\��~Ty�w|P�K�!�w)%I
I also tried to send the bytes as string, by
r = requests.post("http://127.0.0.1:5000/predict", data={'imgb64' : str(data)})
But, I can't decompress the data on the server end.
It seems, I can't send the zlib compressed bytes directly, so I used base64 to encode the data into ascii string.
So, in summary this worked for me, numpy array/any non-string data -> zlib compression -> base64 encode -> post request -> flask -> base64 decode -> zlib decompress
client.py
import numpy as np
import base64
import zlib
import requests
frame = np.random.randint(0,255,(5,5,3)) # dummy rgb image
# compress
data = zlib.compress(frame)
data = base64.b64encode(data)
data_send = data
data2 = base64.b64decode(data)
data2 = zlib.decompress(data2)
fdata = np.frombuffer(data2, dtype=np.uint8)
print(fdata)
r = requests.post("http://127.0.0.1:5000/predict", data={'imgb64' : data_send})
server.py
from flask import Flask
from flask_restful import Resource, Api, reqparse
import json
import numpy as np
import base64
# compression
import zlib
import codecs
app = Flask(__name__)
api = Api(app)
parser = reqparse.RequestParser()
parser.add_argument('imgb64', help = 'type error')
class Predict(Resource):
def post(self):
data = parser.parse_args()
#print(data)
if data['imgb64'] == "":
return {
'data':'',
'message':'No file found',
'status':'error'
}
#img = open(data['imgb64'], 'r').read() # doesn't work
img = data['imgb64']
data2 = img.encode()
data2 = base64.b64decode(data2)
data2 = zlib.decompress(data2)
fdata = np.frombuffer(data2, dtype=np.uint8)
print(fdata)
if img:
return json.dumps({
'data': 'done',
'message':'darknet processed',
'status':'success'
})
return {
'data':'',
'message':'Something when wrong',
'status':'error'
}
api.add_resource(Predict,'/predict')
if __name__ == '__main__':
app.run(debug=True, host = '0.0.0.0', port = 5000, threaded=True)
Here is the related code
import logging
logging.getLogger('googleapicliet.discovery_cache').setLevel(logging.ERROR)
import datetime
import json
from flask import Flask, render_template, request
from flask import make_response
from googleapiclient.discovery import build
from googleapiclient.http import MediaIoBaseDownload
from oauth2client.client import AccessTokenCredentials
...
#app.route('/callback_download')
def userselectioncallback_with_drive_api():
"""
Need to make it a background process
"""
logging.info("In download callback...")
code = request.args.get('code')
fileId = request.args.get('fileId')
logging.info("code %s", code)
logging.info("fileId %s", fileId)
credentials = AccessTokenCredentials(
code,
'flex-env/1.0')
http = httplib2.Http()
http_auth = credentials.authorize(http)
# Exports a Google Doc to the requested MIME type and returns the exported content. Please note that the exported content is limited to 10MB.
# v3 does not work? over quota?
drive_service = build('drive', 'v3', http=http_auth)
drive_request = drive_service.files().export(
fileId=fileId,
mimeType='application/pdf')
b = bytes()
fh = io.BytesIO(b)
downloader = MediaIoBaseDownload(fh, drive_request)
done = False
try:
while done is False:
status, done = downloader.next_chunk()
logging.log("Download %d%%.", int(status.progress() * 100))
except Exception as err:
logging.error(err)
logging.error(err.__class__)
response = make_response(fh.getbuffer())
response.headers['Content-Type'] = 'application/pdf'
response.headers['Content-Disposition'] = \
'inline; filename=%s.pdf' % 'yourfilename'
return response
It is based on some code example of drive api. I am trying to export some files from google drive to pdf format.
The exception comes from the line
response = make_response(fh.getbuffer())
It throws the exception:
TypeError: 'memoryview' object is not callable
How can I retrieve the pdf content properly from the fh? Do I need to further apply some base 64 encoding?
My local runtime is python 3.4.3
I have used an incorrect API. I should do this instead:
response = make_response(fh.getvalue())