I have a sample.json file and I want to dump the data into a DynamoDB table but I keep getting this error - ValueError: Expected object or value.
Here is what I have done so far.
import boto3
import requests
import json
import re
import json_stream
import pandas as pd
from sys import getsizeof
from datetime import datetime
from botocore.exceptions import ClientError
from boto3.dynamodb.conditions import Key
from decimal import Decimal
dynamodb_client = boto3.resource("dynamodb", region_name="eu-central-1")
url_table = dynamodb_client.Table("URLConfig")
global_config = dynamodb_client.Table("GlobalConfigs")
chunks = pd.read_json("sample.json", lines=True, chunksize = 500)
for c in chunks:
c = c.to_dict(orient="records")
try:
with url_table.batch_writer() as batch:
for i in range(len(c)):
c[i]["confidence"] = Decimal(str(c[i]["confidence"]))
batch.put_item(Item=c[i])
except Exception as e:
print(e)
# records which generated exceptions must be collected in a file.
# as of now it is being ignored.
continue
I want to scrape tables of this persian pdf file and get the results as a pandas dataframe but I get error "NameError: name 'PDFResourceManager' is not defined" and no good content is extracted.
please help me to find a true encoded solution for it. Including your tested code is appreciated.
from pdfminer.converter import TextConverter
from io import StringIO
from io import open
from urllib.request import urlopen
import pdfminer as pm
urlpdf="https://www.codal.ir/Reports/DownloadFile.aspx?id=jck8NF9OtmFW6fpyefK09w%3d%3d"
response = requests.get(urlpdf, verify=False, timeout=5)
f=io.BytesIO(response.content)
def readPDF(f):
rsrcmgr=PDFResourceManager()
retstr=StringIO()
laparams=LAParams()
device=TextConverter(rsrcmgr,retstr,laparams=laparams)
process_pdf(rsrcmgr,device,pdfFile)
device.close()
content=retstr.getvalue()
retstr.close()
return content
pdfFile=urlopen(urlpdf)
outputString=readPDF(pdfFile)
proceedings=outputString.encode('utf-8') # creates a UTF-8 byte object
proceedings=str(proceedings) # creates string representation <- the source of your issue
file=open("extract.txt","w", encoding="utf-8") # encodes str to platform specific encoding.
file.write(proceedings)
file.close()
I have a pre-trained keras model which I have hosted on AWS using AWS SageMaker. I've got an endpoint and I can make successful predictions using the Amazon SageMaker Notebook instance.
What I do there is that I serve a .PNG image like the following and the model gives me correct prediction.
file= s3.Bucket(bucketname).download_file(filename_1, 'normal.png')
file_name_1='normal.png'
import sagemaker
from sagemaker.tensorflow.model import TensorFlowModel
endpoint = 'tensorflow-inference-0000-11-22-33-44-55-666' #endpoint
predictor=sagemaker.tensorflow.model.TensorFlowPredictor(endpoint, sagemaker_session)
data = np.array([resize(imread(file_name), (137, 310, 3))])
predictor.predict(data)
Now I wanted to make predictions using a mobile application. For that I have to wrote a Lambda function in python and attached an API gateway to it. My Lambda function is the following.
import os
import sys
CWD = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.join(CWD, "lib"))
import json
import base64
import boto3
import numpy as np
from scipy import signal
from scipy.signal import butter, lfilter
from scipy.io import wavfile
import scipy.signal as sps
import io
from io import BytesIO
import matplotlib.pylab as plt
from matplotlib import pyplot as plt
import matplotlib.image as mpimg
from datetime import datetime
from skimage.io import imread
from skimage.transform import resize
from PIL import Image
ENDPOINT_NAME = 'tensorflow-inference-0000-11-22-33-44-55-666'
runtime= boto3.client('runtime.sagemaker')
def lambda_handler(event, context):
s3 = boto3.client("s3")
# retrieving data from event.
get_file_content_from_postman = event["content"]
# decoding data.
decoded_file_name = base64.b64decode(get_file_content_from_postman)
image = Image.open(io.BytesIO(decoded_file_name))
data = np.array([resize(imread(image), (137, 310, 3))])
response = runtime.invoke_endpoint(EndpointName=ENDPOINT_NAME, ContentType='text/csv', Body=data)
result = json.loads(response['Body'].read().decode())
return result
The third last line is giving me error 'PngImageFile' object has no attribute 'read'.
Any idea what I am missing here?
If io.BytesIO(decoded_file_name) correctly represents your image data (though the name decoded_file_name suggests that its only file name, not actual image data), then you don't need to use PIL. Just use it directly:
data = np.array([resize(imread(io.BytesIO(decoded_file_name)), (137, 310, 3))])
I was missing one thing which was causing this error. After receiving the image data I used python list and then json.dump that list (of lists). Below is the code for reference.
import os
import sys
CWD = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.join(CWD, "lib"))
import json
import base64
import boto3
import numpy as np
import io
from io import BytesIO
from skimage.io import imread
from skimage.transform import resize
# grab environment variable of Lambda Function
ENDPOINT_NAME = os.environ['ENDPOINT_NAME']
runtime= boto3.client('runtime.sagemaker')
def lambda_handler(event, context):
s3 = boto3.client("s3")
# retrieving data from event.
get_file_content_from_postman = event["content"]
# decoding data.
decoded_file_name = base64.b64decode(get_file_content_from_postman)
data = np.array([resize(imread(io.BytesIO(decoded_file_name)), (137, 310, 3))])
payload = json.dumps(data.tolist())
response = runtime.invoke_endpoint(EndpointName=ENDPOINT_NAME, ContentType='application/json', Body=payload)
result = json.loads(response['Body'].read().decode())
return result
I used python3.7 in windows7.
When I tried to run this line: suinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
error occurs: module 'subprocess' has no attribute '_subprocess'
import os
import sqlite3
import subprocess
import time
import re
from django.core.mail import send_mail
from django.http import HttpResponse
suinfo = subprocess.STARTUPINFO()
suinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
How to deal with that?
There is no such thing as subprocess._subprocess, the constant is straight under subprocess:
suinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
See the docs: https://docs.python.org/3/library/subprocess.html#subprocess.STARTF_USESHOWWINDOW
I am writing a python code to receive the can data via a USB2CAN device. I receive the following error:
AttributeError: 'SerialBus' object has no attribute 'can'
from can.interfaces import serial
import random
import time
import datetime
import matplotlib.pyplot as plt
ser= can.interfaces.serial.serial_can.SerialBus('COM5',115200,timeout=None,rtscts=0)
for i in range(100) :
s= ser.can.interfaces.serial.SerialBus._recv_internal(timeout=None)
print(s)