I want to do some data analysis on a json file generated from twitter.
The field that I need is the text field which contain the tweet itself.
your contribution is highly appreciated
while tweetCount < maxTweets:
try:
if (max_id <= 0):
if (not sinceId):
new_tweets = api.search(q=search_query, count=tw_block_size)
else:
new_tweets = api.search(q=search_query, count=tw_block_size, since_id=sinceId)
else:
if (not sinceId):
new_tweets = api.search(q=search_query, count=tw_block_size, max_id=str(max_id - 1))
else:
new_tweets = api.search(q=search_query, count=tw_block_size, max_id=str(max_id - 1), since_id=sinceId)
if not new_tweets:
print("Collecte terminee.")
break
for tweet in new_tweets:
day = tweet.created_at.strftime('%Y-%m-%d')
with open( "%s/%s_tweets.json" % (output_dir, day), 'a') as f:
f.write(json.dumps(tweet._json))
f.write('\n')
tweetCount += len(new_tweets)
print("{0} tweets téléchargés".format(tweetCount))
max_id = new_tweets[-1].id
except tweepy.TweepError as e:
print("an error was occured to continue , run the following command:")
print("python collect.py -s {0} -o {1} -u {2}".format(search_query, output_dir, max_id))
print("")
print("Error : " + str(e))
break
Related
I'm writing a program to create a txt file and store data in the file. The program takes a persons height, weight, age, to calculate their BMI and Fat Burning Heart Rate. I have a general except statement that will be called if there is an error saving the information. Anyways the error is called and I can't figure out why the data won't write to the txt file.
choice = int(input("Select:\n1. Create a fle and add results\n2. Open a file and add results \n3. Read results from file \n4. Quit \n"))
while choice < 4:
BMICalc = []
if choice == 1:
try:
BMIcalculations = open('BMIcalculations.txt')
for n in BMIcalculations:
BMICalc.append(n.strip())
except FileNotFoundError as err:
print(err)
print('Creating file for BMI calculations')
BMIcalculations = open("BMIcalculations.txt", "x")
print("File created.")
BMIcalculations.close()
while True:
try:
age = int(input("Enter your age: "))
if age <= 0:
raise TypeError("Enter a number greater than zero")
break
except ValueError:
print("Invalid age. Must be a number.")
except TypeError as err:
print(err)
except:
print('Invalid input')
while True:
try:
height = float(input('Enter your height in inches: '))
if height <= 0:
raise TypeError("Enter a number greater than 0")
break
except ValueError:
print("Height must be a number.")
except TypeError as err:
print(err)
except:
print('Invalid input')
while True:
try:
weight = float(input("Enter your weight in pounds: "))
if weight <= 0:
raise TypeError("Enter a number greater than 0")
break
except ValueError:
print("Weight must be a number")
except TypeError as err:
print(err)
except:
print('Invalid value')
print('Age = ',age)
print('Height = ',height)
print('Weight = ',weight)
BMI = 703 * weight/pow(height,2)
heartRate = (220 - age)
heartRate1 = heartRate * .7
print("Fat Burning Heart Rate = ",round(heartRate1, 1),"bpm")
heartrate2 = round(heartRate1, 1)
print("Body Mass Index = ", round(BMI, 2))
FatBHR = "Fat Burning Heart Rate = ",round(heartRate1, 1),"bpm"
BMIList = "Body Mass Index = ", round(BMI, 2)
BMIList2 = round(BMI, 2)
BMICalc.append(BMIList2)
BMICalc.append(heartrate2)
try:
BMIcalculations = open('BMIcalculations.txt', 'w')
for n in BMICalc:
BMIcalculations.write(n + '\n')
except:
print('Error writing to file')
BMIcalculations.close()
My project is to compare file names located on a folder with names saved in a PostgreSQL database.
I looked into different responses on the SO but nothing worked for me.
Please see below my comments within the source code.
try
cur = db_conn().cursor()
report query = "SELECT ev_report FROM stc_events_master WHERE
si_station_num = %s AND" +
" ev_ins_date BETWEEN to_date(" + "'" + "1/1/2018" + "'" + ",'mm/dd/yyyy')" + "
and TO_DATE('12/31/2018', 'mm/dd/yyyy')"
# files_dict content-> k = '123p34" , value = "DV03S_120124_5-7-2018.pdf"
for k, v in files_dict.items():
cur.execute(report_query, (k,))
# the reports are unique in the database table
query_result = cur.fetchmany(1) #incoming value:DV03S_120124_5-7-2018.pdf
# query returns:
sta = str(query_result[0])
# if a file is not found in the database, the exception happens
# and the code below is not executed. It jumps
# directly to Exception
if len(sta) < 1:
print("no report found in the database")
print("query_results" + sta)
else:
if v.lower() == q_ry.lower():
print("match " + v)
else:
print("no match " + v)
except Exception:
pass
print("In Exception area")
finally:
cur.close()
I want to continue the loop even if nothing is found the database and save a name of the file from the folder which caused the exception. Now, the code stops after the exception.
Thank you for your help.
for k, v in files_dict.items():
cur.execute(report_query, (k,v))
if cur.rowcount == 0:
print("The file is not in the database: " + v)
else:
query_result = cur.fetchmany(3)
sta = str(query_result[0])
file_name_length = len(sta)
q_ry = sta[2: file_name_length-3]
if v.lower() == q_ry.lower():
pass
else:
print("no match after comparing names" + v)
except (Exception, psycopg2.Error) as error:
print("Error fetching data from PG table " + k, error)
finally:
cur.close()
I have a folder in which there are some video files. I want to extract frames from the videos but only those videos should be processed whose names are not present in the csv. It should check for the present video file names in the csv before processing the videos
def extractFrames(m):
global vid_name
vid_files=glob(m)
print(vid_files)
complete_videos = get_completed_videos()
print(complete_videos)
new_vid_files = [x for x in vid_files if get_vid_name(x) not in complete_videos]
for vid in new_vid_files:
print("path of video========>>>>.",vid)
v1=os.path.basename(vid)
try:
vid_name = get_vid_name(vid)
vidcap = cv2.VideoCapture(vid)
except cv2.error as e:
print(e)
except:
print('error')
#condition
fsize=os.stat(vid)
print('=============size of video ===================:' , fsize.st_size)
try:
if (fsize.st_size > 1000):
fps = vidcap.get(cv2.CAP_PROP_FPS) # OpenCV2 version 2 used "CV_CAP_PROP_FPS"
frameCount = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
duration = frameCount/fps
minutes = int(duration/60)
print('fps = ' + str(fps))
print('number of frames = ' + str(frameCount))
print('duration (S) = ' + str(duration))
if (duration > 1):
success,image = vidcap.read()
count=0
success=True
while success:
img_name = vid_name + '_f' + str(count) + ".jpg"
success,image = vidcap.read()
if count % 10 == 0 or count ==0:
target_non_target(img_name, image)
count+=1
vidcap.release()
cv2.destroyAllWindows()
except:
print("error")
print('finished processing video ',vid)
with open("C:\\multi_cat_3\\models\\research\\object_detection\\my_imgs\\"+'video_info.csv', 'a') as csv_file:
fieldnames = ['Video_Name','Process']
file_is_empty = os.stat("C:\\multi_cat_3\\models\\research\\object_detection\\my_imgs\\"+'video_info.csv').st_size == 0
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
if file_is_empty:
writer.writeheader()
writer.writerow({'Video_Name':vid_name,'Process':'done'})
def get_vid_name(vid):
return os.path.splitext(os.path.basename(vid))[0]
def get_completed_videos():
completed_videos = []
with open("C:\\multi_cat_3\\models\\research\\object_detection\\my_imgs\\video_info.csv") as csv_file:
for row in csv.reader(csv_file):
for col in range(0,len(row)):
try:
completed_videos.append(row[col])
except Exception as e:
print(str(e))
print(completed_videos[0])
return completed_videos
Suppose there are 3 videos in a folder. Code is successfully run for those 3 videos and their names are written in the csv. Now if i paste video number 4 in the folder then it should process only the 4th video after checking for the video names present in the csv. Currently it is repeatedly processing all the video files everytime the script is run.
First off, inside the for loop
v1=os.path.basename(vid_files[v_f])
Should be
v1=os.path.basename(new_vid_files[v_f])
Since you are looping over the new_vid_files range. Using those indices on the original list will give you unexpected items. Better yet, you can directly use a for-each loop (since you don't seem to be using v_f for anything other than list access) as follows:
for vid in new_vid_files:
And this vid would replace all instances of new_vid_files[v_f].
Next, you are using vid_name to write to the csv, so you need to perform the same operation for each item from vid_files before matching against complete_videos while creating the new_vid_files list.
If you create a method for getting the video name as follows:
def get_vid_name(vid_file):
return os.path.splitext(os.path.basename(vid_file))[0]
Then you can change the list comprehension to be
new_vid_files = [x for x in vid_files if get_vid_name(x) not in complete_videos]
Edit: As mentioned in the comments to the other answer, the output for complete_videos indicates it isn't being parsed properly. It is appending both the column headers and other unneeded columns. This code will work despite that, but it needs to be fixed. I am not solving it because it is a relatively simple change, and I want the OP to understand what they're doing wrong.
def extractFrames(m):
global vid_name
vid_files=glob(m)
print(vid_files)
complete_videos = get_completed_videos()
new_vid_files = [x for x in vid_files if get_vid_name(x) not in complete_videos]
for vid in new_vid_files:
print("path of video========>>>>.",vid)
v1=os.path.basename(vid)
try:
vid_name = get_vid_name(vid)
vidcap = cv2.VideoCapture(vid)
except cv2.error as e:
print(e)
except:
print('error')
#condition
fsize=os.stat(vid)
print('=============size of video ===================:' , fsize.st_size)
try:
if (fsize.st_size > 1000):
fps = vidcap.get(cv2.CAP_PROP_FPS) # OpenCV2 version 2 used "CV_CAP_PROP_FPS"
frameCount = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
duration = frameCount/fps
minutes = int(duration/60)
print('fps = ' + str(fps))
print('number of frames = ' + str(frameCount))
print('duration (S) = ' + str(duration))
if (duration > 1):
success,image = vidcap.read()
count=0
success=True
while success:
img_name = vid_name + '_f' + str(count) + ".jpg"
success,image = vidcap.read()
if count % 10 == 0 or count ==0:
target_non_target(img_name, image)
count+=1
vidcap.release()
cv2.destroyAllWindows()
except:
print("error")
print('finished processing video ',vid)
with open("C:\\multi_cat_3\\models\\research\\object_detection\\my_imgs\\"+'video_info.csv', 'a') as csv_file:
fieldnames = ['Video_Name','Process']
file_is_empty = os.stat("C:\\multi_cat_3\\models\\research\\object_detection\\my_imgs\\"+'video_info.csv').st_size == 0
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
if file_is_empty:
writer.writeheader()
writer.writerow({'Video_Name':vid,'Process':'done'})
def get_vid_name(vid):
return os.path.splitext(os.path.basename(vid))[0]
def get_completed_videos():
completed_videos = []
with open("C:\\multi_cat_3\\models\\research\\object_detection\\my_imgs\\video_info.csv") as csv_file:
for row in csv.reader(csv_file):
for col in range(0,len(row)):
try:
completed_videos.append(row[col])
except Exception as e:
print(str(e))
print(completed_videos[0])
return completed_videos
The number of "*" printed fluctuates according to the length of string entered
def main():
# check command line argument
if len(argv) != 2:
print("Usage: python bleep.py dictionary")
exit(1)
else:
ban = set()
# Load txt file
with open(argv[1], "r") as f:
for line in f:
# strip the space and add to set
ban.add(line.strip())
# prompt user input
input = get_string("What message would you like to censor?\n")
# Split input into word tokens
token = input.split()
censored_msg = ""
for i in token:
if i.lower() in ban:
censored_msg = (censored_msg + "*"*(len(token)+1) + " ")
else:
censored_msg += i + " "
# print censored message
print(censored_msg.strip())
if __name__ == "__main__":
main()
It prints fine for some cases, such as
Input: heck my gosh
Output: **** my ****
But not so in others (should be **** ****)
Input: heck gosh
Output: * * (just 6 * for 8 letters)
Is it a typo? Check this line very carefully
censored_msg = (censored_msg + "*"*(len(token)+1) + " ")
and remember which is what here for i in token:
I am a noob in python and i need help.I have made a phonebook where you can add the contacts.But the problem is that when i exit the program the changes to the list are not saved.I want the user to be able to make permanent changes to the list.I have seen posts about a file=open("something",'w') code to do this(I think) but i dont know where to insert this code and i dont really understand what it is.Could someone help me understand what this is about..Here is the full code:
name = ["ranga","hari"]
number = [9895497777,9]
book = {name[0]:number[0],name[1]:number[1]}
def search():
print("Contacts:")
for x in book:
print(x,':',book[x])
while 1:
count = 0
a = 0
ch1 = input("search: ")
try:
ch1 = int(ch1)
except ValueError:
while a < len(name):
result = name[a].find(ch1)
if result == -1:
a = a + 1
else:
print(name[a],number[a])
a = a + 1
count = count + 1
if count == 0:
print("Not available.Try again")
continue
else:
break
ch1 = str(ch1)
while a < len(number):
sumber = str(number[a])
result = sumber.find(ch1)
if result == -1:
a = a + 1
else:
print(name[a],number[a])
a = a + 1
count += 1
if count == 0:
print("Not available.try again")
continue
else:
break
def add():
print("What is the name of the contact you want to add?")
name1 = input()
name.append(name1)
while 1:
print("What is the number of this contact?")
number1 = input()
try:
number1 = int(number1)
except ValueError:
print("Please type a number..")
continue
number.append(number1)
book[name1] = number1
break
def remoe():
print("Reference:")
for x in book:
print(x,':',book[x])
while 1:
print("What is the name of the contact you want to remove?")
name2 = input()
if name2 in book:
increment = name.index(name2)
name.pop(increment)
number.pop(increment)
del book[name2]
break
else:
print("Not available.Please try again")
while 1:
print("Contacts:")
for x in book:
print(x, ':', book[x])
print("\nWhat do you want to do?\n1.Search for a person\n2.edit the phone book\n3.exit")
choice = input()
try:
choice = int(choice)
except ValueError:
print("Type 1,2 or 3")
continue
if choice == 1:
search()
elif choice == 2:
while 1:
print("Do you want to:\n1.Add a contact\n2.Remove a contact\n3.Go back to main menu")
ch2 = input()
if ch2 in['3']:
break
else:
try:
ch2 = int(ch2)
except ValueError:
print("Type 1 or 2..")
if ch2 == 1:
add()
elif ch2 == 2:
remoe()
elif choice == 3:
exit()
else:
print("Type 1,2 or 3")
I appreciate the help.
When you choose to add a contact, it does properly add the name and number to the list. But, that is it.
When you re-run the program, the list gets re-assigned due to the first 2 lines of your code:
name = ["ranga","hari"]
number = [9895497777,9]
So, you won't see the last changes.
This is where you should maintain a file which lives outside the scope of your code, rather than a list.
You can modify your add function like this:
def add():
print("What is the name of the contact you want to add?")
name1 = input()
#name.append(name1)
# Just add the name1 variable's value to the file
with open('contacts_list.txt', 'a+') as f:
f.write(name1 + '\n')
while 1:
print("What is the number of this contact?")
number1 = input()
try:
number1 = int(number1)
except ValueError:
print("Please type a number..")
continue
#number.append(number1)
# Similarly, append the number1 variable's value to file again.
with open('contacts_list.txt', 'w+') as f:
f.write(number1)
#book[name1] = number1
with open('contacts_list.txt', 'r') as f:
print(f.read())
break
Note: You would also need to change the other functions search and remove to read and write from the file. I've just given you a taste of how things are done. You need to modify your code and make it work.
Let me know if it helps.
I took your advice and made a new text file but i still did not know how to do it but after reading ur answers i understood and at last i came to this..
removelist = []
def search():
while 1:
search = str(input("Search: "))
if search not in["exit", "Exit"]:
with open('output.txt', 'r+') as f:
line = f.readline()
while line:
data = line.find(search)
if not data == -1:
print(line.rstrip('\n'))
line = f.readline()
else:
line = f.readline()
else:
break
f.close()
def add():
print("Type the name of the contact:")
name = input()
while 1:
print("Type the number of this contact:")
number = input()
try:
number = int(number)
except ValueError:
print("Please type a number")
continue
number = str(number)
with open('output.txt', 'a+') as f:
f.write('\n' + name +' ' + number)
break
def remoe(): #this is where the problem comes in
while 1:
remove = str(input("Remove: "))
with open('output.txt', 'r+') as f:
line = f.readline()
while line:
if not remove in["Remove", "remove"]:
removelist.clear()
data = line.find(remove)
if not data == -1:
removelist.append(line) #This saves all the lines coming from the search to a
print(removelist) #removelist which can be accessed when you type in remove
line = f.readline() #But the problem is that if there is a \n at the end of the
else: #string then the remove function does not work
line = f.readline()
else:
print(removelist)
with open('output.txt', 'r') as f:
d = f.readlines()
f.close()
with open('output.txt', 'w') as f:
for i in d:
if i not in removelist:
f.write(i)
f.truncate()
f.close()
break
while 1:
with open('output.txt', 'r') as f:
data = f.read()
print("Contacts:")
print(data)
print('''What do you want to do?
1.Search for a contact
2.Edit contacts
3.Exit''')
f.close()
choice = input()
if choice in["1"]:
search()
elif choice in["2"]:
while 1:
print('''What do you wanna do:
1.Add a contact
2.Remove a contact
3.Exit to main menu''')
ch1 = input()
if ch1 in["1"]:
add()
elif ch1 in["2"]:
remoe()
elif ch1 in["3"]:
break
else:
print("Please type 1,2 or 3")
elif choice in[3]:
print("Ok bye")
else:
print("Please type 1,2 or 3")
Now the problem seems to be the remove function..if i try to remove a line with \n at the end of it then it wont work while the opp. seems to work.Any guess what i am doing here?
And thanks for the help Mayank porwal
At the first you should know name = ["ranga","hari"], number = [9895497777,9] that you have defined are in the code and you can not change those value, and after exit() they will reset to default value.
you should use of file (for example .txt file) in this issue:
1. you must create a .txt file in your project (for example Contacts.txt)
2. and write your information in there (for example in first line: Kourosh +98938....)
3. at the first step in your program you must read Contact.txt and load it in a structure like a list or dictionary (for example
>>> with open('workfile') as f:
... read_data = f.read()
>>> f.closed
)
4.now you can edit, add, remove structure.
5.and finally you can write structure in the file, before exit()
for example:
>>> with open('workfile') as f:
... f.write(s)
>>> f.closed