I have this function where the price of a stock gets logged in real time every 2 seconds and save it into a csv file however I cant see anything in the csv when I open it. What am I missing from the script?
import pandas as pd
import time
import urllib
import sys
import fix_yahoo_finance as yf
def stocks():
# Enter stock symbol
stock = input("Enter stock: ")
# Name CSV file
csvy= str(stock) + time.strftime('.%A.%d.%b.%Y').replace(' ', '') + ".csv"
csvy = csvy.replace(':' , '')
with open(csvy, 'w') as f:
sys.stdout = f
while 1 > 0:
print(yf.get_live_price(stock))
time.sleep(2)
stocks()
You wrote:
print(yf.get_live_price(stock))
You want to additionally flush the buffer so your new text is immediately visible:
print(yf.get_live_price(stock), flush=True)
Alternatively, consider assigning the live price to a temp variable,
and then outputting it twice, with print() and f.write(),
rather than assigning a new value to stdout.
Then you'd be able to flush them independently according to your need,
f.flush() or sys.stdout.flush().
Related
I have to take the first line of the csv file and after having processed it, delete it, and then resume the line after.
I'm trying to build a login system that takes the accounts from the csv file, and logs in one by one.
the problem is that every time you start the loop it always takes the same account, how can I fix it?
import pandas as pd
import pyperclip
import selenium
import random
from selenium import webdriver
import undetected_chromedriver as uc
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.common.keys import Keys
import names
df = pd.read_csv('/Users/giuseppeleonardi/Downloads/scraping2.csv')
def instagram_login():
df2=df.at[0,'ID'] #Find the first row id
pyperclip.copy(df2) #Copy the first row id to the clipboard
print(pyperclip.paste()) #Print the first row id
#apro il sito
driver.get('https://www.instagram.com/')
driver.maximize_window() #schermo intero
time.sleep(2)
try:
consent= driver.find_element(By.XPATH,"/html/body/div[2]/div/div/div/div[2]/div/div/div[1]/div/div[2]/div/div/div/div/div[2]/div/button[2]").click() #clicco il consenso
except:
pass
time.sleep(5)
put_username = driver.find_element(By.NAME,("username")).send_keys(pyperclip.paste()) #inserisco username
df2=df.at[0,'PASSWORD'] #find the password
pyperclip.copy(df2) #copy the password
put_password = driver.find_element(By.NAME,("password")).send_keys(pyperclip.paste()) #inserisco password
time.sleep(2)
login = driver.find_element(By.XPATH,"//div[contains(text(),'Accedi')]").click() #Click login
time.sleep(6)
#here is where the first row got deleted and saved on csv
df= pd.read_csv('/Users/giuseppeleonardi/Downloads/scraping2.csv').drop(0, axis=0)
df.to_csv(r'/Users/giuseppeleonardi/Downloads/scraping2.csv', index=False)
#this is the loop that always takes the same line of the file every time even though this is canceled at the end of the operation:
for line in len(df):
instagram_login()
time.sleep(5)
driver.delete_all_cookies()
i've googled a lot but cannot figure it out, i've read that file handle will read the file once, i need the loop for reset the list everytime and take the first value, how can i do it?
sorry but i'm still learning
Google for local and global variables. You are changing df inside a function. This does not change the 'global' df. You either need to return your df from the function or declare it first as a global variable.
First option:
df = pd.read_csv('/Users/giuseppeleonardi/Downloads/scraping2.csv')
def instagram_login():
df2=df.at[0,'ID'] #Find the first row id
.....
#here is where the first row got deleted and saved on csv
df= pd.read_csv('/Users/giuseppeleonardi/Downloads/scraping2.csv').drop(0, axis=0)
df.to_csv(r'/Users/giuseppeleonardi/Downloads/scraping2.csv', index=False)
return df
for line in len(df):
df = instagram_login()
time.sleep(5)
driver.delete_all_cookies()
Second option:
df = pd.read_csv('/Users/giuseppeleonardi/Downloads/scraping2.csv')
def instagram_login():
df2=df.at[0,'ID'] #Find the first row id
.....
#here is where the first row got deleted and saved on csv
global df
df = pd.read_csv('/Users/giuseppeleonardi/Downloads/scraping2.csv').drop(0, axis=0)
df.to_csv(r'/Users/giuseppeleonardi/Downloads/scraping2.csv', index=False)
for line in len(df):
instagram_login()
time.sleep(5)
driver.delete_all_cookies()
Your definition of df inside the function doesn't change the outside df.
So you can return the df and save it to outside df.
data_frame= pd.read_csv('/Users/giuseppeleonardi/Downloads/scraping2.csv')
def instagram_login(df):
......
#here is where the first row got deleted and saved on csv
df= pd.read_csv('/Users/giuseppeleonardi/Downloads/scraping2.csv').drop(0, axis=0)
df.to_csv(r'/Users/giuseppeleonardi/Downloads/scraping2.csv', index=False)
return df
#this is the loop that always takes the same line of the file every time even though this is canceled at the end of the operation:
for line in len(df):
data_frame = instagram_login(data_frame)
time.sleep(5)
driver.delete_all_cookies()
EEG device is OpenBCI and board is Cyton and daisy.My sample code is here:
from datetime import datetime
import argparse
import numpy as np
import pandas as pd
import csv
import mne
from brainflow.board_shim import BoardShim, BrainFlowInputParams, LogLevels, BoardIds
from brainflow.data_filter import DataFilter
import time
def main():
BoardShim.enable_dev_board_logger()
# use synthetic board for demo
params = BrainFlowInputParams()
board = BoardShim(BoardIds.SYNTHETIC_BOARD.value, params)
board.prepare_session()
board.start_stream()
now = datetime.now()
current_time = now.strftime("%H:%M:%S")
BoardShim.log_message(LogLevels.LEVEL_INFO.value, 'start sleeping in the main thread')
time.sleep(10)
data = board.get_board_data()
board.stop_stream()
board.release_session()
eeg_channels = BoardShim.get_eeg_channels(BoardIds.SYNTHETIC_BOARD.value)
# demo for downsampling, it just aggregates data
for count, channel in enumerate(eeg_channels):
print('Original data for channel %d:' % channel)
print(data[channel])
print("Current Time =", current_time)
# demo how to convert it to pandas DF and plot data
# eeg_channels = BoardShim.get_eeg_channels(BoardIds.SYNTHETIC_BOARD.value)
# df = pd.DataFrame(np.transpose(data))
# print('Data From the Board')
# print(df.head(10))
# demo for data serialization using brainflow API, we recommend to use it instead
pandas.to_csv()
DataFilter.write_file(data, 'test.csv', 'w') # use 'a' for append mode
# writing the data in csv file
restored_data = DataFilter.read_file('test.csv')
restored_df = pd.DataFrame(np.transpose(restored_data))
print('Data From the File')
print(restored_df.head(5))
if __name__ == "__main__":
main()
In this program i am not using request or beautiful soup function. I'm instead only using the datetime to extract the URLs. Now in the current program, I have written to extract the values for a long period. I want to make it in such a way that, if I automate this program and it runs today, it will extract yesterday's data. Similarly if it runs tomorrow, it will extract todays data and so on.
here is the code,
import datetime
from datetime import date, datetime,timedelta
import warnings
import datetime
import pandas as pd
import wget
import glob
import os
warnings.filterwarnings("ignore")
import ssl
ssl._create_default_https_context = ssl._create_unverified_context
from urllib.error import HTTPError
def date_range(start_date,end_date):
for n in range(int((end_date-start_date).days)):
yield start_date + timedelta(n)
def get_urls(base_url):
part_two = "/dailyCoal1-"
end_part = ".xlsx"
start_date = date(2020,11,1)
end_date = datetime.datetime.now().date()
start_urls = list()
for single_date in date_range(start_date, end_date):
start_urls.append(single_date.strftime(base_url+'%d-%m-%Y'+part_two+'%Y-%m-%d'+end_part))
return start_urls
def excel_download(link,out):
#downloads a given link provided to a output directory in out
wget.download(link,out)
if __name__ =="__main__":
base_url = "https://npp.gov.in/public-reports/cea/daily/fuel/"
mypath = "/Users/vp/Desktop/temp"
temp_folder = '/Users/vp/Desktop/temp'
out_folder = "/Users/vp/Desktop/NPP"
log_file = os.path.join(out_folder,'debug_log_npp.log')
out_file = os.path.join(out_folder,'Energy_inputs_npp.csv')
file_links = get_urls(base_url)
for link in file_links:
try:
excel_download(link,temp_folder)
except HTTPError:
content = "HTTP issue while capturing data for this link - " + link
log_writer(log_file,content)
continue
file = glob.glob(os.path.join(temp_folder,'*.xlsx'),recursive=True)[0]
df = pd.read_excel(file)
To capture yesterday's data, i created this in the main function where i check for yesterday = and then cancel if it isnt yesterday. But then its throwing error as it constantly picks the start date as its day one.
if(date_time_obj != Yesterday):
os.remove(file)
content = "Date mis-matched - " + str(date_time_obj) + " " + str(Yesterday)
In this program, date_time_obj - is the date it is currently trying to extract data for.
Everyday if this program runs at 8pm, it needs to only capture one day before data on a daily basis.
if this cannot be done in datetime, but only on request or bs4, then how do i approach this problem?
I don't know if you wanted a valid link as your code doesn't seem to produce those for me but you only need to tweak to work off start_date only and return a single item to return yesterday's link matching with your current output for same date.
import datetime
from datetime import date, datetime,timedelta
import warnings
import datetime
import pandas as pd
import glob
import os
warnings.filterwarnings("ignore")
import ssl
ssl._create_default_https_context = ssl._create_unverified_context
from urllib.error import HTTPError
def get_url(base_url):
part_two = "/dailyCoal1-"
end_part = ".xlsx"
start_date = datetime.datetime.now().date() + timedelta(-1)
start_url = start_date.strftime(base_url+'%d-%m-%Y'+part_two+'%Y-%m-%d'+end_part)
return start_url
def excel_download(link,out):
#downloads a given link provided to a output directory in out
wget.download(link,out)
if __name__ =="__main__":
base_url = "https://npp.gov.in/public-reports/cea/daily/fuel/"
mypath = "/Users/vp/Desktop/temp"
temp_folder = '/Users/vp/Desktop/temp'
out_folder = "/Users/vp/Desktop/NPP"
log_file = os.path.join(out_folder,'debug_log_npp.log')
out_file = os.path.join(out_folder,'Energy_inputs_npp.csv')
file_link = get_url(base_url)
print(file_link)
This script downloads images, renames them based on line[0] adds a number to the end of the file name and saves them to a file folder. My goal here is to create the file folder name based on line[0] of my csv file, I'm new to python and need to download/sort 15000+ images. Any help would be appreciated! Using python 3.8.6
Note: 1 model may contain many images so the idea is to create the folder, place images for that model inside, move on to the next model, etc.
Csv file content
RHV3-484-L,https://www.fisherpaykel.com/on/demandware.static/-/Sites-fpa-master-catalog/default/dw0c85e188/product-mugs/cooking/ranges/mug/retail/RHV3-484-N-RHV3-484-L-external-mug-rs-84.png
RHV3-484-L,https://www.fisherpaykel.com/on/demandware.static/-/Sites-fpa-master-catalog/default/dwcbd711e5/inspire/caitlin-wilson-portland-dk-339-rs-84.png
RHV3-484-L,https://www.fisherpaykel.com/on/demandware.static/-/Sites-fpa-master-catalog/default/dw3702e52a/inspire/caitlin-wilson-portland-dk-385-rs-84.jpg
RHV3-484-L,https://www.fisherpaykel.com/on/demandware.static/-/Sites-fpa-master-catalog/default/dw0c85e188/product-mugs/cooking/ranges/mug/retail/RHV3-484-N-RHV3-484-L-external-mug-rs-84.png
RHV3-484-L,https://www.fisherpaykel.com/on/demandware.static/-/Sites-fpa-master-catalog/default/dwf99a5a9d/inspire/david-berridge-project-brooklyn-mw-6457-rs-84.jpg
Python script
import sys
import urllib
import urllib.request
from csv import reader
import os.path
import os
csv_filename = "images"
with open(csv_filename+".csv".format(csv_filename), 'r') as csv_file:
n = 1
for line in reader(csv_file):
if not os.path.exists("ImageID"):
os.makedirs("ImageID")
print("Image skipped for {0}".format(line[0]))
else:
if line[1] != '' and line[0] != "ImageID":
urllib.request.urlretrieve(line[1], "ImageID/" + line[0] + "-" + str(n) + ".jpg")
n += 1
print("Image saved for {0}".format(line[0]))
else:
print("No result for {0}".format(line[0]))
This seems to work as desired....
Couple comments in middle. Notably, you need to respect the .jpg or .png file. If you have file extensions that are longer (4 chars) you may need to split out the file name and then split by "."
Good Luck!
import sys
import urllib
import urllib.request
from csv import reader
import os.path
import os
csv_filename = "images.csv"
with open(csv_filename, 'r') as csv_file:
n = 1 # starting point
for line in reader(csv_file):
tgt_folder = line[0]
if not os.path.exists(tgt_folder):
os.makedirs(tgt_folder)
n = 1 # restart n if you find a NEW folder
# there should be no "else" clause here. Just test the folder name above, but don't waste a file
if line[1] != '' and line[0] != "ImageID": # not clear what this is for... ??
filename = ''.join([line[0], '-', str(n), line[1][-4:]])
destination = os.path.join(tgt_folder, filename)
urllib.request.urlretrieve(line[1], destination)
n += 1
print("Image saved for {0}".format(line[1]))
else:
print("No result for {0}".format(line[1]))
This is the program as given below
import os
import time
index=0
file=open("out_{index}.txt", 'a')
while True:
ps=os.system(f"speedtest.exe")
file.append(ps)
index+=1
time.sleep(4)
I believe it is file.write() and not file.append().
I think you want to write to a new file every time.
I would do this.
import time
import os
index = 0
while True:
ps = os.system("speedtest.exe")
with open(f"out_{index}.txt", 'w+') as f:
f.write(ps)
index += 1
time.sleep(4)
However, if you want to write in the same file.
I would do this.
import time
import os
with open("out_file.txt", 'a+') as f:
index = 0
while True:
ps = os.system("speedtest.exe")
f.write(ps)
index += 1
time.sleep(4)