I am performing spam detection and want to visualize spam and ham keywords separately in Wordcloud. Here's my .csv file.
data = pd.read_csv("spam.csv",encoding='latin-1')
data = data.rename(columns = {"v1":"label", "v2":"message"})
data = data.replace({"spam":"1","ham":"0"})
Here's my code for WordCloud. I need help with spam_words. I cannot generate the right graph.
import matplotlib.pyplot as plt
from wordcloud import WordCloud
spam_words = ' '.join(list(data[data['label'] == 1 ]['message']))
spam_wc = WordCloud(width = 512, height = 512).generate(spam_words)
plt.figure(figsize = (10,8), facecolor = 'k')
plt.imshow(spam_wc)
plt.axis('off')
plt.tight_layout(pad = 0)
plt.show()
The issue is that the current code replaces "spam" and "ham" with the one-character strings "1" and "0", but you filter the DataFrame based on comparison with the integer 1. Change the replace line to this:
data = data.replace({"spam": 1, "ham": 0})
Related
I am making a stock prediction web app and I am trying to overlay the current data with the forecasted data I have produced using NeuralProphet. Here is my full code right now:
import streamlit as st
from datetime import date
import yfinance as yf
from neuralprophet import NeuralProphet
from plotly import graph_objs as go
START = "2015-01-01"
TODAY = date.today().strftime("%Y-%m-%d")
st.title("Prediction")
stocks = ("AAPL", "GOOG", "MSFT", "GME", "TSLA", "RIVN")
selected_stocks = st.selectbox("Select Dataset for Prediction", stocks)
n_years = st.slider("Years of Prediction:", 1, 10)
period = n_years * 365
#st.cache
def load_data(ticker):
data = yf.download(ticker, START, TODAY)
data.reset_index(inplace=True)
return data
data_load_state = st.text("Load data...")
data = load_data(selected_stocks)
data_load_state.text("Loading data...done!")
st.subheader('Raw Data')
st.write(data.tail())
def plot_raw_data():
global fig
fig = go.Figure()
fig.add_trace(go.Line(x=data['Date'], y=data['Open'], name='stock_open'))
fig.add_trace(go.Line(x=data['Date'], y=data['Close'], name='stock_close'))
fig.layout.update(title_text="Time Series Data", xaxis_rangeslider_visible=True)
st.plotly_chart(fig, use_container_width=True)
plot_raw_data()
# Forecasting
df_train = data[['Date', 'Close']]
df_train = df_train.rename(columns={"Date": "ds", "Close": "y"})
m = NeuralProphet()
metrics = m.fit(df_train)
future = m.make_future_dataframe(df=df_train, periods=period)
forecast = m.predict(df=future)
st.subheader('Forecast data')
st.write(forecast.tail())
st.write('Forecast Data')
fig1 = m.plot(forecast)
st.plotly_chart(fig1, use_container_width=True)
st.write("Forecast Components")
fig2 = m.plot_components(forecast)
st.write(fig2)
I can't seem to figure out what to do. I have tried implementing different st.plotly_chart() in attempt to overlay two charts, but no such luck... Help.
Add n_historic_predictions=True param in make_future_dataframe as in:
future = m.make_future_dataframe(df=df_train, periods=period, n_historic_predictions=True)
Output
So, I'm trying to get the colors from the dictionary 'Disaster_type' to draw the markers in geoscatters depending of the type of disaster.
Basically, I want to reprensent in the graphic the natural diasasters with it's color code. eg; it's is a volcanic activity paint it 'orange'. I want to change the size of the marker as well depending of the magnitude of the disaster, but that's for another day.
here's the link of the dataset: https://www.kaggle.com/datasets/brsdincer/all-natural-disasters-19002021-eosdis
import plotly.graph_objects as go
import pandas as pd
import plotly as plt
df = pd.read_csv('1900_2021_DISASTERS - main.csv')
df.head()
df.tail()
disaster_set = {disaster for disaster in df['Disaster Type']}
disaster_type = {'Storm':'aliceblue',
'Volcanic activity':'orange',
'Flood':'royalblue',
'Mass movement (dry)':'darkorange',
'Landslide':'#C76114',
'Extreme temperature':'#FF0000',
'Animal accident':'gray55',
'Glacial lake outburst':'#7D9EC0',
'Earthquake':'#CD8C95',
'Insect infestation':'#EEE8AA',
'Wildfire':' #FFFF00',
'Fog':'#00E5EE',
'Drought':'#FFEFD5',
'Epidemic':'#00CD66 ',
'Impact':'#FF6347'}
# disaster_type_lower = {(k, v.lower()) for k, v in disaster_type.items()}
# print(disaster_type_lower)
# for values in disaster_type.values():
# disaster_type[values] = disaster_type.lowercase()
fig = go.Figure(data=go.Scattergeo(
lon = df['Longitude'],
lat = df['Latitude'],
text = df['Country'],
mode = 'markers',
marker_color = disaster_type_.values()
)
)
fig.show()
I cant figure how, I've left in comments after the dict how I tried to do that.
It changes them to lowercase, but know I dont know hot to get them...My brain is completly melted
it's a simple case of pandas map
found data that appears same as yours on kaggle so have used that
one type is unmapped Extreme temperature so used a fillna("red") to remove any errors
gray55 gave me an error so replaced it with RGB equivalent
import kaggle.cli
import sys
import pandas as pd
from zipfile import ZipFile
import urllib
import plotly.graph_objects as go
# fmt: off
# download data set
url = "https://www.kaggle.com/brsdincer/all-natural-disasters-19002021-eosdis"
sys.argv = [sys.argv[0]] + f"datasets download {urllib.parse.urlparse(url).path[1:]}".split(" ")
kaggle.cli.main()
zfile = ZipFile(f'{urllib.parse.urlparse(url).path.split("/")[-1]}.zip')
dfs = {f.filename: pd.read_csv(zfile.open(f)) for f in zfile.infolist()}
# fmt: on
df = dfs["DISASTERS/1970-2021_DISASTERS.xlsx - emdat data.csv"]
disaster_type = {
"Storm": "aliceblue",
"Volcanic activity": "orange",
"Flood": "royalblue",
"Mass movement (dry)": "darkorange",
"Landslide": "#C76114",
"Extreme temperature": "#FF0000",
"Animal accident": "#8c8c8c", # gray55
"Glacial lake outburst": "#7D9EC0",
"Earthquake": "#CD8C95",
"Insect infestation": "#EEE8AA",
"Wildfire": " #FFFF00",
"Fog": "#00E5EE",
"Drought": "#FFEFD5",
"Epidemic": "#00CD66 ",
"Impact": "#FF6347",
}
fig = go.Figure(
data=go.Scattergeo(
lon=df["Longitude"],
lat=df["Latitude"],
text=df["Country"],
mode="markers",
marker_color=df["Disaster Type"].map(disaster_type).fillna("red"),
)
)
fig.show()
I plotted some data which has 70 classes, so when I built the color bar it's very difficult to distinguish between each legend as shown below:
The code that I'm using is:
formation_colors = # 70 colors
formation_labels = # 70 labels
data = # the section of the entire dataset which only has 13 labels
data = data.sort_values(by='DEPTH_MD')
ztop=data.DEPTH_MD.min(); zbot=data.DEPTH_MD.max()
cmap_formations = colors.ListedColormap(formation_colors[0:len(formation_colors)], 'indexed')
cluster_f = np.repeat(np.expand_dims(data['Formations'].values,1), 100, 1)
fig = plt.figure(figsize=(2,10))
ax = fig.add_subplot()
im_f = ax.imshow(cluster_f, interpolation='none', aspect='auto', cmap = cmap_formations, vmin=0, vmax=69)
ax.set_xlabel('FORMATION')
ax.set_xticklabels(['']);
divider_f = make_axes_locatable(ax)
cax_f = divider_f.append_axes("right", size="20%", pad=0.05)
cbar_f = plt.colorbar(im_f, cax = cax_f,)
cbar_f.set_ticks(range(0,len(formation_labels))); cbar_f.set_ticklabels(formation_labels)
So far, if I just change:
1. cmap_formations = colors.ListedColormap(formation_colors[0:len(formation_colors)], 'indexed')
2. cbar_f.set_ticks(range(0,len(formation_labels))); cbar_f.set_ticklabels(formation_labels)
to:
cmap_formations = colors.ListedColormap(formation_colors[0:len(data['FORMATION'].unique())], 'indexed')
cbar_f.set_ticks(range(0,len(data['FORMATION'].unique()))); cbar_f.set_ticklabels(data['FORMATION'].unique())
I get, the corresponding colors in the cbar, however the plot is no longer correct and also the legends are out of square
Thank you so much if you have any idea how to do this.
Although not explicitly mentioned in the question, I suppose data['FORMATION'] contains indices from 0 till 69 into the lists of formation_colors and formation_labels
The main problem is that data['FORMATION'] needs to be renumbered to be new indices (with numbers 0 till 12) into the new list of unique colors. np.unique(..., return_inverse=True) returns both the list of unique numbers, and the renumbering for the values.
To be able to reindex the list of colors and of labels, it helps to convert them to numpy arrays.
To make the code easier to debug, the following test uses a simple relation between the list of colors and the list of labels.
from matplotlib import pyplot as plt
from matplotlib import colors
from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable
import numpy as np
import pandas as pd
formation_colors = np.random.choice(list(colors.CSS4_COLORS), 70, replace=False) # 70 random color names
formation_labels = ['lbl_' + c for c in formation_colors] # 70 labels
formation_colors = np.asarray(formation_colors)
formation_labels = np.asarray(formation_labels)
f = np.random.randint(0, 70, 13)
d = np.sort(np.random.randint(0, 5300, 13))
data = pd.DataFrame({'FORMATION': np.repeat(f, np.diff(np.append(0, d))),
'DEPTH_MD': np.arange(d[-1])})
data = data.sort_values(by='DEPTH_MD')
ztop = data['DEPTH_MD'].min()
zbot = data['DEPTH_MD'].max()
unique_values, formation_new_values = np.unique(data['FORMATION'], return_inverse=True)
cmap_formations = colors.ListedColormap(formation_colors[unique_values], 'indexed')
cluster_f = formation_new_values.reshape(-1, 1)
fig = plt.figure(figsize=(3, 10))
ax = fig.add_subplot()
im_f = ax.imshow(cluster_f, extent=[0, 1, zbot, ztop],
interpolation='none', aspect='auto', cmap=cmap_formations, vmin=0, vmax=len(unique_values)-1)
ax.set_xlabel('FORMATION')
ax.set_xticks([])
divider_f = make_axes_locatable(ax)
cax_f = divider_f.append_axes("right", size="20%", pad=0.05)
cbar_f = plt.colorbar(im_f, cax=cax_f)
cbar_f.set_ticks(np.linspace(0, len(unique_values)-1, 2*len(unique_values)+1)[1::2])
cbar_f.set_ticklabels(formation_labels[unique_values])
plt.subplots_adjust(left=0.2, right=0.5)
plt.show()
Here is a comparison plot:
I am working with a lot of images (120k), each image is a component of RGB + yellow of a single image (so I have 30k unique images all break down in 4 images: one for Red, Green, Blue and Yellow)
For each image ID, I merge the 4 components (RGB + yellow) into a (M,N,4) array (where M and N are the dimensions of the image).
I work with the following code:
import pandas as pd
import numpy as np
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
from os import listdir
train_labels_data = pd.read_csv('/Documents/train.csv')
def merge_rgb(img_id, colours=['red','blue','green','yellow'], path = 'train'):
"""
For each images, returns an array of shape (M,N,4)
where each dimension in the 4 are red, blue, green and yellow.
"""
merged_colour_img = []
for colour in colours:
full_path = path + '/' + img_id + '_' + colour + '.png'
colour_img=mpimg.imread(full_path)
merged_colour_img.append(colour_img)
merged_colour_img = np.dstack((merged_colour_img))
return merged_colour_img
def train_data_label(train_labels_data):
"""
From the train_labels csv file, create a list of labels, and create a large
array for the train data in same order.
"""
train_ids = [img_id for img_id in train_labels_data['Id']]
train_labels = [label for label in train_labels_data['Target']]
print ('Labels and Ids collected')
train_data = []
i=0
for img_id in train_ids:
print ('Merging Image')
train_data_img = merge_rgb (img_id)
print ('Merging done, appending the (M,N,4) array to a list')
train_data.append(train_data_img)
i += 1
print ('Done appending, going to next image')
print(i)
print('Stacking all images in one big array')
train_data = np.stack(train_data)
return train_labels, train_data
train_labels, train_data = train_data_label(train_labels_data)
# SAVE OUTPUT
data_pickle_train = pickle.dumps(train_data)
data = open("/Documents/train_data.pkl","wb")
data.write(data_pickle_train)
data.close()
data_pickle_train_labels = pickle.dumps(train_labels)
data = open("/Documents/train_data_labels.pkl","wb")
data.write(data_pickle_train_labels)
data.close()
However this code uses a lot of memory and crashes half way before all images are processed. Since I am working with images I suspect I could improve the merge_rgb function, any advice how to?
Thanks,
Python 3.6
Bokeh 12.15
I have tried to implement the bokeh example line_on_off.py, but in a for loop with a hover tool and data of varying length. What happens though is that when a line is turned off it turns off the tool tip of any line created after it. For example if I turn off line 1, line 2,3,4 tool tips are disabled, or if I turn off line 3 line 4's tool tip is disabled.
Can I use a hover tool and checkbox widget in a for loop like this? I have seen this multiline example, but my data is of varying length and I do not want to resample because I would like to see if there is bad or missing data.
Code
from bokeh.plotting import figure
from bokeh.models import CheckboxGroup, CustomJS
from bokeh.models import ColumnDataSource
import pandas as pd
from bokeh.models import HoverTool
def create_plot(df_list):
p = figure(x_axis_type = 'datetime')
glyph_dict = {}
labels = []
active = []
items = []
names = 'abcdefghijklmnopqrstuvwxyz'
callback_string = '{}.visible = {} in checkbox.active;'
code_string = ''
i = 0
sources = []
for df in df_list:
legend = df.columns[0]
series = df.iloc[:,0]
labels.append(legend)
x = series.index
y = series.values
source =ColumnDataSource(data = {'x':x,'y':y, 'date': [str(x) for x in x]})
sources.append(source)
line = p.line('x', 'y', source = sources[i])
items.append((legend, [line]))
name = names[i]
line.name = name
code_string += callback_string.format(name, str(i))
glyph_dict.update({name:line})
active.append(i)
i+=1
hover = HoverTool(tooltips=[('date', '#date'),('y', '#y')])
p.add_tools(hover)
checkbox = CheckboxGroup(labels=labels, active=active, width=200)
glyph_dict.update({'checkbox':checkbox})
checkbox.callback = CustomJS.from_coffeescript(args=glyph_dict, code=code_string)
return checkbox, p
Minimal example
import numpy as np
from datetime import datetime, timedelta
from bokeh.layouts import row
from bokeh.plotting import show
df_list = []
start = datetime(2017, 4,1)
end = datetime(2017,5,1)
for i in range(1,5):
date = pd.date_range(start, end, freq = '1w')
shape = len(date)
df = pd.DataFrame(index = date, data = np.random.randn(shape,1))
name = 'df'+ str(i)
df.columns = [name]
end = end + timedelta(weeks = 1)
df_list.append(df)
c,p = create_plot(df_list)
r=row([c,p])
show(r)
In a situation like this, you should probably create a new, separate hover tool for each line, by restricting the renderers property of each hover tool. So, in relation to your code, move the hover tool creation inside the loop, and have it set renderers each time:
line = p.line('x', 'y', source = sources[i])
hover = HoverTool(tooltips=[('date', '#date'),('y', '#y')]
renderers=[line])
p.add_tools(hover)