I would like to put this DataFrame (DataFrame ) into influxdb. Unfortunately I have the following error message: "ValueError: The truth value of a DataFrame is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all()."
Trouble with this 2 lines?
tags = { 'tag1': df[['tag1']], 'tag2': df[['tag2']] }
client.write_points(df, 'test', tags = tags)
I do not understand why. Thank you for your help
def main(host='10.0.0.3', port=8086):
"""Instantiate the connection to the InfluxDB client."""
user = ''
password = ''
dbname = 'ess_2'
protocol = 'line'
client = DataFrameClient(host, port, user, password, dbname)
print("Create database: " + dbname)
client.create_database(dbname)
t = time.strftime('%Y-%m-%d %H:%M:%S')
headers = ['time','tag1','tag2','field1','field2']
data = [[t,'MSFT','NYSE',1.3,2.5],[t,'APPL','NYSE',3.5,4.24]]
df = pd.DataFrame(data, columns = headers)
df['time'] = pd.to_datetime(df['time'])
df = df.set_index('time')
tags = { 'tag1': df[['tag1']], 'tag2': df[['tag2']] } #DEBUG
client.write_points(df, 'test', tags = tags) #DEBUG
print("Read DataFrame")
client.query("select * from demo")
def parse_args():
"""Parse the args from main."""
parser = argparse.ArgumentParser(
description='example code to play with InfluxDB')
parser.add_argument('--host', type=str, required=False,
default='10.0.0.3',
help='hostname of InfluxDB http API')
parser.add_argument('--port', type=int, required=False, default=8086,
help='port of InfluxDB http API')
return parser.parse_args()
if __name__ == '__main__':
args = parse_args()
main(host=args.host, port=args.port)
Related
I am working on a script which i have to modify in order loop through the multiple resources within a functions.
Below are the items which we need to loop through to get the data from and, this is coming from Config_local
BASE_URL = "https://synergy.hpe.example.com/rest/"
RES_EXT = [ 'resource-alerts?count=500&start=1',
'resource-alerts?count=500&start=501'
'resource-alerts?count=500&start=1001,
'resource-alerts?count=500&start=1501'
]
While i am looping through above list under def main(): section and taking get_resource_alerts_response() under loop the data coming out of the loop getting over-written and thus returning only last loop data only.
Main Script:
import os
import shutil
import smtplib
from email.message import EmailMessage
import pandas as pd
pd.set_option('expand_frame_repr', True)
import requests
from Config_local import (
BASE_URL,
DST_DIR,
OUTFILE,
PASSWORD,
SRC_DIR,
TIMESTAMP_DST,
USERNAME,
SUBJECT,
FROM,
TO,
EMAIL_TEMPLATE,
SMTP_SERVER,
RES_EXT,
)
class FileMoveFailure(Exception):
pass
class SynergyRequestFailure(Exception):
pass
class SessionIdRetrievalFailure(SynergyRequestFailure):
pass
class ResourceAlertsRetrievalFailure(SynergyRequestFailure):
pass
def move_csv_files():
for csv_file in os.listdir(SRC_DIR):
if csv_file.endswith(".csv") and os.path.isfile(os.path.join(SRC_DIR, csv_file)):
try:
shutil.move(
os.path.join(f"{SRC_DIR}/{csv_file}"),
f"{DST_DIR}/{csv_file}-{TIMESTAMP_DST}.log"
)
except OSError as os_error:
raise FileMoveFailure(
f'Moving file {csv_file} has failed: {os_error}'
)
def get_session_id(session):
try:
response = session.post(
url=f"{BASE_URL}/login-sessions",
headers={
"accept": "application/json",
"content-type": "application/json",
"x-api-version": "120",
},
json={
"userName": USERNAME,
"password": PASSWORD
},
verify=False
)
except requests.exceptions.RequestException as req_exception:
# you should also get this logged somewhere, or at least
# printed depending on your use case
raise SessionIdRetrievalFailure(
f"Could not get session id: {req_exception}"
)
json_response = response.json()
if not json_response.get("sessionID"):
# always assume the worse and do sanity checks & validations
# on fetched data
raise KeyError("Could not fetch session id")
return json_response["sessionID"]
#def get_all_text(session, session_id):
# all_text = ''
# for res in RES_EXT:
# url= f"{BASE_URL}{res}"
# newresult = get_resource_alerts_response(session, session_id, url)
# all_text += newresult
# print(f"{all_text}")
# return str(all_text)
#
def get_resource_alerts_response(session, session_id, res):
try:
return session.get(
url=f"{BASE_URL}{res}",
headers={
"accept": "application/json",
"content-type": "text/csv",
"x-api-version": "2",
"auth": session_id,
},
verify=False,
stream=True
)
except requests.exceptions.RequestException as req_exception:
# you should also get this logged somewhere, or at least
# printed depending on your use case
raise ResourceAlertsRetrievalFailure(
f"Could not fetch resource alerts: {req_exception}"
)
def resource_alerts_to_df(resource_alerts_response):
with open(OUTFILE, 'wb') as f:
for chunk in resource_alerts_response.iter_content(chunk_size=1024*36):
f.write(chunk)
return pd.read_csv(OUTFILE)
def send_email(df):
server = smtplib.SMTP(SMTP_SERVER)
msg = EmailMessage()
msg['Subject'], msg['From'], msg['To'] = SUBJECT, FROM, TO
msg.set_content("Text version of your html template")
msg.add_alternative(
EMAIL_TEMPLATE.format(df.to_html(index=False)),
subtype='html'
)
server.send_message(msg)
def main():
move_csv_files()
session = requests.Session()
session_id = get_session_id(session)
for res in RES_EXT:
resource_alerts_response = get_resource_alerts_response(session,
session_id, res)
print(resource_alerts_response)
df = resource_alerts_to_df(resource_alerts_response)
print(df)
send_email(df)
if __name__ == '__main__':
main()
any help or hint will be much appreciated.
This is a copy of the code which I recall we had over SO but not what you want now, However, as the whole code body is okay and the idea of for loop is also looks good, you Just need to tweek it to meet the requirement.
1- You need to create and empty DataFrame assignment Synergy_Data = pd.DataFrame()
2- then you can append the data you received from for loop ie resource_alerts_response which becomes df = resource_alerts_to_df(resource_alerts_response)
3- lastly, you can append this df to the empty Synergy_Data and then call that under your if __name__ == '__main__' to send an e-mail. Also don't forget to declare Synergy_Data as a global variable.
Synergy_Data = pd.DataFrame()
def main():
global Synergy_Data
move_csv_files()
session = requests.Session()
session_id = get_session_id(session)
for res in RES_EXT:
resource_alerts_response = get_resource_alerts_response(session,
session_id, res)
df = resource_alerts_to_df(resource_alerts_response)
Synergy_Data = Synergy_Data.append(df)
if __name__ == '__main__':
main()
send_email(Synergy_Data)
Hope this will be helpful.
Only the last response is being used because this value is overwritten in resource_alerts_response on each iteration of the loop. You may consider acting on the data on each iteration or storing it for use later i.e. after the loop. I've included these options with modifications to the main() function below.
Option 1
Send an email for each resource alert response
def main():
move_csv_files()
session = requests.Session()
session_id = get_session_id(session)
for res in RES_EXT:
resource_alerts_response = get_resource_alerts_response(session,
session_id, res)
print(resource_alerts_response)
# Indent lines below so that the operations below are executed in each loop iteration
df = resource_alerts_to_df(resource_alerts_response)
print(df)
send_email(df)
Option 2
Merge all resource alert responses and send one email
def main():
move_csv_files()
session = requests.Session()
session_id = get_session_id(session)
df_resource_alerts_responses = None
for res in RES_EXT:
resource_alerts_response = get_resource_alerts_response(session,
session_id, res)
print(resource_alerts_response)
df = resource_alerts_to_df(resource_alerts_response)
if df_resource_alerts_responses is None:
df_resource_alerts_responses = df
else:
df_resource_alerts_responses = df_resource_alerts_responses.append(df, ignore_index=True)
print(df_resource_alerts_responses)
if df_resource_alerts_responses is not None:
send_email(df_resource_alerts_responses)
I am trying to Create a new partition Bigquery table on runtime with following code, but i am not getting option to pass column names "_time" over which partition need to be done on my new BQ table.
Can any please please help me on it.
My Code
#------------Import Lib-----------------------#
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions, StandardOptions
import os, sys
import argparse
import logging
from apache_beam.options.pipeline_options import SetupOptions
from datetime import datetime
#------------Set up BQ parameters-----------------------#
# Replace with Project Id
project = 'xxxx'
#plitting Of Records----------------------#
class Transaction_DB_UC2(beam.DoFn):
def process(self, element):
logging.info(element)
result = json.loads(element)
data_time = result.get('_time', 'null')
data_dest = result.get('dest', 'null')
data_DBID = result.get('DBID', 'null')
data_SESSIONID = result.get('SESSIONID', 'null')
data_USERHOST = result.get('USERHOST', 'null')
data_raw = result.get('_raw', 'null')
data_ACTION = result.get('ACTION', 'null')
data_host = result.get('host', 'null')
data_result = result.get('result', 'null')
data_DBUSER = result.get('DBUSER', 'null')
data_OS_USERNAME = result.get('OS_USERNAME', 'null')
data_ACTION_NAME = result.get('ACTION', 'null').replace('100','LOGON').replace('101','LOGOFF')
return [{"_time": data_time[:-8], "dest": data_dest, "DBID": data_DBID, "SESSIONID": data_SESSIONID, "_raw": data_raw, "USERHOST": data_USERHOST, "ACTION": data_ACTION, "host": data_host, "result": data_result, "DBUSER": data_DBUSER, "OS_USERNAME": data_OS_USERNAME, "ACTION_NAME": data_ACTION_NAME}]
def run(argv=None, save_main_session=True):
parser = argparse.ArgumentParser()
parser.add_argument(
'--input',
dest='input',
help='Input file to process.')
parser.add_argument(
'--pro_id',
dest='pro_id',
type=str,
default='ORACLE_SEC_DEFAULT',
help='project id')
known_args, pipeline_args = parser.parse_known_args(argv)
pipeline_options = PipelineOptions(pipeline_args)
pipeline_options.view_as(SetupOptions).save_main_session = save_main_session
p1 = beam.Pipeline(options=pipeline_options)
#data_f = sys.argv[1]
logging.info('***********')
logging.info(known_args.input)
data_loading = (
p1
|'Read from File' >> beam.io.ReadFromText(known_args.input,skip_header_lines=0)
)
project_id = "xxxxx"
dataset_id = 'test123'
table_schema_DB_UC2 = ('_time:DATETIME, dest:STRING, DBID:STRING, SESSIONID:STRING, _raw:STRING, USERHOST:STRING, ACTION:STRING, host:STRING, result:STRING, DBUSER:STRING, OS_USERNAME:STRING, ACTION_NAME:STRING')
# Persist to BigQuery
# WriteToBigQuery accepts the data as list of JSON objects
#---------------------Index = DB-UC2----------------------------------------------------------------------------------------------------------------------
result = (
data_loading
| 'Clean-DB-UC2' >> beam.ParDo(Transaction_DB_UC2())
| 'Write-DB-UC2' >> beam.io.WriteToBigQuery(
table=known_args.pro_id,
dataset=dataset_id,
project=project_id,
schema=table_schema_DB_UC2,
create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND))
result = p1.run()
result.wait_until_finish()
if __name__ == '__main__':
#logging.getLogger().setLevel(logging.INFO)
path_service_account = 'ml-fbf8cabcder.json'
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = path_service_account
run()
I want to create partition on filed "_time", please suggest how it can be achieved.
Thanks.
I believe that you can do that with additional_bq_parameters (Note the limitations) with the timePartitioning parameter.
When creating a new BigQuery table, there are a number of extra parameters
that one may need to specify. For example, clustering, partitioning, data
encoding, etc. It is possible to provide these additional parameters by
passing a Python dictionary as additional_bq_parameters (Reference).
In your case, you could add to your WriteToBigQuery transform the timePartitioning parameter with the required type and optional field fields (Note that field must be a top-level TIMESTAMP or DATE field):
additional_bq_parameters={'timePartitioning': {
'type': 'DAY',
'field': '_time'
}}
I didn't have the time to try it out yet. I'll try to reproduce tomorrow.
Let me know if it works for you.
EDIT
Finally got the chance to try the timePartitioning parameter to create a partitioned table and it worked.
Here is a simple pipeline code to test it.
#!/usr/bin/env python
import apache_beam as beam
PROJECT='YOUR_PROJECT'
BUCKET='YOUR_BUCKET'
def run():
argv = [
'--project={0}'.format(PROJECT),
'--job_name=YOUR_JOB_NAME',
'--save_main_session',
'--staging_location=gs://{0}/staging/'.format(BUCKET),
'--temp_location=gs://{0}/staging/'.format(BUCKET),
'--region=us-central1',
'--runner=DataflowRunner'
]
p = beam.Pipeline(argv=argv)
table_schema = {'fields': [
{'name': 'country', 'type': 'STRING', 'mode': 'NULLABLE'},
{'name': '_time', 'type': 'DATETIME', 'mode': 'NULLABLE'},
{'name': 'query', 'type': 'STRING', 'mode': 'NULLABLE'}]}
additional_bq_parameters = {
'timePartitioning': {'type': 'DAY', 'field': '_time'}}
elements = (p | beam.Create([
{'country': 'mexico', '_time': '2020-06-10 22:19:26', 'query': 'acapulco'},
{'country': 'canada', '_time': '2020-12-11 15:42:32', 'query': 'influenza'},
]))
elements | beam.io.WriteToBigQuery(
table='YOUR_DATASET.YOUR_NEW_TABLE',
schema=table_schema,
additional_bq_parameters=additional_bq_parameters,
create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE
)
p.run()
if __name__ == '__main__':
run()
I'm trying to get quotes from the etrade API. I'm able to list accounts, get transactions, but not get quotes. I've tried removing the accounts and transactions api calls but it makes no difference. I get an "oauth_problem=signature_invalid" response. Any ideas what I need to do differently?
from rauth import OAuth1Service
import webbrowser
import hmac
# required for google sheets
# from __future__ import print_function
import pickle
import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
class ETradeManager():
def __init__(self):
session = None
service = None
def connect_to_etrade(self):
self.service = OAuth1Service(
name='etrade',
consumer_key='',
consumer_secret='',
request_token_url='https://apisb.etrade.com/oauth/request_token',
access_token_url='https://apisb.etrade.com/oauth/access_token',
authorize_url='https://us.etrade.com/e/t/etws/authorize?key={}&token={}',
base_url='https://etsw.etrade.com')
oauth_token, oauth_token_secret = self.service.get_request_token(params=
{'oauth_callback': 'oob',
'format': 'json'})
auth_url = self.service.authorize_url.format(self.service.consumer_key, oauth_token)
webbrowser.open(auth_url)
verifier = input('Please input the verifier: ')
print("Attempting to get session")
self.session = self.service.get_auth_session(oauth_token, oauth_token_secret, params={'oauth_verifier': verifier})
url = 'https://apisb.etrade.com/v1/accounts/list'
resp = self.session.get(url, params={'format': 'json'})
accountid = ""
print(resp.text)
trans_url_template = "https://apisb.etrade.com/v1/accounts/{}/transactions"
trans_url = trans_url_template.format(accountid)
resp = self.session.get(trans_url, params={'format': 'json'})
f = open("trans.xml", "w")
f.write(resp.text)
# key = self.service.consumer_secret + \
# '&' + \
# oauth_token_secret
# hashed = hmac.new(key.encode(), base_string.encode(), sha1)
# def get_quote(self):
quote_url_template = "https://apisb.etrade.com/v1/market/quote/{}"
quote_url = quote_url_template.format("TSLA")
resp = self.session.get(quote_url_template, params={'format': 'json'})
f = open("quote.xml", "w")
f.write(resp.text)
trade_manager = ETradeManager()
trade_manager.connect_to_etrade()
# trade_manager.get_quote()
Not sure if you figured this out but you had a typo here:
resp = self.session.get(quote_url_template, params={'format': 'json'})
Should be using quote_url, not quote_url_template
I am new to python. I am trying to send a response showing a custom keyboard with two buttons to the user in Telegram app but it gives me an error about encode.
I would like to know where is my mistake.
import json
from urllib.request import urlopen
from urllib.parse import quote, unquote
import time
def Decode(telegram_response):
decoded=''
for line in telegram_response:
decoded=decoded+line.decode('utf-8')
return decoded
TOKEN = 'mytoken'
URL = 'https://api.telegram.org/bot{}/'.format(TOKEN)
cmd = 'getme'
telegram_response = urlopen(URL + cmd)
decoded = Decode(telegram_response)
gtm = json.loads(decoded)
status = True
while status:
cmd = 'getUpdates'
telegram_response = urlopen(URL + cmd)
decoded = Decode(telegram_response)
upds = json.loads(decoded)
new_message = len(upds['result'])
if new_message !=0:
msg = upds['result'][0]['message']
chat_id = str(msg['chat']['id'])
reply_markup = {'keyboard': [[{'text':'first button'}],[{'text':'second button'}]], 'resize_keyboard': True, 'one_time_keyboard': True}
reply_markup = json.dumps(reply_markup)
params = ({'chat_id': chat_id, 'reply_markup': reply_markup, 'disable_web_page_preview': 'true'})
myresponse =urlopen(URL + 'sendMessage' + quote((params).encode('utf-8')))
Easy way to build powerful bots is to use python-telegram-bot library.
I re-writed your code with few major fixes and features. Hope it help you to learn bots more deeper.
my version of bot:
###############################################################################
!#/usr/bin/python3
from sys import exc_info as error
from urllib.request import urlopen
from urllib.parse import urlencode
import json
TOKEN = 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
URL = 'https://api.telegram.org/bot{}'.format(TOKEN)
STATUS = True
OFFSET = 0
def call_api_method(method='getMe', data=None):
# Call API method with data.
data = urlencode(data).encode("utf-8") if data else data
response = urlopen('{}/{}'.format(URL, method), data)
return json.loads(response.read())
def get_me():
# Get bot info.
bot = call_api_method()
return type('Bot', (), dict(bot['result']))
def get_updates():
# Get new updates from Telegram.
data = {'offset': OFFSET, 'limit': 0, 'timeout': 0}
return type('Updates', (), call_api_method('getUpdates', data))
def handle(update):
# Make usefull objects.
message = type('Message', (object,), dict(update['message']))
user = type('User', (), dict(update['message']['from']))
chat = type('Chat', (), dict(update['message']['chat']))
return message, user, chat
def send_message(chat_id, message):
# Send message to specific chat.
data = {'text': message,
'chat_id': chat_id,
'parse_mode': 'Markdown',
'disable_web_page_preview': True}
call_api_method('sendMessage', data)
def send_keyboard(chat_id, message, keyboard):
# Send message and keyboard to specific chat.
data = {'text': message,
'chat_id': chat_id,
'parse_mode': 'Markdown',
'reply_markup': reply_markup(keyboard),
'disable_web_page_preview': 'true'}
call_api_method('sendMessage', data)
def reply_markup(keyboard):
# Serialize keyboard data to JSON.
return json.dumps({'keyboard': keyboard,
'resize_keyboard': True,
'one_time_keyboard': True,
'selective': True})
def main_keyboard():
# Main menu.
return [first_button(), second_button()]
def one_line_keyboard():
# Menu with buttons in one line.
return [two_buttons()]
def first_button():
# Single keyboard button.
return ['first button']
def second_button():
# Single keyboard button.
return ['second button']
def two_buttons():
# Two buttons on one line.
return ['left button', 'right button']
while STATUS:
# Get updates forever. Except if get Error.
try:
if not OFFSET:
OFFSET = 1
# Print bot info on the start.
bot = get_me()
print('Bot #{} is running...'.format(bot.username))
updates = get_updates()
for update in updates.result:
# Handle last update.
OFFSET = update['update_id'] + 1
message, user, chat = handle(update)
# Greeting user by full name.
greeting = 'Hello, {} {}!'.format(user.first_name, user.last_name)
#send_message(chat.id, greeting)
send_keyboard(chat.id, greeting, one_line_keyboard())
except:
STATUS = False
print('\nERROR:\t', error()[1])
###############################################################################
you can this code: I hope useful for you .
i change :
params = ({'chat_id': chat_id, 'reply_markup': reply_markup, 'disable_web_page_preview': 'true'})
myresponse =urlopen(URL + 'sendMessage' + quote((params).encode('utf-8')))
with:
params = ({'text': 'ss', 'chat_id': chat_id, 'reply_markup': reply_markup, 'disable_web_page_preview': 'true'})
data = urllib.parse.urlencode(params).encode("utf-8")
myresponse = urlopen(URL + 'sendMessage', data)
complate code :
import json
import urllib
from urllib.parse import quote
from urllib.request import urlopen
def Decode(telegram_response):
decoded = ''
for line in telegram_response:
decoded = decoded + line.decode('utf-8')
return decoded
TOKEN = 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
URL = 'https://api.telegram.org/bot{}/'.format(TOKEN)
cmd = 'getme'
telegram_response = urlopen(URL + cmd)
decoded = Decode(telegram_response)
gtm = json.loads(decoded)
status = True
while status:
cmd = 'getUpdates'
telegram_response = urlopen(URL + cmd)
decoded = Decode(telegram_response)
upds = json.loads(decoded)
new_message = len(upds['result'])
if new_message != 0:
msg = upds['result'][0]['message']
chat_id = str(msg['chat']['id'])
reply_markup = {'keyboard': [[{'text': 'first button'}], [{'text': 'second button'}]], 'resize_keyboard': True,
'one_time_keyboard': True}
reply_markup = json.dumps(reply_markup)
params = ({'text': 'ss', 'chat_id': chat_id, 'reply_markup': reply_markup, 'disable_web_page_preview': 'true'})
data = urllib.parse.urlencode(params).encode("utf-8")
myresponse = urlopen(URL + 'sendMessage', data)
I need to compare two values in data format
import groovy.sql.*
com.eviware.soapui.support.GroovyUtils.registerJdbcDriver( "oracle.jdbc.driver.OracleDriver" )
def messageId1 = context.expand( '${#Project#Id1}' )
def messageId2 = context.expand( '${#Project#Id2}' )
def first = sql.firstRow("select timestamp from table where Messageid = '"+messageId1+"'")
def second = sql.firstRow("select timestamp from table where Messageid = '"+messageId2+"'")
assert first < second, 'OK'
I get following error:
error: groovy.lang.MissingPropertyException No such property: sql for class: Script9
I had the same issue and the mistake that I made is, I didn't create a SQL Instance...So this would help you
def DBurl = '<Your database URL>'
def DBuser = '<Database user name>'
def DBpassword = '<Database Password>'
def DBdriver = '< Your database driver>'
def sql = Sql.newInstance(DBurl, DBuser, DBpassword, DBdriver)