Updating Cols in google Sheets - python-3.x

I am trying to copy columns from one sheet to another sheet. I get the columns in response from source sheet. I need to insert them into the sheet. Since methods like insertDimension and insertRange cannot do it. I used request = service.spreadsheets().values().update(spreadsheetId=to_spreadsheet_id, range=range_, valueInputOption = "USER_ENTERED", body={"values": response}) but it gives me error like this :- googleapiclient.errors.HttpError: <HttpError 400 when requesting https://sheets.googleapis.com/v4/spreadsheets/1sERXk6YshuNOKi4ggp11a36uf5SGutLg7DAP5vitOoQ/values/Working%20Analysis%21G2?valueInputOption=USER_ENTERED&alt=json returned "Invalid values[1][0]: struct_value {
fields {
key: "effectiveFormat"
value {
struct_value {
fields {
key: "backgroundColor"
value {
struct_value {
fields {
key: "blue"
value {
number_value: 1.0
}
}
fields {
key: "green"
value {
number_value: 1.0
}
Its a very long text......
AND if I use other way which is commented in the code block I get the following error
Details: "Invalid requests[0].updateCells: Attempting to write row: 15000, beyond the last requested row of: 14999">
def copy_column(service, from_spreadsheet_id, to_spreadsheet_id, from_sheet='Analysis', to_sheet_id, from_column='F', from_column_till='K', to_column='G'):
request = service.spreadsheets().get(spreadsheetId=from_spreadsheet_id, ranges=[
from_sheet + "!" + from_column + ":" + from_column_till], includeGridData=True)
response = request.execute()["sheets"][0]["data"][0]["rowData"]
range_ = "Working Analysis!G2"
print(response)
# value_range_body = {
# "requests": {
# "insertDimension": {
# "range": {
# "sheetId": to_sheet_id,
# "dimension": "COLUMNS",
# "startIndex": 6,
# "endIndex": 11
# },
# "inheritFromBefore": True
# }
# }
# }
# request_1 = service.spreadsheets().batchUpdate(spreadsheetId=to_spreadsheet_id, body=value_range_body)
# response_1 = request_1.execute()
# body = {
# "requests": {
# "updateCells": {
# "rows": response,
# "fields": "userEnteredFormat, userEnteredValue",
# # "start":{
# # "sheetId": to_sheet_id,
# # "rowIndex": 1,
# # "columnIndex": 6
# # },
# "range": {
# "sheetId": to_sheet_id,
# "startRowIndex": 1,
# "startColumnIndex": 6,
# "endColumnIndex": 13
# },
# }
# }
# }
request = service.spreadsheets().values().update(spreadsheetId=to_spreadsheet_id,
range=range_, valueInputOption = "USER_ENTERED", body={"values": response})
response = request.execute()
return print('Done')

I think that the response values from service.spreadsheets().get() cannot be directly used to service.spreadsheets().values().update(). From your commented script, I guessed that you might want to copy not only the values but also the cell format.
In this case, how about the following modification?
Modified script:
sheetId = "###" # Please set the sheet ID of the sheet "Working Analysis"
request = service.spreadsheets().get(spreadsheetId=from_spreadsheet_id, ranges=[from_sheet + "!" + from_column + ":" + from_column_till], includeGridData=True)
response = request.execute()["sheets"][0]["data"][0]["rowData"]
requests = {
"requests": [
{
"updateCells": {
"start": {"sheetId": sheetId, "rowIndex": 1, "columnIndex": 6},
"rows": response,
"fields": "*",
}
}
]
}
request = service.spreadsheets().batchUpdate(spreadsheetId=to_spreadsheet_id, body=requests)
response = request.execute()
print("Done")
For example, if you want to copy only the values, you can use service.spreadsheets().values().update() as follows.
request = service.spreadsheets().values().get(spreadsheetId=from_spreadsheet_id, range=from_sheet + "!" + from_column + ":" + from_column_till)
response = request.execute()["values"]
range_ = "Working Analysis!G2"
request = service.spreadsheets().values().update(spreadsheetId=to_spreadsheet_id, range=range_, valueInputOption="USER_ENTERED", body={"values": response})
response = request.execute()
return print("Done")
References:
Method: spreadsheets.batchUpdate
Method: spreadsheets.values.get

Related

SF OCAPI pagination loop

I am trying to get a pagination loop inside of another loop where I query multiple environments in salesforce OCAPI. My code goes like this. I first declare some variables to use in the loop:
URLs=["it","at","lv","ee","lt"]
base_url = "https://example.net/s/-/dw/data/v22_4/customer_lists/"
start = 0
count = 200
Then I create the body for the requests:
body = """
{
"query": {
"bool_query": {
"must": [
{
"filtered_query": {
"query": {
"match_all_query": {}
},
"filter": {
"range_filter": {
"field": "creation_date",
"from": """ + start_time_formatted + """,
"to": """ + end_time_formatted + """
}
}
}
}
]
}
},
"expand":["primary_address"],
"select": "(**)",
"count": """ + str(count) + """,
"start": """ + str(start) + """
}"""
Based on these variables I create an initial For loop where I query multiple URL environments, I tried to input a while loop for pagination within the For loop but it doesn't give me the expected result. The code is below:
for country in URLs:
url = base_url + str(country) + "/customer_search"
response = requests.post(url, headers={'Content-Type': 'application/json','Accept': 'application/json','Authorization': 'Bearer ' + oauth_token}, data=body)
data = response.json()
while (start < total):
start = start * count
response = requests.post(url, headers={'Content-Type': 'application/json','Accept': 'application/json','Authorization': 'Bearer ' + oauth_token}, data=body)
data = response.json()
dataframe = pd.json_normalize(data['hits'], max_level=2)
dfs.append(dataframe)
df = pd.concat(dfs, ignore_index=True)
The json output that I usually get from the query looks similar to this(with records inside hits course):
{{'_v': '22.4',
'_type': 'customer_search_result',
'count': 200,
'expand': ['primary_address'],
'hits': [],
'next': {'_type': 'result_page', 'count': 200, 'start': 200},
'query': {'bool_query': {'_type': 'bool_query',
'must': [{'filtered_query': {'_type': 'filtered_query',
'filter': {'range_filter': {'_type': 'range_filter',
'field': 'creation_date',
'from': '2022-01-01T00:00:00.000Z',
'to': '2022-05-09T17:03:16.000Z'}},
'query': {'match_all_query': {'_type': 'match_all_query'}}}}]}},
'select': '(**)',
'start': 0,
'total': 650}
Now what I need is that the while loop increases the starting point with each iteration and collects all total records for each URL by then stopping in the end and give the results. Do you have any idea on how I can create this loop? count is 200 because that's the maximum amount of records I'm allowed to query with each call. Also count should then change based on total amount of records so for example if it goes in batches of 200 then for the last 150 count should change to 150 instead of 200.

Python nested json

Can any one have solution for this, i want there should be api data in this manner ??
I wanted api data in for similar state comes in one hood rather than seprate, different state data can be different obj,
data = [{
state_name:New_jersi, data:{
category:Phishing,
sub_cat_data:[{
name:SubCat1,
count:20
},
{
name:SubCat2,
count:30
}]
}
category: malware,
sub_cat_data:[{
name:SubCat1,
count:20
},
{
name:SubCat2,
count:30
}]
},
{
state_name:Washinton, data:{
category:Phishing,
data:[{
name:SubCat1,
count:20
},
{
name:SubCat2,
count:30
}]
}
}]
But may api response be:
{
"state": "South Carolina",
"state_count": 2,
"Website Compromise/Intrusion": {
"sub_category": {
"Insecure Direct Object Reference": 2,
"Memory Corruption": 2,
"SQLI": 1,
"Stack Overflow": 1,
"XSRF": 1,
"Heap Overflow": 1,
"Security Misconfiguration": 1
}
}
},
{
"state": "South Carolina",
"state_count": 1,
"Phishing": {
"sub_category": {
"Spear Phishing Attacks": 2,
"Fast Flux": 2,
"Rock fish": 2,
"Identify Theft/Social Engineering": 1,
"Phishing Redirector": 1,
"Pharming": 1,
"Exploitation of Hardware Vulnerability": 1
}
}
},
i wanted same state data be in same object buut in my case state data comes in seprate object because of data comes through category, rather that seprate.
My logic are below
cat_count = incnum.values('incident_category__cat_name','incident_category__cat_id').annotate(count=Count('incident_category__cat_id'))
subcat_count = incnum.values('incident_sub_category__sub_cat_name','incident_sub_category__cat_id','incident_sub_category__id').annotate(count=Count('incident_sub_category__cat_id'))
reporter_state_count1 = incnum.values('incident_category__cat_id','reporter__comp_individual_state','reporter__comp_individual_state__name').annotate(count=Count('incident_category__cat_id'))
for x, state_ in enumerate(reporter_state_count1):
for i, cat_ in enumerate(cat_count):
if state_['incident_category__cat_id'] == cat_['incident_category__cat_id']:
for i, cat_ in enumerate(cat_count):
if state_['incident_category__cat_id'] == cat_['incident_category__cat_id']:
arr16.append({'state':state_['reporter__comp_individual_state__name'], 'state_count':state_['count'], cat_['incident_category__cat_name']:{'sub_category':{}}})
for sub_ in subcat_count:
if cat_['incident_category__cat_id'] == sub_['incident_sub_category__cat_id']:
arr16[i][cat_['incident_category__cat_name']]['sub_category'].update({sub_['incident_sub_category__sub_cat_name']:sub_['count']})
cat_count = incnum.values('incident_category__cat_name', 'incident_category__cat_id').annotate(
count=Count('incident_category__cat_id'))
subcat_count = incnum.values('incident_sub_category__sub_cat_name', 'incident_sub_category__cat_id',
'incident_sub_category__id').annotate(count=Count('incident_sub_category__cat_id'))
reporter_state_count1 = incnum.values('incident_category__cat_id', 'reporter__comp_individual_state',
'reporter__comp_individual_state__name').annotate(
count=Count('incident_category__cat_id'))
arr16 = []
for state_ in reporter_state_count1:
state_data = {"state_name" : state_['reporter__comp_individual_state__name'], "data":[]}
for cat_ in cat_count:
if state_['incident_category__cat_id'] == cat_['incident_category__cat_id']:
sub_cat_data = [{sub_['incident_sub_category__sub_cat_name']: sub_['count']} for sub_ in subcat_count if cat_['incident_category__cat_id'] == sub_['incident_sub_category__cat_id']]
category_data = {"category": cat_['incident_category__cat_name'], "sub_cat_data": sub_cat_data}
state_data["data"].append(category_data)
arr16.append(state_data)
1 State might have multiple category, the way you are trying to make your api, it won't be able to show multiple category for a state. This is why i modify a little bit. you will find all the category in state object
Edit
Creating a dictionary which will store category_id as key and all the subcategory of that category as value
cat_to_subcat_list = {}
for cat_ in cat_count:
sub_cat_data = [{"name":sub_['incident_sub_category__sub_cat_name'],"count": sub_['count']} for sub_ in subcat_count if
cat_['incident_category__cat_id'] == sub_['incident_sub_category__cat_id']]
cat_to_subcat_list[cat_['incident_category__cat_id']] = {"category": cat_['incident_category__cat_name'], "sub_cat_data": sub_cat_data}
Createing a dictionary which will store state__name as key and a list of category object will save as value
state_data = {}
for state_ in reporter_state_count1:
if state_['reporter__comp_individual_state__name'] not in state_data:
'''This if statement is checking whether state_name exit or not.
if state_name does not exist in dictionary it'll create a empty list as it's value'''
state_data[state_['reporter__comp_individual_state__name']] = []
state_data[state_['reporter__comp_individual_state__name']].append(cat_to_subcat_list[state_['incident_category__cat_id']])
Re-formatting json as api needed
arr16 = [
{
"state_name": state_name,
"data": state_data
}for state_name, state_data in state_data.items()
]

search a list in api/ format url seach for api

I want to generate 6 random numbers for pokemon api ID.
Put in list.
Then use the 6 numbers in url search.
The url doesn't recognise the list.
I need to convert the list to numbers. I'm not sure how to format them into the url.
import random
import requests
pokemon_ID = []
# pokemon_ID_add = str(pokemon_ID)[1:-1]
# pokemon_ID2 = str(pokemon_ID)[1:-1]
for i in range(0,6):
number = random.randint(1 ,151)
while i in pokemon_ID:
number = random.randint(1, 151)
pokemon_ID.append(number)
url = 'https://pokeapi.co/api/v2/pokemon/{}/'.format(pokemon_ID)
response = requests.get(url)
pokemon = response.json()
print(pokemon)
You can use loop to iterate over random IDs and store the result to a list:
import json
import random
import requests
url = "https://pokeapi.co/api/v2/pokemon/{}/"
random_pokemon_ids = [random.randint(1, 151) for i in range(6)]
result = []
for id_ in random_pokemon_ids:
pokemon = requests.get(url.format(id_)).json()
result.append(pokemon)
# pretty print the result:
print(json.dumps(result, indent=4))
Prints:
[
{
"abilities": [
{
"ability": {
"name": "rock-head",
"url": "https://pokeapi.co/api/v2/ability/69/"
},
"is_hidden": false,
"slot": 1
},
{
"ability": {
"name": "lightning-rod",
"url": "https://pokeapi.co/api/v2/ability/31/"
},
"is_hidden": false,
"slot": 2
},
{
"ability": {
"name": "battle-armor",
"url": "https://pokeapi.co/api/v2/ability/4/"
},
"is_hidden": true,
"slot": 3
}
],
"base_experience": 64,
"forms": [
{
"name": "cubone",
"url": "https://pokeapi.co/api/v2/pokemon-form/104/"
}
],
...

Convert Nested Json to a single Key Value Pair

I have a list of Keys :
Server
Date
Severity
process
message
If there is a key "events" in my Dictionary, I need to replace the key with "Message" and the value of "indicator-type" will become the value of the message
List1=[
{
"server": "10.10.0.123",
"date": 1561994754,
"severity": "INFO",
"process": "webapp",
"message": "server started."
},
{
"server": "10.10.0.202",
"date": 1561994757000,
"source": "jvm-x994a",
"events": [{
"indicator-type": "memory-low"
}]
},
{
"server": "10.10.0.202",
"date": 1561994773000,
"source": "jvm-x994a",
"events": [{
"indicator-type": "memory-low"
}
]
}
]
Expected output:
List1=[
{
"server": "10.10.0.123",
"date": 1561994754,
"severity": "INFO",
"process": "webapp",
"message": "server started."
},
{
"server": "10.10.0.202",
"date": 1561994757000,
"source": "jvm-x994a",
"message": "memory-low"
},
{
"server": "10.10.0.202",
"date": 1561994773000,
"source": "jvm-x994a",
"message": "memory-low"
}
]
}
]
So Far I have Tried :
In the below code I am trying to check for the "events" key and replacing it with the "message" key, but when I am printing it is not replacing they key-value
output = []
d={}
for x in a:
d['server']=x['server']
if (str(x['date']).isdigit() == True):
d['date'] =2
# datetime.datetime.fromtimestamp(x['date'] / 1000.0).strftime('%Y-%m-%d %H:%M:%S.%f')
if ("events" in x):
for i in (x['events']):
if (i['indicator-level'] == 3):
d['severity'] = "INFO"
if (i['indicator-level'] == 7):
d['severity'] = "WARN"
if (i['indicator-type'] != None):
d['message'] = i['indicator-type']
Clean out the dictionary first before moving on to the next step.
def handle_events_key(dict_):
"""
Adds a new 'message' key to 'dict_' based on 'events' key.
Returns a new dictionary without the 'events' key.
"""
if 'events' in dict_:
dict_['message'] = dict_['events'][0]['indicator-type']
return {key: value for key, value in dict_.items() if key != 'events'}
def events_key_to_message(list_):
return [handle_events_key(dict_) for dict_ in list_]
# Usage
# List1 = ...
# Cleaning out the dictionary.
ListUpdated = events_key_to_message(List1)
Try this. I have modified the events handling parts.
output = []
for item in List1:
d = {}
d['server'] = item['server']
if (str(item['date']).isdigit() == True):
d['date'] =2
# datetime.datetime.fromtimestamp(x['date'] / 1000.0).strftime('%Y-%m-%d %H:%M:%S.%f')
# get will give you None and not raise an exception if there is no events key.
if item.get('events') is not None:
for event in item['events']:
if event.get('indicator-level') == 3: d['severity'] = "INFO"
if event.get('indicator-level') == 7: d['severity'] = "WARN"
if event.get('indicator-type') is not None: d['message'] = event['indicator-type']
output.append(d)
print(output)
# [{'server': '10.10.0.123', 'date': 2}, {'server': '10.10.0.202', 'date': 2, 'message': 'memory-low'}, {'server': '10.10.0.202', 'date': 2, 'message': 'memory-low'}]

Data alignment in Python

I am new to Python. I am writhing a code to generate a excel file having the data sourced by calling API and correlate those to get desired result.
basically taking input from one database and search that in others and fetch related information.
The 4 databases have below data :
EEp
---------------------
{u'data': [{u'_id': u'5c30702c8ca9f51da8178df4',
u'encap': u'vlan-24',
u'ip': u'7.12.12.16',
u'mac': u'5B:P9:01:9E:42:08'}]}
PathEp
-----------
{u'data': [{u'_id': u'5c54a81a8ca9f51da84ae08e',
u'paths': u'paths-1507',
u'endpoint': u'eth1/10',
u'cep': u'5B:P9:01:9E:42:08',
u'tenant': u'ESX'}]}
ip4_address
-----------------------
{u'data': [{u'Allocation': u'Build_Reserved',
u'address': u'7.12.12.16',
u'name': u'fecitrix-1',
u'state': u'RESERVED'}]}
asset
---------------
{u'data': [{u'_id': u'57ccce8110dd54f02881fedc',
u'client': u'CES',
u'hostname': u'fecitrix-1'
u'os_team': u'Window'}]}
Logic:
If "mac" of EEp and "cep" of PathEp is same than take "encap","ip" ,"mac"
"paths" ,'endpoint","cep" and "tenant" (these values need to be exported
to excel)
Take ip of EEp and search in "ip4_address"
and get the "name" from ip4_address ( name need to be exported to excel).
If "name" of ip4_address is equal to "hostname" of database "asset" then take
"client" and "os_team" ( export that to excel)
I have written the script but not getting the desired result.
def get_host_details(self):
data = {
"find": {
"hostname": self.controller
},
"projection":{
"tenant": 1,
"paths": 1,
"endpoint":1
}
}
host_details = self.post("https://database.app.com/api/data/devices/PathEp/find", data)
#print host_details
hosts = []
for record in host_details:
if "mig" not in record["endpoint"]:
hosts.append(record)
return hosts
def get_ipaddress(self, controller):
host_record = {"tenant": "UNKNOWN",
"paths": "UNKNOWN",
"endpoint": "UNKNOWN",
"ip": "UNKNOWN",
"mac": "UNKNOWN",
"encap": "UNKNOWN"}
data = {
"find": {
"hostname": controller,
"ip": {
"$ne": "0.0.0.0"
}
},
"projection": {
"ip": 1,
"mac":1,
"encap":1,
}
}
endpoints = self.post("https://database.app.com/api/data/devices/EEp/find", data)
IPAM = self.get_dns()
print endpoints
host_details = self.get_host_details()
host_details_record = []
for record in endpoints:
for host in host_details:
if record["mac"] == host["cep"]:
host_record = {"tenant": host["tenant"],
"paths": host["paths"],
"endpoint": host["endpoint"],
"ip": record["ip"],
"mac": record["mac"],
"encap": record["encap"]}
host_details_record.append(host_record)
self.get_excel(host_details_record)
def get_dns(self, endpoints):
ip_dns_record = []
for each_endpoint in endpoints:
data = {
"find":
{
"address": {
"$eq": each_endpoint["ip"]
},
},
"projection":
{
"name": 1
}
}
dns_record = {"client":"UNKNOWN",
"os_team":"UNKNOWN",
ipam_record = self.post("https://database.app.com/api/data/"
"internal/ip4_address/find", data)
if ipam_record:
dns_record["ip_address"] = each_endpoint["ip"]
dns_record["hostname"] = ipam_record[0]["name"]
dns_record = self.get_remedy_details(ipam_record[0]["name"],
dns_record)
ip_dns_record.append(dns_record)
else:
dns_record["ip_address"] = each_endpoint["ip"]
dns_record["hostname"] = "UNKNOWN"
ip_dns_record.append(dns_record)
self.get_excel(ip_dns_record)
def get_remedy_details(self, hostname, dns_record):
data = {
"find":
{
"hostname": hostname.upper(),
}
}
remedy_data = self.post("https://database.app.com/api/data/internal/asset/find", data)
print(remedy_data)
#remedy_data = remedy_data["data"]
if remedy_data:
dns_record["client"] = remedy_data[0].get("client","UNKNOWN")
dns_record["os_team"] = remedy_data[0].get("os_team", "UNKNOWN")
else:
dns_record["client"] = "UNKNOWN"
dns_record["os_team"] = "UNKNOWN"
return dns_record
def get_excel(self, ip_dns_record):
filename = self.controller + ".xls"
excel_file = xlwt.Workbook()
sheet = excel_file.add_sheet('HOSTLIST')
sheet.write(0, 0, "IP Address")
sheet.write(0, 1, "HostName")
sheet.write(0, 2, "Client")
sheet.write(0, 3, "OS Team")
for count in xrange(1, len(ip_dns_record)+1):
sheet.write(count, 0,ip_dns_record[count - 1]["ip_address"])
sheet.write(count, 1,ip_dns_record[count - 1]["hostname"])
sheet.write(count, 2,ip_dns_record[count - 1]["client"])
sheet.write(count, 3,ip_dns_record[count - 1]["os_team"])
excel_file.save(filename)
if __name__ == "__main__":
controller = sys.argv[1]
OBJ = ACIHostList(controller)
print "SCRIPT COMPLETED"
No idea where I am going wrong and what needs to be done .
Your question leaves too much out. You should include all errors that you get. You should also comment your code as well so we can understand what you are trying to achieve in each step.
This is not an answer but something to try:
Rather than trying to wrap your head around a module like excel, wright your data to a simple CSV file. A CSV file can be opened up in excel and it formats correctly but is a lot easier to create.
import csv
data = [["a", "b"], ["c", "d"]]
with open("file.csv", "w+") as csv_file:
create_csv = csv.writer(csv_file)
create_csv .writerows(data)
simply grab all your data into a 2D list and using the above code dump it into a file so you can easily read it.
check the output of the file and see if you are getting the data you expect.
If you are not getting the desired data into this CSV file then there is an issue with your database queries.

Resources