So I have a collection in ArangoDB where subnets are assigned to teams and I would like to re-format that data to have a single line per team with an array of their assigned subnets
The collection (Assigment) looks like this:
Query
FOR a IN Assigment
RETURN a
Result
[
{
"assignment_type": "subnet",
"assigment_crteria": "192.168.1.0/24",
"team_assignment": "Team1",
"prime_contact": "John",
},
{
"assignment_type": "subnet",
"assigment_crteria": "192.168.2.0/24",
"team_assignment": "Team1",
"prime_contact": "John",
},
{
"assignment_type": "subnet",
"assigment_crteria": "192.168.3.0/24",
"team_assignment": "Team1",
"prime_contact": "John",
},
{
"assignment_type": "subnet",
"assigment_crteria": "192.168.4.0/24",
"team_assignment": "Team2",
"prime_contact": "Bob",
},
{
"assignment_type": "subnet",
"assigment_crteria": "192.168.5.0/24",
"team_assignment": "Team2",
"prime_contact": "Bob",
},
{
"assignment_type": "subnet",
"assigment_crteria": "192.168.8.0/24",
"team_assignment": "Team5",
"prime_contact": "Anna",
},
{
"assignment_type": "subnet",
"assigment_crteria": "192.168.9.0/24",
"team_assignment": "Team2",
"prime_contact": "Bob",
},
{
"assignment_type": "subnet",
"assigment_crteria": "192.168.10.0/24",
"team_assignment": "Team2",
"prime_contact": "Bob",
},
{
"assignment_type": "subnet",
"assigment_crteria": "192.168.12.0/24",
"team_assignment": "Team5",
"prime_contact": "Anna",
},
{
"assignment_type": "subnet",
"assigment_crteria": "192.168.13.0/24",
"team_assignment": "Team1",
"prime_contact": "John",
},
{
"assignment_type": "subnet",
"assigment_crteria": "192.168.15.0/24",
"team_assignment": "Team1",
"prime_contact": "John",
},
{
"assignment_type": "subnet",
"assigment_crteria": "192.168.17.0/24",
"team_assignment": "Team7",
"prime_contact": "Erick",
}
]
And I would like to insert the re-formated documents in a different collection (Team_to_Subnet) with this format:
Result
[
{
"team": "Team1",
"cidrs": ["192.168.1.0/24","192.168.2.0/24","192.168.3.0/24","192.168.13.0/24","192.168.15.0/24"],
"prime_contact": "John"
},
{
"team": "Team2",
"cidrs": ["192.168.4.0/24","192.168.5.0/24","192.168.9.0/24","192.168.10.0/24"],
"prime_contact": "Bob"
},
{
"team": "Team5",
"cidrs": ["192.168.8.0/24","192.168.12.0/24"],
"prime_contact": "Anna"
},
{
"team": "Team7",
"cidrs": ["192.168.17.0/24"],
"prime_contact": "Erick"
},
]
How would I go about doing this? Is it even possible to do it? I've looked into using COLLECT but I just cant figure it out.
Any help would be appreciated
You can group by team and contact with COLLECT, and use a projection expression to get an array of subsets that fall into the respective group:
FOR a IN Assigment
COLLECT team = a.team_assignment, prime_contact = a.prime_contact
INTO cidrs = a.assigment_crteria
RETURN { team, prime_contact, cidrs }
To save the results, just replace the last line with:
INSERT { team, prime_contact, cidrs } INTO Team_to_Subnet
Related
I have the following two data sets, and I'm looking to merge them in a very specific way:
Stream
stream = {
"sdd": [{
"eid": "e532fcdb-2f3f-4c51-9afb-996b8679a5da",
"nid": "ebef3cb1-9053-4d1e-b409-b682236445b7",
"nname": "client"
}, {
"eid": "901b51b1-851f-448e-b2cc-9730267f824b",
"nid": "0b5d79b5-0d21-4186-b3db-8f4a9f2b25fb",
"nname": "server"
}]
}
Profile
profile = [{
"ackid": "5dfe5dd4-e863-4474-bdad-54231b925f12",
"nid": "ebef3cb1-9053-4d1e-b409-b682236445b7",
"name": "default",
"resources": {
"cpu": {
"min_cores": "2",
"max_cores": "8"
},
"memory": {
"min": "2000",
"max": "10000"
},
"storage": {
"min": "50",
"max": "100"
}
}
}]
What I want to do is add to 'stream' from 'profile' the matching (min) values from 'resources' where both 'nid' match to get something like this:
Output
{
"sdd": [{
"eid": "e532fcdb-2f3f-4c51-9afb-996b8679a5da",
"nid": "ebef3cb1-9053-4d1e-b409-b682236445b7",
"nname": "client",
"cpu": "2",
"mem": "2000",
"storage":"50"
}, {
"eid": "901b51b1-851f-448e-b2cc-9730267f824b",
"nid": "0b5d79b5-0d21-4186-b3db-8f4a9f2b25fb",
"nname": "server"
}]
}
def get_dict(x):
return {'min':x['memory']['min'], 'cpu':x['cpu']['min_cores'],
'storage':x['storage']['min']}
Out = {'ssd':[i|get_dict(j['resources']) if i['nid']==j['nid'] else i for i in
stream['sdd'] for j in profile]}
Out:
{'ssd': [{'eid': 'e532fcdb-2f3f-4c51-9afb-996b8679a5da',
'nid': 'ebef3cb1-9053-4d1e-b409-b682236445b7',
'nname': 'client',
'min': '2000',
'cpu': '2',
'storage': '50'},
{'eid': '901b51b1-851f-448e-b2cc-9730267f824b',
'nid': '0b5d79b5-0d21-4186-b3db-8f4a9f2b25fb',
'nname': 'server'}]}
Try:
tmp = {}
for p in profile:
tmp[p["nid"]] = {
"cpu": p["resources"]["cpu"]["min_cores"],
"memory": p["resources"]["memory"]["min"],
"storage": p["resources"]["storage"]["min"],
}
out = {}
for k, v in stream.items():
out[k] = [{**i, **tmp.get(i["nid"], {})} for i in v]
print(out)
Prints:
{
"sdd": [
{
"eid": "e532fcdb-2f3f-4c51-9afb-996b8679a5da",
"nid": "ebef3cb1-9053-4d1e-b409-b682236445b7",
"nname": "client",
"cpu": "2",
"memory": "2000",
"storage": "50",
},
{
"eid": "901b51b1-851f-448e-b2cc-9730267f824b",
"nid": "0b5d79b5-0d21-4186-b3db-8f4a9f2b25fb",
"nname": "server",
},
]
}
I would like to group the dictionaries returned from api inside for loop by its given key.Values for type and query are dynamic, depends from request params from user
def process_item(self, item, spider):
print(f"item {item}")
prints
{
"type": "sponsored_product",
"query": "leather wallet",
"asin": "B092S9S3G3"
},
{
"type": "search_product",
"query": "leather wallet",
"asin": "B092S91234"
},
{
"type": "search_product",
"query": "mens wallet",
"asin": "B092S9789"
}
So what I am trying to achieve is to group them and save in to json file as list of dicts
[
"mens wallet": [
{
"search_product": [
{
"asin": "asin number123"
},
{
"asin": "asin number567"
},
],
"sponsored_product": [
{
"asin": "asin number123"
},
{
"asin": "asin number567"
},
]
}
],
"leather wallet": [
{
"search_product": [
{
"asin": "asin number123"
},
{
"asin": "asin number567"
},
],
"sponsored_product": [
{
"asin": "asin number123"
},
{
"asin": "asin number567"
},
]
}
],
]
I was thinking of saving it first to the file and perform grouping later but I cant make it work.Do you have Idea on how to group them before saving to a json file?
for k, v in item.items():
with open("list1.json", "a", encoding="utf-8") as f:
json.dump(v, f, ensure_ascii=False, indent=4)
UPDATE:
So i tried the following code
import itertools
import json
buck = [
{"type": "sponsored_product", "query": "leather wallet", "asin": "B092S9S3G3"},
{"type": "search_product", "query": "leather wallet", "asin": "B092S91234"},
{"type": "search_product", "query": "mens wallet", "asin": "B092S9789"},
]
def key_func(k):
return k["query"]
def key_func2(k):
return k["type"]
info = sorted(buck, key=key_func)
groups = []
for key, group in itertools.groupby(buck, key_func):
# print(list(group))
groups.append({key: list(group)})
print(groups)
And with output
[
{
'leather wallet': [
{
'type': 'sponsored_product',
'query': 'leather wallet',
'asin': 'B092S9S3G3'
},
{
'type': 'search_product',
'query': 'leather wallet',
'asin': 'B092S91234'
}
]
},
{
'mens wallet': [
{
'type': 'search_product',
'query': 'mens wallet',
'asin': 'B092S9789'
}
]
}
]
What's remaining is to group by type
I have this structure :
{
"_id": "blablabla",
"UUID": "50ED0565C5BF",
"Email": "foobar#bar.ca",
"FirstName": "Bacel",
"LastName": "ELbs",
"ClientID": "sa-sa",
"UserSettings": {
"dateRangeMode": "1",
"skillsMode": "1",
"CalendarViewMode": "One Month"
},
"Filters": [{
"Type": "Calendar",
"Filter": [],
"DefaultFilterID": ""
}, {
"Type": "Calendar",
"desc": "test",
"Filter": [],
"isDefault": false
}]
}
My objective is to first insert one new object to the Filters array and insert multiple items to the Filters.Filter embedded array :
so I'm trying the following approch :
static async saveFilter(UUID, Type, desc, isDefault, aFilters) {
//let objectID = UUID.replace(/-/g, "");
return await user.update({
UUID: UUID
}, {
$push: {
Filters: {
Type: Type,
desc: desc,
isDefault: isDefault
},
"Filters.Filter": ["a","b","c"]
}
});
}
I keep having this error :
Cannot create field 'Filter:' in element {Filters: [ { Type:
"Calendar", Filter: [], DefaultFilterID: "" }, { Type: "Calendar",
desc: "test", isDefault: false } ]}
But I don't want to creat a new field, I simply want to add elements,
desired output is something like :
{
"_id": "blablabla",
"UUID": "50ED0565C5BF",
"Email": "foobar#bar.ca",
"FirstName": "Bacel",
"LastName": "ELbs",
"ClientID": "sa-sa",
"UserSettings": {
"dateRangeMode": "1",
"skillsMode": "1",
"CalendarViewMode": "One Month"
},
"Filters": [{
"Type": "Calendar",
"Filter": [],
"DefaultFilterID": ""
}, {
"Type": "Calendar",
"desc": "test",
"Filter": [],
"isDefault": false
},
{
"Type": "Tower",
"desc": "Tesla",
"Filter": ["a","b","c"],
"isDefault": false
}]
Is this what you're looking for?
(playground)
db.users.update({
UUID: UUID
},
{
$push: {
Filters: {
Type: Type,
desc: desc,
isDefault: isDefault,
"Filter": [
"a",
"b",
"c"
]
}
}
})
{
"success": true,
"message": "Result",
"data": [
{
"Here": [
{
"_id": "5ee97ee7f25d1c1482717bdf",
"email": "test1#test.io",
"profileImages": [],
"username": "test1",
"birthday": "2020-06-11T10:11:32.000Z",
"phoneNumber": "+910000000000",
"location": "Test Location",
"firstName": "test1",
"lastName": "test1",
}
]
},
{
"Here": [
{
"_id": "5ee97ef2f25d1c1482717be1",
"email": "test2#test.io",
"profileImages": [],
"username": "test2",
"birthday": "2020-06-11T10:11:32.000Z",
"phoneNumber": "+910000000000",
"location": "Test Location"
}
]
}
],
}
What I am expecting is this
{
"success": true,
"message": "Result",
data: [
{
"_id": "5ee97ee7f25d1c1482717bdf",
"email": "test1#test.io",
"profileImages": [],
"username": "test1",
"birthday": "2020-06-11T10:11:32.000Z",
"phoneNumber": "+910000000000",
"location": "Test Location",
"firstName": "test1",
"lastName": "test1"},
{
"_id": "5ee97ef2f25d1c1482717be1",
"email": "test2#test.io",
"profileImages": [],
"username": "test2",
"birthday": "2020-06-11T10:11:32.000Z",
"phoneNumber": "+910000000000",
"location": "Test Location"
}
]
}
Query I am using is for this response is below using aggregation in mongodb, lookup and project which is leading me to the some undesired response
db.collections.aggregate( [
{
$lookup: {
from: 'users',
as: 'Here',
let: {
whoDid: '$whoDid'
},
pipeline: [
{
"$match": { "$expr": { "$eq": ["$_id", "$$whoDid"] } }
},
{
$project: {
_id: 1,
email: 1,
profileImages: 1,
username: 1,
birthday: 1,
phoneNumber: 1,
firstName: 1,
lastName: 1,
fullName: 1,
// age: {$year: "$birthday"}
age: {
$divide: [{ $subtract: [new Date(), "$birthday"] },
(31558464000)]
}
}
}
],
}
},
{
$project:{
Here:1,
_id:0
}
} ,
])
who did table is one of the collection I have where I have stored the user Id and later I am populating the data using lookup
{
"_id" : ObjectId("5ee988eb1aac0022e15dbb7b"),
"whoDid" : ObjectId("5ee97ef2f25d1c1482717be1"),
"toWhomDid" : ObjectId("5ee97ec0f25d1c1482717bdd"),
"modified_at" : ISODate("2020-06-17T03:07:23.217Z"),
"created_at" : ISODate("2020-06-17T03:07:23.217Z"),
"__v" : 0
}
{
"_id" : ObjectId("5ee988eb1aac0022e15dbb7c"),
"whoDid" : ObjectId("5ee97ec0f25d1c1482717bdd"),
"toWhomDid" : ObjectId("5ee97ef2f25d1c1482717be1"),
"modified_at" : ISODate("2020-06-17T03:07:23.220Z"),
"created_at" : ISODate("2020-06-17T03:07:23.220Z"),
"__v" : 0
}
Can anyone suggest me any better option so that I can get a desired respose?
It is possible to use reduce method:
obj.data = obj.data.reduce((a, c) => {
a.push(...c.Here);
return a;
}, [])
An example:
let obj = {
"success": true,
"message": "Result",
"data": [ {
"Here": [ {
"_id": "5ee97ee7f25d1c1482717bdf", "email": "test1#test.io",
"profileImages": [], "username": "test1",
"birthday": "2020-06-11T10:11:32.000Z", "phoneNumber": "+910000000000", "location": "Test Location",
"firstName": "test1", "lastName": "test1",
}
]
},
{
"Here": [ {
"_id": "5ee97ef2f25d1c1482717be1",
"email": "test2#test.io",
"profileImages": [],
"username": "test2",
"birthday": "2020-06-11T10:11:32.000Z",
"phoneNumber": "+910000000000",
"location": "Test Location"
}
]
}
]
};
obj.data = obj.data.reduce((a, c) => {
a.push(...c.Here);
return a;
}, [])
console.log(obj);
Add these extra steps into your aggregation pipeline:
{
$unwind: "$Here"
},
{
$replaceWith: "$Here"
}
MongoPlayground
Note: You can replace $project: { _id: 1, email: 1, ... to this:
{
$addFields:{
age: {
$divide: [{ $subtract: [new Date(), "$birthday"] },(31558464000)]
}
}
}
I have a collections of bunch of cities and zip codes, where name of a city can be a same, while zip code is different.
I'm trying to query thru all cities starting with 'San' for example and want to filter results where San Antonio appears only ones along with any other cities starting with San.
Here is a code I use:
Zipcodes.find(
{ city: { $regex: /San/, $options: 'i' } },
(err, result) => {
if (err) {
return res.send({ error: err });
}
res.send({ data: result });
},
).limit(20);
This given me following results:
{
"loc": [
-94.132581,
31.515173
],
"_id": "75972",
"city": "SAN AUGUSTINE",
"pop": 5916,
"state": "TX"
},
{
"loc": [
-98.730929,
31.162678
],
"_id": "76877",
"city": "SAN SABA",
"pop": 4023,
"state": "TX"
},
{
"loc": [
-100.481752,
31.478165
],
"_id": "76901",
"city": "SAN ANGELO",
"pop": 23800,
"state": "TX"
},
{
"loc": [
-100.480036,
31.419411
],
"_id": "76904",
"city": "SAN ANGELO",
"pop": 25535,
"state": "TX"
},
{
"loc": [
-100.390005,
31.464738
],
"_id": "76905",
"city": "SAN ANGELO",
"pop": 11284,
"state": "TX"
},
{
"loc": [
-100.438586,
31.470735
],
"_id": "76903",
"city": "SAN ANGELO",
"pop": 32471,
"state": "TX"
},
{
"loc": [
-95.034496,
29.466033
],
"_id": "77539",
"city": "SAN LEON",
"pop": 21905,
"state": "TX"
},
{
"loc": [
-99.427148,
27.062523
],
"_id": "78067",
"city": "SAN YGNACIO",
"pop": 871,
"state": "TX"
},
{
"loc": [
-98.460127,
29.414799
],
"_id": "78203",
"city": "SAN ANTONIO",
"pop": 7261,
"state": "TX"
},
{
"loc": [
-98.525967,
29.422855
],
"_id": "78207",
"city": "SAN ANTONIO",
"pop": 58355,
"state": "TX"
},
{
"loc": [
-98.5063,
29.400217
],
"_id": "78204",
"city": "SAN ANTONIO",
"pop": 11526,
"state": "TX"
},
{
"loc": [
-98.479338,
29.441338
],
"_id": "78215",
"city": "SAN ANTONIO",
"pop": 1264,
"state": "TX"
},
{
"loc": [
-98.545219,
29.358366
],
"_id": "78211",
"city": "SAN ANTONIO",
"pop": 30417,
"state": "TX"
},
{
"loc": [
-98.492509,
29.423711
],
"_id": "78205",
"city": "SAN ANTONIO",
"pop": 1714,
"state": "TX"
},
{
"loc": [
-98.497511,
29.533387
],
"_id": "78216",
"city": "SAN ANTONIO",
"pop": 30435,
"state": "TX"
},
{
"loc": [
-98.419444,
29.539525
],
"_id": "78217",
"city": "SAN ANTONIO",
"pop": 27925,
"state": "TX"
}
It's returning San Antonio many times. I need only ones.
Please help with a correct query. Thanks.
You can use $group + $replaceRoot to get values unique by city:
db.collection.aggregate([
{
$match: { city: { $regex: "San", $options: "i" } }
},
{
$group: {
_id: "$city",
doc: { $first: "$$ROOT" }
}
},
{
$replaceRoot: {
newRoot: "$doc"
}
}
])
Mongo Playground