cannot translate vega-light color condition to altair - altair

I have this vlspec (it is working)
I need to translate it to altair
vlspec.encoding.color =
{
"scale": {
"type": "threshold",
"domain": [
"< 10",
"10 - 70",
"≥ 70"
],
"range": [
"#cb5047",
"#f1ad4b",
"#486d49"
]
},
"condition": [
{
"test": {
"field": "rate",
"lt": 10
},
"value": "#cb5047"
},
{
"test": {
"field": "rate",
"lt": 70
},
"value": "#f1ad4b"
},
{
"test": {
"field": "rate",
"gte": 70
},
"value": "#486d49"
}
]
}
I trying a lot of options for alt.Condition, but failed to translate this to altair. Can somebody help please?

Altair uses Vega-Lite 4.8, in which it is not permitted to include a "scale" specification in a "value" encoding (it has no effect anyway). Once you take that out, you can use your Vega-Lite specification directly in your Altair chart. For example:
import altair as alt
import pandas as pd
df = pd.DataFrame({
'x': [1, 2, 3, 4],
'rate': [5, 20, 40, 80]
})
color = {
"condition": [
{
"test": {
"field": "rate",
"lt": 10
},
"value": "#cb5047"
},
{
"test": {
"field": "rate",
"lt": 70
},
"value": "#f1ad4b"
},
{
"test": {
"field": "rate",
"gte": 70
},
"value": "#486d49"
}
]
}
alt.Chart(df).mark_point().encode(
x='x',
y='rate',
color=color
)

Related

Filter by Product Properties with Store API

For the store API endpoint /store-api/product is it possible to filter on the properties of a product? Not the defaults such as whether it's active or stock levels, but the properties we've defined on the product, for example colour or farbe? For the search endpoint it supports passing in a list of properties ID's which this one does not.
None of the below queries work, and return the various errors below or Call to a member function buildAccessor() on null.
{
"limit": 40,
"filter": [
{
"type": "contains",
"field": "Farbe",
"value": "red"
}
]
}
"Field \"Farbe\" in entity \"product\" was not found."
{
"limit": 40,
"filter": [
{
"type": "contains",
"field": "properties.Farbe",
"value": "red"
}
]
}
"Field \"Farbe\" in entity \"property_group_option\" was not found."
You can combine filters for the name of the property value and their respective group in a multi filter. The following example will only give you products that have the "shoe-color" property with the value "coral".
{
"limit": 1,
"includes": {
"product": ["id", "productNumber", "properties"],
"property_group_option": ["name", "group"],
"property_group": ["name"]
},
"associations": {
"properties": {
"associations": {
"group": []
}
}
},
"filter": [
{
"type": "multi",
"operator": "and",
"queries": [
{
"type": "equals",
"field": "properties.group.name",
"value": "shoe-color"
},
{
"type": "equals",
"field": "properties.name",
"value": "coral"
}
]
}
]
}
Example response:
{
"entity": "product",
"total": 1,
"aggregations": [],
"page": 1,
"limit": 1,
"elements": [
{
"productNumber": "6bbfe1f608504c9b9a7bf92d6a071734",
"properties": [
{
"name": "coral",
"group": {
"name": "shoe-color",
"apiAlias": "property_group"
},
"apiAlias": "property_group_option"
},
{
"name": "cotton",
"group": {
"name": "textile",
"apiAlias": "property_group"
},
"apiAlias": "property_group_option"
}
],
"id": "062ba988aa1840fa84371c9c43b2f838",
"apiAlias": "product"
}
],
"states": [],
"apiAlias": "dal_entity_search_result"
}

python elasticsearch-dsl return all unique values for specific key

I have a field called account_number . It contains random 6 character string.
I can't seem to get python elasticsearch dsl to return just those unique values.
search = Search(using=client, index=index_name).query(
{
"range": {
"date": {
"gte": "2021-08-01T08:00:00.000Z",
"lte": "2021-08-31T23:59:59.599Z"
#"format": "strict_date_optional_time"
}
}
})
search.aggs.bucket("account_number","terms",field="account_number",size="1000")
es_data = search.execute()
Not sure if I need to define the account_number in the query or if its in the agg bucket?. Right now I just get random full rows returned with all columns
Here is an example of a working query in non-dsl form. I didnt think the metric was necessary but maybe it is.
{
"aggs": {
"3": {
"terms": {
"field": "account_number",
"order": {
"1": "desc"
},
"size": 5
},
"aggs": {
"1": {
"sum": {
"field": "hits"
}
}
}
}
},
"size": 0,
"stored_fields": [
"*"
],
"script_fields": {},
"docvalue_fields": [
{
"field": "#timestamp",
"format": "date_time"
},
{
"field": "date",
"format": "date_time"
}
],
"_source": {
"excludes": []
},
"query": {
"bool": {
"must": [],
"filter": [
{
"match_all": {}
},
{
"range": {
"date": {
"gte": "2021-04-08T21:00:00.000Z",
"lte": "2021-10-08T21:00:00.000Z",
"format": "strict_date_optional_time"
}
}
}
]
}
}
}
You can add extra(size=0) to your query:
search = Search(using=client, index=index_name).query(
{
"range": {
"date": {
"gte": "2021-08-01T08:00:00.000Z",
"lte": "2021-08-31T23:59:59.599Z"
#"format": "strict_date_optional_time"
}
}
}).extra(size=0)
Then your es_data will be empty and es_data.aggregations.account_number.buckets will contain only unique account numbers.
Hope it helps.

How to format grafana Alert on teams via webhooks

when the alert is sent to my team’s channel, the pipeline name is truncated [see image below]
alert on teams triggerd on grafana
I would like to know if there is anything I can do to make the entire variable name appear
I’m using grafana v7.5 and my data comes from Azure logs
{
"firing": true,
"state": "pending",
"conditionEvals": "true = true",
"timeMs": "430.287ms",
"matches": [
{
"metric": "n {PipelineName=PPLCRGLOADORLTOAPOSENTEVT}",
"value": 11
}
],
"logs": [
{
"message": "Condition[0]: Query",
"data": {
"from":number,
"queries": [
{
"refId": "A",
"model": {
"appInsights": {
"dimension": [],
"metricName": "select",
"timeGrain": "auto"
},
"azureLogAnalytics": {
"query": "ADFActivityRun\r\n| where (Status == 'Failed') and (substring(ResourceId,135,3) == 'PRD') and TimeGenerated > ago(8h)\r\n| summarize n=count() by bin(TimeGenerated, 5min),PipelineName\r\n\r\n| project n,TimeGenerated,PipelineName",
"resultFormat": "time_series",
"workspace": "workspace"
},
"azureMonitor": {
"aggOptions": [],
"dimensionFilter": "*",
"dimensionFilters": [],
"metricDefinition": "select",
"metricName": "select",
"metricNamespace": "select",
"resourceGroup": "select",
"resourceName": "select",
"timeGrain": "auto",
"timeGrains": [],
"top": "10"
},
"insightsAnalytics": {
"query": "",
"resultFormat": "time_series"
},
"queryType": "Azure Log Analytics",
"refId": "A",
"subscription": "subscription"
},
"datasource": {
"id": 3,
"name": "LA-DATAANALYTICS-PRD"
},
"maxDataPoints": 0,
"intervalMs": 0
}
],
"to": #number
}
},
{
"message": "Condition[0]: Query Result",
"data": {
"fromDataframe": true,
"series": [
{
"name": "n {PipelineName=PPLCRGLOADORLTOAPOSENTEVT}",
"points": [
[
1,
1628064600000
],
[
1,
1628064900000
],
[
1,
1628065200000
],
[
1,
1628065500000
],
[
1,
1628065800000
],
[
1,
1628066100000
],
[
1,
1628066700000
],
[
1,
1628067000000
],
[
1,
1628067300000
],
[
1,
1628067600000
],
[
2,
1628067900000
]
],
"tags": {
"PipelineName": "PPLCRGLOADORLTOAPOSENTEVT"
}
}
]
}
},
{
"message": "Condition[0]: Eval: true, Metric: n {PipelineName=PPLCRGLOADORLTOAPOSENTEVT}, Value: 11.000",
"data": null
}
]
}
Here is the JSON generated by test rule
And the length of pipeline name is something like 25 chars
(PPLCRGLOADORLTOAPOSENTEVT)

Vega Lite: Color scale for bar chart

I have this bar chart in vega lite in my Observable notebook
But I want to add a color scale to the bars so that the smallest numbers are red and the largest numbers are green and the numbers in between are yellow.
In order to do this -- I was thinking of setting up an ordinal scale with the domain as [0,5] since the numbers range from 0 to 5. The range of that scale would be ["red", "yellow", "green"]. But I'm just not sure how to apply that ordinal color scale to a vega lite chart. My code is below
barchart = vegalite ({
"data": {"values": barChartData},
"height": {"step": 17},
"title": "Gold",
"encoding": {
"y": {
"field": "program",
"type": "ordinal",
"sort": "-x"
},
"x": {
"aggregate": "sum",
"field": "index",
"title": "Gold",
"axis": null
}
},
"layer": [{
"mark": "bar"
}, {
"mark": {
"type": "text",
"align": "left",
"baseline": "middle",
"dx": 3
},
"encoding": {
"text": {"field": "index", "type": "quantitative"}
}
}]
})
Provide color or fill encoding in your bar chart and add your color as scales as done below or in editor:
var yourVlSpec = {
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
"description": "A simple bar chart with embedded data.",
"data": {
"values": [{
"index": 0.03,
"program": "Deliciousness"
},
{
"index": 0.43,
"program": "Reno 911!"
},
{
"index": 0.07,
"program": "Curious Life and Death of"
},
{
"index": 0.01,
"program": "True Life"
},
{
"index": 4.21,
"program": "Two and a Half Men"
},
{
"index": 0.06,
"program": "How Far is Tattoo Far"
},
{
"index": 0.39,
"program": "Cheaters"
},
{
"index": 4.72,
"program": "Bar Rescue"
},
{
"index": 0.81,
"program": "Key & Peele"
},
{
"index": 0.25,
"program": "Drunk History"
},
{
"index": 1.32,
"program": "Tosh.O"
},
{
"index": 0.11,
"program": "Revenge Prank"
},
{
"index": 4.88,
"program": "Workaholics"
},
{
"index": 0.04,
"program": "My Wife and Kids"
},
{
"index": 0.05,
"program": "World of Weapons"
},
{
"index": 0.03,
"program": "Roseanne"
},
{
"index": 1.98,
"program": "Everybody Loves Raymond"
},
{
"index": 1.2,
"program": "Aerial America"
}
]
},
"height": {
"step": 17
},
"title": "Gold",
"encoding": {
"y": {
"field": "program",
"type": "ordinal",
"sort": "-x"
},
"x": {
"aggregate": "sum",
"field": "index",
"title": "Gold",
"axis": null
}
},
"layer": [{
"mark": "bar",
"encoding": {
"color": {
"field": "index",
"scale": {
"range": ["red", "yellow", "green"],
"type": "linear"
},
"legend": null
}
}
},
{
"mark": {
"type": "text",
"align": "left",
"baseline": "middle",
"dx": 3
},
"encoding": {
"text": {
"field": "index",
"type": "quantitative"
}
}
}
]
};
vegaEmbed("#vis", yourVlSpec);
<script src="https://cdn.jsdelivr.net/combine/npm/vega#5.20.2,npm/vega-lite#5.0.0,npm/vega-embed#6.17.0"></script>
<body>
<div id="vis"></div>
</body>

How can I get DAG of Spark Sql Query execution plan?

I am doing some analysis on spark sql query execution plans. the execution plans that explain() api prints are not much readable. If we see spark web UI, a DAG graph is created which is divided into jobs, stages and tasks and much more readable. Is there any way to create that graph from execution plans or any apis in the code? if not, are there any apis that can read that grap from UI?
As close I can see, this project (https://github.com/AbsaOSS/spline-spark-agent) is able to interpret the execution plan and generate it in a readable way.
This spark job is reading a file, convert it to a CSV file, write to local.
A sample output in JSON look like
{
"id": "3861a1a7-ca31-4fab-b0f5-6dbcb53387ca",
"operations": {
"write": {
"outputSource": "file:/output.csv",
"append": false,
"id": 0,
"childIds": [
1
],
"params": {
"path": "output.csv"
},
"extra": {
"name": "InsertIntoHadoopFsRelationCommand",
"destinationType": "csv"
}
},
"reads": [
{
"inputSources": [
"file:/Users/liajiang/Downloads/spark-onboarding-demo-application/src/main/resources/wikidata.csv"
],
"id": 2,
"schema": [
"6742cfd4-d8b6-4827-89f2-4b2f7e060c57",
"62c022d9-c506-4e6e-984a-ee0c48f9df11",
"26f1d7b5-74a4-459c-87f3-46a3df781400",
"6e4063cf-4fd0-465d-a0ee-0e5c53bd52b0",
"2e019926-3adf-4ece-8ea7-0e01befd296b"
],
"params": {
"inferschema": "true",
"header": "true"
},
"extra": {
"name": "LogicalRelation",
"sourceType": "csv"
}
}
],
"other": [
{
"id": 1,
"childIds": [
2
],
"params": {
"name": "`source`"
},
"extra": {
"name": "SubqueryAlias"
}
}
]
},
"systemInfo": {
"name": "spark",
"version": "2.4.2"
},
"agentInfo": {
"name": "spline",
"version": "0.5.5"
},
"extraInfo": {
"appName": "spark-spline-demo-application",
"dataTypes": [
{
"_typeHint": "dt.Simple",
"id": "f0dede5e-8fe1-4c22-ab24-98f7f44a9a5a",
"name": "timestamp",
"nullable": true
},
{
"_typeHint": "dt.Simple",
"id": "dbe1d206-3d87-442c-837d-dfa47c88b9c1",
"name": "string",
"nullable": true
},
{
"_typeHint": "dt.Simple",
"id": "0d786d1e-030b-4997-b005-b4603aa247d7",
"name": "integer",
"nullable": true
}
],
"attributes": [
{
"id": "6742cfd4-d8b6-4827-89f2-4b2f7e060c57",
"name": "date",
"dataTypeId": "f0dede5e-8fe1-4c22-ab24-98f7f44a9a5a"
},
{
"id": "62c022d9-c506-4e6e-984a-ee0c48f9df11",
"name": "domain_code",
"dataTypeId": "dbe1d206-3d87-442c-837d-dfa47c88b9c1"
},
{
"id": "26f1d7b5-74a4-459c-87f3-46a3df781400",
"name": "page_title",
"dataTypeId": "dbe1d206-3d87-442c-837d-dfa47c88b9c1"
},
{
"id": "6e4063cf-4fd0-465d-a0ee-0e5c53bd52b0",
"name": "count_views",
"dataTypeId": "0d786d1e-030b-4997-b005-b4603aa247d7"
},
{
"id": "2e019926-3adf-4ece-8ea7-0e01befd296b",
"name": "total_response_size",
"dataTypeId": "0d786d1e-030b-4997-b005-b4603aa247d7"
}
]
}
}

Resources