Export data from mixpanel to Pandas - python-3.x

I am trying multiple ways to export data from mix panel to pandas. But I am unable to export in Step1 with API response.
import base64
import urllib.request
import ssl
try:
import json
except ImportError:
import simplejson as json
class Mixpanel(object):
ENDPOINT = 'https://mixpanel.com/api'
DATA_ENDPOINT = 'http://data.mixpanel.com/api'
VERSION = '2.0'
def __init__(self, api_secret):
self.api_secret = api_secret
def request(self, methods, params, http_method='GET', format='json'):
"""
methods - List of methods to be joined, e.g. ['events', 'properties', 'values']
will give us http://mixpanel.com/api/2.0/events/properties/values/
params - Extra parameters associated with method
"""
params['format'] = format
# print(base64.b64encode(self.api_secret).decode("ascii"))
request_url = '/'.join([self.ENDPOINT, str(self.VERSION)] + methods)
if http_method == 'GET':
data = None
request_url = request_url + '/?' + self.unicode_urlencode(params)
else:
data = self.unicode_urlencode(params)
auth = base64.b64encode(self.api_secret).decode("ascii")
headers = {'Authorization': 'Basic {encoded_secret}'.format(encoded_secret=auth)}
request = urllib.request.Request(request_url, data, headers)
# print(request)
context = ssl._create_unverified_context()
response = urllib.request.urlopen(request, context=context, timeout=120)
str_response = response.read().decode('utf8')
lines = str_response.splitlines(True)
records = []
for line in lines:
obj = json.loads(line)
records.append(obj)
return records
def unicode_urlencode(self, params):
"""
Convert lists to JSON encoded strings, and correctly handle any
unicode URL parameters.
"""
if isinstance(params, dict):
params = list(params.items())
for i,param in enumerate(params):
if isinstance(param[1], list):
params.remove(param)
params.append ((param[0], json.dumps(param[1]),))
return urllib.parse.urlencode(
[(k, v) for k, v in params]
)
if __name__ == '__main__':
encoded_secret = b'my_secret'
# byteAPISecret = bytes(encoded_secret + ':', "utf-8")
api = Mixpanel(api_secret=encoded_secret)
data = api.request(['events'], {
'event': ['sample event'],
'unit': 'hour',
'interval': 24,
'type': 'general'
})
print(data)
data_iter = api.request(['export'], {
'event': ['sample event'],
'to_date': "2022-02-16",
'from_date': "2022-02-16"
})
print(data_iter)
it is printing for data like
[{'data': {'series': ['2022-02-20 19:00:00', '2022-02-20 20:00:00', '2022-02-20 21:00:00', '2022-02-20 22:00:00', '2022-02-20 23:00:00', '2022-02-21 00:00:00', '2022-02-21 01:00:00', '2022-02-21 02:00:00', '2022-02-21 03:00:00', '2022-02-21 04:00:00', '2022-02-21 05:00:00', '2022-02-21 06:00:00', '2022-02-21 07:00:00', '2022-02-21 08:00:00', '2022-02-21 09:00:00', '2022-02-21 10:00:00', '2022-02-21 11:00:00', '2022-02-21 12:00:00', '2022-02-21 13:00:00', '2022-02-21 14:00:00', '2022-02-21 15:00:00', '2022-02-21 16:00:00', '2022-02-21 17:00:00', '2022-02-21 18:00:00', '2022-02-21 19:00:00', '2022-02-21 20:00:00'], 'values': {'Course Content Start': {'2022-02-20 00:00:00': 79, '2022-02-20 01:00:00': 52, '2022-02-20 02:00:00': 69, '2022-02-20 03:00:00': 101, '2022-02-20 04:00:00': 92, '2022-02-20 05:00:00': 77, '2022-02-20 06:00:00': 79, '2022-02-20 07:00:00': 77, '2022-02-20 08:00:00': 112, '2022-02-20 09:00:00': 134, '2022-02-20 10:00:00': 164, '2022-02-20 11:00:00': 173, '2022-02-20 12:00:00': 124, '2022-02-20 13:00:00': 144, '2022-02-20 14:00:00': 154, '2022-02-20 15:00:00': 95, '2022-02-20 16:00:00': 88, '2022-02-20 17:00:00': 71, '2022-02-20 18:00:00': 89, '2022-02-20 19:00:00': 45, '2022-02-20 20:00:00': 39, '2022-02-20 21:00:00': 32, '2022-02-20 22:00:00': 46, '2022-02-20 23:00:00': 73, '2022-02-21 00:00:00': 65, '2022-02-21 01:00:00': 72, '2022-02-21 02:00:00': 65, '2022-02-21 03:00:00': 92, '2022-02-21 04:00:00': 124, '2022-02-21 05:00:00': 147, '2022-02-21 06:00:00': 159, '2022-02-21 07:00:00': 136, '2022-02-21 08:00:00': 161, '2022-02-21 09:00:00': 159, '2022-02-21 10:00:00': 108, '2022-02-21 11:00:00': 141, '2022-02-21 12:00:00': 143, '2022-02-21 13:00:00': 126, '2022-02-21 14:00:00': 96, '2022-02-21 15:00:00': 109, '2022-02-21 16:00:00': 139, '2022-02-21 17:00:00': 150, '2022-02-21 18:00:00': 81, '2022-02-21 19:00:00': 21, '2022-02-21 20:00:00': 0, '2022-02-21 21:00:00': 0, '2022-02-21 22:00:00': 0, '2022-02-21 23:00:00': 0}}}, 'legend_size': 1, 'computed_at': '2022-02-22T03:29:41.928367+00:00'}]
but it is not working for data_iter(even changed to data point).getting errors like
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 400: Bad Request
I have to fetch the data and store that into pandas.Any way to export the data from Mixpanel and store that to pandas or iterate in python?

Related

Why serde_json rust so slow when deserializing in Untagged Enums

When running code:
#![allow(unused)]
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Instant;
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
#[serde(untagged)]
enum NumberOrString {
String(String),
Int(i64),
Float(f64),
}
fn main() {
let json_str = r#"{
"17594136111": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499],
"0000000017704043101": ["5", "7"],
"features": ["a1"]
}"#;
let start_time = Instant::now();
let parsed: HashMap<&str, Vec<serde_json::Value>> = serde_json::from_str(json_str).expect("panicking !!! ");
println!("Elapsed time: {:.2?}", start_time.elapsed());
let start_time = Instant::now();
let parsed2: HashMap<&str, Vec<NumberOrString>> = serde_json::from_str(json_str).expect("panicking !!! ");
println!("Elapsed time: {:.2?}", start_time.elapsed());
}
And the output comes as:
$ cargo run
Compiling rust_tutorial v0.1.0 (/Users/sandeep.yadav/code/codetest/rust/rust_tutorial)
Finished dev [unoptimized + debuginfo] target(s) in 2.26s
Running `target/debug/rust_tutorial`
Elapsed time: 360.78µs
Elapsed time: 2.22ms
$ cargo run --release
Compiling rust_tutorial v0.1.0 (/Users/sandeep.yadav/code/codetest/rust/rust_tutorial)
Finished release [optimized] target(s) in 2.47s
Running `target/release/rust_tutorial`
Elapsed time: 74.82µs
Elapsed time: 439.90µs
$ cargo run --release
Finished release [optimized] target(s) in 0.03s
Running `target/release/rust_tutorial`
Elapsed time: 63.13µs
Elapsed time: 354.89µs
Why is untaggedJson so slow, when compared to another enum that is defined in serde_json::Value?
As serde_json::Value contains much more than String int64 and f64., it contains, Null, Bool, List and Object. I'm actually reducing the possible acceptable value set and still time taken increases by atleast 5 times?
Any alternates I can use to achieve same result?
After implementing a custom Visitor pattern for the NumberOrString Enum -- as #Chayim correctly mentions is how serde-json impls Deserialize for Value here -- and finally, after removing the default #derive(Deserialize), it looks like the performance times are now much improved, as shown below.
#![allow(unused)]
use std::collections::HashMap;
use std::fmt;
use std::time::Instant;
use serde::de::{Error, Visitor};
use serde::{Deserialize, Deserializer, Serialize};
#[derive(Serialize, Debug, Clone, PartialEq)]
// note: it appears that an "untagged enum" is not needed anymore
// #[serde(untagged)]
enum NumberOrString {
String(String),
Int(i64),
Float(f64),
}
impl<'de> Deserialize<'de> for NumberOrString {
#[inline]
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
use crate::NumberOrString::*;
struct NumberOrStringVisitor;
impl<'de> Visitor<'de> for NumberOrStringVisitor {
type Value = NumberOrString;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("a number or string")
}
#[inline]
fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
where
E: Error,
{
Ok(Int(v))
}
#[inline]
fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
where
E: Error,
{
Ok(Int(v as i64))
}
#[inline]
fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E>
where
E: Error,
{
Ok(Float(v))
}
#[inline]
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: Error,
{
Ok(String(v.to_owned()))
}
#[inline]
fn visit_string<E>(self, v: std::string::String) -> Result<Self::Value, E>
where
E: Error,
{
Ok(String(v))
}
}
deserializer.deserialize_any(NumberOrStringVisitor)
}
}
fn main() {
let json_str = r#"{
"17594136111": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499],
"0000000017704043101": ["5", "7"],
"features": ["a1"]
}"#;
let start_time = Instant::now();
let parsed: HashMap<&str, Vec<serde_json::Value>> =
serde_json::from_str(json_str).expect("panicking !!! ");
println!("Elapsed time: {:.2?}", start_time.elapsed());
let start_time = Instant::now();
let parsed2: HashMap<&str, Vec<NumberOrString>> =
serde_json::from_str(json_str).expect("panicking !!! ");
println!("Elapsed time: {:.2?}", start_time.elapsed());
}
My times (on my Windows 11 PC) were as follows:
$ cargo run --release
Finished release [optimized] target(s) in 0.03s
Running `target/release/rust_tutorial`
Elapsed time: 286.00µs
Elapsed time: 20.50µs
$ cargo run --release
Finished release [optimized] target(s) in 0.03s
Running `target/release/rust_tutorial`
Elapsed time: 303.90µs
Elapsed time: 24.00µs

django.urls.exceptions.NoReverseMatch - django-smart-selects

I am trying to implement django-smart-selects in my project but get errors when trying to add a record in my model from django admin. Here's my code:
models.py
class PricingConfiguration(CreateUpdateMixin):
name = models.CharField(max_length=50)
description = models.CharField(max_length=250)
appointent_category = models.ForeignKey(
AppointmentCategory,
null=True,
blank=True,
related_name="pricing_appointment_category",
on_delete=models.CASCADE,
)
speciality = models.ForeignKey(
Speciality,
null=True,
blank=True,
related_name="pricing_speciality",
on_delete=models.CASCADE,
)
sub_speciality = ChainedForeignKey(
SubSpeciality,
chained_field="speciality",
chained_model_field="speciality",
show_all=False,
auto_choose=True,
sort=True,
null=True,
blank=True,
)
urls.py
from django.urls import include, path, re_path
from django.contrib import admin
from rest_framework.routers import DefaultRouter
app_name = "configuration"
urlpatterns = [
re_path(r"^admin/", admin.site.urls),
re_path(r"^chaining/", include("smart_selects.urls")),
]
Errors:
django.urls.exceptions.NoReverseMatch
django.urls.exceptions.NoReverseMatch: Reverse for 'chained_filter' not found. 'chained_filter' is not a valid view function or pattern name.
Traceback (most recent call last)
File "/usr/local/lib/python3.9/site-packages/django/contrib/staticfiles/handlers.py", line 76, in __call__
return self.application(environ, start_response)
File "/usr/local/lib/python3.9/site-packages/django/core/handlers/wsgi.py", line 133, in __call__
response = self.get_response(request)
File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 130, in get_response
response = self._middleware_chain(request)
File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 49, in inner
response = response_for_exception(request, exc)
File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 114, in response_for_exception
response = handle_uncaught_exception(request, get_resolver(get_urlconf()), sys.exc_info())
File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 149, in handle_uncaught_exception
return debug.technical_500_response(request, *exc_info)
File "/usr/local/lib/python3.9/site-packages/django_extensions/management/technical_response.py", line 40, in null_technical_500_response
raise exc_value.with_traceback(tb)
File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 47, in inner
response = get_response(request)
File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 204, in _get_response
response = response.render()
File "/usr/local/lib/python3.9/site-packages/django/template/response.py", line 105, in render
self.content = self.rendered_content
File "/usr/local/lib/python3.9/site-packages/django/template/response.py", line 83, in rendered_content
return template.render(context, self._request)
File "/usr/local/lib/python3.9/site-packages/django/template/backends/django.py", line 61, in render
return self.template.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 170, in render
return self._render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 162, in _render
return self.nodelist.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render
bit = node.render_annotated(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated
return self.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/loader_tags.py", line 150, in render
return compiled_parent._render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 162, in _render
return self.nodelist.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render
bit = node.render_annotated(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated
return self.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/loader_tags.py", line 150, in render
return compiled_parent._render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 162, in _render
return self.nodelist.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render
bit = node.render_annotated(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated
return self.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/loader_tags.py", line 150, in render
return compiled_parent._render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 162, in _render
return self.nodelist.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render
bit = node.render_annotated(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated
return self.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/loader_tags.py", line 62, in render
result = block.nodelist.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render
bit = node.render_annotated(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated
return self.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/loader_tags.py", line 62, in render
result = block.nodelist.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render
bit = node.render_annotated(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated
return self.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/defaulttags.py", line 211, in render
nodelist.append(node.render_annotated(context))
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated
return self.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/loader_tags.py", line 195, in render
return template.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 172, in render
return self._render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 162, in _render
return self.nodelist.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render
bit = node.render_annotated(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated
return self.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/defaulttags.py", line 211, in render
nodelist.append(node.render_annotated(context))
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated
return self.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/defaulttags.py", line 211, in render
nodelist.append(node.render_annotated(context))
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated
return self.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/defaulttags.py", line 312, in render
return nodelist.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render
bit = node.render_annotated(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated
return self.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/defaulttags.py", line 312, in render
return nodelist.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render
bit = node.render_annotated(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated
return self.render(context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 994, in render
return render_value_in_context(output, context)
File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 973, in render_value_in_context
value = str(value)
File "/usr/local/lib/python3.9/site-packages/django/utils/html.py", line 376, in <lambda>
klass.__str__ = lambda self: mark_safe(klass_str(self))
File "/usr/local/lib/python3.9/site-packages/django/forms/boundfield.py", line 34, in __str__
return self.as_widget()
File "/usr/local/lib/python3.9/site-packages/django/forms/boundfield.py", line 93, in as_widget
return widget.render(
File "/usr/local/lib/python3.9/site-packages/django/forms/widgets.py", line 246, in render
context = self.get_context(name, value, attrs)
File "/usr/local/lib/python3.9/site-packages/django/contrib/admin/widgets.py", line 283, in get_context
'rendered_widget': self.widget.render(name, value, attrs),
File "/usr/local/lib/python3.9/site-packages/smart_selects/widgets.py", line 107, in render
url = URL_PREFIX + ("/".join(reverse(view_name, kwargs=kwargs).split("/")[:-2]))
File "/usr/local/lib/python3.9/site-packages/django/urls/base.py", line 86, in reverse
return resolver._reverse_with_prefix(view, prefix, *args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/django/urls/resolvers.py", line 694, in _reverse_with_prefix
raise NoReverseMatch(msg)
django.urls.exceptions.NoReverseMatch: Reverse for 'chained_filter' not found. 'chained_filter' is not a valid view function or pattern name.
I have also tried different variations in urls.py like following
urlpatterns = [
re_path(r"^admin/", admin.site.urls),
re_path(r"^admin/", include("smart_selects.urls")),
]
and
urlpatterns = [
re_path(r"^chaining/", include("smart_selects.urls")),
]
but nothing seems to be working.
My django version is 3.2.5 and I'm using latest version of django-smart-selects. If anyone can let me know how to fix this issue, it would be really appreciated. Thanks.

TypeError: _bulk_create() got an unexpected keyword argument 'ignore_conflicts'

While adding groups with permission from Django Admin Panel and adding other M2M relationships too. I got this error!!
It says : TypeError: _bulk_create() got an unexpected keyword argument 'ignore_conflicts'
I can't find the error, Probably a noob mistake.
class GroupSerializer(serializers.ModelSerializer):
permissions = PermissionSerializerGroup(many=True, required=False)
class Meta:
model = Group
fields = ('id', 'name', 'permissions')
extra_kwargs = {
'name': {'validators': []},
}
def create(self, validated_data):
print(validated_data)
permissions_data = validated_data.pop("permissions")
obj, group = Group.objects.update_or_create(name=validated_data["name"])
obj.permissions.clear()
for permission in permissions_data:
per = Permission.objects.get(codename=permission["codename"])
obj.permissions.add(per)
obj.save()
return obj
Here is the Traceback:
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/core/handlers/exception.py", line 34, in inner
response = get_response(request)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/core/handlers/base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/core/handlers/base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/contrib/admin/options.py", line 607, in wrapper
return self.admin_site.admin_view(view)(*args, **kwargs)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/utils/decorators.py", line 130, in _wrapped_view
response = view_func(request, *args, **kwargs)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/views/decorators/cache.py", line 44, in _wrapped_view_func
response = view_func(request, *args, **kwargs)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/contrib/admin/sites.py", line 231, in inner
return view(request, *args, **kwargs)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/contrib/admin/options.py", line 1638, in add_view
return self.changeform_view(request, None, form_url, extra_context)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/utils/decorators.py", line 43, in _wrapper
return bound_method(*args, **kwargs)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/utils/decorators.py", line 130, in _wrapped_view
response = view_func(request, *args, **kwargs)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/contrib/admin/options.py", line 1522, in changeform_view
return self._changeform_view(request, object_id, form_url, extra_context)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/contrib/admin/options.py", line 1566, in _changeform_view
self.save_related(request, form, formsets, not add)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/contrib/admin/options.py", line 1107, in save_related
form.save_m2m()
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/forms/models.py", line 442, in _save_m2m
f.save_form_data(self.instance, cleaned_data[f.name])
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/db/models/fields/related.py", line 1618, in save_form_data
getattr(instance, self.attname).set(data)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/db/models/fields/related_descriptors.py", line 1008, in set
self.add(*new_objs, through_defaults=through_defaults)
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/db/models/fields/related_descriptors.py", line 946, in add
through_defaults=through_defaults,
File "/home/suman/Desktop/suman1234/myvenv/lib/python3.6/site-packages/django/db/models/fields/related_descriptors.py", line 1129, in _add_items
], ignore_conflicts=True)
TypeError: _bulk_create() got an unexpected keyword argument 'ignore_conflicts'
I solved this issue by downgrading Django version to 2.2.12
It seems that Django v3.0+ has this issue
This has been fixed in github: https://github.com/martinphellwig/django-query-signals/commit/a3ed59614287f1ef9e7398d325a8bbcc11bf0b3c but it's not in pypi, you need to install django-query-signals from git

HazelcastSerializationException: When Jet job is running over multiple cluster

I am trying to run one of the Hazelcast-jet example over a Distributed System. My objective is to run code over Disributed System, Utilize Multiple machine's processing power. I have two Laptops connected via LAN. When I run this example in One Machine it works fine, to run it both system I Start Machine 1 and Machine 2 with only jet Instance. code on both Machines are,
Machine 1
public class PrimeFinder {
public static void main(String[] args) {
System.setProperty("hazelcast.logging.type", "log4j");
try {
JetConfig cfg = new JetConfig();
cfg.setInstanceConfig(new InstanceConfig().setCooperativeThreadCount(
Math.max(1, getRuntime().availableProcessors() / 2)));
cfg.setProperty("hazelcast.initial.min.cluster.size","3");
Jet.newJetInstance(cfg);
JetInstance jet = Jet.newJetInstance(cfg);
DAG dag = new DAG();
final int limit = 15_485_864;
Vertex generator = dag.newVertex("number-generator", new GenerateNumbersMetaSupplier(limit));
Vertex primeChecker = dag.newVertex("filter-primes", filterP(PrimeFinder::isPrime));
Vertex writer = dag.newVertex("writer", writeListP("primes"));
dag.edge(between(generator, primeChecker));
dag.edge(between(primeChecker, writer));
jet.newJob(dag).join();
IListJet<Integer> primes = jet.getList("primes");
List<Integer> sortedPrimes = primes.stream().sorted().limit(1000).collect(toList());
System.out.println("Found " + primes.size() + " primes.");
System.out.println("Some of the primes found are: " + sortedPrimes);
} finally {
Jet.shutdownAll();
}
}
private static boolean isPrime(int n) {
if (n <= 1) {
return false;
}
int endValue = (int) Math.sqrt(n);
for (int i = 2; i <= endValue; i++) {
if (n % i == 0) {
return false;
}
}
return true;
}
static class GenerateNumbersMetaSupplier implements ProcessorMetaSupplier {
private final int limit;
private transient int totalParallelism;
private transient int localParallelism;
GenerateNumbersMetaSupplier(int limit) {
this.limit = limit;
}
#Override
public void init(#Nonnull Context context) {
totalParallelism = context.totalParallelism();
localParallelism = context.localParallelism();
}
#Override #Nonnull
public Function<Address, ProcessorSupplier> get(#Nonnull List<Address> addresses) {
Map<Address, ProcessorSupplier> map = new HashMap<>();
for (int i = 0; i < addresses.size(); i++) {
Address address = addresses.get(i);
int start = i * localParallelism;
int end = (i + 1) * localParallelism;
int mod = totalParallelism;
map.put(address, count -> range(start, end)
.mapToObj(index -> new GenerateNumbersP(range(0, limit).filter(f -> f % mod == index)))
.collect(toList())
);
}
return map::get;
}
}
static class GenerateNumbersP extends AbstractProcessor {
private final Traverser<Integer> traverser;
GenerateNumbersP(IntStream stream) {
traverser = traverseStream(stream.boxed());
}
#Override
public boolean complete() {
return emitFromTraverser(traverser);
}
}
}
Machine 2
public class PrimeFinder {
public static void main(String[] args) {
System.setProperty("hazelcast.logging.type", "log4j");
try {
JetConfig cfg = new JetConfig();
cfg.setInstanceConfig(new InstanceConfig().setCooperativeThreadCount(
Math.max(1, getRuntime().availableProcessors() / 2)));
Jet.newJetInstance(cfg);
JetInstance jet = Jet.newJetInstance(cfg);
}
}
Error
Members {size:4, ver:4} [
Member [192.168.43.5]:5701 - e9ff45b4-50a2-4918-b51f-9fc3012cce7c
Member [192.168.43.224]:5701 - 5931daa7-4452-4275-a3d5-a9daaf247f50
Member [192.168.43.224]:5702 - 57bce3eb-71f7-40d3-86b3-64481feb84e9
Member [192.168.43.5]:5702 - 9fff0e81-748d-4774-8556-2f45941bd59d this
]
06:10,158 [192.168.43.5]:5701 [jet] [3.2] Starting job 0368-8053-f940-0004 based on submit request
06:12,285 [192.168.43.5]:5701 [jet] [3.2] Didn't find any snapshot to restore for job '0368-8053-f940-0004', execution 0368-8056-5ec0-0001
06:12,286 [192.168.43.5]:5701 [jet] [3.2] Start executing job '0368-8053-f940-0004', execution 0368-8056-5ec0-0001, execution graph in DOT format:
digraph DAG {
"number-generator" [tooltip="local-parallelism=2"];
"filter-primes" [tooltip="local-parallelism=2"];
"writer" [tooltip="local-parallelism=1"];
"number-generator" -> "filter-primes";
"filter-primes" -> "writer";
}
HINT: You can use graphviz or http://viz-js.com to visualize the printed graph.
06:12,733 [192.168.43.5]:5701 [jet] [3.2] Execution plan for jobId=0368-8053-f940-0004, jobName='0368-8053-f940-0004', executionId=0368-8056-5ec0-0001 initialized
06:12,784 [192.168.43.5]:5702 [jet] [3.2] Execution plan for jobId=0368-8053-f940-0004, jobName='0368-8053-f940-0004', executionId=0368-8056-5ec0-0001 initialized
06:12,824 [192.168.43.5]:5701 [jet] [3.2] Execution of job '0368-8053-f940-0004', execution 0368-8056-5ec0-0001 failed after 648 ms
com.hazelcast.nio.serialization.HazelcastSerializationException: java.io.IOException: unexpected exception type
at com.hazelcast.internal.serialization.impl.SerializationUtil.handleException(SerializationUtil.java:70)
at com.hazelcast.internal.serialization.impl.AbstractSerializationService.readObject(AbstractSerializationService.java:275)
at com.hazelcast.internal.serialization.impl.ByteArrayObjectDataInput.readObject(ByteArrayObjectDataInput.java:574)
at com.hazelcast.jet.impl.execution.init.CustomClassLoadedObject.read(CustomClassLoadedObject.java:56)
at com.hazelcast.jet.impl.execution.init.VertexDef.readData(VertexDef.java:153)
at com.hazelcast.internal.serialization.impl.DataSerializableSerializer.readInternal(DataSerializableSerializer.java:160)
at com.hazelcast.internal.serialization.impl.DataSerializableSerializer.read(DataSerializableSerializer.java:106)
at com.hazelcast.internal.serialization.impl.DataSerializableSerializer.read(DataSerializableSerializer.java:51)
at com.hazelcast.internal.serialization.impl.StreamSerializerAdapter.read(StreamSerializerAdapter.java:48)
at com.hazelcast.internal.serialization.impl.AbstractSerializationService.readObject(AbstractSerializationService.java:269)
at com.hazelcast.internal.serialization.impl.ByteArrayObjectDataInput.readObject(ByteArrayObjectDataInput.java:574)
at com.hazelcast.jet.impl.util.ImdgUtil.readList(ImdgUtil.java:444)
at com.hazelcast.jet.impl.execution.init.ExecutionPlan.readData(ExecutionPlan.java:307)
at com.hazelcast.internal.serialization.impl.DataSerializableSerializer.readInternal(DataSerializableSerializer.java:160)
at com.hazelcast.internal.serialization.impl.DataSerializableSerializer.read(DataSerializableSerializer.java:106)
at com.hazelcast.internal.serialization.impl.DataSerializableSerializer.read(DataSerializableSerializer.java:51)
at com.hazelcast.internal.serialization.impl.StreamSerializerAdapter.read(StreamSerializerAdapter.java:48)
at com.hazelcast.internal.serialization.impl.AbstractSerializationService.toObject(AbstractSerializationService.java:187)
at com.hazelcast.jet.impl.execution.init.CustomClassLoadedObject.deserializeWithCustomClassLoader(CustomClassLoadedObject.java:65)
at com.hazelcast.jet.impl.operation.InitExecutionOperation.deserializePlan(InitExecutionOperation.java:116)
at com.hazelcast.jet.impl.operation.InitExecutionOperation.run(InitExecutionOperation.java:71)
at com.hazelcast.spi.Operation.call(Operation.java:170)
at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.call(OperationRunnerImpl.java:210)
at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:199)
at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:416)
at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:153)
at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:123)
at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.run(OperationThread.java:110)
at ------ submitted from ------.(Unknown Source)
at com.hazelcast.spi.impl.operationservice.impl.InvocationFuture.resolve(InvocationFuture.java:126)
at com.hazelcast.spi.impl.AbstractInvocationFuture$1.run(AbstractInvocationFuture.java:251)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
at com.hazelcast.util.executor.HazelcastManagedThread.executeRun(HazelcastManagedThread.java:64)
at com.hazelcast.util.executor.HazelcastManagedThread.run(HazelcastManagedThread.java:80)
Caused by: java.io.IOException: unexpected exception type.....
When I run this code on a single Machine, Example is working fine and output is as expected.
Members {size:2, ver:2} [
Member [172.29.97.33]:5701 - e2e6a9d5-9bdd-49d4-934c-e7becee8ad58
Member [172.29.97.33]:5702 - 9e121fda-ca89-4d8c-a20e-3e345ce91fc2 this
]
36:20,086 [172.29.97.33]:5701 [jet] [3.2] Starting job 0368-94f5-7bc0-0002 based on submit request
36:20,316 [172.29.97.33]:5701 [jet] [3.2] Didn't find any snapshot to restore for job '0368-94f5-7bc0-0002', execution 0368-94f7-a640-0001
36:20,316 [172.29.97.33]:5701 [jet] [3.2] Start executing job '0368-94f5-7bc0-0002', execution 0368-94f7-a640-0001, execution graph in DOT format:
digraph DAG {
"number-generator" [tooltip="local-parallelism=2"];
"filter-primes" [tooltip="local-parallelism=2"];
"writer" [tooltip="local-parallelism=1"];
"number-generator" -> "filter-primes";
"filter-primes" -> "writer";
}
HINT: You can use graphviz or http://viz-js.com to visualize the printed graph.
36:21,185 [172.29.97.33]:5701 [jet] [3.2] Execution plan for jobId=0368-94f5-7bc0-0002, jobName='0368-94f5-7bc0-0002', executionId=0368-94f7-a640-0001 initialized
36:21,237 [172.29.97.33]:5702 [jet] [3.2] Execution plan for jobId=0368-94f5-7bc0-0002, jobName='0368-94f5-7bc0-0002', executionId=0368-94f7-a640-0001 initialized
36:21,250 [172.29.97.33]:5701 [jet] [3.2] Start execution of job '0368-94f5-7bc0-0002', execution 0368-94f7-a640-0001 from coordinator [172.29.97.33]:5701
36:21,327 [172.29.97.33]:5702 [jet] [3.2] Start execution of job '0368-94f5-7bc0-0002', execution 0368-94f7-a640-0001 from coordinator [172.29.97.33]:5701
37:11,263 [172.29.97.33]:5701 [jet] [3.2] Execution of job '0368-94f5-7bc0-0002', execution 0368-94f7-a640-0001 completed in 51,171 ms
Found 1000000 primes.
Some of the primes found are: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211, 223, 227, 229, 233, 239, 241, 251, 257, 263, 269, 271, 277, 281, 283, 293, 307, 311, 313, 317, 331, 337, 347, 349, 353, 359, 367, 373, 379, 383, 389, 397, 401, 409, 419, 421, 431, 433, 439, 443, 449, 457, 461, 463, 467, 479, 487, 491, 499, 503, 509, 521, 523, 541, 547, 557, 563, 569, 571, 577, 587, 593, 599, 601, 607, 613, 617, 619, 631, 641, 643, 647, 653, 659, 661, 673, 677, 683, 691, 701, 709, 719, 727, 733, 739, 743, 751, 757, 761, 769, 773, 787, 797, 809, 811, 821, 823, 827, 829, 839, 853, 857, 859, 863, 877, 881, 883, 887, 907, 911, 919, 929, 937, 941, 947, 953, 967, 971, 977, 983, 991, 997, 1009, 1013, 1019, 1021, 1031, 1033, 1039, 1049, 1051, 1061, 1063, 1069, 1087, 1091, 1093, 1097, 1103, 1109, 1117, 1123, 1129, 1151, 1153, 1163, 1171, 1181, 1187, 1193, 1201, 1213, 1217, 1223, 1229, 1231, 1237, 1249, 1259, 1277, 1279, 1283, 1289, 1291, 1297, 1301, 1303, 1307, 1319, 1321, 1327, 1361, 1367, 1373, 1381, 1399, 1409, 1423, 1427, 1429, 1433, 1439, 1447, 1451, 1453, 1459, 1471, 1481, 1483, 1487, 1489, 1493, 1499, 1511, 1523, 1531, 1543, 1549, 1553, 1559, 1567, 1571, 1579, 1583, 1597, 1601, 1607, 1609, 1613, 1619, 1621, 1627, 1637, 1657, 1663, 1667, 1669, 1693, 1697, 1699, 1709, 1721, 1723, 1733, 1741, 1747, 1753, 1759, 1777, 1783, 1787, 1789, 1801, 1811, 1823, 1831, 1847, 1861, 1867, 1871, 1873, 1877, 1879, 1889, 1901, 1907, 1913, 1931, 1933, 1949, 1951, 1973, 1979, 1987, 1993, 1997, 1999, 2003, 2011, 2017, 2027, 2029, 2039, 2053, 2063, 2069, 2081, 2083, 2087, 2089, 2099, 2111, 2113, 2129, 2131, 2137, 2141, 2143, 2153, 2161, 2179, 2203, 2207, 2213, 2221, 2237, 2239, 2243, 2251, 2267, 2269, 2273, 2281, 2287, 2293, 2297, 2309, 2311, 2333, 2339, 2341, 2347, 2351, 2357, 2371, 2377, 2381, 2383, 2389, 2393, 2399, 2411, 2417, 2423, 2437, 2441, 2447, 2459, 2467, 2473, 2477, 2503, 2521, 2531, 2539, 2543, 2549, 2551, 2557, 2579, 2591, 2593, 2609, 2617, 2621, 2633, 2647, 2657, 2659, 2663, 2671, 2677, 2683, 2687, 2689, 2693, 2699, 2707, 2711, 2713, 2719, 2729, 2731, 2741, 2749, 2753, 2767, 2777, 2789, 2791, 2797, 2801, 2803, 2819, 2833, 2837, 2843, 2851, 2857, 2861, 2879, 2887, 2897, 2903, 2909, 2917, 2927, 2939, 2953, 2957, 2963, 2969, 2971, 2999, 3001, 3011, 3019, 3023, 3037, 3041, 3049, 3061, 3067, 3079, 3083, 3089, 3109, 3119, 3121, 3137, 3163, 3167, 3169, 3181, 3187, 3191, 3203, 3209, 3217, 3221, 3229, 3251, 3253, 3257, 3259, 3271, 3299, 3301, 3307, 3313, 3319, 3323, 3329, 3331, 3343, 3347, 3359, 3361, 3371, 3373, 3389, 3391, 3407, 3413, 3433, 3449, 3457, 3461, 3463, 3467, 3469, 3491, 3499, 3511, 3517, 3527, 3529, 3533, 3539, 3541, 3547, 3557, 3559, 3571, 3581, 3583, 3593, 3607, 3613, 3617, 3623, 3631, 3637, 3643, 3659, 3671, 3673, 3677, 3691, 3697, 3701, 3709, 3719, 3727, 3733, 3739, 3761, 3767, 3769, 3779, 3793, 3797, 3803, 3821, 3823, 3833, 3847, 3851, 3853, 3863, 3877, 3881, 3889, 3907, 3911, 3917, 3919, 3923, 3929, 3931, 3943, 3947, 3967, 3989, 4001, 4003, 4007, 4013, 4019, 4021, 4027, 4049, 4051, 4057, 4073, 4079, 4091, 4093, 4099, 4111, 4127, 4129, 4133, 4139, 4153, 4157, 4159, 4177, 4201, 4211, 4217, 4219, 4229, 4231, 4241, 4243, 4253, 4259, 4261, 4271, 4273, 4283, 4289, 4297, 4327, 4337, 4339, 4349, 4357, 4363, 4373, 4391, 4397, 4409, 4421, 4423, 4441, 4447, 4451, 4457, 4463, 4481, 4483, 4493, 4507, 4513, 4517, 4519, 4523, 4547, 4549, 4561, 4567, 4583, 4591, 4597, 4603, 4621, 4637, 4639, 4643, 4649, 4651, 4657, 4663, 4673, 4679, 4691, 4703, 4721, 4723, 4729, 4733, 4751, 4759, 4783, 4787, 4789, 4793, 4799, 4801, 4813, 4817, 4831, 4861, 4871, 4877, 4889, 4903, 4909, 4919, 4931, 4933, 4937, 4943, 4951, 4957, 4967, 4969, 4973, 4987, 4993, 4999, 5003, 5009, 5011, 5021, 5023, 5039, 5051, 5059, 5077, 5081, 5087, 5099, 5101, 5107, 5113, 5119, 5147, 5153, 5167, 5171, 5179, 5189, 5197, 5209, 5227, 5231, 5233, 5237, 5261, 5273, 5279, 5281, 5297, 5303, 5309, 5323, 5333, 5347, 5351, 5381, 5387, 5393, 5399, 5407, 5413, 5417, 5419, 5431, 5437, 5441, 5443, 5449, 5471, 5477, 5479, 5483, 5501, 5503, 5507, 5519, 5521, 5527, 5531, 5557, 5563, 5569, 5573, 5581, 5591, 5623, 5639, 5641, 5647, 5651, 5653, 5657, 5659, 5669, 5683, 5689, 5693, 5701, 5711, 5717, 5737, 5741, 5743, 5749, 5779, 5783, 5791, 5801, 5807, 5813, 5821, 5827, 5839, 5843, 5849, 5851, 5857, 5861, 5867, 5869, 5879, 5881, 5897, 5903, 5923, 5927, 5939, 5953, 5981, 5987, 6007, 6011, 6029, 6037, 6043, 6047, 6053, 6067, 6073, 6079, 6089, 6091, 6101, 6113, 6121, 6131, 6133, 6143, 6151, 6163, 6173, 6197, 6199, 6203, 6211, 6217, 6221, 6229, 6247, 6257, 6263, 6269, 6271, 6277, 6287, 6299, 6301, 6311, 6317, 6323, 6329, 6337, 6343, 6353, 6359, 6361, 6367, 6373, 6379, 6389, 6397, 6421, 6427, 6449, 6451, 6469, 6473, 6481, 6491, 6521, 6529, 6547, 6551, 6553, 6563, 6569, 6571, 6577, 6581, 6599, 6607, 6619, 6637, 6653, 6659, 6661, 6673, 6679, 6689, 6691, 6701, 6703, 6709, 6719, 6733, 6737, 6761, 6763, 6779, 6781, 6791, 6793, 6803, 6823, 6827, 6829, 6833, 6841, 6857, 6863, 6869, 6871, 6883, 6899, 6907, 6911, 6917, 6947, 6949, 6959, 6961, 6967, 6971, 6977, 6983, 6991, 6997, 7001, 7013, 7019, 7027, 7039, 7043, 7057, 7069, 7079, 7103, 7109, 7121, 7127, 7129, 7151, 7159, 7177, 7187, 7193, 7207, 7211, 7213, 7219, 7229, 7237, 7243, 7247, 7253, 7283, 7297, 7307, 7309, 7321, 7331, 7333, 7349, 7351, 7369, 7393, 7411, 7417, 7433, 7451, 7457, 7459, 7477, 7481, 7487, 7489, 7499, 7507, 7517, 7523, 7529, 7537, 7541, 7547, 7549, 7559, 7561, 7573, 7577, 7583, 7589, 7591, 7603, 7607, 7621, 7639, 7643, 7649, 7669, 7673, 7681, 7687, 7691, 7699, 7703, 7717, 7723, 7727, 7741, 7753, 7757, 7759, 7789, 7793, 7817, 7823, 7829, 7841, 7853, 7867, 7873, 7877, 7879, 7883, 7901, 7907, 7919]
37:21,664 [172.29.97.33]:5702 [jet] [3.2] Removing connection to endpoint [172.29.97.33]:5701 Cause => java.net.SocketException {Connection refused: no further information to address /172.29.97.33:5701}, Error-Count: 5
37:21,697 [172.29.97.33]:5702 [jet] [3.2] Member [172.29.97.33]:5701 - e2e6a9d5-9bdd-49d4-934c-e7becee8ad58 is suspected to be dead for reason: No connection
37:21,697 [172.29.97.33]:5702 [jet] [3.2] Starting mastership claim process...
37:21,698 [172.29.97.33]:5702 [jet] [3.2] Local MembersView{version=2, members=[MemberInfo{address=[172.29.97.33]:5701, uuid=e2e6a9d5-9bdd-49d4-934c-e7becee8ad58, liteMember=false, memberListJoinVersion=1}, MemberInfo{address=[172.29.97.33]:5702, uuid=9e121fda-ca89-4d8c-a20e-3e345ce91fc2, liteMember=false, memberListJoinVersion=2}]} with suspected members: [[172.29.97.33]:5701] and initial addresses to ask: []
37:21,772 [172.29.97.33]:5702 [jet] [3.2]
Members {size:1, ver:3} [
Member [172.29.97.33]:5702 - 9e121fda-ca89-4d8c-a20e-3e345ce91fc2 this
]
37:21,773 [172.29.97.33]:5702 [jet] [3.2] Mastership is claimed with: MembersView{version=3, members=[MemberInfo{address=[172.29.97.33]:5702, uuid=9e121fda-ca89-4d8c-a20e-3e345ce91fc2, liteMember=false, memberListJoinVersion=2}]}
Can anyone help me out what am I doing wrong?
You need to add the class to the JobConfig and pass it when submitting the pipeline:
JobConfig jobConfig = new JobConfig();
jobConfig.addClass(PrimeFinder.class);
jet.newJob(dag, jobConfig).join();

Python requests - retry for any status code but not 400

I am using the following snippet(please note, only partial snippets shown):
from urllib3.util import Retry
status_forcelist = (500, 502, 504)
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
method_whitelist=frozenset(['GET', 'POST'])
As of now, the retry would be done for status codes 500,502,504. However, I intend to retry for any status code but not 400. Is there a graceful way to achieve this instead of populating/hardcoding the status_forcelist for all status codes ?
You can take the codes from request module - which saves you writing them out. In case any new ones get introduced you would have them as well:
import requests
from urllib3.util import Retry
status_forcelist = tuple( x for x in requests.status_codes._codes if x != 400)
print(status_forcelist)
Output:
(100, 101, 102, 103, 122, 200, 201, 202, 203, 204, 205, 206, 207, 208, 226, 300, 301, 302,
303, 304, 305, 306, 307, 308, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412,
413, 414, 415, 416, 417, 418, 421, 422, 423, 424, 425, 426, 428, 429, 431, 444, 449, 450,
451, 499, 500, 501, 502, 503, 504, 505, 506, 507, 509, 510, 511)
Caveat: wikipedia list "self created" / unofficial statuscodes as well as proprietary ones around 9xx which are not covered by this list though. #zwer solution would cover those as well.
status_forcelist, unfortunately, applies as a whitelist only (given the use, it can be called a blacklist, tho) so there are no built-in ways to do it gracefully. However, you can override (or monkey-patch if that's your thing) urllib3.util.retry.Retry.is_retry() to treat it as a blacklist, something like:
import urllib3.util.retry
class BlacklistRetry(urllib3.util.retry.Retry):
def is_retry(self, method, status_code, has_retry_after=False):
if not self._is_method_retryable(method):
return False
if self.status_forcelist and status_code not in self.status_forcelist:
# threat as a blacklist --------------^
return True
return (self.total and self.respect_retry_after_header and
has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES))
Then use it instead of the original Retry and just add 400 to its status_forcelist.
The current accepted answer has a bug, it retries for successful http status codes, which means it will unnecessarily retry - if you want proof, you can test this with the code from this answer here.
You could just follow a similar implementation to Response.raise_for_status.
from urllib3.util import Retry
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=tuple(range(401, 600)),
method_whitelist=frozenset(['GET', 'POST'])

Resources