Why serde_json rust so slow when deserializing in Untagged Enums - rust
When running code:
#![allow(unused)]
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Instant;
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
#[serde(untagged)]
enum NumberOrString {
String(String),
Int(i64),
Float(f64),
}
fn main() {
let json_str = r#"{
"17594136111": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499],
"0000000017704043101": ["5", "7"],
"features": ["a1"]
}"#;
let start_time = Instant::now();
let parsed: HashMap<&str, Vec<serde_json::Value>> = serde_json::from_str(json_str).expect("panicking !!! ");
println!("Elapsed time: {:.2?}", start_time.elapsed());
let start_time = Instant::now();
let parsed2: HashMap<&str, Vec<NumberOrString>> = serde_json::from_str(json_str).expect("panicking !!! ");
println!("Elapsed time: {:.2?}", start_time.elapsed());
}
And the output comes as:
$ cargo run
Compiling rust_tutorial v0.1.0 (/Users/sandeep.yadav/code/codetest/rust/rust_tutorial)
Finished dev [unoptimized + debuginfo] target(s) in 2.26s
Running `target/debug/rust_tutorial`
Elapsed time: 360.78µs
Elapsed time: 2.22ms
$ cargo run --release
Compiling rust_tutorial v0.1.0 (/Users/sandeep.yadav/code/codetest/rust/rust_tutorial)
Finished release [optimized] target(s) in 2.47s
Running `target/release/rust_tutorial`
Elapsed time: 74.82µs
Elapsed time: 439.90µs
$ cargo run --release
Finished release [optimized] target(s) in 0.03s
Running `target/release/rust_tutorial`
Elapsed time: 63.13µs
Elapsed time: 354.89µs
Why is untaggedJson so slow, when compared to another enum that is defined in serde_json::Value?
As serde_json::Value contains much more than String int64 and f64., it contains, Null, Bool, List and Object. I'm actually reducing the possible acceptable value set and still time taken increases by atleast 5 times?
Any alternates I can use to achieve same result?
After implementing a custom Visitor pattern for the NumberOrString Enum -- as #Chayim correctly mentions is how serde-json impls Deserialize for Value here -- and finally, after removing the default #derive(Deserialize), it looks like the performance times are now much improved, as shown below.
#![allow(unused)]
use std::collections::HashMap;
use std::fmt;
use std::time::Instant;
use serde::de::{Error, Visitor};
use serde::{Deserialize, Deserializer, Serialize};
#[derive(Serialize, Debug, Clone, PartialEq)]
// note: it appears that an "untagged enum" is not needed anymore
// #[serde(untagged)]
enum NumberOrString {
String(String),
Int(i64),
Float(f64),
}
impl<'de> Deserialize<'de> for NumberOrString {
#[inline]
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
use crate::NumberOrString::*;
struct NumberOrStringVisitor;
impl<'de> Visitor<'de> for NumberOrStringVisitor {
type Value = NumberOrString;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("a number or string")
}
#[inline]
fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
where
E: Error,
{
Ok(Int(v))
}
#[inline]
fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
where
E: Error,
{
Ok(Int(v as i64))
}
#[inline]
fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E>
where
E: Error,
{
Ok(Float(v))
}
#[inline]
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: Error,
{
Ok(String(v.to_owned()))
}
#[inline]
fn visit_string<E>(self, v: std::string::String) -> Result<Self::Value, E>
where
E: Error,
{
Ok(String(v))
}
}
deserializer.deserialize_any(NumberOrStringVisitor)
}
}
fn main() {
let json_str = r#"{
"17594136111": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499],
"0000000017704043101": ["5", "7"],
"features": ["a1"]
}"#;
let start_time = Instant::now();
let parsed: HashMap<&str, Vec<serde_json::Value>> =
serde_json::from_str(json_str).expect("panicking !!! ");
println!("Elapsed time: {:.2?}", start_time.elapsed());
let start_time = Instant::now();
let parsed2: HashMap<&str, Vec<NumberOrString>> =
serde_json::from_str(json_str).expect("panicking !!! ");
println!("Elapsed time: {:.2?}", start_time.elapsed());
}
My times (on my Windows 11 PC) were as follows:
$ cargo run --release
Finished release [optimized] target(s) in 0.03s
Running `target/release/rust_tutorial`
Elapsed time: 286.00µs
Elapsed time: 20.50µs
$ cargo run --release
Finished release [optimized] target(s) in 0.03s
Running `target/release/rust_tutorial`
Elapsed time: 303.90µs
Elapsed time: 24.00µs
Related
django.urls.exceptions.NoReverseMatch - django-smart-selects
I am trying to implement django-smart-selects in my project but get errors when trying to add a record in my model from django admin. Here's my code: models.py class PricingConfiguration(CreateUpdateMixin): name = models.CharField(max_length=50) description = models.CharField(max_length=250) appointent_category = models.ForeignKey( AppointmentCategory, null=True, blank=True, related_name="pricing_appointment_category", on_delete=models.CASCADE, ) speciality = models.ForeignKey( Speciality, null=True, blank=True, related_name="pricing_speciality", on_delete=models.CASCADE, ) sub_speciality = ChainedForeignKey( SubSpeciality, chained_field="speciality", chained_model_field="speciality", show_all=False, auto_choose=True, sort=True, null=True, blank=True, ) urls.py from django.urls import include, path, re_path from django.contrib import admin from rest_framework.routers import DefaultRouter app_name = "configuration" urlpatterns = [ re_path(r"^admin/", admin.site.urls), re_path(r"^chaining/", include("smart_selects.urls")), ] Errors: django.urls.exceptions.NoReverseMatch django.urls.exceptions.NoReverseMatch: Reverse for 'chained_filter' not found. 'chained_filter' is not a valid view function or pattern name. Traceback (most recent call last) File "/usr/local/lib/python3.9/site-packages/django/contrib/staticfiles/handlers.py", line 76, in __call__ return self.application(environ, start_response) File "/usr/local/lib/python3.9/site-packages/django/core/handlers/wsgi.py", line 133, in __call__ response = self.get_response(request) File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 130, in get_response response = self._middleware_chain(request) File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 49, in inner response = response_for_exception(request, exc) File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 114, in response_for_exception response = handle_uncaught_exception(request, get_resolver(get_urlconf()), sys.exc_info()) File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 149, in handle_uncaught_exception return debug.technical_500_response(request, *exc_info) File "/usr/local/lib/python3.9/site-packages/django_extensions/management/technical_response.py", line 40, in null_technical_500_response raise exc_value.with_traceback(tb) File "/usr/local/lib/python3.9/site-packages/django/core/handlers/exception.py", line 47, in inner response = get_response(request) File "/usr/local/lib/python3.9/site-packages/django/core/handlers/base.py", line 204, in _get_response response = response.render() File "/usr/local/lib/python3.9/site-packages/django/template/response.py", line 105, in render self.content = self.rendered_content File "/usr/local/lib/python3.9/site-packages/django/template/response.py", line 83, in rendered_content return template.render(context, self._request) File "/usr/local/lib/python3.9/site-packages/django/template/backends/django.py", line 61, in render return self.template.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 170, in render return self._render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 162, in _render return self.nodelist.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render bit = node.render_annotated(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated return self.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/loader_tags.py", line 150, in render return compiled_parent._render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 162, in _render return self.nodelist.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render bit = node.render_annotated(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated return self.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/loader_tags.py", line 150, in render return compiled_parent._render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 162, in _render return self.nodelist.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render bit = node.render_annotated(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated return self.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/loader_tags.py", line 150, in render return compiled_parent._render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 162, in _render return self.nodelist.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render bit = node.render_annotated(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated return self.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/loader_tags.py", line 62, in render result = block.nodelist.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render bit = node.render_annotated(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated return self.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/loader_tags.py", line 62, in render result = block.nodelist.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render bit = node.render_annotated(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated return self.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/defaulttags.py", line 211, in render nodelist.append(node.render_annotated(context)) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated return self.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/loader_tags.py", line 195, in render return template.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 172, in render return self._render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 162, in _render return self.nodelist.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render bit = node.render_annotated(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated return self.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/defaulttags.py", line 211, in render nodelist.append(node.render_annotated(context)) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated return self.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/defaulttags.py", line 211, in render nodelist.append(node.render_annotated(context)) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated return self.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/defaulttags.py", line 312, in render return nodelist.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render bit = node.render_annotated(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated return self.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/defaulttags.py", line 312, in render return nodelist.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 938, in render bit = node.render_annotated(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 905, in render_annotated return self.render(context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 994, in render return render_value_in_context(output, context) File "/usr/local/lib/python3.9/site-packages/django/template/base.py", line 973, in render_value_in_context value = str(value) File "/usr/local/lib/python3.9/site-packages/django/utils/html.py", line 376, in <lambda> klass.__str__ = lambda self: mark_safe(klass_str(self)) File "/usr/local/lib/python3.9/site-packages/django/forms/boundfield.py", line 34, in __str__ return self.as_widget() File "/usr/local/lib/python3.9/site-packages/django/forms/boundfield.py", line 93, in as_widget return widget.render( File "/usr/local/lib/python3.9/site-packages/django/forms/widgets.py", line 246, in render context = self.get_context(name, value, attrs) File "/usr/local/lib/python3.9/site-packages/django/contrib/admin/widgets.py", line 283, in get_context 'rendered_widget': self.widget.render(name, value, attrs), File "/usr/local/lib/python3.9/site-packages/smart_selects/widgets.py", line 107, in render url = URL_PREFIX + ("/".join(reverse(view_name, kwargs=kwargs).split("/")[:-2])) File "/usr/local/lib/python3.9/site-packages/django/urls/base.py", line 86, in reverse return resolver._reverse_with_prefix(view, prefix, *args, **kwargs) File "/usr/local/lib/python3.9/site-packages/django/urls/resolvers.py", line 694, in _reverse_with_prefix raise NoReverseMatch(msg) django.urls.exceptions.NoReverseMatch: Reverse for 'chained_filter' not found. 'chained_filter' is not a valid view function or pattern name. I have also tried different variations in urls.py like following urlpatterns = [ re_path(r"^admin/", admin.site.urls), re_path(r"^admin/", include("smart_selects.urls")), ] and urlpatterns = [ re_path(r"^chaining/", include("smart_selects.urls")), ] but nothing seems to be working. My django version is 3.2.5 and I'm using latest version of django-smart-selects. If anyone can let me know how to fix this issue, it would be really appreciated. Thanks.
discord.py Error - ClientConnectorCertificateError
I have an error trying to run my python discord bot. Here's my code: import discord client = discord.Client() #client.event async def on_ready(): print('We have logged in as {0.user}'.format(client)) #client.event async def on_message(message): if message.author == client.user: return None if message.content.startswith('$hello'): await message.channel.send('Hello!') client.run('<redacted>') When I run this, I get the following error code: aiohttp.client_exceptions.ClientConnectorCertificateError: Cannot connect to host discord.com:443 ssl:True [SSLCertVerificationError: (1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1123)')] Full error traceback: Traceback (most recent call last): File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\aiohttp\connector.py", line 936, in _wrap_create_connection return await self._loop.create_connection(*args, **kwargs) # type: ignore # noqa File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\asyncio\base_events.py", line 1050, in create_connection transport, protocol = await self._create_connection_transport( File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\asyncio\base_events.py", line 1080, in _create_connection_transport await waiter File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\asyncio\sslproto.py", line 529, in data_received ssldata, appdata = self._sslpipe.feed_ssldata(data) File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\asyncio\sslproto.py", line 189, in feed_ssldata self._sslobj.do_handshake() File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\ssl.py", line 944, in do_handshake self._sslobj.do_handshake() ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1123) The above exception was the direct cause of the following exception: Traceback (most recent call last): File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\runpy.py", line 194, in _run_module_as_main return _run_code(code, main_globals, None, File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\runpy.py", line 87, in _run_code exec(code, run_globals) File "c:\Users\shalev\.vscode\extensions\ms-python.python-2020.9.111407\pythonFiles\lib\python\debugpy\__main__.py", line 45, in <module> cli.main() File "c:\Users\shalev\.vscode\extensions\ms-python.python-2020.9.111407\pythonFiles\lib\python\debugpy/..\debugpy\server\cli.py", line 430, in main run() File "c:\Users\shalev\.vscode\extensions\ms-python.python-2020.9.111407\pythonFiles\lib\python\debugpy/..\debugpy\server\cli.py", line 267, in run_file runpy.run_path(options.target, run_name=compat.force_str("__main__")) File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\runpy.py", line 265, in run_path return _run_module_code(code, init_globals, run_name, File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\runpy.py", line 97, in _run_module_code _run_code(code, mod_globals, init_globals, File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\runpy.py", line 87, in _run_code exec(code, run_globals) File "f:\python projects\among us bot.py", line 17, in <module> client.run('<redacted>') File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\discord\client.py", line 678, in run return future.result() File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\discord\client.py", line 658, in runner await self.start(*args, **kwargs) File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\discord\client.py", line 621, in start await self.login(*args, bot=bot) File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\discord\client.py", line 472, in login await self.http.static_login(token.strip(), bot=bot) File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\discord\http.py", line 288, in static_login data = await self.request(Route('GET', '/users/#me')) File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\discord\http.py", line 185, in request async with self.__session.request(method, url, **kwargs) as r: File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\aiohttp\client.py", line 1012, in __aenter__ self._resp = await self._coro File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\aiohttp\client.py", line 480, in _request conn = await self._connector.connect( File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\aiohttp\connector.py", line 523, in connect proto = await self._create_connection(req, traces, timeout) File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\aiohttp\connector.py", line 858, in _create_connection _, proto = await self._create_direct_connection( File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\aiohttp\connector.py", line 1004, in _create_direct_connection raise last_exc File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\aiohttp\connector.py", line 980, in _create_direct_connection transp, proto = await self._wrap_create_connection( File "C:\Users\shalev\AppData\Local\Programs\Python\Python38\lib\site-packages\aiohttp\connector.py", line 938, in _wrap_create_connection raise ClientConnectorCertificateError( aiohttp.client_exceptions.ClientConnectorCertificateError: Cannot connect to host discord.com:443 ssl:True [SSLCertVerificationError: (1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1123)')] Can anyone help me with this?
Try updating discord.py. pip install -U discord.py Worked personally for me.
HazelcastSerializationException: When Jet job is running over multiple cluster
I am trying to run one of the Hazelcast-jet example over a Distributed System. My objective is to run code over Disributed System, Utilize Multiple machine's processing power. I have two Laptops connected via LAN. When I run this example in One Machine it works fine, to run it both system I Start Machine 1 and Machine 2 with only jet Instance. code on both Machines are, Machine 1 public class PrimeFinder { public static void main(String[] args) { System.setProperty("hazelcast.logging.type", "log4j"); try { JetConfig cfg = new JetConfig(); cfg.setInstanceConfig(new InstanceConfig().setCooperativeThreadCount( Math.max(1, getRuntime().availableProcessors() / 2))); cfg.setProperty("hazelcast.initial.min.cluster.size","3"); Jet.newJetInstance(cfg); JetInstance jet = Jet.newJetInstance(cfg); DAG dag = new DAG(); final int limit = 15_485_864; Vertex generator = dag.newVertex("number-generator", new GenerateNumbersMetaSupplier(limit)); Vertex primeChecker = dag.newVertex("filter-primes", filterP(PrimeFinder::isPrime)); Vertex writer = dag.newVertex("writer", writeListP("primes")); dag.edge(between(generator, primeChecker)); dag.edge(between(primeChecker, writer)); jet.newJob(dag).join(); IListJet<Integer> primes = jet.getList("primes"); List<Integer> sortedPrimes = primes.stream().sorted().limit(1000).collect(toList()); System.out.println("Found " + primes.size() + " primes."); System.out.println("Some of the primes found are: " + sortedPrimes); } finally { Jet.shutdownAll(); } } private static boolean isPrime(int n) { if (n <= 1) { return false; } int endValue = (int) Math.sqrt(n); for (int i = 2; i <= endValue; i++) { if (n % i == 0) { return false; } } return true; } static class GenerateNumbersMetaSupplier implements ProcessorMetaSupplier { private final int limit; private transient int totalParallelism; private transient int localParallelism; GenerateNumbersMetaSupplier(int limit) { this.limit = limit; } #Override public void init(#Nonnull Context context) { totalParallelism = context.totalParallelism(); localParallelism = context.localParallelism(); } #Override #Nonnull public Function<Address, ProcessorSupplier> get(#Nonnull List<Address> addresses) { Map<Address, ProcessorSupplier> map = new HashMap<>(); for (int i = 0; i < addresses.size(); i++) { Address address = addresses.get(i); int start = i * localParallelism; int end = (i + 1) * localParallelism; int mod = totalParallelism; map.put(address, count -> range(start, end) .mapToObj(index -> new GenerateNumbersP(range(0, limit).filter(f -> f % mod == index))) .collect(toList()) ); } return map::get; } } static class GenerateNumbersP extends AbstractProcessor { private final Traverser<Integer> traverser; GenerateNumbersP(IntStream stream) { traverser = traverseStream(stream.boxed()); } #Override public boolean complete() { return emitFromTraverser(traverser); } } } Machine 2 public class PrimeFinder { public static void main(String[] args) { System.setProperty("hazelcast.logging.type", "log4j"); try { JetConfig cfg = new JetConfig(); cfg.setInstanceConfig(new InstanceConfig().setCooperativeThreadCount( Math.max(1, getRuntime().availableProcessors() / 2))); Jet.newJetInstance(cfg); JetInstance jet = Jet.newJetInstance(cfg); } } Error Members {size:4, ver:4} [ Member [192.168.43.5]:5701 - e9ff45b4-50a2-4918-b51f-9fc3012cce7c Member [192.168.43.224]:5701 - 5931daa7-4452-4275-a3d5-a9daaf247f50 Member [192.168.43.224]:5702 - 57bce3eb-71f7-40d3-86b3-64481feb84e9 Member [192.168.43.5]:5702 - 9fff0e81-748d-4774-8556-2f45941bd59d this ] 06:10,158 [192.168.43.5]:5701 [jet] [3.2] Starting job 0368-8053-f940-0004 based on submit request 06:12,285 [192.168.43.5]:5701 [jet] [3.2] Didn't find any snapshot to restore for job '0368-8053-f940-0004', execution 0368-8056-5ec0-0001 06:12,286 [192.168.43.5]:5701 [jet] [3.2] Start executing job '0368-8053-f940-0004', execution 0368-8056-5ec0-0001, execution graph in DOT format: digraph DAG { "number-generator" [tooltip="local-parallelism=2"]; "filter-primes" [tooltip="local-parallelism=2"]; "writer" [tooltip="local-parallelism=1"]; "number-generator" -> "filter-primes"; "filter-primes" -> "writer"; } HINT: You can use graphviz or http://viz-js.com to visualize the printed graph. 06:12,733 [192.168.43.5]:5701 [jet] [3.2] Execution plan for jobId=0368-8053-f940-0004, jobName='0368-8053-f940-0004', executionId=0368-8056-5ec0-0001 initialized 06:12,784 [192.168.43.5]:5702 [jet] [3.2] Execution plan for jobId=0368-8053-f940-0004, jobName='0368-8053-f940-0004', executionId=0368-8056-5ec0-0001 initialized 06:12,824 [192.168.43.5]:5701 [jet] [3.2] Execution of job '0368-8053-f940-0004', execution 0368-8056-5ec0-0001 failed after 648 ms com.hazelcast.nio.serialization.HazelcastSerializationException: java.io.IOException: unexpected exception type at com.hazelcast.internal.serialization.impl.SerializationUtil.handleException(SerializationUtil.java:70) at com.hazelcast.internal.serialization.impl.AbstractSerializationService.readObject(AbstractSerializationService.java:275) at com.hazelcast.internal.serialization.impl.ByteArrayObjectDataInput.readObject(ByteArrayObjectDataInput.java:574) at com.hazelcast.jet.impl.execution.init.CustomClassLoadedObject.read(CustomClassLoadedObject.java:56) at com.hazelcast.jet.impl.execution.init.VertexDef.readData(VertexDef.java:153) at com.hazelcast.internal.serialization.impl.DataSerializableSerializer.readInternal(DataSerializableSerializer.java:160) at com.hazelcast.internal.serialization.impl.DataSerializableSerializer.read(DataSerializableSerializer.java:106) at com.hazelcast.internal.serialization.impl.DataSerializableSerializer.read(DataSerializableSerializer.java:51) at com.hazelcast.internal.serialization.impl.StreamSerializerAdapter.read(StreamSerializerAdapter.java:48) at com.hazelcast.internal.serialization.impl.AbstractSerializationService.readObject(AbstractSerializationService.java:269) at com.hazelcast.internal.serialization.impl.ByteArrayObjectDataInput.readObject(ByteArrayObjectDataInput.java:574) at com.hazelcast.jet.impl.util.ImdgUtil.readList(ImdgUtil.java:444) at com.hazelcast.jet.impl.execution.init.ExecutionPlan.readData(ExecutionPlan.java:307) at com.hazelcast.internal.serialization.impl.DataSerializableSerializer.readInternal(DataSerializableSerializer.java:160) at com.hazelcast.internal.serialization.impl.DataSerializableSerializer.read(DataSerializableSerializer.java:106) at com.hazelcast.internal.serialization.impl.DataSerializableSerializer.read(DataSerializableSerializer.java:51) at com.hazelcast.internal.serialization.impl.StreamSerializerAdapter.read(StreamSerializerAdapter.java:48) at com.hazelcast.internal.serialization.impl.AbstractSerializationService.toObject(AbstractSerializationService.java:187) at com.hazelcast.jet.impl.execution.init.CustomClassLoadedObject.deserializeWithCustomClassLoader(CustomClassLoadedObject.java:65) at com.hazelcast.jet.impl.operation.InitExecutionOperation.deserializePlan(InitExecutionOperation.java:116) at com.hazelcast.jet.impl.operation.InitExecutionOperation.run(InitExecutionOperation.java:71) at com.hazelcast.spi.Operation.call(Operation.java:170) at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.call(OperationRunnerImpl.java:210) at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:199) at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:416) at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:153) at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:123) at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.run(OperationThread.java:110) at ------ submitted from ------.(Unknown Source) at com.hazelcast.spi.impl.operationservice.impl.InvocationFuture.resolve(InvocationFuture.java:126) at com.hazelcast.spi.impl.AbstractInvocationFuture$1.run(AbstractInvocationFuture.java:251) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) at com.hazelcast.util.executor.HazelcastManagedThread.executeRun(HazelcastManagedThread.java:64) at com.hazelcast.util.executor.HazelcastManagedThread.run(HazelcastManagedThread.java:80) Caused by: java.io.IOException: unexpected exception type..... When I run this code on a single Machine, Example is working fine and output is as expected. Members {size:2, ver:2} [ Member [172.29.97.33]:5701 - e2e6a9d5-9bdd-49d4-934c-e7becee8ad58 Member [172.29.97.33]:5702 - 9e121fda-ca89-4d8c-a20e-3e345ce91fc2 this ] 36:20,086 [172.29.97.33]:5701 [jet] [3.2] Starting job 0368-94f5-7bc0-0002 based on submit request 36:20,316 [172.29.97.33]:5701 [jet] [3.2] Didn't find any snapshot to restore for job '0368-94f5-7bc0-0002', execution 0368-94f7-a640-0001 36:20,316 [172.29.97.33]:5701 [jet] [3.2] Start executing job '0368-94f5-7bc0-0002', execution 0368-94f7-a640-0001, execution graph in DOT format: digraph DAG { "number-generator" [tooltip="local-parallelism=2"]; "filter-primes" [tooltip="local-parallelism=2"]; "writer" [tooltip="local-parallelism=1"]; "number-generator" -> "filter-primes"; "filter-primes" -> "writer"; } HINT: You can use graphviz or http://viz-js.com to visualize the printed graph. 36:21,185 [172.29.97.33]:5701 [jet] [3.2] Execution plan for jobId=0368-94f5-7bc0-0002, jobName='0368-94f5-7bc0-0002', executionId=0368-94f7-a640-0001 initialized 36:21,237 [172.29.97.33]:5702 [jet] [3.2] Execution plan for jobId=0368-94f5-7bc0-0002, jobName='0368-94f5-7bc0-0002', executionId=0368-94f7-a640-0001 initialized 36:21,250 [172.29.97.33]:5701 [jet] [3.2] Start execution of job '0368-94f5-7bc0-0002', execution 0368-94f7-a640-0001 from coordinator [172.29.97.33]:5701 36:21,327 [172.29.97.33]:5702 [jet] [3.2] Start execution of job '0368-94f5-7bc0-0002', execution 0368-94f7-a640-0001 from coordinator [172.29.97.33]:5701 37:11,263 [172.29.97.33]:5701 [jet] [3.2] Execution of job '0368-94f5-7bc0-0002', execution 0368-94f7-a640-0001 completed in 51,171 ms Found 1000000 primes. Some of the primes found are: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211, 223, 227, 229, 233, 239, 241, 251, 257, 263, 269, 271, 277, 281, 283, 293, 307, 311, 313, 317, 331, 337, 347, 349, 353, 359, 367, 373, 379, 383, 389, 397, 401, 409, 419, 421, 431, 433, 439, 443, 449, 457, 461, 463, 467, 479, 487, 491, 499, 503, 509, 521, 523, 541, 547, 557, 563, 569, 571, 577, 587, 593, 599, 601, 607, 613, 617, 619, 631, 641, 643, 647, 653, 659, 661, 673, 677, 683, 691, 701, 709, 719, 727, 733, 739, 743, 751, 757, 761, 769, 773, 787, 797, 809, 811, 821, 823, 827, 829, 839, 853, 857, 859, 863, 877, 881, 883, 887, 907, 911, 919, 929, 937, 941, 947, 953, 967, 971, 977, 983, 991, 997, 1009, 1013, 1019, 1021, 1031, 1033, 1039, 1049, 1051, 1061, 1063, 1069, 1087, 1091, 1093, 1097, 1103, 1109, 1117, 1123, 1129, 1151, 1153, 1163, 1171, 1181, 1187, 1193, 1201, 1213, 1217, 1223, 1229, 1231, 1237, 1249, 1259, 1277, 1279, 1283, 1289, 1291, 1297, 1301, 1303, 1307, 1319, 1321, 1327, 1361, 1367, 1373, 1381, 1399, 1409, 1423, 1427, 1429, 1433, 1439, 1447, 1451, 1453, 1459, 1471, 1481, 1483, 1487, 1489, 1493, 1499, 1511, 1523, 1531, 1543, 1549, 1553, 1559, 1567, 1571, 1579, 1583, 1597, 1601, 1607, 1609, 1613, 1619, 1621, 1627, 1637, 1657, 1663, 1667, 1669, 1693, 1697, 1699, 1709, 1721, 1723, 1733, 1741, 1747, 1753, 1759, 1777, 1783, 1787, 1789, 1801, 1811, 1823, 1831, 1847, 1861, 1867, 1871, 1873, 1877, 1879, 1889, 1901, 1907, 1913, 1931, 1933, 1949, 1951, 1973, 1979, 1987, 1993, 1997, 1999, 2003, 2011, 2017, 2027, 2029, 2039, 2053, 2063, 2069, 2081, 2083, 2087, 2089, 2099, 2111, 2113, 2129, 2131, 2137, 2141, 2143, 2153, 2161, 2179, 2203, 2207, 2213, 2221, 2237, 2239, 2243, 2251, 2267, 2269, 2273, 2281, 2287, 2293, 2297, 2309, 2311, 2333, 2339, 2341, 2347, 2351, 2357, 2371, 2377, 2381, 2383, 2389, 2393, 2399, 2411, 2417, 2423, 2437, 2441, 2447, 2459, 2467, 2473, 2477, 2503, 2521, 2531, 2539, 2543, 2549, 2551, 2557, 2579, 2591, 2593, 2609, 2617, 2621, 2633, 2647, 2657, 2659, 2663, 2671, 2677, 2683, 2687, 2689, 2693, 2699, 2707, 2711, 2713, 2719, 2729, 2731, 2741, 2749, 2753, 2767, 2777, 2789, 2791, 2797, 2801, 2803, 2819, 2833, 2837, 2843, 2851, 2857, 2861, 2879, 2887, 2897, 2903, 2909, 2917, 2927, 2939, 2953, 2957, 2963, 2969, 2971, 2999, 3001, 3011, 3019, 3023, 3037, 3041, 3049, 3061, 3067, 3079, 3083, 3089, 3109, 3119, 3121, 3137, 3163, 3167, 3169, 3181, 3187, 3191, 3203, 3209, 3217, 3221, 3229, 3251, 3253, 3257, 3259, 3271, 3299, 3301, 3307, 3313, 3319, 3323, 3329, 3331, 3343, 3347, 3359, 3361, 3371, 3373, 3389, 3391, 3407, 3413, 3433, 3449, 3457, 3461, 3463, 3467, 3469, 3491, 3499, 3511, 3517, 3527, 3529, 3533, 3539, 3541, 3547, 3557, 3559, 3571, 3581, 3583, 3593, 3607, 3613, 3617, 3623, 3631, 3637, 3643, 3659, 3671, 3673, 3677, 3691, 3697, 3701, 3709, 3719, 3727, 3733, 3739, 3761, 3767, 3769, 3779, 3793, 3797, 3803, 3821, 3823, 3833, 3847, 3851, 3853, 3863, 3877, 3881, 3889, 3907, 3911, 3917, 3919, 3923, 3929, 3931, 3943, 3947, 3967, 3989, 4001, 4003, 4007, 4013, 4019, 4021, 4027, 4049, 4051, 4057, 4073, 4079, 4091, 4093, 4099, 4111, 4127, 4129, 4133, 4139, 4153, 4157, 4159, 4177, 4201, 4211, 4217, 4219, 4229, 4231, 4241, 4243, 4253, 4259, 4261, 4271, 4273, 4283, 4289, 4297, 4327, 4337, 4339, 4349, 4357, 4363, 4373, 4391, 4397, 4409, 4421, 4423, 4441, 4447, 4451, 4457, 4463, 4481, 4483, 4493, 4507, 4513, 4517, 4519, 4523, 4547, 4549, 4561, 4567, 4583, 4591, 4597, 4603, 4621, 4637, 4639, 4643, 4649, 4651, 4657, 4663, 4673, 4679, 4691, 4703, 4721, 4723, 4729, 4733, 4751, 4759, 4783, 4787, 4789, 4793, 4799, 4801, 4813, 4817, 4831, 4861, 4871, 4877, 4889, 4903, 4909, 4919, 4931, 4933, 4937, 4943, 4951, 4957, 4967, 4969, 4973, 4987, 4993, 4999, 5003, 5009, 5011, 5021, 5023, 5039, 5051, 5059, 5077, 5081, 5087, 5099, 5101, 5107, 5113, 5119, 5147, 5153, 5167, 5171, 5179, 5189, 5197, 5209, 5227, 5231, 5233, 5237, 5261, 5273, 5279, 5281, 5297, 5303, 5309, 5323, 5333, 5347, 5351, 5381, 5387, 5393, 5399, 5407, 5413, 5417, 5419, 5431, 5437, 5441, 5443, 5449, 5471, 5477, 5479, 5483, 5501, 5503, 5507, 5519, 5521, 5527, 5531, 5557, 5563, 5569, 5573, 5581, 5591, 5623, 5639, 5641, 5647, 5651, 5653, 5657, 5659, 5669, 5683, 5689, 5693, 5701, 5711, 5717, 5737, 5741, 5743, 5749, 5779, 5783, 5791, 5801, 5807, 5813, 5821, 5827, 5839, 5843, 5849, 5851, 5857, 5861, 5867, 5869, 5879, 5881, 5897, 5903, 5923, 5927, 5939, 5953, 5981, 5987, 6007, 6011, 6029, 6037, 6043, 6047, 6053, 6067, 6073, 6079, 6089, 6091, 6101, 6113, 6121, 6131, 6133, 6143, 6151, 6163, 6173, 6197, 6199, 6203, 6211, 6217, 6221, 6229, 6247, 6257, 6263, 6269, 6271, 6277, 6287, 6299, 6301, 6311, 6317, 6323, 6329, 6337, 6343, 6353, 6359, 6361, 6367, 6373, 6379, 6389, 6397, 6421, 6427, 6449, 6451, 6469, 6473, 6481, 6491, 6521, 6529, 6547, 6551, 6553, 6563, 6569, 6571, 6577, 6581, 6599, 6607, 6619, 6637, 6653, 6659, 6661, 6673, 6679, 6689, 6691, 6701, 6703, 6709, 6719, 6733, 6737, 6761, 6763, 6779, 6781, 6791, 6793, 6803, 6823, 6827, 6829, 6833, 6841, 6857, 6863, 6869, 6871, 6883, 6899, 6907, 6911, 6917, 6947, 6949, 6959, 6961, 6967, 6971, 6977, 6983, 6991, 6997, 7001, 7013, 7019, 7027, 7039, 7043, 7057, 7069, 7079, 7103, 7109, 7121, 7127, 7129, 7151, 7159, 7177, 7187, 7193, 7207, 7211, 7213, 7219, 7229, 7237, 7243, 7247, 7253, 7283, 7297, 7307, 7309, 7321, 7331, 7333, 7349, 7351, 7369, 7393, 7411, 7417, 7433, 7451, 7457, 7459, 7477, 7481, 7487, 7489, 7499, 7507, 7517, 7523, 7529, 7537, 7541, 7547, 7549, 7559, 7561, 7573, 7577, 7583, 7589, 7591, 7603, 7607, 7621, 7639, 7643, 7649, 7669, 7673, 7681, 7687, 7691, 7699, 7703, 7717, 7723, 7727, 7741, 7753, 7757, 7759, 7789, 7793, 7817, 7823, 7829, 7841, 7853, 7867, 7873, 7877, 7879, 7883, 7901, 7907, 7919] 37:21,664 [172.29.97.33]:5702 [jet] [3.2] Removing connection to endpoint [172.29.97.33]:5701 Cause => java.net.SocketException {Connection refused: no further information to address /172.29.97.33:5701}, Error-Count: 5 37:21,697 [172.29.97.33]:5702 [jet] [3.2] Member [172.29.97.33]:5701 - e2e6a9d5-9bdd-49d4-934c-e7becee8ad58 is suspected to be dead for reason: No connection 37:21,697 [172.29.97.33]:5702 [jet] [3.2] Starting mastership claim process... 37:21,698 [172.29.97.33]:5702 [jet] [3.2] Local MembersView{version=2, members=[MemberInfo{address=[172.29.97.33]:5701, uuid=e2e6a9d5-9bdd-49d4-934c-e7becee8ad58, liteMember=false, memberListJoinVersion=1}, MemberInfo{address=[172.29.97.33]:5702, uuid=9e121fda-ca89-4d8c-a20e-3e345ce91fc2, liteMember=false, memberListJoinVersion=2}]} with suspected members: [[172.29.97.33]:5701] and initial addresses to ask: [] 37:21,772 [172.29.97.33]:5702 [jet] [3.2] Members {size:1, ver:3} [ Member [172.29.97.33]:5702 - 9e121fda-ca89-4d8c-a20e-3e345ce91fc2 this ] 37:21,773 [172.29.97.33]:5702 [jet] [3.2] Mastership is claimed with: MembersView{version=3, members=[MemberInfo{address=[172.29.97.33]:5702, uuid=9e121fda-ca89-4d8c-a20e-3e345ce91fc2, liteMember=false, memberListJoinVersion=2}]} Can anyone help me out what am I doing wrong?
You need to add the class to the JobConfig and pass it when submitting the pipeline: JobConfig jobConfig = new JobConfig(); jobConfig.addClass(PrimeFinder.class); jet.newJob(dag, jobConfig).join();
Python requests - retry for any status code but not 400
I am using the following snippet(please note, only partial snippets shown): from urllib3.util import Retry status_forcelist = (500, 502, 504) retry = Retry( total=retries, read=retries, connect=retries, backoff_factor=backoff_factor, status_forcelist=status_forcelist, method_whitelist=frozenset(['GET', 'POST']) As of now, the retry would be done for status codes 500,502,504. However, I intend to retry for any status code but not 400. Is there a graceful way to achieve this instead of populating/hardcoding the status_forcelist for all status codes ?
You can take the codes from request module - which saves you writing them out. In case any new ones get introduced you would have them as well: import requests from urllib3.util import Retry status_forcelist = tuple( x for x in requests.status_codes._codes if x != 400) print(status_forcelist) Output: (100, 101, 102, 103, 122, 200, 201, 202, 203, 204, 205, 206, 207, 208, 226, 300, 301, 302, 303, 304, 305, 306, 307, 308, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 421, 422, 423, 424, 425, 426, 428, 429, 431, 444, 449, 450, 451, 499, 500, 501, 502, 503, 504, 505, 506, 507, 509, 510, 511) Caveat: wikipedia list "self created" / unofficial statuscodes as well as proprietary ones around 9xx which are not covered by this list though. #zwer solution would cover those as well.
status_forcelist, unfortunately, applies as a whitelist only (given the use, it can be called a blacklist, tho) so there are no built-in ways to do it gracefully. However, you can override (or monkey-patch if that's your thing) urllib3.util.retry.Retry.is_retry() to treat it as a blacklist, something like: import urllib3.util.retry class BlacklistRetry(urllib3.util.retry.Retry): def is_retry(self, method, status_code, has_retry_after=False): if not self._is_method_retryable(method): return False if self.status_forcelist and status_code not in self.status_forcelist: # threat as a blacklist --------------^ return True return (self.total and self.respect_retry_after_header and has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES)) Then use it instead of the original Retry and just add 400 to its status_forcelist.
The current accepted answer has a bug, it retries for successful http status codes, which means it will unnecessarily retry - if you want proof, you can test this with the code from this answer here. You could just follow a similar implementation to Response.raise_for_status. from urllib3.util import Retry retry = Retry( total=retries, read=retries, connect=retries, backoff_factor=backoff_factor, status_forcelist=tuple(range(401, 600)), method_whitelist=frozenset(['GET', 'POST'])
unit testing for unverified user trying to login
I've been trying to test for a user that hasn't been verified. class TestLoginApi(TestCase): URL = '/rest-auth/login/' EMAIL = 'testuser#test PASSWORD = 'password' DATA = {'email': EMAIL, 'password': PASSWORD} #classmethod def setUpTestData(cls): cls.user = get_user_model().objects.create_user(username='testuser', email=cls.EMAIL, password=cls.PASSWORD) def test_login_api_without_verification(self): response = self.client.post(self.URL, self.DATA, format='json') The "response" line throws the following error Error Traceback (most recent call last): File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/unittest/case.py", line 59, in testPartExecutor yield File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/unittest/case.py", line 601, in run testMethod() File "/Users/docdocengineering3/GitHub/website/project/authorization/Tests/test_login.py", line 34, in test_login_api_without_verification response = self.client.post(self.URL, self.DATA, format='json') File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/test/client.py", line 548, in post secure=secure, **extra) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/test/client.py", line 350, in post secure=secure, **extra) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/test/client.py", line 416, in generic return self.request(**r) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/test/client.py", line 501, in request six.reraise(*exc_info) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/utils/six.py", line 686, in reraise raise value File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/core/handlers/exception.py", line 41, in inner response = get_response(request) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/core/handlers/base.py", line 187, in _get_response response = self.process_exception_by_middleware(e, request) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/core/handlers/base.py", line 185, in _get_response response = wrapped_callback(request, *callback_args, **callback_kwargs) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/views/decorators/csrf.py", line 58, in wrapped_view return view_func(*args, **kwargs) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/views/generic/base.py", line 68, in view return self.dispatch(request, *args, **kwargs) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/utils/decorators.py", line 67, in _wrapper return bound_func(*args, **kwargs) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/views/decorators/debug.py", line 76, in sensitive_post_parameters_wrapper return view(request, *args, **kwargs) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/utils/decorators.py", line 63, in bound_func return func.get(self, type(self))(*args2, **kwargs2) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/rest_auth/views.py", line 49, in dispatch return super(LoginView, self).dispatch(*args, **kwargs) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/rest_framework/views.py", line 489, in dispatch response = self.handle_exception(exc) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/rest_framework/views.py", line 449, in handle_exception self.raise_uncaught_exception(exc) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/rest_framework/views.py", line 486, in dispatch response = handler(request, *args, **kwargs) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/rest_auth/views.py", line 92, in post self.serializer.is_valid(raise_exception=True) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/rest_framework/serializers.py", line 237, in is_valid self._validated_data = self.run_validation(self.initial_data) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/rest_framework/serializers.py", line 435, in run_validation value = self.validate(value) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/rest_auth/serializers.py", line 105, in validate email_address = user.emailaddress_set.get(email=user.email) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/db/models/manager.py", line 85, in manager_method return getattr(self.get_queryset(), name)(*args, **kwargs) File "/Users/docdocengineering3/.virtualenvs/docdoc/lib/python3.6/site-packages/django/db/models/query.py", line 380, in get self.model._meta.object_name Exception: EmailAddress matching query does not exist. I have no idea why this is happening. It isn't a setup problem as I've tried actually running the system, creating a user (not verifying) and when I "PostMan" the same link, I get the correct error back {"non_field_errors": [ "E-mail is not verified." ] } So it works in the actual product but not when testing. Anyone know why? Any help is greatly appreciated. This is the only test that I can't get to work, all the rest runs correctly.
You probably have your setting EMAIL_VERIFICATION set to mandatory. So you either change it to none in tests or manually create EmailAddress object (it's required by django-allauth when mandatory verification is on)