I have 1106 columns and 6689 rows to be filled in a sparse matrix that is filled by reading a csv file using DictReader from the csv library.
There are some bad columns so I want to remove those (currently trying to identify which ones are the bad columns).
Anyway here's the code:
def read_annotations(filehandle):
with open(filehandle, 'r', encoding = 'utf-8') as csvfile:
reader = csv.DictReader(csvfile)
workers = defaultdict(list)
ids = []
for row in reader:
# CF annotators given a choice to choose link on their request, We treat it as neither (so do expert raters
if row['which_form_of_hate_speech_is_this'].lower() == 'link':
row['which_form_of_hate_speech_is_this'] = 'neither'
# Create a dictionary containing each annotation by a worker,
# save their trust score, their annotation, the id of the tweet and the tweet in a tuple
workers[row['_worker_id']].append((row['_trust'], row['which_form_of_hate_speech_is_this'], row['id'], row['tweet']))
return workers, set(ids)
def annotation_writer(workers, ids):
""" Writes a annotations to file where each column is a worker and each row is a tweet ID
:param workers: Dictionary containing worker ID as key, trust, annotation, tweet id and tweet as tuple
"""
# Create empty dataframe with tweet IDS as indexes and workers as columns
df = pd.DataFrame(index = ids, columns= list(workers.keys()))
# Go through each worker
bad_annotators = [683, 691, 694, 691]
for i, w_idx in enumerate(workers):
# Go through each list item
if i <= 693:
if i in bad_annotators:
print(w_idx)
continue
for item in workers[w_idx]:
_, label, idx, tweet = item
# Set the label they have chosen for the item at index: TweetID and row worker_id
df.set_value(idx, w_idx, label)
return df
The profiling output ordered by cumulative time:
tottime percall cumtime percall filename:lineno(function)
466/1 0.020 0.000 361.865 361.865 {built-in method builtins.exec}
1 0.000 0.000 361.865 361.865 annotator.py:1(<module>)
1 93.955 93.955 360.789 360.789 annotator.py:24(annotation_writer)
20923 0.143 0.000 266.759 0.013 frame.py:1840(set_value)
6594 0.088 0.000 265.032 0.040 indexing.py:126(__setitem__)
6594 0.478 0.000 263.543 0.040 indexing.py:224(_setitem_with_indexer)
6594 0.077 0.000 254.250 0.039 frame.py:2743(reindex_axis)
6594 0.071 0.000 254.173 0.039 generic.py:2307(reindex_axis)
6594 0.144 0.000 250.682 0.038 generic.py:2320(_reindex_with_indexers)
6594 0.095 0.000 250.327 0.038 internals.py:3560(reindex_indexer)
6594 0.029 0.000 249.124 0.038 internals.py:3595(<listcomp>)
6594 0.155 0.000 249.096 0.038 internals.py:975(take_nd)
6594 0.239 0.000 248.633 0.038 algorithms.py:840(take_nd)
34098 140.266 0.004 140.266 0.004 {built-in method numpy.core.multiarray.empty}
6594 107.927 0.016 107.927 0.016 {pandas.algos.take_2d_axis1_object_object}
40942 0.136 0.000 4.503 0.000 base.py:1915(get_loc)
54130 4.213 0.000 4.242 0.000 {method 'get_loc' of 'pandas.index.IndexEngine' objects}
6594 0.052 0.000 3.299 0.001 base.py:2295(reindex)
6594 0.079 0.000 3.099 0.000 base.py:2028(get_indexer)
6594 2.871 0.000 2.923 0.000 {method 'get_indexer' of 'pandas.index.IndexEngine' objects}
6594 0.075 0.000 2.321 0.000 base.py:3011(insert)
13189 0.112 0.000 1.707 0.000 internals.py:2578(__init__)
19787/13191 0.368 0.000 1.620 0.000 base.py:124(__new__)
6594 0.027 0.000 1.519 0.000 internals.py:2916(setitem)
6594 0.108 0.000 1.492 0.000 internals.py:2811(apply)
20923 0.084 0.000 1.434 0.000 generic.py:1345(_get_item_cache)
6594 0.034 0.000 1.359 0.000 indexing.py:101(_get_setitem_indexer)
6594 0.048 0.000 1.281 0.000 indexing.py:163(_convert_tuple)
13188 0.105 0.000 1.229 0.000 indexing.py:1102(_convert_to_indexer)
13189 0.456 0.000 1.155 0.000 internals.py:2674(_rebuild_blknos_and_blklocs)
6594 0.043 0.000 1.008 0.000 base.py:522(_coerce_scalar_to_index)
13192 0.869 0.000 0.895 0.000 {pandas.lib.infer_dtype}
6594 0.031 0.000 0.761 0.000 base.py:354(_shallow_copy_with_infer)
6594 0.130 0.000 0.720 0.000 internals.py:628(setitem)
7283 0.082 0.000 0.694 0.000 internals.py:3283(get)
20473 0.167 0.000 0.669 0.000 internals.py:2482(make_block)
13188 0.020 0.000 0.639 0.000 base.py:950(is_integer)
6595 0.010 0.000 0.619 0.000 base.py:1172(inferred_type)
571/2 0.004 0.000 0.610 0.305 <frozen importlib._bootstrap>:966(_find_and_load)
571/2 0.003 0.000 0.610 0.305 <frozen importlib._bootstrap>:939(_find_and_load_unlocked)
452/2 0.003 0.000 0.610 0.305 <frozen importlib._bootstrap>:659(_load_unlocked)
387/2 0.002 0.000 0.610 0.305 <frozen importlib._bootstrap_external>:656(exec_module)
599/2 0.001 0.000 0.609 0.305 <frozen importlib._bootstrap>:214(_call_with_frames_removed)
2 0.000 0.000 0.609 0.304 __init__.py:5(<module>)
989631/989629 0.370 0.000 0.603 0.000 {built-in method builtins.isinstance}
7283 0.047 0.000 0.593 0.000 frame.py:2331(_box_item_values)
7283 0.031 0.000 0.480 0.000 frame.py:2338(_box_col_values)
1 0.068 0.068 0.465 0.465 annotator.py:7(read_annotations)
13877 0.069 0.000 0.463 0.000 internals.py:183(make_block_same_class)
7283 0.038 0.000 0.449 0.000 series.py:236(from_array)
432/41 0.001 0.000 0.440 0.011 {built-in method builtins.__import__}
7283 0.133 0.000 0.438 0.000 internals.py:3312(iget)
31512 0.141 0.000 0.389 0.000 csv.py:106(__next__)
20472 0.176 0.000 0.381 0.000 internals.py:1657(__init__)
7284 0.054 0.000 0.369 0.000 series.py:120(__init__)
70099/69243 0.051 0.000 0.367 0.000 <frozen importlib._bootstrap>:996(_handle_fromlist)
6594 0.350 0.000 0.350 0.000 {built-in method numpy.core.multiarray.concatenate}
19787 0.116 0.000 0.329 0.000 common.py:1380(_asarray_tuplesafe)
6594 0.032 0.000 0.310 0.000 internals.py:171(make_block)
533564/425149 0.221 0.000 0.290 0.000 {built-in method builtins.len}
32973 0.122 0.000 0.282 0.000 internals.py:2619(shape)
232315 0.230 0.000 0.270 0.000 {built-in method builtins.getattr}
41781 0.055 0.000 0.267 0.000 common.py:1710(is_datetimetz)
54821 0.182 0.000 0.260 0.000 generic.py:2674(__setattr__)
40673 0.061 0.000 0.251 0.000 numeric.py:414(asarray)
58149 0.225 0.000 0.238 0.000 {built-in method numpy.core.multiarray.array}
126058 0.096 0.000 0.231 0.000 generic.py:7(_check)
103351 0.124 0.000 0.230 0.000 dtypes.py:74(is_dtype)
61570 0.068 0.000 0.227 0.000 common.py:1736(is_categorical_dtype)
278263/278240 0.222 0.000 0.223 0.000 {built-in method builtins.hasattr}
31532 0.214 0.000 0.222 0.000 {built-in method builtins.next}
13188 0.043 0.000 0.220 0.000 indexing.py:183(_convert_scalar_indexer)
15213 0.030 0.000 0.208 0.000 {method 'any' of 'numpy.ndarray' objects}
52752 0.059 0.000 0.206 0.000 generic.py:333(_get_axis)
6595 0.054 0.000 0.201 0.000 internals.py:2799(_verify_integrity)
27 0.001 0.000 0.195 0.007 __init__.py:1(<module>)
20473 0.097 0.000 0.194 0.000 internals.py:77(__init__)
6595 0.060 0.000 0.194 0.000 frame.py:210(__init__)
19782 0.042 0.000 0.179 0.000 generic.py:2713(_protect_consolidate)
15213 0.015 0.000 0.178 0.000 _methods.py:37(_any)
6594 0.013 0.000 0.171 0.000 internals.py:555(_try_coerce_and_cast_result)
13188 0.046 0.000 0.168 0.000 generic.py:1407(_maybe_update_cacher)
1 0.000 0.000 0.167 0.167 api.py:5(<module>)
13189 0.069 0.000 0.164 0.000 internals.py:3004(_consolidate_check)
15213 0.163 0.000 0.163 0.000 {method 'reduce' of 'numpy.ufunc' objects}
98919 0.034 0.000 0.159 0.000 internals.py:2621(<genexpr>)
7284 0.032 0.000 0.156 0.000 series.py:270(_set_axis)
6594 0.029 0.000 0.156 0.000 internals.py:510(_try_cast_result)
1 0.000 0.000 0.155 0.155 groupby.py:1(<module>)
13188 0.023 0.000 0.150 0.000 generic.py:2723(_consolidate_inplace)
1 0.000 0.000 0.145 0.145 frame.py:10(<module>)
6594 0.047 0.000 0.141 0.000 common.py:527(_maybe_promote)
15402 0.024 0.000 0.138 0.000 common.py:1731(is_categorical)
13188 0.046 0.000 0.136 0.000 base.py:972(_convert_scalar_indexer)
13191 0.073 0.000 0.131 0.000 base.py:309(_simple_new)
59346 0.094 0.000 0.128 0.000 generic.py:320(_get_axis_name)
1 0.000 0.000 0.126 0.126 __init__.py:106(<module>)
2 0.000 0.000 0.125 0.063 __init__.py:9(<module>)
1514/1502 0.050 0.000 0.125 0.000 {built-in method builtins.__build_class__}
2 0.000 0.000 0.123 0.062 __init__.py:15(<module>)
54132 0.091 0.000 0.117 0.000 {pandas.lib.values_from_object}
6595 0.029 0.000 0.115 0.000 base.py:1507(equals)
41781 0.037 0.000 0.108 0.000 common.py:1575(is_datetime64tz_dtype)
6595 0.033 0.000 0.106 0.000 base.py:1180(is_all_dates)
52760 0.057 0.000 0.104 0.000 base.py:440(values)
1 0.000 0.000 0.104 0.104 series.py:3(<module>)
1 0.000 0.000 0.102 0.102 add_newdocs.py:10(<module>)
387 0.004 0.000 0.102 0.000 <frozen importlib._bootstrap_external>:726(get_code)
1 0.000 0.000 0.100 0.100 config_init.py:11(<module>)
107722 0.069 0.000 0.097 0.000 base.py:409(__len__)
13188 0.038 0.000 0.095 0.000 generic.py:2726(f)
6594 0.025 0.000 0.093 0.000 indexing.py:1860(maybe_convert_ix)
1 0.000 0.000 0.093 0.093 plotting.py:3(<module>)
1 0.000 0.000 0.090 0.090 converter.py:1(<module>)
1 0.000 0.000 0.089 0.089 format.py:2(<module>)
13189 0.022 0.000 0.089 0.000 internals.py:3005(<listcomp>)
27482 0.087 0.000 0.087 0.000 {method 'fill' of 'numpy.ndarray' objects}
1 0.000 0.000 0.087 0.087 type_check.py:3(<module>)
6596 0.085 0.000 0.085 0.000 {built-in method pandas.lib.list_to_object_array}
435/433 0.001 0.000 0.084 0.000 <frozen importlib._bootstrap>:570(module_from_spec)
20923 0.082 0.000 0.082 0.000 {method 'set_value' of 'pandas.index.IndexEngine' objects}
1 0.002 0.002 0.075 0.075 frame.py:307(_init_dict)
13190 0.075 0.000 0.075 0.000 {built-in method numpy.core.multiarray.arange}
6594 0.024 0.000 0.075 0.000 internals.py:1665(is_bool)
6594 0.057 0.000 0.074 0.000 algorithms.py:807(_get_take_nd_function)
13188 0.054 0.000 0.073 0.000 base.py:506(_get_attributes_dict)
764 0.002 0.000 0.072 0.000 re.py:278(_compile)
192 0.000 0.000 0.071 0.000 re.py:222(compile)
43/42 0.000 0.000 0.071 0.002 <frozen importlib._bootstrap_external>:900(create_module)
43/42 0.055 0.001 0.070 0.002 {built-in method _imp.create_dynamic}
167 0.001 0.000 0.070 0.000 sre_compile.py:531(compile)
387 0.001 0.000 0.070 0.000 <frozen importlib._bootstrap_external>:471(_compile_bytecode)
1 0.000 0.000 0.068 0.068 __init__.py:101(<module>)
387 0.067 0.000 0.067 0.000 {built-in method marshal.loads}
13188 0.028 0.000 0.067 0.000 common.py:1532(is_dtype_equal)
13189 0.060 0.000 0.066 0.000 internals.py:276(ftype)
540/539 0.006 0.000 0.066 0.000 <frozen importlib._bootstrap>:879(_find_spec)
20472 0.065 0.000 0.065 0.000 generic.py:2658(__getattr__)
1 0.000 0.000 0.064 0.064 frame.py:5224(_arrays_to_mgr)
6594 0.011 0.000 0.064 0.000 generic.py:2753(_is_mixed_type)
13189 0.038 0.000 0.063 0.000 internals.py:2579(<listcomp>)
13879 0.062 0.000 0.062 0.000 generic.py:94(__init__)
13191 0.019 0.000 0.057 0.000 common.py:1696(is_bool_dtype)
7283 0.009 0.000 0.057 0.000 common.py:73(isnull)
20923 0.017 0.000 0.057 0.000 series.py:366(_values)
2 0.000 0.000 0.056 0.028 common.py:1(<module>)
535 0.001 0.000 0.055 0.000 <frozen importlib._bootstrap_external>:1130(find_spec)
7284 0.036 0.000 0.055 0.000 internals.py:3778(__init__)
535 0.004 0.000 0.055 0.000 <frozen importlib._bootstrap_external>:1098(_get_spec)
53448 0.044 0.000 0.054 0.000 base.py:3381(_ensure_index)
20473 0.046 0.000 0.054 0.000 internals.py:191(mgr_locs)
59355 0.053 0.000 0.053 0.000 {method 'view' of 'numpy.ndarray' objects}
6594 0.036 0.000 0.052 0.000 common.py:733(_possibly_downcast_to_dtype)
990 0.011 0.000 0.052 0.000 <frozen importlib._bootstrap_external>:1212(find_spec)
6594 0.013 0.000 0.051 0.000 generic.py:1476(_check_is_chained_assignment_possible)
2 0.001 0.000 0.051 0.026 pyparsing.py:58(<module>)
1 0.000 0.000 0.051 0.051 rcsetup.py:15(<module>)
7283 0.022 0.000 0.049 0.000 generic.py:1359(_set_as_cached)
1 0.000 0.000 0.048 0.048 fontconfig_pattern.py:7(<module>)
26380 0.036 0.000 0.048 0.000 common.py:1511(_get_dtype_type)
7283 0.033 0.000 0.048 0.000 common.py:94(_isnull_new)
6594 0.029 0.000 0.048 0.000 base.py:2201(_possibly_promote)
7284 0.020 0.000 0.045 0.000 series.py:302(name)
1 0.002 0.002 0.045 0.045 frame.py:5521(_homogenize)
95791 0.044 0.000 0.044 0.000 {method 'get' of 'dict' objects}
19783 0.017 0.000 0.043 0.000 base.py:870(_values)
13188 0.014 0.000 0.043 0.000 indexing.py:1890(is_list_like_indexer)
1105 0.003 0.000 0.043 0.000 series.py:2787(_sanitize_array)
6594 0.042 0.000 0.042 0.000 {pandas.lib.is_bool_array}
169/167 0.001 0.000 0.042 0.000 sre_parse.py:819(parse)
26377 0.027 0.000 0.041 0.000 internals.py:3276(_consolidate_inplace)
6595 0.041 0.000 0.041 0.000 {pandas.lib.is_datetime_array}
19782 0.027 0.000 0.041 0.000 indexing.py:128(<genexpr>)
20923 0.026 0.000 0.040 0.000 internals.py:3911(internal_values)
194369/194289 0.038 0.000 0.040 0.000 {built-in method builtins.issubclass}
557/169 0.003 0.000 0.040 0.000 sre_parse.py:429(_parse_sub)
751/186 0.014 0.000 0.039 0.000 sre_parse.py:491(_parse)
26376 0.031 0.000 0.039 0.000 common.py:1491(_get_dtype)
6594 0.013 0.000 0.038 0.000 generic.py:1402(_is_view)
6 0.000 0.000 0.038 0.006 api.py:3(<module>)
1 0.000 0.000 0.038 0.038 __init__.py:27(<module>)
2 0.000 0.000 0.037 0.019 __init__.py:7(<module>)
1105 0.002 0.000 0.037 0.000 series.py:2804(_try_cast)
6595 0.019 0.000 0.037 0.000 common.py:272(array_equivalent)
6594 0.010 0.000 0.036 0.000 generic.py:2755(<lambda>)
13207 0.025 0.000 0.036 0.000 __init__.py:168(iteritems)
2 0.000 0.000 0.036 0.018 __init__.py:26(<module>)
6596 0.024 0.000 0.036 0.000 base.py:1143(_engine)
6594 0.014 0.000 0.035 0.000 generic.py:337(_get_block_manager_axis)
7284 0.022 0.000 0.033 0.000 series.py:306(name)
34350 0.032 0.000 0.032 0.000 {pandas.lib.isscalar}
6877 0.006 0.000 0.032 0.000 {built-in method builtins.all}
14294 0.022 0.000 0.031 0.000 common.py:1763(is_list_like)
13188 0.021 0.000 0.030 0.000 indexing.py:547(<genexpr>)
53445 0.030 0.000 0.030 0.000 internals.py:160(mgr_locs)
6596 0.010 0.000 0.030 0.000 base.py:1146(<lambda>)
2 0.000 0.000 0.029 0.014 __init__.py:2912(_call_aside)
1 0.000 0.000 0.029 0.029 __init__.py:2927(_initialize_master_working_set)
7283 0.026 0.000 0.028 0.000 base.py:1247(__getitem__)
1 0.000 0.000 0.028 0.028 generic.py:2(<module>)
6608 0.028 0.000 0.028 0.000 {built-in method builtins.sorted}
13188 0.019 0.000 0.028 0.000 generic.py:1441(_clear_item_cache)
1 0.000 0.000 0.027 0.027 requirements.py:4(<module>)
13190 0.009 0.000 0.026 0.000 base.py:445(get_values)
167 0.000 0.000 0.026 0.000 sre_compile.py:516(_code)
6594 0.012 0.000 0.026 0.000 internals.py:3009(is_mixed_type)
21161 0.026 0.000 0.026 0.000 internals.py:2695(_get_items)
6595 0.011 0.000 0.025 0.000 {built-in method builtins.sum}
6594 0.015 0.000 0.025 0.000 internals.py:3027(is_view)
13188 0.015 0.000 0.025 0.000 internals.py:3260(consolidate)
2 0.000 0.000 0.024 0.012 __init__.py:45(<module>)
39565 0.024 0.000 0.024 0.000 internals.py:2996(is_consolidated)
523 0.001 0.000 0.024 0.000 decorators.py:181(__call__)
7283 0.024 0.000 0.024 0.000 internals.py:301(iget)
545 0.004 0.000 0.023 0.000 textwrap.py:415(dedent)
6594 0.021 0.000 0.023 0.000 common.py:446(_infer_dtype_from_scalar)
2 0.000 0.000 0.023 0.011 index.py:2(<module>)
6594 0.020 0.000 0.022 0.000 generic.py:124(_init_mgr)
63023 0.022 0.000 0.022 0.000 csv.py:92(fieldnames)
6594 0.017 0.000 0.021 0.000 generic.py:307(_get_axis_number)
23 0.000 0.000 0.021 0.001 pyparsing.py:798(_trim_arity)
22 0.000 0.000 0.021 0.001 pyparsing.py:806(extract_stack)
22 0.000 0.000 0.021 0.001 traceback.py:192(extract_stack)
22 0.004 0.000 0.021 0.001 traceback.py:303(extract)
30 0.001 0.000 0.021 0.001 __init__.py:349(namedtuple)
1 0.000 0.000 0.021 0.021 internals.py:1(<module>)
6594 0.012 0.000 0.020 0.000 common.py:1550(is_integer_dtype)
1 0.000 0.000 0.020 0.020 parser.py:5(<module>)
13188 0.016 0.000 0.020 0.000 base.py:508(<listcomp>)
1320/167 0.006 0.000 0.019 0.000 sre_compile.py:64(_compile)
19 0.000 0.000 0.019 0.001 pyparsing.py:958(setParseAction)
1 0.000 0.000 0.019 0.019 feedparser.py:20(<module>)
1 0.000 0.000 0.019 0.019 internals.py:3996(create_block_manager_from_arrays)
19784 0.015 0.000 0.019 0.000 internals.py:2623(ndim)
1 0.000 0.000 0.019 0.019 dates.py:111(<module>)
1 0.003 0.003 0.019 0.019 internals.py:4007(form_blocks)
13188 0.019 0.000 0.019 0.000 internals.py:650(<lambda>)
13198 0.019 0.000 0.019 0.000 dtypes.py:122(construct_from_string)
7284 0.018 0.000 0.018 0.000 series.py:292(_set_subtyp)
13188 0.012 0.000 0.018 0.000 missing.py:559(clean_reindex_fill_method)
1105 0.005 0.000 0.018 0.000 common.py:1011(_possibly_cast_to_datetime)
88922 0.018 0.000 0.018 0.000 {method 'append' of 'list' objects}
3288 0.018 0.000 0.018 0.000 {built-in method posix.stat}
6594 0.005 0.000 0.017 0.000 base.py:3425(_ensure_has_len)
6594 0.012 0.000 0.017 0.000 internals.py:4301(_extend_blocks)
17/16 0.000 0.000 0.017 0.001 <frozen importlib._bootstrap>:630(_load_backward_compatible)
5 0.000 0.000 0.017 0.003 __init__.py:34(load_module)
1 0.000 0.000 0.017 0.017 message.py:5(<module>)
2634 0.002 0.000 0.016 0.000 <frozen importlib._bootstrap_external>:68(_path_stat)
14 0.000 0.000 0.016 0.001 __init__.py:663(add_entry)
Related
I am running cProfile on my python script in order to see where i can improve performance and using snakeviz to visualize. The results are pretty vague however; how do I interpret them? Here are the first few lines of it:
Ordered by: internal time
ncalls tottime percall cumtime percall filename:lineno(function)
242059 0.626 0.000 0.914 0.000 pulp.py:585(__init__)
1 0.413 0.413 0.413 0.413 {built-in method _winapi.WaitForSingleObject}
343978/302 0.293 0.000 0.557 0.002 pulp.py:750(addInPlace)
4159617 0.288 0.000 0.288 0.000 pulp.py:165(__hash__)
112 0.282 0.003 0.282 0.003 {method 'read' of '_ssl._SSLSocket' objects}
1913398 0.172 0.000 0.245 0.000 {built-in method builtins.isinstance}
1 0.171 0.171 0.185 0.185 Betfair_Run_Sheet.pyx:243(betfairFinalArray)
377866 0.168 0.000 0.293 0.000 pulp.py:637(addterm)
2255 0.161 0.000 0.161 0.000 mps_lp.py:249(<listcomp>)
1 0.148 0.148 0.570 0.570 mps_lp.py:174(writeMPS)
117214 0.139 0.000 0.444 0.000 pulp.py:820(__mul__)
2 0.136 0.068 0.196 0.098 pulp.py:1465(variablesDict)
5 0.135 0.027 0.135 0.027 {method 'do_handshake' of '_ssl._SSLSocket' objects}
427 0.111 0.000 0.129 0.000 <frozen importlib._bootstrap_external>:914(get_data)
71 0.108 0.002 0.108 0.002 {built-in method _imp.create_dynamic}
2093 0.102 0.000 0.102 0.000 {built-in method nt.stat}
I am using Pulp so aware that takes the lion's share of the time, but the specifics of the setup is not clear from the above, e.g. for the first line of output it seems to be alluding to a line 585 in my script but that is not where I have called or set up the PULP part in it at all.
Same with the <listcomp> 9th one down, there is no list comprehension on that line of my script.
Other things like {method 'do_handshake' of '_ssl._SSLSocket' objects} I don't have a clue what they mean.
In running cProfile on anything (for example a mergeSort) I'm getting all 000's in the runtimes, and key lines/vars/methods not listed/tested in the process. Only seems to test under methods, internals. Please advise.
below are my results for a mergeSort, I've tried running the
python -m cProfile [mergeSort(lst)] w/ and w/out brackets--saw both in documentation.
Only version I can get to work is the:
import cProfile
cProfile.run(mergeSort(lst))
or the enable() disable() method shown.
formatting doesn't turn out well, so attached image.
cProfile Results
results:
'''
[17, 20, 26, 31, 44, 54, 55, 77, 93]
127 function calls (111 primitive calls) in 0.000 seconds
Ordered by: standard name
ncalls tottime percall cumtime percall filename:lineno(function)
17/1 0.000 0.000 0.000 0.000 :1(mergeSort)
1 0.000 0.000 0.000 0.000 :36()
1 0.000 0.000 0.000 0.000 :37()
2 0.000 0.000 0.000 0.000 codeop.py:132(call)
2 0.000 0.000 0.000 0.000 hooks.py:142(call)
2 0.000 0.000 0.000 0.000 hooks.py:207(pre_run_code_hook)
2 0.000 0.000 0.000 0.000 interactiveshell.py:1104(user_global_ns)
2 0.000 0.000 0.000 0.000 interactiveshell.py:2933(run_code)
2 0.000 0.000 0.000 0.000 ipstruct.py:125(getattr)
2 0.000 0.000 0.000 0.000 {built-in method builtins.compile}
2 0.000 0.000 0.000 0.000 {built-in method builtins.exec}
91 0.000 0.000 0.000 0.000 {built-in method builtins.len}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
'''
Hello I have an (numpy) optimizing problem.
Below i have writen an piece of code that's quite common for my type of calculations.
The caclulation take always some time that i think should be shorter.
I think the problem is the loop. I have looked at the linalg part of numpy but i can't find an solution there. I also searched for a method vectorize the data but since i haven't much experience with that... i can't find any solution...
I hope somebody can help me...
import numpy as np
from scipy import signal
from scipy.fftpack import fft
fs = 44100 # frequency sample
T = 5 # time max
t = np.arange(0, T*fs)/fs # time array
x = np.sin(2 * np.pi * 100 * t) + 0.7 * np.sin(2 * np.pi * 880 * t) + 0.2 * np.sin(2 * np.pi * 2400 * t)
# Define Window length and window:
wl = 4 # window lenght
overlap = 0.5
W = signal.get_window('hanning', wl) # window
Wx = np.zeros(len(x))
ul = wl
# loop added for window
if (len(x) / wl) % wl == 0:
while ul <= len(Wx):
Wx[ul-wl:ul] += x[ul-wl:ul] * W
ul += wl * overlap
else:
dsample = (len(x)/wl) % wl # delta in samples between mod (x/windw length)
x = np.append(x, np.zeros(wl - dsample))
while ul <= len(Wx):
Wx[ul-wl:ul] += x[ul-wl:ul] * W
ul += wl * overlap
NFFT = np.int(2 ** np.ceil(np.log2(len(x))))
NFFW = np.int(2 ** np.ceil(np.log2(len(Wx))))
# Frequency spectrums
X = fft(x, NFFT)
WX = fft(Wx, NFFW)
Profiler:
%run -p example.py
110367 function calls (110366 primitive calls) in 19.998 seconds
Ordered by: internal time
ncalls tottime percall cumtime percall filename:lineno(function)
1 19.561 19.561 19.994 19.994 example.py:6(<module>)
110258 0.233 0.000 0.233 0.000 {built-in method len}
2 0.181 0.091 0.189 0.095 basic.py:169(fft)
2 0.008 0.004 0.008 0.004 basic.py:131(_fix_shape)
2 0.008 0.004 0.008 0.004 {built-in method concatenate}
1 0.003 0.003 0.003 0.003 {built-in method compile}
2 0.002 0.001 0.002 0.001 {built-in method arange}
2 0.001 0.000 0.001 0.000 {built-in method open}
4 0.000 0.000 0.000 0.000 {built-in method zeros}
1 0.000 0.000 19.998 19.998 interactiveshell.py:2496(safe_execfile)
2/1 0.000 0.000 19.998 19.998 {built-in method exec}
1 0.000 0.000 0.000 0.000 windows.py:615(hann)
1 0.000 0.000 19.997 19.997 py3compat.py:108(execfile)
1 0.000 0.000 0.000 0.000 {method 'read' of '_io.BufferedReader' objects}
2 0.000 0.000 0.008 0.004 function_base.py:3503(append)
1 0.000 0.000 0.000 0.000 posixpath.py:318(normpath)
1 0.000 0.000 0.000 0.000 windows.py:1380(get_window)
1 0.000 0.000 0.000 0.000 posixpath.py:145(dirname)
4 0.000 0.000 0.000 0.000 {built-in method array}
2 0.000 0.000 0.000 0.000 {built-in method round}
1 0.000 0.000 0.000 0.000 {built-in method getcwd}
2 0.000 0.000 0.000 0.000 <frozen importlib._bootstrap>:2264(_handle_fromlist)
2 0.000 0.000 0.000 0.000 basic.py:116(_asfarray)
4 0.000 0.000 0.000 0.000 basic.py:24(istype)
2 0.000 0.000 0.000 0.000 fromnumeric.py:1281(ravel)
8 0.000 0.000 0.000 0.000 {built-in method isinstance}
1 0.000 0.000 0.000 0.000 posixpath.py:70(join)
2 0.000 0.000 0.000 0.000 numeric.py:462(asanyarray)
1 0.000 0.000 0.000 0.000 posixpath.py:355(abspath)
8 0.000 0.000 0.000 0.000 {built-in method hasattr}
1 0.000 0.000 19.998 19.998 <string>:1(<module>)
1 0.000 0.000 0.000 0.000 syspathcontext.py:64(__exit__)
1 0.000 0.000 0.000 0.000 posixpath.py:221(expanduser)
1 0.000 0.000 0.000 0.000 _bootlocale.py:23(getpreferredencoding)
1 0.000 0.000 0.000 0.000 syspathcontext.py:57(__enter__)
1 0.000 0.000 0.000 0.000 syspathcontext.py:54(__init__)
4 0.000 0.000 0.000 0.000 {built-in method issubclass}
3 0.000 0.000 0.000 0.000 posixpath.py:38(_get_sep)
2 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}
2 0.000 0.000 0.000 0.000 numeric.py:392(asarray)
7 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects}
1 0.000 0.000 0.000 0.000 {built-in method nl_langinfo}
5 0.000 0.000 0.000 0.000 {method 'startswith' of 'str' objects}
1 0.000 0.000 0.000 0.000 codecs.py:306(__init__)
1 0.000 0.000 0.000 0.000 posixpath.py:60(isabs)
1 0.000 0.000 0.000 0.000 {method 'split' of 'str' objects}
1 0.000 0.000 0.000 0.000 codecs.py:257(__init__)
2 0.000 0.000 0.000 0.000 {method 'setdefault' of 'dict' objects}
1 0.000 0.000 0.000 0.000 {method 'rfind' of 'str' objects}
1 0.000 0.000 0.000 0.000 {method 'remove' of 'list' objects}
1 0.000 0.000 0.000 0.000 {method 'join' of 'str' objects}
1 0.000 0.000 0.000 0.000 {method 'rstrip' of 'str' objects}
1 0.000 0.000 0.000 0.000 {method 'endswith' of 'str' objects}
1 0.000 0.000 0.000 0.000 {method 'insert' of 'list' objects}
1 0.000 0.000 0.000 0.000 {built-in method getdefaultencoding}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
1 0.000 0.000 0.000 0.000 py3compat.py:13(no_code)
Precalculating static values shortens my loop from ~4s to 0.7s execution time:
nEntries = len(Wx)
step = int(wl * overlap)
while ul <= nEntries:
Wx[ul-wl:ul] += x[ul-wl:ul] * W
ul += step
I have a grammar that is an extension of Python grammar. And small programs parse about 2 seconds on a Macbook Pro. I have taken the SLL trick and applied it:
# Set up the lexer
inputStream = InputStream(s)
lexer = CustomLexer(inputStream)
stream = CommonTokenStream(lexer)
# Set up the error handling stuff
error_handler = CustomErrorStrategy()
error_listener = CustomErrorListener()
buffered_errors = BufferedErrorListener()
error_listener.addDelegatee(buffered_errors)
# Set up the fast parser
parser = PythonQLParser(stream)
parser._interp.predictionMode = PredictionMode.SLL
parser.removeErrorListeners()
parser.errHandler = BailErrorStrategy()
try:
tree = parser.file_input()
return (tree,parser)
But it didn't do the trick, the time didn't change significantly. Any hints on what to do?
I'm using Python3 with antlr4-python3-runtime-4.5.3
The grammar file is here: Grammar File
And the project github page is here: Github
I have also ran a profiler, here are significant entries from the parser:
ncalls tottime percall cumtime percall filename:lineno(function)
21 0.000 0.000 0.094 0.004 PythonQLParser.py:7483(argument)
8 0.000 0.000 0.195 0.024 PythonQLParser.py:7379(arglist)
9 0.000 0.000 0.196 0.022 PythonQLParser.py:6836(trailer)
5/3 0.000 0.000 0.132 0.044 PythonQLParser.py:6765(testlist_comp)
1 0.000 0.000 0.012 0.012 PythonQLParser.py:6154(window_end_cond)
1 0.000 0.000 0.057 0.057 PythonQLParser.py:6058(sliding_window)
1 0.000 0.000 0.057 0.057 PythonQLParser.py:5941(window_clause)
1 0.000 0.000 0.004 0.004 PythonQLParser.py:5807(for_clause_entry)
1 0.000 0.000 0.020 0.020 PythonQLParser.py:5752(for_clause)
2/1 0.000 0.000 0.068 0.068 PythonQLParser.py:5553(query_expression)
48/10 0.000 0.000 0.133 0.013 PythonQLParser.py:5370(atom)
48/7 0.000 0.000 0.315 0.045 PythonQLParser.py:5283(power)
48/7 0.000 0.000 0.315 0.045 PythonQLParser.py:5212(factor)
48/7 0.000 0.000 0.331 0.047 PythonQLParser.py:5132(term)
47/7 0.000 0.000 0.346 0.049 PythonQLParser.py:5071(arith_expr)
47/7 0.000 0.000 0.361 0.052 PythonQLParser.py:5010(shift_expr)
47/7 0.000 0.000 0.376 0.054 PythonQLParser.py:4962(and_expr)
47/7 0.000 0.000 0.390 0.056 PythonQLParser.py:4914(xor_expr)
47/7 0.000 0.000 0.405 0.058 PythonQLParser.py:4866(expr)
44/7 0.000 0.000 0.405 0.058 PythonQLParser.py:4823(star_expr)
43/7 0.000 0.000 0.422 0.060 PythonQLParser.py:4615(not_test)
43/7 0.000 0.000 0.438 0.063 PythonQLParser.py:4563(and_test)
43/7 0.000 0.000 0.453 0.065 PythonQLParser.py:4509(or_test)
43/7 0.000 0.000 0.467 0.067 PythonQLParser.py:4293(old_test)
43/7 0.000 0.000 0.467 0.067 PythonQLParser.py:4179(try_catch_expr)
43/7 0.000 0.000 0.482 0.069 PythonQLParser.py:3978(test)
1 0.000 0.000 0.048 0.048 PythonQLParser.py:2793(import_from)
1 0.000 0.000 0.048 0.048 PythonQLParser.py:2702(import_stmt)
7 0.000 0.000 1.728 0.247 PythonQLParser.py:2251(testlist_star_expr)
4 0.000 0.000 1.770 0.443 PythonQLParser.py:2161(expr_stmt)
5 0.000 0.000 1.822 0.364 PythonQLParser.py:2063(small_stmt)
5 0.000 0.000 1.855 0.371 PythonQLParser.py:1980(simple_stmt)
5 0.000 0.000 1.859 0.372 PythonQLParser.py:1930(stmt)
1 0.000 0.000 1.898 1.898 PythonQLParser.py:1085(file_input)
176 0.002 0.000 0.993 0.006 Lexer.py:127(nextToken)
420 0.000 0.000 0.535 0.001 ParserATNSimulator.py:1120(closure)
705 0.003 0.000 1.642 0.002 ParserATNSimulator.py:315(adaptivePredict)
The PythonQL program that I was parsing is this one:
# This example illustrates the window query in PythonQL
from collections import namedtuple
trade = namedtuple('Trade', ['day','ammount', 'stock_id'])
trades = [ trade(1, 15.34, 'APPL'),
trade(2, 13.45, 'APPL'),
trade(3, 8.34, 'APPL'),
trade(4, 9.87, 'APPL'),
trade(5, 10.99, 'APPL'),
trade(6, 76.16, 'APPL') ]
# Maximum 3-day sum
res = (select win
for sliding window win in ( select t.ammount for t in trades )
start at s when True
only end at e when (e-s == 2))
print (res)
E: After testing the same on OS X and Linux, I can confirm that the following only happens on OS X. On Linux it literally runs at a thousand fps, as I happened to wonder. Any explanation? I would much prefer developing on Mac, thanks to TextMate.
Here's a simple loop that does almost nothing, and still runs very slowly. Can anyone explain why? FPS averages at little over 30, it takes a little over 30ms for each pass over the loop. Window size does not seem to affect this at all, as even setting a tiny window size like (50,50) has the same fps.
I find this weird, I would expect that any contemporary hardware could do a thousand fps for such a simple loop, even when we update every pixel every time. From the profile I can see that {built-in method get} and {built-in method update} combined seem to take around 30ms of time per call, is that really the best we can get out without using dirty rects?
pygame.init()
clock = pygame.time.Clock()
fps = 1000
#milliseconds from last frame
new_time, old_time = None, None
done = False
while not done:
clock.tick(fps)
for event in pygame.event.get():
if event.type == pygame.QUIT:
done = True
# show fps and milliseconds
if new_time:
old_time = new_time
new_time = pygame.time.get_ticks()
if new_time and old_time:
pygame.display.set_caption("fps: " + str(int(clock.get_fps())) + " ms: " + str(new_time-old_time))
pygame.display.update()
Here's the beginning of a cProfile of the main function.
94503 function calls (92211 primitive calls) in 21.011 seconds
Ordered by: cumulative time
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.026 0.026 21.011 21.011 new_main.py:34(main)
652 14.048 0.022 14.048 0.022 {built-in method get}
652 5.864 0.009 5.864 0.009 {built-in method update}
1 0.444 0.444 0.634 0.634 {built-in method init}
651 0.278 0.000 0.278 0.000 {built-in method set_caption}
72/1 0.000 0.000 0.151 0.151 <frozen importlib._bootstrap>:2234(_find_and_load)
72/1 0.000 0.000 0.151 0.151 <frozen importlib._bootstrap>:2207(_find_and_load_unlocked)
71/1 0.000 0.000 0.151 0.151 <frozen importlib._bootstrap>:1186(_load_unlocked)
46/1 0.000 0.000 0.151 0.151 <frozen importlib._bootstrap>:1122(_exec)
46/1 0.000 0.000 0.151 0.151 <frozen importlib._bootstrap>:1465(exec_module)
74/1 0.000 0.000 0.151 0.151 <frozen importlib._bootstrap>:313(_call_with_frames_removed)
54/1 0.004 0.000 0.151 0.151 {built-in method exec}
1 0.000 0.000 0.151 0.151 macosx.py:1(<module>)
1 0.000 0.000 0.150 0.150 pkgdata.py:18(<module>)
25/3 0.000 0.000 0.122 0.041 <frozen importlib._bootstrap>:1156(_load_backward_compatible)
8/1 0.026 0.003 0.121 0.121 {method 'load_module' of 'zipimport.zipimporter' objects}
1 0.000 0.000 0.101 0.101 __init__.py:15(<module>)
1 0.000 0.000 0.079 0.079 config_reader.py:115(build_from_config)
2 0.000 0.000 0.056 0.028 common.py:43(reset_screen)
2 0.055 0.027 0.055 0.027 {built-in method set_mode}
72/71 0.001 0.000 0.045 0.001 <frozen importlib._bootstrap>:2147(_find_spec)
70/69 0.000 0.000 0.043 0.001 <frozen importlib._bootstrap>:1934(find_spec)
70/69 0.001 0.000 0.043 0.001 <frozen importlib._bootstrap>:1902(_get_spec)
92 0.041 0.000 0.041 0.000 {built-in method load_extended}
6 0.000 0.000 0.041 0.007 new_map.py:74(add_character)
6 0.000 0.000 0.041 0.007 new_character.py:32(added_to_map)
6 0.001 0.000 0.041 0.007 new_character.py:265(__init__)
1 0.000 0.000 0.038 0.038 macosx.py:14(Video_AutoInit)
1 0.038 0.038 0.038 0.038 {built-in method InstallNSApplication}
1 0.036 0.036 0.036 0.036 {built-in method quit}
65 0.001 0.000 0.036 0.001 re.py:277(_compile)
49 0.000 0.000 0.036 0.001 re.py:221(compile)
The answer to this ended up being that the retina display under OS X is the differentiating factor. Running it even on an external display on the same Mac works fine. But moving the window to the retina display makes it sluggish. With or without an external monitor connected.
On the other hand, it runs just fine on the same retina display under Linux. It is unclear what the difference in the display managers / rendering is that causes this, but I doubt there is anything one could do about it.
Changing the game resolution to fullscreen helped me.
Try this:
window = pygame.display.set_mode((0, 0), pygame.FULLSCREEN)
instead of:
window = pygame.display.set_mode((winx, winy))