Repository navigation
Expand file tree
/
Copy pathmisc.py
More file actions
1961 lines (1594 loc) · 67.9 KB
/
Copy pathmisc.py
File metadata and controls
1961 lines (1594 loc) · 67.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Part of Odoo. See LICENSE file for full copyright and licensing details.
"""
Miscellaneous tools used by Odoo.
"""
from __future__ import annotations
import base64
import collections
import csv
import datetime
import enum
import hashlib
import hmac as hmac_lib
import itertools
import json
import logging
import os
import re
import sys
import tempfile
import threading
import time
import traceback
import typing
import unicodedata
import warnings
import zlib
from collections import defaultdict
from collections.abc import Iterable, Iterator, Mapping, MutableMapping, MutableSet, Reversible
from contextlib import ContextDecorator, contextmanager
from difflib import HtmlDiff
from functools import reduce, wraps
from itertools import islice, groupby as itergroupby
from operator import itemgetter
import babel
import babel.dates
import markupsafe
import pytz
from lxml import etree, objectify
# get_encodings, ustr and exception_to_unicode were originally from tools.misc.
# There are moved to loglevels until we refactor tools.
from odoo.loglevels import exception_to_unicode, get_encodings, ustr # noqa: F401
from .config import config
from .float_utils import float_round
from .which import which
K = typing.TypeVar('K')
T = typing.TypeVar('T')
if typing.TYPE_CHECKING:
from collections.abc import Callable, Collection, Sequence
from odoo.api import Environment
from odoo.addons.base.models.res_lang import LangData
P = typing.TypeVar('P')
__all__ = [
'DEFAULT_SERVER_DATETIME_FORMAT',
'DEFAULT_SERVER_DATE_FORMAT',
'DEFAULT_SERVER_TIME_FORMAT',
'NON_BREAKING_SPACE',
'SKIPPED_ELEMENT_TYPES',
'DotDict',
'LastOrderedSet',
'OrderedSet',
'Reverse',
'babel_locale_parse',
'clean_context',
'consteq',
'discardattr',
'exception_to_unicode',
'file_open',
'file_open_temporary_directory',
'file_path',
'find_in_path',
'formatLang',
'format_amount',
'format_date',
'format_datetime',
'format_duration',
'format_time',
'frozendict',
'get_encodings',
'get_iso_codes',
'get_lang',
'groupby',
'hmac',
'hash_sign',
'verify_hash_signed',
'html_escape',
'human_size',
'is_list_of',
'merge_sequences',
'mod10r',
'mute_logger',
'parse_date',
'partition',
'posix_to_ldml',
'remove_accents',
'replace_exceptions',
'reverse_enumerate',
'split_every',
'str2bool',
'street_split',
'topological_sort',
'unique',
'ustr',
'real_time',
]
_logger = logging.getLogger(__name__)
# List of etree._Element subclasses that we choose to ignore when parsing XML.
# We include the *Base ones just in case, currently they seem to be subclasses of the _* ones.
SKIPPED_ELEMENT_TYPES = (etree._Comment, etree._ProcessingInstruction, etree.CommentBase, etree.PIBase, etree._Entity)
# Configure default global parser
etree.set_default_parser(etree.XMLParser(resolve_entities=False))
default_parser = etree.XMLParser(resolve_entities=False, remove_blank_text=True)
default_parser.set_element_class_lookup(objectify.ObjectifyElementClassLookup())
objectify.set_default_parser(default_parser)
NON_BREAKING_SPACE = u'\N{NO-BREAK SPACE}'
# ensure we have a non patched time for query times when using freezegun
real_time = time.time.__call__ # type: ignore
class Sentinel(enum.Enum):
"""Class for typing parameters with a sentinel as a default"""
SENTINEL = -1
SENTINEL = Sentinel.SENTINEL
#----------------------------------------------------------
# Subprocesses
#----------------------------------------------------------
def find_in_path(name):
path = os.environ.get('PATH', os.defpath).split(os.pathsep)
if config.get('bin_path') and config['bin_path'] != 'None':
path.append(config['bin_path'])
return which(name, path=os.pathsep.join(path))
# ----------------------------------------------------------
# Postgres subprocesses
# ----------------------------------------------------------
def find_pg_tool(name):
path = None
if config['pg_path'] and config['pg_path'] != 'None':
path = config['pg_path']
try:
return which(name, path=path)
except OSError:
raise Exception('Command `%s` not found.' % name)
def exec_pg_environ():
"""
Force the database PostgreSQL environment variables to the database
configuration of Odoo.
Note: On systems where pg_restore/pg_dump require an explicit password
(i.e. on Windows where TCP sockets are used), it is necessary to pass the
postgres user password in the PGPASSWORD environment variable or in a
special .pgpass file.
See also https://www.postgresql.org/docs/current/libpq-envars.html
"""
env = os.environ.copy()
if config['db_host']:
env['PGHOST'] = config['db_host']
if config['db_port']:
env['PGPORT'] = str(config['db_port'])
if config['db_user']:
env['PGUSER'] = config['db_user']
if config['db_password']:
env['PGPASSWORD'] = config['db_password']
if config['db_app_name']:
env['PGAPPNAME'] = config['db_app_name'].replace('{pid}', f'env{os.getpid()}')[:63]
if config['db_sslmode']:
env['PGSSLMODE'] = config['db_sslmode']
return env
# ----------------------------------------------------------
# File paths
# ----------------------------------------------------------
def file_path(file_path: str, filter_ext: tuple[str, ...] = ('',), env: Environment | None = None, *, check_exists: bool = True) -> str:
"""Verify that a file exists under a known `addons_path` directory and return its full path.
Examples::
>>> file_path('hr')
>>> file_path('hr/static/description/icon.png')
>>> file_path('hr/static/description/icon.png', filter_ext=('.png', '.jpg'))
:param str file_path: absolute file path, or relative path within any `addons_path` directory
:param list[str] filter_ext: optional list of supported extensions (lowercase, with leading dot)
:param env: optional environment, required for a file path within a temporary directory
created using `file_open_temporary_directory()`
:param check_exists: check that the file exists (default: True)
:return: the absolute path to the file
:raise FileNotFoundError: if the file is not found under the known `addons_path` directories
:raise ValueError: if the file doesn't have one of the supported extensions (`filter_ext`)
"""
import odoo.addons # noqa: PLC0415
is_abs = os.path.isabs(file_path)
normalized_path = os.path.normpath(os.path.normcase(file_path))
if filter_ext and not normalized_path.lower().endswith(filter_ext):
raise ValueError("Unsupported file: " + file_path)
# ignore leading 'addons/' if present, it's the final component of root_path, but
# may sometimes be included in relative paths
normalized_path = normalized_path.removeprefix('addons' + os.sep)
# if path is relative and represents a loaded module, accept only the
# __path__ for that module; otherwise, search in all accepted paths
file_path_split = normalized_path.split(os.path.sep)
if not is_abs and (module := sys.modules.get(f'odoo.addons.{file_path_split[0]}')):
addons_paths = list(map(os.path.dirname, module.__path__))
else:
root_path = os.path.abspath(config.root_path)
temporary_paths = env.transaction._Transaction__file_open_tmp_paths if env else []
addons_paths = [*odoo.addons.__path__, root_path, *temporary_paths]
for addons_dir in addons_paths:
# final path sep required to avoid partial match
parent_path = os.path.normpath(os.path.normcase(addons_dir)) + os.sep
if is_abs:
fpath = normalized_path
else:
fpath = os.path.normpath(os.path.join(parent_path, normalized_path))
if fpath.startswith(parent_path) and (
# we check existence when asked or we have multiple paths to check
# (there is one possibility for absolute paths)
(not check_exists and (is_abs or len(addons_paths) == 1))
or os.path.exists(fpath)
):
return fpath
raise FileNotFoundError("File not found: " + file_path)
def file_open(name: str, mode: str = "r", filter_ext: tuple[str, ...] = (), env: Environment | None = None):
"""Open a file from within the addons_path directories, as an absolute or relative path.
Examples::
>>> file_open('hr/static/description/icon.png')
>>> file_open('hr/static/description/icon.png', filter_ext=('.png', '.jpg'))
>>> with file_open('/opt/odoo/addons/hr/static/description/icon.png', 'rb') as f:
... contents = f.read()
:param name: absolute or relative path to a file located inside an addon
:param mode: file open mode, as for `open()`
:param list[str] filter_ext: optional list of supported extensions (lowercase, with leading dot)
:param env: optional environment, required to open a file within a temporary directory
created using `file_open_temporary_directory()`
:return: file object, as returned by `open()`
:raise FileNotFoundError: if the file is not found under the known `addons_path` directories
:raise ValueError: if the file doesn't have one of the supported extensions (`filter_ext`)
"""
path = file_path(name, filter_ext=filter_ext, env=env, check_exists=False)
encoding = None
if 'b' not in mode:
# Force encoding for text mode, as system locale could affect default encoding,
# even with the latest Python 3 versions.
# Note: This is not covered by a unit test, due to the platform dependency.
# For testing purposes you should be able to force a non-UTF8 encoding with:
# `sudo locale-gen fr_FR; LC_ALL=fr_FR.iso8859-1 python3 ...'
# See also PEP-540, although we can't rely on that at the moment.
encoding = "utf-8"
if any(m in mode for m in ('w', 'x', 'a')) and not os.path.isfile(path):
# Don't let create new files
raise FileNotFoundError(f"Not a file: {path}")
return open(path, mode, encoding=encoding)
@contextmanager
def file_open_temporary_directory(env: Environment):
"""Create and return a temporary directory added to the directories `file_open` is allowed to read from.
`file_open` will be allowed to open files within the temporary directory
only for environments of the same transaction than `env`.
Meaning, other transactions/requests from other users or even other databases
won't be allowed to open files from this directory.
Examples::
>>> with odoo.tools.file_open_temporary_directory(self.env) as module_dir:
... with zipfile.ZipFile('foo.zip', 'r') as z:
... z.extract('foo/__manifest__.py', module_dir)
... with odoo.tools.file_open('foo/__manifest__.py', env=self.env) as f:
... manifest = f.read()
:param env: environment for which the temporary directory is created.
:return: the absolute path to the created temporary directory
"""
with tempfile.TemporaryDirectory() as module_dir:
try:
env.transaction._Transaction__file_open_tmp_paths.append(module_dir)
yield module_dir
finally:
env.transaction._Transaction__file_open_tmp_paths.remove(module_dir)
#----------------------------------------------------------
# iterables
#----------------------------------------------------------
def flatten(list):
"""Flatten a list of elements into a unique list
Author: Christophe Simonis (christophe@tinyerp.com)
Examples::
>>> flatten(['a'])
['a']
>>> flatten('b')
['b']
>>> flatten( [] )
[]
>>> flatten( [[], [[]]] )
[]
>>> flatten( [[['a','b'], 'c'], 'd', ['e', [], 'f']] )
['a', 'b', 'c', 'd', 'e', 'f']
>>> t = (1,2,(3,), [4, 5, [6, [7], (8, 9), ([10, 11, (12, 13)]), [14, [], (15,)], []]])
>>> flatten(t)
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
"""
warnings.warn(
"deprecated since 18.0",
category=DeprecationWarning,
stacklevel=2,
)
r = []
for e in list:
if isinstance(e, (bytes, str)) or not isinstance(e, collections.abc.Iterable):
r.append(e)
else:
r.extend(flatten(e))
return r
def reverse_enumerate(lst: Sequence[T]) -> Iterator[tuple[int, T]]:
"""Like enumerate but in the other direction
Usage::
>>> a = ['a', 'b', 'c']
>>> it = reverse_enumerate(a)
>>> it.next()
(2, 'c')
>>> it.next()
(1, 'b')
>>> it.next()
(0, 'a')
>>> it.next()
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
StopIteration
"""
return zip(range(len(lst) - 1, -1, -1), reversed(lst))
def partition(pred: Callable[[T], bool], elems: Iterable[T]) -> tuple[list[T], list[T]]:
""" Return a pair equivalent to:
``filter(pred, elems), filter(lambda x: not pred(x), elems)`` """
yes: list[T] = []
nos: list[T] = []
for elem in elems:
(yes if pred(elem) else nos).append(elem)
return yes, nos
def topological_sort(elems: Mapping[T, Collection[T]]) -> list[T]:
""" Return a list of elements sorted so that their dependencies are listed
before them in the result.
:param elems: specifies the elements to sort with their dependencies; it is
a dictionary like `{element: dependencies}` where `dependencies` is a
collection of elements that must appear before `element`. The elements
of `dependencies` are not required to appear in `elems`; they will
simply not appear in the result.
:returns: a list with the keys of `elems` sorted according to their
specification.
"""
# the algorithm is inspired by [Tarjan 1976],
# http://en.wikipedia.org/wiki/Topological_sorting#Algorithms
result = []
visited = set()
def visit(n):
if n not in visited:
visited.add(n)
if n in elems:
# first visit all dependencies of n, then append n to result
for it in elems[n]:
visit(it)
result.append(n)
for el in elems:
visit(el)
return result
def merge_sequences(*iterables: Iterable[T]) -> list[T]:
""" Merge several iterables into a list. The result is the union of the
iterables, ordered following the partial order given by the iterables,
with a bias towards the end for the last iterable::
seq = merge_sequences(['A', 'B', 'C'])
assert seq == ['A', 'B', 'C']
seq = merge_sequences(
['A', 'B', 'C'],
['Z'], # 'Z' can be anywhere
['Y', 'C'], # 'Y' must precede 'C';
['A', 'X', 'Y'], # 'X' must follow 'A' and precede 'Y'
)
assert seq == ['A', 'B', 'X', 'Y', 'C', 'Z']
"""
# dict is ordered
deps: defaultdict[T, list[T]] = defaultdict(list) # {item: elems_before_item}
for iterable in iterables:
prev: T | Sentinel = SENTINEL
for item in iterable:
if prev is SENTINEL:
deps[item] # just set the default
else:
deps[item].append(prev)
prev = item
return topological_sort(deps)
def get_iso_codes(lang: str) -> str:
if lang.find('_') != -1:
lang_items = lang.split('_')
if lang_items[0] == lang_items[1].lower():
lang = lang_items[0]
return lang
def scan_languages() -> list[tuple[str, str]]:
""" Returns all languages supported by OpenERP for translation
:returns: a list of (lang_code, lang_name) pairs
:rtype: [(str, unicode)]
"""
try:
# read (code, name) from languages in base/data/res.lang.csv
with file_open('base/data/res.lang.csv') as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='"')
fields = next(reader)
code_index = fields.index("code")
name_index = fields.index("name")
result = [
(row[code_index], row[name_index])
for row in reader
]
except Exception:
_logger.error("Could not read res.lang.csv")
result = []
return sorted(result or [('en_US', u'English')], key=itemgetter(1))
def mod10r(number: str) -> str:
"""
Input number : account or invoice number
Output return: the same number completed with the recursive mod10
key
"""
codec=[0,9,4,6,8,2,7,1,3,5]
report = 0
result=""
for digit in number:
result += digit
if digit.isdigit():
report = codec[ (int(digit) + report) % 10 ]
return result + str((10 - report) % 10)
def str2bool(s: str, default: bool | None = None) -> bool:
# allow this (for now?) because it's used for get_param
if type(s) is bool:
return s # type: ignore
if not isinstance(s, str):
warnings.warn(
f"Passed a non-str to `str2bool`: {s}",
DeprecationWarning,
stacklevel=2,
)
if default is None:
raise ValueError('Use 0/1/yes/no/true/false/on/off')
return bool(default)
s = s.lower()
if s in ('y', 'yes', '1', 'true', 't', 'on'):
return True
if s in ('n', 'no', '0', 'false', 'f', 'off'):
return False
if default is None:
raise ValueError('Use 0/1/yes/no/true/false/on/off')
return bool(default)
def human_size(sz: float | str) -> str | typing.Literal[False]:
"""
Return the size in a human readable format
"""
if not sz:
return False
units = ('bytes', 'Kb', 'Mb', 'Gb', 'Tb')
if isinstance(sz, str):
sz=len(sz)
s, i = float(sz), 0
while s >= 1024 and i < len(units)-1:
s /= 1024
i += 1
return "%0.2f %s" % (s, units[i])
DEFAULT_SERVER_DATE_FORMAT = "%Y-%m-%d"
DEFAULT_SERVER_TIME_FORMAT = "%H:%M:%S"
DEFAULT_SERVER_DATETIME_FORMAT = "%s %s" % (
DEFAULT_SERVER_DATE_FORMAT,
DEFAULT_SERVER_TIME_FORMAT)
DATE_LENGTH = len(datetime.date.today().strftime(DEFAULT_SERVER_DATE_FORMAT))
# Python's strftime supports only the format directives
# that are available on the platform's libc, so in order to
# be cross-platform we map to the directives required by
# the C standard (1989 version), always available on platforms
# with a C standard implementation.
DATETIME_FORMATS_MAP = {
'%C': '', # century
'%D': '%m/%d/%Y', # modified %y->%Y
'%e': '%d',
'%E': '', # special modifier
'%F': '%Y-%m-%d',
'%g': '%Y', # modified %y->%Y
'%G': '%Y',
'%h': '%b',
'%k': '%H',
'%l': '%I',
'%n': '\n',
'%O': '', # special modifier
'%P': '%p',
'%R': '%H:%M',
'%r': '%I:%M:%S %p',
'%s': '', #num of seconds since epoch
'%T': '%H:%M:%S',
'%t': ' ', # tab
'%u': ' %w',
'%V': '%W',
'%y': '%Y', # Even if %y works, it's ambiguous, so we should use %Y
'%+': '%Y-%m-%d %H:%M:%S',
# %Z is a special case that causes 2 problems at least:
# - the timezone names we use (in res_user.context_tz) come
# from pytz, but not all these names are recognized by
# strptime(), so we cannot convert in both directions
# when such a timezone is selected and %Z is in the format
# - %Z is replaced by an empty string in strftime() when
# there is not tzinfo in a datetime value (e.g when the user
# did not pick a context_tz). The resulting string does not
# parse back if the format requires %Z.
# As a consequence, we strip it completely from format strings.
# The user can always have a look at the context_tz in
# preferences to check the timezone.
'%z': '',
'%Z': '',
}
POSIX_TO_LDML = {
'a': 'E',
'A': 'EEEE',
'b': 'MMM',
'B': 'MMMM',
#'c': '',
'd': 'dd',
'-d': 'd',
'H': 'HH',
'I': 'hh',
'j': 'DDD',
'm': 'MM',
'-m': 'M',
'M': 'mm',
'p': 'a',
'S': 'ss',
'U': 'w',
'w': 'e',
'W': 'w',
'y': 'yy',
'Y': 'yyyy',
# see comments above, and babel's format_datetime assumes an UTC timezone
# for naive datetime objects
#'z': 'Z',
#'Z': 'z',
}
def posix_to_ldml(fmt: str, locale: babel.Locale) -> str:
""" Converts a posix/strftime pattern into an LDML date format pattern.
:param fmt: non-extended C89/C90 strftime pattern
:param locale: babel locale used for locale-specific conversions (e.g. %x and %X)
:return: unicode
"""
buf = []
pc = False
minus = False
quoted = []
for c in fmt:
# LDML date format patterns uses letters, so letters must be quoted
if not pc and c.isalpha():
quoted.append(c if c != "'" else "''")
continue
if quoted:
buf.append("'")
buf.append(''.join(quoted))
buf.append("'")
quoted = []
if pc:
if c == '%': # escaped percent
buf.append('%')
elif c == 'x': # date format, short seems to match
buf.append(locale.date_formats['short'].pattern)
elif c == 'X': # time format, seems to include seconds. short does not
buf.append(locale.time_formats['medium'].pattern)
elif c == '-':
minus = True
continue
else: # look up format char in static mapping
if minus:
c = '-' + c
minus = False
buf.append(POSIX_TO_LDML[c])
pc = False
elif c == '%':
pc = True
else:
buf.append(c)
# flush anything remaining in quoted buffer
if quoted:
buf.append("'")
buf.append(''.join(quoted))
buf.append("'")
return ''.join(buf)
@typing.overload
def split_every(n: int, iterable: Iterable[T]) -> Iterator[tuple[T, ...]]:
...
@typing.overload
def split_every(n: int, iterable: Iterable[T], piece_maker: type[Collection[T]]) -> Iterator[Collection[T]]:
...
@typing.overload
def split_every(n: int, iterable: Iterable[T], piece_maker: Callable[[Iterable[T]], P]) -> Iterator[P]:
...
def split_every(n: int, iterable: Iterable[T], piece_maker=tuple):
"""Splits an iterable into length-n pieces. The last piece will be shorter
if ``n`` does not evenly divide the iterable length.
:param int n: maximum size of each generated chunk
:param Iterable iterable: iterable to chunk into pieces
:param piece_maker: callable taking an iterable and collecting each
chunk from its slice, *must consume the entire slice*.
"""
iterator = iter(iterable)
piece = piece_maker(islice(iterator, n))
while piece:
yield piece
piece = piece_maker(islice(iterator, n))
def discardattr(obj: object, key: str) -> None:
""" Perform a ``delattr(obj, key)`` but without crashing if ``key`` is not present. """
try:
delattr(obj, key)
except AttributeError:
pass
# ---------------------------------------------
# String management
# ---------------------------------------------
# Inspired by http://stackoverflow.com/questions/517923
def remove_accents(input_str: str) -> str:
"""Suboptimal-but-better-than-nothing way to replace accented
latin letters by an ASCII equivalent. Will obviously change the
meaning of input_str and work only for some cases"""
if not input_str:
return input_str
nkfd_form = unicodedata.normalize('NFKD', input_str)
return ''.join(c for c in nkfd_form if not unicodedata.combining(c))
class unquote(str):
"""A subclass of str that implements repr() without enclosing quotation marks
or escaping, keeping the original string untouched. The name come from Lisp's unquote.
One of the uses for this is to preserve or insert bare variable names within dicts during eval()
of a dict's repr(). Use with care.
Some examples (notice that there are never quotes surrounding
the ``active_id`` name:
>>> unquote('active_id')
active_id
>>> d = {'test': unquote('active_id')}
>>> d
{'test': active_id}
>>> print d
{'test': active_id}
"""
__slots__ = ()
def __repr__(self):
return self
class mute_logger(logging.Handler):
"""Temporary suppress the logging.
Can be used as context manager or decorator::
@mute_logger('odoo.plic.ploc')
def do_stuff():
blahblah()
with mute_logger('odoo.foo.bar'):
do_suff()
"""
def __init__(self, *loggers):
super().__init__()
self.loggers = loggers
self.old_params = {}
def __enter__(self):
for logger_name in self.loggers:
logger = logging.getLogger(logger_name)
self.old_params[logger_name] = (logger.handlers, logger.propagate)
logger.propagate = False
logger.handlers = [self]
def __exit__(self, exc_type=None, exc_val=None, exc_tb=None):
for logger_name in self.loggers:
logger = logging.getLogger(logger_name)
logger.handlers, logger.propagate = self.old_params[logger_name]
def __call__(self, func):
@wraps(func)
def deco(*args, **kwargs):
with self:
return func(*args, **kwargs)
return deco
def emit(self, record):
pass
class lower_logging(logging.Handler):
"""Temporary lower the max logging level.
"""
def __init__(self, max_level, to_level=None):
super().__init__()
self.old_handlers = None
self.old_propagate = None
self.had_error_log = False
self.max_level = max_level
self.to_level = to_level or max_level
def __enter__(self):
logger = logging.getLogger()
self.old_handlers = logger.handlers[:]
self.old_propagate = logger.propagate
logger.propagate = False
logger.handlers = [self]
self.had_error_log = False
return self
def __exit__(self, exc_type=None, exc_val=None, exc_tb=None):
logger = logging.getLogger()
logger.handlers = self.old_handlers
logger.propagate = self.old_propagate
def emit(self, record):
if record.levelno > self.max_level:
record.levelname = f'_{record.levelname}'
record.levelno = self.to_level
self.had_error_log = True
if MungedTracebackLogRecord.__base__ is logging.LogRecord:
MungedTracebackLogRecord.__bases__ = (record.__class__,)
record.__class__ = MungedTracebackLogRecord
if logging.getLogger(record.name).isEnabledFor(record.levelno):
for handler in self.old_handlers:
if handler.level <= record.levelno:
handler.emit(record)
class MungedTracebackLogRecord(logging.LogRecord):
def getMessage(self):
return super().getMessage().replace(
'Traceback (most recent call last):',
'_Traceback_ (most recent call last):',
)
def stripped_sys_argv(*strip_args):
"""Return sys.argv with some arguments stripped, suitable for reexecution or subprocesses"""
strip_args = sorted(set(strip_args) | set(['-s', '--save', '-u', '--update', '-i', '--init', '--i18n-overwrite']))
assert all(config.parser.has_option(s) for s in strip_args)
takes_value = dict((s, config.parser.get_option(s).takes_value()) for s in strip_args)
longs, shorts = list(tuple(y) for _, y in itergroupby(strip_args, lambda x: x.startswith('--')))
longs_eq = tuple(l + '=' for l in longs if takes_value[l])
args = sys.argv[:]
def strip(args, i):
return args[i].startswith(shorts) \
or args[i].startswith(longs_eq) or (args[i] in longs) \
or (i >= 1 and (args[i - 1] in strip_args) and takes_value[args[i - 1]])
return [x for i, x in enumerate(args) if not strip(args, i)]
C7E4
class ConstantMapping(Mapping[typing.Any, T], typing.Generic[T]):
"""
An immutable mapping returning the provided value for every single key.
Useful for default value to methods
"""
__slots__ = ['_value']
def __init__(self, val: T):
self._value = val
def __len__(self):
"""
defaultdict updates its length for each individually requested key, is
that really useful?
"""
return 0
def __iter__(self):
"""
same as len, defaultdict updates its iterable keyset with each key
requested, is there a point for this?
"""
return iter([])
def __getitem__(self, item) -> T:
return self._value
def dumpstacks(sig=None, frame=None, thread_idents=None, log_level=logging.INFO):
""" Signal handler: dump a stack trace for each existing thread or given
thread(s) specified through the ``thread_idents`` sequence.
"""
code = []
def extract_stack(stack):
for filename, lineno, name, line in traceback.extract_stack(stack):
yield 'File: "%s", line %d, in %s' % (filename, lineno, name)
if line:
yield " %s" % (line.strip(),)
# code from http://stackoverflow.com/questions/132058/getting-stack-trace-from-a-running-python-application#answer-2569696
# modified for python 2.5 compatibility
threads_info = {th.ident: {'repr': repr(th),
'uid': getattr(th, 'uid', 'n/a'),
'dbname': getattr(th, 'dbname', 'n/a'),
'url': getattr(th, 'url', 'n/a'),
'query_count': getattr(th, 'query_count', 'n/a'),
'query_time': getattr(th, 'query_time', None),
'perf_t0': getattr(th, 'perf_t0', None)}
for th in threading.enumerate()}
for threadId, stack in sys._current_frames().items():
if not thread_idents or threadId in thread_idents:
thread_info = threads_info.get(threadId, {})
query_time = thread_info.get('query_time')
perf_t0 = thread_info.get('perf_t0')
remaining_time = None
if query_time is not None and perf_t0:
remaining_time = '%.3f' % (real_time() - perf_t0 - query_time)
query_time = '%.3f' % query_time
# qc:query_count qt:query_time pt:python_time (aka remaining time)
code.append("\n# Thread: %s (db:%s) (uid:%s) (url:%s) (qc:%s qt:%s pt:%s)" %
(thread_info.get('repr', threadId),
thread_info.get('dbname', 'n/a'),
thread_info.get('uid', 'n/a'),
thread_info.get('url', 'n/a'),
thread_info.get('query_count', 'n/a'),
query_time or 'n/a',
remaining_time or 'n/a'))
for line in extract_stack(stack):
code.append(line)
import odoo # eventd
if odoo.evented:
# code from http://stackoverflow.com/questions/12510648/in-gevent-how-can-i-dump-stack-traces-of-all-running-greenlets
import gc
from greenlet import greenlet
for ob in gc.get_objects():
if not isinstance(ob, greenlet) or not ob:
continue
code.append("\n# Greenlet: %r" % (ob,))
for line in extract_stack(ob.gr_frame):
code.append(line)
_logger.log(log_level, "\n".join(code))
def freehash(arg: typing.Any) -> int:
try:
return hash(arg)
except Exception:
if isinstance(arg, Mapping):
return hash(frozendict(arg))
elif isinstance(arg, Iterable):
return hash(frozenset(freehash(item) for item in arg))
else:
return id(arg)
def clean_context(context: dict[str, typing.Any]) -> dict[str, typing.Any]:
""" This function take a dictionary and remove each entry with its key
starting with ``default_``
"""
return {k: v for k, v in context.items() if not k.startswith('default_')}
class frozendict(dict[K, T], typing.Generic[K, T]):
""" An implementation of an immutable dictionary. """
__slots__ = ()
def __delitem__(self, key):
raise NotImplementedError("'__delitem__' not supported on frozendict")
def __setitem__(self, key, val):
raise NotImplementedError("'__setitem__' not supported on frozendict")
def clear(self):
raise NotImplementedError("'clear' not supported on frozendict")
def pop(self, key, default=None):
raise NotImplementedError("'pop' not supported on frozendict")
def popitem(self):
raise NotImplementedError("'popitem' not supported on frozendict")
def setdefault(self, key, default=None):
raise NotImplementedError("'setdefault' not supported on frozendict")
def update(self, *args, **kwargs):
raise NotImplementedError("'update' not supported on frozendict")
def __hash__(self) -> int: # type: ignore
return hash(frozenset((key, freehash(val)) for key, val in self.items()))
class Collector(dict[K, tuple[T, ...]], typing.Generic[K, T]):
""" A mapping from keys to tuples. This implements a relation, and can be
seen as a space optimization for ``defaultdict(tuple)``.
"""
__slots__ = ()
def __getitem__(self, key: K) -> tuple[T, ...]:
return self.get(key, ())
def __setitem__(self, key: K, val: Iterable[T]):
val = tuple(val)
if val:
super().__setitem__(key, val)