| Home | Trees | Indices | Help |
|
|---|
|
|
1 # -*- coding: utf-8 -*-
2
3
4
5 __doc__ = """GNUmed general tools."""
6
7 #===========================================================================
8 __author__ = "K. Hilbert <Karsten.Hilbert@gmx.net>"
9 __license__ = "GPL v2 or later (details at http://www.gnu.org)"
10
11 # std libs
12 import sys
13 import os
14 import os.path
15 import csv
16 import tempfile
17 import logging
18 import hashlib
19 import platform
20 import subprocess
21 import decimal
22 import getpass
23 import io
24 import functools
25 import json
26 import shutil
27 import zipfile
28 import datetime as pydt
29 import re as regex
30 import xml.sax.saxutils as xml_tools
31 # old:
32 import pickle, zlib
33 # docutils
34 du_core = None
35
36
37 # GNUmed libs
38 if __name__ == '__main__':
39 sys.path.insert(0, '../../')
40 from Gnumed.pycommon import gmBorg
41
42
43 _log = logging.getLogger('gm.tools')
44
45 # CAPitalization modes:
46 ( CAPS_NONE, # don't touch it
47 CAPS_FIRST, # CAP first char, leave rest as is
48 CAPS_ALLCAPS, # CAP all chars
49 CAPS_WORDS, # CAP first char of every word
50 CAPS_NAMES, # CAP in a way suitable for names (tries to be smart)
51 CAPS_FIRST_ONLY # CAP first char, lowercase the rest
52 ) = range(6)
53
54
55 u_currency_pound = '\u00A3' # Pound sign
56 u_currency_sign = '\u00A4' # generic currency sign
57 u_currency_yen = '\u00A5' # Yen sign
58 u_right_double_angle_quote = '\u00AB' # <<
59 u_registered_trademark = '\u00AE'
60 u_plus_minus = '\u00B1'
61 u_superscript_one = '\u00B9' # ^1
62 u_left_double_angle_quote = '\u00BB' # >>
63 u_one_quarter = '\u00BC'
64 u_one_half = '\u00BD'
65 u_three_quarters = '\u00BE'
66 u_multiply = '\u00D7' # x
67 u_greek_ALPHA = '\u0391'
68 u_greek_alpha = '\u03b1'
69 u_greek_OMEGA = '\u03A9'
70 u_greek_omega = '\u03c9'
71 u_dagger = '\u2020'
72 u_triangular_bullet = '\u2023' # triangular bullet (>)
73 u_ellipsis = '\u2026' # ...
74 u_euro = '\u20AC' # EURO sign
75 u_numero = '\u2116' # No. / # sign
76 u_down_left_arrow = '\u21B5' # <-'
77 u_left_arrow = '\u2190' # <--
78 u_up_arrow = '\u2191'
79 u_arrow2right = '\u2192' # -->
80 u_down_arrow = '\u2193'
81 u_left_arrow_with_tail = '\u21a2' # <--<
82 u_arrow2right_from_bar = '\u21a6' # |->
83 u_arrow2right_until_vertical_bar = '\u21e5' # -->|
84 u_sum = '\u2211' # sigma
85 u_almost_equal_to = '\u2248' # approximately / nearly / roughly
86 u_corresponds_to = '\u2258'
87 u_infinity = '\u221E'
88 u_arrow2right_until_vertical_bar2 = '\u2b72' # -->|
89 u_diameter = '\u2300'
90 u_checkmark_crossed_out = '\u237B'
91 u_box_vert_left = '\u23b8'
92 u_box_vert_right = '\u23b9'
93 u_box_horiz_single = '\u2500' # -
94 u_box_vert_light = '\u2502'
95 u_box_horiz_light_3dashes = '\u2504' # ...
96 u_box_vert_light_4dashes = '\u2506'
97 u_box_horiz_4dashes = '\u2508' # ....
98 u_box_T_right = '\u251c' # |-
99 u_box_T_left = '\u2524' # -|
100 u_box_T_down = '\u252c'
101 u_box_T_up = '\u2534'
102 u_box_plus = '\u253c'
103 u_box_top_double = '\u2550'
104 u_box_top_left_double_single = '\u2552'
105 u_box_top_right_double_single = '\u2555'
106 u_box_top_left_arc = '\u256d'
107 u_box_top_right_arc = '\u256e'
108 u_box_bottom_right_arc = '\u256f'
109 u_box_bottom_left_arc = '\u2570'
110 u_box_horiz_light_heavy = '\u257c'
111 u_box_horiz_heavy_light = '\u257e'
112 u_skull_and_crossbones = '\u2620'
113 u_caduceus = '\u2624'
114 u_frowning_face = '\u2639'
115 u_smiling_face = '\u263a'
116 u_black_heart = '\u2665'
117 u_female = '\u2640'
118 u_male = '\u2642'
119 u_male_female = '\u26a5'
120 u_checkmark_thin = '\u2713'
121 u_checkmark_thick = '\u2714'
122 u_heavy_greek_cross = '\u271a'
123 u_arrow2right_thick = '\u2794'
124 u_writing_hand = '\u270d'
125 u_pencil_1 = '\u270e'
126 u_pencil_2 = '\u270f'
127 u_pencil_3 = '\u2710'
128 u_latin_cross = '\u271d'
129 u_arrow2right_until_black_diamond = '\u291e' # ->*
130 u_kanji_yen = '\u5186' # Yen kanji
131 u_replacement_character = '\ufffd'
132 u_link_symbol = '\u1f517'
133
134 _kB = 1024
135 _MB = 1024 * _kB
136 _GB = 1024 * _MB
137 _TB = 1024 * _GB
138 _PB = 1024 * _TB
139
140 #===========================================================================
142
143 print(".========================================================")
144 print("| Unhandled exception caught !")
145 print("| Type :", t)
146 print("| Value:", v)
147 print("`========================================================")
148 _log.critical('unhandled exception caught', exc_info = (t,v,tb))
149 sys.__excepthook__(t,v,tb)
150
151 #===========================================================================
152 # path level operations
153 #---------------------------------------------------------------------------
155 try:
156 if mode is None:
157 os.makedirs(directory)
158 else:
159 old_umask = os.umask(0)
160 os.makedirs(directory, mode)
161 os.umask(old_umask)
162 except OSError as e:
163 if (e.errno == 17) and not os.path.isdir(directory):
164 raise
165 return True
166
167 #---------------------------------------------------------------------------
169 #-------------------------------
170 def _on_rm_error(func, path, exc):
171 _log.error('error while shutil.rmtree(%s)', path, exc_info=exc)
172 return True
173 #-------------------------------
174 error_count = 0
175 try:
176 shutil.rmtree(directory, False, _on_rm_error)
177 except Exception:
178 _log.exception('cannot shutil.rmtree(%s)', directory)
179 error_count += 1
180 return error_count
181
182 #---------------------------------------------------------------------------
184 _log.debug('cleaning out [%s]', directory)
185 try:
186 items = os.listdir(directory)
187 except OSError:
188 return False
189 for item in items:
190 # attempt file/link removal and ignore (but log) errors
191 full_item = os.path.join(directory, item)
192 try:
193 os.remove(full_item)
194 except OSError: # as per the docs, this is a directory
195 _log.debug('[%s] seems to be a subdirectory', full_item)
196 errors = rmdir(full_item)
197 if errors > 0:
198 return False
199 except Exception:
200 _log.exception('cannot os.remove(%s) [a file or a link]', full_item)
201 return False
202
203 return True
204
205 #---------------------------------------------------------------------------
207 if prefix is None:
208 if base_dir is None:
209 prefix = 'sandbox-'
210 else:
211 prefix = 'gm_sandbox-'
212 return tempfile.mkdtemp (
213 prefix = prefix,
214 suffix = '',
215 dir = base_dir
216 )
217
218 #---------------------------------------------------------------------------
221
222 #---------------------------------------------------------------------------
224 # /home/user/dir/ -> dir
225 # /home/user/dir -> dir
226 return os.path.basename(os.path.normpath(directory)) # normpath removes trailing slashes if any
227
228 #---------------------------------------------------------------------------
230 try:
231 return len(os.listdir(directory)) == 0
232 except OSError as exc:
233 if exc.errno == 2:
234 return None
235 raise
236
237 #---------------------------------------------------------------------------
239 """This class provides the following paths:
240
241 .home_dir user home
242 .local_base_dir script installation dir
243 .working_dir current dir
244 .user_config_dir
245 .system_config_dir
246 .system_app_data_dir (not writable)
247 .tmp_dir instance-local
248 .user_tmp_dir user-local (NOT per instance)
249 .bytea_cache_dir caches downloaded BYTEA data
250 """
252 """Setup pathes.
253
254 <app_name> will default to (name of the script - .py)
255 """
256 try:
257 self.already_inited
258 return
259 except AttributeError:
260 pass
261
262 self.init_paths(app_name=app_name, wx=wx)
263 self.already_inited = True
264
265 #--------------------------------------
266 # public API
267 #--------------------------------------
269
270 if wx is None:
271 _log.debug('wxPython not available')
272 _log.debug('detecting paths directly')
273
274 if app_name is None:
275 app_name, ext = os.path.splitext(os.path.basename(sys.argv[0]))
276 _log.info('app name detected as [%s]', app_name)
277 else:
278 _log.info('app name passed in as [%s]', app_name)
279
280 # the user home, doesn't work in Wine so work around that
281 self.__home_dir = None
282
283 # where the main script (the "binary") is installed
284 if getattr(sys, 'frozen', False):
285 _log.info('frozen app, installed into temporary path')
286 # this would find the path of *THIS* file
287 #self.local_base_dir = os.path.dirname(__file__)
288 # while this is documented on the web, the ${_MEIPASS2} does not exist
289 #self.local_base_dir = os.environ.get('_MEIPASS2')
290 # this is what Martin Zibricky <mzibr.public@gmail.com> told us to use
291 # when asking about this on pyinstaller@googlegroups.com
292 #self.local_base_dir = sys._MEIPASS
293 # however, we are --onedir, so we should look at sys.executable
294 # as per the pyinstaller manual
295 self.local_base_dir = os.path.dirname(sys.executable)
296 else:
297 self.local_base_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
298
299 # the current working dir at the OS
300 self.working_dir = os.path.abspath(os.curdir)
301
302 # user-specific config dir, usually below the home dir
303 mkdir(os.path.join(self.home_dir, '.%s' % app_name))
304 self.user_config_dir = os.path.join(self.home_dir, '.%s' % app_name)
305
306 # system-wide config dir, usually below /etc/ under UN*X
307 try:
308 self.system_config_dir = os.path.join('/etc', app_name)
309 except ValueError:
310 #self.system_config_dir = self.local_base_dir
311 self.system_config_dir = self.user_config_dir
312
313 # system-wide application data dir
314 try:
315 self.system_app_data_dir = os.path.join(sys.prefix, 'share', app_name)
316 except ValueError:
317 self.system_app_data_dir = self.local_base_dir
318
319 # temporary directory
320 try:
321 self.__tmp_dir_already_set
322 _log.debug('temp dir already set')
323 except AttributeError:
324 _log.info('temp file prefix: %s', tempfile.gettempprefix())
325 _log.info('initial (user level) temp dir: %s', tempfile.gettempdir())
326 # $TMP/gnumed-$USER/
327 self.user_tmp_dir = os.path.join(tempfile.gettempdir(), app_name + '-' + getpass.getuser())
328 mkdir(self.user_tmp_dir, 0o700)
329 tempfile.tempdir = self.user_tmp_dir
330 _log.info('intermediate (app level) temp dir: %s', tempfile.gettempdir())
331 # $TMP/gnumed-$USER/g$UNIQUE/
332 self.tmp_dir = tempfile.mkdtemp(prefix = 'g-')
333 _log.info('final (app instance level) temp dir: %s', tempfile.gettempdir())
334
335 # BYTEA cache dir
336 cache_dir = os.path.join(self.user_tmp_dir, '.bytea_cache')
337 try:
338 stat = os.stat(cache_dir)
339 _log.warning('reusing BYTEA cache dir: %s', cache_dir)
340 _log.debug(stat)
341 except FileNotFoundError:
342 mkdir(cache_dir, mode = 0o0700)
343 self.bytea_cache_dir = cache_dir
344
345 self.__log_paths()
346 if wx is None:
347 return True
348
349 # retry with wxPython
350 _log.debug('re-detecting paths with wxPython')
351
352 std_paths = wx.StandardPaths.Get()
353 _log.info('wxPython app name is [%s]', wx.GetApp().GetAppName())
354
355 # user-specific config dir, usually below the home dir
356 mkdir(os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name))
357 self.user_config_dir = os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name)
358
359 # system-wide config dir, usually below /etc/ under UN*X
360 try:
361 tmp = std_paths.GetConfigDir()
362 if not tmp.endswith(app_name):
363 tmp = os.path.join(tmp, app_name)
364 self.system_config_dir = tmp
365 except ValueError:
366 # leave it at what it was from direct detection
367 pass
368
369 # system-wide application data dir
370 # Robin attests that the following doesn't always
371 # give sane values on Windows, so IFDEF it
372 if 'wxMSW' in wx.PlatformInfo:
373 _log.warning('this platform (wxMSW) sometimes returns a broken value for the system-wide application data dir')
374 else:
375 try:
376 self.system_app_data_dir = std_paths.GetDataDir()
377 except ValueError:
378 pass
379
380 self.__log_paths()
381 return True
382
383 #--------------------------------------
385 _log.debug('sys.argv[0]: %s', sys.argv[0])
386 _log.debug('sys.executable: %s', sys.executable)
387 _log.debug('sys._MEIPASS: %s', getattr(sys, '_MEIPASS', '<not found>'))
388 _log.debug('os.environ["_MEIPASS2"]: %s', os.environ.get('_MEIPASS2', '<not found>'))
389 _log.debug('__file__ : %s', __file__)
390 _log.debug('local application base dir: %s', self.local_base_dir)
391 _log.debug('current working dir: %s', self.working_dir)
392 _log.debug('user home dir: %s', self.home_dir)
393 _log.debug('user-specific config dir: %s', self.user_config_dir)
394 _log.debug('system-wide config dir: %s', self.system_config_dir)
395 _log.debug('system-wide application data dir: %s', self.system_app_data_dir)
396 _log.debug('temporary dir (user): %s', self.user_tmp_dir)
397 _log.debug('temporary dir (instance): %s', self.tmp_dir)
398 _log.debug('BYTEA cache dir: %s', self.bytea_cache_dir)
399
400 #--------------------------------------
401 # properties
402 #--------------------------------------
404 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
405 msg = '[%s:user_config_dir]: invalid path [%s]' % (self.__class__.__name__, path)
406 _log.error(msg)
407 raise ValueError(msg)
408 self.__user_config_dir = path
409
412
413 user_config_dir = property(_get_user_config_dir, _set_user_config_dir)
414 #--------------------------------------
416 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
417 msg = '[%s:system_config_dir]: invalid path [%s]' % (self.__class__.__name__, path)
418 _log.error(msg)
419 raise ValueError(msg)
420 self.__system_config_dir = path
421
424
425 system_config_dir = property(_get_system_config_dir, _set_system_config_dir)
426 #--------------------------------------
428 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
429 msg = '[%s:system_app_data_dir]: invalid path [%s]' % (self.__class__.__name__, path)
430 _log.error(msg)
431 raise ValueError(msg)
432 self.__system_app_data_dir = path
433
436
437 system_app_data_dir = property(_get_system_app_data_dir, _set_system_app_data_dir)
438 #--------------------------------------
441
443 if self.__home_dir is not None:
444 return self.__home_dir
445
446 tmp = os.path.expanduser('~')
447 if tmp == '~':
448 _log.error('this platform does not expand ~ properly')
449 try:
450 tmp = os.environ['USERPROFILE']
451 except KeyError:
452 _log.error('cannot access $USERPROFILE in environment')
453
454 if not (
455 os.access(tmp, os.R_OK)
456 and
457 os.access(tmp, os.X_OK)
458 and
459 os.access(tmp, os.W_OK)
460 ):
461 msg = '[%s:home_dir]: invalid path [%s]' % (self.__class__.__name__, tmp)
462 _log.error(msg)
463 raise ValueError(msg)
464
465 self.__home_dir = tmp
466 return self.__home_dir
467
468 home_dir = property(_get_home_dir, _set_home_dir)
469
470 #--------------------------------------
472 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
473 msg = '[%s:tmp_dir]: invalid path [%s]' % (self.__class__.__name__, path)
474 _log.error(msg)
475 raise ValueError(msg)
476 _log.debug('previous temp dir: %s', tempfile.gettempdir())
477 self.__tmp_dir = path
478 tempfile.tempdir = self.__tmp_dir
479 _log.debug('new temp dir: %s', tempfile.gettempdir())
480 self.__tmp_dir_already_set = True
481
484
485 tmp_dir = property(_get_tmp_dir, _set_tmp_dir)
486
487 #===========================================================================
488 # file related tools
489 #---------------------------------------------------------------------------
490 -def recode_file(source_file=None, target_file=None, source_encoding='utf8', target_encoding=None, base_dir=None, error_mode='replace'):
491 if target_encoding is None:
492 return source_file
493 if target_encoding == source_encoding:
494 return source_file
495 if target_file is None:
496 target_file = get_unique_filename (
497 prefix = '%s-%s_%s-' % (fname_stem(source_file), source_encoding, target_encoding),
498 suffix = fname_extension(source_file, '.txt'),
499 tmp_dir = base_dir
500 )
501
502 _log.debug('[%s] -> [%s] (%s -> %s)', source_encoding, target_encoding, source_file, target_file)
503
504 in_file = io.open(source_file, mode = 'rt', encoding = source_encoding)
505 out_file = io.open(target_file, mode = 'wt', encoding = target_encoding, errors = error_mode)
506 for line in in_file:
507 out_file.write(line)
508 out_file.close()
509 in_file.close()
510
511 return target_file
512
513 #---------------------------------------------------------------------------
515 _log.debug('unzipping [%s] -> [%s]', archive_name, target_dir)
516 success = False
517 try:
518 with zipfile.ZipFile(archive_name) as archive:
519 archive.extractall(target_dir)
520 success = True
521 except Exception:
522 _log.exception('cannot unzip')
523 return False
524 if remove_archive:
525 remove_file(archive_name)
526 return success
527
528 #---------------------------------------------------------------------------
530 if not os.path.lexists(filename):
531 return True
532
533 # attempt file remove and ignore (but log) errors
534 try:
535 os.remove(filename)
536 return True
537
538 except Exception:
539 if log_error:
540 _log.exception('cannot os.remove(%s)', filename)
541
542 if force:
543 tmp_name = get_unique_filename(tmp_dir = fname_dir(filename))
544 _log.debug('attempting os.replace() to: %s', tmp_name)
545 try:
546 os.replace(filename, tmp_name)
547 return True
548
549 except BaseException:
550 if log_error:
551 _log.exception('cannot os.remove(%s)', filename)
552
553 return False
554
555 #---------------------------------------------------------------------------
557
558 if platform.system() == 'Windows':
559 exec_name = 'gpg.exe'
560 else:
561 exec_name = 'gpg'
562
563 tmp, fname = os.path.split(filename)
564 basename, tmp = os.path.splitext(fname)
565 filename_decrypted = get_unique_filename(prefix = '%s-decrypted-' % basename)
566
567 args = [exec_name, '--verbose', '--batch', '--yes', '--passphrase-fd', '0', '--output', filename_decrypted, '--decrypt', filename]
568 _log.debug('GnuPG args: %s' % str(args))
569
570 try:
571 gpg = subprocess.Popen (
572 args = args,
573 stdin = subprocess.PIPE,
574 stdout = subprocess.PIPE,
575 stderr = subprocess.PIPE,
576 close_fds = False
577 )
578 except (OSError, ValueError, subprocess.CalledProcessError):
579 _log.exception('there was a problem executing gpg')
580 gmDispatcher.send(signal = 'statustext', msg = _('Error running GnuPG. Cannot decrypt data.'), beep = True)
581 return
582
583 out, error = gpg.communicate(passphrase)
584 _log.debug('gpg returned [%s]', gpg.returncode)
585 if gpg.returncode != 0:
586 _log.debug('GnuPG STDOUT:\n%s', out)
587 _log.debug('GnuPG STDERR:\n%s', error)
588 return None
589
590 return filename_decrypted
591
592 #---------------------------------------------------------------------------
594 blocksize = 2**10 * 128 # 128k, since md5 uses 128 byte blocks
595 _log.debug('md5(%s): <%s> byte blocks', filename, blocksize)
596
597 f = io.open(filename, mode = 'rb')
598
599 md5 = hashlib.md5()
600 while True:
601 data = f.read(blocksize)
602 if not data:
603 break
604 md5.update(data)
605 f.close()
606
607 _log.debug('md5(%s): %s', filename, md5.hexdigest())
608
609 if return_hex:
610 return md5.hexdigest()
611 return md5.digest()
612
613 #---------------------------------------------------------------------------
615 _log.debug('chunked_md5(%s, chunk_size=%s bytes)', filename, chunk_size)
616 md5_concat = ''
617 f = open(filename, 'rb')
618 while True:
619 md5 = hashlib.md5()
620 data = f.read(chunk_size)
621 if not data:
622 break
623 md5.update(data)
624 md5_concat += md5.hexdigest()
625 f.close()
626 md5 = hashlib.md5()
627 md5.update(md5_concat)
628 hex_digest = md5.hexdigest()
629 _log.debug('md5("%s"): %s', md5_concat, hex_digest)
630 return hex_digest
631
632 #---------------------------------------------------------------------------
636
637 #def utf_8_encoder(unicode_csv_data):
638 # for line in unicode_csv_data:
639 # yield line.encode('utf-8')
640
641 default_csv_reader_rest_key = 'list_of_values_of_unknown_fields'
642
644
645 # csv.py doesn't do Unicode; encode temporarily as UTF-8:
646 try:
647 is_dict_reader = kwargs['dict']
648 del kwargs['dict']
649 if is_dict_reader is not True:
650 raise KeyError
651 kwargs['restkey'] = default_csv_reader_rest_key
652 csv_reader = csv.DictReader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs)
653 except KeyError:
654 is_dict_reader = False
655 csv_reader = csv.reader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs)
656
657 for row in csv_reader:
658 # decode ENCODING back to Unicode, cell by cell:
659 if is_dict_reader:
660 for key in row.keys():
661 if key == default_csv_reader_rest_key:
662 old_data = row[key]
663 new_data = []
664 for val in old_data:
665 new_data.append(str(val, encoding))
666 row[key] = new_data
667 if default_csv_reader_rest_key not in csv_reader.fieldnames:
668 csv_reader.fieldnames.append(default_csv_reader_rest_key)
669 else:
670 row[key] = str(row[key], encoding)
671 yield row
672 else:
673 yield [ str(cell, encoding) for cell in row ]
674 #yield [str(cell, 'utf-8') for cell in row]
675
676 #---------------------------------------------------------------------------
678 """Normalizes unicode, removes non-alpha characters, converts spaces to underscores."""
679
680 dir_part, name_part = os.path.split(filename)
681 if name_part == '':
682 return filename
683
684 import unicodedata
685 name_part = unicodedata.normalize('NFKD', name_part)
686 # remove everything not in group []
687 name_part = regex.sub (
688 '[^.\w\s[\]()%§+-]',
689 '',
690 name_part,
691 flags = regex.UNICODE
692 ).strip()
693 # translate whitespace to underscore
694 name_part = regex.sub (
695 '\s+',
696 '_',
697 name_part,
698 flags = regex.UNICODE
699 )
700 return os.path.join(dir_part, name_part)
701
702 #---------------------------------------------------------------------------
704 """/home/user/dir/filename.ext -> filename"""
705 return os.path.splitext(os.path.basename(filename))[0]
706
707 #---------------------------------------------------------------------------
709 """/home/user/dir/filename.ext -> /home/user/dir/filename"""
710 return os.path.splitext(filename)[0]
711
712 #---------------------------------------------------------------------------
714 """ /home/user/dir/filename.ext -> .ext
715 '' or '.' -> fallback if any else ''
716 """
717 ext = os.path.splitext(filename)[1]
718 if ext.strip() not in ['.', '']:
719 return ext
720 if fallback is None:
721 return ''
722 return fallback
723
724 #---------------------------------------------------------------------------
728
729 #---------------------------------------------------------------------------
733
734 #---------------------------------------------------------------------------
736 """This function has a race condition between
737 its file.close()
738 and actually
739 using the filename in callers.
740
741 The file will NOT exist after calling this function.
742 """
743 if tmp_dir is not None:
744 if (
745 not os.access(tmp_dir, os.F_OK)
746 or
747 not os.access(tmp_dir, os.X_OK | os.W_OK)
748 ):
749 _log.warning('cannot os.access() temporary dir [%s], using system default', tmp_dir)
750 tmp_dir = None
751
752 if include_timestamp:
753 ts = pydt.datetime.now().strftime('%m%d-%H%M%S-')
754 else:
755 ts = ''
756
757 kwargs = {
758 'dir': tmp_dir,
759 # make sure file gets deleted as soon as
760 # .close()d so we can "safely" open it again
761 'delete': True
762 }
763
764 if prefix is None:
765 kwargs['prefix'] = 'gm-%s' % ts
766 else:
767 kwargs['prefix'] = prefix + ts
768
769 if suffix in [None, '']:
770 kwargs['suffix'] = '.tmp'
771 else:
772 if not suffix.startswith('.'):
773 suffix = '.' + suffix
774 kwargs['suffix'] = suffix
775
776 f = tempfile.NamedTemporaryFile(**kwargs)
777 filename = f.name
778 f.close()
779
780 return filename
781
782 #---------------------------------------------------------------------------
784 import ctypes
785 #windows_create_symlink = ctypes.windll.kernel32.CreateSymbolicLinkW
786 kernel32 = ctype.WinDLL('kernel32', use_last_error = True)
787 windows_create_symlink = kernel32.CreateSymbolicLinkW
788 windows_create_symlink.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
789 windows_create_symlink.restype = ctypes.c_ubyte
790 if os.path.isdir(physical_name):
791 flags = 1
792 else:
793 flags = 0
794 ret_code = windows_create_symlink(link_name, physical_name.replace('/', '\\'), flags)
795 _log.debug('ctypes.windll.kernel32.CreateSymbolicLinkW() [%s] exit code: %s', windows_create_symlink, ret_code)
796 if ret_code == 0:
797 raise ctypes.WinError()
798 return ret_code
799
800 #---------------------------------------------------------------------------
802
803 _log.debug('creating symlink (overwrite = %s):', overwrite)
804 _log.debug('link [%s] =>', link_name)
805 _log.debug('=> physical [%s]', physical_name)
806
807 if os.path.exists(link_name):
808 _log.debug('link exists')
809 if overwrite:
810 return True
811 return False
812
813 try:
814 os.symlink(physical_name, link_name)
815 except (AttributeError, NotImplementedError):
816 _log.debug('this Python does not have os.symlink(), resorting to ctypes')
817 __make_symlink_on_windows(physical_name, link_name)
818 #except OSError:
819 # unpriviledged on Windows
820
821 return True
822
823 #===========================================================================
824 -def import_module_from_directory(module_path=None, module_name=None, always_remove_path=False):
825 """Import a module from any location."""
826
827 _log.debug('CWD: %s', os.getcwd())
828
829 remove_path = always_remove_path or False
830 if module_path not in sys.path:
831 _log.info('appending to sys.path: [%s]' % module_path)
832 sys.path.append(module_path)
833 remove_path = True
834
835 _log.debug('will remove import path: %s', remove_path)
836
837 if module_name.endswith('.py'):
838 module_name = module_name[:-3]
839
840 try:
841 module = __import__(module_name)
842 except Exception:
843 _log.exception('cannot __import__() module [%s] from [%s]' % (module_name, module_path))
844 while module_path in sys.path:
845 sys.path.remove(module_path)
846 raise
847
848 _log.info('imported module [%s] as [%s]' % (module_name, module))
849 if remove_path:
850 while module_path in sys.path:
851 sys.path.remove(module_path)
852
853 return module
854
855 #===========================================================================
856 # text related tools
857 #---------------------------------------------------------------------------
859 if size == 1:
860 return template % _('1 Byte')
861 if size < 10 * _kB:
862 return template % _('%s Bytes') % size
863 if size < _MB:
864 return template % '%.1f kB' % (float(size) / _kB)
865 if size < _GB:
866 return template % '%.1f MB' % (float(size) / _MB)
867 if size < _TB:
868 return template % '%.1f GB' % (float(size) / _GB)
869 if size < _PB:
870 return template % '%.1f TB' % (float(size) / _TB)
871 return template % '%.1f PB' % (float(size) / _PB)
872
873 #---------------------------------------------------------------------------
875 if boolean is None:
876 return none_return
877 if boolean:
878 return true_return
879 if not boolean:
880 return false_return
881 raise ValueError('bool2subst(): <boolean> arg must be either of True, False, None')
882
883 #---------------------------------------------------------------------------
885 return bool2subst (
886 boolean = bool(boolean),
887 true_return = true_str,
888 false_return = false_str
889 )
890
891 #---------------------------------------------------------------------------
893 """Modelled after the SQL NULLIF function."""
894 if value is None:
895 return None
896 if strip_string:
897 stripped = value.strip()
898 else:
899 stripped = value
900 if stripped == none_equivalent:
901 return None
902 return value
903
904 #---------------------------------------------------------------------------
905 -def coalesce(initial=None, instead=None, template_initial=None, template_instead=None, none_equivalents=None, function_initial=None):
906 """Modelled after the SQL coalesce function.
907
908 To be used to simplify constructs like:
909
910 if initial is None (or in none_equivalents):
911 real_value = (template_instead % instead) or instead
912 else:
913 real_value = (template_initial % initial) or initial
914 print real_value
915
916 @param initial: the value to be tested for <None>
917 @type initial: any Python type, must have a __str__ method if template_initial is not None
918 @param instead: the value to be returned if <initial> is None
919 @type instead: any Python type, must have a __str__ method if template_instead is not None
920 @param template_initial: if <initial> is returned replace the value into this template, must contain one <%s>
921 @type template_initial: string or None
922 @param template_instead: if <instead> is returned replace the value into this template, must contain one <%s>
923 @type template_instead: string or None
924
925 example:
926 function_initial = ('strftime', '%Y-%m-%d')
927
928 Ideas:
929 - list of insteads: initial, [instead, template], [instead, template], [instead, template], template_initial, ...
930 """
931 if none_equivalents is None:
932 none_equivalents = [None]
933
934 if initial in none_equivalents:
935
936 if template_instead is None:
937 return instead
938
939 return template_instead % instead
940
941 if function_initial is not None:
942 funcname, args = function_initial
943 func = getattr(initial, funcname)
944 initial = func(args)
945
946 if template_initial is None:
947 return initial
948
949 try:
950 return template_initial % initial
951 except TypeError:
952 return template_initial
953
954 #---------------------------------------------------------------------------
956 val = match_obj.group(0).lower()
957 if val in ['von', 'van', 'de', 'la', 'l', 'der', 'den']: # FIXME: this needs to expand, configurable ?
958 return val
959 buf = list(val)
960 buf[0] = buf[0].upper()
961 for part in ['mac', 'mc', 'de', 'la']:
962 if len(val) > len(part) and val[:len(part)] == part:
963 buf[len(part)] = buf[len(part)].upper()
964 return ''.join(buf)
965
966 #---------------------------------------------------------------------------
968 """Capitalize the first character but leave the rest alone.
969
970 Note that we must be careful about the locale, this may
971 have issues ! However, for UTF strings it should just work.
972 """
973 if (mode is None) or (mode == CAPS_NONE):
974 return text
975
976 if len(text) == 0:
977 return text
978
979 if mode == CAPS_FIRST:
980 if len(text) == 1:
981 return text[0].upper()
982 return text[0].upper() + text[1:]
983
984 if mode == CAPS_ALLCAPS:
985 return text.upper()
986
987 if mode == CAPS_FIRST_ONLY:
988 # if len(text) == 1:
989 # return text[0].upper()
990 return text[0].upper() + text[1:].lower()
991
992 if mode == CAPS_WORDS:
993 #return regex.sub(ur'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text)
994 return regex.sub(r'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text)
995
996 if mode == CAPS_NAMES:
997 #return regex.sub(r'\w+', __cap_name, text)
998 return capitalize(text=text, mode=CAPS_FIRST) # until fixed
999
1000 print("ERROR: invalid capitalization mode: [%s], leaving input as is" % mode)
1001 return text
1002
1003 #---------------------------------------------------------------------------
1005
1006 if isinstance(initial, decimal.Decimal):
1007 return True, initial
1008
1009 val = initial
1010
1011 # float ? -> to string first
1012 if type(val) == type(float(1.4)):
1013 val = str(val)
1014
1015 # string ? -> "," to "."
1016 if isinstance(val, str):
1017 val = val.replace(',', '.', 1)
1018 val = val.strip()
1019
1020 try:
1021 d = decimal.Decimal(val)
1022 return True, d
1023 except (TypeError, decimal.InvalidOperation):
1024 return False, val
1025
1026 #---------------------------------------------------------------------------
1028
1029 val = initial
1030
1031 # string ? -> "," to "."
1032 if isinstance(val, str):
1033 val = val.replace(',', '.', 1)
1034 val = val.strip()
1035
1036 try:
1037 int_val = int(val)
1038 except (TypeError, ValueError):
1039 _log.exception('int(%s) failed', val)
1040 return False, initial
1041
1042 if minval is not None:
1043 if int_val < minval:
1044 _log.debug('%s < min (%s)', val, minval)
1045 return False, initial
1046 if maxval is not None:
1047 if int_val > maxval:
1048 _log.debug('%s > max (%s)', val, maxval)
1049 return False, initial
1050
1051 return True, int_val
1052
1053 #---------------------------------------------------------------------------
1055 if remove_repeats:
1056 if remove_whitespace:
1057 while text.lstrip().startswith(prefix):
1058 text = text.lstrip().replace(prefix, '', 1).lstrip()
1059 return text
1060 while text.startswith(prefix):
1061 text = text.replace(prefix, '', 1)
1062 return text
1063 if remove_whitespace:
1064 return text.lstrip().replace(prefix, '', 1).lstrip()
1065 return text.replace(prefix, '', 1)
1066
1067 #---------------------------------------------------------------------------
1069 suffix_len = len(suffix)
1070 if remove_repeats:
1071 if remove_whitespace:
1072 while text.rstrip().endswith(suffix):
1073 text = text.rstrip()[:-suffix_len].rstrip()
1074 return text
1075 while text.endswith(suffix):
1076 text = text[:-suffix_len]
1077 return text
1078 if remove_whitespace:
1079 return text.rstrip()[:-suffix_len].rstrip()
1080 return text[:-suffix_len]
1081
1082 #---------------------------------------------------------------------------
1084 if lines is None:
1085 lines = text.split(eol)
1086
1087 while True:
1088 if lines[0].strip(eol).strip() != '':
1089 break
1090 lines = lines[1:]
1091
1092 if return_list:
1093 return lines
1094
1095 return eol.join(lines)
1096
1097 #---------------------------------------------------------------------------
1099 if lines is None:
1100 lines = text.split(eol)
1101
1102 while True:
1103 if lines[-1].strip(eol).strip() != '':
1104 break
1105 lines = lines[:-1]
1106
1107 if return_list:
1108 return lines
1109
1110 return eol.join(lines)
1111
1112 #---------------------------------------------------------------------------
1114 return strip_trailing_empty_lines (
1115 lines = strip_leading_empty_lines(lines = lines, text = text, eol = eol, return_list = True),
1116 text = None,
1117 eol = eol,
1118 return_list = return_list
1119 )
1120
1121 #---------------------------------------------------------------------------
1122 -def list2text(lines, initial_indent='', subsequent_indent='', eol='\n', strip_leading_empty_lines=True, strip_trailing_empty_lines=True, strip_trailing_whitespace=True):
1123
1124 if len(lines) == 0:
1125 return ''
1126
1127 if strip_leading_empty_lines:
1128 lines = strip_leading_empty_lines(lines = lines, eol = eol, return_list = True)
1129
1130 if strip_trailing_empty_lines:
1131 lines = strip_trailing_empty_lines(lines = lines, eol = eol, return_list = True)
1132
1133 if strip_trailing_whitespace:
1134 lines = [ l.rstrip() for l in lines ]
1135
1136 indented_lines = [initial_indent + lines[0]]
1137 indented_lines.extend([ subsequent_indent + l for l in lines[1:] ])
1138
1139 return eol.join(indented_lines)
1140
1141 #---------------------------------------------------------------------------
1143 """A word-wrap function that preserves existing line breaks
1144 and most spaces in the text. Expects that existing line
1145 breaks are posix newlines (\n).
1146 """
1147 if width is None:
1148 return text
1149 wrapped = initial_indent + functools.reduce (
1150 lambda line, word, width=width: '%s%s%s' % (
1151 line,
1152 ' \n'[(len(line) - line.rfind('\n') - 1 + len(word.split('\n',1)[0]) >= width)],
1153 word
1154 ),
1155 text.split(' ')
1156 )
1157
1158 if subsequent_indent != '':
1159 wrapped = ('\n%s' % subsequent_indent).join(wrapped.split('\n'))
1160
1161 if eol != '\n':
1162 wrapped = wrapped.replace('\n', eol)
1163
1164 return wrapped
1165
1166 #---------------------------------------------------------------------------
1167 -def unwrap(text=None, max_length=None, strip_whitespace=True, remove_empty_lines=True, line_separator = ' // '):
1168
1169 text = text.replace('\r', '')
1170 lines = text.split('\n')
1171 text = ''
1172 for line in lines:
1173
1174 if strip_whitespace:
1175 line = line.strip().strip('\t').strip()
1176
1177 if remove_empty_lines:
1178 if line == '':
1179 continue
1180
1181 text += ('%s%s' % (line, line_separator))
1182
1183 text = text.rstrip(line_separator)
1184
1185 if max_length is not None:
1186 text = text[:max_length]
1187
1188 text = text.rstrip(line_separator)
1189
1190 return text
1191
1192 #---------------------------------------------------------------------------
1194
1195 if len(text) <= max_length:
1196 return text
1197
1198 return text[:max_length-1] + u_ellipsis
1199
1200 #---------------------------------------------------------------------------
1201 -def shorten_words_in_line(text=None, max_length=None, min_word_length=None, ignore_numbers=True, ellipsis=u_ellipsis):
1202 if text is None:
1203 return None
1204 if max_length is None:
1205 max_length = len(text)
1206 else:
1207 if len(text) <= max_length:
1208 return text
1209 old_words = regex.split('\s+', text, flags = regex.UNICODE)
1210 no_old_words = len(old_words)
1211 max_word_length = max(min_word_length, (max_length // no_old_words))
1212 words = []
1213 for word in old_words:
1214 if len(word) <= max_word_length:
1215 words.append(word)
1216 continue
1217 if ignore_numbers:
1218 tmp = word.replace('-', '').replace('+', '').replace('.', '').replace(',', '').replace('/', '').replace('&', '').replace('*', '')
1219 if tmp.isdigit():
1220 words.append(word)
1221 continue
1222 words.append(word[:max_word_length] + ellipsis)
1223 return ' '.join(words)
1224
1225 #---------------------------------------------------------------------------
1229
1230 #---------------------------------------------------------------------------
1231 -def tex_escape_string(text=None, replace_known_unicode=True, replace_eol=False, keep_visual_eol=False):
1232 """Check for special TeX characters and transform them.
1233
1234 replace_eol:
1235 replaces "\n" with "\\newline"
1236 keep_visual_eol:
1237 replaces "\n" with "\\newline \n" such that
1238 both LaTeX will know to place a line break
1239 at this point as well as the visual formatting
1240 is preserved in the LaTeX source (think multi-
1241 row table cells)
1242 """
1243 text = text.replace('\\', '\\textbackslash') # requires \usepackage{textcomp} in LaTeX source
1244
1245 text = text.replace('{', '\\{')
1246 text = text.replace('}', '\\}')
1247 text = text.replace('%', '\\%')
1248 text = text.replace('&', '\\&')
1249 text = text.replace('#', '\\#')
1250 text = text.replace('$', '\\$')
1251 text = text.replace('_', '\\_')
1252
1253 text = text.replace('\\textbackslash', '\\textbackslash{}')
1254 text = text.replace('^', '\\textasciicircum{}') # requires \usepackage{textcomp} in LaTeX source
1255 text = text.replace('~', '\\textasciitilde{}') # requires \usepackage{textcomp} in LaTeX source
1256
1257 if replace_eol:
1258 if keep_visual_eol:
1259 text = text.replace('\n', '\\newline \n')
1260 else:
1261 text = text.replace('\n', '\\newline ')
1262
1263 if replace_known_unicode:
1264 # this should NOT be replaced for Xe(La)Tex
1265 text = text.replace(u_euro, '\\EUR')
1266
1267 return text
1268
1269 #---------------------------------------------------------------------------
1271 global du_core
1272 if du_core is None:
1273 try:
1274 from docutils import core as du_core
1275 except ImportError:
1276 _log.warning('cannot turn ReST into LaTeX: docutils not installed')
1277 return tex_escape_string(text = rst_text)
1278
1279 parts = du_core.publish_parts (
1280 source = rst_text.replace('\\', '\\\\'),
1281 source_path = '<internal>',
1282 writer_name = 'latex',
1283 #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex',
1284 settings_overrides = {
1285 'input_encoding': 'unicode' # un-encoded unicode
1286 },
1287 enable_exit_status = True # how to use ?
1288 )
1289 return parts['body']
1290
1291 #---------------------------------------------------------------------------
1293 global du_core
1294 if du_core is None:
1295 try:
1296 from docutils import core as du_core
1297 except ImportError:
1298 _log.warning('cannot turn ReST into HTML: docutils not installed')
1299 return html_escape_string(text = rst_text, replace_eol=False, keep_visual_eol=False)
1300
1301 parts = du_core.publish_parts (
1302 source = rst_text.replace('\\', '\\\\'),
1303 source_path = '<internal>',
1304 writer_name = 'latex',
1305 #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex',
1306 settings_overrides = {
1307 'input_encoding': 'unicode' # un-encoded unicode
1308 },
1309 enable_exit_status = True # how to use ?
1310 )
1311 return parts['body']
1312
1313 #---------------------------------------------------------------------------
1315 # a web search did not reveal anything else for Xe(La)Tex
1316 # as opposed to LaTeX, except true unicode chars
1317 return tex_escape_string(text = text, replace_known_unicode = False)
1318
1319 #---------------------------------------------------------------------------
1320 __html_escape_table = {
1321 "&": "&",
1322 '"': """,
1323 "'": "'",
1324 ">": ">",
1325 "<": "<",
1326 }
1327
1329 text = ''.join(__html_escape_table.get(char, char) for char in text)
1330 if replace_eol:
1331 if keep_visual_eol:
1332 text = text.replace('\n', '<br>\n')
1333 else:
1334 text = text.replace('\n', '<br>')
1335 return text
1336
1337 #---------------------------------------------------------------------------
1340
1341 #---------------------------------------------------------------------------
1343 if isinstance(obj, pydt.datetime):
1344 return obj.isoformat()
1345 raise TypeError('cannot json_serialize(%s)' % type(obj))
1346
1347 #---------------------------------------------------------------------------
1348 #---------------------------------------------------------------------------
1350 _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title1, '', '"%s" '), type(d1), coalesce(title2, '', '"%s" '), type(d2))
1351 try:
1352 d1 = dict(d1)
1353 except TypeError:
1354 pass
1355 try:
1356 d2 = dict(d2)
1357 except TypeError:
1358 pass
1359 keys_d1 = frozenset(d1.keys())
1360 keys_d2 = frozenset(d2.keys())
1361 different = False
1362 if len(keys_d1) != len(keys_d2):
1363 _log.info('different number of keys: %s vs %s', len(keys_d1), len(keys_d2))
1364 different = True
1365 for key in keys_d1:
1366 if key in keys_d2:
1367 if type(d1[key]) != type(d2[key]):
1368 _log.info('%25.25s: type(dict1) = %s = >>>%s<<<' % (key, type(d1[key]), d1[key]))
1369 _log.info('%25.25s type(dict2) = %s = >>>%s<<<' % ('', type(d2[key]), d2[key]))
1370 different = True
1371 continue
1372 if d1[key] == d2[key]:
1373 _log.info('%25.25s: both = >>>%s<<<' % (key, d1[key]))
1374 else:
1375 _log.info('%25.25s: dict1 = >>>%s<<<' % (key, d1[key]))
1376 _log.info('%25.25s dict2 = >>>%s<<<' % ('', d2[key]))
1377 different = True
1378 else:
1379 _log.info('%25.25s: %50.50s | <MISSING>' % (key, '>>>%s<<<' % d1[key]))
1380 different = True
1381 for key in keys_d2:
1382 if key in keys_d1:
1383 continue
1384 _log.info('%25.25s: %50.50s | %.50s' % (key, '<MISSING>', '>>>%s<<<' % d2[key]))
1385 different = True
1386 if different:
1387 _log.info('dict-likes appear to be different from each other')
1388 return False
1389 _log.info('dict-likes appear equal to each other')
1390 return True
1391
1392 #---------------------------------------------------------------------------
1393 -def format_dict_likes_comparison(d1, d2, title_left=None, title_right=None, left_margin=0, key_delim=' || ', data_delim=' | ', missing_string='=/=', difference_indicator='! ', ignore_diff_in_keys=None):
1394
1395 _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title_left, '', '"%s" '), type(d1), coalesce(title_right, '', '"%s" '), type(d2))
1396 append_type = False
1397 if None not in [title_left, title_right]:
1398 append_type = True
1399 type_left = type(d1)
1400 type_right = type(d2)
1401 if title_left is None:
1402 title_left = '%s' % type_left
1403 if title_right is None:
1404 title_right = '%s' % type_right
1405
1406 try: d1 = dict(d1)
1407 except TypeError: pass
1408 try: d2 = dict(d2)
1409 except TypeError: pass
1410 keys_d1 = d1.keys()
1411 keys_d2 = d2.keys()
1412 data = {}
1413 for key in keys_d1:
1414 data[key] = [d1[key], ' ']
1415 if key in d2:
1416 data[key][1] = d2[key]
1417 for key in keys_d2:
1418 if key in keys_d1:
1419 continue
1420 data[key] = [' ', d2[key]]
1421 max1 = max([ len('%s' % k) for k in keys_d1 ])
1422 max2 = max([ len('%s' % k) for k in keys_d2 ])
1423 max_len = max(max1, max2, len(_('<type>')))
1424 max_key_len_str = '%' + '%s.%s' % (max_len, max_len) + 's'
1425 max1 = max([ len('%s' % d1[k]) for k in keys_d1 ])
1426 max2 = max([ len('%s' % d2[k]) for k in keys_d2 ])
1427 max_data_len = min(max(max1, max2), 100)
1428 max_data_len_str = '%' + '%s.%s' % (max_data_len, max_data_len) + 's'
1429 diff_indicator_len_str = '%' + '%s.%s' % (len(difference_indicator), len(difference_indicator)) + 's'
1430 line_template = (' ' * left_margin) + diff_indicator_len_str + max_key_len_str + key_delim + max_data_len_str + data_delim + '%s'
1431
1432 lines = []
1433 # debugging:
1434 #lines.append(u' (40 regular spaces)')
1435 #lines.append((u' ' * 40) + u"(u' ' * 40)")
1436 #lines.append((u'%40.40s' % u'') + u"(u'%40.40s' % u'')")
1437 #lines.append((u'%40.40s' % u' ') + u"(u'%40.40s' % u' ')")
1438 #lines.append((u'%40.40s' % u'.') + u"(u'%40.40s' % u'.')")
1439 #lines.append(line_template)
1440 lines.append(line_template % ('', '', title_left, title_right))
1441 if append_type:
1442 lines.append(line_template % ('', _('<type>'), type_left, type_right))
1443
1444 if ignore_diff_in_keys is None:
1445 ignore_diff_in_keys = []
1446
1447 for key in keys_d1:
1448 append_type = False
1449 txt_left_col = '%s' % d1[key]
1450 try:
1451 txt_right_col = '%s' % d2[key]
1452 if type(d1[key]) != type(d2[key]):
1453 append_type = True
1454 except KeyError:
1455 txt_right_col = missing_string
1456 lines.append(line_template % (
1457 bool2subst (
1458 ((txt_left_col == txt_right_col) or (key in ignore_diff_in_keys)),
1459 '',
1460 difference_indicator
1461 ),
1462 key,
1463 shorten_text(txt_left_col, max_data_len),
1464 shorten_text(txt_right_col, max_data_len)
1465 ))
1466 if append_type:
1467 lines.append(line_template % (
1468 '',
1469 _('<type>'),
1470 shorten_text('%s' % type(d1[key]), max_data_len),
1471 shorten_text('%s' % type(d2[key]), max_data_len)
1472 ))
1473
1474 for key in keys_d2:
1475 if key in keys_d1:
1476 continue
1477 lines.append(line_template % (
1478 bool2subst((key in ignore_diff_in_keys), '', difference_indicator),
1479 key,
1480 shorten_text(missing_string, max_data_len),
1481 shorten_text('%s' % d2[key], max_data_len)
1482 ))
1483
1484 return lines
1485
1486 #---------------------------------------------------------------------------
1487 -def format_dict_like(d, relevant_keys=None, template=None, missing_key_template='<[%(key)s] MISSING>', left_margin=0, tabular=False, value_delimiters=('>>>', '<<<'), eol='\n', values2ignore=None):
1488 if values2ignore is None:
1489 values2ignore = []
1490 if template is not None:
1491 # all keys in template better exist in d
1492 try:
1493 return template % d
1494 except KeyError:
1495 # or else
1496 _log.exception('template contains %%()s key(s) which do not exist in data dict')
1497 # try to extend dict <d> to contain all required keys,
1498 # for that to work <relevant_keys> better list all
1499 # keys used in <template>
1500 if relevant_keys is not None:
1501 for key in relevant_keys:
1502 try:
1503 d[key]
1504 except KeyError:
1505 d[key] = missing_key_template % {'key': key}
1506 return template % d
1507
1508 if relevant_keys is None:
1509 relevant_keys = list(d.keys())
1510 lines = []
1511 if value_delimiters is None:
1512 delim_left = ''
1513 delim_right = ''
1514 else:
1515 delim_left, delim_right = value_delimiters
1516 if tabular:
1517 max_len = max([ len('%s' % k) for k in relevant_keys ])
1518 max_len_str = '%s.%s' % (max_len, max_len)
1519 line_template = (' ' * left_margin) + '%' + max_len_str + ('s: %s%%s%s' % (delim_left, delim_right))
1520 else:
1521 line_template = (' ' * left_margin) + '%%s: %s%%s%s' % (delim_left, delim_right)
1522 for key in relevant_keys:
1523 try:
1524 val = d[key]
1525 except KeyError:
1526 continue
1527 if val not in values2ignore:
1528 lines.append(line_template % (key, val))
1529 if eol is None:
1530 return lines
1531 return eol.join(lines)
1532
1533 #---------------------------------------------------------------------------
1535 for key in required_keys:
1536 try:
1537 d[key]
1538 except KeyError:
1539 if missing_key_template is None:
1540 d[key] = None
1541 else:
1542 d[key] = missing_key_template % {'key': key}
1543 return d
1544
1545 #---------------------------------------------------------------------------
1546 #---------------------------------------------------------------------------
1548 """Obtains entry from standard input.
1549
1550 prompt: Prompt text to display in standard output
1551 default: Default value (for user to press enter only)
1552 CTRL-C: aborts and returns None
1553 """
1554 if prompt is None:
1555 msg = '(CTRL-C aborts)'
1556 else:
1557 msg = '%s (CTRL-C aborts)' % prompt
1558
1559 if default is None:
1560 msg = msg + ': '
1561 else:
1562 msg = '%s [%s]: ' % (msg, default)
1563
1564 try:
1565 usr_input = input(msg)
1566 except KeyboardInterrupt:
1567 return None
1568
1569 if usr_input == '':
1570 return default
1571
1572 return usr_input
1573
1574 #===========================================================================
1575 # image handling tools
1576 #---------------------------------------------------------------------------
1577 # builtin (ugly but tried and true) fallback icon
1578 __icon_serpent = \
1579 """x\xdae\x8f\xb1\x0e\x83 \x10\x86w\x9f\xe2\x92\x1blb\xf2\x07\x96\xeaH:0\xd6\
1580 \xc1\x85\xd5\x98N5\xa5\xef?\xf5N\xd0\x8a\xdcA\xc2\xf7qw\x84\xdb\xfa\xb5\xcd\
1581 \xd4\xda;\xc9\x1a\xc8\xb6\xcd<\xb5\xa0\x85\x1e\xeb\xbc\xbc7b!\xf6\xdeHl\x1c\
1582 \x94\x073\xec<*\xf7\xbe\xf7\x99\x9d\xb21~\xe7.\xf5\x1f\x1c\xd3\xbdVlL\xc2\
1583 \xcf\xf8ye\xd0\x00\x90\x0etH \x84\x80B\xaa\x8a\x88\x85\xc4(U\x9d$\xfeR;\xc5J\
1584 \xa6\x01\xbbt9\xceR\xc8\x81e_$\x98\xb9\x9c\xa9\x8d,y\xa9t\xc8\xcf\x152\xe0x\
1585 \xe9$\xf5\x07\x95\x0cD\x95t:\xb1\x92\xae\x9cI\xa8~\x84\x1f\xe0\xa3ec"""
1586
1588
1589 paths = gmPaths(app_name = 'gnumed', wx = wx)
1590
1591 candidates = [
1592 os.path.join(paths.system_app_data_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'),
1593 os.path.join(paths.local_base_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'),
1594 os.path.join(paths.system_app_data_dir, 'bitmaps', 'serpent.png'),
1595 os.path.join(paths.local_base_dir, 'bitmaps', 'serpent.png')
1596 ]
1597
1598 found_as = None
1599 for candidate in candidates:
1600 try:
1601 open(candidate, 'r').close()
1602 found_as = candidate
1603 break
1604 except IOError:
1605 _log.debug('icon not found in [%s]', candidate)
1606
1607 if found_as is None:
1608 _log.warning('no icon file found, falling back to builtin (ugly) icon')
1609 icon_bmp_data = wx.BitmapFromXPMData(pickle.loads(zlib.decompress(__icon_serpent)))
1610 icon.CopyFromBitmap(icon_bmp_data)
1611 else:
1612 _log.debug('icon found in [%s]', found_as)
1613 icon = wx.Icon()
1614 try:
1615 icon.LoadFile(found_as, wx.BITMAP_TYPE_ANY) #_PNG
1616 except AttributeError:
1617 _log.exception("this platform doesn't support wx.Icon().LoadFile()")
1618
1619 return icon
1620
1621 #===========================================================================
1622 # main
1623 #---------------------------------------------------------------------------
1624 if __name__ == '__main__':
1625
1626 if len(sys.argv) < 2:
1627 sys.exit()
1628
1629 if sys.argv[1] != 'test':
1630 sys.exit()
1631
1632 # for testing:
1633 logging.basicConfig(level = logging.DEBUG)
1634 from Gnumed.pycommon import gmI18N
1635 gmI18N.activate_locale()
1636 gmI18N.install_domain()
1637
1638 #-----------------------------------------------------------------------
1640
1641 tests = [
1642 [None, False],
1643
1644 ['', False],
1645 [' 0 ', True, 0],
1646
1647 [0, True, 0],
1648 [0.0, True, 0],
1649 [.0, True, 0],
1650 ['0', True, 0],
1651 ['0.0', True, 0],
1652 ['0,0', True, 0],
1653 ['00.0', True, 0],
1654 ['.0', True, 0],
1655 [',0', True, 0],
1656
1657 [0.1, True, decimal.Decimal('0.1')],
1658 [.01, True, decimal.Decimal('0.01')],
1659 ['0.1', True, decimal.Decimal('0.1')],
1660 ['0,1', True, decimal.Decimal('0.1')],
1661 ['00.1', True, decimal.Decimal('0.1')],
1662 ['.1', True, decimal.Decimal('0.1')],
1663 [',1', True, decimal.Decimal('0.1')],
1664
1665 [1, True, 1],
1666 [1.0, True, 1],
1667 ['1', True, 1],
1668 ['1.', True, 1],
1669 ['1,', True, 1],
1670 ['1.0', True, 1],
1671 ['1,0', True, 1],
1672 ['01.0', True, 1],
1673 ['01,0', True, 1],
1674 [' 01, ', True, 1],
1675
1676 [decimal.Decimal('1.1'), True, decimal.Decimal('1.1')]
1677 ]
1678 for test in tests:
1679 conversion_worked, result = input2decimal(initial = test[0])
1680
1681 expected2work = test[1]
1682
1683 if conversion_worked:
1684 if expected2work:
1685 if result == test[2]:
1686 continue
1687 else:
1688 print("ERROR (conversion result wrong): >%s<, expected >%s<, got >%s<" % (test[0], test[2], result))
1689 else:
1690 print("ERROR (conversion worked but was expected to fail): >%s<, got >%s<" % (test[0], result))
1691 else:
1692 if not expected2work:
1693 continue
1694 else:
1695 print("ERROR (conversion failed but was expected to work): >%s<, expected >%s<" % (test[0], test[2]))
1696 #-----------------------------------------------------------------------
1701 #-----------------------------------------------------------------------
1703
1704 val = None
1705 print(val, coalesce(val, 'is None', 'is not None'))
1706 val = 1
1707 print(val, coalesce(val, 'is None', 'is not None'))
1708 return
1709
1710 import datetime as dt
1711 print(coalesce(initial = dt.datetime.now(), template_initial = '-- %s --', function_initial = ('strftime', '%Y-%m-%d')))
1712
1713 print('testing coalesce()')
1714 print("------------------")
1715 tests = [
1716 [None, 'something other than <None>', None, None, 'something other than <None>'],
1717 ['Captain', 'Mr.', '%s.'[:4], 'Mr.', 'Capt.'],
1718 ['value to test', 'test 3 failed', 'template with "%s" included', None, 'template with "value to test" included'],
1719 ['value to test', 'test 4 failed', 'template with value not included', None, 'template with value not included'],
1720 [None, 'initial value was None', 'template_initial: %s', None, 'initial value was None'],
1721 [None, 'initial value was None', 'template_initial: %%(abc)s', None, 'initial value was None']
1722 ]
1723 passed = True
1724 for test in tests:
1725 result = coalesce (
1726 initial = test[0],
1727 instead = test[1],
1728 template_initial = test[2],
1729 template_instead = test[3]
1730 )
1731 if result != test[4]:
1732 print("ERROR")
1733 print("coalesce: (%s, %s, %s, %s)" % (test[0], test[1], test[2], test[3]))
1734 print("expected:", test[4])
1735 print("received:", result)
1736 passed = False
1737
1738 if passed:
1739 print("passed")
1740 else:
1741 print("failed")
1742 return passed
1743 #-----------------------------------------------------------------------
1745 print('testing capitalize() ...')
1746 success = True
1747 pairs = [
1748 # [original, expected result, CAPS mode]
1749 ['Boot', 'Boot', CAPS_FIRST_ONLY],
1750 ['boot', 'Boot', CAPS_FIRST_ONLY],
1751 ['booT', 'Boot', CAPS_FIRST_ONLY],
1752 ['BoOt', 'Boot', CAPS_FIRST_ONLY],
1753 ['boots-Schau', 'Boots-Schau', CAPS_WORDS],
1754 ['boots-sChau', 'Boots-Schau', CAPS_WORDS],
1755 ['boot camp', 'Boot Camp', CAPS_WORDS],
1756 ['fahrner-Kampe', 'Fahrner-Kampe', CAPS_NAMES],
1757 ['häkkönen', 'Häkkönen', CAPS_NAMES],
1758 ['McBurney', 'McBurney', CAPS_NAMES],
1759 ['mcBurney', 'McBurney', CAPS_NAMES],
1760 ['blumberg', 'Blumberg', CAPS_NAMES],
1761 ['roVsing', 'RoVsing', CAPS_NAMES],
1762 ['Özdemir', 'Özdemir', CAPS_NAMES],
1763 ['özdemir', 'Özdemir', CAPS_NAMES],
1764 ]
1765 for pair in pairs:
1766 result = capitalize(pair[0], pair[2])
1767 if result != pair[1]:
1768 success = False
1769 print('ERROR (caps mode %s): "%s" -> "%s", expected "%s"' % (pair[2], pair[0], result, pair[1]))
1770
1771 if success:
1772 print("... SUCCESS")
1773
1774 return success
1775 #-----------------------------------------------------------------------
1777 print("testing import_module_from_directory()")
1778 path = sys.argv[1]
1779 name = sys.argv[2]
1780 try:
1781 mod = import_module_from_directory(module_path = path, module_name = name)
1782 except:
1783 print("module import failed, see log")
1784 return False
1785
1786 print("module import succeeded", mod)
1787 print(dir(mod))
1788 return True
1789 #-----------------------------------------------------------------------
1793 #-----------------------------------------------------------------------
1795 print("testing gmPaths()")
1796 print("-----------------")
1797 paths = gmPaths(wx=None, app_name='gnumed')
1798 print("user config dir:", paths.user_config_dir)
1799 print("system config dir:", paths.system_config_dir)
1800 print("local base dir:", paths.local_base_dir)
1801 print("system app data dir:", paths.system_app_data_dir)
1802 print("working directory :", paths.working_dir)
1803 print("temp directory :", paths.tmp_dir)
1804 #-----------------------------------------------------------------------
1806 print("testing none_if()")
1807 print("-----------------")
1808 tests = [
1809 [None, None, None],
1810 ['a', 'a', None],
1811 ['a', 'b', 'a'],
1812 ['a', None, 'a'],
1813 [None, 'a', None],
1814 [1, 1, None],
1815 [1, 2, 1],
1816 [1, None, 1],
1817 [None, 1, None]
1818 ]
1819
1820 for test in tests:
1821 if none_if(value = test[0], none_equivalent = test[1]) != test[2]:
1822 print('ERROR: none_if(%s) returned [%s], expected [%s]' % (test[0], none_if(test[0], test[1]), test[2]))
1823
1824 return True
1825 #-----------------------------------------------------------------------
1827 tests = [
1828 [True, 'Yes', 'Yes', 'Yes'],
1829 [False, 'OK', 'not OK', 'not OK']
1830 ]
1831 for test in tests:
1832 if bool2str(test[0], test[1], test[2]) != test[3]:
1833 print('ERROR: bool2str(%s, %s, %s) returned [%s], expected [%s]' % (test[0], test[1], test[2], bool2str(test[0], test[1], test[2]), test[3]))
1834
1835 return True
1836 #-----------------------------------------------------------------------
1838
1839 print(bool2subst(True, 'True', 'False', 'is None'))
1840 print(bool2subst(False, 'True', 'False', 'is None'))
1841 print(bool2subst(None, 'True', 'False', 'is None'))
1842 #-----------------------------------------------------------------------
1844 print(get_unique_filename())
1845 print(get_unique_filename(prefix='test-'))
1846 print(get_unique_filename(suffix='tst'))
1847 print(get_unique_filename(prefix='test-', suffix='tst'))
1848 print(get_unique_filename(tmp_dir='/home/ncq/Archiv/'))
1849 #-----------------------------------------------------------------------
1851 print("testing size2str()")
1852 print("------------------")
1853 tests = [0, 1, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, 100000000000, 1000000000000, 10000000000000]
1854 for test in tests:
1855 print(size2str(test))
1856 #-----------------------------------------------------------------------
1858
1859 test = """
1860 second line\n
1861 3rd starts with tab \n
1862 4th with a space \n
1863
1864 6th
1865
1866 """
1867 print(unwrap(text = test, max_length = 25))
1868 #-----------------------------------------------------------------------
1870 test = 'line 1\nline 2\nline 3'
1871
1872 print("wrap 5-6-7 initial 0, subsequent 0")
1873 print(wrap(test, 5))
1874 print()
1875 print(wrap(test, 6))
1876 print()
1877 print(wrap(test, 7))
1878 print("-------")
1879 input()
1880 print("wrap 5 initial 1-1-3, subsequent 1-3-1")
1881 print(wrap(test, 5, ' ', ' '))
1882 print()
1883 print(wrap(test, 5, ' ', ' '))
1884 print()
1885 print(wrap(test, 5, ' ', ' '))
1886 print("-------")
1887 input()
1888 print("wrap 6 initial 1-1-3, subsequent 1-3-1")
1889 print(wrap(test, 6, ' ', ' '))
1890 print()
1891 print(wrap(test, 6, ' ', ' '))
1892 print()
1893 print(wrap(test, 6, ' ', ' '))
1894 print("-------")
1895 input()
1896 print("wrap 7 initial 1-1-3, subsequent 1-3-1")
1897 print(wrap(test, 7, ' ', ' '))
1898 print()
1899 print(wrap(test, 7, ' ', ' '))
1900 print()
1901 print(wrap(test, 7, ' ', ' '))
1902 #-----------------------------------------------------------------------
1904 print('md5 %s: %s' % (sys.argv[2], file2md5(sys.argv[2])))
1905 print('chunked md5 %s: %s' % (sys.argv[2], file2chunked_md5(sys.argv[2])))
1906 #-----------------------------------------------------------------------
1908 print(u_link_symbol * 10)
1909 #-----------------------------------------------------------------------
1911 print(xml_escape_string('<'))
1912 print(xml_escape_string('>'))
1913 print(xml_escape_string('&'))
1914 #-----------------------------------------------------------------------
1916 tests = ['\\', '^', '~', '{', '}', '%', '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234']
1917 tests.append(' '.join(tests))
1918 for test in tests:
1919 print('%s:' % test, tex_escape_string(test))
1920
1921 #-----------------------------------------------------------------------
1923 tests = ['\\', '^', '~', '{', '}', '%', '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234']
1924 tests.append(' '.join(tests))
1925 tests.append('C:\Windows\Programme\System 32\lala.txt')
1926 tests.extend([
1927 'should be identical',
1928 'text *some text* text',
1929 """A List
1930 ======
1931
1932 1. 1
1933 2. 2
1934
1935 3. ist-list
1936 1. more
1937 2. noch was ü
1938 #. nummer x"""
1939 ])
1940 for test in tests:
1941 print('==================================================')
1942 print('raw:')
1943 print(test)
1944 print('---------')
1945 print('ReST 2 LaTeX:')
1946 latex = rst2latex_snippet(test)
1947 print(latex)
1948 if latex.strip() == test.strip():
1949 print('=> identical')
1950 print('---------')
1951 print('tex_escape_string:')
1952 print(tex_escape_string(test))
1953 input()
1954
1955 #-----------------------------------------------------------------------
1957 fname = gpg_decrypt_file(filename = sys.argv[2], passphrase = sys.argv[3])
1958 if fname is not None:
1959 print("successfully decrypted:", fname)
1960
1961 #-----------------------------------------------------------------------
1963 tests = [
1964 'one line, no embedded line breaks ',
1965 'one line\nwith embedded\nline\nbreaks\n '
1966 ]
1967 for test in tests:
1968 print('as list:')
1969 print(strip_trailing_empty_lines(text = test, eol='\n', return_list = True))
1970 print('as string:')
1971 print('>>>%s<<<' % strip_trailing_empty_lines(text = test, eol='\n', return_list = False))
1972 tests = [
1973 ['list', 'without', 'empty', 'trailing', 'lines'],
1974 ['list', 'with', 'empty', 'trailing', 'lines', '', ' ', '']
1975 ]
1976 for test in tests:
1977 print('as list:')
1978 print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = True))
1979 print('as string:')
1980 print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = False))
1981 #-----------------------------------------------------------------------
1983 tests = [
1984 r'abc.exe',
1985 r'\abc.exe',
1986 r'c:\abc.exe',
1987 r'c:\d\abc.exe',
1988 r'/home/ncq/tmp.txt',
1989 r'~/tmp.txt',
1990 r'./tmp.txt',
1991 r'./.././tmp.txt',
1992 r'tmp.txt'
1993 ]
1994 for t in tests:
1995 print("[%s] -> [%s]" % (t, fname_stem(t)))
1996 #-----------------------------------------------------------------------
1998 print(sys.argv[2], 'empty:', dir_is_empty(sys.argv[2]))
1999
2000 #-----------------------------------------------------------------------
2002 d1 = {}
2003 d2 = {}
2004 d1[1] = 1
2005 d1[2] = 2
2006 d1[3] = 3
2007 # 4
2008 d1[5] = 5
2009
2010 d2[1] = 1
2011 d2[2] = None
2012 # 3
2013 d2[4] = 4
2014
2015 #compare_dict_likes(d1, d2)
2016
2017 d1 = {1: 1, 2: 2}
2018 d2 = {1: 1, 2: 2}
2019
2020 #compare_dict_likes(d1, d2, 'same1', 'same2')
2021 print(format_dict_like(d1, tabular = False))
2022 print(format_dict_like(d1, tabular = True))
2023 #print(format_dict_like(d2))
2024
2025 #-----------------------------------------------------------------------
2027 d1 = {}
2028 d2 = {}
2029 d1[1] = 1
2030 d1[2] = 2
2031 d1[3] = 3
2032 # 4
2033 d1[5] = 5
2034
2035 d2[1] = 1
2036 d2[2] = None
2037 # 3
2038 d2[4] = 4
2039
2040 print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2')))
2041
2042 d1 = {1: 1, 2: 2}
2043 d2 = {1: 1, 2: 2}
2044
2045 print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2')))
2046
2047 #-----------------------------------------------------------------------
2049 rmdir('cx:\windows\system3__2xxxxxxxxxxxxx')
2050
2051 #-----------------------------------------------------------------------
2053 #print(rm_dir_content('cx:\windows\system3__2xxxxxxxxxxxxx'))
2054 print(rm_dir_content('/tmp/user/1000/tmp'))
2055
2056 #-----------------------------------------------------------------------
2058 tests = [
2059 ('', '', ''),
2060 ('a', 'a', ''),
2061 ('\.br\MICROCYTES+1\.br\SPHEROCYTES present\.br\POLYCHROMASIAmoderate\.br\\', '\.br\\', 'MICROCYTES+1\.br\SPHEROCYTES present\.br\POLYCHROMASIAmoderate\.br\\')
2062 ]
2063 for test in tests:
2064 text, prefix, expect = test
2065 result = strip_prefix(text, prefix)
2066 if result == expect:
2067 continue
2068 print('test failed:', test)
2069 print('result:', result)
2070 #-----------------------------------------------------------------------
2072 tst = [
2073 ('123', 1),
2074 ('123', 2),
2075 ('123', 3),
2076 ('123', 4),
2077 ('', 1),
2078 ('1', 1),
2079 ('12', 1),
2080 ('', 2),
2081 ('1', 2),
2082 ('12', 2),
2083 ('123', 2)
2084 ]
2085 for txt, lng in tst:
2086 print('max', lng, 'of', txt, '=', shorten_text(txt, lng))
2087 #-----------------------------------------------------------------------
2089 tests = [
2090 '/tmp/test.txt',
2091 '/tmp/ test.txt',
2092 '/tmp/ tes\\t.txt',
2093 'test'
2094 ]
2095 for test in tests:
2096 print (test, fname_sanitize(test))
2097
2098 #-----------------------------------------------------------------------
2099 #test_coalesce()
2100 #test_capitalize()
2101 #test_import_module()
2102 #test_mkdir()
2103 #test_gmPaths()
2104 #test_none_if()
2105 #test_bool2str()
2106 #test_bool2subst()
2107 #test_get_unique_filename()
2108 #test_size2str()
2109 #test_wrap()
2110 #test_input2decimal()
2111 #test_input2int()
2112 #test_unwrap()
2113 #test_md5()
2114 #test_unicode()
2115 #test_xml_escape()
2116 #test_gpg_decrypt()
2117 #test_strip_trailing_empty_lines()
2118 #test_fname_stem()
2119 #test_tex_escape()
2120 test_rst2latex_snippet()
2121 #test_dir_is_empty()
2122 #test_compare_dicts()
2123 #test_rm_dir()
2124 #test_rm_dir_content()
2125 #test_strip_prefix()
2126 #test_shorten_text()
2127 #test_format_compare_dicts()
2128 #test_fname_sanitize()
2129
2130 #===========================================================================
2131
| Home | Trees | Indices | Help |
|
|---|
| Generated by Epydoc 3.0.1 on Sun Aug 19 01:55:20 2018 | http://epydoc.sourceforge.net |