Source Code for Module web2py.gluon.fileutils

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3   
  4  """ 
  5  This file is part of the web2py Web Framework 
  6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 
  7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 
  8  """ 
  9   
 10  import storage 
 11  import os 
 12  import re 
 13  import tarfile 
 14  import glob 
 15  import time 
 16  from http import HTTP 
 17  from gzip import open as gzopen 
 18  from settings import global_settings 
 19   
 20   
 21  __all__ = [ 
 22      'up', 
 23      'abspath', 
 24      'mktree', 
 25      'listdir', 
 26      'recursive_unlink', 
 27      'cleanpath', 
 28      'tar', 
 29      'untar', 
 30      'tar_compiled', 
 31      'get_session', 
 32      'check_credentials', 
 33      'w2p_pack', 
 34      'w2p_unpack', 
 35      'w2p_pack_plugin', 
 36      'w2p_unpack_plugin', 
 37      'fix_newlines', 
 38      ] 
 39   
 40   
 41 -def abspath(*relpath, **base): 
 42      "convert relative path to absolute path based (by default) on applications_parent" 
 43      path = os.path.join(*relpath) 
 44      gluon = base.get('gluon', False) 
 45      if os.path.isabs(path): 
 46          return path 
 47      if gluon: 
 48          return os.path.join(global_settings.gluon_parent, path) 
 49      return os.path.join(global_settings.applications_parent, path) 
 50   
 51   
 52 -def mktree(path): 
 53      head,tail =os.path.split(path) 
 54      if head: 
 55          if tail: mktree(head) 
 56          if not os.path.exists(head): 
 57              os.mkdir(head) 
 58   
 59 -def listdir( 
 60      path, 
 61      expression='^.+$', 
 62      drop=True, 
 63      add_dirs=False, 
 64      sort=True, 
 65      ): 
 66      """ 
 67      like os.listdir() but you can specify a regex pattern to filter files. 
 68      if add_dirs is True, the returned items will have the full path. 
 69      """ 
 70      if path[-1:] != '/': 
 71          path = path + '/' 
 72      if drop: 
 73          n = len(path) 
 74      else: 
 75          n = 0 
 76      regex = re.compile(expression) 
 77      items = [] 
 78      for (root, dirs, files) in os.walk(path, topdown=True): 
 79          for dir in dirs[:]: 
 80              if dir.startswith('.'): 
 81                  dirs.remove(dir) 
 82          if add_dirs: 
 83              items.append(root[n:]) 
 84          for file in sorted(files): 
 85              if regex.match(file) and not file.startswith('.'): 
 86                  items.append(os.path.join(root, file)[n:]) 
 87      if sort: 
 88          return sorted(items) 
 89      else: 
 90          return items 
 91   
 92   
 93 -def recursive_unlink(f): 
 94      if os.path.isdir(f): 
 95          for s in os.listdir(f): 
 96              recursive_unlink(os.path.join(f,s)) 
 97          os.rmdir(f) 
 98      elif os.path.isfile(f): 
 99          os.unlink(f) 
100   
101   
102 -def cleanpath(path): 
103      """ 
104      turns any expression/path into a valid filename. replaces / with _ and 
105      removes special characters. 
106      """ 
107   
108      items = path.split('.') 
109      if len(items) > 1: 
110          path = re.sub('[^\w\.]+', '_', '_'.join(items[:-1]) + '.' 
111                         + ''.join(items[-1:])) 
112      else: 
113          path = re.sub('[^\w\.]+', '_', ''.join(items[-1:])) 
114      return path 
115   
116   
117 -def _extractall(filename, path='.', members=None): 
118      if not hasattr(tarfile.TarFile, 'extractall'): 
119          from tarfile import ExtractError 
120   
121          class TarFile(tarfile.TarFile): 
122   
123              def extractall(self, path='.', members=None): 
124                  """Extract all members from the archive to the current working 
125               directory and set owner, modification time and permissions on 
126               directories afterwards. `path' specifies a different directory 
127               to extract to. `members' is optional and must be a subset of the 
128               list returned by getmembers(). 
129                  """ 
130   
131                  directories = [] 
132                  if members is None: 
133                      members = self 
134                  for tarinfo in members: 
135                      if tarinfo.isdir(): 
136   
137                          # Extract directory with a safe mode, so that 
138                          # all files below can be extracted as well. 
139   
140                          try: 
141                              os.makedirs(os.path.join(path, 
142                                      tarinfo.name), 0777) 
143                          except EnvironmentError: 
144                              pass 
145                          directories.append(tarinfo) 
146                      else: 
147                          self.extract(tarinfo, path) 
148   
149                  # Reverse sort directories. 
150   
151                  directories.sort(lambda a, b: cmp(a.name, b.name)) 
152                  directories.reverse() 
153   
154                  # Set correct owner, mtime and filemode on directories. 
155   
156                  for tarinfo in directories: 
157                      path = os.path.join(path, tarinfo.name) 
158                      try: 
159                          self.chown(tarinfo, path) 
160                          self.utime(tarinfo, path) 
161                          self.chmod(tarinfo, path) 
162                      except ExtractError, e: 
163                          if self.errorlevel > 1: 
164                              raise 
165                          else: 
166                              self._dbg(1, 'tarfile: %s' % e) 
167   
168   
169          _cls = TarFile 
170      else: 
171          _cls = tarfile.TarFile 
172   
173      tar = _cls(filename, 'r') 
174      ret = tar.extractall(path, members) 
175      tar.close() 
176      return ret 
177   
178 -def tar(file, dir, expression='^.+$'): 
179      """ 
180      tars dir into file, only tars file that match expression 
181      """ 
182   
183      tar = tarfile.TarFile(file, 'w') 
184      for file in listdir(dir, expression, add_dirs=True): 
185          tar.add(os.path.join(dir, file), file, False) 
186      tar.close() 
187   
188 -def untar(file, dir): 
189      """ 
190      untar file into dir 
191      """ 
192   
193      _extractall(file, dir) 
194   
195   
196 -def w2p_pack(filename, path, compiled=False): 
197      filename = abspath(filename) 
198      path = abspath(path) 
199      tarname = filename + '.tar' 
200      if compiled: 
201          tar_compiled(tarname, path, '^[\w\.\-]+$') 
202      else: 
203          tar(tarname, path, '^[\w\.\-]+$') 
204      w2pfp = gzopen(filename, 'wb') 
205      tarfp = open(tarname, 'rb') 
206      w2pfp.write(tarfp.read()) 
207      w2pfp.close() 
208      tarfp.close() 
209      os.unlink(tarname) 
210   
211 -def w2p_unpack(filename, path, delete_tar=True): 
212      filename = abspath(filename) 
213      path = abspath(path) 
214      if filename[-4:] == '.w2p' or filename[-3:] == '.gz': 
215          if filename[-4:] == '.w2p': 
216              tarname = filename[:-4] + '.tar' 
217          else: 
218              tarname = filename[:-3] + '.tar' 
219          fgzipped = gzopen(filename, 'rb') 
220          tarfile = open(tarname, 'wb') 
221          tarfile.write(fgzipped.read()) 
222          tarfile.close() 
223          fgzipped.close() 
224      else: 
225          tarname = filename 
226      untar(tarname, path) 
227      if delete_tar: 
228          os.unlink(tarname) 
229   
230   
231 -def w2p_pack_plugin(filename, path, plugin_name): 
232      """Pack the given plugin into a w2p file. 
233      Will match files at: 
234          <path>/*/plugin_[name].* 
235          <path>/*/plugin_[name]/* 
236      """ 
237      filename = abspath(filename) 
238      path = abspath(path) 
239      if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name): 
240          raise Exception, "Not a web2py plugin name" 
241      plugin_tarball = tarfile.open(filename, 'w:gz') 
242      app_dir = path 
243      while app_dir[-1]=='/': 
244          app_dir = app_dir[:-1] 
245      files1=glob.glob(os.path.join(app_dir,'*/plugin_%s.*' % plugin_name)) 
246      files2=glob.glob(os.path.join(app_dir,'*/plugin_%s/*' % plugin_name)) 
247      for file in files1+files2: 
248          plugin_tarball.add(file, arcname=file[len(app_dir)+1:]) 
249      plugin_tarball.close() 
250   
251   
252 -def w2p_unpack_plugin(filename, path, delete_tar=True): 
253      filename = abspath(filename) 
254      path = abspath(path) 
255      if not os.path.basename(filename).startswith('web2py.plugin.'): 
256          raise Exception, "Not a web2py plugin" 
257      w2p_unpack(filename,path,delete_tar) 
258   
259   
260 -def tar_compiled(file, dir, expression='^.+$'): 
261      """ 
262      used to tar a compiled application. 
263      the content of models, views, controllers is not stored in the tar file. 
264      """ 
265   
266      tar = tarfile.TarFile(file, 'w') 
267      for file in listdir(dir, expression, add_dirs=True): 
268          filename = os.path.join(dir, file) 
269          if os.path.islink(filename): 
270              continue 
271          if os.path.isfile(filename) and file[-4:] != '.pyc': 
272              if file[:6] == 'models': 
273                  continue 
274              if file[:5] == 'views': 
275                  continue 
276              if file[:11] == 'controllers': 
277                  continue 
278              if file[:7] == 'modules': 
279                  continue 
280          tar.add(filename, file, False) 
281      tar.close() 
282   
283 -def up(path): 
284      return os.path.dirname(os.path.normpath(path)) 
285   
286   
287 -def get_session(request, other_application='admin'): 
288      """ checks that user is authorized to access other_application""" 
289      if request.application == other_application: 
290          raise KeyError 
291      try: 
292          session_id = request.cookies['session_id_' + other_application].value 
293          osession = storage.load_storage(os.path.join( 
294                  up(request.folder), other_application, 'sessions', session_id)) 
295      except: 
296          osession = storage.Storage() 
297      return osession 
298   
299   
300 -def check_credentials(request, other_application='admin', expiration = 60*60): 
301      """ checks that user is authorized to access other_application""" 
302      if request.env.web2py_runtime_gae: 
303          from google.appengine.api import users 
304          if users.is_current_user_admin(): 
305              return True 
306          else: 
307              login_html = '<a href="%s">Sign in with your google account</a>.' \ 
308                  % users.create_login_url(request.env.path_info) 
309              raise HTTP(200, '<html><body>%s</body></html>' % login_html) 
310      else: 
311          dt = time.time() - expiration 
312          s = get_session(request, other_application) 
313          return (s.authorized and s.last_time and s.last_time > dt) 
314   
315   
316 -def fix_newlines(path): 
317      regex = re.compile(r'''(\r 
318  |\r| 
319  )''') 
320      for filename in listdir(path, '.*\.(py|html)$', drop=False): 
321          fp = open(filename, 'rb') 
322          rdata = fp.read() 
323          fp.close() 
324          wdata = regex.sub('\n', rdata) 
325          if wdata != rdata: 
326              fp = open(filename, 'wb') 
327              fp.write(wdata) 
328              fp.close() 
329   
330   
331 -def copystream( 
332      src, 
333      dest, 
334      size, 
335      chunk_size=10 ** 5, 
336      ): 
337      """ 
338      this is here because I think there is a bug in shutil.copyfileobj 
339      """ 
340      while size > 0: 
341          if size < chunk_size: 
342              data = src.read(size) 
343          else: 
344              data = src.read(chunk_size) 
345          length = len(data) 
346          if length > size: 
347              (data, length) = (data[:size], size) 
348          size -= length 
349          if length == 0: 
350              break 
351          dest.write(data) 
352          if length < chunk_size: 
353              break 
354      dest.seek(0) 
355      return 
356