{% for f in file.listdir(sortkey=sort_fnc, reverse=sort_reverse) %}
-
+
{% if f.link %}
{{ draw_widget(f, f.link) }}
diff --git a/browsepy/tests/test_file.py b/browsepy/tests/test_file.py
index 1032082..368169c 100644
--- a/browsepy/tests/test_file.py
+++ b/browsepy/tests/test_file.py
@@ -39,8 +39,17 @@ class TestFile(unittest.TestCase):
f.write(text)
return tmp_txt
+ def test_repr(self):
+ self.assertIsInstance(
+ repr(self.module.Node('a')),
+ browsepy.compat.basestring
+ )
+
def test_iter_listdir(self):
- directory = self.module.Directory(path=self.workbench)
+ directory = self.module.Directory(
+ path=self.workbench,
+ app=self.app
+ )
tmp_txt = self.textfile('somefile.txt', 'a')
--
GitLab
From 6cd94b5b1a808eb7feb927ded783f1f8e03c8be6 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sat, 28 Oct 2017 14:01:16 +0200
Subject: [PATCH 007/171] raise coverage, update setup
---
MANIFEST.in | 2 +
browsepy/plugin/file_actions/__init__.py | 8 +-
browsepy/plugin/file_actions/clipboard.py | 95 +++++++++++--------
browsepy/plugin/file_actions/exceptions.py | 90 ++++++++++++++++++
.../templates/400.file_actions.html | 65 +++++++++++++
.../templates/selection.file_actions.html | 60 ++++++++++++
browsepy/plugin/file_actions/tests.py | 68 ++++++++++++-
setup.py | 4 +
8 files changed, 346 insertions(+), 46 deletions(-)
create mode 100644 browsepy/plugin/file_actions/exceptions.py
create mode 100644 browsepy/plugin/file_actions/templates/400.file_actions.html
create mode 100644 browsepy/plugin/file_actions/templates/selection.file_actions.html
diff --git a/MANIFEST.in b/MANIFEST.in
index b32f958..0545255 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -3,3 +3,5 @@ graft browsepy/templates
graft browsepy/static
graft browsepy/plugin/player/templates
graft browsepy/plugin/player/static
+graft browsepy/plugin/file_actions/templates
+graft browsepy/plugin/file_actions/static
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index 3ea6df3..4b7bcc7 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -13,12 +13,12 @@ from browsepy import get_cookie_browse_sorting, browse_sortkey_reverse
from browsepy.file import Node, abspath_to_urlpath, secure_filename, \
current_restricted_chars, common_path_separators
from browsepy.compat import map, re_escape, FileNotFoundError
-from browsepy.exceptions import OutsideDirectoryBase, InvalidFilenameError
+from browsepy.exceptions import OutsideDirectoryBase
from .clipboard import Clipboard
from .exceptions import FileActionsException, ClipboardException, \
InvalidClipboardModeError, InvalidClipboardItemError, \
- MissingClipboardItemError
+ MissingClipboardItemError, InvalidDirnameError
__basedir__ = os.path.dirname(os.path.abspath(__file__))
@@ -56,9 +56,9 @@ def create_directory(path):
basename = request.form['name']
if secure_filename(basename) != basename or not basename:
- raise InvalidFilenameError(
+ raise InvalidDirnameError(
path=directory.path,
- filename=basename,
+ name=basename,
)
os.mkdir(os.path.join(directory.path, basename))
diff --git a/browsepy/plugin/file_actions/clipboard.py b/browsepy/plugin/file_actions/clipboard.py
index 5cc123d..4da670c 100644
--- a/browsepy/plugin/file_actions/clipboard.py
+++ b/browsepy/plugin/file_actions/clipboard.py
@@ -6,25 +6,37 @@ import json
import base64
import logging
import hashlib
+import zlib
from flask import request
from browsepy.compat import range
+from .exceptions import InvalidClipboardSizeError
logger = logging.getLogger(__name__)
class Clipboard(set):
+ '''
+ Clipboard (set) with convenience methods to pick its state from request
+ cookies and save it to response cookies.
+ '''
cookie_secret = os.urandom(256)
- cookie_sign_name = 'clipboard-signature'
- cookie_mode_name = 'clipboard-mode'
- cookie_list_name = 'clipboard-{:x}'
+ cookie_name = 'clipboard-{:x}'
cookie_path = '/'
request_cache_field = '_browsepy_file_actions_clipboard_cache'
- max_pages = 0xffffffff
+ max_pages = 0xffffffff # 2 ** 32 - 1
@classmethod
def count(cls, request=request):
+ '''
+ Get how many clipboard items are stores on request cookies.
+
+ :param request: optional request, defaults to current flask request
+ :type request: werkzeug.Request
+ :return: number of clipboard items on request's cookies
+ :rtype: int
+ '''
return len(cls.from_request(request))
@classmethod
@@ -43,45 +55,35 @@ class Clipboard(set):
return cached
self = cls()
setattr(request, cls.request_cache_field, self)
- signature = cls._cookiebytes(cls.cookie_sign_name, request)
- data = cls._read_paginated_cookie(request)
- mode = cls._cookietext(cls.cookie_mode_name, request)
- if cls._signature(data, mode) == signature:
- try:
- self.update(json.loads(base64.b64decode(data).decode('utf-8')))
- self.mode = mode
- except:
- pass
+ try:
+ self.__setstate__(cls._read_paginated_cookie(request))
+ except:
+ pass
return self
- @classmethod
- def _cookiebytes(cls, name, request=request):
- return request.cookies.get(name, '').encode('ascii')
-
- @classmethod
- def _cookietext(cls, name, request=request):
- return request.cookies.get(name, '')
-
@classmethod
def _paginated_cookie_length(cls, page=0):
- name_fnc = cls.cookie_list_name.format
+ name_fnc = cls.cookie_name.format
return 3990 - len(name_fnc(page) + cls.cookie_path)
@classmethod
def _read_paginated_cookie(cls, request=request):
chunks = []
if request:
- name_fnc = cls.cookie_list_name.format
+ name_fnc = cls.cookie_name.format
for i in range(cls.max_pages): # 2 ** 32 - 1
cookie = request.cookies.get(name_fnc(i), '').encode('ascii')
chunks.append(cookie)
if len(cookie) < cls._paginated_cookie_length(i):
break
- return b''.join(chunks)
+ serialized = zlib.decompress(base64.b64decode(b''.join(chunks)))
+ return json.loads(serialized.decode('utf-8'))
@classmethod
def _write_paginated_cookie(cls, data, response):
- name_fnc = cls.cookie_list_name.format
+ serialized = zlib.compress(json.dumps(data).encode('utf-8'))
+ data = base64.b64encode(serialized)
+ name_fnc = cls.cookie_name.format
start = 0
size = len(data)
for i in range(cls.max_pages):
@@ -90,11 +92,11 @@ class Clipboard(set):
start = end
if start > size: # we need an empty page after start == size
return i
- return 0
+ raise InvalidClipboardSizeError(max_cookies=cls.max_pages)
@classmethod
def _delete_paginated_cookie(cls, response, start=0, request=request):
- name_fnc = cls.cookie_list_name.format
+ name_fnc = cls.cookie_name.format
for i in range(start, cls.max_pages):
name = name_fnc(i)
if name not in request.cookies:
@@ -102,23 +104,40 @@ class Clipboard(set):
response.set_cookie(name, '', expires=0)
@classmethod
- def _signature(cls, data, method):
- data = cls.cookie_secret + method.encode('utf-8') + data
- return base64.b64encode(hashlib.sha512(data).digest())
+ def _signature(cls, items, method):
+ serialized = json.dumps(items).encode('utf-8')
+ data = cls.cookie_secret + method.encode('utf-8') + serialized
+ return base64.b64encode(hashlib.sha512(data).digest()).decode('ascii')
def __init__(self, iterable=(), mode='copy'):
self.mode = mode
super(Clipboard, self).__init__(iterable)
+ def __getstate__(self):
+ items = list(self)
+ return {
+ 'mode': self.mode,
+ 'items': items,
+ 'signature': self._signature(items, self.mode),
+ }
+
+ def __setstate__(self, data):
+ if data['signature'] == self._signature(data['items'], data['mode']):
+ self.update(data['items'])
+ self.mode = data['mode']
+
def to_response(self, response, request=request):
+ '''
+ Save clipboard state to response taking care of disposing old clipboard
+ cookies from request.
+
+ :param response: response object to write cookies on
+ :type response: werkzeug.Response
+ :param request: optional request, defaults to current flask request
+ :type request: werkzeug.Request
+ '''
+ start = 0
if self:
- data = base64.b64encode(json.dumps(list(self)).encode('utf-8'))
- signature = self._signature(data, self.mode)
+ data = self.__getstate__()
start = self._write_paginated_cookie(data, response) + 1
- response.set_cookie(self.cookie_mode_name, self.mode)
- response.set_cookie(self.cookie_sign_name, signature)
- else:
- start = 0
- response.set_cookie(self.cookie_mode_name, '', expires=0)
- response.set_cookie(self.cookie_sign_name, '', expires=0)
self._delete_paginated_cookie(response, start, request)
diff --git a/browsepy/plugin/file_actions/exceptions.py b/browsepy/plugin/file_actions/exceptions.py
new file mode 100644
index 0000000..9de3653
--- /dev/null
+++ b/browsepy/plugin/file_actions/exceptions.py
@@ -0,0 +1,90 @@
+
+
+class FileActionsException(Exception):
+ '''
+ Base class for file-actions exceptions
+ '''
+ code = None
+ template = 'Unhandled error.'
+
+ def __init__(self, message=None, path=None):
+ self.path = path
+ message = self.template.format(self) if message is None else message
+ super(FileActionsException, self).__init__(message)
+
+
+class InvalidDirnameError(FileActionsException):
+ '''
+ Exception raised when a clipboard item is not valid.
+
+ :property name: name which raised this Exception
+ '''
+ code = 'invalid-dirname'
+ template = 'Clipboard item {0.name!r} is not valid.'
+
+ def __init__(self, message=None, path=None, name=None):
+ self.name = name
+ super(InvalidDirnameError, self).__init__(message, path)
+
+
+class ClipboardException(FileActionsException):
+ '''
+ Base class for clipboard exceptions.
+ '''
+ pass
+
+
+class InvalidClipboardItemError(ClipboardException):
+ '''
+ Exception raised when a clipboard item is not valid.
+
+ :property path: item path which raised this Exception
+ '''
+ code = 'invalid-clipboard-item'
+ template = 'Clipboard item {0.item!r} is not valid.'
+
+ def __init__(self, message=None, path=None, item=None):
+ self.item = item
+ super(InvalidClipboardItemError, self).__init__(message, path)
+
+
+class MissingClipboardItemError(InvalidClipboardItemError):
+ '''
+ Exception raised when a clipboard item is not valid.
+
+ :property path: item path which raised this Exception
+ '''
+ code = 'missing-clipboard-item'
+ template = 'Clipboard item {0.item!r} not found.'
+
+ def __init__(self, message=None, path=None, item=None):
+ self.item = item
+ super(InvalidClipboardItemError, self).__init__(message, path)
+
+
+class InvalidClipboardModeError(ClipboardException):
+ '''
+ Exception raised when a clipboard mode is not valid.
+
+ :property mode: mode which raised this Exception
+ '''
+ code = 'invalid-clipboard-mode'
+ template = 'Clipboard mode {0.path!r} is not valid.'
+
+ def __init__(self, message=None, path=None, mode=None):
+ self.mode = mode
+ super(InvalidClipboardModeError, self).__init__(message, path)
+
+
+class InvalidClipboardSizeError(ClipboardException):
+ '''
+ Exception raised when a clipboard size exceeds cookie limit.
+
+ :property max_cookies: maximum allowed size
+ '''
+ code = 'invalid-clipboard-size'
+ template = 'Clipboard has too many items.'
+
+ def __init__(self, message=None, path=None, max_cookies=0):
+ self.max_cookies = max_cookies
+ super(InvalidClipboardSizeError, self).__init__(message, path)
diff --git a/browsepy/plugin/file_actions/templates/400.file_actions.html b/browsepy/plugin/file_actions/templates/400.file_actions.html
new file mode 100644
index 0000000..70816fc
--- /dev/null
+++ b/browsepy/plugin/file_actions/templates/400.file_actions.html
@@ -0,0 +1,65 @@
+{% extends "base.html" %}
+
+{% set description %}
+
The server cannot or will not process the request due to an apparent client error (e.g., malformed request syntax, size too large, invalid request message framing, or deceptive request routing).
+
Please try other parameters or contact server administrator.
+{% endset %}
+
+{% set buttons %}
+ {% if file %}
+ Accept
+ {% else %}
+ Accept
+ {% endif %}
+{% endset %}
+
+{% if error and error.code %}
+ {% if error.code.startswith('invalid-clipboard-') %}
+ {% set workaround %}
+ {% if item %}
Clipboard item: {{ item.urlpath }}
{% endif %}
+
Please try again, if this problem persists contact server administrator.
+ {% endset %}
+ {% if error.code == 'invalid-clipboard-item' %}
+ {% set description -%}
+
A clipboard item is not valid.
+
File or directory is no longer available for given action.
+ {{ workaround }}
+ {% endset %}
{% endif %}
{% endif %}
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index 6ed2345..2bb7743 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -31,6 +31,10 @@ class CookieProxy(object):
}
def set_cookie(self, name, value, **kwargs):
+ for cookie in self._client.cookie_jar:
+ if cookie.name == name:
+ self._client.cookie_jar.clear(
+ cookie.domain, cookie.path, cookie.name)
return self._client.set_cookie(
self._client.environ_base['REMOTE_ADDR'], name, value, **kwargs)
--
GitLab
From 67012eb60b1df963a17da0e64e52fd81f8cd13f4 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 21 Feb 2018 20:49:22 +0100
Subject: [PATCH 012/171] update travis-sphinx call (due breaking change)
---
Makefile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Makefile b/Makefile
index 32ceedb..3c5b529 100644
--- a/Makefile
+++ b/Makefile
@@ -56,7 +56,7 @@ showcoverage: coverage
travis-script: pep8 flake8 coverage
travis-script-sphinx:
- travis-sphinx --nowarn --source=doc build
+ travis-sphinx build --nowarn --source doc
travis-success:
codecov
--
GitLab
From 3d652153da30b1ec06bb8d7b9d09304e60fe18a9 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 22 Feb 2018 19:12:03 +0100
Subject: [PATCH 013/171] simplify OSError handling on paste
---
browsepy/plugin/file_actions/__init__.py | 5 ++-
browsepy/plugin/file_actions/clipboard.py | 9 -----
browsepy/plugin/file_actions/exceptions.py | 19 ++++++++---
browsepy/plugin/file_actions/static/style.css | 4 +++
.../templates/400.file_actions.html | 6 +---
browsepy/plugin/file_actions/tests.py | 34 +++++++++++++++++++
6 files changed, 58 insertions(+), 19 deletions(-)
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index c732ce3..fd8f9b2 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -66,7 +66,10 @@ def create_directory(path):
name=basename,
)
- os.mkdir(os.path.join(directory.path, basename))
+ try:
+ os.mkdir(os.path.join(directory.path, basename))
+ except OSError:
+ raise FileActionsException(path=directory.path)
return redirect(url_for('browse', path=directory.urlpath))
diff --git a/browsepy/plugin/file_actions/clipboard.py b/browsepy/plugin/file_actions/clipboard.py
index 77d1295..753174e 100644
--- a/browsepy/plugin/file_actions/clipboard.py
+++ b/browsepy/plugin/file_actions/clipboard.py
@@ -156,12 +156,3 @@ class Clipboard(set):
data = self.__getstate__()
start = self._write_paginated_cookie(data, response) + 1
self._delete_paginated_cookie(response, start, request)
-
- def copy(self):
- '''
- Create another instance of this class with same items and mode.
-
- :returns: clipboard instance
- :rtype: Clipboard
- '''
- return self.___class__(self, mode=self.mode)
diff --git a/browsepy/plugin/file_actions/exceptions.py b/browsepy/plugin/file_actions/exceptions.py
index 6f81489..9166298 100644
--- a/browsepy/plugin/file_actions/exceptions.py
+++ b/browsepy/plugin/file_actions/exceptions.py
@@ -1,3 +1,5 @@
+import os
+import errno
class FileActionsException(Exception):
@@ -55,9 +57,18 @@ class ItemIssue(tuple):
return self[1]
@property
- def code(self):
- if isinstance(self.error, OSError):
- return 'oserror-%d' % self.error.errno
+ def message(self):
+ if isinstance(self.error, OSError) and \
+ self.error.errno in errno.errorcode:
+ return '%s (%s)' % (
+ os.strerror(self.error.errno),
+ errno.errorcode[self.error.errno]
+ )
+
+ # ensure full path is never returned
+ text = str(self.error)
+ text = text.replace(self.item.path, self.item.name)
+ return text
class InvalidClipboardItemsError(ClipboardException):
@@ -76,7 +87,7 @@ class InvalidClipboardItemsError(ClipboardException):
supa.__init__(message, path, clipboard)
def append(self, item, error):
- self.issues.append(self.pair_class(item, error))
+ self.issues.append(self.pair_class((item, error)))
class InvalidClipboardModeError(ClipboardException):
diff --git a/browsepy/plugin/file_actions/static/style.css b/browsepy/plugin/file_actions/static/style.css
index cb7ddfb..6260294 100644
--- a/browsepy/plugin/file_actions/static/style.css
+++ b/browsepy/plugin/file_actions/static/style.css
@@ -55,3 +55,7 @@ tr.clickable:hover {
tr.clickable:hover > td{
background-color: #ddd;
}
+.issue-name {
+ font-weight: bold;
+ font-family: monospace;
+}
\ No newline at end of file
diff --git a/browsepy/plugin/file_actions/templates/400.file_actions.html b/browsepy/plugin/file_actions/templates/400.file_actions.html
index 1e185c1..9401533 100644
--- a/browsepy/plugin/file_actions/templates/400.file_actions.html
+++ b/browsepy/plugin/file_actions/templates/400.file_actions.html
@@ -15,11 +15,7 @@
{% set messages %}
{% for issue in issues %}
- {% if issue.error.code == 'oserror-2' %}
-
{{ issue.item.name }} is no longer on source directory.
- {% else %}
-
{{ issue.item.name }} {{ issue.error }}
- {% endif %}
+
{{ issue.item.name }}: {{ issue.message }}
{% endfor %}
{% endset %}
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index 2bb7743..82eb014 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -14,6 +14,7 @@ from werkzeug.utils import cached_property
import browsepy.plugin.file_actions as file_actions
import browsepy.plugin.file_actions.clipboard as file_actions_clipboard
import browsepy.plugin.file_actions.exceptions as file_actions_exceptions
+import browsepy.file as browsepy_file
import browsepy.manager as browsepy_manager
import browsepy.exceptions as browsepy_exceptions
import browsepy.compat as compat
@@ -320,6 +321,13 @@ class TestAction(unittest.TestCase):
self.assertEqual(response.status_code, 302)
self.assertExist('asdf')
+ response = client.post(
+ '/file-actions/create/directory',
+ data={
+ 'name': 'asdf'
+ })
+ self.assertEqual(response.status_code, 400) # already exists
+
response = client.post(
'/file-actions/create/directory/..',
data={
@@ -520,3 +528,29 @@ class TestClipboard(unittest.TestCase):
clipboard = self.module.Clipboard()
clipboard.to_response(request, request)
self.assertNotIn(name, request.cookies)
+
+
+class TestException(unittest.TestCase):
+ module = file_actions_exceptions
+ clipboard_class = file_actions_clipboard.Clipboard
+ node_class = browsepy_file.Node
+
+ def test_invalid_clipboard_items_error(self):
+ clipboard = self.clipboard_class((
+ 'asdf',
+ ))
+ pair = (
+ self.node_class('/base/asdf'),
+ Exception('Uncaught exception /base/asdf'),
+ )
+ e = self.module.InvalidClipboardItemsError(
+ path='/',
+ clipboard=clipboard,
+ issues=[pair]
+ )
+ e.append(
+ self.node_class('/base/other'),
+ OSError(2, 'Not found with random message'),
+ )
+ self.assertIn('Uncaught exception asdf', e.issues[0].message)
+ self.assertIn('No such file or directory', e.issues[1].message)
--
GitLab
From c6a1d5002507eb01c55c295390a937e37faf428b Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 23 Feb 2018 14:14:33 +0100
Subject: [PATCH 014/171] refactor file-actions exception handling, clear after
cut paste
---
browsepy/plugin/file_actions/__init__.py | 18 ++--
browsepy/plugin/file_actions/exceptions.py | 41 +++++++--
.../templates/400.file_actions.html | 83 +++++++++++--------
3 files changed, 95 insertions(+), 47 deletions(-)
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index fd8f9b2..9d283da 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -20,7 +20,8 @@ from .clipboard import Clipboard
from .exceptions import FileActionsException, \
InvalidClipboardModeError, \
InvalidClipboardItemsError, \
- InvalidDirnameError
+ InvalidDirnameError, \
+ DirectoryCreationError
__basedir__ = os.path.dirname(os.path.abspath(__file__))
@@ -68,8 +69,12 @@ def create_directory(path):
try:
os.mkdir(os.path.join(directory.path, basename))
- except OSError:
- raise FileActionsException(path=directory.path)
+ except OSError as e:
+ raise DirectoryCreationError.from_exception(
+ e,
+ path=directory.path,
+ name=basename
+ )
return redirect(url_for('browse', path=directory.urlpath))
@@ -184,8 +189,11 @@ def clipboard_paste(path):
clipboard=clipboard,
issues=issues
)
- else:
- response = redirect(url_for('browse', path=directory.urlpath))
+
+ if mode == 'cut':
+ clipboard.clear()
+
+ response = redirect(url_for('browse', path=directory.urlpath))
clipboard.to_response(response)
return response
diff --git a/browsepy/plugin/file_actions/exceptions.py b/browsepy/plugin/file_actions/exceptions.py
index 9166298..35dd68e 100644
--- a/browsepy/plugin/file_actions/exceptions.py
+++ b/browsepy/plugin/file_actions/exceptions.py
@@ -21,7 +21,7 @@ class InvalidDirnameError(FileActionsException):
:property name: name which raised this Exception
'''
- code = 'invalid-dirname'
+ code = 'directory-invalid-name'
template = 'Clipboard item {0.name!r} is not valid.'
def __init__(self, message=None, path=None, name=None):
@@ -29,6 +29,34 @@ class InvalidDirnameError(FileActionsException):
super(InvalidDirnameError, self).__init__(message, path)
+class DirectoryCreationError(FileActionsException):
+ '''
+ Exception raised when a new directory creation fails.
+
+ :property name: name which raised this Exception
+ '''
+ code = 'directory-mkdir-error'
+ template = 'Clipboard item {0.name!r} is not valid.'
+
+ def __init__(self, message=None, path=None, name=None):
+ self.name = name
+ super(DirectoryCreationError, self).__init__(message, path)
+
+ @property
+ def message(self):
+ return self.args[0]
+
+ @classmethod
+ def from_exception(cls, exception, *args, **kwargs):
+ message = None
+ if isinstance(exception, OSError):
+ message = '%s (%s)' % (
+ os.strerror(exception.errno),
+ errno.errorcode[exception.errno]
+ )
+ return cls(message, *args, **kwargs)
+
+
class ClipboardException(FileActionsException):
'''
Base class for clipboard exceptions.
@@ -36,7 +64,7 @@ class ClipboardException(FileActionsException):
:property path: item path which raised this Exception
:property clipboard: :class Clipboard: instance
'''
- code = 'invalid-clipboard'
+ code = 'clipboard-invalid'
template = 'Clipboard is invalid.'
def __init__(self, message=None, path=None, clipboard=None):
@@ -58,8 +86,7 @@ class ItemIssue(tuple):
@property
def message(self):
- if isinstance(self.error, OSError) and \
- self.error.errno in errno.errorcode:
+ if isinstance(self.error, OSError):
return '%s (%s)' % (
os.strerror(self.error.errno),
errno.errorcode[self.error.errno]
@@ -78,7 +105,7 @@ class InvalidClipboardItemsError(ClipboardException):
:property path: item path which raised this Exception
'''
pair_class = ItemIssue
- code = 'invalid-clipboard-items'
+ code = 'clipboard-invalid-items'
template = 'Clipboard has invalid items.'
def __init__(self, message=None, path=None, clipboard=None, issues=()):
@@ -96,7 +123,7 @@ class InvalidClipboardModeError(ClipboardException):
:property mode: mode which raised this Exception
'''
- code = 'invalid-clipboard-mode'
+ code = 'clipboard-invalid-mode'
template = 'Clipboard mode {0.path!r} is not valid.'
def __init__(self, message=None, path=None, clipboard=None, mode=None):
@@ -111,7 +138,7 @@ class InvalidClipboardSizeError(ClipboardException):
:property max_cookies: maximum allowed size
'''
- code = 'invalid-clipboard-size'
+ code = 'clipboard-invalid-size'
template = 'Clipboard has too many items.'
def __init__(self, message=None, path=None, clipboard=None, max_cookies=0):
diff --git a/browsepy/plugin/file_actions/templates/400.file_actions.html b/browsepy/plugin/file_actions/templates/400.file_actions.html
index 9401533..9ac9b30 100644
--- a/browsepy/plugin/file_actions/templates/400.file_actions.html
+++ b/browsepy/plugin/file_actions/templates/400.file_actions.html
@@ -1,10 +1,5 @@
{% extends "base.html" %}
-{% set description %}
-
The server cannot or will not process the request due to an apparent client error (e.g., malformed request syntax, size too large, invalid request message framing, or deceptive request routing).
-
Please try other parameters or contact server administrator.
-{% endset %}
-
{% set buttons %}
{% if file %}
Accept
@@ -20,39 +15,57 @@
{% endset %}
{% if error and error.code %}
- {% set workaround %}
- {% if item %}
Clipboard item: {{ item.urlpath }}
{% endif %}
-
Please try again, if this problem persists contact server administrator.
- {% endset %}
- {% if error.code == 'invalid-clipboard-items' %}
- {% set description -%}
- {% if issues.length == 1 -%}
-
+ {% endset %}
{% endif %}
{% block title %}400 Bad Request{% endblock %}
--
GitLab
From 3503b9e0f070faa5c781b3f1123f3bb6d1b70fd9 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 23 Feb 2018 14:35:05 +0100
Subject: [PATCH 015/171] make paste to raise artificial OSError exceptions
---
browsepy/plugin/file_actions/__init__.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index 9d283da..f318869 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -128,7 +128,7 @@ def clipboard_paste(path):
def copy(target, node, join_fnc=os.path.join):
if node.is_excluded:
- raise Exception()
+ raise OSError(2, os.strerror(2))
dest = join_fnc(
target.path,
target.choose_filename(node.name)
@@ -140,7 +140,8 @@ def clipboard_paste(path):
def cut(target, node, join_fnc=os.path.join):
if node.is_excluded or not node.can_remove:
- raise Exception()
+ code = 2 if node.is_excluded else 1
+ raise OSError(code, os.strerror(code))
if node.parent.path != target.path:
dest = join_fnc(
target.path,
--
GitLab
From 925240a82d1db790a117ff746eb08b9c604c0dfa Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Tue, 13 Mar 2018 16:21:54 +0100
Subject: [PATCH 016/171] drop travis-sphinx
---
.travis.yml | 17 +++++++++--------
Makefile | 13 +------------
requirements.txt | 21 ++-------------------
requirements/base.txt | 2 ++
requirements/compat.txt | 5 +++++
requirements/development.txt | 12 ++++++++++++
6 files changed, 31 insertions(+), 39 deletions(-)
create mode 100644 requirements/base.txt
create mode 100644 requirements/compat.txt
create mode 100644 requirements/development.txt
diff --git a/.travis.yml b/.travis.yml
index 2d7e50e..766c851 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,8 +8,7 @@ matrix:
include:
- python: "2.7"
- python: "pypy"
- # pypy3 disabled until fixed
- #- python: "pypy3"
+ - python: "pypy3"
- python: "3.3"
- python: "3.4"
- python: "3.5"
@@ -20,7 +19,7 @@ matrix:
install:
- pip install --upgrade pip
- - pip install travis-sphinx flake8 pep8 codecov coveralls .
+ - pip install ghp-import sphinx flake8 pep8 codecov coveralls .
- |
if [ "$eslint" = "yes" ]; then
nvm install stable
@@ -30,21 +29,23 @@ install:
fi
script:
- - make travis-script
+ - make pep8 flake8 coverage
- |
if [ "$eslint" = "yes" ]; then
make eslint
fi
- |
if [ "$sphinx" = "yes" ]; then
- make travis-script-sphinx
+ make doc
fi
after_success:
- - make travis-success
+ - codecov
+ - coveralls
- |
- if [ "$sphinx" = "yes" ]; then
- make travis-success-sphinx
+ if [ "$sphinx" = "yes" ] && [ "$TRAVIS_BRANCH" = "master" ] &&
+[ "$TRAVIS_PULL_REQUEST" = "false" ]; then
+ ghp-import --push --force --no-jekyll --branch=gh-pages --remote="https://${GH_TOKEN}@github.com${TRAVIS_REPO_SLUG}.git doc/.build/html"
fi
notifications:
diff --git a/Makefile b/Makefile
index 2b90dcf..9994329 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
.PHONY: doc clean pep8 coverage travis
test: pep8 flake8 eslint
- python -c 'import yaml;yaml.load(open(".travis.yml").read())'
+ python -c 'import yaml, glob;[yaml.load(open(p)) for p in glob.glob(".*.yml")]'
ifdef debug
python setup.py test --debug=$(debug)
else
@@ -55,14 +55,3 @@ showcoverage: coverage
coverage html
xdg-open file://${CURDIR}/htmlcov/index.html >> /dev/null
-travis-script: pep8 flake8 coverage
-
-travis-script-sphinx:
- travis-sphinx --nowarn --source=doc build
-
-travis-success:
- codecov
- coveralls
-
-travis-success-sphinx:
- travis-sphinx deploy
diff --git a/requirements.txt b/requirements.txt
index bbb34a0..79239b1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,19 +1,2 @@
-flask
-unicategories
-
-# for python < 3.6
-scandir
-
-# for python < 3.3
-backports.shutil_get_terminal_size
-
-# for building
-setuptools
-wheel
-
-# for Makefile tasks (testing, coverage, docs)
-pep8
-flake8
-coverage
-pyaml
-sphinx
+-r requirements/compat.txt
+-r requirements/development.txt
diff --git a/requirements/base.txt b/requirements/base.txt
new file mode 100644
index 0000000..af7a758
--- /dev/null
+++ b/requirements/base.txt
@@ -0,0 +1,2 @@
+flask
+unicategories
diff --git a/requirements/compat.txt b/requirements/compat.txt
new file mode 100644
index 0000000..e070c36
--- /dev/null
+++ b/requirements/compat.txt
@@ -0,0 +1,5 @@
+# for python < 3.6
+scandir
+
+# for python < 3.3
+backports.shutil_get_terminal_size
diff --git a/requirements/development.txt b/requirements/development.txt
new file mode 100644
index 0000000..fb9f680
--- /dev/null
+++ b/requirements/development.txt
@@ -0,0 +1,12 @@
+-r base.txt
+
+# for building
+setuptools
+wheel
+
+# for Makefile tasks (testing, coverage, docs)
+pep8
+flake8
+coverage
+pyaml
+sphinx
--
GitLab
From 8952f748198fac11a6ac5e8afd44bf96f8c91cfc Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 5 Apr 2018 23:55:13 +0200
Subject: [PATCH 017/171] simplify tarfilestream usin queue
---
.gitignore | 45 ++++-----
browsepy/compat.py | 55 +++++++++++
browsepy/stream.py | 233 +++++++++++++++++++++++++++------------------
3 files changed, 219 insertions(+), 114 deletions(-)
diff --git a/.gitignore b/.gitignore
index c9eacdc..bf44bca 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,24 +1,25 @@
-.git/*
-.c9/*
-.idea/*
-.vscode/*
+.git
+.c9
+.idea
+.vscode
.coverage
-htmlcov/*
-dist/*
-build/*
-doc/.build/*
-env/*
-env2/*
-env3/*
-wenv2/*
-wenv3/*
-node_modules/*
-.eggs/*
-browsepy.build/*
-browsepy.egg-info/*
-**/__pycache__/*
-**.egg/*
-**.eggs/*
-**.egg-info/*
+htmlcov
+dist
+build
+doc/.build
+env
+env2
+env3
+wenv2
+wenv3
+penv2
+penv3
+.eggs
+browsepy.build
+browsepy.egg-info
+**/__pycache__
+**.egg
+**.eggs
+**.egg-info
**.pyc
-**/node_modules/*
+**/node_modules
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 5ee9f88..6bd7720 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -27,6 +27,61 @@ try:
except ImportError:
from backports.shutil_get_terminal_size import get_terminal_size # noqa
+try:
+ from queue import Queue, Empty, Full
+except ImportError:
+ from Queue import Queue, Empty, Full # noqa
+
+try:
+ from threading import Barrier, BrokenBarrierError
+except ImportError:
+ from threading import Lock # noqa
+
+ class BrokenBarrierError(RuntimeError):
+ pass
+
+ class Barrier(object):
+ def __init__(self, parties, action=None, timeout=0):
+ self.parties = parties
+ self.n_waiting = 0
+ self.broken = False
+ self._queue = Queue(maxsize=1)
+ self._lock = Lock()
+
+ def _uncork(self, broken=False):
+ # allow every exit
+ while self.n_waiting > 1:
+ self._queue.put(broken)
+ self.n_waiting -= 1
+ self.n_waiting = 0
+
+ def reset(self):
+ with self._lock:
+ self._uncork(True)
+
+ def abort(self):
+ with self._lock:
+ self.broken = True
+ self._uncork(True)
+
+ def wait(self):
+ with self._lock:
+ if self.broken:
+ raise BrokenBarrierError()
+
+ self.n_waiting += 1
+
+ if self.n_waiting == self.parties:
+ self._uncork()
+
+ if self.action:
+ self.action()
+
+ return
+
+ if self._queue.get():
+ raise BrokenBarrierError()
+
def isexec(path):
'''
diff --git a/browsepy/stream.py b/browsepy/stream.py
index 5fb0189..fe58b55 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -2,56 +2,136 @@
import os
import os.path
import tarfile
-import functools
import threading
+import browsepy.compat as compat
+
+
+class WriteAbort(RuntimeError):
+ def __init__(self):
+ print('abort')
+ super(WriteAbort, self).__init__()
+
+
+class WritableQueue(object):
+ '''
+ Minimal threading blocking pipe threading with only `write`, `get` and
+ `close` methods.
+
+ This class includes :class:`queue.Queue` specific logic, which itself
+ depends on :module:`threading` so any alternate
+
+ Method `write` is exposed instead of `put` for :class:`tarfile.TarFile`
+ `fileobj` compatibility.
+ '''
+ abort_exception = WriteAbort
+
+ def __init__(self):
+ self._queue = compat.Queue(maxsize=1)
+ self.closed = False
+
+ def write(self, data):
+ '''
+ Put chunk of data onto pipe.
+ This method blocks if pipe is already full.
+
+ :param data: bytes to write to pipe
+ :type data: bytes
+ :returns: number of bytes written
+ :rtype: int
+ :raises WriteAbort: if already closed or closed while blocking
+ '''
+ if self.closed:
+ raise self.abort_exception()
+
+ self._queue.put(data)
+
+ if self.closed:
+ raise self.abort_exception()
+
+ return len(data)
+
+ def retrieve(self):
+ '''
+ Get chunk of data from pipe.
+ This method blocks if pipe is empty.
+
+ :returns: data chunk
+ :rtype: bytes
+ :raises WriteAbort: if already closed or closed while blocking
+ '''
+ if self.closed:
+ raise self.abort_exception()
+
+ data = self._queue.get()
+
+ if self.closed and data is None:
+ raise self.abort_exception()
+
+ return data
+
+ def close(self):
+ '''
+ Closes, so any blocked and future writes or retrieves will raise
+ :attr:`abort_exception` instances.
+ '''
+ self.closed = True
+
class TarFileStream(object):
'''
Tarfile which compresses while reading for streaming.
- Buffsize can be provided, it must be 512 multiple (the tar block size) for
- compression.
+ Buffsize can be provided, it should be 512 multiple (the tar block size)
+ for and will be used as tarfile block size.
Note on corroutines: this class uses threading by default, but
corroutine-based applications can change this behavior overriding the
:attr:`event_class` and :attr:`thread_class` values.
'''
- event_class = threading.Event
+
+ writable_class = WritableQueue
thread_class = threading.Thread
tarfile_class = tarfile.open
- def __init__(self, path, buffsize=10240, exclude=None):
+ extensions = {
+ 'gz': 'tgz',
+ 'bz2': 'tar.bz2',
+ 'xz': 'tar.xz',
+ }
+
+ @property
+ def name(self):
+ return '%s.%s' % (
+ os.path.basename(self.path),
+ self.extensions.get(self._compress, 'tar')
+ )
+
+ def __init__(self, path, buffsize=10240, exclude=None, compress='gz'):
'''
- Internal tarfile object will be created, and compression will start
- on a thread until buffer became full with writes becoming locked until
- a read occurs.
+ Compression will start a thread, and will be pausing until consumed.
+
+ Note that compress parameter will be ignored if buffsize is below 16.
:param path: local path of directory whose content will be compressed.
:type path: str
- :param buffsize: size of internal buffer on bytes, defaults to 10KiB
+ :param buffsize: byte size of tarfile blocks, defaults to 10KiB
:type buffsize: int
- :param exclude: path filter function, defaults to None
+ :param exclude: absolute path filtering function, defaults to None
:type exclude: callable
+ :param compress: compression method ('gz', 'bz2', 'xz' or None)
+ :type compress: str or None
'''
self.path = path
- self.name = os.path.basename(path) + ".tgz"
self.exclude = exclude
- self._finished = 0
- self._want = 0
- self._data = bytes()
- self._add = self.event_class()
- self._result = self.event_class()
- self._tarfile = self.tarfile_class( # stream write
- fileobj=self,
- mode="w|gz",
- bufsize=buffsize
- )
- self._th = self.thread_class(target=self.fill)
- self._th.start()
+ self._started = False
+ self._buffsize = buffsize
+ self._compress = compress if compress and buffsize > 15 else ''
+ self._writable = self.writable_class()
+ self._th = self.thread_class(target=self._fill)
- def fill(self):
+ def _fill(self):
'''
Writes data on internal tarfile instance, which writes to current
object, using :meth:`write`.
@@ -61,88 +141,57 @@ class TarFileStream(object):
This method is called automatically, on a thread, on initialization,
so there is little need to call it manually.
'''
- if self.exclude:
- exclude = self.exclude
- ap = functools.partial(os.path.join, self.path)
- self._tarfile.add(
- self.path, "",
- filter=lambda info: None if exclude(ap(info.name)) else info
- )
- else:
- self._tarfile.add(self.path, "")
- self._tarfile.close() # force stream flush
- self._finished += 1
- if not self._result.is_set():
- self._result.set()
+ exclude = self.exclude
+ path_join = os.path.join
+ path = self.path
- def write(self, data):
- '''
- Write method used by internal tarfile instance to output data.
- This method blocks tarfile execution once internal buffer is full.
+ def infofilter(info):
+ return None if exclude(path_join(path, info.name)) else info
- As this method is blocking, it is used inside the same thread of
- :meth:`fill`.
+ tarfile = self.tarfile_class( # stream write
+ fileobj=self._writable,
+ mode='w|{}'.format(self._compress),
+ bufsize=self._buffsize
+ )
- :param data: bytes to write to internal buffer
- :type data: bytes
- :returns: number of bytes written
- :rtype: int
- '''
- self._add.wait()
- self._data += data
- if len(self._data) > self._want:
- self._add.clear()
- self._result.set()
- return len(data)
+ try:
+ tarfile.add(self.path, "", filter=infofilter if exclude else None)
+ tarfile.close() # force stream flush
+ except self._writable.abort_exception:
+ pass
+ else:
+ self._writable.close()
- def read(self, want=0):
+ def __next__(self):
'''
- Read method, gets data from internal buffer while releasing
- :meth:`write` locks when needed.
-
- The lock usage means it must ran on a different thread than
- :meth:`fill`, ie. the main thread, otherwise will deadlock.
-
- The combination of both write and this method running on different
- threads makes tarfile being streamed on-the-fly, with data chunks being
- processed and retrieved on demand.
+ Pulls chunk from tarfile (which is processed on its own thread).
:param want: number bytes to read, defaults to 0 (all available)
:type want: int
:returns: tarfile data as bytes
:rtype: bytes
'''
- if self._finished:
- if self._finished == 1:
- self._finished += 1
- return ""
- return EOFError("EOF reached")
-
- # Thread communication
- self._want = want
- self._add.set()
- self._result.wait()
- self._result.clear()
-
- if want:
- data = self._data[:want]
- self._data = self._data[want:]
- else:
- data = self._data
- self._data = bytes()
- return data
+ if not self._started:
+ self._started = True
+ self._th.start()
+
+ try:
+ return self._writable.retrieve()
+ except self._writable.abort_exception:
+ raise StopIteration()
def __iter__(self):
'''
- Iterate through tarfile result chunks.
+ This class itself implements iterable protocol, so iter() returns
+ this instance itself.
- Similarly to :meth:`read`, this methos must ran on a different thread
- than :meth:`write` calls.
+ :returns: instance itself
+ :rtype: TarFileStream
+ '''
+ return self
- :yields: data chunks as taken from :meth:`read`.
- :ytype: bytes
+ def close(self):
+ '''
+ Finish processing aborting any pending write.
'''
- data = self.read()
- while data:
- yield data
- data = self.read()
+ self._writable.close()
--
GitLab
From 9b9199117df453438f1ffb5ae080c17fc66746b0 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 12 Apr 2018 09:42:20 +0200
Subject: [PATCH 018/171] harden tarfile stream implementation
---
browsepy/stream.py | 96 +++++++++++++++++++++--------------
browsepy/tests/test_stream.py | 43 ++++++++++++++++
2 files changed, 102 insertions(+), 37 deletions(-)
create mode 100644 browsepy/tests/test_stream.py
diff --git a/browsepy/stream.py b/browsepy/stream.py
index fe58b55..0d33891 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -3,31 +3,41 @@ import os
import os.path
import tarfile
import threading
+import functools
import browsepy.compat as compat
-class WriteAbort(RuntimeError):
- def __init__(self):
- print('abort')
- super(WriteAbort, self).__init__()
-
+class BlockingPipeAbort(RuntimeError):
+ '''
+ Exception used internally be default's :class:`BlockingPipe`
+ implementation.
+ '''
+ pass
-class WritableQueue(object):
+class BlockingPipe(object):
'''
- Minimal threading blocking pipe threading with only `write`, `get` and
- `close` methods.
+ Minimal pipe class with `write`, `retrieve` and `close` blocking methods.
- This class includes :class:`queue.Queue` specific logic, which itself
- depends on :module:`threading` so any alternate
+ This class implementation assumes that :attr:`pipe_class` (set as
+ class:`queue.Queue` in current implementation) instances has both `put`
+ and `get blocking methods.
- Method `write` is exposed instead of `put` for :class:`tarfile.TarFile`
+ Due its blocking implementation, this class is only compatible with
+ python's threading module, any other approach (like coroutines) will
+ require to adapt this class (via inheritance or implementation).
+
+ This class exposes :method:`write` for :class:`tarfile.TarFile`
`fileobj` compatibility.
- '''
- abort_exception = WriteAbort
+ '''''
+ lock_class = threading.Lock
+ pipe_class = functools.partial(compat.Queue, maxsize=1)
+ abort_exception = BlockingPipeAbort
def __init__(self):
- self._queue = compat.Queue(maxsize=1)
+ self._pipe = self.pipe_class()
+ self._wlock = self.lock_class()
+ self._rlock = self.lock_class()
self.closed = False
def write(self, data):
@@ -41,15 +51,12 @@ class WritableQueue(object):
:rtype: int
:raises WriteAbort: if already closed or closed while blocking
'''
- if self.closed:
- raise self.abort_exception()
-
- self._queue.put(data)
- if self.closed:
- raise self.abort_exception()
-
- return len(data)
+ with self._wlock:
+ if self.closed:
+ raise self.abort_exception()
+ self._pipe.put(data)
+ return len(data)
def retrieve(self):
'''
@@ -60,15 +67,14 @@ class WritableQueue(object):
:rtype: bytes
:raises WriteAbort: if already closed or closed while blocking
'''
- if self.closed:
- raise self.abort_exception()
-
- data = self._queue.get()
-
- if self.closed and data is None:
- raise self.abort_exception()
- return data
+ with self._rlock:
+ if self.closed:
+ raise self.abort_exception()
+ data = self._pipe.get()
+ if data is None:
+ raise self.abort_exception()
+ return data
def close(self):
'''
@@ -77,6 +83,20 @@ class WritableQueue(object):
'''
self.closed = True
+ # release locks
+ reading = not self._rlock.acquire(blocking=False)
+ writing = not self._wlock.acquire(blocking=False)
+
+ if not reading:
+ if writing:
+ self._pipe.get()
+ self._rlock.release()
+
+ if not writing:
+ if reading:
+ self._pipe.put(None)
+ self._wlock.release()
+
class TarFileStream(object):
'''
@@ -90,7 +110,8 @@ class TarFileStream(object):
:attr:`event_class` and :attr:`thread_class` values.
'''
- writable_class = WritableQueue
+ pipe_class = BlockingPipe
+ abort_exception = BlockingPipe.abort_exception
thread_class = threading.Thread
tarfile_class = tarfile.open
@@ -128,7 +149,7 @@ class TarFileStream(object):
self._started = False
self._buffsize = buffsize
self._compress = compress if compress and buffsize > 15 else ''
- self._writable = self.writable_class()
+ self._writable = self.pipe_class()
self._th = self.thread_class(target=self._fill)
def _fill(self):
@@ -148,7 +169,7 @@ class TarFileStream(object):
def infofilter(info):
return None if exclude(path_join(path, info.name)) else info
- tarfile = self.tarfile_class( # stream write
+ tarfile = self.tarfile_class(
fileobj=self._writable,
mode='w|{}'.format(self._compress),
bufsize=self._buffsize
@@ -157,10 +178,11 @@ class TarFileStream(object):
try:
tarfile.add(self.path, "", filter=infofilter if exclude else None)
tarfile.close() # force stream flush
- except self._writable.abort_exception:
- pass
+ except self.abort_exception:
+ # expected exception when pipe is closed prematurely
+ tarfile.close() # free fd
else:
- self._writable.close()
+ self.close()
def __next__(self):
'''
@@ -177,7 +199,7 @@ class TarFileStream(object):
try:
return self._writable.retrieve()
- except self._writable.abort_exception:
+ except self.abort_exception:
raise StopIteration()
def __iter__(self):
diff --git a/browsepy/tests/test_stream.py b/browsepy/tests/test_stream.py
new file mode 100644
index 0000000..ee9bac2
--- /dev/null
+++ b/browsepy/tests/test_stream.py
@@ -0,0 +1,43 @@
+
+import os
+import os.path
+import codecs
+import unittest
+import tempfile
+import shutil
+import threading
+
+import browsepy.stream
+
+
+class StreamTest(unittest.TestCase):
+ module = browsepy.stream
+
+ def setUp(self):
+ self.base = tempfile.mkdtemp()
+
+ def tearDown(self):
+ shutil.rmtree(self.base)
+
+ def randfile(self, size=1024):
+ name = codecs.encode(os.urandom(5), 'hex').decode()
+ with open(os.path.join(self.base, name), 'wb') as f:
+ f.write(os.urandom(size))
+
+ def test_chunks(self):
+ self.randfile()
+ self.randfile()
+ stream = self.module.TarFileStream(self.base, buffsize=5)
+ self.assertTrue(next(stream))
+
+ with self.assertRaises(StopIteration):
+ while True:
+ next(stream)
+
+ def test_close(self):
+ self.randfile()
+ stream = self.module.TarFileStream(self.base, buffsize=16)
+ self.assertTrue(next(stream))
+ stream.close()
+ with self.assertRaises(StopIteration):
+ next(stream)
--
GitLab
From 28a13411337e7b04fde303bbe6bf9e2c72ade660 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 12 Apr 2018 13:38:57 +0200
Subject: [PATCH 019/171] implement TarFileStream as a full Generator
---
.python-version | 2 +-
browsepy/compat.py | 55 ++----------------
browsepy/mimetype.py | 2 +-
browsepy/stream.py | 95 ++++++++++++++++++--------------
browsepy/tests/test_stream.py | 1 -
browsepy/tests/test_transform.py | 2 +-
requirements.txt | 3 +-
setup.py | 19 +++++--
8 files changed, 78 insertions(+), 101 deletions(-)
diff --git a/.python-version b/.python-version
index 87ce492..0f44168 100644
--- a/.python-version
+++ b/.python-version
@@ -1 +1 @@
-3.5.2
+3.6.4
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 6bd7720..2cc3a5e 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -18,69 +18,24 @@ PY_LEGACY = sys.version_info < (3, )
TRUE_VALUES = frozenset(('true', 'yes', '1', 'enable', 'enabled', True, 1))
try:
- from os import scandir, walk
+ from os import scandir, walk # python 3.5+
except ImportError:
from scandir import scandir, walk # noqa
try:
- from shutil import get_terminal_size
+ from shutil import get_terminal_size # python 3.3+
except ImportError:
from backports.shutil_get_terminal_size import get_terminal_size # noqa
try:
- from queue import Queue, Empty, Full
+ from queue import Queue, Empty, Full # python 3
except ImportError:
from Queue import Queue, Empty, Full # noqa
try:
- from threading import Barrier, BrokenBarrierError
+ from collections.abc import Generator # python 3.3+
except ImportError:
- from threading import Lock # noqa
-
- class BrokenBarrierError(RuntimeError):
- pass
-
- class Barrier(object):
- def __init__(self, parties, action=None, timeout=0):
- self.parties = parties
- self.n_waiting = 0
- self.broken = False
- self._queue = Queue(maxsize=1)
- self._lock = Lock()
-
- def _uncork(self, broken=False):
- # allow every exit
- while self.n_waiting > 1:
- self._queue.put(broken)
- self.n_waiting -= 1
- self.n_waiting = 0
-
- def reset(self):
- with self._lock:
- self._uncork(True)
-
- def abort(self):
- with self._lock:
- self.broken = True
- self._uncork(True)
-
- def wait(self):
- with self._lock:
- if self.broken:
- raise BrokenBarrierError()
-
- self.n_waiting += 1
-
- if self.n_waiting == self.parties:
- self._uncork()
-
- if self.action:
- self.action()
-
- return
-
- if self._queue.get():
- raise BrokenBarrierError()
+ from backports_abc import Generator # noqa
def isexec(path):
diff --git a/browsepy/mimetype.py b/browsepy/mimetype.py
index 3fe83e0..b908dd2 100644
--- a/browsepy/mimetype.py
+++ b/browsepy/mimetype.py
@@ -8,7 +8,7 @@ import mimetypes
from .compat import FileNotFoundError, which # noqa
generic_mimetypes = frozenset(('application/octet-stream', None))
-re_mime_validate = re.compile('\w+/\w+(; \w+=[^;]+)*')
+re_mime_validate = re.compile(r'\w+/\w+(; \w+=[^;]+)*')
def by_python(path):
diff --git a/browsepy/stream.py b/browsepy/stream.py
index 0d33891..1a9ea00 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -10,11 +10,12 @@ import browsepy.compat as compat
class BlockingPipeAbort(RuntimeError):
'''
- Exception used internally be default's :class:`BlockingPipe`
+ Exception used internally by :class:`BlockingPipe`'s default
implementation.
'''
pass
+
class BlockingPipe(object):
'''
Minimal pipe class with `write`, `retrieve` and `close` blocking methods.
@@ -23,9 +24,7 @@ class BlockingPipe(object):
class:`queue.Queue` in current implementation) instances has both `put`
and `get blocking methods.
- Due its blocking implementation, this class is only compatible with
- python's threading module, any other approach (like coroutines) will
- require to adapt this class (via inheritance or implementation).
+ Due its blocking implementation, this class uses :module:`threading`.
This class exposes :method:`write` for :class:`tarfile.TarFile`
`fileobj` compatibility.
@@ -76,38 +75,48 @@ class BlockingPipe(object):
raise self.abort_exception()
return data
+ def __del__(self):
+ '''
+ Call :method:`BlockingPipe.close`.
+ '''
+ self.close()
+
def close(self):
'''
Closes, so any blocked and future writes or retrieves will raise
:attr:`abort_exception` instances.
'''
- self.closed = True
+ if not self.closed:
+ self.closed = True
- # release locks
- reading = not self._rlock.acquire(blocking=False)
- writing = not self._wlock.acquire(blocking=False)
+ # release locks
+ reading = not self._rlock.acquire(blocking=False)
+ writing = not self._wlock.acquire(blocking=False)
- if not reading:
- if writing:
- self._pipe.get()
- self._rlock.release()
+ if not reading:
+ if writing:
+ self._pipe.get()
+ self._rlock.release()
- if not writing:
- if reading:
- self._pipe.put(None)
- self._wlock.release()
+ if not writing:
+ if reading:
+ self._pipe.put(None)
+ self._wlock.release()
-class TarFileStream(object):
+class TarFileStream(compat.Generator):
'''
- Tarfile which compresses while reading for streaming.
+ Iterable/generator class which yields tarfile chunks for streaming.
+
+ This class implements :class:`collections.abc.Generator` interface
+ (`PEP 325 `_),
+ so it can be appropriately handled by wsgi servers
+ (`PEP 333`_).
Buffsize can be provided, it should be 512 multiple (the tar block size)
for and will be used as tarfile block size.
- Note on corroutines: this class uses threading by default, but
- corroutine-based applications can change this behavior overriding the
- :attr:`event_class` and :attr:`thread_class` values.
+ This class uses :module:`threading` for offloading.
'''
pipe_class = BlockingPipe
@@ -130,8 +139,7 @@ class TarFileStream(object):
def __init__(self, path, buffsize=10240, exclude=None, compress='gz'):
'''
- Compression will start a thread, and will be pausing until consumed.
-
+ Initialize thread and class (thread is not started until interated.)
Note that compress parameter will be ignored if buffsize is below 16.
:param path: local path of directory whose content will be compressed.
@@ -149,7 +157,7 @@ class TarFileStream(object):
self._started = False
self._buffsize = buffsize
self._compress = compress if compress and buffsize > 15 else ''
- self._writable = self.pipe_class()
+ self._pipe = self.pipe_class()
self._th = self.thread_class(target=self._fill)
def _fill(self):
@@ -170,7 +178,7 @@ class TarFileStream(object):
return None if exclude(path_join(path, info.name)) else info
tarfile = self.tarfile_class(
- fileobj=self._writable,
+ fileobj=self._pipe,
mode='w|{}'.format(self._compress),
bufsize=self._buffsize
)
@@ -184,7 +192,7 @@ class TarFileStream(object):
else:
self.close()
- def __next__(self):
+ def send(self, value):
'''
Pulls chunk from tarfile (which is processed on its own thread).
@@ -198,22 +206,29 @@ class TarFileStream(object):
self._th.start()
try:
- return self._writable.retrieve()
+ return self._pipe.retrieve()
except self.abort_exception:
raise StopIteration()
- def __iter__(self):
+ def throw(self, typ, val=None, tb=None):
'''
- This class itself implements iterable protocol, so iter() returns
- this instance itself.
-
- :returns: instance itself
- :rtype: TarFileStream
- '''
- return self
-
- def close(self):
- '''
- Finish processing aborting any pending write.
+ Raise an exception in the coroutine.
+ Return next yielded value or raise StopIteration.
'''
- self._writable.close()
+ try:
+ if val is None:
+ if tb is None:
+ raise typ
+ val = typ()
+ if tb is not None:
+ val = val.with_traceback(tb)
+ raise val
+ except GeneratorExit:
+ self._pipe.close()
+ raise
+
+ def __del__(self):
+ '''
+ Call :method:`TarFileStream.close`,
+ '''
+ self.close()
diff --git a/browsepy/tests/test_stream.py b/browsepy/tests/test_stream.py
index ee9bac2..144ab8f 100644
--- a/browsepy/tests/test_stream.py
+++ b/browsepy/tests/test_stream.py
@@ -5,7 +5,6 @@ import codecs
import unittest
import tempfile
import shutil
-import threading
import browsepy.stream
diff --git a/browsepy/tests/test_transform.py b/browsepy/tests/test_transform.py
index 32b06ea..ac3a16b 100755
--- a/browsepy/tests/test_transform.py
+++ b/browsepy/tests/test_transform.py
@@ -100,7 +100,7 @@ class TestGlob(unittest.TestCase):
translations = [
('[[.a-acute.]]a', '/.a(/|$)'),
('/[[=a=]]a', '^/.a(/|$)'),
- ('/[[=a=]\d]a', '^/.a(/|$)'),
+ ('/[[=a=]\\d]a', '^/.a(/|$)'),
('[[:non-existent-class:]]a', '/.a(/|$)'),
]
for source, result in translations:
diff --git a/requirements.txt b/requirements.txt
index 5e7f6f4..d072e40 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,13 +6,14 @@ scandir
# for python < 3.3
backports.shutil_get_terminal_size
+backports_abc
# for building
setuptools
wheel
# for Makefile tasks (testing, coverage, docs)
-pycodestyle
+pycodestyle<2.4.0
flake8
coverage
pyaml
diff --git a/setup.py b/setup.py
index 9c7882a..af30c56 100644
--- a/setup.py
+++ b/setup.py
@@ -35,10 +35,13 @@ try:
except ImportError:
from distutils.core import setup
-sys_path = sys.path[:]
-sys.path[:] = (os.path.abspath('browsepy'),)
-__import__('__meta__')
-meta = sys.modules['__meta__']
+try:
+ import collections.abc as collections_abc
+except ImportError:
+ collections_abc = None
+
+sys.path[:], sys_path = [os.path.abspath('browsepy')], sys.path[:]
+import __meta__ as meta # noqa
sys.path[:] = sys_path
with open('README.rst') as f:
@@ -46,12 +49,16 @@ with open('README.rst') as f:
extra_requires = []
bdist = 'bdist' in sys.argv or any(a.startswith('bdist_') for a in sys.argv)
-if bdist or not hasattr(os, 'scandir'):
+
+if bdist or not hasattr(os, 'scandir'): # python 3.5+
extra_requires.append('scandir')
-if bdist or not hasattr(shutil, 'get_terminal_size'):
+if bdist or not hasattr(shutil, 'get_terminal_size'): # python 3.3+
extra_requires.append('backports.shutil_get_terminal_size')
+if bdist or not hasattr(collections_abc, 'Generator'): # python 3.3+
+ extra_requires.append('backports_abc')
+
for debugger in ('ipdb', 'pudb', 'pdb'):
opt = '--debug=%s' % debugger
if opt in sys.argv:
--
GitLab
From b5704241351b0fe422ada0e75a1fc88d1dd9db59 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 12 Apr 2018 14:30:11 +0200
Subject: [PATCH 020/171] fix yaml, add ci requirements txt
---
.travis.yml | 16 ++++++++++++----
requirements/ci.txt | 6 ++++++
requirements/development.txt | 2 +-
3 files changed, 19 insertions(+), 5 deletions(-)
create mode 100644 requirements/ci.txt
diff --git a/.travis.yml b/.travis.yml
index 766c851..7e7d543 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,7 +19,8 @@ matrix:
install:
- pip install --upgrade pip
- - pip install ghp-import sphinx flake8 pep8 codecov coveralls .
+ - pip install -r requirements/ci.txt
+ - pip install .
- |
if [ "$eslint" = "yes" ]; then
nvm install stable
@@ -43,9 +44,16 @@ after_success:
- codecov
- coveralls
- |
- if [ "$sphinx" = "yes" ] && [ "$TRAVIS_BRANCH" = "master" ] &&
-[ "$TRAVIS_PULL_REQUEST" = "false" ]; then
- ghp-import --push --force --no-jekyll --branch=gh-pages --remote="https://${GH_TOKEN}@github.com${TRAVIS_REPO_SLUG}.git doc/.build/html"
+ if [ "$sphinx" = "yes" ] && \
+ [ "$TRAVIS_BRANCH" = "master" ] && \
+ [ "$TRAVIS_PULL_REQUEST" = "false" ]; then
+ ghp-import \
+ --push \
+ --force \
+ --no-jekyll \
+ --branch=gh-pages \
+ --remote="https://${GH_TOKEN}@github.com${TRAVIS_REPO_SLUG}.git" \
+ doc/.build/html
fi
notifications:
diff --git a/requirements/ci.txt b/requirements/ci.txt
new file mode 100644
index 0000000..1a30a39
--- /dev/null
+++ b/requirements/ci.txt
@@ -0,0 +1,6 @@
+-r development.txt
+
+# for continuous integration tasks
+ghp-import
+codecov
+coveralls
diff --git a/requirements/development.txt b/requirements/development.txt
index fb9f680..5f03a12 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -5,7 +5,7 @@ setuptools
wheel
# for Makefile tasks (testing, coverage, docs)
-pep8
+pycodestyle<2.4.0
flake8
coverage
pyaml
--
GitLab
From 751e151b788ecfa4707fbef27aa52923a555968c Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 12 Apr 2018 16:56:11 +0200
Subject: [PATCH 021/171] improve/update requirements, fix travis yml
---
.gitignore | 1 +
.travis.yml | 2 +-
Makefile | 45 +++++++++++++++++++-----------------
requirements.txt | 26 +++++++++++++++++++++
requirements/base.txt | 18 +++++++++++++++
requirements/ci.txt | 23 +++++++++++++++++-
requirements/compat.txt | 26 +++++++++++++++++++--
requirements/development.txt | 24 ++++++++++++++++---
setup.cfg | 12 +++++++++-
9 files changed, 148 insertions(+), 29 deletions(-)
diff --git a/.gitignore b/.gitignore
index 93ff4b4..5061893 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
.git/*
.c9/*
.idea/*
+.vscode
.coverage
htmlcov/*
dist/*
diff --git a/.travis.yml b/.travis.yml
index 7e7d543..3ec22de 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -30,7 +30,7 @@ install:
fi
script:
- - make pep8 flake8 coverage
+ - make flake8 coverage
- |
if [ "$eslint" = "yes" ]; then
make eslint
diff --git a/Makefile b/Makefile
index 9994329..1355301 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
-.PHONY: doc clean pep8 coverage travis
+.PHONY: test clean upload doc showdoc eslint pep8 pycodestyle flake8 coverage showcoverage
-test: pep8 flake8 eslint
+test: flake8 eslint
python -c 'import yaml, glob;[yaml.load(open(p)) for p in glob.glob(".*.yml")]'
ifdef debug
python setup.py test --debug=$(debug)
@@ -9,25 +9,27 @@ else
endif
clean:
- rm -rf build dist browsepy.egg-info htmlcov MANIFEST \
- .eggs *.egg .coverage
+ rm -rf \
+ build \
+ dist \
+ browsepy.egg-info
find browsepy -type f -name "*.py[co]" -delete
find browsepy -type d -name "__pycache__" -delete
$(MAKE) -C doc clean
-build-env:
+build/env:
mkdir -p build
- python3 -m venv build/env3
- build/env3/bin/pip install pip --upgrade
- build/env3/bin/pip install wheel
+ python3 -m venv build/env
+ build/env/bin/pip install pip --upgrade
+ build/env/bin/pip install wheel
-build: clean build-env
- build/env3/bin/python setup.py bdist_wheel
- build/env3/bin/python setup.py sdist
+build: clean build/env
+ build/env/bin/python setup.py bdist_wheel
+ build/env/bin/python setup.py sdist
-upload: clean build-env
- build/env3/bin/python setup.py bdist_wheel upload
- build/env3/bin/python setup.py sdist upload
+upload: clean build/env
+ build/env/bin/python setup.py bdist_wheel upload
+ build/env/bin/python setup.py sdist upload
doc:
$(MAKE) -C doc html 2>&1 | grep -v \
@@ -36,17 +38,18 @@ doc:
showdoc: doc
xdg-open file://${CURDIR}/doc/.build/html/index.html >> /dev/null
+eslint:
+ eslint ${CURDIR}/browsepy
+
pep8:
- find browsepy -type f -name "*.py" -exec pep8 --ignore=E123,E126,E121 {} +
+ pycodestyle --show-source browsepy
+ pycodestyle --show-source setup.py
-eslint:
- eslint \
- --ignore-path .gitignore \
- --ignore-pattern *.min.js \
- ${CURDIR}/browsepy
+pycodestyle: pep8
flake8:
- flake8 browsepy/
+ flake8 --show-source browsepy
+ flake8 --show-source setup.py
coverage:
coverage run --source=browsepy setup.py test
diff --git a/requirements.txt b/requirements.txt
index 79239b1..663f429 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,28 @@
+#
+# Default requirements file
+# =========================
+#
+# This file includes all development dependencies, including compatibility
+# modules.
+#
+# If you do not want compatibility dependencies, install
+# `requirements/development.txt` instead.
+#
+# Requirement file inheritance tree
+# ---------------------------------
+#
+# - requirements.txt
+# - requirements/compat.txt
+# - requirements/development.txt
+# - requirements/base.txt
+#
+# See also
+# --------
+#
+# - requirements/ci.txt
+# - requirements/compat.txt
+# - requirements/development.txt
+#
+
-r requirements/compat.txt
-r requirements/development.txt
diff --git a/requirements/base.txt b/requirements/base.txt
index af7a758..65fe872 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -1,2 +1,20 @@
+#
+# Base requirements
+# =================
+#
+# This module lists the strict minimal subset of dependencies required to
+# run this application.
+#
+# Depending on your python environment, optional modules could be also
+# required. They're defined on `requirements/compat.txt`.
+#
+# See also
+# --------
+#
+# - requirements.txt
+# - requirements/compat.txt
+# - requirements/development.txt
+#
+
flask
unicategories
diff --git a/requirements/ci.txt b/requirements/ci.txt
index 1a30a39..a7d54ba 100644
--- a/requirements/ci.txt
+++ b/requirements/ci.txt
@@ -1,6 +1,27 @@
+#
+# Continuous-integration requirements file
+# ========================================
+#
+# This module includes all requirements for continuous-integration tasks
+# (see .travis.yml).
+#
+# Requirement file inheritance tree
+# ---------------------------------
+#
+# - requirements/ci.txt
+# - requirements/development.txt
+# - requirements/base.txt
+#
+# See also
+# --------
+#
+# - requirements.txt
+# - requirements/compat.txt
+# - requirements/development.txt
+#
+
-r development.txt
-# for continuous integration tasks
ghp-import
codecov
coveralls
diff --git a/requirements/compat.txt b/requirements/compat.txt
index e070c36..a593b7a 100644
--- a/requirements/compat.txt
+++ b/requirements/compat.txt
@@ -1,5 +1,27 @@
-# for python < 3.6
+#
+# Compatibility requirements file
+# ===============================
+#
+# This module includes optional dependencies which could be required (or not)
+# based on your python version.
+#
+# They should be forward-compatible so you can install them all if you are not
+# sure.
+#
+# This requirements are conditional dependencies on 'source' distributable
+# packages, but regular dependencies of 'binary' distributable packages
+# (see `setup.py`).
+#
+# See also
+# --------
+#
+# - requirements.txt
+# - requirements/ci.txt
+# - requirements/development.txt
+#
+
+# python < 3.6
scandir
-# for python < 3.3
+# python < 3.3
backports.shutil_get_terminal_size
diff --git a/requirements/development.txt b/requirements/development.txt
index 5f03a12..a40880a 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -1,10 +1,28 @@
+#
+# Development requirements file
+# =============================
+#
+# This module includes only development dependencies for Makefile tasks such as
+# code analysis and building (see Makefile).
+#
+# Requirement file inheritance tree
+# ---------------------------------
+#
+# - requirements/development.txt
+# - requirements/base.txt
+#
+# See also
+# --------
+#
+# - requirements.txt
+# - requirements/ci.txt
+# - requirements/compat.txt
+#
+
-r base.txt
-# for building
setuptools
wheel
-
-# for Makefile tasks (testing, coverage, docs)
pycodestyle<2.4.0
flake8
coverage
diff --git a/setup.cfg b/setup.cfg
index 0de37e5..cd1dd2d 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -5,4 +5,14 @@ description-file = README.rst
universal = 1
[egg_info]
-tag_build =
\ No newline at end of file
+tag_build =
+
+[pycodestyle]
+ignore = E123,E126,E121,W504
+
+[flake8]
+ignore = E123,E126,E121,W504
+max-complexity = 9
+
+[yapf]
+align_closing_bracket_with_visual_indent = true
--
GitLab
From 842b6e6191286fa1ad04fe9b2882fc4062f7e806 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 12 Apr 2018 18:19:02 +0200
Subject: [PATCH 022/171] prevent doc requirements to crash compatibility CI
builds
---
.travis.yml | 4 ++++
requirements/development.txt | 3 ++-
requirements/doc.txt | 26 ++++++++++++++++++++++++++
3 files changed, 32 insertions(+), 1 deletion(-)
create mode 100644 requirements/doc.txt
diff --git a/.travis.yml b/.travis.yml
index 3ec22de..465bcef 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -28,6 +28,10 @@ install:
npm install eslint
export PATH="$PWD/node_modules/.bin:$PATH"
fi
+ - |
+ if [ "$sphinx" = "yes" ]; then
+ pip install -r requirements/doc.txt
+ fi
script:
- make flake8 coverage
diff --git a/requirements/development.txt b/requirements/development.txt
index a40880a..ce54672 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -5,6 +5,8 @@
# This module includes only development dependencies for Makefile tasks such as
# code analysis and building (see Makefile).
#
+# Documentation requirements are on its own file `requirements/doc.txt`.
+#
# Requirement file inheritance tree
# ---------------------------------
#
@@ -27,4 +29,3 @@ pycodestyle<2.4.0
flake8
coverage
pyaml
-sphinx
diff --git a/requirements/doc.txt b/requirements/doc.txt
new file mode 100644
index 0000000..2c2e5f6
--- /dev/null
+++ b/requirements/doc.txt
@@ -0,0 +1,26 @@
+#
+# Documentation requirements file
+# ===============================
+#
+# This module includes only dependencies required to build the documentation.
+#
+# These requirements are kept separated from both development and integration
+# requirement files as they tend to be picky about python versions.
+#
+# Requirement file inheritance tree
+# ---------------------------------
+#
+# - requirements/doc.txt
+# - requirements/base.txt
+#
+# See also
+# --------
+#
+# - requirements.txt
+# - requirements/ci.txt
+# - requirements/compat.txt
+#
+
+-r base.txt
+
+sphinx
--
GitLab
From b2ad981d8edeceebd63dbbbd6d620e44d012b365 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 12 Apr 2018 23:06:56 +0200
Subject: [PATCH 023/171] drop backports_abc and base TarFileStream on Iterator
---
browsepy/compat.py | 17 ++++++++++++-----
browsepy/stream.py | 34 +++++++++++-----------------------
requirements.txt | 1 -
setup.py | 8 --------
4 files changed, 23 insertions(+), 37 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 2cc3a5e..44f7000 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -6,10 +6,8 @@ import os.path
import sys
import abc
import itertools
-
-import warnings
import functools
-
+import warnings
import posixpath
import ntpath
@@ -33,9 +31,9 @@ except ImportError:
from Queue import Queue, Empty, Full # noqa
try:
- from collections.abc import Generator # python 3.3+
+ from collections.abc import Iterator as BaseIterator # python 3.3+
except ImportError:
- from backports_abc import Generator # noqa
+ from collections import Iterator as BaseIterator
def isexec(path):
@@ -328,6 +326,14 @@ def re_escape(pattern, chars=frozenset("()[]{}?*+|^$\\.-#")):
if PY_LEGACY:
+ class Iterator(BaseIterator):
+ def next(self):
+ '''
+ Call :method:`__next__` for compatibility.
+
+ :returns: see :method:`__next__`
+ '''
+ return self.__next__()
class FileNotFoundError(BaseException):
__metaclass__ = abc.ABCMeta
@@ -343,6 +349,7 @@ if PY_LEGACY:
chr = unichr # noqa
bytes = str # noqa
else:
+ Iterator = BaseIterator
FileNotFoundError = FileNotFoundError
range = range
filter = filter
diff --git a/browsepy/stream.py b/browsepy/stream.py
index 1a9ea00..50ea279 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -90,8 +90,8 @@ class BlockingPipe(object):
self.closed = True
# release locks
- reading = not self._rlock.acquire(blocking=False)
- writing = not self._wlock.acquire(blocking=False)
+ reading = not self._rlock.acquire(False)
+ writing = not self._wlock.acquire(False)
if not reading:
if writing:
@@ -104,13 +104,12 @@ class BlockingPipe(object):
self._wlock.release()
-class TarFileStream(compat.Generator):
+class TarFileStream(compat.Iterator):
'''
- Iterable/generator class which yields tarfile chunks for streaming.
+ Iterator class which yields tarfile chunks for streaming.
- This class implements :class:`collections.abc.Generator` interface
- (`PEP 325 `_),
- so it can be appropriately handled by wsgi servers
+ This class implements :class:`collections.abc.Iterator` interface
+ with :method:`close`, so it can be appropriately handled by wsgi servers
(`PEP 333`_).
Buffsize can be provided, it should be 512 multiple (the tar block size)
@@ -192,7 +191,7 @@ class TarFileStream(compat.Generator):
else:
self.close()
- def send(self, value):
+ def __next__(self):
'''
Pulls chunk from tarfile (which is processed on its own thread).
@@ -210,25 +209,14 @@ class TarFileStream(compat.Generator):
except self.abort_exception:
raise StopIteration()
- def throw(self, typ, val=None, tb=None):
+ def close(self):
'''
- Raise an exception in the coroutine.
- Return next yielded value or raise StopIteration.
+ Closes tarfile pipe and stops further processing.
'''
- try:
- if val is None:
- if tb is None:
- raise typ
- val = typ()
- if tb is not None:
- val = val.with_traceback(tb)
- raise val
- except GeneratorExit:
- self._pipe.close()
- raise
+ self._pipe.close()
def __del__(self):
'''
- Call :method:`TarFileStream.close`,
+ Call :method:`TarFileStream.close`.
'''
self.close()
diff --git a/requirements.txt b/requirements.txt
index d072e40..9b6095b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,6 @@ scandir
# for python < 3.3
backports.shutil_get_terminal_size
-backports_abc
# for building
setuptools
diff --git a/setup.py b/setup.py
index af30c56..b2d0ef7 100644
--- a/setup.py
+++ b/setup.py
@@ -35,11 +35,6 @@ try:
except ImportError:
from distutils.core import setup
-try:
- import collections.abc as collections_abc
-except ImportError:
- collections_abc = None
-
sys.path[:], sys_path = [os.path.abspath('browsepy')], sys.path[:]
import __meta__ as meta # noqa
sys.path[:] = sys_path
@@ -56,9 +51,6 @@ if bdist or not hasattr(os, 'scandir'): # python 3.5+
if bdist or not hasattr(shutil, 'get_terminal_size'): # python 3.3+
extra_requires.append('backports.shutil_get_terminal_size')
-if bdist or not hasattr(collections_abc, 'Generator'): # python 3.3+
- extra_requires.append('backports_abc')
-
for debugger in ('ipdb', 'pudb', 'pdb'):
opt = '--debug=%s' % debugger
if opt in sys.argv:
--
GitLab
From ca060378caaa261eb27f2c046988368faf117eef Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 13 Apr 2018 00:02:40 +0200
Subject: [PATCH 024/171] refactor file-actions paste code to reduce complexity
---
browsepy/plugin/file_actions/__init__.py | 54 ++-----------------
browsepy/plugin/file_actions/paste.py | 69 ++++++++++++++++++++++++
2 files changed, 73 insertions(+), 50 deletions(-)
create mode 100644 browsepy/plugin/file_actions/paste.py
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index f318869..5ce4393 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -3,7 +3,6 @@
import os
import os.path
-import shutil
import logging
from flask import Blueprint, render_template, request, redirect, url_for, \
@@ -13,15 +12,15 @@ from werkzeug.exceptions import NotFound
from browsepy import get_cookie_browse_sorting, browse_sortkey_reverse
from browsepy.file import Node, abspath_to_urlpath, secure_filename, \
current_restricted_chars, common_path_separators
-from browsepy.compat import map, re_escape, FileNotFoundError
+from browsepy.compat import re_escape, FileNotFoundError
from browsepy.exceptions import OutsideDirectoryBase
from .clipboard import Clipboard
from .exceptions import FileActionsException, \
- InvalidClipboardModeError, \
InvalidClipboardItemsError, \
InvalidDirnameError, \
DirectoryCreationError
+from .paste import paste_clipboard
__basedir__ = os.path.dirname(os.path.abspath(__file__))
@@ -125,30 +124,6 @@ def selection(path):
@actions.route('/clipboard/paste', defaults={'path': ''})
@actions.route('/clipboard/paste/')
def clipboard_paste(path):
-
- def copy(target, node, join_fnc=os.path.join):
- if node.is_excluded:
- raise OSError(2, os.strerror(2))
- dest = join_fnc(
- target.path,
- target.choose_filename(node.name)
- )
- if node.is_directory:
- shutil.copytree(node.path, dest)
- else:
- shutil.copy2(node.path, dest)
-
- def cut(target, node, join_fnc=os.path.join):
- if node.is_excluded or not node.can_remove:
- code = 2 if node.is_excluded else 1
- raise OSError(code, os.strerror(code))
- if node.parent.path != target.path:
- dest = join_fnc(
- target.path,
- target.choose_filename(node.name)
- )
- shutil.move(node.path, dest)
-
try:
directory = Node.from_urlpath(path)
except OutsideDirectoryBase:
@@ -162,28 +137,7 @@ def clipboard_paste(path):
return NotFound()
clipboard = Clipboard.from_request()
- mode = clipboard.mode
-
- if mode == 'cut':
- paste_fnc = cut
- elif mode == 'copy':
- paste_fnc = copy
- else:
- raise InvalidClipboardModeError(
- path=directory.path,
- clipboard=clipboard,
- mode=mode
- )
-
- issues = []
- clipboard.mode = 'paste' # disables exclusion
- for node in map(Node.from_urlpath, clipboard):
- try:
- paste_fnc(directory, node)
- except BaseException as e:
- issues.append((node, e))
-
- clipboard.mode = mode
+ success, issues = paste_clipboard(directory, clipboard)
if issues:
raise InvalidClipboardItemsError(
path=directory.path,
@@ -191,7 +145,7 @@ def clipboard_paste(path):
issues=issues
)
- if mode == 'cut':
+ if clipboard.mode == 'cut':
clipboard.clear()
response = redirect(url_for('browse', path=directory.urlpath))
diff --git a/browsepy/plugin/file_actions/paste.py b/browsepy/plugin/file_actions/paste.py
new file mode 100644
index 0000000..a1de913
--- /dev/null
+++ b/browsepy/plugin/file_actions/paste.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+
+import os
+import os.path
+import shutil
+import functools
+
+from browsepy.file import Node
+from browsepy.compat import map
+
+from .exceptions import InvalidClipboardModeError
+
+
+def copy(target, node, join_fnc=os.path.join):
+ if node.is_excluded:
+ raise OSError(2, os.strerror(2))
+ dest = join_fnc(
+ target.path,
+ target.choose_filename(node.name)
+ )
+ if node.is_directory:
+ shutil.copytree(node.path, dest)
+ else:
+ shutil.copy2(node.path, dest)
+ return dest
+
+
+def move(target, node, join_fnc=os.path.join):
+ if node.is_excluded or not node.can_remove:
+ code = 2 if node.is_excluded else 1
+ raise OSError(code, os.strerror(code))
+ if node.parent.path != target.path:
+ dest = join_fnc(
+ target.path,
+ target.choose_filename(node.name)
+ )
+ shutil.move(node.path, dest)
+ return dest
+ return node.path
+
+
+def paste_clipboard(target, clipboard):
+ '''
+ Get pasting function for given directory and keyboard.
+ '''
+ mode = clipboard.mode
+ if mode == 'cut':
+ paste_fnc = functools.partial(move, target)
+ elif mode == 'copy':
+ paste_fnc = functools.partial(copy, target)
+ else:
+ raise InvalidClipboardModeError(
+ path=target.path,
+ clipboard=clipboard,
+ mode=mode
+ )
+ success = []
+ issues = []
+ try:
+ clipboard.mode = 'paste' # deactivates excluded_clipboard fnc
+ for node in map(Node.from_urlpath, clipboard):
+ try:
+ success.append(paste_fnc(node))
+ except BaseException as e:
+ issues.append((node, e))
+ finally:
+ clipboard.mode = mode
+ return success, issues
--
GitLab
From fb4fefa96008514930f1ac0f292a3fc41094e38a Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 13 Apr 2018 00:07:41 +0200
Subject: [PATCH 025/171] use hex_codec instead of hex for compatibility
---
browsepy/tests/test_stream.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/browsepy/tests/test_stream.py b/browsepy/tests/test_stream.py
index 144ab8f..cbc5f39 100644
--- a/browsepy/tests/test_stream.py
+++ b/browsepy/tests/test_stream.py
@@ -19,7 +19,7 @@ class StreamTest(unittest.TestCase):
shutil.rmtree(self.base)
def randfile(self, size=1024):
- name = codecs.encode(os.urandom(5), 'hex').decode()
+ name = codecs.encode(os.urandom(5), 'hex_codec').decode()
with open(os.path.join(self.base, name), 'wb') as f:
f.write(os.urandom(size))
--
GitLab
From 9d04f1faad132c865fcd6687131154827b208417 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 13 Apr 2018 21:10:14 +0200
Subject: [PATCH 026/171] send appropiate headers on tarfile responses
---
browsepy/file.py | 21 ++++++++++++++-------
browsepy/http.py | 41 +++++++++++++++++++++++++++++++++++++++++
browsepy/stream.py | 36 ++++++++++++++++++++++++------------
browsepy/widget.py | 2 +-
4 files changed, 80 insertions(+), 20 deletions(-)
create mode 100644 browsepy/http.py
diff --git a/browsepy/file.py b/browsepy/file.py
index 1b3b14c..920bbcc 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -17,6 +17,7 @@ from werkzeug.utils import cached_property
from . import compat
from . import manager
from .compat import range
+from .http import Headers
from .stream import TarFileStream
from .exceptions import OutsideDirectoryBase, OutsideRemovableBase, \
PathTooLongError, FilenameTooLongError
@@ -636,14 +637,20 @@ class Directory(Node):
:returns: Response object
:rtype: flask.Response
'''
-
+ stream = TarFileStream(
+ self.path,
+ self.app.config['directory_tar_buffsize'],
+ self.plugin_manager.check_excluded,
+ )
return self.app.response_class(
- TarFileStream(
- self.path,
- self.app.config['directory_tar_buffsize'],
- self.plugin_manager.check_excluded,
- ),
- mimetype="application/octet-stream"
+ stream,
+ direct_passthrough=True,
+ headers=Headers(
+ content_type=stream.mimetype,
+ content_type_options={'encoding': stream.encoding},
+ content_disposition='attachment',
+ content_disposition_options={'filename': stream.name},
+ )
)
def contains(self, filename):
diff --git a/browsepy/http.py b/browsepy/http.py
new file mode 100644
index 0000000..9fdfada
--- /dev/null
+++ b/browsepy/http.py
@@ -0,0 +1,41 @@
+
+import re
+
+from werkzeug.http import dump_options_header
+from werkzeug.datastructures import Headers as BaseHeaders
+
+
+class Headers(BaseHeaders):
+ '''
+ A wrapper around :class:`werkzeug.datastructures.Headers`, allowing
+ to specify headers with options.
+ '''
+ snake_replace = staticmethod(re.compile(r'(^|-)[a-z]').sub)
+
+ @classmethod
+ def genpair(cls, key, optkey, values):
+ '''
+ Extract value and options from values dict based on given key and
+ options-key.
+
+ :param key: value key
+ :type key: str
+ :param optkey: options key
+ :type optkey: str
+ :param values: value dictionary
+ :type values: dict
+ :returns: tuple of (key, value)
+ :rtype: tuple of str
+ '''
+ return (
+ cls.snake_replace(lambda x: x[0].upper(), key.replace('_', '-')),
+ dump_options_header(values[key], values.get(optkey, {})),
+ )
+
+ def __init__(self, options_suffix='_options', **kwargs):
+ items = [
+ self.genpair(key, '%s_%s' % (key, options_suffix), kwargs)
+ for key in kwargs
+ if not key.endswith(options_suffix)
+ ]
+ return super(Headers, self).__init__(items)
diff --git a/browsepy/stream.py b/browsepy/stream.py
index 50ea279..d48dc47 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -5,7 +5,7 @@ import tarfile
import threading
import functools
-import browsepy.compat as compat
+from . import compat
class BlockingPipeAbort(RuntimeError):
@@ -123,20 +123,29 @@ class TarFileStream(compat.Iterator):
thread_class = threading.Thread
tarfile_class = tarfile.open
- extensions = {
- 'gz': 'tgz',
- 'bz2': 'tar.bz2',
- 'xz': 'tar.xz',
+ mimetype = 'application/x-tar'
+ compresion_modes = {
+ None: ('', 'tar'),
+ 'gzip': ('gz', 'tgz'),
+ 'bzip2': ('bz2', 'tar.bz2'),
+ 'xz': ('xz', 'tar.xz'),
}
@property
def name(self):
- return '%s.%s' % (
- os.path.basename(self.path),
- self.extensions.get(self._compress, 'tar')
- )
+ '''
+ Filename generated from given path and compression method.
+ '''
+ return '%s.%s' % (os.path.basename(self.path), self._extension)
- def __init__(self, path, buffsize=10240, exclude=None, compress='gz'):
+ @property
+ def encoding(self):
+ '''
+ Mimetype parameters (such as encoding).
+ '''
+ return self._compress
+
+ def __init__(self, path, buffsize=10240, exclude=None, compress='gzip'):
'''
Initialize thread and class (thread is not started until interated.)
Note that compress parameter will be ignored if buffsize is below 16.
@@ -155,7 +164,10 @@ class TarFileStream(compat.Iterator):
self._started = False
self._buffsize = buffsize
- self._compress = compress if compress and buffsize > 15 else ''
+
+ self._compress = compress if compress and buffsize > 15 else None
+ self._mode, self._extension = self.compresion_modes[self._compress]
+
self._pipe = self.pipe_class()
self._th = self.thread_class(target=self._fill)
@@ -178,7 +190,7 @@ class TarFileStream(compat.Iterator):
tarfile = self.tarfile_class(
fileobj=self._pipe,
- mode='w|{}'.format(self._compress),
+ mode='w|{}'.format(self._mode),
bufsize=self._buffsize
)
diff --git a/browsepy/widget.py b/browsepy/widget.py
index afbda38..fed6642 100644
--- a/browsepy/widget.py
+++ b/browsepy/widget.py
@@ -55,7 +55,7 @@ class LinkWidget(WidgetBase):
self.icon if self.icon is not None else
'dir-icon' if file.is_directory else
'file-icon',
- )
+ )
return self
--
GitLab
From 944eaac7b204969cdf285fcdd13fe21a064f218a Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Mon, 16 Apr 2018 11:16:49 +0200
Subject: [PATCH 027/171] use backwards-compatible regex match group
---
browsepy/http.py | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/browsepy/http.py b/browsepy/http.py
index 9fdfada..7da14cf 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -8,7 +8,7 @@ from werkzeug.datastructures import Headers as BaseHeaders
class Headers(BaseHeaders):
'''
A wrapper around :class:`werkzeug.datastructures.Headers`, allowing
- to specify headers with options.
+ to specify headers with options on initialization.
'''
snake_replace = staticmethod(re.compile(r'(^|-)[a-z]').sub)
@@ -28,11 +28,19 @@ class Headers(BaseHeaders):
:rtype: tuple of str
'''
return (
- cls.snake_replace(lambda x: x[0].upper(), key.replace('_', '-')),
+ cls.snake_replace(
+ lambda x: x.group(0).upper(),
+ key.replace('_', '-')
+ ),
dump_options_header(values[key], values.get(optkey, {})),
)
def __init__(self, options_suffix='_options', **kwargs):
+ '''
+ :param options_suffix: suffix for header options (default: '_options')
+ :type options_suffix: str
+ :param **kwargs: headers as keyword arguments
+ '''
items = [
self.genpair(key, '%s_%s' % (key, options_suffix), kwargs)
for key in kwargs
--
GitLab
From 4864e3dfdc7ea0a48de6727804b4f1131a6c0e34 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 18 Apr 2018 22:22:33 +0200
Subject: [PATCH 028/171] test tarfilestream filename
---
browsepy/tests/test_file.py | 2 +-
browsepy/tests/test_module.py | 57 ++++++++++++++++++++++++++---------
2 files changed, 44 insertions(+), 15 deletions(-)
diff --git a/browsepy/tests/test_file.py b/browsepy/tests/test_file.py
index 368169c..80f9947 100644
--- a/browsepy/tests/test_file.py
+++ b/browsepy/tests/test_file.py
@@ -304,7 +304,7 @@ class TestFileFunctions(unittest.TestCase):
self.assertRaises(
browsepy.OutsideDirectoryBase,
self.module.relativize_path, '/other', '/parent', '/'
- )
+ )
def test_under_base(self):
self.assertTrue(
diff --git a/browsepy/tests/test_module.py b/browsepy/tests/test_module.py
index b8b1c0a..b43fc9c 100644
--- a/browsepy/tests/test_module.py
+++ b/browsepy/tests/test_module.py
@@ -9,11 +9,13 @@ import tempfile
import tarfile
import io
import mimetypes
+import collections
import flask
import bs4
from werkzeug.exceptions import NotFound
+from werkzeug.http import parse_options_header
import browsepy
import browsepy.file
@@ -40,6 +42,36 @@ class Page(object):
return cls(source, response)
+class DirectoryDownload(Page):
+ file_class = collections.namedtuple('File', ('name', 'size'))
+
+ def __init__(self, filename, content_type, files, response=None):
+ self.filename = filename
+ self.content_type = content_type
+ self.files = files
+ self.response = response
+
+ @classmethod
+ def from_source(cls, source, response=None):
+ iodata = io.BytesIO(source)
+ with tarfile.open('p.tgz', mode="r:gz", fileobj=iodata) as tgz:
+ files = [
+ cls.file_class(member.name, member.size)
+ for member in tgz.getmembers()
+ if member.name
+ ]
+ files.sort()
+ filename = None
+ content_type = None
+ if response:
+ content_type = response.content_type
+ disposition = response.headers.get('Content-Disposition')
+ mode, options = parse_options_header(disposition)
+ if mode == 'attachment' and 'filename' in options:
+ filename = options['filename']
+ return cls(filename, content_type, files, response)
+
+
class ListPage(Page):
def __init__(self, path, directories, files, removable, upload, tarfile,
source, response=None):
@@ -128,6 +160,7 @@ class TestApp(unittest.TestCase):
generic_page_class = Page
list_page_class = ListPage
confirm_page_class = ConfirmPage
+ directory_download_class = DirectoryDownload
page_exceptions = {
404: Page404Exception,
400: Page400Exception,
@@ -198,6 +231,8 @@ class TestApp(unittest.TestCase):
page_class = self.confirm_page_class
elif endpoint == 'sort' and follow_redirects:
page_class = self.list_page_class
+ elif endpoint == 'download_directory':
+ page_class = self.directory_download_class
else:
page_class = self.generic_page_class
with kwargs.pop('client', None) or self.app.test_client() as client:
@@ -391,33 +426,27 @@ class TestApp(unittest.TestCase):
bindata = bytes(range(256))
exclude = self.app.config['exclude_fnc']
- def tarball_files(path):
- page = self.get('download_directory', path=path)
- iodata = io.BytesIO(page.data)
- with tarfile.open('p.tgz', mode="r:gz", fileobj=iodata) as tgz:
- tgz_files = [
- member.name
- for member in tgz.getmembers()
- if member.name
- ]
- tgz_files.sort()
- return tgz_files
-
for path in (binfile, excfile):
with open(path, 'wb') as f:
f.write(bindata)
self.app.config['exclude_fnc'] = None
+ response = self.get('download_directory', path='start')
+ self.assertEqual(response.filename, 'start.tgz')
+ self.assertEqual(response.content_type, 'application/x-tar')
self.assertEqual(
- tarball_files('start'),
+ [f.name for f in response.files],
['testfile.%s' % x for x in ('bin', 'exc', 'txt')]
)
self.app.config['exclude_fnc'] = lambda p: p.endswith('.exc')
+ response = self.get('download_directory', path='start')
+ self.assertEqual(response.filename, 'start.tgz')
+ self.assertEqual(response.content_type, 'application/x-tar')
self.assertEqual(
- tarball_files('start'),
+ [f.name for f in response.files],
['testfile.%s' % x for x in ('bin', 'txt')]
)
--
GitLab
From 34c6feee6837c7fd2400aa81dc489340f82640f7 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 18 Apr 2018 22:55:30 +0200
Subject: [PATCH 029/171] fix tarfile download headers
---
browsepy/http.py | 2 +-
browsepy/tests/test_module.py | 13 ++++++++++---
2 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/browsepy/http.py b/browsepy/http.py
index 7da14cf..6a48e92 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -42,7 +42,7 @@ class Headers(BaseHeaders):
:param **kwargs: headers as keyword arguments
'''
items = [
- self.genpair(key, '%s_%s' % (key, options_suffix), kwargs)
+ self.genpair(key, '%s%s' % (key, options_suffix), kwargs)
for key in kwargs
if not key.endswith(options_suffix)
]
diff --git a/browsepy/tests/test_module.py b/browsepy/tests/test_module.py
index b43fc9c..c2e70ef 100644
--- a/browsepy/tests/test_module.py
+++ b/browsepy/tests/test_module.py
@@ -45,9 +45,10 @@ class Page(object):
class DirectoryDownload(Page):
file_class = collections.namedtuple('File', ('name', 'size'))
- def __init__(self, filename, content_type, files, response=None):
+ def __init__(self, filename, content_type, encoding, files, response=None):
self.filename = filename
self.content_type = content_type
+ self.encoding = encoding
self.files = files
self.response = response
@@ -63,13 +64,17 @@ class DirectoryDownload(Page):
files.sort()
filename = None
content_type = None
+ encoding = None
if response:
- content_type = response.content_type
+ content_type, options = parse_options_header(response.content_type)
+ if 'encoding' in options:
+ encoding = options['encoding']
+
disposition = response.headers.get('Content-Disposition')
mode, options = parse_options_header(disposition)
if mode == 'attachment' and 'filename' in options:
filename = options['filename']
- return cls(filename, content_type, files, response)
+ return cls(filename, content_type, encoding, files, response)
class ListPage(Page):
@@ -435,6 +440,7 @@ class TestApp(unittest.TestCase):
response = self.get('download_directory', path='start')
self.assertEqual(response.filename, 'start.tgz')
self.assertEqual(response.content_type, 'application/x-tar')
+ self.assertEqual(response.encoding, 'gzip')
self.assertEqual(
[f.name for f in response.files],
['testfile.%s' % x for x in ('bin', 'exc', 'txt')]
@@ -445,6 +451,7 @@ class TestApp(unittest.TestCase):
response = self.get('download_directory', path='start')
self.assertEqual(response.filename, 'start.tgz')
self.assertEqual(response.content_type, 'application/x-tar')
+ self.assertEqual(response.encoding, 'gzip')
self.assertEqual(
[f.name for f in response.files],
['testfile.%s' % x for x in ('bin', 'txt')]
--
GitLab
From 457ec2cee6a9b0117972c242ea10f6fa6e793a9c Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sat, 28 Apr 2018 03:36:52 +0200
Subject: [PATCH 030/171] initial datacookie implmentation
---
browsepy/__init__.py | 65 +++---
browsepy/abc.py | 15 ++
browsepy/compat.py | 2 +
browsepy/exceptions.py | 12 ++
browsepy/file.py | 12 +-
browsepy/http.py | 239 ++++++++++++++++++++--
browsepy/plugin/file_actions/clipboard.py | 84 ++------
browsepy/plugin/file_actions/tests.py | 46 +++--
browsepy/tests/test_http.py | 28 +++
browsepy/tests/test_module.py | 8 +-
browsepy/tests/test_transform.py | 1 +
11 files changed, 364 insertions(+), 148 deletions(-)
create mode 100644 browsepy/abc.py
create mode 100644 browsepy/tests/test_http.py
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index 6909a8a..51ce398 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -4,19 +4,19 @@
import logging
import os
import os.path
-import json
-import base64
from flask import Response, request, render_template, redirect, \
url_for, send_from_directory, stream_with_context, \
make_response
from werkzeug.exceptions import NotFound
+from .http import DataCookie
from .appconfig import Flask
from .manager import PluginManager
from .file import Node, secure_filename
from .exceptions import OutsideRemovableBase, OutsideDirectoryBase, \
- InvalidFilenameError, InvalidPathError
+ InvalidFilenameError, InvalidPathError, \
+ InvalidCookieSizeError
from . import compat
from . import __meta__ as meta
@@ -56,21 +56,7 @@ if 'BROWSEPY_SETTINGS' in os.environ:
app.config.from_envvar('BROWSEPY_SETTINGS')
plugin_manager = PluginManager(app)
-
-
-def iter_cookie_browse_sorting(cookies):
- '''
- Get sorting-cookie from cookies dictionary.
-
- :yields: tuple of path and sorting property
- :ytype: 2-tuple of strings
- '''
- try:
- data = cookies.get('browse-sorting', 'e30=').encode('ascii')
- for path, prop in json.loads(base64.b64decode(data).decode('utf-8')):
- yield path, prop
- except (ValueError, TypeError, KeyError) as e:
- logger.exception(e)
+sorting_cookie = DataCookie('browse-sorting')
def get_cookie_browse_sorting(path, default):
@@ -81,9 +67,12 @@ def get_cookie_browse_sorting(path, default):
:rtype: string
'''
if request:
- for cpath, cprop in iter_cookie_browse_sorting(request.cookies):
- if path == cpath:
- return cprop
+ try:
+ for cpath, cprop in sorting_cookie.load_headers(request.headers):
+ if path == cpath:
+ return cprop
+ except BaseException:
+ pass
return default
@@ -92,11 +81,12 @@ def browse_sortkey_reverse(prop):
Get sorting function for directory listing based on given attribute
name, with some caveats:
* Directories will be first.
- * If *name* is given, link widget lowercase text will be used istead.
+ * If *name* is given, link widget lowercase text will be used instead.
* If *size* is given, bytesize will be used.
:param prop: file attribute name
- :returns: tuple with sorting gunction and reverse bool
+ :type prop: str
+ :returns: tuple with sorting function and reverse bool
:rtype: tuple of a dict and a bool
'''
if prop.startswith('-'):
@@ -164,21 +154,26 @@ def sort(property, path):
if not directory.is_directory or directory.is_excluded:
return NotFound()
- data = [
- (cpath, cprop)
- for cpath, cprop in iter_cookie_browse_sorting(request.cookies)
- if cpath != path
- ]
- data.append((path, property))
- raw_data = base64.b64encode(json.dumps(data).encode('utf-8'))
+ data = [(path, property)]
+ try:
+ data[:-1] = [
+ (cpath, cprop)
+ for cpath, cprop in sorting_cookie.load_headers(request.headers)
+ if cpath != path
+ ]
+ except BaseException:
+ pass
- # prevent cookie becoming too large
- while len(raw_data) > 3975: # 4000 - len('browse-sorting=""; Path=/')
- data.pop(0)
- raw_data = base64.b64encode(json.dumps(data).encode('utf-8'))
+ # handle cookie becoming too large
+ while True:
+ try:
+ headers = sorting_cookie.dump_headers(data, request.headers)
+ break
+ except InvalidCookieSizeError:
+ data.pop(0)
response = redirect(url_for(".browse", path=directory.urlpath))
- response.set_cookie('browse-sorting', raw_data)
+ response.headers.extend(headers)
return response
diff --git a/browsepy/abc.py b/browsepy/abc.py
new file mode 100644
index 0000000..1f69753
--- /dev/null
+++ b/browsepy/abc.py
@@ -0,0 +1,15 @@
+
+from abc import ABC
+
+from .compat import NoneType
+
+
+class JSONSerializable(ABC):
+ pass
+
+
+JSONSerializable.register(NoneType)
+JSONSerializable.register(int)
+JSONSerializable.register(float)
+JSONSerializable.register(list)
+JSONSerializable.register(str)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 44f7000..75047fa 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -358,3 +358,5 @@ else:
unicode = str
chr = chr
bytes = bytes
+
+NoneType = type(None)
diff --git a/browsepy/exceptions.py b/browsepy/exceptions.py
index 8a47c03..c72c3df 100644
--- a/browsepy/exceptions.py
+++ b/browsepy/exceptions.py
@@ -69,3 +69,15 @@ class FilenameTooLongError(InvalidFilenameError):
self.limit = limit
super(FilenameTooLongError, self).__init__(
message, path=path, filename=filename)
+
+
+class InvalidCookieSizeError(ValueError):
+ '''
+ Exception raised when a paginated cookie exceeds its maximun page number.
+ '''
+ code = 'invalid-cookie-size'
+ template = 'Value too big to fit in {} cookies.'
+
+ def __init__(self, message=None, max_cookies=None):
+ self.max_cookies = max_cookies
+ message = message or self.template.format(max_cookies=max_cookies)
diff --git a/browsepy/file.py b/browsepy/file.py
index 920bbcc..61c5a37 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -646,10 +646,14 @@ class Directory(Node):
stream,
direct_passthrough=True,
headers=Headers(
- content_type=stream.mimetype,
- content_type_options={'encoding': stream.encoding},
- content_disposition='attachment',
- content_disposition_options={'filename': stream.name},
+ content_type=(
+ stream.mimetype,
+ {'encoding': stream.encoding} if stream.encoding else {},
+ ),
+ content_disposition=(
+ 'attachment',
+ {'filename': stream.name} if stream.name else {},
+ )
)
)
diff --git a/browsepy/http.py b/browsepy/http.py
index 6a48e92..f8140dd 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -1,49 +1,244 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
import re
+import string
+import json
+import base64
+import logging
+import zlib
-from werkzeug.http import dump_options_header
+from werkzeug.http import dump_header, dump_options_header, dump_cookie, \
+ parse_cookie
from werkzeug.datastructures import Headers as BaseHeaders
+from .compat import range, map
+from .exceptions import InvalidCookieSizeError
+
+
+logger = logging.getLogger(__name__)
+
class Headers(BaseHeaders):
'''
A wrapper around :class:`werkzeug.datastructures.Headers`, allowing
to specify headers with options on initialization.
+
+ Headers are provided as keyword arguments while values can be either
+ :type:`str` (no options) or tuple of :type:`str` and :type:`dict`.
'''
snake_replace = staticmethod(re.compile(r'(^|-)[a-z]').sub)
@classmethod
- def genpair(cls, key, optkey, values):
+ def genpair(cls, key, value):
'''
Extract value and options from values dict based on given key and
options-key.
:param key: value key
:type key: str
- :param optkey: options key
- :type optkey: str
- :param values: value dictionary
- :type values: dict
- :returns: tuple of (key, value)
- :rtype: tuple of str
- '''
- return (
- cls.snake_replace(
- lambda x: x.group(0).upper(),
- key.replace('_', '-')
- ),
- dump_options_header(values[key], values.get(optkey, {})),
+ :param value: value or value/options pair
+ :type value: str or pair of (str, dict)
+ :returns: tuple with key and value
+ :rtype: tuple of (str, str)
+ '''
+ rkey = cls.snake_replace(
+ lambda x: x.group(0).upper(),
+ key.replace('_', '-')
+ )
+ rvalue = (
+ dump_header([value])
+ if isinstance(value, str) else
+ dump_options_header(*value)
)
+ return rkey, rvalue
- def __init__(self, options_suffix='_options', **kwargs):
+ def __init__(self, **kwargs):
'''
- :param options_suffix: suffix for header options (default: '_options')
- :type options_suffix: str
- :param **kwargs: headers as keyword arguments
+ :param **kwargs: header and values as keyword arguments
+ :type **kwargs: str or (str, dict)
'''
items = [
- self.genpair(key, '%s%s' % (key, options_suffix), kwargs)
- for key in kwargs
- if not key.endswith(options_suffix)
+ self.genpair(key, value)
+ for key, value in kwargs.items()
]
return super(Headers, self).__init__(items)
+
+
+class DataCookie(object):
+ '''
+ Compressed base64 paginated cookie manager.
+
+ Usage
+ -----
+
+ > from flask import Flask, request, make_response
+ >
+ > app = Flask(__name__)
+ > cookie = DataCookie('my-cookie')
+ >
+ > @app.route('/get')
+ > def get():
+ > return 'Cookie value is %s' % cookie.load(request.cookies)
+ >
+ > @app.route('/set')
+ > def set():
+ > response = make_response('Cookie set')
+ > cookie.dump('setted', response, request.cookies)
+ > return response
+
+ '''
+ NOT_FOUND = object()
+ cookie_path = '/'
+ page_digits = string.digits + string.ascii_letters
+ max_pages = 5
+ max_length = 3990
+ size_error = InvalidCookieSizeError
+ compress_fnc = staticmethod(zlib.compress)
+ decompress_fnc = staticmethod(zlib.decompress)
+ headers_class = BaseHeaders
+
+ def __init__(self, cookie_name, max_pages=None):
+ self.cookie_name = cookie_name
+ self.request_cache_field = '_browsepy.cache.cookie.%s' % cookie_name
+ self.max_pages = max_pages or self.max_pages
+
+ @classmethod
+ def _name_page(cls, page):
+ '''
+ Converts page integer to string, using fewer characters as possible.
+ If string is given, it is returned as is.
+
+ :param page: page number
+ :type page: int or str
+ :return: page id
+ :rtype: str
+ '''
+ if isinstance(page, str):
+ return page
+
+ digits = []
+
+ if page > 1:
+ base = len(cls.page_digits)
+ remaining = page - 1
+ while remaining >= base:
+ remaining, modulus = divmod(remaining, base)
+ digits.append(modulus)
+ digits.append(remaining)
+ digits.reverse()
+
+ return ''.join(map(cls.page_digits.__getitem__, digits))
+
+ def _name_cookie_page(self, page):
+ '''
+ Get name of cookie corresponding to given page.
+
+ By design (see :method:`_name_page`), pages lower than 1 results on
+ cookie names without a page name.
+
+ :param page: page number or name
+ :type page: int or str
+ :returns: cookie name
+ :rtype: str
+ '''
+ return '{}{}'.format(self.cookie_name, self._name_page(page))
+
+ def _available_cookie_size(self, name):
+ '''
+ Get available cookie size for value.
+ '''
+ return self.max_length - len(name + self.cookie_path)
+
+ def _extract_cookies(self, headers):
+ '''
+ Extract relevant cookies from headers.
+ '''
+ regex_page_name = '[%s]' % re.escape(self.page_digits)
+ regex = re.compile('^%s$' % self._name_cookie_page(regex_page_name))
+ return {
+ key: value
+ for header in headers.get_all('cookie')
+ for key, value in parse_cookie(header).items()
+ if regex.match(key)
+ }
+
+ def load_headers(self, headers):
+ '''
+ Parse data from relevant paginated cookie data on request headers.
+
+ :param headers: request headers
+ :type headers: werkzeug.http.Headers
+ :returns: deserialized value
+ :rtype: browsepy.abc.JSONSerializable
+ '''
+ cookies = self._extract_cookies(headers)
+ chunks = []
+ for i in range(self.max_pages):
+ name = self._name_cookie_page(i)
+ cookie = cookies.get(name, '').encode('ascii')
+ chunks.append(cookie)
+ if len(cookie) < self._available_cookie_size(name):
+ break
+ data = b''.join(chunks)
+ try:
+ data = base64.b64decode(data)
+ serialized = self.decompress_fnc(data)
+ return json.loads(serialized.decode('utf-8'))
+ except (json.JSONDecodeError, ValueError, TypeError):
+ return None
+
+ def dump_headers(self, data, headers=None):
+ '''
+ Serialize given object into a :class:`werkzeug.datastructures.Headers`
+ instance.
+
+ :param data: any json-serializable value
+ :type data: browsepy.abc.JSONSerializable
+ :param headers: optional request headers, used to truncate old pages
+ :type headers: werkzeug.http.Headers
+ :return: response headers
+ :rtype: werkzeug.http.Headers
+ '''
+ result = self.headers_class()
+ serialized = self.compress_fnc(json.dumps(data).encode('utf-8'))
+ data = base64.b64encode(serialized)
+ start = 0
+ total = len(data)
+ for i in range(self.max_pages):
+ name = self._name_cookie_page(i)
+ end = start + self._available_cookie_size(name)
+ result.set(name, data[start:end].decode('ascii'))
+ start = end
+ if start > total:
+ # incidentally, an empty page will be added after start == size
+ break
+ else:
+ # pages exhausted, limit reached
+ raise self.size_error(max_cookies=self.max_pages)
+
+ if headers:
+ result.extend(self.truncate_headers(headers, i + 1))
+
+ return headers
+
+ def truncate_headers(self, headers, start=0):
+ '''
+ Evict relevant cookies found on request headers, optionally starting
+ from a given page number.
+
+ :param headers: request headers, required to truncate old pages
+ :type headers: werkzeug.http.Headers
+ :param start: paginated cookie start, defaults to 0
+ :type start: int
+ :return: response headers
+ :rtype: werkzeug.http.Headers
+ '''
+ name_cookie = self._name_cookie_page
+ cookie_names = set(self._extract_cookies(headers))
+ cookie_names.difference_update(name_cookie(i) for i in range(start))
+
+ result = self.headers_class()
+ for name in cookie_names:
+ result.add('Set-Cookie', dump_cookie(name, expires=0))
+ return result
diff --git a/browsepy/plugin/file_actions/clipboard.py b/browsepy/plugin/file_actions/clipboard.py
index 753174e..b023cfa 100644
--- a/browsepy/plugin/file_actions/clipboard.py
+++ b/browsepy/plugin/file_actions/clipboard.py
@@ -6,25 +6,10 @@ import json
import base64
import logging
import hashlib
-import functools
-
-try:
- import lzma
- LZMA_OPTIONS = {
- 'format': lzma.FORMAT_RAW,
- 'filters': [
- {'id': lzma.FILTER_DELTA, 'dist': 5},
- {'id': lzma.FILTER_LZMA2, 'preset': lzma.PRESET_DEFAULT},
- ]
- }
- compress = functools.partial(lzma.compress, **LZMA_OPTIONS)
- decompress = functools.partial(lzma.decompress, **LZMA_OPTIONS)
-except ImportError:
- from zlib import compress, decompress
-
from flask import request
-from browsepy.compat import range
+from browsepy.http import DataCookie
+from browsepy.exceptions import InvalidCookieSizeError
from .exceptions import InvalidClipboardSizeError
@@ -36,11 +21,9 @@ class Clipboard(set):
Clipboard (set) with convenience methods to pick its state from request
cookies and save it to response cookies.
'''
- cookie_secret = os.urandom(256)
- cookie_name = 'clipboard-{:x}'
- cookie_path = '/'
+ data_cookie = DataCookie('clipboard', max_pages=20)
+ secret = os.urandom(256)
request_cache_field = '_browsepy_file_actions_clipboard_cache'
- max_pages = 20
@classmethod
def count(cls, request=request):
@@ -71,57 +54,15 @@ class Clipboard(set):
self = cls()
setattr(request, cls.request_cache_field, self)
try:
- self.__setstate__(cls._read_paginated_cookie(request))
+ self.__setstate__(cls.data_cookie.load_headers(request.headers))
except BaseException:
pass
return self
- @classmethod
- def _paginated_cookie_length(cls, page=0):
- name_fnc = cls.cookie_name.format
- return 3990 - len(name_fnc(page) + cls.cookie_path)
-
- @classmethod
- def _read_paginated_cookie(cls, request=request):
- chunks = []
- if request:
- name_fnc = cls.cookie_name.format
- for i in range(cls.max_pages): # 2 ** 32 - 1
- cookie = request.cookies.get(name_fnc(i), '').encode('ascii')
- chunks.append(cookie)
- if len(cookie) < cls._paginated_cookie_length(i):
- break
- serialized = decompress(base64.b64decode(b''.join(chunks)))
- return json.loads(serialized.decode('utf-8'))
-
- @classmethod
- def _write_paginated_cookie(cls, data, response):
- serialized = compress(json.dumps(data).encode('utf-8'))
- data = base64.b64encode(serialized)
- name_fnc = cls.cookie_name.format
- start = 0
- size = len(data)
- for i in range(cls.max_pages):
- end = cls._paginated_cookie_length(i)
- response.set_cookie(name_fnc(i), data[start:end].decode('ascii'))
- start = end
- if start > size: # we need an empty page after start == size
- return i
- raise InvalidClipboardSizeError(max_cookies=cls.max_pages)
-
- @classmethod
- def _delete_paginated_cookie(cls, response, start=0, request=request):
- name_fnc = cls.cookie_name.format
- for i in range(start, cls.max_pages):
- name = name_fnc(i)
- if name not in request.cookies:
- break
- response.set_cookie(name, '', expires=0)
-
@classmethod
def _signature(cls, items, method):
serialized = json.dumps(items).encode('utf-8')
- data = cls.cookie_secret + method.encode('utf-8') + serialized
+ data = cls.secret + method.encode('utf-8') + serialized
return base64.b64encode(hashlib.sha512(data).digest()).decode('ascii')
def __init__(self, iterable=(), mode='copy'):
@@ -151,8 +92,15 @@ class Clipboard(set):
:param request: optional request, defaults to current flask request
:type request: werkzeug.Request
'''
- start = 0
if self:
data = self.__getstate__()
- start = self._write_paginated_cookie(data, response) + 1
- self._delete_paginated_cookie(response, start, request)
+ try:
+ headers = self.data_cookie.dump_headers(data, request.headers)
+ except InvalidCookieSizeError as e:
+ raise InvalidClipboardSizeError(
+ clipboard=self,
+ max_cookies=e.max_cookies
+ )
+ else:
+ headers = self.data_cookie.truncate_headers(request.headers)
+ response.headers.extend(headers)
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index 82eb014..74dba56 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -10,6 +10,8 @@ import bs4
import flask
from werkzeug.utils import cached_property
+from werkzeug.http import Headers, parse_cookie, dump_cookie, \
+ parse_options_header
import browsepy.plugin.file_actions as file_actions
import browsepy.plugin.file_actions.clipboard as file_actions_clipboard
@@ -17,21 +19,43 @@ import browsepy.plugin.file_actions.exceptions as file_actions_exceptions
import browsepy.file as browsepy_file
import browsepy.manager as browsepy_manager
import browsepy.exceptions as browsepy_exceptions
-import browsepy.compat as compat
import browsepy
-class CookieProxy(object):
+class CookieHeaderMock(object):
+ @property
+ def cookies(self):
+ return {
+ key: value
+ for header in self.headers.get_all('Set-Cookie')
+ for key, value in parse_cookie(header).items()
+ if parse_options_header(header)[1].get('expire', 1) > 0
+ }
+
+ def __init__(self):
+ self.headers = Headers()
+
+ def set_cookie(self, name, value='', **kwargs):
+ self.headers.add('Set-Cookie', dump_cookie(name, value, **kwargs))
+ self.headers.add('Cookie', dump_cookie(name, value))
+
+
+class CookieProxy(CookieHeaderMock):
def __init__(self, client):
self._client = client
+ super(CookieProxy, self).__init__()
@property
def cookies(self):
- return {
+ result = {
cookie.name: cookie.value for cookie in self._client.cookie_jar
}
+ result.update(super(CookieProxy, self).cookies)
+ return result
def set_cookie(self, name, value, **kwargs):
+ super(CookieProxy, self).set_cookie(name, value, **kwargs)
+
for cookie in self._client.cookie_jar:
if cookie.name == name:
self._client.cookie_jar.clear(
@@ -40,17 +64,9 @@ class CookieProxy(object):
self._client.environ_base['REMOTE_ADDR'], name, value, **kwargs)
-class RequestMock(object):
- def __init__(self):
- self.cookies = {}
-
+class RequestMock(CookieHeaderMock):
def set_cookie(self, name, value, expires=sys.maxsize, **kwargs):
- if isinstance(value, compat.bytes):
- value = value.decode('utf-8')
- if expires:
- self.cookies[name] = value
- elif name in self.cookies:
- del self.cookies[name]
+ super(RequestMock, self).set_cookie(name, value, **kwargs)
field = file_actions_clipboard.Clipboard.request_cache_field
setattr(self, field, None)
@@ -515,14 +531,14 @@ class TestClipboard(unittest.TestCase):
)
def test_unreadable(self):
- name = self.module.Clipboard.cookie_name.format(0)
+ name = self.module.Clipboard.data_cookie._name_cookie_page(0)
request = RequestMock()
request.set_cookie(name, 'a')
clipboard = self.module.Clipboard.from_request(request)
self.assertFalse(clipboard)
def test_cookie_cleanup(self):
- name = self.module.Clipboard.cookie_name.format(0)
+ name = self.module.Clipboard.data_cookie._name_cookie_page(0)
request = RequestMock()
request.set_cookie(name, 'value')
clipboard = self.module.Clipboard()
diff --git a/browsepy/tests/test_http.py b/browsepy/tests/test_http.py
new file mode 100644
index 0000000..0048651
--- /dev/null
+++ b/browsepy/tests/test_http.py
@@ -0,0 +1,28 @@
+
+import unittest
+
+import browsepy.http
+
+
+class TestHeaders(unittest.TestCase):
+ module = browsepy.http
+
+ def test_simple(self):
+ headers = self.module.Headers(
+ simple_header='something',
+ other_header='something else',
+ )
+ self.assertEqual(
+ headers.get('Simple-Header'),
+ 'something',
+ )
+
+ def test_options(self):
+ headers = self.module.Headers(
+ option_header=('something', {'option': 1}),
+ other_header='something else',
+ )
+ self.assertEqual(
+ headers.get('Option-Header'),
+ 'something; option=1',
+ )
diff --git a/browsepy/tests/test_module.py b/browsepy/tests/test_module.py
index c2e70ef..5914c8c 100644
--- a/browsepy/tests/test_module.py
+++ b/browsepy/tests/test_module.py
@@ -442,8 +442,8 @@ class TestApp(unittest.TestCase):
self.assertEqual(response.content_type, 'application/x-tar')
self.assertEqual(response.encoding, 'gzip')
self.assertEqual(
- [f.name for f in response.files],
- ['testfile.%s' % x for x in ('bin', 'exc', 'txt')]
+ [f1.name for f1 in response.files],
+ ['testfile.%s' % x1 for x1 in ('bin', 'exc', 'txt')]
)
self.app.config['exclude_fnc'] = lambda p: p.endswith('.exc')
@@ -453,8 +453,8 @@ class TestApp(unittest.TestCase):
self.assertEqual(response.content_type, 'application/x-tar')
self.assertEqual(response.encoding, 'gzip')
self.assertEqual(
- [f.name for f in response.files],
- ['testfile.%s' % x for x in ('bin', 'txt')]
+ [f2.name for f2 in response.files],
+ ['testfile.%s' % x2 for x2 in ('bin', 'txt')]
)
self.app.config['exclude_fnc'] = exclude
diff --git a/browsepy/tests/test_transform.py b/browsepy/tests/test_transform.py
index ac3a16b..ee48793 100755
--- a/browsepy/tests/test_transform.py
+++ b/browsepy/tests/test_transform.py
@@ -78,6 +78,7 @@ class TestGlob(unittest.TestCase):
'/a',
'/ñ',
'/1',
+ b'/\xc3\xa0', # /Ã
), (
'/_',
)),
--
GitLab
From 62b5b6078b190086664665a44f02253269bda72d Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 10 May 2018 00:42:31 +0200
Subject: [PATCH 031/171] reduce complexity
---
browsepy/__init__.py | 36 +++--
browsepy/exceptions.py | 5 +-
browsepy/http.py | 157 +++++++++++++---------
browsepy/plugin/file_actions/__init__.py | 10 +-
browsepy/plugin/file_actions/clipboard.py | 6 +-
browsepy/plugin/file_actions/tests.py | 109 ++++++++-------
6 files changed, 191 insertions(+), 132 deletions(-)
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index 51ce398..aa655bd 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -5,6 +5,8 @@ import logging
import os
import os.path
+from datetime import timedelta
+
from flask import Response, request, render_template, redirect, \
url_for, send_from_directory, stream_with_context, \
make_response
@@ -56,7 +58,21 @@ if 'BROWSEPY_SETTINGS' in os.environ:
app.config.from_envvar('BROWSEPY_SETTINGS')
plugin_manager = PluginManager(app)
-sorting_cookie = DataCookie('browse-sorting')
+sorting_cookie = DataCookie('browse-sorting', max_age=timedelta(days=90))
+
+
+def iter_cookie_browse_sorting(cookies):
+ '''
+ Get sorting-cookie from cookies dictionary.
+
+ :yields: tuple of path and sorting property
+ :ytype: 2-tuple of strings
+ '''
+ try:
+ for path, prop in sorting_cookie.load_cookies(cookies, ()):
+ yield path, prop
+ except ValueError as e:
+ logger.exception(e)
def get_cookie_browse_sorting(path, default):
@@ -67,12 +83,9 @@ def get_cookie_browse_sorting(path, default):
:rtype: string
'''
if request:
- try:
- for cpath, cprop in sorting_cookie.load_headers(request.headers):
- if path == cpath:
- return cprop
- except BaseException:
- pass
+ for cpath, cprop in iter_cookie_browse_sorting(request.cookies):
+ if path == cpath:
+ return cprop
return default
@@ -156,16 +169,17 @@ def sort(property, path):
data = [(path, property)]
try:
- data[:-1] = [
+ data.extend(
(cpath, cprop)
- for cpath, cprop in sorting_cookie.load_headers(request.headers)
+ for cpath, cprop in iter_cookie_browse_sorting(request.cookies)
if cpath != path
- ]
+ )
except BaseException:
pass
# handle cookie becoming too large
- while True:
+ headers = ()
+ while data:
try:
headers = sorting_cookie.dump_headers(data, request.headers)
break
diff --git a/browsepy/exceptions.py b/browsepy/exceptions.py
index c72c3df..1ed1e9d 100644
--- a/browsepy/exceptions.py
+++ b/browsepy/exceptions.py
@@ -76,8 +76,9 @@ class InvalidCookieSizeError(ValueError):
Exception raised when a paginated cookie exceeds its maximun page number.
'''
code = 'invalid-cookie-size'
- template = 'Value too big to fit in {} cookies.'
+ template = 'Value too big to fit in {0.max_cookies} cookies.'
def __init__(self, message=None, max_cookies=None):
self.max_cookies = max_cookies
- message = message or self.template.format(max_cookies=max_cookies)
+ message = self.template.format(self) if message is None else message
+ super(InvalidCookieSizeError, self).__init__(message)
diff --git a/browsepy/http.py b/browsepy/http.py
index f8140dd..052db6e 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -2,7 +2,6 @@
# -*- coding: UTF-8 -*-
import re
-import string
import json
import base64
import logging
@@ -12,7 +11,7 @@ from werkzeug.http import dump_header, dump_options_header, dump_cookie, \
parse_cookie
from werkzeug.datastructures import Headers as BaseHeaders
-from .compat import range, map
+from .compat import range
from .exceptions import InvalidCookieSizeError
@@ -88,47 +87,44 @@ class DataCookie(object):
> return response
'''
- NOT_FOUND = object()
- cookie_path = '/'
- page_digits = string.digits + string.ascii_letters
- max_pages = 5
- max_length = 3990
+ page_length = 4000
size_error = InvalidCookieSizeError
- compress_fnc = staticmethod(zlib.compress)
- decompress_fnc = staticmethod(zlib.decompress)
headers_class = BaseHeaders
- def __init__(self, cookie_name, max_pages=None):
- self.cookie_name = cookie_name
- self.request_cache_field = '_browsepy.cache.cookie.%s' % cookie_name
- self.max_pages = max_pages or self.max_pages
-
- @classmethod
- def _name_page(cls, page):
+ @staticmethod
+ def _serialize(data):
'''
- Converts page integer to string, using fewer characters as possible.
- If string is given, it is returned as is.
-
- :param page: page number
- :type page: int or str
- :return: page id
- :rtype: str
+ :type data: json-serializable
+ :rtype: bytes
'''
- if isinstance(page, str):
- return page
+ serialized = zlib.compress(json.dumps(data).encode('utf-8'))
+ return base64.b64encode(serialized).decode('ascii')
- digits = []
-
- if page > 1:
- base = len(cls.page_digits)
- remaining = page - 1
- while remaining >= base:
- remaining, modulus = divmod(remaining, base)
- digits.append(modulus)
- digits.append(remaining)
- digits.reverse()
+ @staticmethod
+ def _deserialize(data):
+ '''
+ :type data: bytes
+ :rtype: json-serializable
+ '''
+ decoded = base64.b64decode(data)
+ serialized = zlib.decompress(decoded)
+ return json.loads(serialized)
- return ''.join(map(cls.page_digits.__getitem__, digits))
+ def __init__(self, cookie_name, max_pages=1, max_age=None, path='/'):
+ '''
+ :param cookie_name: first cookie name and prefix for the following
+ :type cookie_name: str
+ :param max_pages: maximum allowed cookie parts, defaults to 1
+ :type max_pages: int
+ :param max_age: cookie lifetime in seconds or None (session, default)
+ :type max_age: int, datetime.timedelta or None
+ :param path: cookie path, defaults to /
+ :type path: str
+ '''
+ self.cookie_name = cookie_name
+ self.max_pages = max_pages
+ self.max_age = max_age
+ self.path = path
def _name_cookie_page(self, page):
'''
@@ -142,20 +138,35 @@ class DataCookie(object):
:returns: cookie name
:rtype: str
'''
- return '{}{}'.format(self.cookie_name, self._name_page(page))
+ return '{}{}'.format(
+ self.cookie_name,
+ page if isinstance(page, str) else
+ '-{:x}'.format(page - 1) if page else
+ ''
+ )
def _available_cookie_size(self, name):
'''
Get available cookie size for value.
+
+ :param name: cookie name
+ :type name: str
+ :return: available bytes for cookie value
+ :rtype: int
'''
- return self.max_length - len(name + self.cookie_path)
+ empty = 'Set-Cookie: %s' % dump_cookie(
+ name,
+ value=' ', # force quotes
+ max_age=self.max_age,
+ path=self.path
+ )
+ return self.page_length - len(empty)
def _extract_cookies(self, headers):
'''
Extract relevant cookies from headers.
'''
- regex_page_name = '[%s]' % re.escape(self.page_digits)
- regex = re.compile('^%s$' % self._name_cookie_page(regex_page_name))
+ regex = re.compile('^%s$' % self._name_cookie_page('(-[0-9a-f])?'))
return {
key: value
for header in headers.get_all('cookie')
@@ -163,16 +174,15 @@ class DataCookie(object):
if regex.match(key)
}
- def load_headers(self, headers):
+ def load_cookies(self, cookies, default=None):
'''
- Parse data from relevant paginated cookie data on request headers.
+ Parse data from relevant paginated cookie data given as mapping.
- :param headers: request headers
- :type headers: werkzeug.http.Headers
+ :param cookies: request cookies
+ :type cookies: collections.abc.Mapping
:returns: deserialized value
:rtype: browsepy.abc.JSONSerializable
'''
- cookies = self._extract_cookies(headers)
chunks = []
for i in range(self.max_pages):
name = self._name_cookie_page(i)
@@ -181,12 +191,24 @@ class DataCookie(object):
if len(cookie) < self._available_cookie_size(name):
break
data = b''.join(chunks)
- try:
- data = base64.b64decode(data)
- serialized = self.decompress_fnc(data)
- return json.loads(serialized.decode('utf-8'))
- except (json.JSONDecodeError, ValueError, TypeError):
- return None
+ if data:
+ try:
+ return self._deserialize(data)
+ except BaseException:
+ pass
+ return default
+
+ def load_headers(self, headers, default=None):
+ '''
+ Parse data from relevant paginated cookie data on request headers.
+
+ :param headers: request headers
+ :type headers: werkzeug.http.Headers
+ :returns: deserialized value
+ :rtype: browsepy.abc.JSONSerializable
+ '''
+ cookies = self._extract_cookies(headers)
+ return self.load_cookies(cookies)
def dump_headers(self, data, headers=None):
'''
@@ -201,26 +223,29 @@ class DataCookie(object):
:rtype: werkzeug.http.Headers
'''
result = self.headers_class()
- serialized = self.compress_fnc(json.dumps(data).encode('utf-8'))
- data = base64.b64encode(serialized)
+ data = self._serialize(data)
start = 0
- total = len(data)
+ size = len(data)
for i in range(self.max_pages):
name = self._name_cookie_page(i)
end = start + self._available_cookie_size(name)
- result.set(name, data[start:end].decode('ascii'))
+ result.set(
+ 'Set-Cookie',
+ dump_cookie(
+ name,
+ data[start:end],
+ max_age=self.max_age,
+ path=self.path,
+ )
+ )
+ if end > size:
+ # incidentally, an empty page will be added after end == size
+ if headers:
+ result.extend(self.truncate_headers(headers, i + 1))
+ return result
start = end
- if start > total:
- # incidentally, an empty page will be added after start == size
- break
- else:
- # pages exhausted, limit reached
- raise self.size_error(max_cookies=self.max_pages)
-
- if headers:
- result.extend(self.truncate_headers(headers, i + 1))
-
- return headers
+ # pages exhausted, limit reached
+ raise self.size_error(max_cookies=self.max_pages)
def truncate_headers(self, headers, start=0):
'''
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index 5ce4393..fb27e36 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -18,6 +18,7 @@ from browsepy.exceptions import OutsideDirectoryBase
from .clipboard import Clipboard
from .exceptions import FileActionsException, \
InvalidClipboardItemsError, \
+ InvalidClipboardModeError, \
InvalidDirnameError, \
DirectoryCreationError
from .paste import paste_clipboard
@@ -100,13 +101,18 @@ def selection(path):
mode = action
break
+ clipboard = Clipboard(request.form.getlist('path'), mode)
+
if mode in ('cut', 'copy'):
response = redirect(url_for('browse', path=directory.urlpath))
- clipboard = Clipboard(request.form.getlist('path'), mode)
clipboard.to_response(response)
return response
- return redirect(request.path)
+ raise InvalidClipboardModeError(
+ path=directory.path,
+ clipboard=clipboard,
+ mode=clipboard.mode,
+ )
clipboard = Clipboard.from_request()
clipboard.mode = 'select' # disables exclusion
diff --git a/browsepy/plugin/file_actions/clipboard.py b/browsepy/plugin/file_actions/clipboard.py
index b023cfa..9c0f0b7 100644
--- a/browsepy/plugin/file_actions/clipboard.py
+++ b/browsepy/plugin/file_actions/clipboard.py
@@ -60,9 +60,9 @@ class Clipboard(set):
return self
@classmethod
- def _signature(cls, items, method):
- serialized = json.dumps(items).encode('utf-8')
- data = cls.secret + method.encode('utf-8') + serialized
+ def _signature(cls, items, mode):
+ serialized = json.dumps([items, mode]).encode('utf-8')
+ data = cls.secret + serialized
return base64.b64encode(hashlib.sha512(data).digest()).decode('ascii')
def __init__(self, iterable=(), mode='copy'):
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index 74dba56..ac91556 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -16,6 +16,7 @@ from werkzeug.http import Headers, parse_cookie, dump_cookie, \
import browsepy.plugin.file_actions as file_actions
import browsepy.plugin.file_actions.clipboard as file_actions_clipboard
import browsepy.plugin.file_actions.exceptions as file_actions_exceptions
+import browsepy.http as browsepy_http
import browsepy.file as browsepy_file
import browsepy.manager as browsepy_manager
import browsepy.exceptions as browsepy_exceptions
@@ -23,11 +24,13 @@ import browsepy
class CookieHeaderMock(object):
+ header = 'Cookie'
+
@property
def cookies(self):
return {
key: value
- for header in self.headers.get_all('Set-Cookie')
+ for header in self.headers.get_all(self.header)
for key, value in parse_cookie(header).items()
if parse_options_header(header)[1].get('expire', 1) > 0
}
@@ -36,39 +39,31 @@ class CookieHeaderMock(object):
self.headers = Headers()
def set_cookie(self, name, value='', **kwargs):
- self.headers.add('Set-Cookie', dump_cookie(name, value, **kwargs))
- self.headers.add('Cookie', dump_cookie(name, value))
-
-
-class CookieProxy(CookieHeaderMock):
- def __init__(self, client):
- self._client = client
- super(CookieProxy, self).__init__()
+ self.headers.add(self.header, dump_cookie(name, value))
- @property
- def cookies(self):
- result = {
- cookie.name: cookie.value for cookie in self._client.cookie_jar
- }
- result.update(super(CookieProxy, self).cookies)
- return result
- def set_cookie(self, name, value, **kwargs):
- super(CookieProxy, self).set_cookie(name, value, **kwargs)
+class ResponseMock(CookieHeaderMock):
+ header = 'Set-Cookie'
- for cookie in self._client.cookie_jar:
- if cookie.name == name:
- self._client.cookie_jar.clear(
+ def dump_cookies(self, client):
+ owned = self.cookies
+ for cookie in client.cookie_jar:
+ if cookie.name in owned:
+ client.cookie_jar.clear(
cookie.domain, cookie.path, cookie.name)
- return self._client.set_cookie(
- self._client.environ_base['REMOTE_ADDR'], name, value, **kwargs)
+
+ for name, value in owned.items():
+ client.set_cookie(client.environ_base['REMOTE_ADDR'], name, value)
class RequestMock(CookieHeaderMock):
- def set_cookie(self, name, value, expires=sys.maxsize, **kwargs):
- super(RequestMock, self).set_cookie(name, value, **kwargs)
- field = file_actions_clipboard.Clipboard.request_cache_field
- setattr(self, field, None)
+ def load_cookies(self, client):
+ self.headers.clear()
+ for cookie in client.cookie_jar:
+ self.headers.add(
+ self.header,
+ dump_cookie(cookie.name, cookie.value)
+ )
class Page(object):
@@ -201,11 +196,14 @@ class TestIntegration(unittest.TestCase):
page.widgets
)
- proxy = CookieProxy(client)
- clipboard = self.clipboard_module.Clipboard.from_request(proxy)
+ reqmock = RequestMock()
+ reqmock.load_cookies(client)
+ resmock = ResponseMock()
+ clipboard = self.clipboard_module.Clipboard.from_request(reqmock)
clipboard.mode = 'copy'
clipboard.add('whatever')
- clipboard.to_response(proxy, proxy)
+ clipboard.to_response(resmock, reqmock)
+ resmock.dump_cookies(client)
response = client.get('/')
self.assertEqual(response.status_code, 200)
@@ -253,19 +251,23 @@ class TestIntegration(unittest.TestCase):
page = Page(response.data)
self.assertIn('potato', page.entries)
- proxy = CookieProxy(client)
- clipboard = self.clipboard_module.Clipboard.from_request(proxy)
+ reqmock = RequestMock()
+ reqmock.load_cookies(client)
+ resmock = ResponseMock()
+ clipboard = self.clipboard_module.Clipboard.from_request(reqmock)
clipboard.mode = 'copy'
clipboard.add('potato')
- clipboard.to_response(proxy, proxy)
+ clipboard.to_response(resmock, reqmock)
+ resmock.dump_cookies(client)
response = client.get('/')
self.assertEqual(response.status_code, 200)
page = Page(response.data)
self.assertIn('potato', page.entries)
+ reqmock.load_cookies(client)
clipboard.mode = 'cut'
- clipboard.to_response(proxy, proxy)
+ clipboard.to_response(resmock, reqmock)
response = client.get('/')
self.assertEqual(response.status_code, 200)
@@ -431,35 +433,43 @@ class TestAction(unittest.TestCase):
self.assertEqual(response.status_code, 404)
with self.app.test_client() as client:
- proxy = CookieProxy(client)
- clipboard = self.clipboard_module.Clipboard.from_request(proxy)
+ reqmock = RequestMock()
+ reqmock.load_cookies(client)
+ resmock = ResponseMock()
+ clipboard = self.clipboard_module.Clipboard.from_request(reqmock)
clipboard.mode = 'wrong-mode'
clipboard.add('whatever')
- clipboard.to_response(proxy, proxy)
+ clipboard.to_response(resmock, reqmock)
+ resmock.dump_cookies(client)
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 400)
+ reqmock.load_cookies(client)
clipboard.mode = 'cut'
- clipboard.to_response(proxy, proxy)
+ clipboard.to_response(resmock, reqmock)
+ resmock.dump_cookies(client)
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 302) # same location
clipboard.mode = 'cut'
- clipboard.to_response(proxy, proxy)
+ clipboard.to_response(resmock, reqmock)
+ resmock.dump_cookies(client)
response = client.get('/file-actions/clipboard/paste/target')
self.assertEqual(response.status_code, 400)
clipboard.mode = 'copy'
- clipboard.to_response(proxy, proxy)
+ clipboard.to_response(resmock, reqmock)
+ resmock.dump_cookies(client)
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 400)
clipboard.mode = 'copy'
- clipboard.to_response(proxy, proxy)
+ clipboard.to_response(resmock, reqmock)
+ resmock.dump_cookies(client)
response = client.get('/file-actions/clipboard/paste/target')
self.assertEqual(response.status_code, 400)
@@ -467,13 +477,15 @@ class TestAction(unittest.TestCase):
self.app.config['exclude_fnc'] = lambda n: n.endswith('whatever')
clipboard.mode = 'cut'
- clipboard.to_response(proxy, proxy)
+ clipboard.to_response(resmock, reqmock)
+ resmock.dump_cookies(client)
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 400)
clipboard.mode = 'copy'
- clipboard.to_response(proxy, proxy)
+ clipboard.to_response(resmock, reqmock)
+ resmock.dump_cookies(client)
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 400)
@@ -504,19 +516,20 @@ class TestClipboard(unittest.TestCase):
module = file_actions_clipboard
def test_count(self):
- request = RequestMock()
- self.assertEqual(self.module.Clipboard.count(request), 0)
+ reqmock = RequestMock()
+ resmock = ResponseMock()
+ self.assertEqual(self.module.Clipboard.count(resmock), 0)
clipboard = self.module.Clipboard()
clipboard.mode = 'test'
clipboard.add('item')
- clipboard.to_response(request, request)
+ clipboard.to_response(resmock, reqmock)
- self.assertEqual(self.module.Clipboard.count(request), 1)
+ self.assertEqual(self.module.Clipboard.count(resmock), 1)
def test_oveflow(self):
class TinyClipboard(self.module.Clipboard):
- max_pages = 2
+ data_cookie = browsepy_http.DataCookie('small-clipboard')
request = RequestMock()
clipboard = TinyClipboard()
--
GitLab
From cc842cbc901b9522dee75b706ba9268095e1e067 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 23 May 2018 20:50:51 +0200
Subject: [PATCH 032/171] wip
---
browsepy/plugin/file_actions/tests.py | 112 ++++++++++++++++++++------
browsepy/tests/test_http.py | 30 +++++++
2 files changed, 118 insertions(+), 24 deletions(-)
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index ac91556..4f5773b 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -3,15 +3,15 @@ import tempfile
import shutil
import os
import os.path
-import sys
import functools
+import datetime
import bs4
import flask
from werkzeug.utils import cached_property
from werkzeug.http import Headers, parse_cookie, dump_cookie, \
- parse_options_header
+ parse_options_header, parse_date
import browsepy.plugin.file_actions as file_actions
import browsepy.plugin.file_actions.clipboard as file_actions_clipboard
@@ -25,45 +25,103 @@ import browsepy
class CookieHeaderMock(object):
header = 'Cookie'
+ def parse_cookie(self, header):
+ cookies = parse_cookie(header)
+ if len(cookies) != 1:
+ # multi-cookie header (Cookie), no options
+ return cookies, {}
+ name, value = tuple(cookies.items())[0]
+ cookie, _ = parse_options_header(header)
+ extra = header[len(cookie):]
+
+ for part in .split('; '):
+
+ if part.startswith('Expires='):
+ expire = part[8:].replace('"', '')
+ options['expires'] = parse_date(expire)
+
+
+ def check_expired(self, options):
+
+ if expire and < datetime.datetime.now():
+ return True
+
+ def check_expired(self, header):
+
+
+
+ cookie, options = parse_options_header(header)
+ if 'Expires' in options: # implies Set-Cookie
+
+ if options.get('Max-Age', '1') == '0':
+ return True
+ return False
+
+ @property
+ def _cookies(self):
+ check_expired = self.check_expired
+ return [
+ (name, value, check_expired(header))
+ for header in self.headers.get_all(self.header)
+ for name, value in parse_cookie(header).items()
+ ]
@property
def cookies(self):
return {
- key: value
- for header in self.headers.get_all(self.header)
- for key, value in parse_cookie(header).items()
- if parse_options_header(header)[1].get('expire', 1) > 0
+ name: value
+ for name, value, expired in self._cookies
+ if not expired
}
def __init__(self):
self.headers = Headers()
def set_cookie(self, name, value='', **kwargs):
- self.headers.add(self.header, dump_cookie(name, value))
+ owned = [
+ for name, value, expires in self._cookies
+ if name != name
+ ]
+
+
+ for name, value in owned.items():
+ self.headers.add(self.header, dump_cookie(name, value, **kwargs))
+
+ def clear(self):
+ self.headers.clear()
class ResponseMock(CookieHeaderMock):
header = 'Set-Cookie'
def dump_cookies(self, client):
- owned = self.cookies
- for cookie in client.cookie_jar:
- if cookie.name in owned:
- client.cookie_jar.clear(
- cookie.domain, cookie.path, cookie.name)
-
- for name, value in owned.items():
- client.set_cookie(client.environ_base['REMOTE_ADDR'], name, value)
+ owned = self._cookies
+ if isinstance(client, CookieHeaderMock):
+ client.clear()
+ for name, value in owned.items():
+ client.set_cookie(name, value)
+ else:
+ for cookie in client.cookie_jar:
+ if cookie.name in owned:
+ client.cookie_jar.clear(
+ cookie.domain, cookie.path, cookie.name)
+
+ for name, value in owned.items():
+ client.set_cookie(
+ client.environ_base['REMOTE_ADDR'], name, value)
class RequestMock(CookieHeaderMock):
def load_cookies(self, client):
- self.headers.clear()
+ self.clear()
+
for cookie in client.cookie_jar:
- self.headers.add(
- self.header,
- dump_cookie(cookie.name, cookie.value)
- )
+ self.set_cookie(cookie.name, cookie.value)
+
+ def uncache(self):
+ field = file_actions_clipboard.Clipboard.request_cache_field
+ if hasattr(self, field):
+ delattr(self, field)
class Page(object):
@@ -500,6 +558,7 @@ class TestAction(unittest.TestCase):
'/file-actions/selection',
data={
'path': files,
+ 'action-copy': 'whatever',
})
self.assertEqual(response.status_code, 302)
@@ -518,14 +577,16 @@ class TestClipboard(unittest.TestCase):
def test_count(self):
reqmock = RequestMock()
resmock = ResponseMock()
- self.assertEqual(self.module.Clipboard.count(resmock), 0)
+ self.assertEqual(self.module.Clipboard.count(reqmock), 0)
clipboard = self.module.Clipboard()
clipboard.mode = 'test'
clipboard.add('item')
clipboard.to_response(resmock, reqmock)
+ resmock.dump_cookies(reqmock)
+ reqmock.uncache()
- self.assertEqual(self.module.Clipboard.count(resmock), 1)
+ self.assertEqual(self.module.Clipboard.count(reqmock), 1)
def test_oveflow(self):
class TinyClipboard(self.module.Clipboard):
@@ -554,9 +615,12 @@ class TestClipboard(unittest.TestCase):
name = self.module.Clipboard.data_cookie._name_cookie_page(0)
request = RequestMock()
request.set_cookie(name, 'value')
+ response = ResponseMock()
+ response.set_cookie(name, 'value')
clipboard = self.module.Clipboard()
- clipboard.to_response(request, request)
- self.assertNotIn(name, request.cookies)
+ clipboard.to_response(response, request)
+ print(response.headers)
+ self.assertNotIn(name, response.cookies)
class TestException(unittest.TestCase):
diff --git a/browsepy/tests/test_http.py b/browsepy/tests/test_http.py
index 0048651..ebd2d84 100644
--- a/browsepy/tests/test_http.py
+++ b/browsepy/tests/test_http.py
@@ -1,8 +1,34 @@
+import re
import unittest
+import datetime
+
+import werkzeug
import browsepy.http
+resc = re.compile(r'^((?P[^=;]+)(=(?P[^;]+))?(;|$))')
+
+def parse_set_cookies(headers, dt=None):
+ '''
+ :param headers: header structure
+ :type headers: werkzeug.http.Headers
+ '''
+ if dt is None:
+ dt = datetime.datetime.now()
+ cookies = {}
+ for value in headers.get_all('Set-Cookie'):
+ items = [match.groupdict() for match in resc.finditer(value)]
+ name = items[0]['key']
+ options = {
+ item['key'].strip(): item['value'].strip()
+ for item in item
+ }
+ if 'expires' in options:
+ options['expires'] = werkzeug.parse_date(options['expires'])
+
+
+
class TestHeaders(unittest.TestCase):
module = browsepy.http
@@ -26,3 +52,7 @@ class TestHeaders(unittest.TestCase):
headers.get('Option-Header'),
'something; option=1',
)
+
+
+class TestDataCookie(unittest.TestCase):
+ pass
--
GitLab
From 6140e3c4f0d7b3d8e1763d289ef96491fd8df971 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 24 May 2018 01:37:01 +0200
Subject: [PATCH 033/171] implement set_cookie_parser, add pycodestyle
unittest, disable flake8 pycodestyle
---
Makefile | 4 +-
browsepy/http.py | 53 ++++++++++++++++-
browsepy/plugin/file_actions/tests.py | 83 +++++++++++----------------
browsepy/tests/test_code_quality.py | 65 +++++++++++++++++++++
browsepy/tests/test_http.py | 65 +++++++++++++++------
requirements/development.txt | 2 +-
setup.cfg | 2 +-
setup.py | 3 +
8 files changed, 206 insertions(+), 71 deletions(-)
create mode 100644 browsepy/tests/test_code_quality.py
diff --git a/Makefile b/Makefile
index 1355301..828e39c 100644
--- a/Makefile
+++ b/Makefile
@@ -41,11 +41,11 @@ showdoc: doc
eslint:
eslint ${CURDIR}/browsepy
-pep8:
+pycodestyle:
pycodestyle --show-source browsepy
pycodestyle --show-source setup.py
-pycodestyle: pep8
+pep8: pycodestyle
flake8:
flake8 --show-source browsepy
diff --git a/browsepy/http.py b/browsepy/http.py
index 052db6e..74a169b 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -8,7 +8,7 @@ import logging
import zlib
from werkzeug.http import dump_header, dump_options_header, dump_cookie, \
- parse_cookie
+ parse_cookie, parse_date
from werkzeug.datastructures import Headers as BaseHeaders
from .compat import range
@@ -267,3 +267,54 @@ class DataCookie(object):
for name in cookie_names:
result.add('Set-Cookie', dump_cookie(name, expires=0))
return result
+
+
+def parse_set_cookie_option(name, value):
+ '''
+ Parse Set-Cookie header option (acepting option 'value' as cookie value),
+ both name and value.
+
+ Resulting names are compatible as :func:`werkzeug.http.dump_cookie`
+ keyword arguments.
+
+ :param name: option name
+ :type name: str
+ :param value: option value
+ :type value: str
+ :returns: tuple of parsed name and option, or None if name is unknown
+ :rtype: tuple of str or None
+ '''
+ try:
+ if name == 'Max-Age':
+ return 'max_age', int(value)
+ if name == 'Expires':
+ return 'expires', parse_date(value)
+ if name in ('value', 'Path', 'Domain', 'SameSite'):
+ return name.lower(), value
+ if name in ('Secure', 'HttpOnly'):
+ return name.lower(), True
+ except (AttributeError, ValueError, TypeError):
+ pass
+ except BaseException as e:
+ logger.exception(e)
+
+
+re_parse_set_cookie = re.compile(r'([^=;]+)(?:=([^;]*))?(?:$|;\s*)')
+
+
+def parse_set_cookie(header, option_parse_fnc=parse_set_cookie_option):
+ '''
+ Parse the content of a Set-Type HTTP header.
+
+ Result options are compatible as :func:`werkzeug.http.dump_cookie`
+ keyword arguments.
+
+ :param header: Set-Cookie header value
+ :type header: str
+ :returns: tuple with cookie name and its options
+ :rtype: tuple of str and dict
+ '''
+ pairs = re_parse_set_cookie.findall(header)
+ name, value = pairs[0]
+ pairs[0] = ('value', parse_cookie('v=%s' % value).get('v', None))
+ return name, dict(filter(None, (option_parse_fnc(*p) for p in pairs)))
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index 4f5773b..023439d 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -10,8 +10,7 @@ import bs4
import flask
from werkzeug.utils import cached_property
-from werkzeug.http import Headers, parse_cookie, dump_cookie, \
- parse_options_header, parse_date
+from werkzeug.http import Headers, parse_cookie, dump_cookie
import browsepy.plugin.file_actions as file_actions
import browsepy.plugin.file_actions.clipboard as file_actions_clipboard
@@ -25,67 +24,47 @@ import browsepy
class CookieHeaderMock(object):
header = 'Cookie'
- def parse_cookie(self, header):
- cookies = parse_cookie(header)
- if len(cookies) != 1:
- # multi-cookie header (Cookie), no options
- return cookies, {}
- name, value = tuple(cookies.items())[0]
- cookie, _ = parse_options_header(header)
- extra = header[len(cookie):]
-
- for part in .split('; '):
-
- if part.startswith('Expires='):
- expire = part[8:].replace('"', '')
- options['expires'] = parse_date(expire)
-
-
- def check_expired(self, options):
-
- if expire and < datetime.datetime.now():
- return True
-
- def check_expired(self, header):
-
-
-
- cookie, options = parse_options_header(header)
- if 'Expires' in options: # implies Set-Cookie
-
- if options.get('Max-Age', '1') == '0':
- return True
- return False
@property
def _cookies(self):
- check_expired = self.check_expired
return [
- (name, value, check_expired(header))
+ (name, {'value': value})
for header in self.headers.get_all(self.header)
for name, value in parse_cookie(header).items()
]
@property
def cookies(self):
+ is_valid = self.valid
return {
- name: value
- for name, value, expired in self._cookies
- if not expired
+ name: options.get('value')
+ for name, options in self._cookies
+ if is_valid(options)
}
+ def valid(self, cookie_options):
+ dt = datetime.datetime.now()
+ return (
+ cookie_options.get('max_age', 1) > 0 and
+ cookie_options.get('expiration', dt) >= dt
+ )
+
def __init__(self):
self.headers = Headers()
+ def add_cookie(self, name, value='', **kwargs):
+ self.headers.add(self.header, dump_cookie(name, value, **kwargs))
+
def set_cookie(self, name, value='', **kwargs):
owned = [
- for name, value, expires in self._cookies
+ (name, options)
+ for name, options in self._cookies
if name != name
]
-
-
- for name, value in owned.items():
- self.headers.add(self.header, dump_cookie(name, value, **kwargs))
+ self.clear()
+ for name, options in owned:
+ self.headers.add(self.header, dump_cookie(name, **options))
+ self.add_cookie(name, value, **kwargs)
def clear(self):
self.headers.clear()
@@ -94,21 +73,27 @@ class CookieHeaderMock(object):
class ResponseMock(CookieHeaderMock):
header = 'Set-Cookie'
+ @property
+ def _cookies(self):
+ return [
+ browsepy_http.parse_set_cookie(header)
+ for header in self.headers.get_all(self.header)
+ ]
+
def dump_cookies(self, client):
owned = self._cookies
if isinstance(client, CookieHeaderMock):
client.clear()
- for name, value in owned.items():
- client.set_cookie(name, value)
+ for name, options in owned:
+ client.add_cookie(name, **options)
else:
for cookie in client.cookie_jar:
- if cookie.name in owned:
- client.cookie_jar.clear(
+ client.cookie_jar.clear(
cookie.domain, cookie.path, cookie.name)
- for name, value in owned.items():
+ for name, options in owned:
client.set_cookie(
- client.environ_base['REMOTE_ADDR'], name, value)
+ client.environ_base['REMOTE_ADDR'], name, **options)
class RequestMock(CookieHeaderMock):
diff --git a/browsepy/tests/test_code_quality.py b/browsepy/tests/test_code_quality.py
new file mode 100644
index 0000000..f23029c
--- /dev/null
+++ b/browsepy/tests/test_code_quality.py
@@ -0,0 +1,65 @@
+import re
+import os.path
+import unittest
+import pycodestyle
+import functools
+
+
+class DeferredReport(pycodestyle.StandardReport):
+ def __init__(self, *args, **kwargs):
+ self.print_fnc = kwargs.pop('print_fnc')
+ super(DeferredReport, self).__init__(*args, **kwargs)
+
+ def get_file_results(self):
+ self._deferred_print.sort()
+ for line_number, offset, code, text, doc in self._deferred_print:
+ error = {
+ 'path': self.filename,
+ 'row': self.line_offset + line_number,
+ 'col': offset + 1,
+ 'code': code,
+ 'text': text,
+ }
+ lines = [self._fmt % error]
+ if line_number <= len(self.lines):
+ line = self.lines[line_number - 1]
+ lines.extend((
+ line.rstrip(),
+ re.sub(r'\S', ' ', line[:offset]) + '^'
+ ))
+ if doc:
+ lines.append(' ' + doc.strip())
+ error['message'] = '\n'.join(lines)
+ self.print_fnc(error)
+ return self.file_errors
+
+
+class TestCodeQuality(unittest.TestCase):
+ base = os.path.join(
+ os.path.dirname(__file__),
+ os.path.pardir,
+ )
+ setup_py = os.path.join(base, '..', 'setup.py')
+ config_file = os.path.join(base, '..', 'setup.cfg')
+
+ def test_conformance(self):
+ '''
+ Test pep-8 conformance
+ '''
+ messages = []
+ style = pycodestyle.StyleGuide(
+ config_file=self.config_file,
+ paths=[self.base, self.setup_py],
+ reporter=functools.partial(
+ DeferredReport,
+ print_fnc=messages.append
+ )
+ )
+ result = style.check_files()
+ text = ''.join(
+ '%s\n %s' % ('' if n else '\n', line)
+ for message in messages
+ for n, line in enumerate(message['message'].splitlines())
+ )
+ self.assertEqual(
+ result.total_errors, 0, "Code style errors:%s" % text)
diff --git a/browsepy/tests/test_http.py b/browsepy/tests/test_http.py
index ebd2d84..30f1177 100644
--- a/browsepy/tests/test_http.py
+++ b/browsepy/tests/test_http.py
@@ -1,33 +1,64 @@
import re
+import logging
import unittest
-import datetime
import werkzeug
import browsepy.http
-resc = re.compile(r'^((?P[^=;]+)(=(?P[^;]+))?(;|$))')
+logger = logging.getLogger(__name__)
+rct = re.compile(
+ r'([^=;]+)(?:=([^;]*))?\s*(?:$|;\s+)'
+ )
-def parse_set_cookies(headers, dt=None):
+
+def parse_set_cookie_option(name, value):
'''
- :param headers: header structure
- :type headers: werkzeug.http.Headers
+ Parse Set-Cookie header option (acepting option 'value' as cookie value),
+ both name and value.
+
+ Resulting names are compatible as :func:`werkzeug.http.dump_cookie`
+ keyword arguments.
+
+ :param name: option name
+ :type name: str
+ :param value: option value
+ :type value: str
+ :returns: tuple of parsed name and option, or None if name is unknown
+ :rtype: tuple of str or None
'''
- if dt is None:
- dt = datetime.datetime.now()
- cookies = {}
- for value in headers.get_all('Set-Cookie'):
- items = [match.groupdict() for match in resc.finditer(value)]
- name = items[0]['key']
- options = {
- item['key'].strip(): item['value'].strip()
- for item in item
- }
- if 'expires' in options:
- options['expires'] = werkzeug.parse_date(options['expires'])
+ try:
+ if name == 'Max-Age':
+ return 'max_age', int(value)
+ if name == 'Expires':
+ return 'expires', werkzeug.parse_date(value)
+ if name in ('value', 'Path', 'Domain', 'SameSite'):
+ return name.lower(), value
+ if name in ('Secure', 'HttpOnly'):
+ return name.lower(), True
+ except (AttributeError, ValueError, TypeError):
+ pass
+ except BaseException as e:
+ logger.exception(e)
+def parse_set_cookie(header, option_parse_fnc=parse_set_cookie_option):
+ '''
+ Parse the content of a Set-Type HTTP header.
+
+ Result options are compatible as :func:`werkzeug.http.dump_cookie`
+ keyword arguments.
+
+ :param header: Set-Cookie header value
+ :type header: str
+ :returns: tuple with cookie name and its options
+ :rtype: tuple of str and dict
+ '''
+ pairs = rct.findall(header)
+ name, value = pairs[0]
+ pairs[0] = ('value', werkzeug.parse_cookie('v=%s' % value).get('v', None))
+ return name, dict(filter(None, (option_parse_fnc(*p) for p in pairs)))
class TestHeaders(unittest.TestCase):
diff --git a/requirements/development.txt b/requirements/development.txt
index ce54672..4fe96a5 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -25,7 +25,7 @@
setuptools
wheel
-pycodestyle<2.4.0
+pycodestyle
flake8
coverage
pyaml
diff --git a/setup.cfg b/setup.cfg
index 451da18..76001e3 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -11,7 +11,7 @@ tag_build =
ignore = E123,E126,E121,W504
[flake8]
-ignore = E123,E126,E121,W504
+select = F,C
max-complexity = 9
[yapf]
diff --git a/setup.py b/setup.py
index b2d0ef7..4b540b5 100644
--- a/setup.py
+++ b/setup.py
@@ -108,6 +108,9 @@ setup(
install_requires=(
['flask', 'unicategories'] + extra_requires
),
+ extras_require={
+ 'tests': ['beautifulsoup4'],
+ },
tests_require=['beautifulsoup4'],
test_suite='browsepy.tests',
test_runner='browsepy.tests.runner:DebuggerTextTestRunner',
--
GitLab
From 0a393e64d6c0da55a6cda05d2d3be135ed731866 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 24 May 2018 19:16:29 +0200
Subject: [PATCH 034/171] fixes
---
Makefile | 6 +-
browsepy/http.py | 2 +-
browsepy/tests/test_http.py | 112 +++++++++++++++++-------------------
setup.cfg | 2 +
4 files changed, 59 insertions(+), 63 deletions(-)
diff --git a/Makefile b/Makefile
index 828e39c..527050e 100644
--- a/Makefile
+++ b/Makefile
@@ -42,14 +42,12 @@ eslint:
eslint ${CURDIR}/browsepy
pycodestyle:
- pycodestyle --show-source browsepy
- pycodestyle --show-source setup.py
+ pycodestyle browsepy setup.py
pep8: pycodestyle
flake8:
- flake8 --show-source browsepy
- flake8 --show-source setup.py
+ flake8 browsepy setup.py
coverage:
coverage run --source=browsepy setup.py test
diff --git a/browsepy/http.py b/browsepy/http.py
index 74a169b..e482d7c 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -229,7 +229,7 @@ class DataCookie(object):
for i in range(self.max_pages):
name = self._name_cookie_page(i)
end = start + self._available_cookie_size(name)
- result.set(
+ result.add(
'Set-Cookie',
dump_cookie(
name,
diff --git a/browsepy/tests/test_http.py b/browsepy/tests/test_http.py
index 30f1177..3226f10 100644
--- a/browsepy/tests/test_http.py
+++ b/browsepy/tests/test_http.py
@@ -1,65 +1,11 @@
-import re
-import logging
+import os
import unittest
-
-import werkzeug
+import datetime
+import base64
import browsepy.http
-logger = logging.getLogger(__name__)
-rct = re.compile(
- r'([^=;]+)(?:=([^;]*))?\s*(?:$|;\s+)'
- )
-
-
-def parse_set_cookie_option(name, value):
- '''
- Parse Set-Cookie header option (acepting option 'value' as cookie value),
- both name and value.
-
- Resulting names are compatible as :func:`werkzeug.http.dump_cookie`
- keyword arguments.
-
- :param name: option name
- :type name: str
- :param value: option value
- :type value: str
- :returns: tuple of parsed name and option, or None if name is unknown
- :rtype: tuple of str or None
- '''
- try:
- if name == 'Max-Age':
- return 'max_age', int(value)
- if name == 'Expires':
- return 'expires', werkzeug.parse_date(value)
- if name in ('value', 'Path', 'Domain', 'SameSite'):
- return name.lower(), value
- if name in ('Secure', 'HttpOnly'):
- return name.lower(), True
- except (AttributeError, ValueError, TypeError):
- pass
- except BaseException as e:
- logger.exception(e)
-
-
-def parse_set_cookie(header, option_parse_fnc=parse_set_cookie_option):
- '''
- Parse the content of a Set-Type HTTP header.
-
- Result options are compatible as :func:`werkzeug.http.dump_cookie`
- keyword arguments.
-
- :param header: Set-Cookie header value
- :type header: str
- :returns: tuple with cookie name and its options
- :rtype: tuple of str and dict
- '''
- pairs = rct.findall(header)
- name, value = pairs[0]
- pairs[0] = ('value', werkzeug.parse_cookie('v=%s' % value).get('v', None))
- return name, dict(filter(None, (option_parse_fnc(*p) for p in pairs)))
-
class TestHeaders(unittest.TestCase):
module = browsepy.http
@@ -85,5 +31,55 @@ class TestHeaders(unittest.TestCase):
)
+class TestParseSetCookie(unittest.TestCase):
+ module = browsepy.http
+
+ def test_parse(self):
+ tests = {
+ 'value-cookie=value': {
+ 'name': 'value-cookie',
+ 'value': 'value',
+ },
+ 'expiration-cookie=value; Expires=Thu, 24 May 2018 18:10:26 GMT': {
+ 'name': 'expiration-cookie',
+ 'value': 'value',
+ 'expires': datetime.datetime(2018, 5, 24, 18, 10, 26),
+ },
+ 'maxage-cookie="value with spaces"; Max-Age=0': {
+ 'name': 'maxage-cookie',
+ 'value': 'value with spaces',
+ 'max_age': 0,
+ },
+ 'secret-cookie; HttpOnly; Secure': {
+ 'name': 'secret-cookie',
+ 'value': '',
+ 'httponly': True,
+ 'secure': True,
+ },
+ 'spaced name=value': {
+ 'name': 'spaced name',
+ 'value': 'value',
+ },
+ }
+ for cookie, data in tests.items():
+ name, parsed = self.module.parse_set_cookie(cookie)
+ parsed['name'] = name
+ self.assertEquals(parsed, data)
+
+
class TestDataCookie(unittest.TestCase):
- pass
+ module = browsepy.http
+ manager_cls = module.DataCookie
+
+ def random_text(self, size=2):
+ bytedata = base64.b64encode(os.urandom(size))
+ return bytedata.decode('ascii')[:size]
+
+ def test_pagination(self):
+ data = self.random_text(self.manager_cls.page_length)
+ manager = self.manager_cls('cookie', max_pages=3)
+ headers = manager.dump_headers(data)
+ print(headers)
+ self.assertEqual(len(headers), 2)
+ self.assertEqual(manager.load_headers(headers), data)
+
diff --git a/setup.cfg b/setup.cfg
index 76001e3..57e5abb 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -8,11 +8,13 @@ universal = 1
tag_build =
[pycodestyle]
+show-source = True
ignore = E123,E126,E121,W504
[flake8]
select = F,C
max-complexity = 9
+show-source = True
[yapf]
align_closing_bracket_with_visual_indent = true
--
GitLab
From 4cf75428feb0f903e4086f16a26aebb9c1071347 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sat, 26 May 2018 19:38:25 +0200
Subject: [PATCH 035/171] datacookie tests and fixes
---
browsepy/http.py | 140 ++++++++++++++++------------
browsepy/tests/test_code_quality.py | 10 +-
browsepy/tests/test_http.py | 74 +++++++++++++--
3 files changed, 154 insertions(+), 70 deletions(-)
diff --git a/browsepy/http.py b/browsepy/http.py
index e482d7c..664efd0 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -87,9 +87,10 @@ class DataCookie(object):
> return response
'''
- page_length = 4000
size_error = InvalidCookieSizeError
headers_class = BaseHeaders
+ header_max_size = 4096
+ header_initial_size = len('Set-Cookie: ')
@staticmethod
def _serialize(data):
@@ -110,21 +111,44 @@ class DataCookie(object):
serialized = zlib.decompress(decoded)
return json.loads(serialized)
- def __init__(self, cookie_name, max_pages=1, max_age=None, path='/'):
+ def __init__(self, cookie_name, max_pages=1,
+ max_age=None, expires=None, path='/',
+ domain=None, secure=False, httponly=False,
+ charset='utf-8', sync_expires=True):
'''
:param cookie_name: first cookie name and prefix for the following
:type cookie_name: str
:param max_pages: maximum allowed cookie parts, defaults to 1
:type max_pages: int
- :param max_age: cookie lifetime in seconds or None (session, default)
- :type max_age: int, datetime.timedelta or None
- :param path: cookie path, defaults to /
+ :param max_age: cookie lifetime
+ :type max_age: int or datetime.timedelta
+ :param expires: date (or timestamp) of cookie expiration
+ :type expires: datetime.datetime or int
+ :param path: cookie domain path, defaults to '/'
:type path: str
+ :param domain: cookie domain, defaults to current
+ :type domain: str
+ :param secure: either cookie will only be available via HTTPS or not
+ :type secure: bool
+ :param httponly: either cookie can be accessed via Javascript or not
+ :type httponly: bool
+ :param charset: the encoding for unicode values, defaults to utf-8
+ :type charset: str
+ :param sync_expires: set expires based on max_age as default
+ :type sync_expires: bool
'''
- self.cookie_name = cookie_name
self.max_pages = max_pages
- self.max_age = max_age
- self.path = path
+ self.cookie_name = cookie_name
+ self.cookie_options = {
+ 'max_age': max_age,
+ 'expires': None,
+ 'path': path,
+ 'domain': domain,
+ 'secure': secure,
+ 'httponly': httponly,
+ 'charset': charset,
+ 'sync_expires': sync_expires,
+ }
def _name_cookie_page(self, page):
'''
@@ -141,10 +165,23 @@ class DataCookie(object):
return '{}{}'.format(
self.cookie_name,
page if isinstance(page, str) else
- '-{:x}'.format(page - 1) if page else
+ '-{:x}'.format(page + 1) if page else
''
)
+ def _dump_cookie(self, name, value):
+ '''
+ Dump cookie with configured options.
+
+ :param name: cookie name
+ :type name: str
+ :param value: cookie value
+ :type value: str
+ :returns: Set-Cookie header value
+ :rtype: str
+ '''
+ return dump_cookie(name, value, **self.cookie_options)
+
def _available_cookie_size(self, name):
'''
Get available cookie size for value.
@@ -154,19 +191,19 @@ class DataCookie(object):
:return: available bytes for cookie value
:rtype: int
'''
- empty = 'Set-Cookie: %s' % dump_cookie(
- name,
- value=' ', # force quotes
- max_age=self.max_age,
- path=self.path
- )
- return self.page_length - len(empty)
+ empty = self._dump_cookie(name, ' ') # forces quotes
+ return self.header_max_size - self.header_initial_size - len(empty)
def _extract_cookies(self, headers):
'''
Extract relevant cookies from headers.
+
+ :param headers: request headers
+ :type headers: werkzeug.datasturctures.Headers
+ :returns: cookies
+ :rtype: dict
'''
- regex = re.compile('^%s$' % self._name_cookie_page('(-[0-9a-f])?'))
+ regex = re.compile('^%s$' % self._name_cookie_page('(-[0-9a-f]+)?'))
return {
key: value
for header in headers.get_all('cookie')
@@ -229,15 +266,7 @@ class DataCookie(object):
for i in range(self.max_pages):
name = self._name_cookie_page(i)
end = start + self._available_cookie_size(name)
- result.add(
- 'Set-Cookie',
- dump_cookie(
- name,
- data[start:end],
- max_age=self.max_age,
- path=self.path,
- )
- )
+ result.add('Set-Cookie', self._dump_cookie(name, data[start:end]))
if end > size:
# incidentally, an empty page will be added after end == size
if headers:
@@ -265,44 +294,14 @@ class DataCookie(object):
result = self.headers_class()
for name in cookie_names:
- result.add('Set-Cookie', dump_cookie(name, expires=0))
+ result.add('Set-Cookie', dump_cookie(name, max_age=0, expires=0))
return result
-def parse_set_cookie_option(name, value):
- '''
- Parse Set-Cookie header option (acepting option 'value' as cookie value),
- both name and value.
-
- Resulting names are compatible as :func:`werkzeug.http.dump_cookie`
- keyword arguments.
-
- :param name: option name
- :type name: str
- :param value: option value
- :type value: str
- :returns: tuple of parsed name and option, or None if name is unknown
- :rtype: tuple of str or None
- '''
- try:
- if name == 'Max-Age':
- return 'max_age', int(value)
- if name == 'Expires':
- return 'expires', parse_date(value)
- if name in ('value', 'Path', 'Domain', 'SameSite'):
- return name.lower(), value
- if name in ('Secure', 'HttpOnly'):
- return name.lower(), True
- except (AttributeError, ValueError, TypeError):
- pass
- except BaseException as e:
- logger.exception(e)
-
-
re_parse_set_cookie = re.compile(r'([^=;]+)(?:=([^;]*))?(?:$|;\s*)')
-def parse_set_cookie(header, option_parse_fnc=parse_set_cookie_option):
+def parse_set_cookie(header):
'''
Parse the content of a Set-Type HTTP header.
@@ -314,7 +313,26 @@ def parse_set_cookie(header, option_parse_fnc=parse_set_cookie_option):
:returns: tuple with cookie name and its options
:rtype: tuple of str and dict
'''
+
+ def parse_option(pair):
+ name, value = pair
+ try:
+ if name == 'Max-Age':
+ return 'max_age', int(value)
+ if name == 'Expires':
+ return 'expires', parse_date(value)
+ if name in ('Path', 'Domain', 'SameSite'):
+ return name.lower(), value
+ if name in ('Secure', 'HttpOnly'):
+ return name.lower(), True
+ except (AttributeError, ValueError, TypeError):
+ pass
+ except BaseException as e:
+ logger.exception(e)
+ return None, None
+
pairs = re_parse_set_cookie.findall(header)
- name, value = pairs[0]
- pairs[0] = ('value', parse_cookie('v=%s' % value).get('v', None))
- return name, dict(filter(None, (option_parse_fnc(*p) for p in pairs)))
+ name, value = pairs.pop(0)
+ options = {k: v for k, v in map(parse_option, pairs) if k}
+ options['value'] = parse_cookie('v=%s' % value).get('v', None)
+ return name, options
diff --git a/browsepy/tests/test_code_quality.py b/browsepy/tests/test_code_quality.py
index f23029c..dd07e17 100644
--- a/browsepy/tests/test_code_quality.py
+++ b/browsepy/tests/test_code_quality.py
@@ -8,13 +8,20 @@ import functools
class DeferredReport(pycodestyle.StandardReport):
def __init__(self, *args, **kwargs):
self.print_fnc = kwargs.pop('print_fnc')
+ self.location_base = kwargs.pop('location_base')
super(DeferredReport, self).__init__(*args, **kwargs)
+ @property
+ def location(self):
+ if self.filename:
+ return os.path.relpath(self.filename, self.location_base)
+ return self.filename
+
def get_file_results(self):
self._deferred_print.sort()
for line_number, offset, code, text, doc in self._deferred_print:
error = {
- 'path': self.filename,
+ 'path': self.location,
'row': self.line_offset + line_number,
'col': offset + 1,
'code': code,
@@ -52,6 +59,7 @@ class TestCodeQuality(unittest.TestCase):
paths=[self.base, self.setup_py],
reporter=functools.partial(
DeferredReport,
+ location_base=self.base,
print_fnc=messages.append
)
)
diff --git a/browsepy/tests/test_http.py b/browsepy/tests/test_http.py
index 3226f10..30cbfd2 100644
--- a/browsepy/tests/test_http.py
+++ b/browsepy/tests/test_http.py
@@ -3,8 +3,13 @@ import os
import unittest
import datetime
import base64
+import zlib
+
+import werkzeug.http
+import werkzeug.datastructures
import browsepy.http
+import browsepy.exceptions
class TestHeaders(unittest.TestCase):
@@ -70,16 +75,69 @@ class TestParseSetCookie(unittest.TestCase):
class TestDataCookie(unittest.TestCase):
module = browsepy.http
manager_cls = module.DataCookie
+ header_cls = werkzeug.datastructures.Headers
+ cookie_size_exception = browsepy.exceptions.InvalidCookieSizeError
def random_text(self, size=2):
- bytedata = base64.b64encode(os.urandom(size))
- return bytedata.decode('ascii')[:size]
+ bytedata = zlib.compress(os.urandom(size * 3), 9) # ensure entropy
+ b64data = base64.b64encode(bytedata)
+ return b64data.decode('ascii')[:size]
+
+ def parse_response_cookies(self, headers):
+ '''
+ :type headers: werkzeug.datastructures.Headers
+ '''
+ return {
+ name: options
+ for name, options in map(
+ self.module.parse_set_cookie,
+ headers.get_all('set-cookie')
+ )}
+
+ def response_to_request_headers(self, headers):
+ '''
+ :type headers: werkzeug.datastructures.Headers
+ '''
+ return self.header_cls(
+ ('Cookie', werkzeug.http.dump_cookie(name, options['value']))
+ for name, options in self.parse_response_cookies(headers).items()
+ )
def test_pagination(self):
- data = self.random_text(self.manager_cls.page_length)
- manager = self.manager_cls('cookie', max_pages=3)
- headers = manager.dump_headers(data)
- print(headers)
- self.assertEqual(len(headers), 2)
- self.assertEqual(manager.load_headers(headers), data)
+ data = self.random_text(self.manager_cls.header_max_size)
+ manager = self.manager_cls('cookie', max_pages=2)
+
+ rheaders = manager.dump_headers(data)
+ self.assertEqual(len(rheaders), 2)
+
+ qheaders = self.response_to_request_headers(rheaders)
+ self.assertEqual(manager.load_headers(qheaders), data)
+
+ rheaders = manager.dump_headers('shorter-data', qheaders)
+ self.assertEqual(len(rheaders), 2) # 1 for value, 1 for discard
+
+ cookies = self.parse_response_cookies(rheaders)
+
+ cookie1 = cookies['cookie']
+ deserialized = manager.load_cookies({'cookie': cookie1['value']})
+ self.assertEqual(deserialized, 'shorter-data')
+
+ cookie2 = cookies['cookie-2']
+ self.assertEqual(cookie2['value'], '')
+ self.assertLess(cookie2['expires'], datetime.datetime.now())
+ self.assertLess(cookie2['max_age'], 1)
+
+ def test_max_pagination(self):
+ manager = self.manager_cls('cookie', max_pages=2)
+ self.assertRaises(
+ self.cookie_size_exception,
+ manager.dump_headers,
+ self.random_text(self.manager_cls.header_max_size * 2)
+ )
+ def test_truncate(self):
+ qheaders = self.header_cls([('Cookie', 'cookie=value')])
+ manager = self.manager_cls('cookie', max_pages=2)
+ rheaders = manager.truncate_headers(qheaders)
+ parsed = self.parse_response_cookies(rheaders)
+ self.assertEqual(parsed['cookie']['value'], '')
--
GitLab
From 597e2be79ab465264c3fa194e1b672d40af0fc30 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 27 May 2018 18:06:54 +0200
Subject: [PATCH 036/171] fix clipboard exclude test, add bad datacookie test
---
browsepy/plugin/file_actions/tests.py | 32 ++++++++++++++++++---------
browsepy/tests/test_http.py | 6 +++++
2 files changed, 28 insertions(+), 10 deletions(-)
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index 023439d..d328102 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -35,19 +35,24 @@ class CookieHeaderMock(object):
@property
def cookies(self):
- is_valid = self.valid
+ is_expired = self.expired
return {
name: options.get('value')
for name, options in self._cookies
- if is_valid(options)
+ if not is_expired(options)
}
- def valid(self, cookie_options):
- dt = datetime.datetime.now()
- return (
- cookie_options.get('max_age', 1) > 0 and
- cookie_options.get('expiration', dt) >= dt
- )
+ @property
+ def expired_cookies(self):
+ is_expired = self.expired
+ return {
+ name: options.get('value')
+ for name, options in self._cookies
+ if is_expired(options)
+ }
+
+ def expired(self, options):
+ return False
def __init__(self):
self.headers = Headers()
@@ -80,6 +85,13 @@ class ResponseMock(CookieHeaderMock):
for header in self.headers.get_all(self.header)
]
+ def expired(self, cookie_options):
+ dt = datetime.datetime.now()
+ return (
+ cookie_options.get('max_age', 1) < 1 or
+ cookie_options.get('expiration', dt) < dt
+ )
+
def dump_cookies(self, client):
owned = self._cookies
if isinstance(client, CookieHeaderMock):
@@ -311,6 +323,7 @@ class TestIntegration(unittest.TestCase):
reqmock.load_cookies(client)
clipboard.mode = 'cut'
clipboard.to_response(resmock, reqmock)
+ resmock.dump_cookies(client)
response = client.get('/')
self.assertEqual(response.status_code, 200)
@@ -601,10 +614,9 @@ class TestClipboard(unittest.TestCase):
request = RequestMock()
request.set_cookie(name, 'value')
response = ResponseMock()
- response.set_cookie(name, 'value')
clipboard = self.module.Clipboard()
clipboard.to_response(response, request)
- print(response.headers)
+ self.assertIn(name, response.expired_cookies)
self.assertNotIn(name, response.cookies)
diff --git a/browsepy/tests/test_http.py b/browsepy/tests/test_http.py
index 30cbfd2..862aace 100644
--- a/browsepy/tests/test_http.py
+++ b/browsepy/tests/test_http.py
@@ -141,3 +141,9 @@ class TestDataCookie(unittest.TestCase):
rheaders = manager.truncate_headers(qheaders)
parsed = self.parse_response_cookies(rheaders)
self.assertEqual(parsed['cookie']['value'], '')
+
+ def test_corrupted(self):
+ manager = self.manager_cls('cookie')
+ default = object()
+ data = manager.load_cookies({'cookie': 'corrupted'}, default)
+ self.assertIs(data, default)
--
GitLab
From 35ad46d6d45a8752a56fef3903e7859b1c5d460d Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 30 May 2018 23:53:53 +0200
Subject: [PATCH 037/171] remove unused code, add coverage config excluding
tests
---
Makefile | 2 +-
browsepy/abc.py | 15 ---------------
browsepy/http.py | 3 ---
setup.cfg | 5 +++++
4 files changed, 6 insertions(+), 19 deletions(-)
delete mode 100644 browsepy/abc.py
diff --git a/Makefile b/Makefile
index 527050e..8ea3107 100644
--- a/Makefile
+++ b/Makefile
@@ -50,7 +50,7 @@ flake8:
flake8 browsepy setup.py
coverage:
- coverage run --source=browsepy setup.py test
+ coverage run setup.py test
showcoverage: coverage
coverage html
diff --git a/browsepy/abc.py b/browsepy/abc.py
deleted file mode 100644
index 1f69753..0000000
--- a/browsepy/abc.py
+++ /dev/null
@@ -1,15 +0,0 @@
-
-from abc import ABC
-
-from .compat import NoneType
-
-
-class JSONSerializable(ABC):
- pass
-
-
-JSONSerializable.register(NoneType)
-JSONSerializable.register(int)
-JSONSerializable.register(float)
-JSONSerializable.register(list)
-JSONSerializable.register(str)
diff --git a/browsepy/http.py b/browsepy/http.py
index 664efd0..bc18ddf 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -218,7 +218,6 @@ class DataCookie(object):
:param cookies: request cookies
:type cookies: collections.abc.Mapping
:returns: deserialized value
- :rtype: browsepy.abc.JSONSerializable
'''
chunks = []
for i in range(self.max_pages):
@@ -242,7 +241,6 @@ class DataCookie(object):
:param headers: request headers
:type headers: werkzeug.http.Headers
:returns: deserialized value
- :rtype: browsepy.abc.JSONSerializable
'''
cookies = self._extract_cookies(headers)
return self.load_cookies(cookies)
@@ -253,7 +251,6 @@ class DataCookie(object):
instance.
:param data: any json-serializable value
- :type data: browsepy.abc.JSONSerializable
:param headers: optional request headers, used to truncate old pages
:type headers: werkzeug.http.Headers
:return: response headers
diff --git a/setup.cfg b/setup.cfg
index 57e5abb..8104f2b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -19,3 +19,8 @@ show-source = True
[yapf]
align_closing_bracket_with_visual_indent = true
+[coverage:run]
+source = browsepy
+omit =
+ */tests/*
+ */tests.py
--
GitLab
From 2e373d7f2579e1c2dd279d1a92216eceefe7784f Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 31 May 2018 00:22:02 +0200
Subject: [PATCH 038/171] revert to flake8-compatible pycodestyle dev
requirement
---
browsepy/tests/test_code_quality.py | 3 ++-
requirements/development.txt | 2 +-
setup.py | 4 ++--
3 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/browsepy/tests/test_code_quality.py b/browsepy/tests/test_code_quality.py
index dd07e17..0983943 100644
--- a/browsepy/tests/test_code_quality.py
+++ b/browsepy/tests/test_code_quality.py
@@ -1,9 +1,10 @@
import re
import os.path
import unittest
-import pycodestyle
import functools
+import pycodestyle
+
class DeferredReport(pycodestyle.StandardReport):
def __init__(self, *args, **kwargs):
diff --git a/requirements/development.txt b/requirements/development.txt
index 4fe96a5..ce54672 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -25,7 +25,7 @@
setuptools
wheel
-pycodestyle
+pycodestyle<2.4.0
flake8
coverage
pyaml
diff --git a/setup.py b/setup.py
index 4b540b5..38a56ed 100644
--- a/setup.py
+++ b/setup.py
@@ -109,9 +109,9 @@ setup(
['flask', 'unicategories'] + extra_requires
),
extras_require={
- 'tests': ['beautifulsoup4'],
+ 'tests': ['beautifulsoup4', 'pycodestyle'],
},
- tests_require=['beautifulsoup4'],
+ tests_require=['beautifulsoup4', 'pycodestyle'],
test_suite='browsepy.tests',
test_runner='browsepy.tests.runner:DebuggerTextTestRunner',
zip_safe=False,
--
GitLab
From a6443ef84668071fda20d4b83c6887227e7328b6 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 31 May 2018 00:31:39 +0200
Subject: [PATCH 039/171] fix list-comprenhension redefining variable, add bs4
to dev requirements
---
browsepy/plugin/file_actions/tests.py | 10 +++++-----
requirements/development.txt | 1 +
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index d328102..bc44843 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -62,13 +62,13 @@ class CookieHeaderMock(object):
def set_cookie(self, name, value='', **kwargs):
owned = [
- (name, options)
- for name, options in self._cookies
- if name != name
+ (cname, coptions)
+ for cname, coptions in self._cookies
+ if cname != name
]
self.clear()
- for name, options in owned:
- self.headers.add(self.header, dump_cookie(name, **options))
+ for cname, coptions in owned:
+ self.headers.add(self.header, dump_cookie(cname, **coptions))
self.add_cookie(name, value, **kwargs)
def clear(self):
diff --git a/requirements/development.txt b/requirements/development.txt
index ce54672..dfe20d9 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -28,4 +28,5 @@ wheel
pycodestyle<2.4.0
flake8
coverage
+beautifulsoup4
pyaml
--
GitLab
From 939e48a8fc812f88dbe63425707c3b525dfd29da Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 31 May 2018 01:18:02 +0200
Subject: [PATCH 040/171] fix json-from-bytes failing on old python versions
---
browsepy/http.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/browsepy/http.py b/browsepy/http.py
index bc18ddf..8ff9091 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -108,7 +108,7 @@ class DataCookie(object):
:rtype: json-serializable
'''
decoded = base64.b64decode(data)
- serialized = zlib.decompress(decoded)
+ serialized = zlib.decompress(decoded).decode('utf-8')
return json.loads(serialized)
def __init__(self, cookie_name, max_pages=1,
--
GitLab
From c5fc14dfa7460a648fd6957d835d8abbd3514a06 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 28 Dec 2018 15:37:12 +0000
Subject: [PATCH 041/171] update requirents, doc and ci
---
.appveyor.yml | 8 +++
.python-version | 2 +-
.travis.yml | 1 +
doc/integrations.rst | 107 ++++++++++++++++++++++++++++++++---
requirements/development.txt | 2 +-
5 files changed, 110 insertions(+), 10 deletions(-)
diff --git a/.appveyor.yml b/.appveyor.yml
index 1818bd9..b9408f8 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -42,6 +42,14 @@ environment:
PYTHON_VERSION: "3.6.1"
PYTHON_ARCH: "64"
+ - PYTHON: "C:\\Python37"
+ PYTHON_VERSION: "3.7.1"
+ PYTHON_ARCH: "32"
+
+ - PYTHON: "C:\\Python37-x64"
+ PYTHON_VERSION: "3.7.1"
+ PYTHON_ARCH: "64"
+
build: off
init:
diff --git a/.python-version b/.python-version
index 0f44168..a76ccff 100644
--- a/.python-version
+++ b/.python-version
@@ -1 +1 @@
-3.6.4
+3.7.1
diff --git a/.travis.yml b/.travis.yml
index 465bcef..1307d1e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,6 +13,7 @@ matrix:
- python: "3.4"
- python: "3.5"
- python: "3.6"
+ - python: "3.7"
env:
- eslint=yes
- sphinx=yes
diff --git a/doc/integrations.rst b/doc/integrations.rst
index 6352b77..1d654f2 100644
--- a/doc/integrations.rst
+++ b/doc/integrations.rst
@@ -39,6 +39,99 @@ The other way of loading a plugin programmatically is calling
:meth:`browsepy.manager.PluginManager.load_plugin` for browsepy's plugin
manager.
+.. _integrations-wsgi:
+
+Waitress and any WSGI application
+---------------------------------
+
+Startup script running browsepy inside
+`waitress `_
+along with a root wsgi application.
+
+.. code-block:: python
+
+ #!/env/bin/python
+
+ import os
+ import os.path
+ import sys
+
+ import flask
+ import werkzeug.wsgi
+ import waitress
+
+ import browsepy
+
+
+ class cfg():
+ base_path = os.path.abspath(os.path.dirname(__file__))
+ static_path = os.path.join(base_path, 'static')
+ media_path = os.path.expandvars('$HOME/media')
+
+ stderr_path = os.path.join(base_path, 'stderr.log')
+ stdout_path = os.path.join(base_path, 'stdout.log')
+ pid_path = os.path.join(base_path, 'pidfile.pid')
+
+
+ def setup_browsepy():
+ browsepy.app.config.update(
+ APPLICATION_ROOT='/browse',
+ directory_base=cfg.media_path,
+ directory_start=cfg.media_path,
+ directory_remove=cfg.media_path,
+ directory_upload=cfg.media_path,
+ plugin_modules=['player'],
+ )
+ browsepy.plugin_manager.load_arguments([
+ '--plugin=player',
+ '--player-directory-play'
+ ])
+ browsepy.plugin_manager.reload()
+ return browsepy.app
+
+
+ def setup_app():
+ app = flask.Flask(
+ __name__,
+ static_folder=cfg.static_path,
+ static_url_path='',
+ )
+
+ @app.route('/')
+ def index():
+ return flask.send_from_directory(cfg.static_path, 'index.html')
+
+ return app
+
+
+ def setup_dispatcher():
+ return werkzeug.wsgi.DispatcherMiddleware(
+ setup_app(),
+ {
+ '/browse': setup_browsepy(),
+ # add other wsgi apps here
+ }
+ )
+
+
+ def main():
+ sys.stderr = open(cfg.stderr_path, 'w')
+ sys.stdout = open(cfg.stdout_path, 'w')
+ with open(cfg.pid_path, 'w') as f:
+ f.write('%d' % os.getpid())
+
+ try:
+ print('Starting server')
+ waitress.serve(setup_dispatcher(), listen='127.0.0.1:8080')
+ finally:
+ sys.stderr.close()
+ sys.stdout.close()
+
+
+ if __name__ == '__main__':
+ main()
+
+
.. _integrations-cherrymusic:
Cherrypy and Cherrymusic
@@ -50,7 +143,6 @@ server provided by `cherrymusic `_.
.. code-block:: python
#!/env/bin/python
- # -*- coding: UTF-8 -*-
import os
import sys
@@ -73,7 +165,6 @@ server provided by `cherrymusic `_.
base_path = abspath(dirname(__file__))
static_path = joinpath(base_path, 'static')
media_path = expandvars('$HOME/media')
- download_path = joinpath(media_path, 'downloads')
root_config = {
'/': {
'tools.staticdir.on': True,
@@ -85,12 +176,12 @@ server provided by `cherrymusic `_.
'server.rootpath': '/player',
}
browsepy.config.update(
- APPLICATION_ROOT = '/browse',
- directory_base = media_path,
- directory_start = media_path,
- directory_remove = media_path,
- directory_upload = media_path,
- plugin_modules = ['player'],
+ APPLICATION_ROOT='/browse',
+ directory_base=media_path,
+ directory_start=media_path,
+ directory_remove=media_path,
+ directory_upload=media_path,
+ plugin_modules=['player'],
)
plugin_manager.reload()
diff --git a/requirements/development.txt b/requirements/development.txt
index dfe20d9..634202a 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -25,7 +25,7 @@
setuptools
wheel
-pycodestyle<2.4.0
+pycodestyle
flake8
coverage
beautifulsoup4
--
GitLab
From 6fe73fc4a1c440d53a6bc0b8c04e278b032e0c4b Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sat, 30 Mar 2019 23:50:06 +0100
Subject: [PATCH 042/171] migrate to cookieman
---
.gitignore | 1 -
.vscode/settings.json | 17 ++
browsepy/__init__.py | 63 ++--
browsepy/__main__.py | 6 +-
browsepy/__meta__.py | 12 -
browsepy/exceptions.py | 13 -
browsepy/http.py | 281 +-----------------
browsepy/plugin/file_actions/__init__.py | 73 ++---
browsepy/plugin/file_actions/clipboard.py | 106 -------
browsepy/plugin/file_actions/paste.py | 22 +-
.../templates/selection.file_actions.html | 2 +-
browsepy/plugin/file_actions/tests.py | 25 +-
browsepy/tests/meta.py | 75 +++++
browsepy/tests/test_code_quality.py | 91 ++----
browsepy/tests/test_http.py | 121 +-------
browsepy/utils.py | 15 +
requirements.txt | 3 -
requirements/base.txt | 10 +-
requirements/ci.txt | 1 -
requirements/compat.txt | 27 --
requirements/development.txt | 21 +-
requirements/doc.txt | 1 -
setup.cfg | 14 +-
setup.py | 92 ++----
24 files changed, 264 insertions(+), 828 deletions(-)
create mode 100644 .vscode/settings.json
delete mode 100644 browsepy/__meta__.py
delete mode 100644 browsepy/plugin/file_actions/clipboard.py
create mode 100644 browsepy/tests/meta.py
create mode 100644 browsepy/utils.py
delete mode 100644 requirements/compat.txt
diff --git a/.gitignore b/.gitignore
index bf44bca..82e54ab 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,6 @@
.git
.c9
.idea
-.vscode
.coverage
htmlcov
dist
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000..5534594
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,17 @@
+{
+ "editor.rulers": [
+ 72,
+ 79
+ ],
+ "editor.tabSize": 4,
+ "files.trimTrailingWhitespace": true,
+ "files.insertFinalNewline": true,
+ "python.formatting.provider": "yapf",
+ "python.unitTest.unittestEnabled": true,
+ "python.linting.flake8Enabled": true,
+ "python.linting.enabled": true,
+ "python.linting.pep8Enabled": true,
+ "python.linting.pep8Path": "pycodestyle",
+ "python.linting.pylintEnabled": false,
+ "python.jediEnabled": true
+}
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index aa655bd..729916b 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -1,42 +1,36 @@
-#!/usr/bin/env python
# -*- coding: UTF-8 -*-
+__version__ = '0.5.6'
+
import logging
import os
import os.path
-from datetime import timedelta
+import cookieman
from flask import Response, request, render_template, redirect, \
url_for, send_from_directory, stream_with_context, \
- make_response
+ make_response, session
from werkzeug.exceptions import NotFound
-from .http import DataCookie
from .appconfig import Flask
from .manager import PluginManager
from .file import Node, secure_filename
from .exceptions import OutsideRemovableBase, OutsideDirectoryBase, \
- InvalidFilenameError, InvalidPathError, \
- InvalidCookieSizeError
+ InvalidFilenameError, InvalidPathError
from . import compat
-from . import __meta__ as meta
-
-__app__ = meta.app # noqa
-__version__ = meta.version # noqa
-__license__ = meta.license # noqa
-__author__ = meta.author # noqa
-__basedir__ = os.path.abspath(os.path.dirname(compat.fsdecode(__file__)))
+from . import utils
logger = logging.getLogger(__name__)
app = Flask(
__name__,
static_url_path='/static',
- static_folder=os.path.join(__basedir__, "static"),
- template_folder=os.path.join(__basedir__, "templates")
+ static_folder=utils.ppath('static'),
+ template_folder=utils.ppath('templates'),
)
app.config.update(
+ application_name='browsepy',
directory_base=compat.getcwd(),
directory_start=None,
directory_remove=None,
@@ -53,26 +47,24 @@ app.config.update(
exclude_fnc=None,
)
app.jinja_env.add_extension('browsepy.transform.htmlcompress.HTMLCompress')
+app.secret_key = utils.random_string(4096)
if 'BROWSEPY_SETTINGS' in os.environ:
app.config.from_envvar('BROWSEPY_SETTINGS')
plugin_manager = PluginManager(app)
-sorting_cookie = DataCookie('browse-sorting', max_age=timedelta(days=90))
+session_manager = cookieman.CookieMan()
+app.session_interface = session_manager
-def iter_cookie_browse_sorting(cookies):
- '''
- Get sorting-cookie from cookies dictionary.
- :yields: tuple of path and sorting property
- :ytype: 2-tuple of strings
- '''
- try:
- for path, prop in sorting_cookie.load_cookies(cookies, ()):
- yield path, prop
- except ValueError as e:
- logger.exception(e)
+@session_manager.register('browse:sort')
+def shrink_browse_sort(data, last):
+ if data['browse:sort'] and not last:
+ data['browse:sort'].pop()
+ else:
+ del data['browse:sort']
+ return data
def get_cookie_browse_sorting(path, default):
@@ -83,7 +75,7 @@ def get_cookie_browse_sorting(path, default):
:rtype: string
'''
if request:
- for cpath, cprop in iter_cookie_browse_sorting(request.cookies):
+ for cpath, cprop in session.get('browse:sort', ()):
if path == cpath:
return cprop
return default
@@ -171,24 +163,13 @@ def sort(property, path):
try:
data.extend(
(cpath, cprop)
- for cpath, cprop in iter_cookie_browse_sorting(request.cookies)
+ for cpath, cprop in session.get('browse:sort', ())
if cpath != path
)
except BaseException:
pass
- # handle cookie becoming too large
- headers = ()
- while data:
- try:
- headers = sorting_cookie.dump_headers(data, request.headers)
- break
- except InvalidCookieSizeError:
- data.pop(0)
-
- response = redirect(url_for(".browse", path=directory.urlpath))
- response.headers.extend(headers)
- return response
+ return redirect(url_for(".browse", path=directory.urlpath))
@app.route("/browse", defaults={"path": ""})
diff --git a/browsepy/__main__.py b/browsepy/__main__.py
index 9bdb7a2..d453cfe 100644
--- a/browsepy/__main__.py
+++ b/browsepy/__main__.py
@@ -11,7 +11,6 @@ import warnings
import flask
from . import app
-from . import __meta__ as meta
from .compat import PY_LEGACY, getdebug, get_terminal_size
from .transform.glob import translate
@@ -53,15 +52,16 @@ class ArgParse(argparse.ArgumentParser):
)
default_removable = app.config['directory_remove']
default_upload = app.config['directory_upload']
+ name = app.config['application_name']
default_host = os.getenv('BROWSEPY_HOST', '127.0.0.1')
default_port = os.getenv('BROWSEPY_PORT', '8080')
plugin_action_class = PluginAction
defaults = {
- 'prog': meta.app,
+ 'prog': name,
'formatter_class': HelpFormatter,
- 'description': 'description: starts a %s web file browser' % meta.app
+ 'description': 'description: starts a %s web file browser' % name
}
def __init__(self, sep=os.sep):
diff --git a/browsepy/__meta__.py b/browsepy/__meta__.py
deleted file mode 100644
index 0ca7f53..0000000
--- a/browsepy/__meta__.py
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-
-app = 'browsepy'
-description = 'Simple web file browser'
-version = '0.5.6'
-license = 'MIT'
-author_name = 'Felipe A. Hernandez'
-author_mail = 'ergoithz@gmail.com'
-author = '%s <%s>' % (author_name, author_mail)
-url = 'https://github.com/ergoithz/browsepy'
-tarball = '%s/archive/%s.tar.gz' % (url, version)
diff --git a/browsepy/exceptions.py b/browsepy/exceptions.py
index 1ed1e9d..8a47c03 100644
--- a/browsepy/exceptions.py
+++ b/browsepy/exceptions.py
@@ -69,16 +69,3 @@ class FilenameTooLongError(InvalidFilenameError):
self.limit = limit
super(FilenameTooLongError, self).__init__(
message, path=path, filename=filename)
-
-
-class InvalidCookieSizeError(ValueError):
- '''
- Exception raised when a paginated cookie exceeds its maximun page number.
- '''
- code = 'invalid-cookie-size'
- template = 'Value too big to fit in {0.max_cookies} cookies.'
-
- def __init__(self, message=None, max_cookies=None):
- self.max_cookies = max_cookies
- message = self.template.format(self) if message is None else message
- super(InvalidCookieSizeError, self).__init__(message)
diff --git a/browsepy/http.py b/browsepy/http.py
index 8ff9091..cb73cfa 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -1,19 +1,11 @@
-#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import re
-import json
-import base64
import logging
-import zlib
-from werkzeug.http import dump_header, dump_options_header, dump_cookie, \
- parse_cookie, parse_date
+from werkzeug.http import dump_header, dump_options_header
from werkzeug.datastructures import Headers as BaseHeaders
-from .compat import range
-from .exceptions import InvalidCookieSizeError
-
logger = logging.getLogger(__name__)
@@ -62,274 +54,3 @@ class Headers(BaseHeaders):
for key, value in kwargs.items()
]
return super(Headers, self).__init__(items)
-
-
-class DataCookie(object):
- '''
- Compressed base64 paginated cookie manager.
-
- Usage
- -----
-
- > from flask import Flask, request, make_response
- >
- > app = Flask(__name__)
- > cookie = DataCookie('my-cookie')
- >
- > @app.route('/get')
- > def get():
- > return 'Cookie value is %s' % cookie.load(request.cookies)
- >
- > @app.route('/set')
- > def set():
- > response = make_response('Cookie set')
- > cookie.dump('setted', response, request.cookies)
- > return response
-
- '''
- size_error = InvalidCookieSizeError
- headers_class = BaseHeaders
- header_max_size = 4096
- header_initial_size = len('Set-Cookie: ')
-
- @staticmethod
- def _serialize(data):
- '''
- :type data: json-serializable
- :rtype: bytes
- '''
- serialized = zlib.compress(json.dumps(data).encode('utf-8'))
- return base64.b64encode(serialized).decode('ascii')
-
- @staticmethod
- def _deserialize(data):
- '''
- :type data: bytes
- :rtype: json-serializable
- '''
- decoded = base64.b64decode(data)
- serialized = zlib.decompress(decoded).decode('utf-8')
- return json.loads(serialized)
-
- def __init__(self, cookie_name, max_pages=1,
- max_age=None, expires=None, path='/',
- domain=None, secure=False, httponly=False,
- charset='utf-8', sync_expires=True):
- '''
- :param cookie_name: first cookie name and prefix for the following
- :type cookie_name: str
- :param max_pages: maximum allowed cookie parts, defaults to 1
- :type max_pages: int
- :param max_age: cookie lifetime
- :type max_age: int or datetime.timedelta
- :param expires: date (or timestamp) of cookie expiration
- :type expires: datetime.datetime or int
- :param path: cookie domain path, defaults to '/'
- :type path: str
- :param domain: cookie domain, defaults to current
- :type domain: str
- :param secure: either cookie will only be available via HTTPS or not
- :type secure: bool
- :param httponly: either cookie can be accessed via Javascript or not
- :type httponly: bool
- :param charset: the encoding for unicode values, defaults to utf-8
- :type charset: str
- :param sync_expires: set expires based on max_age as default
- :type sync_expires: bool
- '''
- self.max_pages = max_pages
- self.cookie_name = cookie_name
- self.cookie_options = {
- 'max_age': max_age,
- 'expires': None,
- 'path': path,
- 'domain': domain,
- 'secure': secure,
- 'httponly': httponly,
- 'charset': charset,
- 'sync_expires': sync_expires,
- }
-
- def _name_cookie_page(self, page):
- '''
- Get name of cookie corresponding to given page.
-
- By design (see :method:`_name_page`), pages lower than 1 results on
- cookie names without a page name.
-
- :param page: page number or name
- :type page: int or str
- :returns: cookie name
- :rtype: str
- '''
- return '{}{}'.format(
- self.cookie_name,
- page if isinstance(page, str) else
- '-{:x}'.format(page + 1) if page else
- ''
- )
-
- def _dump_cookie(self, name, value):
- '''
- Dump cookie with configured options.
-
- :param name: cookie name
- :type name: str
- :param value: cookie value
- :type value: str
- :returns: Set-Cookie header value
- :rtype: str
- '''
- return dump_cookie(name, value, **self.cookie_options)
-
- def _available_cookie_size(self, name):
- '''
- Get available cookie size for value.
-
- :param name: cookie name
- :type name: str
- :return: available bytes for cookie value
- :rtype: int
- '''
- empty = self._dump_cookie(name, ' ') # forces quotes
- return self.header_max_size - self.header_initial_size - len(empty)
-
- def _extract_cookies(self, headers):
- '''
- Extract relevant cookies from headers.
-
- :param headers: request headers
- :type headers: werkzeug.datasturctures.Headers
- :returns: cookies
- :rtype: dict
- '''
- regex = re.compile('^%s$' % self._name_cookie_page('(-[0-9a-f]+)?'))
- return {
- key: value
- for header in headers.get_all('cookie')
- for key, value in parse_cookie(header).items()
- if regex.match(key)
- }
-
- def load_cookies(self, cookies, default=None):
- '''
- Parse data from relevant paginated cookie data given as mapping.
-
- :param cookies: request cookies
- :type cookies: collections.abc.Mapping
- :returns: deserialized value
- '''
- chunks = []
- for i in range(self.max_pages):
- name = self._name_cookie_page(i)
- cookie = cookies.get(name, '').encode('ascii')
- chunks.append(cookie)
- if len(cookie) < self._available_cookie_size(name):
- break
- data = b''.join(chunks)
- if data:
- try:
- return self._deserialize(data)
- except BaseException:
- pass
- return default
-
- def load_headers(self, headers, default=None):
- '''
- Parse data from relevant paginated cookie data on request headers.
-
- :param headers: request headers
- :type headers: werkzeug.http.Headers
- :returns: deserialized value
- '''
- cookies = self._extract_cookies(headers)
- return self.load_cookies(cookies)
-
- def dump_headers(self, data, headers=None):
- '''
- Serialize given object into a :class:`werkzeug.datastructures.Headers`
- instance.
-
- :param data: any json-serializable value
- :param headers: optional request headers, used to truncate old pages
- :type headers: werkzeug.http.Headers
- :return: response headers
- :rtype: werkzeug.http.Headers
- '''
- result = self.headers_class()
- data = self._serialize(data)
- start = 0
- size = len(data)
- for i in range(self.max_pages):
- name = self._name_cookie_page(i)
- end = start + self._available_cookie_size(name)
- result.add('Set-Cookie', self._dump_cookie(name, data[start:end]))
- if end > size:
- # incidentally, an empty page will be added after end == size
- if headers:
- result.extend(self.truncate_headers(headers, i + 1))
- return result
- start = end
- # pages exhausted, limit reached
- raise self.size_error(max_cookies=self.max_pages)
-
- def truncate_headers(self, headers, start=0):
- '''
- Evict relevant cookies found on request headers, optionally starting
- from a given page number.
-
- :param headers: request headers, required to truncate old pages
- :type headers: werkzeug.http.Headers
- :param start: paginated cookie start, defaults to 0
- :type start: int
- :return: response headers
- :rtype: werkzeug.http.Headers
- '''
- name_cookie = self._name_cookie_page
- cookie_names = set(self._extract_cookies(headers))
- cookie_names.difference_update(name_cookie(i) for i in range(start))
-
- result = self.headers_class()
- for name in cookie_names:
- result.add('Set-Cookie', dump_cookie(name, max_age=0, expires=0))
- return result
-
-
-re_parse_set_cookie = re.compile(r'([^=;]+)(?:=([^;]*))?(?:$|;\s*)')
-
-
-def parse_set_cookie(header):
- '''
- Parse the content of a Set-Type HTTP header.
-
- Result options are compatible as :func:`werkzeug.http.dump_cookie`
- keyword arguments.
-
- :param header: Set-Cookie header value
- :type header: str
- :returns: tuple with cookie name and its options
- :rtype: tuple of str and dict
- '''
-
- def parse_option(pair):
- name, value = pair
- try:
- if name == 'Max-Age':
- return 'max_age', int(value)
- if name == 'Expires':
- return 'expires', parse_date(value)
- if name in ('Path', 'Domain', 'SameSite'):
- return name.lower(), value
- if name in ('Secure', 'HttpOnly'):
- return name.lower(), True
- except (AttributeError, ValueError, TypeError):
- pass
- except BaseException as e:
- logger.exception(e)
- return None, None
-
- pairs = re_parse_set_cookie.findall(header)
- name, value = pairs.pop(0)
- options = {k: v for k, v in map(parse_option, pairs) if k}
- options['value'] = parse_cookie('v=%s' % value).get('v', None)
- return name, options
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index fb27e36..bdcd74c 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -6,7 +6,7 @@ import os.path
import logging
from flask import Blueprint, render_template, request, redirect, url_for, \
- make_response
+ session
from werkzeug.exceptions import NotFound
from browsepy import get_cookie_browse_sorting, browse_sortkey_reverse
@@ -15,7 +15,6 @@ from browsepy.file import Node, abspath_to_urlpath, secure_filename, \
from browsepy.compat import re_escape, FileNotFoundError
from browsepy.exceptions import OutsideDirectoryBase
-from .clipboard import Clipboard
from .exceptions import FileActionsException, \
InvalidClipboardItemsError, \
InvalidClipboardModeError, \
@@ -41,6 +40,10 @@ re_basename = '^[^ {0}]([^{0}]*[^ {0}])?$'.format(
)
+def get_clipboard():
+ return session.get('clipboard:paths', ('select', ()))
+
+
@actions.route('/create/directory', methods=('GET', 'POST'),
defaults={'path': ''})
@actions.route('/create/directory/', methods=('GET', 'POST'))
@@ -101,24 +104,23 @@ def selection(path):
mode = action
break
- clipboard = Clipboard(request.form.getlist('path'), mode)
+ clipboard = request.form.getlist('path')
if mode in ('cut', 'copy'):
- response = redirect(url_for('browse', path=directory.urlpath))
- clipboard.to_response(response)
- return response
+ session['clipboard:paths'] = (mode, clipboard)
+ return redirect(url_for('browse', path=directory.urlpath))
raise InvalidClipboardModeError(
path=directory.path,
+ mode=mode,
clipboard=clipboard,
- mode=clipboard.mode,
)
- clipboard = Clipboard.from_request()
- clipboard.mode = 'select' # disables exclusion
+ mode, clipboard = get_clipboard()
return render_template(
'selection.file_actions.html',
file=directory,
+ mode=mode,
clipboard=clipboard,
cut_support=any(node.can_remove for node in directory.listdir()),
sort_property=sort_property,
@@ -142,52 +144,50 @@ def clipboard_paste(path):
):
return NotFound()
- clipboard = Clipboard.from_request()
- success, issues = paste_clipboard(directory, clipboard)
+ mode, clipboard = get_clipboard()
+ success, issues, mode = paste_clipboard(directory, mode, clipboard)
if issues:
raise InvalidClipboardItemsError(
path=directory.path,
+ mode=mode,
clipboard=clipboard,
issues=issues
)
- if clipboard.mode == 'cut':
- clipboard.clear()
+ if mode == 'cut':
+ del session['clipboard:paths']
- response = redirect(url_for('browse', path=directory.urlpath))
- clipboard.to_response(response)
- return response
+ return redirect(url_for('browse', path=directory.urlpath))
@actions.route('/clipboard/clear', defaults={'path': ''})
@actions.route('/clipboard/clear/')
def clipboard_clear(path):
- response = redirect(url_for('browse', path=path))
- clipboard = Clipboard.from_request()
- clipboard.clear()
- clipboard.to_response(response)
- return response
+ if 'clipboard:paths' in session:
+ del session['clipboard:paths']
+ return redirect(url_for('browse', path=path))
@actions.errorhandler(FileActionsException)
def clipboard_error(e):
file = Node(e.path) if hasattr(e, 'path') else None
- clipboard = getattr(e, 'clipboard', None)
+ mode, clipboard = get_clipboard()
issues = getattr(e, 'issues', ())
- response = make_response((
+ if clipboard:
+ for issue in issues:
+ if isinstance(issue.error, FileNotFoundError):
+ path = issue.item.urlpath
+ if path in clipboard:
+ clipboard.remove(path)
+
+ return (
render_template(
'400.file_actions.html',
- error=e, file=file, clipboard=clipboard, issues=issues,
+ error=e, file=file, mode=mode, clipboard=clipboard, issues=issues,
),
400
- ))
- if clipboard:
- for issue in issues:
- if isinstance(issue.error, FileNotFoundError):
- clipboard.remove(issue.item.urlpath)
- clipboard.to_response(response)
- return response
+ )
def register_plugin(manager):
@@ -201,11 +201,11 @@ def register_plugin(manager):
return directory.is_directory and directory.can_upload
def detect_clipboard(directory):
- return directory.is_directory and Clipboard.from_request()
+ return directory.is_directory and 'clipboard:paths' in session
def excluded_clipboard(path):
- clipboard = Clipboard.from_request(request)
- if clipboard.mode == 'cut':
+ mode, clipboard = get_clipboard()
+ if mode == 'cut':
base = manager.app.config['directory_base']
return abspath_to_urlpath(path, base) in clipboard
@@ -238,7 +238,10 @@ def register_plugin(manager):
html=lambda file: render_template(
'widget.clipboard.file_actions.html',
file=file,
- clipboard=Clipboard.from_request()
+ **dict(zip(
+ ('mode', 'clipboard'),
+ get_clipboard(),
+ )),
),
filter=detect_clipboard,
)
diff --git a/browsepy/plugin/file_actions/clipboard.py b/browsepy/plugin/file_actions/clipboard.py
deleted file mode 100644
index 9c0f0b7..0000000
--- a/browsepy/plugin/file_actions/clipboard.py
+++ /dev/null
@@ -1,106 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-
-import os
-import json
-import base64
-import logging
-import hashlib
-
-from flask import request
-from browsepy.http import DataCookie
-from browsepy.exceptions import InvalidCookieSizeError
-
-from .exceptions import InvalidClipboardSizeError
-
-logger = logging.getLogger(__name__)
-
-
-class Clipboard(set):
- '''
- Clipboard (set) with convenience methods to pick its state from request
- cookies and save it to response cookies.
- '''
- data_cookie = DataCookie('clipboard', max_pages=20)
- secret = os.urandom(256)
- request_cache_field = '_browsepy_file_actions_clipboard_cache'
-
- @classmethod
- def count(cls, request=request):
- '''
- Get how many clipboard items are stores on request cookies.
-
- :param request: optional request, defaults to current flask request
- :type request: werkzeug.Request
- :return: number of clipboard items on request's cookies
- :rtype: int
- '''
- return len(cls.from_request(request))
-
- @classmethod
- def from_request(cls, request=request):
- '''
- Create clipboard object from request cookies.
- Uses request itself for cache.
-
- :param request: optional request, defaults to current flask request
- :type request: werkzeug.Request
- :returns: clipboard instance
- :rtype: Clipboard
- '''
- cached = getattr(request, cls.request_cache_field, None)
- if cached is not None:
- return cached
- self = cls()
- setattr(request, cls.request_cache_field, self)
- try:
- self.__setstate__(cls.data_cookie.load_headers(request.headers))
- except BaseException:
- pass
- return self
-
- @classmethod
- def _signature(cls, items, mode):
- serialized = json.dumps([items, mode]).encode('utf-8')
- data = cls.secret + serialized
- return base64.b64encode(hashlib.sha512(data).digest()).decode('ascii')
-
- def __init__(self, iterable=(), mode='copy'):
- self.mode = mode
- super(Clipboard, self).__init__(iterable)
-
- def __getstate__(self):
- items = list(self)
- return {
- 'mode': self.mode,
- 'items': items,
- 'signature': self._signature(items, self.mode),
- }
-
- def __setstate__(self, data):
- if data['signature'] == self._signature(data['items'], data['mode']):
- self.update(data['items'])
- self.mode = data['mode']
-
- def to_response(self, response, request=request):
- '''
- Save clipboard state to response taking care of disposing old clipboard
- cookies from request.
-
- :param response: response object to write cookies on
- :type response: werkzeug.Response
- :param request: optional request, defaults to current flask request
- :type request: werkzeug.Request
- '''
- if self:
- data = self.__getstate__()
- try:
- headers = self.data_cookie.dump_headers(data, request.headers)
- except InvalidCookieSizeError as e:
- raise InvalidClipboardSizeError(
- clipboard=self,
- max_cookies=e.max_cookies
- )
- else:
- headers = self.data_cookie.truncate_headers(request.headers)
- response.headers.extend(headers)
diff --git a/browsepy/plugin/file_actions/paste.py b/browsepy/plugin/file_actions/paste.py
index a1de913..2163f1b 100644
--- a/browsepy/plugin/file_actions/paste.py
+++ b/browsepy/plugin/file_actions/paste.py
@@ -40,11 +40,10 @@ def move(target, node, join_fnc=os.path.join):
return node.path
-def paste_clipboard(target, clipboard):
+def paste_clipboard(target, mode, clipboard):
'''
Get pasting function for given directory and keyboard.
'''
- mode = clipboard.mode
if mode == 'cut':
paste_fnc = functools.partial(move, target)
elif mode == 'copy':
@@ -52,18 +51,15 @@ def paste_clipboard(target, clipboard):
else:
raise InvalidClipboardModeError(
path=target.path,
+ mode=mode,
clipboard=clipboard,
- mode=mode
)
success = []
issues = []
- try:
- clipboard.mode = 'paste' # deactivates excluded_clipboard fnc
- for node in map(Node.from_urlpath, clipboard):
- try:
- success.append(paste_fnc(node))
- except BaseException as e:
- issues.append((node, e))
- finally:
- clipboard.mode = mode
- return success, issues
+ mode = 'paste' # deactivates excluded_clipboard fnc
+ for node in map(Node.from_urlpath, clipboard):
+ try:
+ success.append(paste_fnc(node))
+ except BaseException as e:
+ issues.append((node, e))
+ return success, issues, mode
diff --git a/browsepy/plugin/file_actions/templates/selection.file_actions.html b/browsepy/plugin/file_actions/templates/selection.file_actions.html
index 646cb7a..ac333f9 100644
--- a/browsepy/plugin/file_actions/templates/selection.file_actions.html
+++ b/browsepy/plugin/file_actions/templates/selection.file_actions.html
@@ -43,7 +43,7 @@
{{ f.link.text }}
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index bc44843..a5fb70b 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -13,7 +13,6 @@ from werkzeug.utils import cached_property
from werkzeug.http import Headers, parse_cookie, dump_cookie
import browsepy.plugin.file_actions as file_actions
-import browsepy.plugin.file_actions.clipboard as file_actions_clipboard
import browsepy.plugin.file_actions.exceptions as file_actions_exceptions
import browsepy.http as browsepy_http
import browsepy.file as browsepy_file
@@ -108,19 +107,6 @@ class ResponseMock(CookieHeaderMock):
client.environ_base['REMOTE_ADDR'], name, **options)
-class RequestMock(CookieHeaderMock):
- def load_cookies(self, client):
- self.clear()
-
- for cookie in client.cookie_jar:
- self.set_cookie(cookie.name, cookie.value)
-
- def uncache(self):
- field = file_actions_clipboard.Clipboard.request_cache_field
- if hasattr(self, field):
- delattr(self, field)
-
-
class Page(object):
def __init__(self, source):
self.tree = bs4.BeautifulSoup(source, 'html.parser')
@@ -209,7 +195,6 @@ class TestIntegration(unittest.TestCase):
actions_module = file_actions
manager_module = browsepy_manager
browsepy_module = browsepy
- clipboard_module = file_actions_clipboard
def setUp(self):
self.base = tempfile.mkdtemp()
@@ -251,14 +236,8 @@ class TestIntegration(unittest.TestCase):
page.widgets
)
- reqmock = RequestMock()
- reqmock.load_cookies(client)
- resmock = ResponseMock()
- clipboard = self.clipboard_module.Clipboard.from_request(reqmock)
- clipboard.mode = 'copy'
- clipboard.add('whatever')
- clipboard.to_response(resmock, reqmock)
- resmock.dump_cookies(client)
+ with client.request_context():
+ flask.session['clipboard:paths'] = ('copy', ['whatever'])
response = client.get('/')
self.assertEqual(response.status_code, 200)
diff --git a/browsepy/tests/meta.py b/browsepy/tests/meta.py
new file mode 100644
index 0000000..c929d35
--- /dev/null
+++ b/browsepy/tests/meta.py
@@ -0,0 +1,75 @@
+try:
+ import importlib.resources as res
+except ImportError: # pragma: no cover
+ import importlib_resources as res # to support python < 3.7
+
+
+class TestFileMeta(type):
+ '''
+ This metaclass generates test methods for every file matching given
+ rules (as class properties).
+
+ The added test methods will call an existing meta_test method passing
+ module name and filename as arguments.
+
+ Functions from :module:`importlib.resources` (see resource_methods) are
+ also injected for convenience.
+
+ Honored class properties:
+ - meta_test: called with module and filename by injected test methods.
+ - meta_module: module to inspect for files
+ - meta_file_extensions: filename extensions will result on test injection.
+ - meta_prefix: prefix added to injected tests (defaults to `file`)
+ '''
+
+ resource_methods = (
+ 'contents',
+ 'is_resource',
+ 'open_binary',
+ 'open_text',
+ 'path',
+ 'read_binary',
+ 'read_text',
+ )
+
+ @classmethod
+ def iter_contents(cls, module, extensions):
+ for item in res.contents(module):
+ if res.is_resource(module, item):
+ if any(map(item.endswith, extensions)):
+ yield (module, item)
+ continue
+ submodule = '%s.%s' % (module, item)
+ try:
+ for subitem in cls.iter_contents(submodule, extensions):
+ yield subitem
+ except ImportError:
+ pass
+
+ @classmethod
+ def create_method(cls, module, filename, prefix, extensions):
+ def test(self):
+ self.meta_test(module, filename)
+ strip = max(len(ext) for ext in extensions if filename.endswith(ext))
+ test.__name__ = 'test_%s_%s_%s' % (
+ prefix,
+ module.replace('.', '_').strip('_'),
+ filename[:-strip].strip('_'),
+ )
+ return test
+
+ def __init__(self, name, bases, dct):
+ super(TestFileMeta, self).__init__(name, bases, dct)
+
+ # generate tests from files
+ name = self.meta_module
+ prefix = getattr(self, 'meta_prefix', 'file')
+ extensions = self.meta_file_extensions
+ for module, item in self.iter_contents(name, extensions):
+ test = self.create_method(module, item, prefix, extensions)
+ setattr(self, test.__name__, test)
+
+ # add resource methods
+ for method in self.resource_methods:
+ if not hasattr(self, method):
+ setattr(self, method, staticmethod(getattr(res, method)))
diff --git a/browsepy/tests/test_code_quality.py b/browsepy/tests/test_code_quality.py
index 0983943..a0b0b35 100644
--- a/browsepy/tests/test_code_quality.py
+++ b/browsepy/tests/test_code_quality.py
@@ -1,74 +1,29 @@
-import re
-import os.path
import unittest
-import functools
+import six
import pycodestyle
-
-class DeferredReport(pycodestyle.StandardReport):
- def __init__(self, *args, **kwargs):
- self.print_fnc = kwargs.pop('print_fnc')
- self.location_base = kwargs.pop('location_base')
- super(DeferredReport, self).__init__(*args, **kwargs)
-
- @property
- def location(self):
- if self.filename:
- return os.path.relpath(self.filename, self.location_base)
- return self.filename
-
- def get_file_results(self):
- self._deferred_print.sort()
- for line_number, offset, code, text, doc in self._deferred_print:
- error = {
- 'path': self.location,
- 'row': self.line_offset + line_number,
- 'col': offset + 1,
- 'code': code,
- 'text': text,
- }
- lines = [self._fmt % error]
- if line_number <= len(self.lines):
- line = self.lines[line_number - 1]
- lines.extend((
- line.rstrip(),
- re.sub(r'\S', ' ', line[:offset]) + '^'
- ))
- if doc:
- lines.append(' ' + doc.strip())
- error['message'] = '\n'.join(lines)
- self.print_fnc(error)
- return self.file_errors
-
-
-class TestCodeQuality(unittest.TestCase):
- base = os.path.join(
- os.path.dirname(__file__),
- os.path.pardir,
- )
- setup_py = os.path.join(base, '..', 'setup.py')
- config_file = os.path.join(base, '..', 'setup.cfg')
-
- def test_conformance(self):
- '''
- Test pep-8 conformance
- '''
- messages = []
- style = pycodestyle.StyleGuide(
- config_file=self.config_file,
- paths=[self.base, self.setup_py],
- reporter=functools.partial(
- DeferredReport,
- location_base=self.base,
- print_fnc=messages.append
+from . import meta
+
+
+class TestCodeFormat(six.with_metaclass(meta.TestFileMeta, unittest.TestCase)):
+ '''
+ pycodestyle unit test
+ '''
+ meta_module = 'browsepy'
+ meta_prefix = 'code'
+ meta_file_extensions = ('.py',)
+
+ def meta_test(self, module, filename):
+ style = pycodestyle.StyleGuide(quiet=False)
+ with self.path(module, filename) as f:
+ result = style.check_files([str(f)])
+ self.assertFalse(result.total_errors, (
+ 'Found {errors} code style error{s} (or warning{s}) '
+ 'on module {module}, file {filename!r}.').format(
+ errors=result.total_errors,
+ s='s' if result.total_errors > 1 else '',
+ module=module,
+ filename=filename,
)
)
- result = style.check_files()
- text = ''.join(
- '%s\n %s' % ('' if n else '\n', line)
- for message in messages
- for n, line in enumerate(message['message'].splitlines())
- )
- self.assertEqual(
- result.total_errors, 0, "Code style errors:%s" % text)
diff --git a/browsepy/tests/test_http.py b/browsepy/tests/test_http.py
index 862aace..a5b3ec4 100644
--- a/browsepy/tests/test_http.py
+++ b/browsepy/tests/test_http.py
@@ -1,12 +1,6 @@
+# -*- coding: UTF-8 -*-
-import os
import unittest
-import datetime
-import base64
-import zlib
-
-import werkzeug.http
-import werkzeug.datastructures
import browsepy.http
import browsepy.exceptions
@@ -34,116 +28,3 @@ class TestHeaders(unittest.TestCase):
headers.get('Option-Header'),
'something; option=1',
)
-
-
-class TestParseSetCookie(unittest.TestCase):
- module = browsepy.http
-
- def test_parse(self):
- tests = {
- 'value-cookie=value': {
- 'name': 'value-cookie',
- 'value': 'value',
- },
- 'expiration-cookie=value; Expires=Thu, 24 May 2018 18:10:26 GMT': {
- 'name': 'expiration-cookie',
- 'value': 'value',
- 'expires': datetime.datetime(2018, 5, 24, 18, 10, 26),
- },
- 'maxage-cookie="value with spaces"; Max-Age=0': {
- 'name': 'maxage-cookie',
- 'value': 'value with spaces',
- 'max_age': 0,
- },
- 'secret-cookie; HttpOnly; Secure': {
- 'name': 'secret-cookie',
- 'value': '',
- 'httponly': True,
- 'secure': True,
- },
- 'spaced name=value': {
- 'name': 'spaced name',
- 'value': 'value',
- },
- }
- for cookie, data in tests.items():
- name, parsed = self.module.parse_set_cookie(cookie)
- parsed['name'] = name
- self.assertEquals(parsed, data)
-
-
-class TestDataCookie(unittest.TestCase):
- module = browsepy.http
- manager_cls = module.DataCookie
- header_cls = werkzeug.datastructures.Headers
- cookie_size_exception = browsepy.exceptions.InvalidCookieSizeError
-
- def random_text(self, size=2):
- bytedata = zlib.compress(os.urandom(size * 3), 9) # ensure entropy
- b64data = base64.b64encode(bytedata)
- return b64data.decode('ascii')[:size]
-
- def parse_response_cookies(self, headers):
- '''
- :type headers: werkzeug.datastructures.Headers
- '''
- return {
- name: options
- for name, options in map(
- self.module.parse_set_cookie,
- headers.get_all('set-cookie')
- )}
-
- def response_to_request_headers(self, headers):
- '''
- :type headers: werkzeug.datastructures.Headers
- '''
- return self.header_cls(
- ('Cookie', werkzeug.http.dump_cookie(name, options['value']))
- for name, options in self.parse_response_cookies(headers).items()
- )
-
- def test_pagination(self):
- data = self.random_text(self.manager_cls.header_max_size)
- manager = self.manager_cls('cookie', max_pages=2)
-
- rheaders = manager.dump_headers(data)
- self.assertEqual(len(rheaders), 2)
-
- qheaders = self.response_to_request_headers(rheaders)
- self.assertEqual(manager.load_headers(qheaders), data)
-
- rheaders = manager.dump_headers('shorter-data', qheaders)
- self.assertEqual(len(rheaders), 2) # 1 for value, 1 for discard
-
- cookies = self.parse_response_cookies(rheaders)
-
- cookie1 = cookies['cookie']
- deserialized = manager.load_cookies({'cookie': cookie1['value']})
- self.assertEqual(deserialized, 'shorter-data')
-
- cookie2 = cookies['cookie-2']
- self.assertEqual(cookie2['value'], '')
- self.assertLess(cookie2['expires'], datetime.datetime.now())
- self.assertLess(cookie2['max_age'], 1)
-
- def test_max_pagination(self):
- manager = self.manager_cls('cookie', max_pages=2)
- self.assertRaises(
- self.cookie_size_exception,
- manager.dump_headers,
- self.random_text(self.manager_cls.header_max_size * 2)
- )
-
- def test_truncate(self):
- qheaders = self.header_cls([('Cookie', 'cookie=value')])
- manager = self.manager_cls('cookie', max_pages=2)
- rheaders = manager.truncate_headers(qheaders)
- parsed = self.parse_response_cookies(rheaders)
- self.assertEqual(parsed['cookie']['value'], '')
-
- def test_corrupted(self):
- manager = self.manager_cls('cookie')
- default = object()
- data = manager.load_cookies({'cookie': 'corrupted'}, default)
- self.assertIs(data, default)
diff --git a/browsepy/utils.py b/browsepy/utils.py
new file mode 100644
index 0000000..3b37ff8
--- /dev/null
+++ b/browsepy/utils.py
@@ -0,0 +1,15 @@
+
+import os
+import os.path
+import random
+import functools
+
+ppath = functools.partial(
+ os.path.join,
+ os.path.dirname(os.path.realpath(__file__)),
+ )
+
+
+def random_string(size, sample=tuple(map(chr, range(256)))):
+ randrange = functools.partial(random.randrange, 0, len(sample))
+ return ''.join(sample[randrange()] for i in range(size))
diff --git a/requirements.txt b/requirements.txt
index 48f4d82..5968891 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,7 +12,6 @@
# ---------------------------------
#
# - requirements.txt
-# - requirements/compat.txt
# - requirements/development.txt
# - requirements/base.txt
#
@@ -20,10 +19,8 @@
# --------
#
# - requirements/ci.txt
-# - requirements/compat.txt
# - requirements/development.txt
#
--r requirements/compat.txt
-r requirements/development.txt
diff --git a/requirements/base.txt b/requirements/base.txt
index 65fe872..4518665 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -5,16 +5,18 @@
# This module lists the strict minimal subset of dependencies required to
# run this application.
#
-# Depending on your python environment, optional modules could be also
-# required. They're defined on `requirements/compat.txt`.
-#
# See also
# --------
#
# - requirements.txt
-# - requirements/compat.txt
# - requirements/development.txt
#
flask
+cookieman
unicategories
+
+# compat
+six
+scandir # python < 3.6
+backports.shutil_get_terminal_size # python < 3.3
diff --git a/requirements/ci.txt b/requirements/ci.txt
index a7d54ba..c6aa8f1 100644
--- a/requirements/ci.txt
+++ b/requirements/ci.txt
@@ -16,7 +16,6 @@
# --------
#
# - requirements.txt
-# - requirements/compat.txt
# - requirements/development.txt
#
diff --git a/requirements/compat.txt b/requirements/compat.txt
deleted file mode 100644
index a593b7a..0000000
--- a/requirements/compat.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Compatibility requirements file
-# ===============================
-#
-# This module includes optional dependencies which could be required (or not)
-# based on your python version.
-#
-# They should be forward-compatible so you can install them all if you are not
-# sure.
-#
-# This requirements are conditional dependencies on 'source' distributable
-# packages, but regular dependencies of 'binary' distributable packages
-# (see `setup.py`).
-#
-# See also
-# --------
-#
-# - requirements.txt
-# - requirements/ci.txt
-# - requirements/development.txt
-#
-
-# python < 3.6
-scandir
-
-# python < 3.3
-backports.shutil_get_terminal_size
diff --git a/requirements/development.txt b/requirements/development.txt
index 634202a..cd9a5cd 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -18,15 +18,26 @@
#
# - requirements.txt
# - requirements/ci.txt
-# - requirements/compat.txt
#
-r base.txt
-setuptools
-wheel
-pycodestyle
+# linting
flake8
+doc8
+yapf
coverage
+jedi
+
+# dev
+sphinx
+pycodestyle
+importlib_resources # pycodestyle unittest with python < 3.7
+six # compatible metaclass with python 2-3
+pyyaml
beautifulsoup4
-pyaml
+
+# dist
+setuptools
+wheel
+twine
diff --git a/requirements/doc.txt b/requirements/doc.txt
index 2c2e5f6..1617bb9 100644
--- a/requirements/doc.txt
+++ b/requirements/doc.txt
@@ -18,7 +18,6 @@
#
# - requirements.txt
# - requirements/ci.txt
-# - requirements/compat.txt
#
-r base.txt
diff --git a/setup.cfg b/setup.cfg
index 8104f2b..9cce8f5 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,26 +1,34 @@
[metadata]
description-file = README.rst
-[wheel]
+[bdist_wheel]
universal = 1
[egg_info]
tag_build =
[pycodestyle]
-show-source = True
ignore = E123,E126,E121,W504
[flake8]
+ignore = E123,E126,E121,W504
+max-complexity = 5
select = F,C
-max-complexity = 9
show-source = True
[yapf]
align_closing_bracket_with_visual_indent = true
[coverage:run]
+branch = True
source = browsepy
omit =
*/tests/*
*/tests.py
+
+[coverage:report]
+show_missing = True
+fail_under = 100
+exclude_lines =
+ pragma: no cover
+ pass
diff --git a/setup.py b/setup.py
index 38a56ed..cf2ed59 100644
--- a/setup.py
+++ b/setup.py
@@ -1,55 +1,17 @@
# -*- coding: utf-8 -*-
-"""
-browsepy
-========
-
-Simple web file browser with directory gzipped tarball download, file upload,
-removal and plugins.
-
-More details on project's README and
-`github page `_.
-
-
-Development Version
--------------------
-
-The browsepy development version can be installed by cloning the git
-repository from `github`_::
-
- git clone git@github.com:ergoithz/browsepy.git
-
-.. _github: https://github.com/ergoithz/browsepy
-
-License
--------
-MIT (see LICENSE file).
-"""
+import io
+import re
import os
-import os.path
import sys
-import shutil
-
-try:
- from setuptools import setup
-except ImportError:
- from distutils.core import setup
-
-sys.path[:], sys_path = [os.path.abspath('browsepy')], sys.path[:]
-import __meta__ as meta # noqa
-sys.path[:] = sys_path
-with open('README.rst') as f:
- meta_doc = f.read()
+from setuptools import setup, find_packages
-extra_requires = []
-bdist = 'bdist' in sys.argv or any(a.startswith('bdist_') for a in sys.argv)
+with io.open('README.rst', 'rt', encoding='utf8') as f:
+ readme = f.read()
-if bdist or not hasattr(os, 'scandir'): # python 3.5+
- extra_requires.append('scandir')
-
-if bdist or not hasattr(shutil, 'get_terminal_size'): # python 3.3+
- extra_requires.append('backports.shutil_get_terminal_size')
+with io.open('browsepy/__init__.py', 'rt', encoding='utf8') as f:
+ version = re.search(r'__version__ = \'(.*?)\'', f.read()).group(1)
for debugger in ('ipdb', 'pudb', 'pdb'):
opt = '--debug=%s' % debugger
@@ -58,33 +20,23 @@ for debugger in ('ipdb', 'pudb', 'pdb'):
sys.argv.remove(opt)
setup(
- name=meta.app,
- version=meta.version,
- url=meta.url,
- download_url=meta.tarball,
- license=meta.license,
- author=meta.author_name,
- author_email=meta.author_mail,
- description=meta.description,
- long_description=meta_doc,
+ name='browsepy',
+ version=version,
+ url='https://github.com/ergoithz/browsepy',
+ license='MIT',
+ author='Felipe A. Hernandez',
+ author_email='ergoithz@gmail.com',
+ description='Simple web file browser',
+ long_description=readme,
+ long_description_content_type='text/x-rst',
classifiers=[
'Development Status :: 4 - Beta',
'License :: OSI Approved :: MIT License',
'Operating System :: OS Independent',
- 'Programming Language :: Python',
'Programming Language :: Python :: 3',
],
keywords=['web', 'file', 'browser'],
- packages=[
- 'browsepy',
- 'browsepy.tests',
- 'browsepy.tests.deprecated',
- 'browsepy.tests.deprecated.plugin',
- 'browsepy.transform',
- 'browsepy.plugin',
- 'browsepy.plugin.player',
- 'browsepy.plugin.file_actions',
- ],
+ packages=find_packages(),
entry_points={
'console_scripts': (
'browsepy=browsepy.__main__:main'
@@ -105,9 +57,13 @@ setup(
'static/*',
],
},
- install_requires=(
- ['flask', 'unicategories'] + extra_requires
- ),
+ install_requires=[
+ 'flask',
+ 'unicategories',
+ 'scandir',
+ 'importlib_resources',
+ 'backports.shutil_get_terminal_size'
+ ],
extras_require={
'tests': ['beautifulsoup4', 'pycodestyle'],
},
--
GitLab
From 7111ac9c74ceb5c7d910615632811fe6ec4f61d2 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 31 Mar 2019 00:37:21 +0100
Subject: [PATCH 043/171] fix tests, simplify clipboard
---
browsepy/__init__.py | 25 +--
browsepy/__main__.py | 2 +-
browsepy/manager.py | 28 +++
browsepy/plugin/file_actions/__init__.py | 53 +++--
browsepy/plugin/file_actions/exceptions.py | 28 +--
browsepy/plugin/file_actions/tests.py | 223 +++------------------
6 files changed, 105 insertions(+), 254 deletions(-)
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index 729916b..c728e74 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -30,7 +30,8 @@ app = Flask(
template_folder=utils.ppath('templates'),
)
app.config.update(
- application_name='browsepy',
+ SECRET_KEY=utils.random_string(4096),
+ APPLICATION_NAME='browsepy',
directory_base=compat.getcwd(),
directory_start=None,
directory_remove=None,
@@ -47,18 +48,15 @@ app.config.update(
exclude_fnc=None,
)
app.jinja_env.add_extension('browsepy.transform.htmlcompress.HTMLCompress')
-app.secret_key = utils.random_string(4096)
+app.session_interface = cookieman.CookieMan()
if 'BROWSEPY_SETTINGS' in os.environ:
app.config.from_envvar('BROWSEPY_SETTINGS')
plugin_manager = PluginManager(app)
-session_manager = cookieman.CookieMan()
-app.session_interface = session_manager
-
-@session_manager.register('browse:sort')
+@app.session_interface.register('browse:sort')
def shrink_browse_sort(data, last):
if data['browse:sort'] and not last:
data['browse:sort'].pop()
@@ -74,7 +72,7 @@ def get_cookie_browse_sorting(path, default):
:returns: sorting property
:rtype: string
'''
- if request:
+ if session:
for cpath, cprop in session.get('browse:sort', ()):
if path == cpath:
return cprop
@@ -159,15 +157,10 @@ def sort(property, path):
if not directory.is_directory or directory.is_excluded:
return NotFound()
- data = [(path, property)]
- try:
- data.extend(
- (cpath, cprop)
- for cpath, cprop in session.get('browse:sort', ())
- if cpath != path
- )
- except BaseException:
- pass
+ if session:
+ data = session.get('browse:sort', [])
+ data.insert(0, (path, property))
+ session.modified = True
return redirect(url_for(".browse", path=directory.urlpath))
diff --git a/browsepy/__main__.py b/browsepy/__main__.py
index d453cfe..948b01a 100644
--- a/browsepy/__main__.py
+++ b/browsepy/__main__.py
@@ -52,7 +52,7 @@ class ArgParse(argparse.ArgumentParser):
)
default_removable = app.config['directory_remove']
default_upload = app.config['directory_upload']
- name = app.config['application_name']
+ name = app.config['APPLICATION_NAME']
default_host = os.getenv('BROWSEPY_HOST', '127.0.0.1')
default_port = os.getenv('BROWSEPY_PORT', '8080')
diff --git a/browsepy/manager.py b/browsepy/manager.py
index fc9ce13..b8124e9 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -10,6 +10,8 @@ import collections
from flask import current_app
from werkzeug.utils import cached_property
+from cookieman import CookieMan
+
from . import mimetype
from . import compat
from .compat import deprecated, usedoc
@@ -492,6 +494,31 @@ class MimetypePluginManager(RegistrablePluginManager):
self._mimetype_functions.insert(0, fnc)
+class SessionPluginManager(PluginManagerBase):
+ def register_session(self, key_or_keys, shrink_fnc=None):
+ '''
+ Register session shrink function for specific session key or
+ keys. Can be used as decorator.
+
+ Usage:
+ >>> @app.session_interface.register('my_session_key')
+ ... def my_shrink_fnc(data):
+ ... del data['my_session_key']
+ ... return data
+
+ :param key_or_keys: key or iterable of keys would be affected
+ :type key_or_keys: Union[str, Iterable[str]]
+ :param shrink_fnc: shrinking function (optional for decorator)
+ :type shrink_fnc: cookieman.abc.ShrinkFunction
+ :returns: either original given shrink_fnc or decorator
+ :rtype: cookieman.abc.ShrinkFunction
+ '''
+ interface = self.app.session_interface
+ if isinstance(interface, CookieMan):
+ return interface.register(key_or_keys, shrink_fnc)
+ return shrink_fnc
+
+
class ArgumentPluginManager(PluginManagerBase):
'''
Plugin manager for command-line argument registration.
@@ -753,6 +780,7 @@ class PluginManager(MimetypeActionPluginManager,
ExcludePluginManager,
WidgetPluginManager,
MimetypePluginManager,
+ SessionPluginManager,
ArgumentPluginManager):
'''
Main plugin manager
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index bdcd74c..229f801 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -40,10 +40,6 @@ re_basename = '^[^ {0}]([^{0}]*[^ {0}])?$'.format(
)
-def get_clipboard():
- return session.get('clipboard:paths', ('select', ()))
-
-
@actions.route('/create/directory', methods=('GET', 'POST'),
defaults={'path': ''})
@actions.route('/create/directory/', methods=('GET', 'POST'))
@@ -107,7 +103,8 @@ def selection(path):
clipboard = request.form.getlist('path')
if mode in ('cut', 'copy'):
- session['clipboard:paths'] = (mode, clipboard)
+ session['clipboard:mode'] = mode
+ session['clipboard:items'] = clipboard
return redirect(url_for('browse', path=directory.urlpath))
raise InvalidClipboardModeError(
@@ -116,12 +113,11 @@ def selection(path):
clipboard=clipboard,
)
- mode, clipboard = get_clipboard()
return render_template(
'selection.file_actions.html',
file=directory,
- mode=mode,
- clipboard=clipboard,
+ mode=session.get('clipboard:mode'),
+ clipboard=session.get('clipboard:items', ()),
cut_support=any(node.can_remove for node in directory.listdir()),
sort_property=sort_property,
sort_fnc=sort_fnc,
@@ -144,8 +140,14 @@ def clipboard_paste(path):
):
return NotFound()
- mode, clipboard = get_clipboard()
- success, issues, mode = paste_clipboard(directory, mode, clipboard)
+ mode = session.get('clipboard:mode')
+ clipboard = session.get('clipboard:items', ())
+
+ success, issues, nmode = paste_clipboard(directory, mode, clipboard)
+ if clipboard and mode != nmode:
+ session['mode'] = nmode
+ mode = nmode
+
if issues:
raise InvalidClipboardItemsError(
path=directory.path,
@@ -171,20 +173,25 @@ def clipboard_clear(path):
@actions.errorhandler(FileActionsException)
def clipboard_error(e):
file = Node(e.path) if hasattr(e, 'path') else None
- mode, clipboard = get_clipboard()
issues = getattr(e, 'issues', ())
- if clipboard:
+ if session.get('clipboard:items'):
+ clipboard = session['clipboard:mode']
for issue in issues:
if isinstance(issue.error, FileNotFoundError):
path = issue.item.urlpath
if path in clipboard:
clipboard.remove(path)
+ session.modified = True
return (
render_template(
'400.file_actions.html',
- error=e, file=file, mode=mode, clipboard=clipboard, issues=issues,
+ error=e,
+ file=file,
+ mode=session.get('clipboard:mode'),
+ clipboard=session.get('clipboard:items', ()),
+ issues=issues,
),
400
)
@@ -201,14 +208,22 @@ def register_plugin(manager):
return directory.is_directory and directory.can_upload
def detect_clipboard(directory):
- return directory.is_directory and 'clipboard:paths' in session
+ return directory.is_directory and session.get('clipboard:mode')
def excluded_clipboard(path):
- mode, clipboard = get_clipboard()
- if mode == 'cut':
+ if session.get('clipboard:mode') == 'cut':
base = manager.app.config['directory_base']
+ clipboard = session.get('clipboard:items', ())
return abspath_to_urlpath(path, base) in clipboard
+ def shrink(data, last):
+ if last:
+ # TODO: add warning message
+ del data['clipboard:items']
+ del data['clipboard:mode']
+ return data
+
+ manager.register_session(('clipboard:items', 'clipboard:mode'), shrink)
manager.register_exclude_function(excluded_clipboard)
manager.register_blueprint(actions)
manager.register_widget(
@@ -238,10 +253,8 @@ def register_plugin(manager):
html=lambda file: render_template(
'widget.clipboard.file_actions.html',
file=file,
- **dict(zip(
- ('mode', 'clipboard'),
- get_clipboard(),
- )),
+ mode=session.get('clipboard:mode'),
+ clipboard=session.get('clipboard:items', ()),
),
filter=detect_clipboard,
)
diff --git a/browsepy/plugin/file_actions/exceptions.py b/browsepy/plugin/file_actions/exceptions.py
index 35dd68e..d9b4391 100644
--- a/browsepy/plugin/file_actions/exceptions.py
+++ b/browsepy/plugin/file_actions/exceptions.py
@@ -67,7 +67,8 @@ class ClipboardException(FileActionsException):
code = 'clipboard-invalid'
template = 'Clipboard is invalid.'
- def __init__(self, message=None, path=None, clipboard=None):
+ def __init__(self, message=None, path=None, mode=None, clipboard=None):
+ self.mode = mode
self.clipboard = clipboard
super(ClipboardException, self).__init__(message, path)
@@ -108,10 +109,11 @@ class InvalidClipboardItemsError(ClipboardException):
code = 'clipboard-invalid-items'
template = 'Clipboard has invalid items.'
- def __init__(self, message=None, path=None, clipboard=None, issues=()):
+ def __init__(self, message=None, path=None, mode=None, clipboard=None,
+ issues=()):
self.issues = list(map(self.pair_class, issues))
supa = super(InvalidClipboardItemsError, self)
- supa.__init__(message, path, clipboard)
+ supa.__init__(message, path, mode, clipboard)
def append(self, item, error):
self.issues.append(self.pair_class((item, error)))
@@ -126,22 +128,6 @@ class InvalidClipboardModeError(ClipboardException):
code = 'clipboard-invalid-mode'
template = 'Clipboard mode {0.path!r} is not valid.'
- def __init__(self, message=None, path=None, clipboard=None, mode=None):
- self.mode = mode
+ def __init__(self, message=None, path=None, mode=None, clipboard=None):
supa = super(InvalidClipboardModeError, self)
- supa.__init__(message, path, clipboard)
-
-
-class InvalidClipboardSizeError(ClipboardException):
- '''
- Exception raised when a clipboard size exceeds cookie limit.
-
- :property max_cookies: maximum allowed size
- '''
- code = 'clipboard-invalid-size'
- template = 'Clipboard has too many items.'
-
- def __init__(self, message=None, path=None, clipboard=None, max_cookies=0):
- self.max_cookies = max_cookies
- supa = super(InvalidClipboardSizeError, self)
- supa.__init__(message, path, clipboard)
+ supa.__init__(message, path, mode, clipboard)
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index a5fb70b..6f03c46 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -4,109 +4,20 @@ import shutil
import os
import os.path
import functools
-import datetime
import bs4
import flask
from werkzeug.utils import cached_property
-from werkzeug.http import Headers, parse_cookie, dump_cookie
import browsepy.plugin.file_actions as file_actions
import browsepy.plugin.file_actions.exceptions as file_actions_exceptions
-import browsepy.http as browsepy_http
import browsepy.file as browsepy_file
import browsepy.manager as browsepy_manager
import browsepy.exceptions as browsepy_exceptions
import browsepy
-class CookieHeaderMock(object):
- header = 'Cookie'
-
- @property
- def _cookies(self):
- return [
- (name, {'value': value})
- for header in self.headers.get_all(self.header)
- for name, value in parse_cookie(header).items()
- ]
-
- @property
- def cookies(self):
- is_expired = self.expired
- return {
- name: options.get('value')
- for name, options in self._cookies
- if not is_expired(options)
- }
-
- @property
- def expired_cookies(self):
- is_expired = self.expired
- return {
- name: options.get('value')
- for name, options in self._cookies
- if is_expired(options)
- }
-
- def expired(self, options):
- return False
-
- def __init__(self):
- self.headers = Headers()
-
- def add_cookie(self, name, value='', **kwargs):
- self.headers.add(self.header, dump_cookie(name, value, **kwargs))
-
- def set_cookie(self, name, value='', **kwargs):
- owned = [
- (cname, coptions)
- for cname, coptions in self._cookies
- if cname != name
- ]
- self.clear()
- for cname, coptions in owned:
- self.headers.add(self.header, dump_cookie(cname, **coptions))
- self.add_cookie(name, value, **kwargs)
-
- def clear(self):
- self.headers.clear()
-
-
-class ResponseMock(CookieHeaderMock):
- header = 'Set-Cookie'
-
- @property
- def _cookies(self):
- return [
- browsepy_http.parse_set_cookie(header)
- for header in self.headers.get_all(self.header)
- ]
-
- def expired(self, cookie_options):
- dt = datetime.datetime.now()
- return (
- cookie_options.get('max_age', 1) < 1 or
- cookie_options.get('expiration', dt) < dt
- )
-
- def dump_cookies(self, client):
- owned = self._cookies
- if isinstance(client, CookieHeaderMock):
- client.clear()
- for name, options in owned:
- client.add_cookie(name, **options)
- else:
- for cookie in client.cookie_jar:
- client.cookie_jar.clear(
- cookie.domain, cookie.path, cookie.name)
-
- for name, options in owned:
- client.set_cookie(
- client.environ_base['REMOTE_ADDR'], name, **options)
-
-
class Page(object):
def __init__(self, source):
self.tree = bs4.BeautifulSoup(source, 'html.parser')
@@ -200,6 +111,7 @@ class TestIntegration(unittest.TestCase):
self.base = tempfile.mkdtemp()
self.app = self.browsepy_module.app
self.app.config.update(
+ SECRET_KEY='secret',
directory_base=self.base,
directory_start=self.base,
directory_upload=None,
@@ -236,8 +148,9 @@ class TestIntegration(unittest.TestCase):
page.widgets
)
- with client.request_context():
- flask.session['clipboard:paths'] = ('copy', ['whatever'])
+ with client.session_transaction() as session:
+ session['clipboard:mode'] = 'copy'
+ session['clipboard:items'] = ['whatever']
response = client.get('/')
self.assertEqual(response.status_code, 200)
@@ -285,25 +198,16 @@ class TestIntegration(unittest.TestCase):
page = Page(response.data)
self.assertIn('potato', page.entries)
- reqmock = RequestMock()
- reqmock.load_cookies(client)
- resmock = ResponseMock()
- clipboard = self.clipboard_module.Clipboard.from_request(reqmock)
- clipboard.mode = 'copy'
- clipboard.add('potato')
- clipboard.to_response(resmock, reqmock)
- resmock.dump_cookies(client)
-
+ with client.session_transaction() as session:
+ session['clipboard:mode'] = 'copy'
+ session['clipboard:items'] = ['potato']
response = client.get('/')
self.assertEqual(response.status_code, 200)
page = Page(response.data)
self.assertIn('potato', page.entries)
- reqmock.load_cookies(client)
- clipboard.mode = 'cut'
- clipboard.to_response(resmock, reqmock)
- resmock.dump_cookies(client)
-
+ with client.session_transaction() as session:
+ session['clipboard:mode'] = 'cut'
response = client.get('/')
self.assertEqual(response.status_code, 200)
page = Page(response.data)
@@ -312,7 +216,6 @@ class TestIntegration(unittest.TestCase):
class TestAction(unittest.TestCase):
module = file_actions
- clipboard_module = file_actions_clipboard
def setUp(self):
self.base = tempfile.mkdtemp()
@@ -320,6 +223,7 @@ class TestAction(unittest.TestCase):
self.app = flask.Flask('browsepy')
self.app.register_blueprint(self.module.actions)
self.app.config.update(
+ SECRET_KEY='secret',
directory_base=self.base,
directory_upload=self.base,
directory_remove=self.base,
@@ -468,60 +372,41 @@ class TestAction(unittest.TestCase):
self.assertEqual(response.status_code, 404)
with self.app.test_client() as client:
- reqmock = RequestMock()
- reqmock.load_cookies(client)
- resmock = ResponseMock()
- clipboard = self.clipboard_module.Clipboard.from_request(reqmock)
- clipboard.mode = 'wrong-mode'
- clipboard.add('whatever')
- clipboard.to_response(resmock, reqmock)
- resmock.dump_cookies(client)
-
+ with client.session_transaction() as session:
+ session['clipboard:mode'] = 'wrong-mode'
+ session['clipboard:items'] = ['whatever']
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 400)
- reqmock.load_cookies(client)
- clipboard.mode = 'cut'
- clipboard.to_response(resmock, reqmock)
- resmock.dump_cookies(client)
-
+ with client.session_transaction() as session:
+ session['clipboard:mode'] = 'cut'
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 302) # same location
- clipboard.mode = 'cut'
- clipboard.to_response(resmock, reqmock)
- resmock.dump_cookies(client)
-
+ with client.session_transaction() as session:
+ session['clipboard:mode'] = 'cut'
response = client.get('/file-actions/clipboard/paste/target')
self.assertEqual(response.status_code, 400)
- clipboard.mode = 'copy'
- clipboard.to_response(resmock, reqmock)
- resmock.dump_cookies(client)
-
+ with client.session_transaction() as session:
+ session['clipboard:mode'] = 'copy'
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 400)
- clipboard.mode = 'copy'
- clipboard.to_response(resmock, reqmock)
- resmock.dump_cookies(client)
-
+ with client.session_transaction() as session:
+ session['clipboard:mode'] = 'copy'
response = client.get('/file-actions/clipboard/paste/target')
self.assertEqual(response.status_code, 400)
self.app.config['exclude_fnc'] = lambda n: n.endswith('whatever')
- clipboard.mode = 'cut'
- clipboard.to_response(resmock, reqmock)
- resmock.dump_cookies(client)
-
+ with client.session_transaction() as session:
+ session['clipboard:mode'] = 'cut'
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 400)
- clipboard.mode = 'copy'
- clipboard.to_response(resmock, reqmock)
- resmock.dump_cookies(client)
-
+ with client.session_transaction() as session:
+ session['clipboard:mode'] = 'copy'
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 400)
@@ -548,73 +433,19 @@ class TestAction(unittest.TestCase):
self.assertFalse(page.selected)
-class TestClipboard(unittest.TestCase):
- module = file_actions_clipboard
-
- def test_count(self):
- reqmock = RequestMock()
- resmock = ResponseMock()
- self.assertEqual(self.module.Clipboard.count(reqmock), 0)
-
- clipboard = self.module.Clipboard()
- clipboard.mode = 'test'
- clipboard.add('item')
- clipboard.to_response(resmock, reqmock)
- resmock.dump_cookies(reqmock)
- reqmock.uncache()
-
- self.assertEqual(self.module.Clipboard.count(reqmock), 1)
-
- def test_oveflow(self):
- class TinyClipboard(self.module.Clipboard):
- data_cookie = browsepy_http.DataCookie('small-clipboard')
-
- request = RequestMock()
- clipboard = TinyClipboard()
- clipboard.mode = 'test'
- clipboard.update('item-%04d' % i for i in range(4000))
-
- self.assertRaises(
- file_actions_exceptions.InvalidClipboardSizeError,
- clipboard.to_response,
- request,
- request
- )
-
- def test_unreadable(self):
- name = self.module.Clipboard.data_cookie._name_cookie_page(0)
- request = RequestMock()
- request.set_cookie(name, 'a')
- clipboard = self.module.Clipboard.from_request(request)
- self.assertFalse(clipboard)
-
- def test_cookie_cleanup(self):
- name = self.module.Clipboard.data_cookie._name_cookie_page(0)
- request = RequestMock()
- request.set_cookie(name, 'value')
- response = ResponseMock()
- clipboard = self.module.Clipboard()
- clipboard.to_response(response, request)
- self.assertIn(name, response.expired_cookies)
- self.assertNotIn(name, response.cookies)
-
-
class TestException(unittest.TestCase):
module = file_actions_exceptions
- clipboard_class = file_actions_clipboard.Clipboard
node_class = browsepy_file.Node
def test_invalid_clipboard_items_error(self):
- clipboard = self.clipboard_class((
- 'asdf',
- ))
pair = (
self.node_class('/base/asdf'),
Exception('Uncaught exception /base/asdf'),
)
e = self.module.InvalidClipboardItemsError(
path='/',
- clipboard=clipboard,
+ mode='cut',
+ clipboard=('asdf,'),
issues=[pair]
)
e.append(
--
GitLab
From 2ef9e6d6f61a050f120f25e240f684c5882a229d Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 31 Mar 2019 00:46:17 +0100
Subject: [PATCH 044/171] slim down makefile
---
.travis.yml | 9 +++++----
Makefile | 50 +-------------------------------------------------
2 files changed, 6 insertions(+), 53 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 1307d1e..db92f2e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,7 +27,6 @@ install:
nvm install stable
nvm use stable
npm install eslint
- export PATH="$PWD/node_modules/.bin:$PATH"
fi
- |
if [ "$sphinx" = "yes" ]; then
@@ -35,14 +34,16 @@ install:
fi
script:
- - make flake8 coverage
+ - coverage run setup.py test
+ - coverage report
- |
if [ "$eslint" = "yes" ]; then
- make eslint
+ node_modules/.bin/eslint browsepy
fi
- |
if [ "$sphinx" = "yes" ]; then
- make doc
+ make -C doc html 2>&1 | grep -v \
+ 'WARNING: more than one target found for cross-reference'
fi
after_success:
diff --git a/Makefile b/Makefile
index 8ea3107..56f2e1a 100644
--- a/Makefile
+++ b/Makefile
@@ -1,12 +1,4 @@
-.PHONY: test clean upload doc showdoc eslint pep8 pycodestyle flake8 coverage showcoverage
-
-test: flake8 eslint
- python -c 'import yaml, glob;[yaml.load(open(p)) for p in glob.glob(".*.yml")]'
-ifdef debug
- python setup.py test --debug=$(debug)
-else
- python setup.py test
-endif
+.PHONY: clean
clean:
rm -rf \
@@ -16,43 +8,3 @@ clean:
find browsepy -type f -name "*.py[co]" -delete
find browsepy -type d -name "__pycache__" -delete
$(MAKE) -C doc clean
-
-build/env:
- mkdir -p build
- python3 -m venv build/env
- build/env/bin/pip install pip --upgrade
- build/env/bin/pip install wheel
-
-build: clean build/env
- build/env/bin/python setup.py bdist_wheel
- build/env/bin/python setup.py sdist
-
-upload: clean build/env
- build/env/bin/python setup.py bdist_wheel upload
- build/env/bin/python setup.py sdist upload
-
-doc:
- $(MAKE) -C doc html 2>&1 | grep -v \
- 'WARNING: more than one target found for cross-reference'
-
-showdoc: doc
- xdg-open file://${CURDIR}/doc/.build/html/index.html >> /dev/null
-
-eslint:
- eslint ${CURDIR}/browsepy
-
-pycodestyle:
- pycodestyle browsepy setup.py
-
-pep8: pycodestyle
-
-flake8:
- flake8 browsepy setup.py
-
-coverage:
- coverage run setup.py test
-
-showcoverage: coverage
- coverage html
- xdg-open file://${CURDIR}/htmlcov/index.html >> /dev/null
-
--
GitLab
From 538c0f23128125c9cb731ecd89b9b3833102c53f Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 31 Mar 2019 13:52:22 +0200
Subject: [PATCH 045/171] fix session usage and mime test
---
browsepy/__init__.py | 9 ++++-----
browsepy/tests/test_file.py | 2 +-
2 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index c728e74..ee8e23e 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -72,7 +72,7 @@ def get_cookie_browse_sorting(path, default):
:returns: sorting property
:rtype: string
'''
- if session:
+ if request:
for cpath, cprop in session.get('browse:sort', ()):
if path == cpath:
return cprop
@@ -157,10 +157,9 @@ def sort(property, path):
if not directory.is_directory or directory.is_excluded:
return NotFound()
- if session:
- data = session.get('browse:sort', [])
- data.insert(0, (path, property))
- session.modified = True
+ if request:
+ session['browse:sort'] = \
+ [(path, property)] + session.get('browse:sort', [])
return redirect(url_for(".browse", path=directory.urlpath))
diff --git a/browsepy/tests/test_file.py b/browsepy/tests/test_file.py
index 80f9947..cd0ace4 100644
--- a/browsepy/tests/test_file.py
+++ b/browsepy/tests/test_file.py
@@ -112,7 +112,7 @@ class TestFile(unittest.TestCase):
bad_file = os.path.join(bad_path, 'file')
with open(bad_file, 'w') as f:
- f.write('#!/usr/bin/env bash\nexit 1\n')
+ f.write('#!/bin/sh\nexit 1\n')
os.chmod(bad_file, os.stat(bad_file).st_mode | stat.S_IEXEC)
old_path = os.environ['PATH']
--
GitLab
From 00d230ee7f77191917171b2f5b276a58be213c53 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 31 Mar 2019 14:40:23 +0200
Subject: [PATCH 046/171] use six, simplify code, add encoding headers for py2
---
browsepy/appconfig.py | 2 ++
browsepy/compat.py | 1 -
browsepy/exceptions.py | 2 ++
browsepy/file.py | 1 -
browsepy/manager.py | 1 -
browsepy/mimetype.py | 1 -
browsepy/plugin/__init__.py | 1 -
browsepy/plugin/file_actions/__init__.py | 33 +++++++++--------
browsepy/plugin/file_actions/exceptions.py | 2 ++
browsepy/plugin/file_actions/tests.py | 2 ++
.../file_actions/{paste.py => utils.py} | 8 +++--
browsepy/plugin/player/__init__.py | 9 ++---
browsepy/plugin/player/playable.py | 35 +++++++++----------
browsepy/plugin/player/tests.py | 5 +--
browsepy/plugin/player/utils.py | 10 ++++++
browsepy/stream.py | 1 +
browsepy/utils.py | 1 +
browsepy/widget.py | 4 ++-
18 files changed, 67 insertions(+), 52 deletions(-)
rename browsepy/plugin/file_actions/{paste.py => utils.py} (92%)
create mode 100644 browsepy/plugin/player/utils.py
diff --git a/browsepy/appconfig.py b/browsepy/appconfig.py
index 384108e..56aceb2 100644
--- a/browsepy/appconfig.py
+++ b/browsepy/appconfig.py
@@ -1,3 +1,5 @@
+# -*- coding: UTF-8 -*-
+
import flask
import flask.config
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 75047fa..669645d 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os
diff --git a/browsepy/exceptions.py b/browsepy/exceptions.py
index 8a47c03..9f7a81f 100644
--- a/browsepy/exceptions.py
+++ b/browsepy/exceptions.py
@@ -1,3 +1,5 @@
+# -*- coding: UTF-8 -*-
+
class OutsideDirectoryBase(Exception):
'''
diff --git a/browsepy/file.py b/browsepy/file.py
index 61c5a37..845aa5b 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os
diff --git a/browsepy/manager.py b/browsepy/manager.py
index b8124e9..be2b8fa 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import re
diff --git a/browsepy/mimetype.py b/browsepy/mimetype.py
index b908dd2..bcdba47 100644
--- a/browsepy/mimetype.py
+++ b/browsepy/mimetype.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import re
diff --git a/browsepy/plugin/__init__.py b/browsepy/plugin/__init__.py
index 8cb7f45..8d98fed 100644
--- a/browsepy/plugin/__init__.py
+++ b/browsepy/plugin/__init__.py
@@ -1,2 +1 @@
-#!/usr/bin/env python
# -*- coding: UTF-8 -*-
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index 229f801..ce7a39c 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -1,9 +1,7 @@
-#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os
import os.path
-import logging
from flask import Blueprint, render_template, request, redirect, url_for, \
session
@@ -20,19 +18,16 @@ from .exceptions import FileActionsException, \
InvalidClipboardModeError, \
InvalidDirnameError, \
DirectoryCreationError
-from .paste import paste_clipboard
+from . import utils
-__basedir__ = os.path.dirname(os.path.abspath(__file__))
-
-logger = logging.getLogger(__name__)
actions = Blueprint(
'file_actions',
__name__,
url_prefix='/file-actions',
- template_folder=os.path.join(__basedir__, 'templates'),
- static_folder=os.path.join(__basedir__, 'static'),
+ template_folder=utils.ppath('templates'),
+ static_folder=utils.ppath('static'),
)
re_basename = '^[^ {0}]([^{0}]*[^ {0}])?$'.format(
@@ -143,7 +138,7 @@ def clipboard_paste(path):
mode = session.get('clipboard:mode')
clipboard = session.get('clipboard:items', ())
- success, issues, nmode = paste_clipboard(directory, mode, clipboard)
+ success, issues, nmode = utils.paste(directory, mode, clipboard)
if clipboard and mode != nmode:
session['mode'] = nmode
mode = nmode
@@ -197,6 +192,14 @@ def clipboard_error(e):
)
+def shrink_session(data, last):
+ if last:
+ # TODO: add warning message
+ del data['clipboard:items']
+ del data['clipboard:mode']
+ return data
+
+
def register_plugin(manager):
'''
Register blueprints and actions using given plugin manager.
@@ -216,14 +219,6 @@ def register_plugin(manager):
clipboard = session.get('clipboard:items', ())
return abspath_to_urlpath(path, base) in clipboard
- def shrink(data, last):
- if last:
- # TODO: add warning message
- del data['clipboard:items']
- del data['clipboard:mode']
- return data
-
- manager.register_session(('clipboard:items', 'clipboard:mode'), shrink)
manager.register_exclude_function(excluded_clipboard)
manager.register_blueprint(actions)
manager.register_widget(
@@ -258,3 +253,7 @@ def register_plugin(manager):
),
filter=detect_clipboard,
)
+ manager.register_session(
+ ('clipboard:items', 'clipboard:mode'),
+ shrink_session,
+ )
diff --git a/browsepy/plugin/file_actions/exceptions.py b/browsepy/plugin/file_actions/exceptions.py
index d9b4391..8df75a0 100644
--- a/browsepy/plugin/file_actions/exceptions.py
+++ b/browsepy/plugin/file_actions/exceptions.py
@@ -1,3 +1,5 @@
+# -*- coding: UTF-8 -*-
+
import os
import errno
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index 6f03c46..0ae0458 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -1,3 +1,5 @@
+# -*- coding: UTF-8 -*-
+
import unittest
import tempfile
import shutil
diff --git a/browsepy/plugin/file_actions/paste.py b/browsepy/plugin/file_actions/utils.py
similarity index 92%
rename from browsepy/plugin/file_actions/paste.py
rename to browsepy/plugin/file_actions/utils.py
index 2163f1b..039e973 100644
--- a/browsepy/plugin/file_actions/paste.py
+++ b/browsepy/plugin/file_actions/utils.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os
@@ -11,6 +10,11 @@ from browsepy.compat import map
from .exceptions import InvalidClipboardModeError
+ppath = functools.partial(
+ os.path.join,
+ os.path.dirname(os.path.realpath(__file__)),
+ )
+
def copy(target, node, join_fnc=os.path.join):
if node.is_excluded:
@@ -40,7 +44,7 @@ def move(target, node, join_fnc=os.path.join):
return node.path
-def paste_clipboard(target, mode, clipboard):
+def paste(target, mode, clipboard):
'''
Get pasting function for given directory and keyboard.
'''
diff --git a/browsepy/plugin/player/__init__.py b/browsepy/plugin/player/__init__.py
index ccfbaf1..dcc0f8a 100644
--- a/browsepy/plugin/player/__init__.py
+++ b/browsepy/plugin/player/__init__.py
@@ -1,8 +1,5 @@
-#!/usr/bin/env python
# -*- coding: UTF-8 -*-
-import os.path
-
from flask import Blueprint, render_template
from werkzeug.exceptions import NotFound
@@ -13,15 +10,15 @@ from browsepy.file import OutsideDirectoryBase
from .playable import PlayableFile, PlayableDirectory, \
PlayListFile, detect_playable_mimetype
+from . import utils
-__basedir__ = os.path.dirname(os.path.abspath(__file__))
player = Blueprint(
'player',
__name__,
url_prefix='/play',
- template_folder=os.path.join(__basedir__, 'templates'),
- static_folder=os.path.join(__basedir__, 'static'),
+ template_folder=utils.ppath('templates'),
+ static_folder=utils.ppath('static'),
)
diff --git a/browsepy/plugin/player/playable.py b/browsepy/plugin/player/playable.py
index ea967ea..5b96487 100644
--- a/browsepy/plugin/player/playable.py
+++ b/browsepy/plugin/player/playable.py
@@ -1,28 +1,20 @@
+# -*- coding: UTF-8 -*-
+
import sys
import codecs
import os.path
import warnings
+import six
+import six.moves
+
from werkzeug.utils import cached_property
-from browsepy.compat import range, PY_LEGACY # noqa
from browsepy.file import Node, File, Directory, \
underscore_replace, check_under_base
-if PY_LEGACY:
- import ConfigParser as configparser
-else:
- import configparser
-
-ConfigParserBase = (
- configparser.SafeConfigParser
- if hasattr(configparser, 'SafeConfigParser') else
- configparser.ConfigParser
- )
-
-
class PLSFileParser(object):
'''
ConfigParser wrapper accepting fallback on get for convenience.
@@ -30,10 +22,14 @@ class PLSFileParser(object):
This wraps instead of inheriting due ConfigParse being classobj on python2.
'''
NOT_SET = type('NotSetType', (object,), {})
- parser_class = (
- configparser.SafeConfigParser
- if hasattr(configparser, 'SafeConfigParser') else
- configparser.ConfigParser
+ option_exceptions = (
+ six.moves.configparser.NoOptionError,
+ ValueError,
+ )
+ parser_class = getattr(
+ six.moves.configparser,
+ 'SafeConfigParser',
+ six.moves.configparser.ConfigParser
)
def __init__(self, path):
@@ -46,7 +42,7 @@ class PLSFileParser(object):
def getint(self, section, key, fallback=NOT_SET):
try:
return self._parser.getint(section, key)
- except (configparser.NoOptionError, ValueError):
+ except self.option_exceptions:
if fallback is self.NOT_SET:
raise
return fallback
@@ -54,7 +50,7 @@ class PLSFileParser(object):
def get(self, section, key, fallback=NOT_SET):
try:
return self._parser.get(section, key)
- except (configparser.NoOptionError, ValueError):
+ except self.option_exceptions:
if fallback is self.NOT_SET:
raise
return fallback
@@ -159,6 +155,7 @@ class PLSFile(PlayListFile):
def _entries(self):
parser = self.ini_parser_class(self.path)
maxsize = parser.getint('playlist', 'NumberOfEntries', None)
+ range = six.moves.range
for i in range(1, self.maxsize if maxsize is None else maxsize + 1):
path = parser.get('playlist', 'File%d' % i, None)
if not path:
diff --git a/browsepy/plugin/player/tests.py b/browsepy/plugin/player/tests.py
index fca8dc6..c40e2e2 100644
--- a/browsepy/plugin/player/tests.py
+++ b/browsepy/plugin/player/tests.py
@@ -1,3 +1,4 @@
+# -*- coding: UTF-8 -*-
import os
import os.path
@@ -5,12 +6,12 @@ import unittest
import shutil
import tempfile
+import six
import flask
from werkzeug.exceptions import NotFound
import browsepy
-import browsepy.compat as compat
import browsepy.file as browsepy_file
import browsepy.manager as browsepy_manager
import browsepy.plugin.player as player
@@ -366,7 +367,7 @@ class TestBlueprint(TestPlayerBase):
def p(*args):
args = [
- arg if isinstance(arg, compat.unicode) else arg.decode('utf-8')
+ arg if isinstance(arg, six.text_type) else arg.decode('utf-8')
for arg in args
]
return os.path.join(*args)
diff --git a/browsepy/plugin/player/utils.py b/browsepy/plugin/player/utils.py
new file mode 100644
index 0000000..907c240
--- /dev/null
+++ b/browsepy/plugin/player/utils.py
@@ -0,0 +1,10 @@
+# -*- coding: UTF-8 -*-
+
+import os
+import os.path
+import functools
+
+ppath = functools.partial(
+ os.path.join,
+ os.path.dirname(os.path.realpath(__file__)),
+ )
diff --git a/browsepy/stream.py b/browsepy/stream.py
index d48dc47..b834039 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -1,3 +1,4 @@
+# -*- coding: UTF-8 -*-
import os
import os.path
diff --git a/browsepy/utils.py b/browsepy/utils.py
index 3b37ff8..5d84629 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -1,3 +1,4 @@
+# -*- coding: UTF-8 -*-
import os
import os.path
diff --git a/browsepy/widget.py b/browsepy/widget.py
index fed6642..0ced542 100644
--- a/browsepy/widget.py
+++ b/browsepy/widget.py
@@ -1,9 +1,11 @@
+# -*- coding: UTF-8 -*-
'''
WARNING: deprecated module.
-API defined in this module has been deprecated in version 0.5 will likely be
+API defined in this module has been deprecated in version 0.5 and it will be
removed at 0.6.
'''
+
import warnings
from markupsafe import Markup
--
GitLab
From 306d74831ed91b9fad9f93a719d1a862d7a96b77 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 5 Apr 2019 16:57:38 +0200
Subject: [PATCH 047/171] refactor stream_template
---
.vscode/settings.json | 3 ++-
browsepy/__init__.py | 20 +++-----------------
browsepy/plugin/file_actions/__init__.py | 3 ++-
browsepy/plugin/player/__init__.py | 4 ++--
browsepy/utils.py | 19 +++++++++++++++++++
5 files changed, 28 insertions(+), 21 deletions(-)
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 5534594..cbbcd79 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -13,5 +13,6 @@
"python.linting.pep8Enabled": true,
"python.linting.pep8Path": "pycodestyle",
"python.linting.pylintEnabled": false,
- "python.jediEnabled": true
+ "python.jediEnabled": true,
+ "python.pythonPath": "env/bin/python"
}
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index ee8e23e..b5733b9 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -8,14 +8,15 @@ import os.path
import cookieman
-from flask import Response, request, render_template, redirect, \
- url_for, send_from_directory, stream_with_context, \
+from flask import request, render_template, redirect, \
+ url_for, send_from_directory, \
make_response, session
from werkzeug.exceptions import NotFound
from .appconfig import Flask
from .manager import PluginManager
from .file import Node, secure_filename
+from .utils import stream_template
from .exceptions import OutsideRemovableBase, OutsideDirectoryBase, \
InvalidFilenameError, InvalidPathError
from . import compat
@@ -123,21 +124,6 @@ def browse_sortkey_reverse(prop):
)
-def stream_template(template_name, **context):
- '''
- Some templates can be huge, this function returns an streaming response,
- sending the content in chunks and preventing from timeout.
-
- :param template_name: template
- :param **context: parameters for templates.
- :yields: HTML strings
- '''
- app.update_template_context(context)
- template = app.jinja_env.get_template(template_name)
- stream = template.generate(context)
- return Response(stream_with_context(stream))
-
-
@app.context_processor
def template_globals():
return {
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index ce7a39c..dd3e7f6 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -12,6 +12,7 @@ from browsepy.file import Node, abspath_to_urlpath, secure_filename, \
current_restricted_chars, common_path_separators
from browsepy.compat import re_escape, FileNotFoundError
from browsepy.exceptions import OutsideDirectoryBase
+from browsepy.utils import stream_template
from .exceptions import FileActionsException, \
InvalidClipboardItemsError, \
@@ -108,7 +109,7 @@ def selection(path):
clipboard=clipboard,
)
- return render_template(
+ return stream_template(
'selection.file_actions.html',
file=directory,
mode=session.get('clipboard:mode'),
diff --git a/browsepy/plugin/player/__init__.py b/browsepy/plugin/player/__init__.py
index dcc0f8a..3abb881 100644
--- a/browsepy/plugin/player/__init__.py
+++ b/browsepy/plugin/player/__init__.py
@@ -3,9 +3,9 @@
from flask import Blueprint, render_template
from werkzeug.exceptions import NotFound
-from browsepy import stream_template, get_cookie_browse_sorting, \
- browse_sortkey_reverse
+from browsepy import get_cookie_browse_sorting, browse_sortkey_reverse
from browsepy.file import OutsideDirectoryBase
+from browsepy.utils import stream_template
from .playable import PlayableFile, PlayableDirectory, \
PlayListFile, detect_playable_mimetype
diff --git a/browsepy/utils.py b/browsepy/utils.py
index 5d84629..979ca21 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -5,6 +5,9 @@ import os.path
import random
import functools
+import flask
+
+
ppath = functools.partial(
os.path.join,
os.path.dirname(os.path.realpath(__file__)),
@@ -14,3 +17,19 @@ ppath = functools.partial(
def random_string(size, sample=tuple(map(chr, range(256)))):
randrange = functools.partial(random.randrange, 0, len(sample))
return ''.join(sample[randrange()] for i in range(size))
+
+
+def stream_template(template_name, **context):
+ '''
+ Some templates can be huge, this function returns an streaming response,
+ sending the content in chunks and preventing from timeout.
+
+ :param template_name: template
+ :param **context: parameters for templates.
+ :yields: HTML strings
+ '''
+ app = flask.current_app
+ app.update_template_context(context)
+ template = app.jinja_env.get_template(template_name)
+ stream = template.generate(context)
+ return flask.Response(flask.stream_with_context(stream))
--
GitLab
From f7aa1e7496a768d6710e1d3cf0edda4f746523d0 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sat, 6 Apr 2019 00:16:35 +0200
Subject: [PATCH 048/171] active app context resolution on Node init
---
browsepy/file.py | 11 +++++++----
browsepy/plugin/player/tests.py | 1 +
browsepy/utils.py | 8 +++++++-
3 files changed, 15 insertions(+), 5 deletions(-)
diff --git a/browsepy/file.py b/browsepy/file.py
index 845aa5b..41b3744 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -15,6 +15,8 @@ from werkzeug.utils import cached_property
from . import compat
from . import manager
+from . import utils
+
from .compat import range
from .http import Headers
from .stream import TarFileStream
@@ -84,6 +86,7 @@ class Node(object):
Get current app's plugin manager.
:returns: plugin manager instance
+ :type: browsepy.manager.PluginManagerBase
'''
return (
self.app.extensions.get('plugin_manager') or
@@ -259,12 +262,12 @@ class Node(object):
'''
:param path: local path
:type path: str
- :param path: optional app instance
- :type path: flask.app
- :param **defaults: attributes will be set to object
+ :param app: app instance (optional inside application context)
+ :type app: flask.Flask
+ :param **defaults: initial property values
'''
self.path = compat.fsdecode(path) if path else None
- self.app = current_app if app is None else app
+ self.app = utils.solve_local(current_app if app is None else app)
self.__dict__.update(defaults) # only for attr and cached_property
def __repr__(self):
diff --git a/browsepy/plugin/player/tests.py b/browsepy/plugin/player/tests.py
index c40e2e2..68e42aa 100644
--- a/browsepy/plugin/player/tests.py
+++ b/browsepy/plugin/player/tests.py
@@ -346,6 +346,7 @@ class TestBlueprint(TestPlayerBase):
self.file('directory/test.mp3')
result = self.get('player.directory', path=name)
self.assertEqual(result.status_code, 200)
+ self.assertIn(b'test.mp3', result.data)
def test_endpoints(self):
with self.app.app_context():
diff --git a/browsepy/utils.py b/browsepy/utils.py
index 979ca21..cffe00f 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -19,6 +19,12 @@ def random_string(size, sample=tuple(map(chr, range(256)))):
return ''.join(sample[randrange()] for i in range(size))
+def solve_local(context_local):
+ if callable(getattr(context_local, '_get_current_object', None)):
+ return context_local._get_current_object()
+ return context_local
+
+
def stream_template(template_name, **context):
'''
Some templates can be huge, this function returns an streaming response,
@@ -28,7 +34,7 @@ def stream_template(template_name, **context):
:param **context: parameters for templates.
:yields: HTML strings
'''
- app = flask.current_app
+ app = solve_local(context.get('current_app') or flask.current_app)
app.update_template_context(context)
template = app.jinja_env.get_template(template_name)
stream = template.generate(context)
--
GitLab
From 5d13bd7f7cef747ac9ad2ea390d901b221c661a7 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Mon, 8 Apr 2019 12:50:14 +0200
Subject: [PATCH 049/171] fix tests
---
browsepy/plugin/file_actions/tests.py | 7 +++++--
browsepy/plugin/player/__init__.py | 2 +-
browsepy/plugin/player/tests.py | 9 ++++++++-
browsepy/tests/deprecated/test_plugins.py | 9 +++++----
browsepy/tests/test_file.py | 2 +-
5 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index 0ae0458..1c13497 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -439,9 +439,12 @@ class TestException(unittest.TestCase):
module = file_actions_exceptions
node_class = browsepy_file.Node
+ def setUp(self):
+ self.app = flask.Flask('browsepy')
+
def test_invalid_clipboard_items_error(self):
pair = (
- self.node_class('/base/asdf'),
+ self.node_class('/base/asdf', app=self.app),
Exception('Uncaught exception /base/asdf'),
)
e = self.module.InvalidClipboardItemsError(
@@ -451,7 +454,7 @@ class TestException(unittest.TestCase):
issues=[pair]
)
e.append(
- self.node_class('/base/other'),
+ self.node_class('/base/other', app=self.app),
OSError(2, 'Not found with random message'),
)
self.assertIn('Uncaught exception asdf', e.issues[0].message)
diff --git a/browsepy/plugin/player/__init__.py b/browsepy/plugin/player/__init__.py
index 3abb881..68a38f8 100644
--- a/browsepy/plugin/player/__init__.py
+++ b/browsepy/plugin/player/__init__.py
@@ -48,7 +48,7 @@ def playlist(path):
return NotFound()
-@player.route("/directory", defaults={"path": ""})
+@player.route('/directory', defaults={'path': ''})
@player.route('/directory/')
def directory(path):
sort_property = get_cookie_browse_sorting(path, 'text')
diff --git a/browsepy/plugin/player/tests.py b/browsepy/plugin/player/tests.py
index 68e42aa..b903011 100644
--- a/browsepy/plugin/player/tests.py
+++ b/browsepy/plugin/player/tests.py
@@ -288,19 +288,26 @@ class TestPlayable(TestIntegrationBase):
class TestBlueprint(TestPlayerBase):
def setUp(self):
super(TestBlueprint, self).setUp()
- self.app = browsepy.app # required for our url_for calls
+ self.app = flask.Flask('browsepy')
self.app.config.update(
directory_base=tempfile.mkdtemp(),
SERVER_NAME='test'
)
self.app.register_blueprint(self.module.player)
+ app = self.app
+ @app.route("/open", defaults={"path": ""})
+ @app.route('/open/')
+ def open(path):
+ pass
+
def tearDown(self):
shutil.rmtree(self.app.config['directory_base'])
test_utils.clear_flask_context()
def url_for(self, endpoint, **kwargs):
with self.app.app_context():
+ print(flask.current_app)
return flask.url_for(endpoint, _external=False, **kwargs)
def get(self, endpoint, **kwargs):
diff --git a/browsepy/tests/deprecated/test_plugins.py b/browsepy/tests/deprecated/test_plugins.py
index 1328e96..53b03d9 100644
--- a/browsepy/tests/deprecated/test_plugins.py
+++ b/browsepy/tests/deprecated/test_plugins.py
@@ -209,7 +209,7 @@ class TestIntegration(TestIntegrationBase):
self.assertEqual(actions[5].widget, widget)
def test_register_widget(self):
- file = self.file_module.Node()
+ file = self.file_module.Node(app=self.app)
manager = self.manager_module.MimetypeActionPluginManager(self.app)
widget = self.widget_module.StyleWidget('static', filename='a.css')
manager.register_widget(widget)
@@ -261,9 +261,10 @@ class TestIntegration(TestIntegrationBase):
self.assertEqual(file.link.text, 'asdf.txt')
def test_from_file(self):
- file = self.file_module.File('asdf.txt')
- widget = self.widget_module.LinkWidget.from_file(file)
- self.assertEqual(widget.text, 'asdf.txt')
+ with self.app.app_context():
+ file = self.file_module.File('asdf.txt')
+ widget = self.widget_module.LinkWidget.from_file(file)
+ self.assertEqual(widget.text, 'asdf.txt')
class TestPlayable(TestIntegrationBase):
diff --git a/browsepy/tests/test_file.py b/browsepy/tests/test_file.py
index cd0ace4..d30b74d 100644
--- a/browsepy/tests/test_file.py
+++ b/browsepy/tests/test_file.py
@@ -41,7 +41,7 @@ class TestFile(unittest.TestCase):
def test_repr(self):
self.assertIsInstance(
- repr(self.module.Node('a')),
+ repr(self.module.Node('a', app=self.app)),
browsepy.compat.basestring
)
--
GitLab
From e8718cd3a2727d1b41329a5f1981f2ffcce7f9a4 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Mon, 8 Apr 2019 17:43:55 +0200
Subject: [PATCH 050/171] fix a lot of app context issues
---
.python-version | 2 +-
browsepy/__init__.py | 3 +-
browsepy/compat.py | 7 +-
browsepy/file.py | 83 +++++++++++---------
browsepy/manager.py | 36 +--------
browsepy/plugin/file_actions/__init__.py | 8 +-
browsepy/plugin/file_actions/tests.py | 9 +++
browsepy/plugin/file_actions/utils.py | 5 --
browsepy/plugin/player/__init__.py | 8 +-
browsepy/plugin/player/playable.py | 13 ++--
browsepy/plugin/player/tests.py | 40 +++++-----
browsepy/plugin/player/utils.py | 10 ---
browsepy/tests/deprecated/test_plugins.py | 2 +-
browsepy/tests/meta.py | 5 +-
browsepy/tests/test_file.py | 4 +-
browsepy/tests/test_module.py | 92 ++++++++++++-----------
browsepy/tests/test_plugins.py | 4 +-
browsepy/tests/utils.py | 28 -------
browsepy/utils.py | 74 +++++++++++++++++-
doc/index.rst | 2 +-
doc/tests_utils.rst | 2 +-
setup.cfg | 1 +
22 files changed, 228 insertions(+), 210 deletions(-)
delete mode 100644 browsepy/plugin/player/utils.py
delete mode 100644 browsepy/tests/utils.py
diff --git a/.python-version b/.python-version
index a76ccff..0b2eb36 100644
--- a/.python-version
+++ b/.python-version
@@ -1 +1 @@
-3.7.1
+3.7.2
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index b5733b9..73b90e2 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -132,7 +132,7 @@ def template_globals():
}
-@app.route('/sort/', defaults={"path": ""})
+@app.route('/sort/', defaults={'path': ''})
@app.route('/sort//')
def sort(property, path):
try:
@@ -146,7 +146,6 @@ def sort(property, path):
if request:
session['browse:sort'] = \
[(path, property)] + session.get('browse:sort', [])
-
return redirect(url_for(".browse", path=directory.urlpath))
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 669645d..9be65c6 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -14,6 +14,11 @@ FS_ENCODING = sys.getfilesystemencoding()
PY_LEGACY = sys.version_info < (3, )
TRUE_VALUES = frozenset(('true', 'yes', '1', 'enable', 'enabled', True, 1))
+try:
+ import importlib.resources as res # to support python >= 3.7
+except ImportError:
+ import importlib_resources as res # noqa
+
try:
from os import scandir, walk # python 3.5+
except ImportError:
@@ -32,7 +37,7 @@ except ImportError:
try:
from collections.abc import Iterator as BaseIterator # python 3.3+
except ImportError:
- from collections import Iterator as BaseIterator
+ from collections import Iterator as BaseIterator # noqa
def isexec(path):
diff --git a/browsepy/file.py b/browsepy/file.py
index 41b3744..64d2917 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -9,6 +9,7 @@ import string
import random
import datetime
import logging
+import contextlib
from flask import current_app, send_from_directory
from werkzeug.utils import cached_property
@@ -16,10 +17,11 @@ from werkzeug.utils import cached_property
from . import compat
from . import manager
from . import utils
+from . import stream
from .compat import range
from .http import Headers
-from .stream import TarFileStream
+
from .exceptions import OutsideDirectoryBase, OutsideRemovableBase, \
PathTooLongError, FilenameTooLongError
@@ -27,11 +29,12 @@ from .exceptions import OutsideDirectoryBase, OutsideRemovableBase, \
logger = logging.getLogger(__name__)
unicode_underscore = '_'.decode('utf-8') if compat.PY_LEGACY else '_'
underscore_replace = '%s:underscore' % __name__
-codecs.register_error(underscore_replace,
- lambda error: (unicode_underscore, error.start + 1)
- )
-binary_units = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB")
-standard_units = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+codecs.register_error(
+ underscore_replace,
+ lambda error: (unicode_underscore, error.start + 1)
+ )
+binary_units = ('B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB')
+standard_units = ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB')
common_path_separators = '\\/'
restricted_chars = '/\0'
nt_restricted_chars = '/\0\\<>:"|?*' + ''.join(map(chr, range(1, 32)))
@@ -139,7 +142,7 @@ class Node(object):
:returns: True if current node can be removed, False otherwise.
:rtype: bool
'''
- dirbase = self.app.config["directory_remove"]
+ dirbase = self.app.config.get('directory_remove')
return bool(dirbase and check_under_base(self.path, dirbase))
@cached_property
@@ -171,7 +174,8 @@ class Node(object):
:returns: parent object if available
:rtype: Node instance or None
'''
- if check_path(self.path, self.app.config['directory_base']):
+ directory_base = self.app.config.get('directory_base', self.path)
+ if check_path(self.path, directory_base):
return None
parent = os.path.dirname(self.path) if self.path else None
return self.directory_class(parent, self.app) if parent else None
@@ -214,7 +218,10 @@ class Node(object):
:returns: relative-url-like for node's path
:rtype: str
'''
- return abspath_to_urlpath(self.path, self.app.config['directory_base'])
+ return abspath_to_urlpath(
+ self.path,
+ self.app.config.get('directory_base', self.path),
+ )
@property
def name(self):
@@ -267,7 +274,7 @@ class Node(object):
:param **defaults: initial property values
'''
self.path = compat.fsdecode(path) if path else None
- self.app = utils.solve_local(current_app if app is None else app)
+ self.app = utils.solve_local(app or current_app)
self.__dict__.update(defaults) # only for attr and cached_property
def __repr__(self):
@@ -302,8 +309,8 @@ class Node(object):
:return: file object pointing to path
:rtype: File
'''
- app = app or current_app
- base = app.config['directory_base']
+ app = utils.solve_local(app or current_app)
+ base = app.config.get('directory_base', path)
path = urlpath_to_abspath(path, base)
if not cls.generic:
kls = cls
@@ -425,7 +432,7 @@ class File(Node):
try:
size, unit = fmt_size(
self.stats.st_size,
- self.app.config['use_binary_multiples'] if self.app else False
+ self.app.config.get('use_binary_multiples', False)
)
except OSError:
return None
@@ -481,6 +488,8 @@ class Directory(Node):
* :attr:`generic` is set to False, so static method :meth:`from_urlpath`
will always return instances of this class.
'''
+ stream_class = stream.TarFileStream
+
_listdir_cache = None
mimetype = 'inode/directory'
is_file = False
@@ -584,7 +593,7 @@ class Directory(Node):
:returns: True if downloadable, False otherwise
:rtype: bool
'''
- return self.app.config['directory_downloadable']
+ return self.app.config.get('directory_downloadable', False)
@cached_property
def can_upload(self):
@@ -595,7 +604,7 @@ class Directory(Node):
:returns: True if a file can be upload to directory, False otherwise
:rtype: bool
'''
- dirbase = self.app.config["directory_upload"]
+ dirbase = self.app.config.get('directory_upload', False)
return dirbase and check_base(self.path, dirbase)
@cached_property
@@ -639,9 +648,9 @@ class Directory(Node):
:returns: Response object
:rtype: flask.Response
'''
- stream = TarFileStream(
+ stream = self.stream_class(
self.path,
- self.app.config['directory_tar_buffsize'],
+ self.app.config.get('directory_tar_buffsize', 10240),
self.plugin_manager.check_excluded,
)
return self.app.response_class(
@@ -711,29 +720,31 @@ class Directory(Node):
Iter unsorted entries on this directory.
:yields: Directory or File instance for each entry in directory
- :ytype: Node
+ :rtype: Iterator of browsepy.file.Node
'''
directory_class = self.directory_class
file_class = self.file_class
exclude_fnc = self.plugin_manager.check_excluded
- for entry in compat.scandir(self.path):
- if exclude_fnc(entry.path):
- continue
- kwargs = {
- 'path': entry.path,
- 'app': self.app,
- 'parent': self,
- 'is_excluded': False
- }
- try:
- if precomputed_stats and not entry.is_symlink():
- kwargs['stats'] = entry.stat()
- if entry.is_dir(follow_symlinks=True):
- yield directory_class(**kwargs)
- else:
- yield file_class(**kwargs)
- except OSError as e:
- logger.exception(e)
+ with contextlib.closing(compat.scandir(self.path)) as files:
+ for entry in files:
+ if exclude_fnc(entry.path):
+ continue
+ kwargs = {
+ 'path': entry.path,
+ 'app': self.app,
+ 'parent': self,
+ 'is_excluded': False,
+ }
+ try:
+ if precomputed_stats and not entry.is_symlink():
+ kwargs['stats'] = entry.stat()
+ yield (
+ directory_class(**kwargs)
+ if entry.is_dir(follow_symlinks=True) else
+ file_class(**kwargs)
+ )
+ except OSError as e:
+ logger.exception(e)
def listdir(self, sortkey=None, reverse=False):
'''
diff --git a/browsepy/manager.py b/browsepy/manager.py
index be2b8fa..a582c2f 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -14,6 +14,7 @@ from cookieman import CookieMan
from . import mimetype
from . import compat
from .compat import deprecated, usedoc
+from .utils import get_module
def defaultsnamedtuple(name, fields, defaults=None):
@@ -97,32 +98,6 @@ class PluginManagerBase(object):
'''
pass
- def _expected_import_error(self, error, name):
- '''
- Check if error could be expected when importing given module by name.
-
- :param error: exception
- :type error: BaseException
- :param name: module name
- :type name: str
- :returns: True if error corresponds to module name, False otherwise
- :rtype: bool
- '''
- if error.args:
- message = error.args[0]
- components = (
- name.split('.')
- if isinstance(error, ImportError) else
- (name,)
- )
- for finish in range(1, len(components) + 1):
- for start in range(0, finish):
- part = '.'.join(components[start:finish])
- for fmt in (' \'%s\'', ' "%s"', ' %s'):
- if message.endswith(fmt % part):
- return True
- return False
-
def import_plugin(self, plugin):
'''
Import plugin by given name, looking at :attr:`namespaces`.
@@ -144,12 +119,9 @@ class PluginManagerBase(object):
return sys.modules[name]
for name in names:
- try:
- __import__(name)
- return sys.modules[name]
- except (ImportError, KeyError) as e:
- if not self._expected_import_error(e, name):
- raise e
+ module = get_module(name)
+ if module:
+ return module
raise PluginNotFoundError(
'No plugin module %r found, tried %r' % (plugin, names),
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index dd3e7f6..ddc8c72 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -12,7 +12,7 @@ from browsepy.file import Node, abspath_to_urlpath, secure_filename, \
current_restricted_chars, common_path_separators
from browsepy.compat import re_escape, FileNotFoundError
from browsepy.exceptions import OutsideDirectoryBase
-from browsepy.utils import stream_template
+from browsepy.utils import stream_template, ppath
from .exceptions import FileActionsException, \
InvalidClipboardItemsError, \
@@ -27,9 +27,9 @@ actions = Blueprint(
'file_actions',
__name__,
url_prefix='/file-actions',
- template_folder=utils.ppath('templates'),
- static_folder=utils.ppath('static'),
- )
+ template_folder=ppath('templates', module=__name__),
+ static_folder=ppath('static', module=__name__),
+)
re_basename = '^[^ {0}]([^{0}]*[^ {0}])?$'.format(
re_escape(current_restricted_chars + common_path_separators)
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index 1c13497..30a230b 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -12,6 +12,7 @@ import flask
from werkzeug.utils import cached_property
+import browsepy.utils as utils
import browsepy.plugin.file_actions as file_actions
import browsepy.plugin.file_actions.exceptions as file_actions_exceptions
import browsepy.file as browsepy_file
@@ -73,6 +74,9 @@ class TestRegistration(unittest.TestCase):
exclude_fnc=None,
)
+ def tearDown(self):
+ utils.clear_flask_context()
+
def test_register_plugin(self):
self.app.config.update(self.browsepy_module.app.config)
self.app.config['plugin_namespaces'] = ('browsepy.plugin',)
@@ -130,6 +134,7 @@ class TestIntegration(unittest.TestCase):
shutil.rmtree(self.base)
self.app.config['plugin_modules'] = []
self.manager.clear()
+ utils.clear_flask_context()
def test_detection(self):
with self.app.test_client() as client:
@@ -244,6 +249,7 @@ class TestAction(unittest.TestCase):
def tearDown(self):
shutil.rmtree(self.base)
+ utils.clear_flask_context()
def mkdir(self, *path):
os.mkdir(os.path.join(self.base, *path))
@@ -442,6 +448,9 @@ class TestException(unittest.TestCase):
def setUp(self):
self.app = flask.Flask('browsepy')
+ def tearDown(self):
+ utils.clear_flask_context()
+
def test_invalid_clipboard_items_error(self):
pair = (
self.node_class('/base/asdf', app=self.app),
diff --git a/browsepy/plugin/file_actions/utils.py b/browsepy/plugin/file_actions/utils.py
index 039e973..8666f2a 100644
--- a/browsepy/plugin/file_actions/utils.py
+++ b/browsepy/plugin/file_actions/utils.py
@@ -10,11 +10,6 @@ from browsepy.compat import map
from .exceptions import InvalidClipboardModeError
-ppath = functools.partial(
- os.path.join,
- os.path.dirname(os.path.realpath(__file__)),
- )
-
def copy(target, node, join_fnc=os.path.join):
if node.is_excluded:
diff --git a/browsepy/plugin/player/__init__.py b/browsepy/plugin/player/__init__.py
index 68a38f8..0639131 100644
--- a/browsepy/plugin/player/__init__.py
+++ b/browsepy/plugin/player/__init__.py
@@ -5,20 +5,18 @@ from werkzeug.exceptions import NotFound
from browsepy import get_cookie_browse_sorting, browse_sortkey_reverse
from browsepy.file import OutsideDirectoryBase
-from browsepy.utils import stream_template
+from browsepy.utils import stream_template, ppath
from .playable import PlayableFile, PlayableDirectory, \
PlayListFile, detect_playable_mimetype
-from . import utils
-
player = Blueprint(
'player',
__name__,
url_prefix='/play',
- template_folder=utils.ppath('templates'),
- static_folder=utils.ppath('static'),
+ template_folder=ppath('templates', module=__name__),
+ static_folder=ppath('static', module=__name__),
)
diff --git a/browsepy/plugin/player/playable.py b/browsepy/plugin/player/playable.py
index 5b96487..acb0fc3 100644
--- a/browsepy/plugin/player/playable.py
+++ b/browsepy/plugin/player/playable.py
@@ -64,7 +64,7 @@ class PlayableBase(File):
'm3u': 'audio/x-mpegurl',
'm3u8': 'audio/x-mpegurl',
'pls': 'audio/x-scpls',
- }
+ }
@classmethod
def extensions_from_mimetypes(cls, mimetypes):
@@ -137,12 +137,11 @@ class PlayListFile(PlayableBase):
return None
def _entries(self):
- return
- yield # noqa
+ return ()
def entries(self, sortkey=None, reverse=None):
for file in self._entries():
- if PlayableFile.detect(file):
+ if self.playable_class.detect(file):
yield file
@@ -220,20 +219,20 @@ class PlayableDirectory(Directory):
@cached_property
def parent(self):
- return Directory(self.path)
+ return self.directory_class(self.path, app=self.app)
@classmethod
def detect(cls, node):
if node.is_directory:
for file in node._listdir():
- if PlayableFile.detect(file):
+ if cls.file_class.detect(file):
return cls.mimetype
return None
def entries(self, sortkey=None, reverse=None):
listdir_fnc = super(PlayableDirectory, self).listdir
for file in listdir_fnc(sortkey=sortkey, reverse=reverse):
- if PlayableFile.detect(file):
+ if self.file_class.detect(file):
yield file
diff --git a/browsepy/plugin/player/tests.py b/browsepy/plugin/player/tests.py
index b903011..bda1f59 100644
--- a/browsepy/plugin/player/tests.py
+++ b/browsepy/plugin/player/tests.py
@@ -12,11 +12,11 @@ import flask
from werkzeug.exceptions import NotFound
import browsepy
+import browsepy.utils as utils
import browsepy.file as browsepy_file
import browsepy.manager as browsepy_manager
import browsepy.plugin.player as player
import browsepy.plugin.player.playable as player_playable
-import browsepy.tests.utils as test_utils
class ManagerMock(object):
@@ -61,9 +61,15 @@ class TestPlayerBase(unittest.TestCase):
def setUp(self):
self.base = 'c:\\base' if os.name == 'nt' else '/base'
self.app = flask.Flask(self.__class__.__name__)
- self.app.config['directory_base'] = self.base
+ self.app.config.update(
+ directory_base=self.base,
+ SERVER_NAME='localhost',
+ )
self.manager = ManagerMock()
+ def tearDown(self):
+ utils.clear_flask_context()
+
class TestPlayer(TestPlayerBase):
def test_register_plugin(self):
@@ -109,6 +115,9 @@ class TestIntegrationBase(TestPlayerBase):
browsepy_module = browsepy
manager_module = browsepy_manager
+ def tearDown(self):
+ utils.clear_flask_context()
+
class TestIntegration(TestIntegrationBase):
non_directory_args = ['--plugin', 'player']
@@ -199,7 +208,7 @@ class TestPlayable(TestIntegrationBase):
try:
file = p(tmpdir, 'playable.mp3')
open(file, 'w').close()
- node = browsepy_file.Directory(tmpdir)
+ node = browsepy_file.Directory(tmpdir, app=self.app)
self.assertTrue(self.module.PlayableDirectory.detect(node))
directory = self.module.PlayableDirectory(tmpdir, app=self.app)
@@ -288,27 +297,20 @@ class TestPlayable(TestIntegrationBase):
class TestBlueprint(TestPlayerBase):
def setUp(self):
super(TestBlueprint, self).setUp()
- self.app = flask.Flask('browsepy')
- self.app.config.update(
- directory_base=tempfile.mkdtemp(),
- SERVER_NAME='test'
- )
- self.app.register_blueprint(self.module.player)
-
app = self.app
- @app.route("/open", defaults={"path": ""})
- @app.route('/open/')
- def open(path):
+ app.template_folder = utils.ppath('templates')
+ app.config['directory_base'] = tempfile.mkdtemp()
+ app.register_blueprint(self.module.player)
+
+ @app.route("/browse", defaults={"path": ""}, endpoint='browse')
+ @app.route('/browse/', endpoint='browse')
+ @app.route('/open/', endpoint='open')
+ def dummy(path):
pass
- def tearDown(self):
- shutil.rmtree(self.app.config['directory_base'])
- test_utils.clear_flask_context()
-
def url_for(self, endpoint, **kwargs):
with self.app.app_context():
- print(flask.current_app)
- return flask.url_for(endpoint, _external=False, **kwargs)
+ return flask.url_for(endpoint, **kwargs)
def get(self, endpoint, **kwargs):
with self.app.test_client() as client:
diff --git a/browsepy/plugin/player/utils.py b/browsepy/plugin/player/utils.py
deleted file mode 100644
index 907c240..0000000
--- a/browsepy/plugin/player/utils.py
+++ /dev/null
@@ -1,10 +0,0 @@
-# -*- coding: UTF-8 -*-
-
-import os
-import os.path
-import functools
-
-ppath = functools.partial(
- os.path.join,
- os.path.dirname(os.path.realpath(__file__)),
- )
diff --git a/browsepy/tests/deprecated/test_plugins.py b/browsepy/tests/deprecated/test_plugins.py
index 53b03d9..c6e07f7 100644
--- a/browsepy/tests/deprecated/test_plugins.py
+++ b/browsepy/tests/deprecated/test_plugins.py
@@ -124,7 +124,7 @@ def register_plugin(manager):
class TestPlayerBase(unittest.TestCase):
module = player
scheme = 'test'
- hostname = 'testing'
+ hostname = 'localhost'
urlprefix = '%s://%s' % (scheme, hostname)
def assertUrlEqual(self, a, b):
diff --git a/browsepy/tests/meta.py b/browsepy/tests/meta.py
index c929d35..9ed086a 100644
--- a/browsepy/tests/meta.py
+++ b/browsepy/tests/meta.py
@@ -1,7 +1,4 @@
-try:
- import importlib.resources as res
-except ImportError: # pragma: no cover
- import importlib_resources as res # to support python < 3.7
+from browsepy.compat import res
class TestFileMeta(type):
diff --git a/browsepy/tests/test_file.py b/browsepy/tests/test_file.py
index d30b74d..7acdde3 100644
--- a/browsepy/tests/test_file.py
+++ b/browsepy/tests/test_file.py
@@ -9,7 +9,7 @@ import stat
import browsepy
import browsepy.file
import browsepy.compat
-import browsepy.tests.utils as test_utils
+import browsepy.utils as utils
PY_LEGACY = browsepy.compat.PY_LEGACY
@@ -31,7 +31,7 @@ class TestFile(unittest.TestCase):
def tearDown(self):
shutil.rmtree(self.workbench)
- test_utils.clear_flask_context()
+ utils.clear_flask_context()
def textfile(self, name, text):
tmp_txt = os.path.join(self.workbench, name)
diff --git a/browsepy/tests/test_module.py b/browsepy/tests/test_module.py
index 5914c8c..b162e67 100644
--- a/browsepy/tests/test_module.py
+++ b/browsepy/tests/test_module.py
@@ -22,7 +22,7 @@ import browsepy.file
import browsepy.manager
import browsepy.__main__
import browsepy.compat
-import browsepy.tests.utils as test_utils
+import browsepy.utils as utils
PY_LEGACY = browsepy.compat.PY_LEGACY
range = browsepy.compat.range # noqa
@@ -200,7 +200,7 @@ class TestApp(unittest.TestCase):
directory_remove=self.remove,
directory_upload=self.upload,
exclude_fnc=exclude_fnc,
- SERVER_NAME='test',
+ SERVER_NAME='localhost',
)
self.base_directories = [
@@ -225,7 +225,7 @@ class TestApp(unittest.TestCase):
def tearDown(self):
shutil.rmtree(self.base)
- test_utils.clear_flask_context()
+ utils.clear_flask_context()
def get(self, endpoint, **kwargs):
status_code = kwargs.pop('status_code', 200)
@@ -240,11 +240,11 @@ class TestApp(unittest.TestCase):
page_class = self.directory_download_class
else:
page_class = self.generic_page_class
- with kwargs.pop('client', None) or self.app.test_client() as client:
- response = client.get(
+ mclient = kwargs.pop('client', None) or self.app.test_client()
+ try:
+ response = mclient.get(
self.url_for(endpoint, **kwargs),
- follow_redirects=follow_redirects
- )
+ follow_redirects=follow_redirects)
if response.status_code != status_code:
raise self.page_exceptions.get(
response.status_code,
@@ -252,7 +252,9 @@ class TestApp(unittest.TestCase):
)(response.status_code)
result = page_class.from_source(response.data, response)
response.close()
- test_utils.clear_flask_context()
+ finally:
+ if mclient == kwargs.get('client'):
+ mclient.close()
return result
def post(self, endpoint, **kwargs):
@@ -270,7 +272,7 @@ class TestApp(unittest.TestCase):
self.page_exceptions[None]
)(response.status_code)
result = self.list_page_class.from_source(response.data, response)
- test_utils.clear_flask_context()
+ utils.clear_flask_context()
return result
def url_for(self, endpoint, **kwargs):
@@ -602,44 +604,44 @@ class TestApp(unittest.TestCase):
with open(path, 'wb') as f:
f.write(content.encode('ascii'))
- client = self.app.test_client()
- page = self.get('browse', client=client)
- self.assertListEqual(page.files, by_name)
+ with self.app.test_client() as client:
+ page = self.get('browse', client=client)
+ self.assertListEqual(page.files, by_name)
- self.assertRaises(
- Page302Exception,
- self.get, 'sort', property='text', client=client
- )
+ self.assertRaises(
+ Page302Exception,
+ self.get, 'sort', property='text', client=client
+ )
- page = self.get('browse', client=client)
- self.assertListEqual(page.files, by_name)
+ page = self.get('browse', client=client)
+ self.assertListEqual(page.files, by_name)
- page = self.get('sort', property='-text', client=client,
- follow_redirects=True)
- self.assertListEqual(page.files, by_name_desc)
+ page = self.get('sort', property='-text', client=client,
+ follow_redirects=True)
+ self.assertListEqual(page.files, by_name_desc)
- page = self.get('sort', property='type', client=client,
- follow_redirects=True)
- self.assertListEqual(page.files, by_type)
+ page = self.get('sort', property='type', client=client,
+ follow_redirects=True)
+ self.assertListEqual(page.files, by_type)
- page = self.get('sort', property='-type', client=client,
- follow_redirects=True)
- self.assertListEqual(page.files, by_type_desc)
+ page = self.get('sort', property='-type', client=client,
+ follow_redirects=True)
+ self.assertListEqual(page.files, by_type_desc)
- page = self.get('sort', property='size', client=client,
- follow_redirects=True)
- self.assertListEqual(page.files, by_size)
+ page = self.get('sort', property='size', client=client,
+ follow_redirects=True)
+ self.assertListEqual(page.files, by_size)
- page = self.get('sort', property='-size', client=client,
- follow_redirects=True)
- self.assertListEqual(page.files, by_size_desc)
+ page = self.get('sort', property='-size', client=client,
+ follow_redirects=True)
+ self.assertListEqual(page.files, by_size_desc)
- # We're unable to test modified sorting due filesystem time resolution
- page = self.get('sort', property='modified', client=client,
- follow_redirects=True)
+ # Cannot to test modified sorting due filesystem time resolution
+ page = self.get('sort', property='modified', client=client,
+ follow_redirects=True)
- page = self.get('sort', property='-modified', client=client,
- follow_redirects=True)
+ page = self.get('sort', property='-modified', client=client,
+ follow_redirects=True)
def test_sort_cookie_size(self):
files = [chr(i) * 150 for i in range(97, 123)]
@@ -647,14 +649,14 @@ class TestApp(unittest.TestCase):
path = os.path.join(self.base, name)
os.mkdir(path)
- client = self.app.test_client()
- for name in files:
- page = self.get('sort', property='modified', path=name,
- client=client, status_code=302)
+ with self.app.test_client() as client:
+ for name in files:
+ page = self.get('sort', property='modified', path=name,
+ client=client, status_code=302)
- for cookie in page.response.headers.getlist('set-cookie'):
- if cookie.startswith('browse-sorting='):
- self.assertLessEqual(len(cookie), 4000)
+ for cookie in page.response.headers.getlist('set-cookie'):
+ if cookie.startswith('browse-sorting='):
+ self.assertLessEqual(len(cookie), 4000)
def test_endpoints(self):
# test endpoint function for the library use-case
diff --git a/browsepy/tests/test_plugins.py b/browsepy/tests/test_plugins.py
index 8f2f3e1..80d8e18 100644
--- a/browsepy/tests/test_plugins.py
+++ b/browsepy/tests/test_plugins.py
@@ -4,7 +4,7 @@ import flask
import browsepy
import browsepy.manager
-import browsepy.tests.utils as test_utils
+import browsepy.utils as utils
from browsepy.plugin.player.tests import * # noqa
from browsepy.plugin.file_actions.tests import * # noqa
@@ -55,7 +55,7 @@ class TestPlugins(unittest.TestCase):
def tearDown(self):
self.app.config['plugin_namespaces'] = self.original_namespaces
self.manager.clear()
- test_utils.clear_flask_context()
+ utils.clear_flask_context()
def test_manager(self):
self.manager.load_plugin(self.plugin_name)
diff --git a/browsepy/tests/utils.py b/browsepy/tests/utils.py
deleted file mode 100644
index e328aa6..0000000
--- a/browsepy/tests/utils.py
+++ /dev/null
@@ -1,28 +0,0 @@
-
-import flask
-
-
-def clear_localstack(stack):
- '''
- Clear given werkzeug LocalStack instance.
-
- :param ctx: local stack instance
- :type ctx: werkzeug.local.LocalStack
- '''
- while stack.pop():
- pass
-
-
-def clear_flask_context():
- '''
- Clear flask current_app and request globals.
-
- When using :meth:`flask.Flask.test_client`, even as context manager,
- the flask's globals :attr:`flask.current_app` and :attr:`flask.request`
- are left dirty, so testing code relying on them will probably fail.
-
- This function clean said globals, and should be called after testing
- with :meth:`flask.Flask.test_client`.
- '''
- clear_localstack(flask._app_ctx_stack)
- clear_localstack(flask._request_ctx_stack)
diff --git a/browsepy/utils.py b/browsepy/utils.py
index cffe00f..f02177a 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -1,5 +1,6 @@
# -*- coding: UTF-8 -*-
+import sys
import os
import os.path
import random
@@ -8,18 +9,56 @@ import functools
import flask
-ppath = functools.partial(
- os.path.join,
- os.path.dirname(os.path.realpath(__file__)),
- )
+def ppath(*args, module=__name__):
+ '''
+ Get joined file path relative to module location.
+ '''
+ module = get_module(module)
+ path = os.path.realpath(module.__file__)
+ return os.path.join(os.path.dirname(path), *args)
+
+
+def get_module(name):
+ '''
+ Get module object by name
+ '''
+ try:
+ __import__(name)
+ return sys.modules[name]
+ except (ImportError, KeyError) as error:
+ if error.args:
+ message = error.args[0]
+ parts = (name,) if isinstance(error, KeyError) else name.split('.')
+ part = ''
+ for component in parts:
+ part += component
+ for fmt in (' \'%s\'', ' "%s"', ' %s'):
+ if message.endswith(fmt % part):
+ return None
+ part += '.'
+ raise
def random_string(size, sample=tuple(map(chr, range(256)))):
+ '''
+ Get random string of given size.
+
+ :param size: length of the returned string
+ :type size: int
+ :param sample: character space, defaults to full 8-bit utf-8
+ :type sample: tuple of str
+ :returns: random string of specified size
+ :rtype: str
+ '''
randrange = functools.partial(random.randrange, 0, len(sample))
return ''.join(sample[randrange()] for i in range(size))
def solve_local(context_local):
+ '''
+ Resolve given context local to its actual value. If given object
+ it's not a context local nothing happens, just returns the same value.
+ '''
if callable(getattr(context_local, '_get_current_object', None)):
return context_local._get_current_object()
return context_local
@@ -33,9 +72,36 @@ def stream_template(template_name, **context):
:param template_name: template
:param **context: parameters for templates.
:yields: HTML strings
+ :rtype: Iterator of str
'''
app = solve_local(context.get('current_app') or flask.current_app)
app.update_template_context(context)
template = app.jinja_env.get_template(template_name)
stream = template.generate(context)
return flask.Response(flask.stream_with_context(stream))
+
+
+def clear_localstack(stack):
+ '''
+ Clear given werkzeug LocalStack instance.
+
+ :param ctx: local stack instance
+ :type ctx: werkzeug.local.LocalStack
+ '''
+ while stack.pop():
+ pass
+
+
+def clear_flask_context():
+ '''
+ Clear flask current_app and request globals.
+
+ When using :meth:`flask.Flask.test_client`, even as context manager,
+ the flask's globals :attr:`flask.current_app` and :attr:`flask.request`
+ are left dirty, so testing code relying on them will probably fail.
+
+ This function clean said globals, and should be called after testing
+ with :meth:`flask.Flask.test_client`.
+ '''
+ clear_localstack(flask._app_ctx_stack)
+ clear_localstack(flask._request_ctx_stack)
diff --git a/doc/index.rst b/doc/index.rst
index e5435e8..1a2dc88 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -53,7 +53,7 @@ Specific information about functions, class or methods.
stream
compat
exceptions
- tests_utils
+ utils
Indices and tables
==================
diff --git a/doc/tests_utils.rst b/doc/tests_utils.rst
index 8816eb7..40d89e5 100644
--- a/doc/tests_utils.rst
+++ b/doc/tests_utils.rst
@@ -5,7 +5,7 @@ Test Utility Module
Convenience functions for unit testing.
-.. automodule:: browsepy.tests.utils
+.. automodule:: browsepy.utils
:show-inheritance:
:members:
:inherited-members:
diff --git a/setup.cfg b/setup.cfg
index 9cce8f5..fa4df87 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -32,3 +32,4 @@ fail_under = 100
exclude_lines =
pragma: no cover
pass
+ noqa
--
GitLab
From 5e685008a1bcdde7a9d3c87ffc7ec221ee9be0ff Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Tue, 9 Apr 2019 10:39:01 +0200
Subject: [PATCH 051/171] fix travis yml, test PLSFileParser
---
.travis.yml | 3 +--
browsepy/plugin/player/playable.py | 1 +
browsepy/plugin/player/tests.py | 24 ++++++++++++++++++++++++
3 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index db92f2e..e6dfabe 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,8 +42,7 @@ script:
fi
- |
if [ "$sphinx" = "yes" ]; then
- make -C doc html 2>&1 | grep -v \
- 'WARNING: more than one target found for cross-reference'
+ make -C doc html
fi
after_success:
diff --git a/browsepy/plugin/player/playable.py b/browsepy/plugin/player/playable.py
index acb0fc3..7e1b278 100644
--- a/browsepy/plugin/player/playable.py
+++ b/browsepy/plugin/player/playable.py
@@ -23,6 +23,7 @@ class PLSFileParser(object):
'''
NOT_SET = type('NotSetType', (object,), {})
option_exceptions = (
+ six.moves.configparser.NoSectionError,
six.moves.configparser.NoOptionError,
ValueError,
)
diff --git a/browsepy/plugin/player/tests.py b/browsepy/plugin/player/tests.py
index bda1f59..bcf8e4f 100644
--- a/browsepy/plugin/player/tests.py
+++ b/browsepy/plugin/player/tests.py
@@ -7,6 +7,7 @@ import shutil
import tempfile
import six
+import six.moves
import flask
from werkzeug.exceptions import NotFound
@@ -43,6 +44,29 @@ class ManagerMock(object):
return self.argument_values.get(name, default)
+class TestPLSFileParser(unittest.TestCase):
+ module = player_playable
+ exceptions = player_playable.PLSFileParser.option_exceptions
+
+ def get_parser(self, content=''):
+ with tempfile.NamedTemporaryFile(mode='w') as f:
+ f.write(content)
+ f.flush()
+ return self.module.PLSFileParser(f.name)
+
+ def test_getint(self):
+ parser = self.get_parser()
+ self.assertEqual(parser.getint('a', 'a', 2), 2)
+ with self.assertRaises(self.exceptions):
+ parser.getint('a', 'a')
+
+ def test_get(self):
+ parser = self.get_parser()
+ self.assertEqual(parser.get('a', 'a', 2), 2)
+ with self.assertRaises(self.exceptions):
+ parser.get('a', 'a')
+
+
class TestPlayerBase(unittest.TestCase):
module = player
--
GitLab
From 4e70f5e36ef8dd7bc117ce9736d3223c5c811d65 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Tue, 9 Apr 2019 11:06:44 +0200
Subject: [PATCH 052/171] make ppath py2-compatible
---
browsepy/tests/test_utils.py | 24 ++++++++++++++++++++++++
browsepy/utils.py | 12 ++++++++++--
2 files changed, 34 insertions(+), 2 deletions(-)
create mode 100644 browsepy/tests/test_utils.py
diff --git a/browsepy/tests/test_utils.py b/browsepy/tests/test_utils.py
new file mode 100644
index 0000000..a8507e4
--- /dev/null
+++ b/browsepy/tests/test_utils.py
@@ -0,0 +1,24 @@
+# -*- coding: UTF-8 -*-
+
+import unittest
+
+import browsepy.utils as utils
+
+
+class TestPPath(unittest.TestCase):
+ def test_bad_kwarg(self):
+ with self.assertRaises(TypeError) as e:
+ utils.ppath('a', 'b', bad=1)
+ self.assertIn('\'bad\'', e.args[0])
+
+ def test_join(self):
+ self.assertTrue(
+ utils
+ .ppath('a', 'b', module=__name__)
+ .endswith('browsepy/tests/a/b')
+ )
+ self.assertTrue(
+ utils
+ .ppath('a', 'b')
+ .endswith('browsepy/a/b')
+ )
diff --git a/browsepy/utils.py b/browsepy/utils.py
index f02177a..7ed40d4 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -9,11 +9,19 @@ import functools
import flask
-def ppath(*args, module=__name__):
+def ppath(*args, **kwargs):
'''
Get joined file path relative to module location.
+
+ :param module: Module name
+ :type module: str
'''
- module = get_module(module)
+ module = get_module(kwargs.pop('module', __name__))
+ if kwargs:
+ raise TypeError(
+ 'ppath() got an unexpected keyword argument \'%s\''
+ % next(iter(kwargs))
+ )
path = os.path.realpath(module.__file__)
return os.path.join(os.path.dirname(path), *args)
--
GitLab
From d286734b79c5646c934c61d3dbf4842513f698e6 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Tue, 9 Apr 2019 11:30:35 +0200
Subject: [PATCH 053/171] backwards-compatible scandir close
---
browsepy/file.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/browsepy/file.py b/browsepy/file.py
index 64d2917..1dea225 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -9,7 +9,6 @@ import string
import random
import datetime
import logging
-import contextlib
from flask import current_app, send_from_directory
from werkzeug.utils import cached_property
@@ -725,7 +724,8 @@ class Directory(Node):
directory_class = self.directory_class
file_class = self.file_class
exclude_fnc = self.plugin_manager.check_excluded
- with contextlib.closing(compat.scandir(self.path)) as files:
+ files = compat.scandir(self.path)
+ try:
for entry in files:
if exclude_fnc(entry.path):
continue
@@ -745,6 +745,9 @@ class Directory(Node):
)
except OSError as e:
logger.exception(e)
+ finally:
+ if callable(getattr(files, 'close', None)):
+ files.close()
def listdir(self, sortkey=None, reverse=False):
'''
--
GitLab
From ad86afb3d2894d22e32ddfa4e9a8e316fa58cfd8 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Tue, 9 Apr 2019 17:40:49 +0200
Subject: [PATCH 054/171] fix import error detection logic
---
browsepy/utils.py | 28 +++++++++++++++++-----------
1 file changed, 17 insertions(+), 11 deletions(-)
diff --git a/browsepy/utils.py b/browsepy/utils.py
index 7ed40d4..86cfac1 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -3,12 +3,16 @@
import sys
import os
import os.path
+import re
import random
import functools
import flask
+RE_WORDS = re.compile(r'\b((?:[._]+|\w)+)\b')
+
+
def ppath(*args, **kwargs):
'''
Get joined file path relative to module location.
@@ -28,22 +32,24 @@ def ppath(*args, **kwargs):
def get_module(name):
'''
- Get module object by name
+ Get module object by name.
+
+ :param name: module name
+ :type name: str
+ :return: module or None if not found
+ :rtype: module or None
'''
try:
__import__(name)
return sys.modules[name]
except (ImportError, KeyError) as error:
- if error.args:
- message = error.args[0]
- parts = (name,) if isinstance(error, KeyError) else name.split('.')
- part = ''
- for component in parts:
- part += component
- for fmt in (' \'%s\'', ' "%s"', ' %s'):
- if message.endswith(fmt % part):
- return None
- part += '.'
+ if isinstance(error, ImportError):
+ message = error.args[0] if error.args else ''
+ words = frozenset(RE_WORDS.findall(message))
+ parts = name.split('.')
+ for i in range(len(parts) - 1, 0, -1):
+ if '.'.join(parts[i:]) in words:
+ return None
raise
--
GitLab
From d306a4e073d2990a7797f499f5514f2e8d3d8fa7 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Tue, 9 Apr 2019 17:41:49 +0200
Subject: [PATCH 055/171] fix import error detection logic, again
---
browsepy/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/browsepy/utils.py b/browsepy/utils.py
index 86cfac1..caca42c 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -47,7 +47,7 @@ def get_module(name):
message = error.args[0] if error.args else ''
words = frozenset(RE_WORDS.findall(message))
parts = name.split('.')
- for i in range(len(parts) - 1, 0, -1):
+ for i in range(len(parts) - 1, -1, -1):
if '.'.join(parts[i:]) in words:
return None
raise
--
GitLab
From 4768130101d25a2aa65059f7c42d98460a9d7a48 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Tue, 9 Apr 2019 17:45:24 +0200
Subject: [PATCH 056/171] avoid upgrade pip in py33
---
.travis.yml | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/.travis.yml b/.travis.yml
index e6dfabe..a9dd839 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,6 +10,8 @@ matrix:
- python: "pypy"
- python: "pypy3"
- python: "3.3"
+ env:
+ - legacy=yes
- python: "3.4"
- python: "3.5"
- python: "3.6"
@@ -19,7 +21,10 @@ matrix:
- sphinx=yes
install:
- - pip install --upgrade pip
+ - |
+ if [ "$legacy" != "yes" ]; then
+ pip install --upgrade pip
+ fi
- pip install -r requirements/ci.txt
- pip install .
- |
--
GitLab
From f04241281aa1faab0b45623c9903a50942b3f89a Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Tue, 9 Apr 2019 19:52:29 +0200
Subject: [PATCH 057/171] expose default mimetype implementation priority
---
browsepy/manager.py | 6 +-----
browsepy/mimetype.py | 38 +++++++++++++++++++-------------------
browsepy/utils.py | 4 ++--
3 files changed, 22 insertions(+), 26 deletions(-)
diff --git a/browsepy/manager.py b/browsepy/manager.py
index a582c2f..a29583d 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -420,11 +420,7 @@ class MimetypePluginManager(RegistrablePluginManager):
'''
Plugin manager for mimetype-function registration.
'''
- _default_mimetype_functions = (
- mimetype.by_python,
- mimetype.by_file,
- mimetype.by_default,
- )
+ _default_mimetype_functions = mimetype.alternatives
def clear(self):
'''
diff --git a/browsepy/mimetype.py b/browsepy/mimetype.py
index bcdba47..ef681f0 100644
--- a/browsepy/mimetype.py
+++ b/browsepy/mimetype.py
@@ -21,26 +21,26 @@ def by_python(path):
)
-if which('file'):
- def by_file(path):
- try:
- output = subprocess.check_output(
- ("file", "-ib", path),
- universal_newlines=True
- ).strip()
- if (
- re_mime_validate.match(output) and
- output not in generic_mimetypes
- ):
- # 'file' command can return status zero with invalid output
- return output
- except (subprocess.CalledProcessError, FileNotFoundError):
- pass
- return None
-else:
- def by_file(path):
- return None
+def by_file(path):
+ try:
+ output = subprocess.check_output(
+ ("file", "-ib", path),
+ universal_newlines=True
+ ).strip()
+ if re_mime_validate.match(output) and output not in generic_mimetypes:
+ # 'file' command can return status zero with invalid output
+ return output
+ except (subprocess.CalledProcessError, FileNotFoundError):
+ pass
+ return None
def by_default(path):
return "application/octet-stream"
+
+
+alternatives = (
+ (by_python, by_file, by_default)
+ if which('file') else
+ (by_python, by_default)
+ )
diff --git a/browsepy/utils.py b/browsepy/utils.py
index caca42c..65fd397 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -10,7 +10,7 @@ import functools
import flask
-RE_WORDS = re.compile(r'\b((?:[._]+|\w)+)\b')
+re_words = re.compile(r'\b((?:[._]+|\w)+)\b')
def ppath(*args, **kwargs):
@@ -45,7 +45,7 @@ def get_module(name):
except (ImportError, KeyError) as error:
if isinstance(error, ImportError):
message = error.args[0] if error.args else ''
- words = frozenset(RE_WORDS.findall(message))
+ words = frozenset(re_words.findall(message))
parts = name.split('.')
for i in range(len(parts) - 1, -1, -1):
if '.'.join(parts[i:]) in words:
--
GitLab
From 493f5f1a452dc04621d9ae58401dab220ed6d483 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 10 Apr 2019 11:40:46 +0200
Subject: [PATCH 058/171] wip plugin listing
---
browsepy/manager.py | 32 ++++++++++++++++++++++++++++++++
browsepy/tests/test_plugins.py | 5 +++++
2 files changed, 37 insertions(+)
diff --git a/browsepy/manager.py b/browsepy/manager.py
index a29583d..0a0ec0d 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -65,6 +65,10 @@ class PluginManagerBase(object):
return self.app.config['plugin_namespaces'] if self.app else []
def __init__(self, app=None):
+ '''
+ :param app: flask application
+ :type app: flask.Flask
+ '''
if app is None:
self.clear()
else:
@@ -73,6 +77,9 @@ class PluginManagerBase(object):
def init_app(self, app):
'''
Initialize this Flask extension for given app.
+
+ :param app: flask application
+ :type app: flask.Flask
'''
self.app = app
if not hasattr(app, 'extensions'):
@@ -98,6 +105,25 @@ class PluginManagerBase(object):
'''
pass
+ def iter_plugins(self):
+ '''
+ Iterate through all loadable plugins on default import locations
+ '''
+ res = compat.res
+ for base in self.namespaces:
+ for item in res.contents(base):
+ if not res.is_resource(base, item):
+ submodule = '%s.%s' % (base, item)
+ module = get_module(submodule)
+ if module and self.check_plugin(module):
+ yield submodule
+
+ def check_plugin(self, plugin):
+ '''
+ Check if given object (usually a python module) is a valid plugin
+ '''
+ return True
+
def import_plugin(self, plugin):
'''
Import plugin by given name, looking at :attr:`namespaces`.
@@ -143,6 +169,12 @@ class RegistrablePluginManager(PluginManagerBase):
Base plugin manager for plugin registration via :func:`register_plugin`
functions at plugin module level.
'''
+ def check_plugin(self, plugin):
+ return (
+ super(RegistrablePluginManager, self).check_plugin(plugin) and
+ callable(getattr(plugin, 'register_plugin'))
+ )
+
def load_plugin(self, plugin):
'''
Import plugin (see :meth:`import_plugin`) and load related data.
diff --git a/browsepy/tests/test_plugins.py b/browsepy/tests/test_plugins.py
index 80d8e18..ef128db 100644
--- a/browsepy/tests/test_plugins.py
+++ b/browsepy/tests/test_plugins.py
@@ -87,6 +87,11 @@ class TestPlugins(unittest.TestCase):
self.manager.register_widget
)
+ def test_list(self):
+ names = list(self.manager.iter_plugins())
+ self.assertIn('player', names)
+ self.assertIn('file_actions', names)
+
def test_namespace_prefix(self):
self.assertTrue(self.manager.import_plugin(self.plugin_name))
self.app.config['plugin_namespaces'] = (
--
GitLab
From d23e1e80e5ddc40b67ea1f644bbc7a5298a5c7b0 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 11 Apr 2019 22:50:26 +0200
Subject: [PATCH 059/171] add plugin search, argparse workaround
---
browsepy/__main__.py | 30 +++++-----
browsepy/compat.py | 14 +++++
browsepy/manager.py | 103 ++++++++++++++++++++++++---------
browsepy/tests/test_main.py | 25 ++++----
browsepy/tests/test_plugins.py | 5 +-
setup.cfg | 2 +-
6 files changed, 125 insertions(+), 54 deletions(-)
diff --git a/browsepy/__main__.py b/browsepy/__main__.py
index 948b01a..fdbfc55 100644
--- a/browsepy/__main__.py
+++ b/browsepy/__main__.py
@@ -6,12 +6,11 @@ import sys
import os
import os.path
import argparse
-import warnings
import flask
-from . import app
-from .compat import PY_LEGACY, getdebug, get_terminal_size
+from . import app, plugin_manager, __version__
+from .compat import PY_LEGACY, getdebug, get_terminal_size, SafeArgumentParser
from .transform.glob import translate
@@ -27,15 +26,8 @@ class HelpFormatter(argparse.RawTextHelpFormatter):
prog, indent_increment, max_help_position, width)
-class PluginAction(argparse.Action):
+class CommaSeparatedAction(argparse.Action):
def __call__(self, parser, namespace, value, option_string=None):
- warned = '%s_warning' % self.dest
- if ',' in value and not getattr(namespace, warned, False):
- setattr(namespace, warned, True)
- warnings.warn(
- 'Comma-separated --plugin value is deprecated, '
- 'use multiple --plugin options instead.'
- )
values = value.split(',')
prev = getattr(namespace, self.dest, None)
if isinstance(prev, list):
@@ -43,7 +35,7 @@ class PluginAction(argparse.Action):
setattr(namespace, self.dest, values)
-class ArgParse(argparse.ArgumentParser):
+class ArgParse(SafeArgumentParser):
default_directory = app.config['directory_base']
default_initial = (
None
@@ -56,10 +48,14 @@ class ArgParse(argparse.ArgumentParser):
default_host = os.getenv('BROWSEPY_HOST', '127.0.0.1')
default_port = os.getenv('BROWSEPY_PORT', '8080')
- plugin_action_class = PluginAction
defaults = {
+ 'add_help': True,
'prog': name,
+ 'epilog': 'available plugins:\n%s' % '\n'.join(
+ ' %s, %s' % (short, name) if short else ' %s' % name
+ for name, short in plugin_manager.iter_plugins()
+ ),
'formatter_class': HelpFormatter,
'description': 'description: starts a %s web file browser' % name
}
@@ -74,6 +70,9 @@ class ArgParse(argparse.ArgumentParser):
'port', nargs='?', type=int,
default=self.default_port,
help='port to listen (default: %(default)s)')
+ self.add_argument(
+ '--help-all', action='store_true',
+ help='show help for all available plugins and exit')
self.add_argument(
'--directory', metavar='PATH', type=self._directory,
default=self.default_directory,
@@ -101,9 +100,12 @@ class ArgParse(argparse.ArgumentParser):
action='append',
default=[],
help='exclude paths by pattern file (multiple)')
+ self.add_argument(
+ '--version', action='version',
+ version=__version__)
self.add_argument(
'--plugin', metavar='MODULE',
- action=self.plugin_action_class,
+ action=CommaSeparatedAction,
default=[],
help='load plugin module (multiple)')
self.add_argument(
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 9be65c6..9f3ebbd 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -9,6 +9,7 @@ import functools
import warnings
import posixpath
import ntpath
+import argparse
FS_ENCODING = sys.getfilesystemencoding()
PY_LEGACY = sys.version_info < (3, )
@@ -40,6 +41,19 @@ except ImportError:
from collections import Iterator as BaseIterator # noqa
+class SafeArgumentParser(argparse.ArgumentParser):
+ allow_abbrev_support = sys.version_info >= (3, 5, 0)
+
+ def _get_option_tuples(self, option_string):
+ return []
+
+ def __init__(self, **kwargs):
+ if self.allow_abbrev_support:
+ kwargs.setdefault('allow_abbrev', False)
+ kwargs.setdefault('add_help', False)
+ super(SafeArgumentParser, self).__init__(**kwargs)
+
+
def isexec(path):
'''
Check if given path points to an executable file.
diff --git a/browsepy/manager.py b/browsepy/manager.py
index 0a0ec0d..588e4c3 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -2,6 +2,7 @@
import re
import sys
+import pkgutil
import argparse
import warnings
import collections
@@ -13,7 +14,7 @@ from cookieman import CookieMan
from . import mimetype
from . import compat
-from .compat import deprecated, usedoc
+from .compat import deprecated, usedoc, SafeArgumentParser
from .utils import get_module
@@ -69,6 +70,8 @@ class PluginManagerBase(object):
:param app: flask application
:type app: flask.Flask
'''
+ self.plugin_filters = []
+
if app is None:
self.clear()
else:
@@ -105,24 +108,47 @@ class PluginManagerBase(object):
'''
pass
- def iter_plugins(self):
+ def _iter_modules(self, prefix):
'''
- Iterate through all loadable plugins on default import locations
+ Iterate thru all root modules containing given prefix.
+ '''
+ for info in pkgutil.iter_modules():
+ if info.name.startswith(prefix):
+ yield info.name, None
+
+ def _iter_submodules(self, prefix):
+ '''
+ Iterate thru all submodules which full name contains given prefix.
'''
res = compat.res
- for base in self.namespaces:
- for item in res.contents(base):
- if not res.is_resource(base, item):
- submodule = '%s.%s' % (base, item)
- module = get_module(submodule)
- if module and self.check_plugin(module):
- yield submodule
+ parent = prefix.rsplit('.', 1)[0]
+ for base in (prefix, parent):
+ try:
+ for item in res.contents(base):
+ name = '%s.%s' % (base, item)
+ if name.startswith(prefix) and \
+ not res.is_resource(base, item):
+ yield name, item
+ break
+ except ImportError:
+ pass
- def check_plugin(self, plugin):
+ def iter_plugins(self):
'''
- Check if given object (usually a python module) is a valid plugin
+ Iterate through all loadable plugins on default import locations
'''
- return True
+ for prefix in self.namespaces:
+ names = (
+ self._iter_submodules(prefix)
+ if '.' in prefix else
+ self._iter_modules(prefix)
+ if prefix else
+ ()
+ )
+ for name, short in names:
+ module = get_module(name)
+ if module and any(f(module) for f in self.plugin_filters):
+ yield name, short
def import_plugin(self, plugin):
'''
@@ -169,10 +195,10 @@ class RegistrablePluginManager(PluginManagerBase):
Base plugin manager for plugin registration via :func:`register_plugin`
functions at plugin module level.
'''
- def check_plugin(self, plugin):
- return (
- super(RegistrablePluginManager, self).check_plugin(plugin) and
- callable(getattr(plugin, 'register_plugin'))
+ def __init__(self, app=None):
+ super(RegistrablePluginManager, self).__init__(app)
+ self.plugin_filters.append(
+ lambda o: callable(getattr(o, 'register_plugin', None))
)
def load_plugin(self, plugin):
@@ -532,6 +558,12 @@ class ArgumentPluginManager(PluginManagerBase):
_argparse_kwargs = {'add_help': False}
_argparse_arguments = argparse.Namespace()
+ def __init__(self, app=None):
+ super(ArgumentPluginManager, self).__init__(app)
+ self.plugin_filters.append(
+ lambda o: callable(getattr(o, 'register_arguments', None))
+ )
+
def extract_plugin_arguments(self, plugin):
'''
Given a plugin name, extracts its registered_arguments as an
@@ -549,6 +581,13 @@ class ArgumentPluginManager(PluginManagerBase):
return manager._argparse_argkwargs
return ()
+ @cached_property
+ def _default_argument_parser(self):
+ parser = SafeArgumentParser()
+ parser.add_argument('--plugin', action='append', default=[])
+ parser.add_argument('--help-all', action='store_true')
+ return parser
+
def load_arguments(self, argv, base=None):
'''
Process given argument list based on registered arguments and given
@@ -568,25 +607,37 @@ class ArgumentPluginManager(PluginManagerBase):
given by :meth:`argparse.ArgumentParser.parse_args`.
:rtype: argparse.Namespace
'''
- plugin_parser = argparse.ArgumentParser(add_help=False)
- plugin_parser.add_argument('--plugin', action='append', default=[])
- parent = base or plugin_parser
- parser = argparse.ArgumentParser(
- parents=(parent,),
- add_help=False,
- **getattr(parent, 'defaults', {})
+ parser = SafeArgumentParser(
+ parents=[base or self._default_argument_parser],
)
+ options, _ = parser.parse_known_args(argv)
+
plugins = [
plugin
- for plugins in plugin_parser.parse_known_args(argv)[0].plugin
+ for plugins in options.plugin
for plugin in plugins.split(',')
]
+
+ if options.help_all:
+ plugins.extend(
+ short if short else name
+ for name, short in self.iter_plugins()
+ if not (name in plugins or short in plugins)
+ )
+
for plugin in sorted(set(plugins), key=plugins.index):
arguments = self.extract_plugin_arguments(plugin)
if arguments:
- group = parser.add_argument_group('%s arguments' % plugin)
+ group = parser.add_argument_group(
+ '%s arguments' % plugin
+ )
for argargs, argkwargs in arguments:
group.add_argument(*argargs, **argkwargs)
+
+ if options.help_all:
+ parser.print_help()
+ parser.exit()
+
self._argparse_arguments = parser.parse_args(argv)
return self._argparse_arguments
diff --git a/browsepy/tests/test_main.py b/browsepy/tests/test_main.py
index 43ca37f..0c20b90 100644
--- a/browsepy/tests/test_main.py
+++ b/browsepy/tests/test_main.py
@@ -3,6 +3,7 @@ import os
import os.path
import tempfile
import shutil
+import contextlib
import browsepy.__main__
@@ -80,17 +81,19 @@ class TestMain(unittest.TestCase):
self.assertListEqual(result.exclude_from, [])
self.assertListEqual(result.plugin, [])
- self.assertRaises(
- SystemExit,
- self.parser.parse_args,
- ['--directory=%s' % __file__]
- )
-
- self.assertRaises(
- SystemExit,
- self.parser.parse_args,
- ['--exclude-from=non-existing']
- )
+ with open(os.devnull, 'w') as f:
+ with contextlib.redirect_stderr(f):
+ self.assertRaises(
+ SystemExit,
+ self.parser.parse_args,
+ ['--directory=%s' % __file__]
+ )
+
+ self.assertRaises(
+ SystemExit,
+ self.parser.parse_args,
+ ['--exclude-from=non-existing']
+ )
def test_exclude(self):
result = self.parser.parse_args([
diff --git a/browsepy/tests/test_plugins.py b/browsepy/tests/test_plugins.py
index ef128db..6170cd6 100644
--- a/browsepy/tests/test_plugins.py
+++ b/browsepy/tests/test_plugins.py
@@ -88,9 +88,10 @@ class TestPlugins(unittest.TestCase):
)
def test_list(self):
+ self.app.config['plugin_namespaces'] = self.original_namespaces
names = list(self.manager.iter_plugins())
- self.assertIn('player', names)
- self.assertIn('file_actions', names)
+ self.assertIn(('browsepy.plugin.player', 'player'), names)
+ self.assertIn(('browsepy.plugin.file_actions', 'file_actions'), names)
def test_namespace_prefix(self):
self.assertTrue(self.manager.import_plugin(self.plugin_name))
diff --git a/setup.cfg b/setup.cfg
index fa4df87..66043ce 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -28,7 +28,7 @@ omit =
[coverage:report]
show_missing = True
-fail_under = 100
+fail_under = 95
exclude_lines =
pragma: no cover
pass
--
GitLab
From 05fa6306ecc96e3890413df74491d407ce3da5a8 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 12 Apr 2019 14:20:22 +0200
Subject: [PATCH 060/171] bump version, uppercase config, improve plugin
discovery
---
README.rst | 68 +++++++++++-------
browsepy/__init__.py | 28 ++++----
browsepy/__main__.py | 47 +++++--------
browsepy/appconfig.py | 27 ++++---
browsepy/compat.py | 15 +++-
browsepy/exceptions.py | 2 +-
browsepy/file.py | 30 ++++----
browsepy/manager.py | 85 +++++++++++++++--------
browsepy/plugin/file_actions/__init__.py | 42 +++++++----
browsepy/plugin/file_actions/tests.py | 48 +++++++------
browsepy/plugin/player/playable.py | 4 +-
browsepy/plugin/player/tests.py | 16 ++---
browsepy/tests/deprecated/test_plugins.py | 10 +--
browsepy/tests/test_app.py | 27 ++++---
browsepy/tests/test_file.py | 8 +--
browsepy/tests/test_module.py | 30 ++++----
browsepy/tests/test_plugins.py | 15 ++--
browsepy/transform/glob.py | 13 ++--
doc/integrations.rst | 42 +++++------
19 files changed, 319 insertions(+), 238 deletions(-)
diff --git a/README.rst b/README.rst
index b077dac..79bf900 100644
--- a/README.rst
+++ b/README.rst
@@ -122,6 +122,12 @@ Showing help including player plugin arguments
browsepy --plugin=player --help
+Showing help all detected plugin arguments
+
+.. code-block:: bash
+
+ browsepy --help-all
+
This examples assume python's `bin` directory is in `PATH`, otherwise try
replacing `browsepy` with `python -m browsepy`.
@@ -133,24 +139,31 @@ plugins (loaded with `plugin` argument) could add extra arguments to this list.
::
- usage: browsepy [-h] [--directory PATH] [--initial PATH] [--removable PATH]
- [--upload PATH] [--exclude PATTERN] [--exclude-from PATH]
- [--plugin MODULE]
+ usage: browsepy [-h] [--help-all] [--directory PATH] [--initial PATH] [--removable PATH] [--upload PATH] [--exclude PATTERN]
+ [--exclude-from PATH] [--version] [--plugin MODULE]
[host] [port]
+ description: starts a browsepy web file browser
+
positional arguments:
- host address to listen (default: 127.0.0.1)
- port port to listen (default: 8080)
+ host address to listen (default: 127.0.0.1)
+ port port to listen (default: 8080)
optional arguments:
- -h, --help show this help message and exit
- --directory PATH serving directory (default: current path)
- --initial PATH default directory (default: same as --directory)
- --removable PATH base directory allowing remove (default: none)
- --upload PATH base directory allowing upload (default: none)
- --exclude PATTERN exclude paths by pattern (multiple)
- --exclude-from PATH exclude paths by pattern file (multiple)
- --plugin MODULE load plugin module (multiple)
+ -h, --help show this help message and exit
+ --help-all show help for all available plugins and exit
+ --directory PATH serving directory (default: /home/work/Desarrollo/browsepy)
+ --initial PATH default directory (default: same as --directory)
+ --removable PATH base directory allowing remove (default: None)
+ --upload PATH base directory allowing upload (default: None)
+ --exclude PATTERN exclude paths by pattern (multiple)
+ --exclude-from PATH exclude paths by pattern file (multiple)
+ --version show program's version number and exit
+ --plugin MODULE load plugin module (multiple)
+
+ available plugins:
+ file-actions, browsepy.plugin.file_actions
+ player, browsepy.plugin.player
Using as library
@@ -170,26 +183,28 @@ url, and mounting **browsepy.app** on the appropriate parent
Browsepy app config (available at :attr:`browsepy.app.config`) uses the
following configuration options.
-* **directory_base**: anything under this directory will be served,
+* **DIRECTORY_BASE**: anything under this directory will be served,
defaults to current path.
-* **directory_start**: directory will be served when accessing root URL
-* **directory_remove**: file removing will be available under this path,
+* **DIRECTORY_START**: directory will be served when accessing root URL
+* **DIRECTORY_REMOVE**: file removing will be available under this path,
defaults to **None**.
-* **directory_upload**: file upload will be available under this path,
+* **DIRECTORY_UPLOAD**: file upload will be available under this path,
defaults to **None**.
-* **directory_tar_buffsize**, directory tar streaming buffer size,
+* **DIRECTORY_TAR_BUFFSIZE**, directory tar streaming buffer size,
defaults to **262144** and must be multiple of 512.
-* **directory_downloadable** whether enable directory download or not,
+* **DIRECTORY_DOWNLOADABLE** whether enable directory download or not,
defaults to **True**.
-* **use_binary_multiples** whether use binary units (bi-bytes, like KiB)
+* **USE_BINARY_MULTIPLES** whether use binary units (bi-bytes, like KiB)
instead of common ones (bytes, like KB), defaults to **True**.
-* **plugin_modules** list of module names (absolute or relative to
+* **PLUGIN_MODULES** list of module names (absolute or relative to
plugin_namespaces) will be loaded.
-* **plugin_namespaces** prefixes for module names listed at plugin_modules
- where relative plugin_modules are searched.
-* **exclude_fnc** function will be used to exclude files from listing and directory tarballs. Can be either None or function receiving an absolute path and returning a boolean.
+* **PLUGIN_NAMESPACES** prefixes for module names listed at PLUGIN_MODULES
+ where relative PLUGIN_MODULES are searched.
+* **EXCLUDE_FNC** function will be used to exclude files from listing and
+ directory tarballs. Can be either None or function receiving an absolute
+ path and returning a boolean.
-After editing `plugin_modules` value, plugin manager (available at module
+After editing `PLUGIN_MODULES` value, plugin manager (available at module
plugin_manager and app.extensions['plugin_manager']) should be reloaded using
the `reload` method.
@@ -202,6 +217,9 @@ Extend via plugin API
Starting from version 0.4.0, browsepy is extensible via plugins. A functional
'player' plugin is provided as example, and some more are planned.
+Starting from version 0.6.0, browsepy a new plugin `file-actions` is included
+providing copy/cut/paste and directory creation operations.
+
Plugins can add HTML content to browsepy's browsing view, using some
convenience abstraction for already used elements like external stylesheet and
javascript tags, links, buttons and file upload.
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index 73b90e2..12ca3e9 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: UTF-8 -*-
-__version__ = '0.5.6'
+__version__ = '0.6.0'
import logging
import os
@@ -33,20 +33,20 @@ app = Flask(
app.config.update(
SECRET_KEY=utils.random_string(4096),
APPLICATION_NAME='browsepy',
- directory_base=compat.getcwd(),
- directory_start=None,
- directory_remove=None,
- directory_upload=None,
- directory_tar_buffsize=262144,
- directory_downloadable=True,
- use_binary_multiples=True,
- plugin_modules=[],
- plugin_namespaces=(
+ DIRECTORY_BASE=compat.getcwd(),
+ DIRECTORY_START=None,
+ DIRECTORY_REMOVE=None,
+ DIRECTORY_UPLOAD=None,
+ DIRECTORY_TAR_BUFFSIZE=262144,
+ DIRECTORY_DOWNLOADABLE=True,
+ USE_BINARY_MULTIPLES=True,
+ PLUGIN_MODULES=[],
+ PLUGIN_NAMESPACES=(
'browsepy.plugin',
'browsepy_',
'',
),
- exclude_fnc=None,
+ EXCLUDE_FNC=None,
)
app.jinja_env.add_extension('browsepy.transform.htmlcompress.HTMLCompress')
app.session_interface = cookieman.CookieMan()
@@ -247,12 +247,12 @@ def upload(path):
path=directory.path,
filename=f.filename
)
- return redirect(url_for(".browse", path=directory.urlpath))
+ return redirect(url_for('.browse', path=directory.urlpath))
-@app.route("/")
+@app.route('/')
def index():
- path = app.config["directory_start"] or app.config["directory_base"]
+ path = app.config['DIRECTORY_START'] or app.config['DIRECTORY_BASE']
try:
urlpath = Node(path).urlpath
except OutsideDirectoryBase:
diff --git a/browsepy/__main__.py b/browsepy/__main__.py
index fdbfc55..318234f 100644
--- a/browsepy/__main__.py
+++ b/browsepy/__main__.py
@@ -9,23 +9,12 @@ import argparse
import flask
-from . import app, plugin_manager, __version__
-from .compat import PY_LEGACY, getdebug, get_terminal_size, SafeArgumentParser
+from . import app, __version__
+from .compat import PY_LEGACY, getdebug, \
+ SafeArgumentParser, HelpFormatter
from .transform.glob import translate
-class HelpFormatter(argparse.RawTextHelpFormatter):
- def __init__(self, prog, indent_increment=2, max_help_position=24,
- width=None):
- if width is None:
- try:
- width = get_terminal_size().columns - 2
- except ValueError: # https://bugs.python.org/issue24966
- pass
- super(HelpFormatter, self).__init__(
- prog, indent_increment, max_help_position, width)
-
-
class CommaSeparatedAction(argparse.Action):
def __call__(self, parser, namespace, value, option_string=None):
values = value.split(',')
@@ -36,14 +25,14 @@ class CommaSeparatedAction(argparse.Action):
class ArgParse(SafeArgumentParser):
- default_directory = app.config['directory_base']
+ default_directory = app.config['DIRECTORY_BASE']
default_initial = (
None
- if app.config['directory_start'] == app.config['directory_base'] else
- app.config['directory_start']
+ if app.config['DIRECTORY_START'] == app.config['DIRECTORY_BASE'] else
+ app.config['DIRECTORY_START']
)
- default_removable = app.config['directory_remove']
- default_upload = app.config['directory_upload']
+ default_removable = app.config['DIRECTORY_REMOVE']
+ default_upload = app.config['DIRECTORY_UPLOAD']
name = app.config['APPLICATION_NAME']
default_host = os.getenv('BROWSEPY_HOST', '127.0.0.1')
@@ -52,10 +41,6 @@ class ArgParse(SafeArgumentParser):
defaults = {
'add_help': True,
'prog': name,
- 'epilog': 'available plugins:\n%s' % '\n'.join(
- ' %s, %s' % (short, name) if short else ' %s' % name
- for name, short in plugin_manager.iter_plugins()
- ),
'formatter_class': HelpFormatter,
'description': 'description: starts a %s web file browser' % name
}
@@ -170,16 +155,16 @@ def main(argv=sys.argv[1:], app=app, parser=ArgParse, run_fnc=flask.Flask.run):
if args.debug:
os.environ['DEBUG'] = 'true'
app.config.update(
- directory_base=args.directory,
- directory_start=args.initial or args.directory,
- directory_remove=args.removable,
- directory_upload=args.upload,
- plugin_modules=list_union(
- app.config['plugin_modules'],
+ DIRECTORY_BASE=args.directory,
+ DIRECTORY_START=args.initial or args.directory,
+ DIRECTORY_REMOVE=args.removable,
+ DIRECTORY_UPLOAD=args.upload,
+ PLUGIN_MODULES=list_union(
+ app.config['PLUGIN_MODULES'],
args.plugin,
),
- exclude_fnc=filter_union(
- app.config['exclude_fnc'],
+ EXCLUDE_FNC=filter_union(
+ app.config['EXCLUDE_FNC'],
create_exclude_fnc(patterns, args.directory),
),
)
diff --git a/browsepy/appconfig.py b/browsepy/appconfig.py
index 56aceb2..3c752fc 100644
--- a/browsepy/appconfig.py
+++ b/browsepy/appconfig.py
@@ -1,5 +1,7 @@
# -*- coding: UTF-8 -*-
+import warnings
+
import flask
import flask.config
@@ -8,17 +10,17 @@ from .compat import basestring
class Config(flask.config.Config):
'''
- Flask-compatible case-insensitive Config classt.
+ Flask-compatible case-insensitive Config class.
See :type:`flask.config.Config` for more info.
'''
def __init__(self, root, defaults=None):
+ self._warned = set()
if defaults:
defaults = self.gendict(defaults)
super(Config, self).__init__(root, defaults)
- @classmethod
- def genkey(cls, k):
+ def genkey(self, k):
'''
Key translation function.
@@ -27,10 +29,19 @@ class Config(flask.config.Config):
:returns: uppercase key
;rtype: str
'''
- return k.upper() if isinstance(k, basestring) else k
-
- @classmethod
- def gendict(cls, *args, **kwargs):
+ if isinstance(k, basestring):
+ if k not in self._warned and k != k.upper():
+ self._warned.add(k)
+ warnings.warn(
+ 'Config accessed with lowercase key '
+ '%r, lowercase config is deprecated.' % k,
+ DeprecationWarning,
+ 3
+ )
+ return k.upper()
+ return k
+
+ def gendict(self, *args, **kwargs):
'''
Pre-translated key dictionary constructor.
@@ -39,7 +50,7 @@ class Config(flask.config.Config):
:returns: dictionary with uppercase keys
:rtype: dict
'''
- gk = cls.genkey
+ gk = self.genkey
return dict((gk(k), v) for k, v in dict(*args, **kwargs).items())
def __getitem__(self, k):
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 9f3ebbd..e797d29 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -54,6 +54,18 @@ class SafeArgumentParser(argparse.ArgumentParser):
super(SafeArgumentParser, self).__init__(**kwargs)
+class HelpFormatter(argparse.RawTextHelpFormatter):
+ def __init__(self, prog, indent_increment=2, max_help_position=24,
+ width=None):
+ if width is None:
+ try:
+ width = get_terminal_size().columns - 2
+ except ValueError: # https://bugs.python.org/issue24966
+ pass
+ super(HelpFormatter, self).__init__(
+ prog, indent_increment, max_help_position, width)
+
+
def isexec(path):
'''
Check if given path points to an executable file.
@@ -173,8 +185,7 @@ def deprecated(func_or_text, environ=os.environ):
with warnings.catch_warnings():
if getdebug(environ):
warnings.simplefilter('always', DeprecationWarning)
- warnings.warn(message, category=DeprecationWarning,
- stacklevel=3)
+ warnings.warn(message, DeprecationWarning, 3)
return func(*args, **kwargs)
return new_func
return inner(func_or_text) if callable(func_or_text) else inner
diff --git a/browsepy/exceptions.py b/browsepy/exceptions.py
index 9f7a81f..ff766d9 100644
--- a/browsepy/exceptions.py
+++ b/browsepy/exceptions.py
@@ -4,7 +4,7 @@
class OutsideDirectoryBase(Exception):
'''
Exception raised when trying to access to a file outside path defined on
- `directory_base` config property.
+ `DIRECTORY_BASE` config property.
'''
pass
diff --git a/browsepy/file.py b/browsepy/file.py
index 1dea225..4edb48e 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -121,7 +121,7 @@ class Node(object):
@cached_property
def link(self):
'''
- Get last widget with place "entry-link".
+ Get last widget with place `entry-link`.
:returns: widget on entry-link (ideally a link one)
:rtype: namedtuple instance
@@ -136,18 +136,18 @@ class Node(object):
def can_remove(self):
'''
Get if current node can be removed based on app config's
- directory_remove.
+ `DIRECTORY_REMOVE`.
:returns: True if current node can be removed, False otherwise.
:rtype: bool
'''
- dirbase = self.app.config.get('directory_remove')
+ dirbase = self.app.config.get('DIRECTORY_REMOVE')
return bool(dirbase and check_under_base(self.path, dirbase))
@cached_property
def stats(self):
'''
- Get current stats object as returned by os.stat function.
+ Get current stats object as returned by `os.stat` function.
:returns: stats object
:rtype: posix.stat_result or nt.stat_result
@@ -168,12 +168,12 @@ class Node(object):
@cached_property
def parent(self):
'''
- Get parent node if available based on app config's directory_base.
+ Get parent node if available based on app config's `DIRECTORY_BASE`.
:returns: parent object if available
:rtype: Node instance or None
'''
- directory_base = self.app.config.get('directory_base', self.path)
+ directory_base = self.app.config.get('DIRECTORY_BASE', self.path)
if check_path(self.path, directory_base):
return None
parent = os.path.dirname(self.path) if self.path else None
@@ -182,7 +182,7 @@ class Node(object):
@cached_property
def ancestors(self):
'''
- Get list of ancestors until app config's directory_base is reached.
+ Get list of ancestors until app config's `DIRECTORY_BASE` is reached.
:returns: list of ancestors starting from nearest.
:rtype: list of Node objects
@@ -219,7 +219,7 @@ class Node(object):
'''
return abspath_to_urlpath(
self.path,
- self.app.config.get('directory_base', self.path),
+ self.app.config.get('DIRECTORY_BASE', self.path),
)
@property
@@ -309,7 +309,7 @@ class Node(object):
:rtype: File
'''
app = utils.solve_local(app or current_app)
- base = app.config.get('directory_base', path)
+ base = app.config.get('DIRECTORY_BASE', path)
path = urlpath_to_abspath(path, base)
if not cls.generic:
kls = cls
@@ -431,7 +431,7 @@ class File(Node):
try:
size, unit = fmt_size(
self.stats.st_size,
- self.app.config.get('use_binary_multiples', False)
+ self.app.config.get('USE_BINARY_MULTIPLES', False)
)
except OSError:
return None
@@ -586,24 +586,24 @@ class Directory(Node):
@cached_property
def can_download(self):
'''
- Get if path is downloadable (if app's `directory_downloadable` config
+ Get if path is downloadable (if app's `DIRECTORY_DOWNLOADABLE` config
property is True).
:returns: True if downloadable, False otherwise
:rtype: bool
'''
- return self.app.config.get('directory_downloadable', False)
+ return self.app.config.get('DIRECTORY_DOWNLOADABLE', False)
@cached_property
def can_upload(self):
'''
Get if a file can be uploaded to path (if directory path is under app's
- `directory_upload` config property).
+ `DIRECTORY_UPLOAD` config property).
:returns: True if a file can be upload to directory, False otherwise
:rtype: bool
'''
- dirbase = self.app.config.get('directory_upload', False)
+ dirbase = self.app.config.get('DIRECTORY_UPLOAD', False)
return dirbase and check_base(self.path, dirbase)
@cached_property
@@ -649,7 +649,7 @@ class Directory(Node):
'''
stream = self.stream_class(
self.path,
- self.app.config.get('directory_tar_buffsize', 10240),
+ self.app.config.get('DIRECTORY_TAR_BUFFSIZE', 10240),
self.plugin_manager.check_excluded,
)
return self.app.response_class(
diff --git a/browsepy/manager.py b/browsepy/manager.py
index 588e4c3..ba0ef2e 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -4,6 +4,7 @@ import re
import sys
import pkgutil
import argparse
+import functools
import warnings
import collections
@@ -14,7 +15,7 @@ from cookieman import CookieMan
from . import mimetype
from . import compat
-from .compat import deprecated, usedoc, SafeArgumentParser
+from .compat import deprecated, usedoc
from .utils import get_module
@@ -63,7 +64,7 @@ class PluginManagerBase(object):
'''
List of plugin namespaces taken from app config.
'''
- return self.app.config['plugin_namespaces'] if self.app else []
+ return self.app.config['PLUGIN_NAMESPACES'] if self.app else []
def __init__(self, app=None):
'''
@@ -96,10 +97,10 @@ class PluginManagerBase(object):
This method will make use of :meth:`clear` and :meth:`load_plugin`,
so all internal state will be cleared, and all plugins defined in
- :data:`self.app.config['plugin_modules']` will be loaded.
+ :data:`self.app.config['PLUGIN_MODULES']` will be loaded.
'''
self.clear()
- for plugin in self.app.config.get('plugin_modules', ()):
+ for plugin in self.app.config.get('PLUGIN_MODULES', ()):
self.load_plugin(plugin)
def clear(self):
@@ -114,11 +115,11 @@ class PluginManagerBase(object):
'''
for info in pkgutil.iter_modules():
if info.name.startswith(prefix):
- yield info.name, None
+ yield info.name
def _iter_submodules(self, prefix):
'''
- Iterate thru all submodules which full name contains given prefix.
+ Iterate thru all modules which full name contains given prefix.
'''
res = compat.res
parent = prefix.rsplit('.', 1)[0]
@@ -128,27 +129,40 @@ class PluginManagerBase(object):
name = '%s.%s' % (base, item)
if name.startswith(prefix) and \
not res.is_resource(base, item):
- yield name, item
+ yield name
break
except ImportError:
pass
- def iter_plugins(self):
+ def _iter_plugin_modules(self):
'''
- Iterate through all loadable plugins on default import locations
+ Iterate plugin modules, yielding both full qualified name and
+ short plugin name as tuple.
'''
+ nameset = set()
+ shortset = set()
+ filters = self.plugin_filters
for prefix in self.namespaces:
- names = (
- self._iter_submodules(prefix)
- if '.' in prefix else
- self._iter_modules(prefix)
- if prefix else
- ()
- )
- for name, short in names:
- module = get_module(name)
- if module and any(f(module) for f in self.plugin_filters):
+ for name in (self._iter_submodules(prefix)
+ if '.' in prefix else
+ self._iter_modules(prefix)
+ if prefix else
+ ()):
+ module = get_module(name) if name not in nameset else None
+ if module and any(f(module) for f in filters):
+ short = name[len(prefix):].lstrip('.').replace('_', '-')
+ if short in shortset or '.' in short or not short:
+ short = None
yield name, short
+ nameset.add(name)
+ shortset.add(short)
+
+ @cached_property
+ def available_plugins(self):
+ '''
+ Iterate through all loadable plugins on default import locations
+ '''
+ return list(self._iter_plugin_modules())
def import_plugin(self, plugin):
'''
@@ -269,7 +283,7 @@ class ExcludePluginManager(PluginManagerBase):
'''
Check if given path is excluded.
'''
- exclude_fnc = self.app.config.get('exclude_fnc')
+ exclude_fnc = self.app.config.get('EXCLUDE_FNC')
if exclude_fnc and exclude_fnc(path):
return True
for fnc in self._exclude_functions:
@@ -581,13 +595,30 @@ class ArgumentPluginManager(PluginManagerBase):
return manager._argparse_argkwargs
return ()
- @cached_property
def _default_argument_parser(self):
- parser = SafeArgumentParser()
+ parser = compat.SafeArgumentParser()
parser.add_argument('--plugin', action='append', default=[])
parser.add_argument('--help-all', action='store_true')
return parser
+ def _plugin_argument_parser(self, base=None):
+ plugins = self.available_plugins
+ parent = base or self._default_argument_parser
+ prop = functools.partial(getattr, parent)
+ epilog = prop('epilog') or ''
+ if plugins:
+ epilog += '\n\navailable plugins:\n%s' % '\n'.join(
+ ' %s, %s' % (short, name) if short else ' %s' % name
+ for name, short in plugins
+ )
+ return compat.SafeArgumentParser(
+ parents=[parent],
+ prog=prop('prog', self.app.config['APPLICATION_NAME']),
+ description=prop('description'),
+ formatter_class=prop('formatter_class', compat.HelpFormatter),
+ epilog=epilog.strip(),
+ )
+
def load_arguments(self, argv, base=None):
'''
Process given argument list based on registered arguments and given
@@ -607,9 +638,7 @@ class ArgumentPluginManager(PluginManagerBase):
given by :meth:`argparse.ArgumentParser.parse_args`.
:rtype: argparse.Namespace
'''
- parser = SafeArgumentParser(
- parents=[base or self._default_argument_parser],
- )
+ parser = self._plugin_argument_parser(base)
options, _ = parser.parse_known_args(argv)
plugins = [
@@ -621,7 +650,7 @@ class ArgumentPluginManager(PluginManagerBase):
if options.help_all:
plugins.extend(
short if short else name
- for name, short in self.iter_plugins()
+ for name, short in self.available_plugins
if not (name in plugins or short in plugins)
)
@@ -797,7 +826,7 @@ class MimetypeActionPluginManager(WidgetPluginManager, MimetypePluginManager):
if isinstance(place or widget, self._widget.WidgetBase):
warnings.warn(
'Deprecated use of register_widget',
- category=DeprecationWarning
+ DeprecationWarning
)
widget = place or widget
props = self._widget_props(widget)
@@ -813,7 +842,7 @@ class MimetypeActionPluginManager(WidgetPluginManager, MimetypePluginManager):
place in self._deprecated_places:
warnings.warn(
'Deprecated use of get_widgets',
- category=DeprecationWarning
+ DeprecationWarning
)
place = file or place
return [
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index ddc8c72..568914a 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -2,9 +2,10 @@
import os
import os.path
+import functools
from flask import Blueprint, render_template, request, redirect, url_for, \
- session
+ session, current_app
from werkzeug.exceptions import NotFound
from browsepy import get_cookie_browse_sorting, browse_sortkey_reverse
@@ -201,6 +202,27 @@ def shrink_session(data, last):
return data
+def detect_upload(directory):
+ return directory.is_directory and directory.can_upload
+
+
+def detect_clipboard(directory):
+ return directory.is_directory and session.get('clipboard:mode')
+
+
+def detect_selection(directory):
+ return directory.is_directory and \
+ current_app.config.get('DIRECTORY_UPLOAD')
+
+
+def excluded_clipboard(manager, path):
+ if session.get('clipboard:mode') == 'cut':
+ base = manager.app.config['DIRECTORY_BASE']
+ clipboard = session.get('clipboard:items', ())
+ return abspath_to_urlpath(path, base) in clipboard
+ return False
+
+
def register_plugin(manager):
'''
Register blueprints and actions using given plugin manager.
@@ -208,19 +230,9 @@ def register_plugin(manager):
:param manager: plugin manager
:type manager: browsepy.manager.PluginManager
'''
- def detect_upload(directory):
- return directory.is_directory and directory.can_upload
-
- def detect_clipboard(directory):
- return directory.is_directory and session.get('clipboard:mode')
-
- def excluded_clipboard(path):
- if session.get('clipboard:mode') == 'cut':
- base = manager.app.config['directory_base']
- clipboard = session.get('clipboard:items', ())
- return abspath_to_urlpath(path, base) in clipboard
-
- manager.register_exclude_function(excluded_clipboard)
+ manager.register_exclude_function(
+ functools.partial(excluded_clipboard, manager)
+ )
manager.register_blueprint(actions)
manager.register_widget(
place='styles',
@@ -240,7 +252,7 @@ def register_plugin(manager):
place='header',
type='button',
endpoint='file_actions.selection',
- filter=lambda directory: directory.is_directory,
+ filter=detect_selection,
text='Selection...',
)
manager.register_widget(
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index 30a230b..9f22ce6 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -70,8 +70,8 @@ class TestRegistration(unittest.TestCase):
self.base = 'c:\\base' if os.name == 'nt' else '/base'
self.app = flask.Flask(self.__class__.__name__)
self.app.config.update(
- directory_base=self.base,
- exclude_fnc=None,
+ DIRECTORY_BASE=self.base,
+ EXCLUDE_FNC=None,
)
def tearDown(self):
@@ -79,7 +79,7 @@ class TestRegistration(unittest.TestCase):
def test_register_plugin(self):
self.app.config.update(self.browsepy_module.app.config)
- self.app.config['plugin_namespaces'] = ('browsepy.plugin',)
+ self.app.config['PLUGIN_NAMESPACES'] = ('browsepy.plugin',)
manager = self.manager_module.PluginManager(self.app)
manager.load_plugin('file-actions')
self.assertIn(
@@ -89,8 +89,8 @@ class TestRegistration(unittest.TestCase):
def test_reload(self):
self.app.config.update(
- plugin_modules=[],
- plugin_namespaces=[]
+ PLUGIN_MODULES=[],
+ PLUGIN_NAMESPACES=[]
)
manager = self.manager_module.PluginManager(self.app)
self.assertNotIn(
@@ -98,8 +98,8 @@ class TestRegistration(unittest.TestCase):
self.app.blueprints.values()
)
self.app.config.update(
- plugin_modules=['file-actions'],
- plugin_namespaces=['browsepy.plugin']
+ PLUGIN_MODULES=['file-actions'],
+ PLUGIN_NAMESPACES=['browsepy.plugin']
)
manager.reload()
self.assertIn(
@@ -118,12 +118,12 @@ class TestIntegration(unittest.TestCase):
self.app = self.browsepy_module.app
self.app.config.update(
SECRET_KEY='secret',
- directory_base=self.base,
- directory_start=self.base,
- directory_upload=None,
- exclude_fnc=None,
- plugin_modules=['file-actions'],
- plugin_namespaces=[
+ DIRECTORY_BASE=self.base,
+ DIRECTORY_START=self.base,
+ DIRECTORY_UPLOAD=None,
+ EXCLUDE_FNC=None,
+ PLUGIN_MODULES=['file-actions'],
+ PLUGIN_NAMESPACES=[
'browsepy.plugin'
]
)
@@ -132,17 +132,18 @@ class TestIntegration(unittest.TestCase):
def tearDown(self):
shutil.rmtree(self.base)
- self.app.config['plugin_modules'] = []
+ self.app.config['PLUGIN_MODULES'] = []
self.manager.clear()
utils.clear_flask_context()
def test_detection(self):
+ url = functools.partial(flask.url_for, path='')
+
with self.app.test_client() as client:
+ self.app.config['DIRECTORY_UPLOAD'] = self.base
response = client.get('/')
self.assertEqual(response.status_code, 200)
- url = functools.partial(flask.url_for, path='')
-
page = Page(response.data)
with self.app.app_context():
self.assertIn(
@@ -159,6 +160,7 @@ class TestIntegration(unittest.TestCase):
session['clipboard:mode'] = 'copy'
session['clipboard:items'] = ['whatever']
+ self.app.config['DIRECTORY_UPLOAD'] = 'whatever'
response = client.get('/')
self.assertEqual(response.status_code, 200)
@@ -177,7 +179,7 @@ class TestIntegration(unittest.TestCase):
page.widgets
)
- self.app.config['directory_upload'] = self.base
+ self.app.config['DIRECTORY_UPLOAD'] = self.base
response = client.get('/')
self.assertEqual(response.status_code, 200)
@@ -231,11 +233,11 @@ class TestAction(unittest.TestCase):
self.app.register_blueprint(self.module.actions)
self.app.config.update(
SECRET_KEY='secret',
- directory_base=self.base,
- directory_upload=self.base,
- directory_remove=self.base,
- exclude_fnc=None,
- use_binary_multiples=True,
+ DIRECTORY_BASE=self.base,
+ DIRECTORY_UPLOAD=self.base,
+ DIRECTORY_REMOVE=self.base,
+ EXCLUDE_FNC=None,
+ USE_BINARY_MULTIPLES=True,
)
self.app.add_url_rule(
'/browse/',
@@ -406,7 +408,7 @@ class TestAction(unittest.TestCase):
response = client.get('/file-actions/clipboard/paste/target')
self.assertEqual(response.status_code, 400)
- self.app.config['exclude_fnc'] = lambda n: n.endswith('whatever')
+ self.app.config['EXCLUDE_FNC'] = lambda n: n.endswith('whatever')
with client.session_transaction() as session:
session['clipboard:mode'] = 'cut'
diff --git a/browsepy/plugin/player/playable.py b/browsepy/plugin/player/playable.py
index 7e1b278..c354dcd 100644
--- a/browsepy/plugin/player/playable.py
+++ b/browsepy/plugin/player/playable.py
@@ -36,7 +36,7 @@ class PLSFileParser(object):
def __init__(self, path):
with warnings.catch_warnings():
# We already know about SafeConfigParser deprecation!
- warnings.filterwarnings('ignore', category=DeprecationWarning)
+ warnings.simplefilter('ignore', category=DeprecationWarning)
self._parser = self.parser_class()
self._parser.read(path)
@@ -133,7 +133,7 @@ class PlayListFile(PlayableBase):
drive = os.path.splitdrive(self.path)[0]
if drive and not os.path.splitdrive(path)[0]:
path = drive + path
- if check_under_base(path, self.app.config['directory_base']):
+ if check_under_base(path, self.app.config['DIRECTORY_BASE']):
return path
return None
diff --git a/browsepy/plugin/player/tests.py b/browsepy/plugin/player/tests.py
index bcf8e4f..a2c7003 100644
--- a/browsepy/plugin/player/tests.py
+++ b/browsepy/plugin/player/tests.py
@@ -86,7 +86,7 @@ class TestPlayerBase(unittest.TestCase):
self.base = 'c:\\base' if os.name == 'nt' else '/base'
self.app = flask.Flask(self.__class__.__name__)
self.app.config.update(
- directory_base=self.base,
+ DIRECTORY_BASE=self.base,
SERVER_NAME='localhost',
)
self.manager = ManagerMock()
@@ -149,14 +149,14 @@ class TestIntegration(TestIntegrationBase):
def test_register_plugin(self):
self.app.config.update(self.browsepy_module.app.config)
- self.app.config['plugin_namespaces'] = ('browsepy.plugin',)
+ self.app.config['PLUGIN_NAMESPACES'] = ('browsepy.plugin',)
manager = self.manager_module.PluginManager(self.app)
manager.load_plugin('player')
self.assertIn(self.player_module.player, self.app.blueprints.values())
def test_register_arguments(self):
self.app.config.update(self.browsepy_module.app.config)
- self.app.config['plugin_namespaces'] = ('browsepy.plugin',)
+ self.app.config['PLUGIN_NAMESPACES'] = ('browsepy.plugin',)
manager = self.manager_module.ArgumentPluginManager(self.app)
manager.load_arguments(self.non_directory_args)
@@ -166,8 +166,8 @@ class TestIntegration(TestIntegrationBase):
def test_reload(self):
self.app.config.update(
- plugin_modules=['player'],
- plugin_namespaces=['browsepy.plugin']
+ PLUGIN_MODULES=['player'],
+ PLUGIN_NAMESPACES=['browsepy.plugin']
)
manager = self.manager_module.PluginManager(self.app)
manager.load_arguments(self.non_directory_args)
@@ -323,7 +323,7 @@ class TestBlueprint(TestPlayerBase):
super(TestBlueprint, self).setUp()
app = self.app
app.template_folder = utils.ppath('templates')
- app.config['directory_base'] = tempfile.mkdtemp()
+ app.config['DIRECTORY_BASE'] = tempfile.mkdtemp()
app.register_blueprint(self.module.player)
@app.route("/browse", defaults={"path": ""}, endpoint='browse')
@@ -343,13 +343,13 @@ class TestBlueprint(TestPlayerBase):
return response
def file(self, path, data=''):
- apath = p(self.app.config['directory_base'], path)
+ apath = p(self.app.config['DIRECTORY_BASE'], path)
with open(apath, 'w') as f:
f.write(data)
return apath
def directory(self, path):
- apath = p(self.app.config['directory_base'], path)
+ apath = p(self.app.config['DIRECTORY_BASE'], path)
os.mkdir(apath)
return apath
diff --git a/browsepy/tests/deprecated/test_plugins.py b/browsepy/tests/deprecated/test_plugins.py
index c6e07f7..492dfd0 100644
--- a/browsepy/tests/deprecated/test_plugins.py
+++ b/browsepy/tests/deprecated/test_plugins.py
@@ -71,12 +71,12 @@ class TestPlugins(unittest.TestCase):
def setUp(self):
self.app = self.app_module.app
self.manager = self.manager_module.PluginManager(self.app)
- self.original_namespaces = self.app.config['plugin_namespaces']
+ self.original_namespaces = self.app.config['PLUGIN_NAMESPACES']
self.plugin_namespace, self.plugin_name = __name__.rsplit('.', 1)
- self.app.config['plugin_namespaces'] = (self.plugin_namespace,)
+ self.app.config['PLUGIN_NAMESPACES'] = (self.plugin_namespace,)
def tearDown(self):
- self.app.config['plugin_namespaces'] = self.original_namespaces
+ self.app.config['PLUGIN_NAMESPACES'] = self.original_namespaces
self.manager.clear()
def test_manager(self):
@@ -132,7 +132,7 @@ class TestPlayerBase(unittest.TestCase):
def setUp(self):
self.app = flask.Flask(self.__class__.__name__)
- self.app.config['directory_remove'] = None
+ self.app.config['DIRECTORY_REMOVE'] = None
self.app.config['SERVER_NAME'] = self.hostname
self.app.config['PREFERRED_URL_SCHEME'] = self.scheme
self.manager = ManagerMock()
@@ -171,7 +171,7 @@ class TestIntegration(TestIntegrationBase):
self.app.config.update(
SERVER_NAME=self.hostname,
PREFERRED_URL_SCHEME=self.scheme,
- plugin_namespaces=('browsepy.tests.deprecated.plugin',)
+ PLUGIN_NAMESPACES=('browsepy.tests.deprecated.plugin',)
)
manager = self.manager_module.PluginManager(self.app)
manager.load_plugin('player')
diff --git a/browsepy/tests/test_app.py b/browsepy/tests/test_app.py
index 83cb7f3..cf30aee 100644
--- a/browsepy/tests/test_app.py
+++ b/browsepy/tests/test_app.py
@@ -2,6 +2,7 @@ import os
import os.path
import unittest
import tempfile
+import warnings
import browsepy
import browsepy.appconfig
@@ -11,17 +12,25 @@ class TestApp(unittest.TestCase):
module = browsepy
app = browsepy.app
+ def setUp(self):
+ self.app.config._warned.clear()
+
def test_config(self):
- try:
- with tempfile.NamedTemporaryFile(delete=False) as f:
- f.write(b'DIRECTORY_DOWNLOADABLE = False\n')
- name = f.name
- os.environ['BROWSEPY_TEST_SETTINGS'] = name
- self.app.config['directory_downloadable'] = True
+ with tempfile.NamedTemporaryFile() as f:
+ f.write(b'DIRECTORY_DOWNLOADABLE = False\n')
+ f.flush()
+
+ os.environ['BROWSEPY_TEST_SETTINGS'] = f.name
+ with warnings.catch_warnings(record=True) as warns:
+ warnings.simplefilter('always')
+ self.app.config['directory_downloadable'] = True
+ self.assertTrue(warns)
+
self.app.config.from_envvar('BROWSEPY_TEST_SETTINGS')
- self.assertFalse(self.app.config['directory_downloadable'])
- finally:
- os.remove(name)
+ with warnings.catch_warnings(record=True) as warns:
+ warnings.simplefilter('always')
+ self.assertFalse(self.app.config['directory_downloadable'])
+ self.assertFalse(warns)
class TestConfig(unittest.TestCase):
diff --git a/browsepy/tests/test_file.py b/browsepy/tests/test_file.py
index 7acdde3..3bd1802 100644
--- a/browsepy/tests/test_file.py
+++ b/browsepy/tests/test_file.py
@@ -130,15 +130,15 @@ class TestFile(unittest.TestCase):
f.write(b',\n' * 512)
f = self.module.File(test_file, app=self.app)
- default = self.app.config['use_binary_multiples']
+ default = self.app.config['USE_BINARY_MULTIPLES']
- self.app.config['use_binary_multiples'] = True
+ self.app.config['USE_BINARY_MULTIPLES'] = True
self.assertEqual(f.size, '1.00 KiB')
- self.app.config['use_binary_multiples'] = False
+ self.app.config['USE_BINARY_MULTIPLES'] = False
self.assertEqual(f.size, '1.02 KB')
- self.app.config['use_binary_multiples'] = default
+ self.app.config['USE_BINARY_MULTIPLES'] = default
def test_stats(self):
virtual_file = os.path.join(self.workbench, 'file.txt')
diff --git a/browsepy/tests/test_module.py b/browsepy/tests/test_module.py
index b162e67..2a7d962 100644
--- a/browsepy/tests/test_module.py
+++ b/browsepy/tests/test_module.py
@@ -195,11 +195,11 @@ class TestApp(unittest.TestCase):
or path.startswith(self.exclude + os.sep)
self.app.config.update(
- directory_base=self.base,
- directory_start=self.start,
- directory_remove=self.remove,
- directory_upload=self.upload,
- exclude_fnc=exclude_fnc,
+ DIRECTORY_BASE=self.base,
+ DIRECTORY_START=self.start,
+ DIRECTORY_REMOVE=self.remove,
+ DIRECTORY_UPLOAD=self.upload,
+ EXCLUDE_FNC=exclude_fnc,
SERVER_NAME='localhost',
)
@@ -284,14 +284,14 @@ class TestApp(unittest.TestCase):
self.assertEqual(page.path, '%s/start' % os.path.basename(self.base))
start = os.path.abspath(os.path.join(self.base, '..'))
- self.app.config['directory_start'] = start
+ self.app.config['DIRECTORY_START'] = start
self.assertRaises(
Page404Exception,
self.get, 'index'
)
- self.app.config['directory_start'] = self.start
+ self.app.config['DIRECTORY_START'] = self.start
def test_browse(self):
basename = os.path.basename(self.base)
@@ -334,10 +334,10 @@ class TestApp(unittest.TestCase):
self.get, 'browse', path='exclude'
)
- self.app.config['directory_downloadable'] = True
+ self.app.config['DIRECTORY_DOWNLOADABLE'] = True
page = self.get('browse')
self.assertTrue(page.tarfile)
- self.app.config['directory_downloadable'] = False
+ self.app.config['DIRECTORY_DOWNLOADABLE'] = False
page = self.get('browse')
self.assertFalse(page.tarfile)
@@ -382,14 +382,14 @@ class TestApp(unittest.TestCase):
self.post, 'remove', path='start/testfile.txt'
)
- self.app.config['directory_remove'] = None
+ self.app.config['DIRECTORY_REMOVE'] = None
self.assertRaises(
Page404Exception,
self.get, 'remove', path='remove/testfile.txt'
)
- self.app.config['directory_remove'] = self.remove
+ self.app.config['DIRECTORY_REMOVE'] = self.remove
self.assertRaises(
Page404Exception,
@@ -431,13 +431,13 @@ class TestApp(unittest.TestCase):
binfile = os.path.join(self.start, 'testfile.bin')
excfile = os.path.join(self.start, 'testfile.exc')
bindata = bytes(range(256))
- exclude = self.app.config['exclude_fnc']
+ exclude = self.app.config['EXCLUDE_FNC']
for path in (binfile, excfile):
with open(path, 'wb') as f:
f.write(bindata)
- self.app.config['exclude_fnc'] = None
+ self.app.config['EXCLUDE_FNC'] = None
response = self.get('download_directory', path='start')
self.assertEqual(response.filename, 'start.tgz')
@@ -448,7 +448,7 @@ class TestApp(unittest.TestCase):
['testfile.%s' % x1 for x1 in ('bin', 'exc', 'txt')]
)
- self.app.config['exclude_fnc'] = lambda p: p.endswith('.exc')
+ self.app.config['EXCLUDE_FNC'] = lambda p: p.endswith('.exc')
response = self.get('download_directory', path='start')
self.assertEqual(response.filename, 'start.tgz')
@@ -459,7 +459,7 @@ class TestApp(unittest.TestCase):
['testfile.%s' % x2 for x2 in ('bin', 'txt')]
)
- self.app.config['exclude_fnc'] = exclude
+ self.app.config['EXCLUDE_FNC'] = exclude
self.assertRaises(
Page404Exception,
diff --git a/browsepy/tests/test_plugins.py b/browsepy/tests/test_plugins.py
index 6170cd6..1899ce8 100644
--- a/browsepy/tests/test_plugins.py
+++ b/browsepy/tests/test_plugins.py
@@ -1,4 +1,3 @@
-
import unittest
import flask
@@ -47,13 +46,13 @@ class TestPlugins(unittest.TestCase):
def setUp(self):
self.app = self.app_module.app
- self.original_namespaces = self.app.config['plugin_namespaces']
+ self.original_namespaces = self.app.config['PLUGIN_NAMESPACES']
self.plugin_namespace, self.plugin_name = __name__.rsplit('.', 1)
- self.app.config['plugin_namespaces'] = (self.plugin_namespace,)
+ self.app.config['PLUGIN_NAMESPACES'] = (self.plugin_namespace,)
self.manager = self.manager_module.PluginManager(self.app)
def tearDown(self):
- self.app.config['plugin_namespaces'] = self.original_namespaces
+ self.app.config['PLUGIN_NAMESPACES'] = self.original_namespaces
self.manager.clear()
utils.clear_flask_context()
@@ -88,14 +87,14 @@ class TestPlugins(unittest.TestCase):
)
def test_list(self):
- self.app.config['plugin_namespaces'] = self.original_namespaces
- names = list(self.manager.iter_plugins())
+ self.app.config['PLUGIN_NAMESPACES'] = self.original_namespaces
+ names = self.manager.available_plugins
self.assertIn(('browsepy.plugin.player', 'player'), names)
- self.assertIn(('browsepy.plugin.file_actions', 'file_actions'), names)
+ self.assertIn(('browsepy.plugin.file_actions', 'file-actions'), names)
def test_namespace_prefix(self):
self.assertTrue(self.manager.import_plugin(self.plugin_name))
- self.app.config['plugin_namespaces'] = (
+ self.app.config['PLUGIN_NAMESPACES'] = (
self.plugin_namespace + '.test_',
)
self.assertTrue(self.manager.import_plugin('module'))
diff --git a/browsepy/transform/glob.py b/browsepy/transform/glob.py
index 64783af..428fb5c 100644
--- a/browsepy/transform/glob.py
+++ b/browsepy/transform/glob.py
@@ -146,15 +146,18 @@ class GlobTransform(StateMachine):
def transform_posix_collating_symbol(self, data, mark, next):
warnings.warn(
'Posix collating symbols (like %s%s) are not supported.'
- % (data, mark))
+ % (data, mark),
+ RuntimeWarning
+ )
return None
def transform_posix_character_class(self, data, mark, next):
name = data[len(self.start):]
if name not in self.character_classes:
warnings.warn(
- 'Posix character class %s is not supported.'
- % name)
+ 'Posix character class %s is not supported.' % name,
+ RuntimeWarning
+ )
return None
return ''.join(
chr(start)
@@ -166,7 +169,9 @@ class GlobTransform(StateMachine):
def transform_posix_equivalence_class(self, data, mark, next):
warnings.warn(
'Posix equivalence class expresions (like %s%s) are not supported.'
- % (data, mark))
+ % (data, mark),
+ RuntimeWarning
+ )
return None
def transform_wildcard(self, data, mark, next):
diff --git a/doc/integrations.rst b/doc/integrations.rst
index 1d654f2..8abd594 100644
--- a/doc/integrations.rst
+++ b/doc/integrations.rst
@@ -10,25 +10,25 @@ is supported. Also, browsepy's public API could be easily reused.
Browsepy app config (available at :attr:`browsepy.app.config`) exposes the
following configuration options.
-* **directory_base**: anything under this directory will be served,
+* **DIRECTORY_BASE**: anything under this directory will be served,
defaults to current path.
-* **directory_start**: directory will be served when accessing root URL
-* **directory_remove**: file removing will be available under this path,
+* **DIRECTORY_START**: directory will be served when accessing root URL
+* **DIRECTORY_REMOVE**: file removing will be available under this path,
defaults to **None**.
-* **directory_upload**: file upload will be available under this path,
+* **DIRECTORY_UPLOAD**: file upload will be available under this path,
defaults to **None**.
-* **directory_tar_buffsize**, directory tar streaming buffer size,
+* **DIRECTORY_TAR_BUFFSIZE**, directory tar streaming buffer size,
defaults to **262144** and must be multiple of 512.
-* **directory_downloadable** whether enable directory download or not,
+* **DIRECTORY_DOWNLOADABLE** whether enable directory download or not,
defaults to **True**.
-* **use_binary_multiples** whether use binary units (bi-bytes, like KiB)
+* **USE_BINARY_MULTIPLES** whether use binary units (bi-bytes, like KiB)
instead of common ones (bytes, like KB), defaults to **True**.
-* **plugin_modules** list of module names (absolute or relative to
+* **PLUGIN_MODULES** list of module names (absolute or relative to
plugin_namespaces) will be loaded.
-* **plugin_namespaces** prefixes for module names listed at plugin_modules
- where relative plugin_modules are searched.
+* **PLUGIN_NAMESPACES** prefixes for module names listed at PLUGIN_MODULES
+ where relative PLUGIN_MODULES are searched.
-Please note: After editing `plugin_modules` value, plugin manager (available
+Please note: After editing `PLUGIN_MODULES` value, plugin manager (available
at module :data:`browsepy.plugin_manager` and
:data:`browsepy.app.extensions['plugin_manager']`) should be reloaded using
the :meth:`browsepy.plugin_manager.reload` instance method of :meth:`browsepy.manager.PluginManager.reload` for browsepy's plugin
@@ -76,11 +76,11 @@ along with a root wsgi application.
def setup_browsepy():
browsepy.app.config.update(
APPLICATION_ROOT='/browse',
- directory_base=cfg.media_path,
- directory_start=cfg.media_path,
- directory_remove=cfg.media_path,
- directory_upload=cfg.media_path,
- plugin_modules=['player'],
+ DIRECTORY_BASE=cfg.media_path,
+ DIRECTORY_START=cfg.media_path,
+ DIRECTORY_REMOVE=cfg.media_path,
+ DIRECTORY_UPLOAD=cfg.media_path,
+ PLUGIN_MODULES=['player'],
)
browsepy.plugin_manager.load_arguments([
'--plugin=player',
@@ -177,11 +177,11 @@ server provided by `cherrymusic `_.
}
browsepy.config.update(
APPLICATION_ROOT='/browse',
- directory_base=media_path,
- directory_start=media_path,
- directory_remove=media_path,
- directory_upload=media_path,
- plugin_modules=['player'],
+ DIRECTORY_BASE=media_path,
+ DIRECTORY_START=media_path,
+ DIRECTORY_REMOVE=media_path,
+ DIRECTORY_UPLOAD=media_path,
+ PLUGIN_MODULES=['player'],
)
plugin_manager.reload()
--
GitLab
From 8861b00d5eba83129b47d200401ce4df00b64ddb Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 14 Apr 2019 09:59:42 +0200
Subject: [PATCH 061/171] raise coverage, reduce complexity
---
browsepy/compat.py | 28 +++++---
browsepy/exceptions.py | 16 +++++
browsepy/file.py | 2 +-
browsepy/manager.py | 90 +++++++++---------------
browsepy/plugin/file_actions/__init__.py | 2 +-
browsepy/plugin/file_actions/tests.py | 12 +++-
browsepy/tests/test_main.py | 12 ++--
browsepy/tests/test_plugins.py | 65 ++++++++++++++++-
browsepy/tests/test_utils.py | 53 +++++++++++++-
browsepy/utils.py | 35 ++++++---
10 files changed, 228 insertions(+), 87 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index e797d29..8713fcb 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -6,6 +6,7 @@ import sys
import abc
import itertools
import functools
+import contextlib
import warnings
import posixpath
import ntpath
@@ -354,16 +355,26 @@ def re_escape(pattern, chars=frozenset("()[]{}?*+|^$\\.-#")):
)
-if PY_LEGACY:
- class Iterator(BaseIterator):
- def next(self):
- '''
- Call :method:`__next__` for compatibility.
+@contextlib.contextmanager
+def redirect_stderr(f):
+ old = sys.stderr
+ sys.stderr = f
+ yield
+ if sys.stderr is f:
+ sys.stderr = old
+
+
+class Iterator(BaseIterator):
+ def next(self):
+ '''
+ Call :method:`__next__` for compatibility.
- :returns: see :method:`__next__`
- '''
- return self.__next__()
+ :returns: see :method:`__next__`
+ '''
+ return self.__next__()
+
+if PY_LEGACY:
class FileNotFoundError(BaseException):
__metaclass__ = abc.ABCMeta
@@ -378,7 +389,6 @@ if PY_LEGACY:
chr = unichr # noqa
bytes = str # noqa
else:
- Iterator = BaseIterator
FileNotFoundError = FileNotFoundError
range = range
filter = filter
diff --git a/browsepy/exceptions.py b/browsepy/exceptions.py
index ff766d9..2a49f74 100644
--- a/browsepy/exceptions.py
+++ b/browsepy/exceptions.py
@@ -71,3 +71,19 @@ class FilenameTooLongError(InvalidFilenameError):
self.limit = limit
super(FilenameTooLongError, self).__init__(
message, path=path, filename=filename)
+
+
+class PluginNotFoundError(ImportError):
+ pass
+
+
+class WidgetException(Exception):
+ pass
+
+
+class WidgetParameterException(WidgetException):
+ pass
+
+
+class InvalidArgumentError(ValueError):
+ pass
diff --git a/browsepy/file.py b/browsepy/file.py
index 4edb48e..0bcb9b8 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -860,7 +860,7 @@ def generic_filename(path):
return path
-def clean_restricted_chars(path, restricted_chars=restricted_chars):
+def clean_restricted_chars(path, restricted_chars=current_restricted_chars):
'''
Get path without restricted characters.
diff --git a/browsepy/manager.py b/browsepy/manager.py
index ba0ef2e..78cbc8e 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -1,7 +1,6 @@
# -*- coding: UTF-8 -*-
import re
-import sys
import pkgutil
import argparse
import functools
@@ -16,48 +15,16 @@ from cookieman import CookieMan
from . import mimetype
from . import compat
from .compat import deprecated, usedoc
-from .utils import get_module
-
-
-def defaultsnamedtuple(name, fields, defaults=None):
- '''
- Generate namedtuple with default values.
-
- :param name: name
- :param fields: iterable with field names
- :param defaults: iterable or mapping with field defaults
- :returns: defaultdict with given fields and given defaults
- :rtype: collections.defaultdict
- '''
- nt = collections.namedtuple(name, fields)
- nt.__new__.__defaults__ = (None,) * len(nt._fields)
- if isinstance(defaults, collections.Mapping):
- nt.__new__.__defaults__ = tuple(nt(**defaults))
- elif defaults:
- nt.__new__.__defaults__ = tuple(nt(*defaults))
- return nt
-
-
-class PluginNotFoundError(ImportError):
- pass
-
-
-class WidgetException(Exception):
- pass
-
-
-class WidgetParameterException(WidgetException):
- pass
-
-
-class InvalidArgumentError(ValueError):
- pass
+from .utils import get_module, defaultsnamedtuple
+from .exceptions import PluginNotFoundError, InvalidArgumentError, \
+ WidgetParameterException
class PluginManagerBase(object):
'''
Base plugin manager for plugin module loading and Flask extension logic.
'''
+ _pyfile_extensions = ('.py', '.pyc', '.pyd', '.pyo')
@property
def namespaces(self):
@@ -117,6 +84,19 @@ class PluginManagerBase(object):
if info.name.startswith(prefix):
yield info.name
+ def _content_import_name(self, module, item, prefix):
+ '''
+ Get importable module contnt import name..
+ '''
+ res = compat.res
+ name = '%s.%s' % (module, item)
+ if name.startswith(prefix):
+ for ext in self._pyfile_extensions:
+ if name.endswith(ext):
+ return name[:-len(ext)]
+ if not res.is_resource(module, item):
+ return name
+
def _iter_submodules(self, prefix):
'''
Iterate thru all modules which full name contains given prefix.
@@ -126,11 +106,9 @@ class PluginManagerBase(object):
for base in (prefix, parent):
try:
for item in res.contents(base):
- name = '%s.%s' % (base, item)
- if name.startswith(prefix) and \
- not res.is_resource(base, item):
- yield name
- break
+ content = self._content_import_name(base, item, prefix)
+ if content:
+ yield content
except ImportError:
pass
@@ -174,21 +152,18 @@ class PluginManagerBase(object):
'''
plugin = plugin.replace('-', '_')
names = [
- '%s%s%s' % (namespace, '' if namespace[-1] == '_' else '.', plugin)
- if namespace else
- plugin
- for namespace in self.namespaces
+ name
+ for ns in self.namespaces
+ for name in (
+ '%s%s' % (ns, plugin),
+ '%s.%s' % (ns.rstrip('.'), plugin),
+ )
]
-
- for name in names:
- if name in sys.modules:
- return sys.modules[name]
-
+ names = sorted(frozenset(names), key=names.index)
for name in names:
module = get_module(name)
if module:
return module
-
raise PluginNotFoundError(
'No plugin module %r found, tried %r' % (plugin, names),
plugin, names)
@@ -399,18 +374,19 @@ class WidgetPluginManager(PluginManagerBase):
'''
for filter, dynamic, cwidget in self._widgets:
try:
- if file and filter and not filter(file):
+ if (
+ (file and filter and not filter(file)) or
+ (place and place != cwidget.place)
+ ):
continue
except BaseException as e:
- # Exception is handled as this method execution is deffered,
- # making hard to debug for plugin developers.
+ # Exception is catch as this execution is deferred,
+ # making debugging harder for plugin developers.
warnings.warn(
'Plugin action filtering failed with error: %s' % e,
RuntimeWarning
)
continue
- if place and place != cwidget.place:
- continue
if file and dynamic:
cwidget = self._resolve_widget(file, cwidget)
yield cwidget
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index 568914a..ffb4cf4 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -65,7 +65,7 @@ def create_directory(path):
try:
os.mkdir(os.path.join(directory.path, basename))
- except OSError as e:
+ except BaseException as e:
raise DirectoryCreationError.from_exception(
e,
path=directory.path,
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index 9f22ce6..668051a 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -116,6 +116,7 @@ class TestIntegration(unittest.TestCase):
def setUp(self):
self.base = tempfile.mkdtemp()
self.app = self.browsepy_module.app
+ self.original_config = dict(self.app.config)
self.app.config.update(
SECRET_KEY='secret',
DIRECTORY_BASE=self.base,
@@ -132,7 +133,8 @@ class TestIntegration(unittest.TestCase):
def tearDown(self):
shutil.rmtree(self.base)
- self.app.config['PLUGIN_MODULES'] = []
+ self.app.config.clear()
+ self.app.config.update(self.original_config)
self.manager.clear()
utils.clear_flask_context()
@@ -308,6 +310,7 @@ class TestAction(unittest.TestCase):
'name': 'asdf',
})
self.assertEqual(response.status_code, 404)
+
response = client.post(
'/file-actions/create/directory',
data={
@@ -315,6 +318,13 @@ class TestAction(unittest.TestCase):
})
self.assertEqual(response.status_code, 400)
+ response = client.post(
+ '/file-actions/create/directory',
+ data={
+ 'name': '\0',
+ })
+ self.assertEqual(response.status_code, 400)
+
def test_selection(self):
self.touch('a')
self.touch('b')
diff --git a/browsepy/tests/test_main.py b/browsepy/tests/test_main.py
index 0c20b90..cb88d3c 100644
--- a/browsepy/tests/test_main.py
+++ b/browsepy/tests/test_main.py
@@ -3,12 +3,14 @@ import os
import os.path
import tempfile
import shutil
-import contextlib
-import browsepy.__main__
+
+import browsepy
+import browsepy.__main__ as main
+import browsepy.compat as compat
class TestMain(unittest.TestCase):
- module = browsepy.__main__
+ module = main
def setUp(self):
self.app = browsepy.app
@@ -82,7 +84,7 @@ class TestMain(unittest.TestCase):
self.assertListEqual(result.plugin, [])
with open(os.devnull, 'w') as f:
- with contextlib.redirect_stderr(f):
+ with compat.redirect_stderr(f):
self.assertRaises(
SystemExit,
self.parser.parse_args,
@@ -99,7 +101,7 @@ class TestMain(unittest.TestCase):
result = self.parser.parse_args([
'--exclude', '/.*',
'--exclude-from', self.exclude_file,
- ])
+ ])
extra = self.module.collect_exclude_patterns(result.exclude_from)
self.assertListEqual(extra, ['.ignore'])
match = self.module.create_exclude_fnc(
diff --git a/browsepy/tests/test_plugins.py b/browsepy/tests/test_plugins.py
index 1899ce8..0a9c1c9 100644
--- a/browsepy/tests/test_plugins.py
+++ b/browsepy/tests/test_plugins.py
@@ -1,9 +1,18 @@
+# -*- coding: UTF-8 -*-
+
+import sys
+import os
+import os.path
+import io
import unittest
+import tempfile
+
import flask
import browsepy
import browsepy.manager
import browsepy.utils as utils
+import browsepy.exceptions as exceptions
from browsepy.plugin.player.tests import * # noqa
from browsepy.plugin.file_actions.tests import * # noqa
@@ -56,7 +65,15 @@ class TestPlugins(unittest.TestCase):
self.manager.clear()
utils.clear_flask_context()
- def test_manager(self):
+ def test_manager_init(self):
+ class App(object):
+ config = {}
+
+ app = App()
+ manager = self.manager_module.PluginManagerBase(app)
+ self.assertDictEqual(app.extensions, {'plugin_manager': manager})
+
+ def test_manager_load(self):
self.manager.load_plugin(self.plugin_name)
self.assertTrue(self.manager._plugin_loaded)
@@ -92,12 +109,56 @@ class TestPlugins(unittest.TestCase):
self.assertIn(('browsepy.plugin.player', 'player'), names)
self.assertIn(('browsepy.plugin.file_actions', 'file-actions'), names)
- def test_namespace_prefix(self):
+ def test_namespace_submodule(self):
self.assertTrue(self.manager.import_plugin(self.plugin_name))
self.app.config['PLUGIN_NAMESPACES'] = (
self.plugin_namespace + '.test_',
)
self.assertTrue(self.manager.import_plugin('module'))
+ self.assertIn(
+ (self.plugin_namespace + '.' + self.plugin_name, 'plugins'),
+ self.manager.available_plugins,
+ )
+
+ def test_namespace_module(self):
+ self.app.config['PLUGIN_NAMESPACES'] = (
+ 'browsepy_test_',
+ 'browsepy_testing_',
+ )
+
+ with tempfile.TemporaryDirectory() as base:
+ names = (
+ 'browsepy_test_another_plugin',
+ 'browsepy_testing_another_plugin',
+ )
+ for name in names:
+ path = os.path.join(base, name) + '.py'
+ with io.open(path, 'w', encoding='utf8') as f:
+ f.write(
+ '\n'
+ 'def register_plugin(manager):\n'
+ ' pass\n'
+ )
+ try:
+ sys.path.insert(0, base)
+ self.assertIn(
+ (names[0], 'another-plugin'),
+ self.manager.available_plugins,
+ )
+ self.assertIn(
+ (names[1], None),
+ self.manager.available_plugins,
+ )
+ finally:
+ sys.path.remove(base)
+
+ def test_widget(self):
+ with self.assertRaises(exceptions.WidgetParameterException):
+ self.manager.register_widget(
+ place='header',
+ type='html',
+ bad_attr=True,
+ )
def register_plugin(manager):
diff --git a/browsepy/tests/test_utils.py b/browsepy/tests/test_utils.py
index a8507e4..a656ce1 100644
--- a/browsepy/tests/test_utils.py
+++ b/browsepy/tests/test_utils.py
@@ -1,24 +1,71 @@
# -*- coding: UTF-8 -*-
+import sys
+import io
+import os
+import os.path
+import functools
import unittest
+import tempfile
import browsepy.utils as utils
class TestPPath(unittest.TestCase):
+ module = utils
+
def test_bad_kwarg(self):
with self.assertRaises(TypeError) as e:
- utils.ppath('a', 'b', bad=1)
+ self.module.ppath('a', 'b', bad=1)
self.assertIn('\'bad\'', e.args[0])
+ def test_defaultsnamedtuple(self):
+ dnt = self.module.defaultsnamedtuple
+
+ tup = dnt('a', ('a', 'b', 'c'))
+ self.assertListEqual(list(tup(1, 2, 3)), [1, 2, 3])
+
+ tup = dnt('a', ('a', 'b', 'c'), {'b': 2})
+ self.assertListEqual(list(tup(1, c=3)), [1, 2, 3])
+
+ tup = dnt('a', ('a', 'b', 'c'), (1, 2, 3))
+ self.assertListEqual(list(tup(c=10)), [1, 2, 10])
+
def test_join(self):
self.assertTrue(
- utils
+ self.module
.ppath('a', 'b', module=__name__)
.endswith('browsepy/tests/a/b')
)
self.assertTrue(
- utils
+ self.module
.ppath('a', 'b')
.endswith('browsepy/a/b')
)
+
+ def test_get_module(self):
+ oldpath = sys.path[:]
+ try:
+ with tempfile.TemporaryDirectory() as base:
+ ppath = functools.partial(os.path.join, base)
+ sys.path[:] = [base]
+
+ p = ppath('_test_zderr.py')
+ with io.open(p, 'w', encoding='utf8') as f:
+ f.write('\na = 1 / 0\n')
+ with self.assertRaises(ZeroDivisionError):
+ self.module.get_module('_test_zderr')
+
+ p = ppath('_test_importerr.py')
+ with io.open(p, 'w', encoding='utf8') as f:
+ f.write(
+ '\n'
+ 'import os\n'
+ 'import _this_module_should_not_exist_ as m\n'
+ 'm.something()\n'
+ )
+ with self.assertRaises(ImportError):
+ self.module.get_module('_test_importerr')
+
+ finally:
+ sys.path[:] = oldpath
diff --git a/browsepy/utils.py b/browsepy/utils.py
index 65fd397..f8b352f 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -6,6 +6,7 @@ import os.path
import re
import random
import functools
+import collections
import flask
@@ -42,14 +43,13 @@ def get_module(name):
try:
__import__(name)
return sys.modules[name]
- except (ImportError, KeyError) as error:
- if isinstance(error, ImportError):
- message = error.args[0] if error.args else ''
- words = frozenset(re_words.findall(message))
- parts = name.split('.')
- for i in range(len(parts) - 1, -1, -1):
- if '.'.join(parts[i:]) in words:
- return None
+ except ImportError as error:
+ message = error.args[0] if error.args else ''
+ words = frozenset(re_words.findall(message))
+ parts = name.split('.')
+ for i in range(len(parts) - 1, -1, -1):
+ if '.'.join(parts[i:]) in words:
+ return None
raise
@@ -119,3 +119,22 @@ def clear_flask_context():
'''
clear_localstack(flask._app_ctx_stack)
clear_localstack(flask._request_ctx_stack)
+
+
+def defaultsnamedtuple(name, fields, defaults=None):
+ '''
+ Generate namedtuple with default values.
+
+ :param name: name
+ :param fields: iterable with field names
+ :param defaults: iterable or mapping with field defaults
+ :returns: defaultdict with given fields and given defaults
+ :rtype: collections.defaultdict
+ '''
+ nt = collections.namedtuple(name, fields)
+ nt.__new__.__defaults__ = (None, ) * len(nt._fields)
+ if isinstance(defaults, collections.Mapping):
+ nt.__new__.__defaults__ = tuple(nt(**defaults))
+ elif defaults:
+ nt.__new__.__defaults__ = tuple(nt(*defaults))
+ return nt
--
GitLab
From a67f5dfdf32ad3f6fa3c040047d83c86c4b0f0c6 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 25 Apr 2019 22:10:39 +0200
Subject: [PATCH 062/171] raise coverage, fixes
---
.vscode/settings.json | 2 +-
browsepy/__init__.py | 4 +-
browsepy/file.py | 17 ++++---
browsepy/manager.py | 48 +++++++++++--------
browsepy/plugin/file_actions/__init__.py | 35 +++++++-------
browsepy/plugin/file_actions/exceptions.py | 42 ++++++++++++++++
.../templates/400.file_actions.html | 6 +--
browsepy/plugin/file_actions/tests.py | 11 ++++-
browsepy/plugin/file_actions/utils.py | 16 +++++--
browsepy/plugin/player/__init__.py | 3 +-
browsepy/stream.py | 20 ++++++++
browsepy/utils.py | 29 +++++------
12 files changed, 161 insertions(+), 72 deletions(-)
diff --git a/.vscode/settings.json b/.vscode/settings.json
index cbbcd79..1f7909a 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -7,7 +7,7 @@
"files.trimTrailingWhitespace": true,
"files.insertFinalNewline": true,
"python.formatting.provider": "yapf",
- "python.unitTest.unittestEnabled": true,
+ "python.testing.unittestEnabled": true,
"python.linting.flake8Enabled": true,
"python.linting.enabled": true,
"python.linting.pep8Enabled": true,
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index 12ca3e9..5dcf37c 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -16,7 +16,7 @@ from werkzeug.exceptions import NotFound
from .appconfig import Flask
from .manager import PluginManager
from .file import Node, secure_filename
-from .utils import stream_template
+from .stream import stream_template
from .exceptions import OutsideRemovableBase, OutsideDirectoryBase, \
InvalidFilenameError, InvalidPathError
from . import compat
@@ -149,7 +149,7 @@ def sort(property, path):
return redirect(url_for(".browse", path=directory.urlpath))
-@app.route("/browse", defaults={"path": ""})
+@app.route('/browse', defaults={'path': ''})
@app.route('/browse/')
def browse(path):
sort_property = get_cookie_browse_sorting(path, 'text')
diff --git a/browsepy/file.py b/browsepy/file.py
index 0bcb9b8..5beb4da 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -9,8 +9,10 @@ import string
import random
import datetime
import logging
+import functools
+
+import flask
-from flask import current_app, send_from_directory
from werkzeug.utils import cached_property
from . import compat
@@ -273,7 +275,7 @@ class Node(object):
:param **defaults: initial property values
'''
self.path = compat.fsdecode(path) if path else None
- self.app = utils.solve_local(app or current_app)
+ self.app = utils.solve_local(app or flask.current_app)
self.__dict__.update(defaults) # only for attr and cached_property
def __repr__(self):
@@ -308,7 +310,7 @@ class Node(object):
:return: file object pointing to path
:rtype: File
'''
- app = utils.solve_local(app or current_app)
+ app = utils.solve_local(app or flask.current_app)
base = app.config.get('DIRECTORY_BASE', path)
path = urlpath_to_abspath(path, base)
if not cls.generic:
@@ -469,7 +471,7 @@ class File(Node):
:rtype: flask.Response
'''
directory, name = os.path.split(self.path)
- return send_from_directory(directory, name, as_attachment=True)
+ return flask.send_from_directory(directory, name, as_attachment=True)
@Node.register_directory_class
@@ -650,10 +652,13 @@ class Directory(Node):
stream = self.stream_class(
self.path,
self.app.config.get('DIRECTORY_TAR_BUFFSIZE', 10240),
- self.plugin_manager.check_excluded,
+ functools.partial(
+ self.plugin_manager.check_excluded,
+ request=utils.solve_local(flask.request),
+ )
)
return self.app.response_class(
- stream,
+ flask.stream_with_context(stream),
direct_passthrough=True,
headers=Headers(
content_type=(
diff --git a/browsepy/manager.py b/browsepy/manager.py
index 78cbc8e..5b1f682 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -7,15 +7,17 @@ import functools
import warnings
import collections
-from flask import current_app
-from werkzeug.utils import cached_property
+import flask
+from werkzeug.utils import cached_property
from cookieman import CookieMan
from . import mimetype
from . import compat
+from . import utils
+
from .compat import deprecated, usedoc
-from .utils import get_module, defaultsnamedtuple
+from .utils import defaultsnamedtuple
from .exceptions import PluginNotFoundError, InvalidArgumentError, \
WidgetParameterException
@@ -31,7 +33,7 @@ class PluginManagerBase(object):
'''
List of plugin namespaces taken from app config.
'''
- return self.app.config['PLUGIN_NAMESPACES'] if self.app else []
+ return self.app.config.get('PLUGIN_NAMESPACES', []) if self.app else []
def __init__(self, app=None):
'''
@@ -112,7 +114,7 @@ class PluginManagerBase(object):
except ImportError:
pass
- def _iter_plugin_modules(self):
+ def _iter_plugin_modules(self, get_module_fnc=utils.get_module):
'''
Iterate plugin modules, yielding both full qualified name and
short plugin name as tuple.
@@ -120,13 +122,14 @@ class PluginManagerBase(object):
nameset = set()
shortset = set()
filters = self.plugin_filters
- for prefix in self.namespaces:
- for name in (self._iter_submodules(prefix)
- if '.' in prefix else
- self._iter_modules(prefix)
- if prefix else
- ()):
- module = get_module(name) if name not in nameset else None
+ for prefix in filter(None, self.namespaces):
+ name_iter_fnc = (
+ self._iter_submodules
+ if '.' in prefix else
+ self._iter_modules
+ )
+ for name in name_iter_fnc(prefix):
+ module = get_module_fnc(name) if name not in nameset else None
if module and any(f(module) for f in filters):
short = name[len(prefix):].lstrip('.').replace('_', '-')
if short in shortset or '.' in short or not short:
@@ -142,7 +145,7 @@ class PluginManagerBase(object):
'''
return list(self._iter_plugin_modules())
- def import_plugin(self, plugin):
+ def import_plugin(self, plugin, get_module_fnc=utils.get_module):
'''
Import plugin by given name, looking at :attr:`namespaces`.
@@ -161,7 +164,7 @@ class PluginManagerBase(object):
]
names = sorted(frozenset(names), key=names.index)
for name in names:
- module = get_module(name)
+ module = get_module_fnc(name)
if module:
return module
raise PluginNotFoundError(
@@ -254,16 +257,19 @@ class ExcludePluginManager(PluginManagerBase):
'''
self._exclude_functions.add(exclude_fnc)
- def check_excluded(self, path):
+ def check_excluded(self, path, request=flask.request):
'''
Check if given path is excluded.
'''
- exclude_fnc = self.app.config.get('EXCLUDE_FNC')
- if exclude_fnc and exclude_fnc(path):
- return True
- for fnc in self._exclude_functions:
- if fnc(path):
+ if not request:
+ request = utils.dummy_context()
+ with request:
+ exclude_fnc = self.app.config.get('EXCLUDE_FNC')
+ if exclude_fnc and exclude_fnc(path):
return True
+ for fnc in self._exclude_functions:
+ if fnc(path):
+ return True
return False
def clear(self):
@@ -712,7 +718,7 @@ class MimetypeActionPluginManager(WidgetPluginManager, MimetypePluginManager):
def _widget_attrgetter(self, widget, name):
def handler(f):
- app = f.app or self.app or current_app
+ app = f.app or self.app or flask.current_app
with app.app_context():
return getattr(widget.for_file(f), name)
return handler
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index ffb4cf4..0a1e68a 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -5,7 +5,7 @@ import os.path
import functools
from flask import Blueprint, render_template, request, redirect, url_for, \
- session, current_app
+ session, current_app, g
from werkzeug.exceptions import NotFound
from browsepy import get_cookie_browse_sorting, browse_sortkey_reverse
@@ -13,13 +13,15 @@ from browsepy.file import Node, abspath_to_urlpath, secure_filename, \
current_restricted_chars, common_path_separators
from browsepy.compat import re_escape, FileNotFoundError
from browsepy.exceptions import OutsideDirectoryBase
-from browsepy.utils import stream_template, ppath
+from browsepy.utils import ppath
+from browsepy.stream import stream_template
from .exceptions import FileActionsException, \
InvalidClipboardItemsError, \
InvalidClipboardModeError, \
InvalidDirnameError, \
- DirectoryCreationError
+ DirectoryCreationError, \
+ InvalidClipboardSizeError
from . import utils
@@ -140,10 +142,8 @@ def clipboard_paste(path):
mode = session.get('clipboard:mode')
clipboard = session.get('clipboard:items', ())
- success, issues, nmode = utils.paste(directory, mode, clipboard)
- if clipboard and mode != nmode:
- session['mode'] = nmode
- mode = nmode
+ g.file_actions_paste = True # disable exclude function
+ success, issues = utils.paste(directory, mode, clipboard)
if issues:
raise InvalidClipboardItemsError(
@@ -154,7 +154,8 @@ def clipboard_paste(path):
)
if mode == 'cut':
- del session['clipboard:paths']
+ session.pop('clipboard:mode', None)
+ session.pop('clipboard:items', None)
return redirect(url_for('browse', path=directory.urlpath))
@@ -162,8 +163,8 @@ def clipboard_paste(path):
@actions.route('/clipboard/clear', defaults={'path': ''})
@actions.route('/clipboard/clear/')
def clipboard_clear(path):
- if 'clipboard:paths' in session:
- del session['clipboard:paths']
+ session.pop('clipboard:mode', None)
+ session.pop('clipboard:items', None)
return redirect(url_for('browse', path=path))
@@ -172,8 +173,8 @@ def clipboard_error(e):
file = Node(e.path) if hasattr(e, 'path') else None
issues = getattr(e, 'issues', ())
- if session.get('clipboard:items'):
- clipboard = session['clipboard:mode']
+ clipboard = session.get('clipboard:items')
+ if clipboard and issues:
for issue in issues:
if isinstance(issue.error, FileNotFoundError):
path = issue.item.urlpath
@@ -196,9 +197,10 @@ def clipboard_error(e):
def shrink_session(data, last):
if last:
- # TODO: add warning message
- del data['clipboard:items']
- del data['clipboard:mode']
+ raise InvalidClipboardSizeError(
+ mode=data.pop('clipboard:mode', None),
+ clipboard=data.pop('clipboard:items', None),
+ )
return data
@@ -216,7 +218,8 @@ def detect_selection(directory):
def excluded_clipboard(manager, path):
- if session.get('clipboard:mode') == 'cut':
+ if not getattr(g, 'file_actions_paste', False) and \
+ session.get('clipboard:mode') == 'cut':
base = manager.app.config['DIRECTORY_BASE']
clipboard = session.get('clipboard:items', ())
return abspath_to_urlpath(path, base) in clipboard
diff --git a/browsepy/plugin/file_actions/exceptions.py b/browsepy/plugin/file_actions/exceptions.py
index 8df75a0..aaa0285 100644
--- a/browsepy/plugin/file_actions/exceptions.py
+++ b/browsepy/plugin/file_actions/exceptions.py
@@ -7,6 +7,8 @@ import errno
class FileActionsException(Exception):
'''
Base class for file-actions exceptions
+
+ :property path: item path which raised this Exception
'''
code = None
template = 'Unhandled error.'
@@ -21,6 +23,7 @@ class InvalidDirnameError(FileActionsException):
'''
Exception raised when a new directory name is invalid.
+ :property path: item path which raised this Exception
:property name: name which raised this Exception
'''
code = 'directory-invalid-name'
@@ -35,6 +38,7 @@ class DirectoryCreationError(FileActionsException):
'''
Exception raised when a new directory creation fails.
+ :property path: item path which raised this Exception
:property name: name which raised this Exception
'''
code = 'directory-mkdir-error'
@@ -64,6 +68,7 @@ class ClipboardException(FileActionsException):
Base class for clipboard exceptions.
:property path: item path which raised this Exception
+ :property mode: mode which raised this Exception
:property clipboard: :class Clipboard: instance
'''
code = 'clipboard-invalid'
@@ -106,6 +111,9 @@ class InvalidClipboardItemsError(ClipboardException):
Exception raised when a clipboard item is not valid.
:property path: item path which raised this Exception
+ :property mode: mode which raised this Exception
+ :property clipboard: :class Clipboard: instance
+ :property issues: iterable of issues
'''
pair_class = ItemIssue
code = 'clipboard-invalid-items'
@@ -125,7 +133,9 @@ class InvalidClipboardModeError(ClipboardException):
'''
Exception raised when a clipboard mode is not valid.
+ :property path: item path which raised this Exception
:property mode: mode which raised this Exception
+ :property clipboard: :class Clipboard: instance
'''
code = 'clipboard-invalid-mode'
template = 'Clipboard mode {0.path!r} is not valid.'
@@ -133,3 +143,35 @@ class InvalidClipboardModeError(ClipboardException):
def __init__(self, message=None, path=None, mode=None, clipboard=None):
supa = super(InvalidClipboardModeError, self)
supa.__init__(message, path, mode, clipboard)
+
+
+class InvalidEmptyClipboardError(ClipboardException):
+ '''
+ Exception raised when an invalid action is requested on an empty clipboard.
+
+ :property path: item path which raised this Exception
+ :property mode: mode which raised this Exception
+ :property clipboard: :class Clipboard: instance
+ '''
+ code = 'clipboard-invalid-empty'
+ template = 'Clipboard action {0.mode!r} cannot be performed without items.'
+
+ def __init__(self, message=None, path=None, mode=None, clipboard=None):
+ supa = super(InvalidEmptyClipboardError, self)
+ supa.__init__(message, path, mode, clipboard)
+
+
+class InvalidClipboardSizeError(ClipboardException):
+ '''
+ Exception raised when session manager evicts clipboard data.
+
+ :property path: item path which raised this Exception
+ :property mode: mode which raised this Exception
+ :property clipboard: :class Clipboard: instance
+ '''
+ code = 'clipboard-invalid-size'
+ template = 'Clipboard evicted due session size limit.'
+
+ def __init__(self, message=None, path=None, mode=None, clipboard=None):
+ supa = super(InvalidClipboardSizeError, self)
+ supa.__init__(message, path, mode, clipboard)
diff --git a/browsepy/plugin/file_actions/templates/400.file_actions.html b/browsepy/plugin/file_actions/templates/400.file_actions.html
index 9ac9b30..969458a 100644
--- a/browsepy/plugin/file_actions/templates/400.file_actions.html
+++ b/browsepy/plugin/file_actions/templates/400.file_actions.html
@@ -48,9 +48,9 @@
{%- endset %}
{% elif error.code == 'clipboard-invalid-size' %}
{% set description -%}
-
Clipboard size limit exceeded
-
Clipboard became too big to be effectively stored in your browser cookies.
-
Please try again with fewer items.
+
Session size limit exceeded, clipboard lost.
+
Clipboard became too big to be stored in your browser cookies.
+
Please try again with fewer items and avoid other operations in the process.
{%- endset %}
{% else %}
{% set description %}
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index 668051a..d1ecf86 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -342,6 +342,9 @@ class TestAction(unittest.TestCase):
response = client.get('/file-actions/selection/nowhere')
self.assertEqual(response.status_code, 404)
+ response = client.post('/file-actions/selection', data={})
+ self.assertEqual(response.status_code, 400)
+
def test_paste(self):
files = ['a', 'b', 'c']
self.touch('a')
@@ -393,27 +396,31 @@ class TestAction(unittest.TestCase):
with self.app.test_client() as client:
with client.session_transaction() as session:
- session['clipboard:mode'] = 'wrong-mode'
session['clipboard:items'] = ['whatever']
+ session['clipboard:mode'] = 'wrong-mode'
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 400)
with client.session_transaction() as session:
+ session['clipboard:items'] = ['whatever']
session['clipboard:mode'] = 'cut'
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 302) # same location
with client.session_transaction() as session:
+ session['clipboard:items'] = ['whatever']
session['clipboard:mode'] = 'cut'
response = client.get('/file-actions/clipboard/paste/target')
self.assertEqual(response.status_code, 400)
with client.session_transaction() as session:
+ session['clipboard:items'] = ['whatever']
session['clipboard:mode'] = 'copy'
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 400)
with client.session_transaction() as session:
+ session['clipboard:items'] = ['whatever']
session['clipboard:mode'] = 'copy'
response = client.get('/file-actions/clipboard/paste/target')
self.assertEqual(response.status_code, 400)
@@ -421,11 +428,13 @@ class TestAction(unittest.TestCase):
self.app.config['EXCLUDE_FNC'] = lambda n: n.endswith('whatever')
with client.session_transaction() as session:
+ session['clipboard:items'] = ['whatever']
session['clipboard:mode'] = 'cut'
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 400)
with client.session_transaction() as session:
+ session['clipboard:items'] = ['whatever']
session['clipboard:mode'] = 'copy'
response = client.get('/file-actions/clipboard/paste')
self.assertEqual(response.status_code, 400)
diff --git a/browsepy/plugin/file_actions/utils.py b/browsepy/plugin/file_actions/utils.py
index 8666f2a..28a5772 100644
--- a/browsepy/plugin/file_actions/utils.py
+++ b/browsepy/plugin/file_actions/utils.py
@@ -8,20 +8,23 @@ import functools
from browsepy.file import Node
from browsepy.compat import map
-from .exceptions import InvalidClipboardModeError
+from .exceptions import InvalidClipboardModeError, InvalidEmptyClipboardError
def copy(target, node, join_fnc=os.path.join):
if node.is_excluded:
raise OSError(2, os.strerror(2))
+
dest = join_fnc(
target.path,
target.choose_filename(node.name)
)
+
if node.is_directory:
shutil.copytree(node.path, dest)
else:
shutil.copy2(node.path, dest)
+
return dest
@@ -53,12 +56,19 @@ def paste(target, mode, clipboard):
mode=mode,
clipboard=clipboard,
)
+
+ if not clipboard:
+ raise InvalidEmptyClipboardError(
+ path=target.path,
+ mode=mode,
+ clipboard=clipboard,
+ )
+
success = []
issues = []
- mode = 'paste' # deactivates excluded_clipboard fnc
for node in map(Node.from_urlpath, clipboard):
try:
success.append(paste_fnc(node))
except BaseException as e:
issues.append((node, e))
- return success, issues, mode
+ return success, issues
diff --git a/browsepy/plugin/player/__init__.py b/browsepy/plugin/player/__init__.py
index 0639131..fbc5e2d 100644
--- a/browsepy/plugin/player/__init__.py
+++ b/browsepy/plugin/player/__init__.py
@@ -5,7 +5,8 @@ from werkzeug.exceptions import NotFound
from browsepy import get_cookie_browse_sorting, browse_sortkey_reverse
from browsepy.file import OutsideDirectoryBase
-from browsepy.utils import stream_template, ppath
+from browsepy.utils import ppath
+from browsepy.stream import stream_template
from .playable import PlayableFile, PlayableDirectory, \
PlayListFile, detect_playable_mimetype
diff --git a/browsepy/stream.py b/browsepy/stream.py
index b834039..f17f131 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -6,6 +6,9 @@ import tarfile
import threading
import functools
+import flask
+
+from . import utils
from . import compat
@@ -233,3 +236,20 @@ class TarFileStream(compat.Iterator):
Call :method:`TarFileStream.close`.
'''
self.close()
+
+
+def stream_template(template_name, **context):
+ '''
+ Some templates can be huge, this function returns an streaming response,
+ sending the content in chunks and preventing from timeout.
+
+ :param template_name: template
+ :param **context: parameters for templates.
+ :yields: HTML strings
+ :rtype: Iterator of str
+ '''
+ app = utils.solve_local(context.get('current_app') or flask.current_app)
+ app.update_template_context(context)
+ template = app.jinja_env.get_template(template_name)
+ stream = template.generate(context)
+ return flask.Response(flask.stream_with_context(stream))
diff --git a/browsepy/utils.py b/browsepy/utils.py
index f8b352f..0609d54 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -6,6 +6,7 @@ import os.path
import re
import random
import functools
+import contextlib
import collections
import flask
@@ -31,6 +32,15 @@ def ppath(*args, **kwargs):
return os.path.join(os.path.dirname(path), *args)
+@contextlib.contextmanager
+def dummy_context():
+ '''
+ Context manager which does nothing besides exposing the context
+ manger interface
+ '''
+ yield
+
+
def get_module(name):
'''
Get module object by name.
@@ -47,7 +57,7 @@ def get_module(name):
message = error.args[0] if error.args else ''
words = frozenset(re_words.findall(message))
parts = name.split('.')
- for i in range(len(parts) - 1, -1, -1):
+ for i in range(len(parts) - 1):
if '.'.join(parts[i:]) in words:
return None
raise
@@ -78,23 +88,6 @@ def solve_local(context_local):
return context_local
-def stream_template(template_name, **context):
- '''
- Some templates can be huge, this function returns an streaming response,
- sending the content in chunks and preventing from timeout.
-
- :param template_name: template
- :param **context: parameters for templates.
- :yields: HTML strings
- :rtype: Iterator of str
- '''
- app = solve_local(context.get('current_app') or flask.current_app)
- app.update_template_context(context)
- template = app.jinja_env.get_template(template_name)
- stream = template.generate(context)
- return flask.Response(flask.stream_with_context(stream))
-
-
def clear_localstack(stack):
'''
Clear given werkzeug LocalStack instance.
--
GitLab
From dd6ba35407827416d965067f7d1c01cbee639259 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 26 Apr 2019 13:04:10 +0200
Subject: [PATCH 063/171] compatibility fixes
---
browsepy/compat.py | 20 ++++++++++++++++++--
browsepy/file.py | 7 ++-----
browsepy/manager.py | 6 +++---
browsepy/tests/test_file.py | 11 ++++++-----
setup.py | 8 +++++---
5 files changed, 34 insertions(+), 18 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 8713fcb..afe1709 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -22,9 +22,9 @@ except ImportError:
import importlib_resources as res # noqa
try:
- from os import scandir, walk # python 3.5+
+ from os import scandir as _scandir, walk # python 3.5+
except ImportError:
- from scandir import scandir, walk # noqa
+ from scandir import scandir as _scandir, walk # noqa
try:
from shutil import get_terminal_size # python 3.3+
@@ -67,6 +67,22 @@ class HelpFormatter(argparse.RawTextHelpFormatter):
prog, indent_increment, max_help_position, width)
+@contextlib.contextmanager
+def scandir(path):
+ '''
+ Backwards-compatible scandir context manager
+
+ :param path: path to iterate
+ :type path: str
+ '''
+ files = _scandir(path)
+ try:
+ yield files
+ finally:
+ if callable(getattr(files, 'close', None)):
+ files.close()
+
+
def isexec(path):
'''
Check if given path points to an executable file.
diff --git a/browsepy/file.py b/browsepy/file.py
index 5beb4da..f214f26 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -729,8 +729,8 @@ class Directory(Node):
directory_class = self.directory_class
file_class = self.file_class
exclude_fnc = self.plugin_manager.check_excluded
- files = compat.scandir(self.path)
- try:
+
+ with compat.scandir(self.path) as files:
for entry in files:
if exclude_fnc(entry.path):
continue
@@ -750,9 +750,6 @@ class Directory(Node):
)
except OSError as e:
logger.exception(e)
- finally:
- if callable(getattr(files, 'close', None)):
- files.close()
def listdir(self, sortkey=None, reverse=False):
'''
diff --git a/browsepy/manager.py b/browsepy/manager.py
index 5b1f682..6994626 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -82,9 +82,9 @@ class PluginManagerBase(object):
'''
Iterate thru all root modules containing given prefix.
'''
- for info in pkgutil.iter_modules():
- if info.name.startswith(prefix):
- yield info.name
+ for finder, name, ispkg in pkgutil.iter_modules():
+ if name.startswith(prefix):
+ yield name
def _content_import_name(self, module, item, prefix):
'''
diff --git a/browsepy/tests/test_file.py b/browsepy/tests/test_file.py
index 3bd1802..08ca9cc 100644
--- a/browsepy/tests/test_file.py
+++ b/browsepy/tests/test_file.py
@@ -23,11 +23,12 @@ class TestFile(unittest.TestCase):
self.workbench = tempfile.mkdtemp()
def clear_workbench(self):
- for entry in browsepy.compat.scandir(self.workbench):
- if entry.is_dir():
- shutil.rmtree(entry.path)
- else:
- os.remove(entry.path)
+ with browsepy.compat.scandir(self.workbench) as files:
+ for entry in files:
+ if entry.is_dir():
+ shutil.rmtree(entry.path)
+ else:
+ os.remove(entry.path)
def tearDown(self):
shutil.rmtree(self.workbench)
diff --git a/setup.py b/setup.py
index cf2ed59..7f33865 100644
--- a/setup.py
+++ b/setup.py
@@ -60,9 +60,11 @@ setup(
install_requires=[
'flask',
'unicategories',
- 'scandir',
- 'importlib_resources',
- 'backports.shutil_get_terminal_size'
+ 'cookieman',
+ 'backports.shutil_get_terminal_size ; python_version<\'3.3\'',
+ 'scandir ; python_version<\'3.5\'',
+ 'pathlib2 ; python_version<\'3.5\'',
+ 'importlib_resources ; python_version<\'3.7\'',
],
extras_require={
'tests': ['beautifulsoup4', 'pycodestyle'],
--
GitLab
From ceb6a9d5caed10efcdb2dd9ada75ab276ec1bcb0 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 26 Apr 2019 13:29:12 +0200
Subject: [PATCH 064/171] add note on readme regarding setuptools, update
requirements
---
.travis.yml | 2 +-
README.rst | 4 ++++
requirements/base.txt | 6 ++++--
3 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index a9dd839..10e463b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,7 +23,7 @@ matrix:
install:
- |
if [ "$legacy" != "yes" ]; then
- pip install --upgrade pip
+ pip install --upgrade pip setuptools
fi
- pip install -r requirements/ci.txt
- pip install .
diff --git a/README.rst b/README.rst
index 79bf900..190fd8a 100644
--- a/README.rst
+++ b/README.rst
@@ -83,6 +83,10 @@ New in 0.5
Install
-------
+*Note*: with some legacy Python versions shiping outdated libraries, both
+`pip` and `setuptools` library should be upgraded with
+`pip install --upgrade pip setuptools`.
+
It's on `pypi` so...
.. _pypi: https://pypi.python.org/pypi/browsepy/
diff --git a/requirements/base.txt b/requirements/base.txt
index 4518665..5d1f523 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -18,5 +18,7 @@ unicategories
# compat
six
-scandir # python < 3.6
-backports.shutil_get_terminal_size # python < 3.3
+backports.shutil_get_terminal_size ; python_version<'3.3'
+scandir ; python_version<'3.5'
+pathlib2 ; python_version<'3.5'
+importlib_resources ; python_version<'3.7'
--
GitLab
From 304006a4781d6821fdaf432b33bb0743798803ac Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 26 Apr 2019 13:34:44 +0200
Subject: [PATCH 065/171] retry with distutils upgrade
---
.appveyor.yml | 2 +-
.travis.yml | 2 +-
README.rst | 6 +++---
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/.appveyor.yml b/.appveyor.yml
index b9408f8..8c32029 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -73,7 +73,7 @@ install:
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
- "python --version"
- "python -c \"import struct; print(struct.calcsize('P') * 8)\""
- - "pip install --disable-pip-version-check --user --upgrade pip setuptools"
+ - "pip install --disable-pip-version-check --user --upgrade pip setuptools distutils"
test_script:
- "python setup.py test"
diff --git a/.travis.yml b/.travis.yml
index 10e463b..3f5a0b6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,7 +23,7 @@ matrix:
install:
- |
if [ "$legacy" != "yes" ]; then
- pip install --upgrade pip setuptools
+ pip install --upgrade pip setuptools distutils
fi
- pip install -r requirements/ci.txt
- pip install .
diff --git a/README.rst b/README.rst
index 190fd8a..53749a8 100644
--- a/README.rst
+++ b/README.rst
@@ -83,9 +83,9 @@ New in 0.5
Install
-------
-*Note*: with some legacy Python versions shiping outdated libraries, both
-`pip` and `setuptools` library should be upgraded with
-`pip install --upgrade pip setuptools`.
+*Note*: with some legacy Python versions shipping outdated libraries, all
+`pip`, `setuptools` and `distutils` libraries should be upgraded with
+`pip install --upgrade pip setuptools distutils`.
It's on `pypi` so...
--
GitLab
From 1bc65cf41e4aabd54d9a8665926bd792bf7a8b1e Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 26 Apr 2019 13:46:56 +0200
Subject: [PATCH 066/171] drop python3.3 for good
---
.appveyor.yml | 10 +---------
.travis.yml | 5 +----
README.rst | 23 ++++++++++++++++-------
3 files changed, 18 insertions(+), 20 deletions(-)
diff --git a/.appveyor.yml b/.appveyor.yml
index 8c32029..54f91e3 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -8,14 +8,6 @@ environment:
PYTHON_VERSION: "2.7.x" # currently 2.7.9
PYTHON_ARCH: "64"
- - PYTHON: "C:\\Python33"
- PYTHON_VERSION: "3.3.x" # currently 3.3.5
- PYTHON_ARCH: "32"
-
- - PYTHON: "C:\\Python33-x64"
- PYTHON_VERSION: "3.3.x" # currently 3.3.5
- PYTHON_ARCH: "64"
-
- PYTHON: "C:\\Python34"
PYTHON_VERSION: "3.4.x" # currently 3.4.3
PYTHON_ARCH: "32"
@@ -73,7 +65,7 @@ install:
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
- "python --version"
- "python -c \"import struct; print(struct.calcsize('P') * 8)\""
- - "pip install --disable-pip-version-check --user --upgrade pip setuptools distutils"
+ - "pip install --disable-pip-version-check --user --upgrade pip setuptools"
test_script:
- "python setup.py test"
diff --git a/.travis.yml b/.travis.yml
index 3f5a0b6..7aa44db 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,9 +9,6 @@ matrix:
- python: "2.7"
- python: "pypy"
- python: "pypy3"
- - python: "3.3"
- env:
- - legacy=yes
- python: "3.4"
- python: "3.5"
- python: "3.6"
@@ -23,7 +20,7 @@ matrix:
install:
- |
if [ "$legacy" != "yes" ]; then
- pip install --upgrade pip setuptools distutils
+ pip install --upgrade pip setuptools
fi
- pip install -r requirements/ci.txt
- pip install .
diff --git a/README.rst b/README.rst
index 53749a8..e15e756 100644
--- a/README.rst
+++ b/README.rst
@@ -25,9 +25,9 @@ browsepy
:target: https://pypi.python.org/pypi/browsepy/
:alt: Version: 0.5.6
-.. image:: https://img.shields.io/badge/python-2.7%2B%2C%203.3%2B-FFC100.svg?style=flat-square
+.. image:: https://img.shields.io/badge/python-2.7%2B%2C%203.4%2B-FFC100.svg?style=flat-square
:target: https://pypi.python.org/pypi/browsepy/
- :alt: Python 2.7+, 3.3+
+ :alt: Python 2.7+, 3.4+
The simple web file browser.
@@ -56,6 +56,15 @@ Features
* **Optional upload** for directories under given path.
* **Player** audio player plugin is provided (without transcoding).
+New in 0.6
+----------
+
+* Drop Python 3.3.
+* Plugin discovery.
+* Plugin file-actions providing copy/cut/paste and directory creation.
+* Smarter cookie sessions with session shrinking mechanisms available
+ for plugin implementors.
+
New in 0.5
----------
@@ -83,9 +92,9 @@ New in 0.5
Install
-------
-*Note*: with some legacy Python versions shipping outdated libraries, all
-`pip`, `setuptools` and `distutils` libraries should be upgraded with
-`pip install --upgrade pip setuptools distutils`.
+*Note*: with some legacy Python versions shipping outdated libraries, both
+`pip` and `setuptools` libraries should be upgraded with
+`pip install --upgrade pip setuptools`.
It's on `pypi` so...
@@ -156,7 +165,7 @@ plugins (loaded with `plugin` argument) could add extra arguments to this list.
optional arguments:
-h, --help show this help message and exit
--help-all show help for all available plugins and exit
- --directory PATH serving directory (default: /home/work/Desarrollo/browsepy)
+ --directory PATH serving directory (default: ...)
--initial PATH default directory (default: same as --directory)
--removable PATH base directory allowing remove (default: None)
--upload PATH base directory allowing upload (default: None)
@@ -166,8 +175,8 @@ plugins (loaded with `plugin` argument) could add extra arguments to this list.
--plugin MODULE load plugin module (multiple)
available plugins:
- file-actions, browsepy.plugin.file_actions
player, browsepy.plugin.player
+ file-actions, browsepy.plugin.file_actions
Using as library
--
GitLab
From 4b491be44faf4560a1fab6d0c4f6d3c6d02c8ccf Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 26 Apr 2019 18:34:11 +0200
Subject: [PATCH 067/171] fix get_module importerror detection
---
browsepy/utils.py | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/browsepy/utils.py b/browsepy/utils.py
index 0609d54..6e2affb 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -12,7 +12,7 @@ import collections
import flask
-re_words = re.compile(r'\b((?:[._]+|\w)+)\b')
+re_words = re.compile(r'\b((?:[._]+|\w+)+)\b')
def ppath(*args, **kwargs):
@@ -57,10 +57,8 @@ def get_module(name):
message = error.args[0] if error.args else ''
words = frozenset(re_words.findall(message))
parts = name.split('.')
- for i in range(len(parts) - 1):
- if '.'.join(parts[i:]) in words:
- return None
- raise
+ if not any('.'.join(parts[i:]) in words for i in range(len(parts))):
+ raise
def random_string(size, sample=tuple(map(chr, range(256)))):
--
GitLab
From 7d233af4288cec04a924a47718933173efd0196f Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 26 May 2019 18:34:47 +0100
Subject: [PATCH 068/171] improve readability, drop 3.4, add flake8 to ci
---
.travis.yml | 3 ++-
README.rst | 2 +-
browsepy/compat.py | 19 +++++++++++++++++-
browsepy/manager.py | 33 ++++++++++++++++++++-----------
browsepy/stream.py | 36 ++++++++++++++++++++--------------
browsepy/tests/test_plugins.py | 7 +++----
browsepy/tests/test_utils.py | 16 ++++++---------
browsepy/utils.py | 15 ++------------
setup.cfg | 2 +-
9 files changed, 76 insertions(+), 57 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 7aa44db..3b448c3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,10 +9,10 @@ matrix:
- python: "2.7"
- python: "pypy"
- python: "pypy3"
- - python: "3.4"
- python: "3.5"
- python: "3.6"
- python: "3.7"
+ dist: xenial
env:
- eslint=yes
- sphinx=yes
@@ -36,6 +36,7 @@ install:
fi
script:
+ - flake8 browsepy
- coverage run setup.py test
- coverage report
- |
diff --git a/README.rst b/README.rst
index e15e756..b6b2330 100644
--- a/README.rst
+++ b/README.rst
@@ -27,7 +27,7 @@ browsepy
.. image:: https://img.shields.io/badge/python-2.7%2B%2C%203.4%2B-FFC100.svg?style=flat-square
:target: https://pypi.python.org/pypi/browsepy/
- :alt: Python 2.7+, 3.4+
+ :alt: Python 2.7+, 3.5+
The simple web file browser.
diff --git a/browsepy/compat.py b/browsepy/compat.py
index afe1709..109e83b 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -4,6 +4,8 @@ import os
import os.path
import sys
import abc
+import shutil
+import tempfile
import itertools
import functools
import contextlib
@@ -70,7 +72,7 @@ class HelpFormatter(argparse.RawTextHelpFormatter):
@contextlib.contextmanager
def scandir(path):
'''
- Backwards-compatible scandir context manager
+ Backwards-compatible :func:`scandir.scandir` context manager
:param path: path to iterate
:type path: str
@@ -83,6 +85,21 @@ def scandir(path):
files.close()
+@contextlib.contextmanager
+def mkdtemp(suffix='', prefix='', dir=None):
+ '''
+ Backwards-compatible :class:`tmpfile.TemporaryDirectory` context manager.
+
+ :param path: path to iterate
+ :type path: str
+ '''
+ path = tempfile.mkdtemp(suffix, prefix, dir)
+ try:
+ yield path
+ finally:
+ shutil.rmtree(path)
+
+
def isexec(path):
'''
Check if given path points to an executable file.
diff --git a/browsepy/manager.py b/browsepy/manager.py
index 6994626..42fc063 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -129,14 +129,24 @@ class PluginManagerBase(object):
self._iter_modules
)
for name in name_iter_fnc(prefix):
- module = get_module_fnc(name) if name not in nameset else None
- if module and any(f(module) for f in filters):
- short = name[len(prefix):].lstrip('.').replace('_', '-')
- if short in shortset or '.' in short or not short:
- short = None
- yield name, short
- nameset.add(name)
- shortset.add(short)
+ if name in nameset:
+ continue
+
+ try:
+ module = get_module_fnc(name)
+ except ImportError:
+ continue
+
+ if not any(f(module) for f in filters):
+ continue
+
+ short = name[len(prefix):].lstrip('.').replace('_', '-')
+ yield (
+ name,
+ None if short in shortset or '.' in short else short
+ )
+ nameset.add(name)
+ shortset.add(short)
@cached_property
def available_plugins(self):
@@ -164,9 +174,10 @@ class PluginManagerBase(object):
]
names = sorted(frozenset(names), key=names.index)
for name in names:
- module = get_module_fnc(name)
- if module:
- return module
+ try:
+ return get_module_fnc(name)
+ except ImportError:
+ pass
raise PluginNotFoundError(
'No plugin module %r found, tried %r' % (plugin, names),
plugin, names)
diff --git a/browsepy/stream.py b/browsepy/stream.py
index f17f131..20d4878 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -8,7 +8,6 @@ import functools
import flask
-from . import utils
from . import compat
@@ -90,22 +89,29 @@ class BlockingPipe(object):
Closes, so any blocked and future writes or retrieves will raise
:attr:`abort_exception` instances.
'''
+
+ def blocked():
+ '''
+ NOOP lock release function for non-owned locks.
+ '''
+ pass
+
if not self.closed:
self.closed = True
- # release locks
- reading = not self._rlock.acquire(False)
- writing = not self._wlock.acquire(False)
+ releasing = writing, reading = [
+ lock.release if lock.acquire(False) else blocked
+ for lock in (self._wlock, self._rlock)
+ ]
+
+ if writing is blocked and reading is not blocked:
+ self._pipe.get()
- if not reading:
- if writing:
- self._pipe.get()
- self._rlock.release()
+ if reading is blocked and writing is not blocked:
+ self._pipe.put(None)
- if not writing:
- if reading:
- self._pipe.put(None)
- self._wlock.release()
+ for release in releasing:
+ release()
class TarFileStream(compat.Iterator):
@@ -200,11 +206,11 @@ class TarFileStream(compat.Iterator):
try:
tarfile.add(self.path, "", filter=infofilter if exclude else None)
- tarfile.close() # force stream flush
+ tarfile.close() # force stream flush (may raise)
except self.abort_exception:
# expected exception when pipe is closed prematurely
tarfile.close() # free fd
- else:
+ finally:
self.close()
def __next__(self):
@@ -248,7 +254,7 @@ def stream_template(template_name, **context):
:yields: HTML strings
:rtype: Iterator of str
'''
- app = utils.solve_local(context.get('current_app') or flask.current_app)
+ app = context.get('current_app', flask.current_app)
app.update_template_context(context)
template = app.jinja_env.get_template(template_name)
stream = template.generate(context)
diff --git a/browsepy/tests/test_plugins.py b/browsepy/tests/test_plugins.py
index 0a9c1c9..cc877c6 100644
--- a/browsepy/tests/test_plugins.py
+++ b/browsepy/tests/test_plugins.py
@@ -3,15 +3,14 @@
import sys
import os
import os.path
-import io
import unittest
-import tempfile
import flask
import browsepy
import browsepy.manager
import browsepy.utils as utils
+import browsepy.compat as compat
import browsepy.exceptions as exceptions
from browsepy.plugin.player.tests import * # noqa
@@ -126,14 +125,14 @@ class TestPlugins(unittest.TestCase):
'browsepy_testing_',
)
- with tempfile.TemporaryDirectory() as base:
+ with compat.mkdtemp() as base:
names = (
'browsepy_test_another_plugin',
'browsepy_testing_another_plugin',
)
for name in names:
path = os.path.join(base, name) + '.py'
- with io.open(path, 'w', encoding='utf8') as f:
+ with open(path, 'w') as f:
f.write(
'\n'
'def register_plugin(manager):\n'
diff --git a/browsepy/tests/test_utils.py b/browsepy/tests/test_utils.py
index a656ce1..7be70f6 100644
--- a/browsepy/tests/test_utils.py
+++ b/browsepy/tests/test_utils.py
@@ -1,14 +1,13 @@
# -*- coding: UTF-8 -*-
import sys
-import io
import os
import os.path
import functools
import unittest
-import tempfile
import browsepy.utils as utils
+import browsepy.compat as compat
class TestPPath(unittest.TestCase):
@@ -46,22 +45,19 @@ class TestPPath(unittest.TestCase):
def test_get_module(self):
oldpath = sys.path[:]
try:
- with tempfile.TemporaryDirectory() as base:
- ppath = functools.partial(os.path.join, base)
+ with compat.mkdtemp() as base:
+ p = functools.partial(os.path.join, base)
sys.path[:] = [base]
- p = ppath('_test_zderr.py')
- with io.open(p, 'w', encoding='utf8') as f:
+ with open(p('_test_zderr.py'), 'w') as f:
f.write('\na = 1 / 0\n')
with self.assertRaises(ZeroDivisionError):
self.module.get_module('_test_zderr')
- p = ppath('_test_importerr.py')
- with io.open(p, 'w', encoding='utf8') as f:
+ with open(p('_test_importerr.py'), 'w') as f:
f.write(
'\n'
- 'import os\n'
- 'import _this_module_should_not_exist_ as m\n'
+ 'import browsepy.tests.test_utils.failing\n'
'm.something()\n'
)
with self.assertRaises(ImportError):
diff --git a/browsepy/utils.py b/browsepy/utils.py
index 6e2affb..d6dd38f 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -3,7 +3,6 @@
import sys
import os
import os.path
-import re
import random
import functools
import contextlib
@@ -12,9 +11,6 @@ import collections
import flask
-re_words = re.compile(r'\b((?:[._]+|\w+)+)\b')
-
-
def ppath(*args, **kwargs):
'''
Get joined file path relative to module location.
@@ -50,15 +46,8 @@ def get_module(name):
:return: module or None if not found
:rtype: module or None
'''
- try:
- __import__(name)
- return sys.modules[name]
- except ImportError as error:
- message = error.args[0] if error.args else ''
- words = frozenset(re_words.findall(message))
- parts = name.split('.')
- if not any('.'.join(parts[i:]) in words for i in range(len(parts))):
- raise
+ __import__(name)
+ return sys.modules.get(name, None)
def random_string(size, sample=tuple(map(chr, range(256)))):
diff --git a/setup.cfg b/setup.cfg
index 66043ce..73203c0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -12,7 +12,7 @@ ignore = E123,E126,E121,W504
[flake8]
ignore = E123,E126,E121,W504
-max-complexity = 5
+max-complexity = 7
select = F,C
show-source = True
--
GitLab
From 65e67eab9c54a8c2a37e2affda6c1571dacbebea Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Mon, 27 May 2019 22:30:46 +0100
Subject: [PATCH 069/171] reduce complexity
---
browsepy/plugin/file_actions/__init__.py | 67 ++++++++-------------
browsepy/plugin/file_actions/exceptions.py | 2 +-
browsepy/plugin/file_actions/utils.py | 17 +++++-
browsepy/transform/htmlcompress.py | 69 ++++++++++++----------
4 files changed, 78 insertions(+), 77 deletions(-)
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index 0a1e68a..810aced 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -1,7 +1,5 @@
# -*- coding: UTF-8 -*-
-import os
-import os.path
import functools
from flask import Blueprint, render_template, request, redirect, url_for, \
@@ -9,8 +7,8 @@ from flask import Blueprint, render_template, request, redirect, url_for, \
from werkzeug.exceptions import NotFound
from browsepy import get_cookie_browse_sorting, browse_sortkey_reverse
-from browsepy.file import Node, abspath_to_urlpath, secure_filename, \
- current_restricted_chars, common_path_separators
+from browsepy.file import Node, abspath_to_urlpath, current_restricted_chars, \
+ common_path_separators
from browsepy.compat import re_escape, FileNotFoundError
from browsepy.exceptions import OutsideDirectoryBase
from browsepy.utils import ppath
@@ -19,8 +17,6 @@ from browsepy.stream import stream_template
from .exceptions import FileActionsException, \
InvalidClipboardItemsError, \
InvalidClipboardModeError, \
- InvalidDirnameError, \
- DirectoryCreationError, \
InvalidClipboardSizeError
from . import utils
@@ -51,30 +47,15 @@ def create_directory(path):
if not directory.is_directory or not directory.can_upload:
return NotFound()
- if request.method == 'GET':
- return render_template(
- 'create_directory.file_actions.html',
- file=directory,
- re_basename=re_basename,
- )
-
- basename = request.form['name']
- if secure_filename(basename) != basename or not basename:
- raise InvalidDirnameError(
- path=directory.path,
- name=basename,
- )
-
- try:
- os.mkdir(os.path.join(directory.path, basename))
- except BaseException as e:
- raise DirectoryCreationError.from_exception(
- e,
- path=directory.path,
- name=basename
- )
+ if request.method == 'POST':
+ utils.mkdir(directory.path, request.form['name'])
+ return redirect(url_for('browse', path=directory.urlpath))
- return redirect(url_for('browse', path=directory.urlpath))
+ return render_template(
+ 'create_directory.file_actions.html',
+ file=directory,
+ re_basename=re_basename,
+ )
@actions.route('/selection', methods=('GET', 'POST'), defaults={'path': ''})
@@ -92,25 +73,23 @@ def selection(path):
return NotFound()
if request.method == 'POST':
- action_fmt = 'action-{}'.format
- mode = None
- for action in ('cut', 'copy'):
- if request.form.get(action_fmt(action)):
- mode = action
- break
+ mode = (
+ 'cut' if request.form.get('action-cut') else
+ 'copy' if request.form.get('action-copy') else
+ None
+ )
clipboard = request.form.getlist('path')
- if mode in ('cut', 'copy'):
- session['clipboard:mode'] = mode
- session['clipboard:items'] = clipboard
- return redirect(url_for('browse', path=directory.urlpath))
+ if mode is None:
+ raise InvalidClipboardModeError(
+ path=directory.path,
+ clipboard=clipboard,
+ )
- raise InvalidClipboardModeError(
- path=directory.path,
- mode=mode,
- clipboard=clipboard,
- )
+ session['clipboard:mode'] = mode
+ session['clipboard:items'] = clipboard
+ return redirect(url_for('browse', path=directory.urlpath))
return stream_template(
'selection.file_actions.html',
diff --git a/browsepy/plugin/file_actions/exceptions.py b/browsepy/plugin/file_actions/exceptions.py
index aaa0285..8a00a34 100644
--- a/browsepy/plugin/file_actions/exceptions.py
+++ b/browsepy/plugin/file_actions/exceptions.py
@@ -138,7 +138,7 @@ class InvalidClipboardModeError(ClipboardException):
:property clipboard: :class Clipboard: instance
'''
code = 'clipboard-invalid-mode'
- template = 'Clipboard mode {0.path!r} is not valid.'
+ template = 'Clipboard mode {0.mode!r} is not valid.'
def __init__(self, message=None, path=None, mode=None, clipboard=None):
supa = super(InvalidClipboardModeError, self)
diff --git a/browsepy/plugin/file_actions/utils.py b/browsepy/plugin/file_actions/utils.py
index 28a5772..f7d91c1 100644
--- a/browsepy/plugin/file_actions/utils.py
+++ b/browsepy/plugin/file_actions/utils.py
@@ -5,10 +5,13 @@ import os.path
import shutil
import functools
-from browsepy.file import Node
+from browsepy.file import Node, secure_filename
from browsepy.compat import map
-from .exceptions import InvalidClipboardModeError, InvalidEmptyClipboardError
+from .exceptions import InvalidClipboardModeError, \
+ InvalidEmptyClipboardError,\
+ InvalidDirnameError, \
+ DirectoryCreationError
def copy(target, node, join_fnc=os.path.join):
@@ -72,3 +75,13 @@ def paste(target, mode, clipboard):
except BaseException as e:
issues.append((node, e))
return success, issues
+
+
+def mkdir(path, name):
+ if secure_filename(name) != name or not name:
+ raise InvalidDirnameError(path=path, name=name)
+
+ try:
+ os.mkdir(os.path.join(path, name))
+ except BaseException as e:
+ raise DirectoryCreationError.from_exception(e, path=path, name=name)
diff --git a/browsepy/transform/htmlcompress.py b/browsepy/transform/htmlcompress.py
index ccd05de..f737d85 100755
--- a/browsepy/transform/htmlcompress.py
+++ b/browsepy/transform/htmlcompress.py
@@ -7,7 +7,39 @@ import jinja2.lexer
from . import StateMachine
-class SGMLCompressContext(StateMachine):
+class CompressContext(StateMachine):
+ token_class = jinja2.lexer.Token
+ block_tokens = {
+ 'variable_begin': 'variable_end',
+ 'block_begin': 'block_end'
+ }
+ skip_until_token = None
+ lineno = 0
+
+ def feed(self, token):
+ if self.skip_until_token:
+ yield token
+ if token.type == self.skip_until_token:
+ self.skip_until_token = None
+ elif token.type != 'data':
+ for ftoken in self.finish():
+ yield ftoken
+ yield token
+ self.skip_until_token = self.block_tokens.get(token.type)
+ else:
+ if not self.pending:
+ self.lineno = token.lineno
+
+ for data in super(CompressContext, self).feed(token.value):
+ yield self.token_class(self.lineno, 'data', data)
+ self.lineno = token.lineno
+
+ def finish(self):
+ for data in super(CompressContext, self).finish():
+ yield self.token_class(self.lineno, 'data', data)
+
+
+class SGMLCompressContext(CompressContext):
re_whitespace = re.compile('[ \\t\\r\\n]+')
block_tags = {} # block content will be treated as literal text
jumps = { # state machine jumps
@@ -70,36 +102,13 @@ class HTMLCompressContext(SGMLCompressContext):
class HTMLCompress(jinja2.ext.Extension):
context_class = HTMLCompressContext
- token_class = jinja2.lexer.Token
- block_tokens = {
- 'variable_begin': 'variable_end',
- 'block_begin': 'block_end'
- }
def filter_stream(self, stream):
transform = self.context_class()
- lineno = 0
- skip_until_token = None
+
for token in stream:
- if skip_until_token:
- yield token
- if token.type == skip_until_token:
- skip_until_token = None
- continue
-
- if token.type != 'data':
- for data in transform.finish():
- yield self.token_class(lineno, 'data', data)
- yield token
- skip_until_token = self.block_tokens.get(token.type)
- continue
-
- if not transform.pending:
- lineno = token.lineno
-
- for data in transform.feed(token.value):
- yield self.token_class(lineno, 'data', data)
- lineno = token.lineno
-
- for data in transform.finish():
- yield self.token_class(lineno, 'data', data)
+ for compressed in transform.feed(token):
+ yield compressed
+
+ for compressed in transform.finish():
+ yield compressed
--
GitLab
From 0b129f1f37bf676c1688eb5507f3971efbd19e79 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 16 Jun 2019 16:52:50 +0200
Subject: [PATCH 070/171] add download folder head button, fix directory stream
premature retrieve abort
---
.python-version | 2 +-
browsepy/file.py | 26 ++++++++++++++++++++------
browsepy/stream.py | 2 +-
3 files changed, 22 insertions(+), 8 deletions(-)
diff --git a/.python-version b/.python-version
index 0b2eb36..c1e43e6 100644
--- a/.python-version
+++ b/.python-version
@@ -1 +1 @@
-3.7.2
+3.7.3
diff --git a/browsepy/file.py b/browsepy/file.py
index f214f26..70abb3c 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -390,15 +390,22 @@ class File(Node):
)
]
if self.can_download:
- widgets.append(
+ widgets.extend((
self.plugin_manager.create_widget(
'entry-actions',
'button',
file=self,
css='download',
endpoint='download_file'
- )
- )
+ ),
+ self.plugin_manager.create_widget(
+ 'header',
+ 'button',
+ file=self,
+ text='Download file',
+ endpoint='download_file'
+ ),
+ ))
return widgets + super(File, self).widgets
@cached_property
@@ -554,15 +561,22 @@ class Directory(Node):
)
))
if self.can_download:
- widgets.append(
+ widgets.extend((
self.plugin_manager.create_widget(
'entry-actions',
'button',
file=self,
css='download',
endpoint='download_directory'
- )
- )
+ ),
+ self.plugin_manager.create_widget(
+ 'header',
+ 'button',
+ file=self,
+ text='Download all',
+ endpoint='download_directory'
+ ),
+ ))
return widgets + super(Directory, self).widgets
@cached_property
diff --git a/browsepy/stream.py b/browsepy/stream.py
index 20d4878..c42214e 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -71,7 +71,7 @@ class BlockingPipe(object):
'''
with self._rlock:
- if self.closed:
+ if self.closed and self._pipe.empty():
raise self.abort_exception()
data = self._pipe.get()
if data is None:
--
GitLab
From f65688cacbd48a674ad449b03683850670ba9b2d Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 16 Jun 2019 20:33:07 +0200
Subject: [PATCH 071/171] reorder download directory header button, fixes
---
browsepy/file.py | 34 ++++++++++++------------
browsepy/manager.py | 16 +++++------
browsepy/plugin/file_actions/__init__.py | 12 +++++----
browsepy/plugin/file_actions/utils.py | 4 ++-
browsepy/templates/remove.html | 2 +-
5 files changed, 36 insertions(+), 32 deletions(-)
diff --git a/browsepy/file.py b/browsepy/file.py
index 70abb3c..d020226 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -536,6 +536,23 @@ class Directory(Node):
endpoint='browse'
)
]
+ if self.can_download:
+ widgets.extend((
+ self.plugin_manager.create_widget(
+ 'entry-actions',
+ 'button',
+ file=self,
+ css='download',
+ endpoint='download_directory'
+ ),
+ self.plugin_manager.create_widget(
+ 'header',
+ 'button',
+ file=self,
+ text='Download all',
+ endpoint='download_directory'
+ ),
+ ))
if self.can_upload:
widgets.extend((
self.plugin_manager.create_widget(
@@ -560,23 +577,6 @@ class Directory(Node):
endpoint='upload'
)
))
- if self.can_download:
- widgets.extend((
- self.plugin_manager.create_widget(
- 'entry-actions',
- 'button',
- file=self,
- css='download',
- endpoint='download_directory'
- ),
- self.plugin_manager.create_widget(
- 'header',
- 'button',
- file=self,
- text='Download all',
- endpoint='download_directory'
- ),
- ))
return widgets + super(Directory, self).widgets
@cached_property
diff --git a/browsepy/manager.py b/browsepy/manager.py
index 42fc063..f35de48 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -271,16 +271,16 @@ class ExcludePluginManager(PluginManagerBase):
def check_excluded(self, path, request=flask.request):
'''
Check if given path is excluded.
+
+ :type path: str
+ :type request: flask.Request
'''
- if not request:
- request = utils.dummy_context()
- with request:
- exclude_fnc = self.app.config.get('EXCLUDE_FNC')
- if exclude_fnc and exclude_fnc(path):
+ exclude_fnc = self.app.config.get('EXCLUDE_FNC')
+ if exclude_fnc and exclude_fnc(path):
+ return True
+ for fnc in self._exclude_functions:
+ if fnc(path):
return True
- for fnc in self._exclude_functions:
- if fnc(path):
- return True
return False
def clear(self):
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index 810aced..fc6ca4f 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -48,8 +48,9 @@ def create_directory(path):
return NotFound()
if request.method == 'POST':
- utils.mkdir(directory.path, request.form['name'])
- return redirect(url_for('browse', path=directory.urlpath))
+ path = utils.mkdir(directory.path, request.form['name'])
+ base = current_app.config['DIRECTORY_BASE']
+ return redirect(url_for('browse', path=abspath_to_urlpath(path, base)))
return render_template(
'create_directory.file_actions.html',
@@ -196,10 +197,11 @@ def detect_selection(directory):
current_app.config.get('DIRECTORY_UPLOAD')
-def excluded_clipboard(manager, path):
+def excluded_clipboard(app, path):
+ # TODO: get this working outside requests (for tarfile streaming)
if not getattr(g, 'file_actions_paste', False) and \
session.get('clipboard:mode') == 'cut':
- base = manager.app.config['DIRECTORY_BASE']
+ base = app.config['DIRECTORY_BASE']
clipboard = session.get('clipboard:items', ())
return abspath_to_urlpath(path, base) in clipboard
return False
@@ -213,7 +215,7 @@ def register_plugin(manager):
:type manager: browsepy.manager.PluginManager
'''
manager.register_exclude_function(
- functools.partial(excluded_clipboard, manager)
+ functools.partial(excluded_clipboard, manager.app)
)
manager.register_blueprint(actions)
manager.register_widget(
diff --git a/browsepy/plugin/file_actions/utils.py b/browsepy/plugin/file_actions/utils.py
index f7d91c1..694af61 100644
--- a/browsepy/plugin/file_actions/utils.py
+++ b/browsepy/plugin/file_actions/utils.py
@@ -82,6 +82,8 @@ def mkdir(path, name):
raise InvalidDirnameError(path=path, name=name)
try:
- os.mkdir(os.path.join(path, name))
+ path = os.path.join(path, name)
+ os.mkdir(path)
+ return path
except BaseException as e:
raise DirectoryCreationError.from_exception(e, path=path, name=name)
diff --git a/browsepy/templates/remove.html b/browsepy/templates/remove.html
index ee81315..8f953e5 100644
--- a/browsepy/templates/remove.html
+++ b/browsepy/templates/remove.html
@@ -6,7 +6,7 @@
{% endblock %}
{% block content %}
-
Do you really want to remove this file?
+
Do you really want to remove this {% if file.is_directory %}directory{% else %}file{% endif %}?
Cancel
--
GitLab
From 5ff6003d7a950c8374ae71a4e7fc86efe43e1fa7 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Tue, 2 Jul 2019 18:37:03 +0100
Subject: [PATCH 072/171] simplify stream logic via custom queue implementation
---
.vscode/settings.json | 11 ++-
README.rst | 2 +-
browsepy/stream.py | 162 ++++++++++++++++++++++--------------------
3 files changed, 94 insertions(+), 81 deletions(-)
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 1f7909a..6b2ba5b 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -14,5 +14,14 @@
"python.linting.pep8Path": "pycodestyle",
"python.linting.pylintEnabled": false,
"python.jediEnabled": true,
- "python.pythonPath": "env/bin/python"
+ "python.pythonPath": "env/bin/python",
+ "python.testing.unittestArgs": [
+ "-v",
+ "-s",
+ "./browsepy/tests",
+ "-p",
+ "test_*.py"
+ ],
+ "python.testing.pytestEnabled": false,
+ "python.testing.nosetestsEnabled": false
}
diff --git a/README.rst b/README.rst
index b6b2330..734c610 100644
--- a/README.rst
+++ b/README.rst
@@ -105,7 +105,7 @@ It's on `pypi` so...
pip install browsepy
-You can get the development version from our `github repository`.
+You can get the same version from our `github repository`.
.. _github repository: https://github.com/ergoithz/browsepy
diff --git a/browsepy/stream.py b/browsepy/stream.py
index c42214e..024b5b0 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -11,36 +11,56 @@ import flask
from . import compat
-class BlockingPipeAbort(RuntimeError):
+class ByteQueue(compat.Queue):
'''
- Exception used internally by :class:`BlockingPipe`'s default
- implementation.
+ Small synchronized queue backed with bytearray, with an additional
+ finish method with turns the queue into non-blocking.
+
+ Once bytequeue became finished, all :method:`get` calls return empty bytes,
+ and :method:`put` calls raise an exception.
'''
- pass
+ def _init(self, maxsize):
+ self.queue = bytearray()
+ self.finished = False
+
+ def _qsize(self):
+ return -1 if self.finished and not self.queue else len(self.queue)
+
+ def _put(self, item):
+ if self.finished:
+ raise RuntimeError('PUT operation on finished byte queue')
+ self.queue.extend(item)
+
+ def _get(self):
+ num = self.maxsize
+ data, self.queue = bytes(self.queue[:num]), bytearray(self.queue[num:])
+ return data
+
+ def finish(self):
+ if not self.finished:
+ with self.not_full:
+ self.not_full.notify_all()
+
+ with self.not_empty:
+ self.not_empty.notify_all()
+
+ self.finished = True
class BlockingPipe(object):
'''
- Minimal pipe class with `write`, `retrieve` and `close` blocking methods.
-
- This class implementation assumes that :attr:`pipe_class` (set as
- class:`queue.Queue` in current implementation) instances has both `put`
- and `get blocking methods.
+ Minimal pipe class with `write` and `read` blocking methods.
Due its blocking implementation, this class uses :module:`threading`.
This class exposes :method:`write` for :class:`tarfile.TarFile`
`fileobj` compatibility.
- '''''
- lock_class = threading.Lock
- pipe_class = functools.partial(compat.Queue, maxsize=1)
- abort_exception = BlockingPipeAbort
-
- def __init__(self):
- self._pipe = self.pipe_class()
- self._wlock = self.lock_class()
- self._rlock = self.lock_class()
- self.closed = False
+ '''
+ pipe_class = ByteQueue
+
+ def __init__(self, buffsize=None):
+ self._pipe = self.pipe_class(buffsize)
+ self._raises = None
def write(self, data):
'''
@@ -53,14 +73,14 @@ class BlockingPipe(object):
:rtype: int
:raises WriteAbort: if already closed or closed while blocking
'''
+ if self._raises:
+ raise self._raises
+
+ self._pipe.put(data, timeout=1)
- with self._wlock:
- if self.closed:
- raise self.abort_exception()
- self._pipe.put(data)
- return len(data)
+ return len(data)
- def retrieve(self):
+ def read(self):
'''
Get chunk of data from pipe.
This method blocks if pipe is empty.
@@ -69,49 +89,34 @@ class BlockingPipe(object):
:rtype: bytes
:raises WriteAbort: if already closed or closed while blocking
'''
+ if self._raises:
+ raise self._raises
+ return self._pipe.get()
- with self._rlock:
- if self.closed and self._pipe.empty():
- raise self.abort_exception()
- data = self._pipe.get()
- if data is None:
- raise self.abort_exception()
- return data
-
- def __del__(self):
+ def finish(self):
'''
- Call :method:`BlockingPipe.close`.
+ Notify queue that we're finished, so it became non-blocking returning
+ empty bytes.
'''
- self.close()
+ self._pipe.finish()
- def close(self):
+ def abort(self, exception):
'''
- Closes, so any blocked and future writes or retrieves will raise
- :attr:`abort_exception` instances.
- '''
-
- def blocked():
- '''
- NOOP lock release function for non-owned locks.
- '''
- pass
+ Make further writes to raise an exception.
- if not self.closed:
- self.closed = True
-
- releasing = writing, reading = [
- lock.release if lock.acquire(False) else blocked
- for lock in (self._wlock, self._rlock)
- ]
-
- if writing is blocked and reading is not blocked:
- self._pipe.get()
+ :param exception: exception to raise on write
+ :type exception: Exception
+ '''
+ self._raises = exception
+ self.finish()
- if reading is blocked and writing is not blocked:
- self._pipe.put(None)
- for release in releasing:
- release()
+class StreamError(RuntimeError):
+ '''
+ Exception used internally by :class:`TarFileStream`'s default
+ implementation to stop tarfile compression.
+ '''
+ pass
class TarFileStream(compat.Iterator):
@@ -129,7 +134,7 @@ class TarFileStream(compat.Iterator):
'''
pipe_class = BlockingPipe
- abort_exception = BlockingPipe.abort_exception
+ abort_exception = StreamError
thread_class = threading.Thread
tarfile_class = tarfile.open
@@ -171,6 +176,7 @@ class TarFileStream(compat.Iterator):
'''
self.path = path
self.exclude = exclude
+ self.closed = False
self._started = False
self._buffsize = buffsize
@@ -178,7 +184,7 @@ class TarFileStream(compat.Iterator):
self._compress = compress if compress and buffsize > 15 else None
self._mode, self._extension = self.compresion_modes[self._compress]
- self._pipe = self.pipe_class()
+ self._pipe = self.pipe_class(buffsize)
self._th = self.thread_class(target=self._fill)
def _fill(self):
@@ -201,17 +207,17 @@ class TarFileStream(compat.Iterator):
tarfile = self.tarfile_class(
fileobj=self._pipe,
mode='w|{}'.format(self._mode),
- bufsize=self._buffsize
+ bufsize=self._buffsize,
+ encoding='utf-8',
)
-
try:
- tarfile.add(self.path, "", filter=infofilter if exclude else None)
+ tarfile.add(self.path, '', filter=infofilter if exclude else None)
tarfile.close() # force stream flush (may raise)
- except self.abort_exception:
- # expected exception when pipe is closed prematurely
+ self._pipe.finish()
+ except self.abort_exception: # probably closed prematurely
tarfile.close() # free fd
- finally:
- self.close()
+ except Exception as e:
+ self._pipe.abort(e)
def __next__(self):
'''
@@ -226,22 +232,20 @@ class TarFileStream(compat.Iterator):
self._started = True
self._th.start()
- try:
- return self._pipe.retrieve()
- except self.abort_exception:
+ data = self._pipe.read()
+ if not data:
raise StopIteration()
+ return data
def close(self):
'''
Closes tarfile pipe and stops further processing.
'''
- self._pipe.close()
-
- def __del__(self):
- '''
- Call :method:`TarFileStream.close`.
- '''
- self.close()
+ if not self.closed:
+ self.closed = True
+ if self._started:
+ self._pipe.abort(self.abort_exception())
+ self._th.join()
def stream_template(template_name, **context):
--
GitLab
From 4fa96f3e7ad6bf8dad7f3bb8f0d507662d772a35 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Tue, 2 Jul 2019 22:27:33 +0100
Subject: [PATCH 073/171] simplify tarfile stream implementation
---
browsepy/stream.py | 160 ++++++++++++++++++++-------------------------
1 file changed, 70 insertions(+), 90 deletions(-)
diff --git a/browsepy/stream.py b/browsepy/stream.py
index 024b5b0..1eb8749 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -4,7 +4,6 @@ import os
import os.path
import tarfile
import threading
-import functools
import flask
@@ -13,105 +12,55 @@ from . import compat
class ByteQueue(compat.Queue):
'''
- Small synchronized queue backed with bytearray, with an additional
- finish method with turns the queue into non-blocking.
+ Small synchronized queue storing bytes, with an additional finish method
+ with turns the queue :method:`get` into non-blocking (returns empty bytes).
- Once bytequeue became finished, all :method:`get` calls return empty bytes,
- and :method:`put` calls raise an exception.
+ On a finished queue all :method:`put` will raise Full exceptions,
+ regardless of the parameters given.
'''
def _init(self, maxsize):
- self.queue = bytearray()
+ self.queue = []
+ self.bytes = 0
self.finished = False
+ self.closed = False
def _qsize(self):
- return -1 if self.finished and not self.queue else len(self.queue)
+ return -1 if self.finished else self.bytes
def _put(self, item):
if self.finished:
- raise RuntimeError('PUT operation on finished byte queue')
- self.queue.extend(item)
+ raise compat.Full
+ self.queue.append(item)
+ self.bytes += len(item)
def _get(self):
- num = self.maxsize
- data, self.queue = bytes(self.queue[:num]), bytearray(self.queue[num:])
+ size = self.maxsize
+ data = b''.join(self.queue)
+ data, tail = data[:size], data[size:]
+ self.queue[:] = (tail,)
+ self.bytes = len(tail)
return data
- def finish(self):
- if not self.finished:
- with self.not_full:
- self.not_full.notify_all()
-
- with self.not_empty:
- self.not_empty.notify_all()
-
- self.finished = True
-
-
-class BlockingPipe(object):
- '''
- Minimal pipe class with `write` and `read` blocking methods.
-
- Due its blocking implementation, this class uses :module:`threading`.
-
- This class exposes :method:`write` for :class:`tarfile.TarFile`
- `fileobj` compatibility.
- '''
- pipe_class = ByteQueue
-
- def __init__(self, buffsize=None):
- self._pipe = self.pipe_class(buffsize)
- self._raises = None
-
- def write(self, data):
+ def qsize(self):
'''
- Put chunk of data onto pipe.
- This method blocks if pipe is already full.
-
- :param data: bytes to write to pipe
- :type data: bytes
- :returns: number of bytes written
- :rtype: int
- :raises WriteAbort: if already closed or closed while blocking
+ Return the number of bytes in the queue.
'''
- if self._raises:
- raise self._raises
-
- self._pipe.put(data, timeout=1)
-
- return len(data)
-
- def read(self):
- '''
- Get chunk of data from pipe.
- This method blocks if pipe is empty.
-
- :returns: data chunk
- :rtype: bytes
- :raises WriteAbort: if already closed or closed while blocking
- '''
- if self._raises:
- raise self._raises
- return self._pipe.get()
+ with self.mutex:
+ return self.bytes
def finish(self):
'''
- Notify queue that we're finished, so it became non-blocking returning
- empty bytes.
+ Turn queue into finished mode: :method:`get` becomes non-blocking
+ and returning empty bytes if empty, and :method:`put` raising
+ :class:`queue.Full` exceptions unconditionally.
'''
- self._pipe.finish()
+ self.finished = True
- def abort(self, exception):
- '''
- Make further writes to raise an exception.
-
- :param exception: exception to raise on write
- :type exception: Exception
- '''
- self._raises = exception
- self.finish()
+ with self.not_full:
+ self.not_empty.notify()
-class StreamError(RuntimeError):
+class WriteAbort(Exception):
'''
Exception used internally by :class:`TarFileStream`'s default
implementation to stop tarfile compression.
@@ -133,8 +82,8 @@ class TarFileStream(compat.Iterator):
This class uses :module:`threading` for offloading.
'''
- pipe_class = BlockingPipe
- abort_exception = StreamError
+ queue_class = ByteQueue
+ abort_exception = WriteAbort
thread_class = threading.Thread
tarfile_class = tarfile.open
@@ -184,8 +133,9 @@ class TarFileStream(compat.Iterator):
self._compress = compress if compress and buffsize > 15 else None
self._mode, self._extension = self.compresion_modes[self._compress]
- self._pipe = self.pipe_class(buffsize)
+ self._queue = self.queue_class(buffsize)
self._th = self.thread_class(target=self._fill)
+ self._th_exc = None
def _fill(self):
'''
@@ -205,19 +155,20 @@ class TarFileStream(compat.Iterator):
return None if exclude(path_join(path, info.name)) else info
tarfile = self.tarfile_class(
- fileobj=self._pipe,
+ fileobj=self,
mode='w|{}'.format(self._mode),
bufsize=self._buffsize,
encoding='utf-8',
)
try:
tarfile.add(self.path, '', filter=infofilter if exclude else None)
- tarfile.close() # force stream flush (may raise)
- self._pipe.finish()
- except self.abort_exception: # probably closed prematurely
- tarfile.close() # free fd
+ tarfile.close()
+ except self.abort_exception:
+ tarfile.close()
except Exception as e:
- self._pipe.abort(e)
+ self._th_exc = e
+ finally:
+ self._queue.finish()
def __next__(self):
'''
@@ -228,24 +179,53 @@ class TarFileStream(compat.Iterator):
:returns: tarfile data as bytes
:rtype: bytes
'''
+ if self.closed:
+ raise StopIteration()
+
if not self._started:
self._started = True
self._th.start()
- data = self._pipe.read()
+ data = self._queue.get()
if not data:
raise StopIteration()
+
return data
+ def write(self, data):
+ '''
+ Put chunk of data into data queue, used on the tarfile thread.
+
+ This method blocks when pipe is already, applying backpressure to
+ writers.
+
+ :param data: bytes to write to pipe
+ :type data: bytes
+ :returns: number of bytes written
+ :rtype: int
+ :raises WriteAbort: if already closed or closed while blocking
+ '''
+ if self.closed:
+ raise self.abort_exception()
+
+ try:
+ self._queue.put(data)
+ except compat.Full:
+ raise self.abort_exception()
+
+ return len(data)
+
def close(self):
'''
Closes tarfile pipe and stops further processing.
'''
if not self.closed:
self.closed = True
- if self._started:
- self._pipe.abort(self.abort_exception())
+ self._queue.finish()
+ if self._started and self._th.is_alive():
self._th.join()
+ if self._th_exc:
+ raise self._th_exc
def stream_template(template_name, **context):
--
GitLab
From 6c0392f747d9a82a1ab17febe660dd3152f228ff Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Tue, 2 Jul 2019 22:39:34 +0100
Subject: [PATCH 074/171] pass config to directory stream
---
browsepy/__init__.py | 2 ++
browsepy/file.py | 4 +++-
browsepy/stream.py | 5 ++++-
3 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index 5dcf37c..2f8a6f7 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -38,6 +38,8 @@ app.config.update(
DIRECTORY_REMOVE=None,
DIRECTORY_UPLOAD=None,
DIRECTORY_TAR_BUFFSIZE=262144,
+ DIRECTORY_TAR_COMPRESSION='gzip',
+ DIRECTORY_TAR_COMPRESSLEVEL=1,
DIRECTORY_DOWNLOADABLE=True,
USE_BINARY_MULTIPLES=True,
PLUGIN_MODULES=[],
diff --git a/browsepy/file.py b/browsepy/file.py
index d020226..b465500 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -669,7 +669,9 @@ class Directory(Node):
functools.partial(
self.plugin_manager.check_excluded,
request=utils.solve_local(flask.request),
- )
+ ),
+ self.app.config.get('DIRECTORY_TAR_COMPRESS', 'gzip'),
+ self.app.config.get('DIRECTORY_TAR_COMPRESSLEVEL', 1),
)
return self.app.response_class(
flask.stream_with_context(stream),
diff --git a/browsepy/stream.py b/browsepy/stream.py
index 1eb8749..c2c64ab 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -109,7 +109,8 @@ class TarFileStream(compat.Iterator):
'''
return self._compress
- def __init__(self, path, buffsize=10240, exclude=None, compress='gzip'):
+ def __init__(self, path, buffsize=10240, exclude=None, compress='gzip',
+ compresslevel=1):
'''
Initialize thread and class (thread is not started until interated.)
Note that compress parameter will be ignored if buffsize is below 16.
@@ -122,6 +123,8 @@ class TarFileStream(compat.Iterator):
:type exclude: callable
:param compress: compression method ('gz', 'bz2', 'xz' or None)
:type compress: str or None
+ :param compresslevel: compression level [1-9] defaults to 1 (fastest)
+ :type compresslevel: int
'''
self.path = path
self.exclude = exclude
--
GitLab
From b6bd7caf86911680754c87fac6caa70b1b8cce7e Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 3 Jul 2019 17:26:23 +0100
Subject: [PATCH 075/171] better handling of request on tarfile stream
---
browsepy/__init__.py | 5 ++--
browsepy/file.py | 5 ++--
browsepy/manager.py | 6 +++--
browsepy/stream.py | 46 +++++++++++++++++++++++------------
browsepy/tests/test_stream.py | 28 ++++++++++++++++++++-
5 files changed, 65 insertions(+), 25 deletions(-)
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index 2f8a6f7..c469bb5 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -145,9 +145,8 @@ def sort(property, path):
if not directory.is_directory or directory.is_excluded:
return NotFound()
- if request:
- session['browse:sort'] = \
- [(path, property)] + session.get('browse:sort', [])
+ session['browse:sort'] = \
+ [(path, property)] + session.get('browse:sort', [])
return redirect(url_for(".browse", path=directory.urlpath))
diff --git a/browsepy/file.py b/browsepy/file.py
index b465500..563b49c 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -666,9 +666,8 @@ class Directory(Node):
stream = self.stream_class(
self.path,
self.app.config.get('DIRECTORY_TAR_BUFFSIZE', 10240),
- functools.partial(
- self.plugin_manager.check_excluded,
- request=utils.solve_local(flask.request),
+ flask.copy_current_request_context(
+ self.plugin_manager.check_excluded
),
self.app.config.get('DIRECTORY_TAR_COMPRESS', 'gzip'),
self.app.config.get('DIRECTORY_TAR_COMPRESSLEVEL', 1),
diff --git a/browsepy/manager.py b/browsepy/manager.py
index f35de48..d42bcf3 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -268,12 +268,14 @@ class ExcludePluginManager(PluginManagerBase):
'''
self._exclude_functions.add(exclude_fnc)
- def check_excluded(self, path, request=flask.request):
+ def check_excluded(self, path):
'''
Check if given path is excluded.
+ :param path: absolute path to check against config and plugins
:type path: str
- :type request: flask.Request
+ :return: wether if path should be excluded or not
+ :rtype: bool
'''
exclude_fnc = self.app.config.get('EXCLUDE_FNC')
if exclude_fnc and exclude_fnc(path):
diff --git a/browsepy/stream.py b/browsepy/stream.py
index c2c64ab..86af22a 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -140,6 +140,28 @@ class TarFileStream(compat.Iterator):
self._th = self.thread_class(target=self._fill)
self._th_exc = None
+ @property
+ def _infofilter(self):
+ '''
+ TarInfo filtering function based on :attr:`exclude`.
+ '''
+ path = self.path
+ path_join = os.path.join
+ exclude = self.exclude
+
+ def infofilter(info):
+ '''
+ Filter TarInfo objects for TarFile.
+
+ :param info:
+ :type info: tarfile.TarInfo
+ :return: infofile or None if file must be excluded
+ :rtype: tarfile.TarInfo or None
+ '''
+ return None if exclude(path_join(path, info.name)) else info
+
+ return infofilter if exclude else None
+
def _fill(self):
'''
Writes data on internal tarfile instance, which writes to current
@@ -150,24 +172,16 @@ class TarFileStream(compat.Iterator):
This method is called automatically, on a thread, on initialization,
so there is little need to call it manually.
'''
- exclude = self.exclude
- path_join = os.path.join
- path = self.path
-
- def infofilter(info):
- return None if exclude(path_join(path, info.name)) else info
-
- tarfile = self.tarfile_class(
- fileobj=self,
- mode='w|{}'.format(self._mode),
- bufsize=self._buffsize,
- encoding='utf-8',
- )
try:
- tarfile.add(self.path, '', filter=infofilter if exclude else None)
- tarfile.close()
+ with self.tarfile_class(
+ fileobj=self,
+ mode='w|{}'.format(self._mode),
+ bufsize=self._buffsize,
+ encoding='utf-8',
+ ) as tarfile:
+ tarfile.add(self.path, '', filter=self._infofilter)
except self.abort_exception:
- tarfile.close()
+ pass
except Exception as e:
self._th_exc = e
finally:
diff --git a/browsepy/tests/test_stream.py b/browsepy/tests/test_stream.py
index cbc5f39..3935b00 100644
--- a/browsepy/tests/test_stream.py
+++ b/browsepy/tests/test_stream.py
@@ -5,6 +5,7 @@ import codecs
import unittest
import tempfile
import shutil
+import time
import browsepy.stream
@@ -27,12 +28,37 @@ class StreamTest(unittest.TestCase):
self.randfile()
self.randfile()
stream = self.module.TarFileStream(self.base, buffsize=5)
- self.assertTrue(next(stream))
+
+ self.assertFalse(stream._queue.qsize()) # not yet compressing
+ self.assertEqual(len(next(stream)), 5)
+
+ while not stream._queue.qsize():
+ time.sleep(0.1)
+
+ self.assertGreater(stream._queue.qsize(), 4)
+ self.assertLess(stream._queue.qsize(), 10)
with self.assertRaises(StopIteration):
while True:
next(stream)
+ def test_exception(self):
+ class MyException(Exception):
+ pass
+
+ class BrokenQueue(self.module.ByteQueue):
+ def put(self, data):
+ raise MyException()
+
+ stream = self.module.TarFileStream(self.base, buffsize=5)
+ stream._queue = BrokenQueue()
+
+ with self.assertRaises(StopIteration):
+ next(stream)
+
+ with self.assertRaises(MyException):
+ stream.close()
+
def test_close(self):
self.randfile()
stream = self.module.TarFileStream(self.base, buffsize=16)
--
GitLab
From dce6bd48ac4e9c1708f558a35331abad0574d551 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 3 Jul 2019 19:05:39 +0100
Subject: [PATCH 076/171] doc updates
---
browsepy/stream.py | 10 ++++++++--
doc/conf.py | 18 ++++++++++++------
doc/stream.rst | 14 +++++++++++---
3 files changed, 31 insertions(+), 11 deletions(-)
diff --git a/browsepy/stream.py b/browsepy/stream.py
index 86af22a..a5a79b3 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -79,7 +79,12 @@ class TarFileStream(compat.Iterator):
Buffsize can be provided, it should be 512 multiple (the tar block size)
for and will be used as tarfile block size.
- This class uses :module:`threading` for offloading.
+ This class uses :module:`threading` for offloading, so if your exclude
+ function would not have access to any thread-local specific data.
+
+ If your exclude function requires accessing to :data:`flask.app`,
+ :data:`flask.g`, :data:`flask.request` or any other flask contexts,
+ ensure is wrapped with :func:`flask.copy_current_request_context`
'''
queue_class = ByteQueue
@@ -112,7 +117,8 @@ class TarFileStream(compat.Iterator):
def __init__(self, path, buffsize=10240, exclude=None, compress='gzip',
compresslevel=1):
'''
- Initialize thread and class (thread is not started until interated.)
+ Initialize thread and class (thread is not started until interation).
+
Note that compress parameter will be ignored if buffsize is below 16.
:param path: local path of directory whose content will be compressed.
diff --git a/doc/conf.py b/doc/conf.py
index 6835895..e565472 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -18,10 +18,16 @@
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
+import io
+import re
import sys
import time
+
sys.path.insert(0, os.path.abspath('..')) # noqa
-import browsepy.__meta__ as meta
+
+with io.open('../browsepy/__init__.py', 'rt', encoding='utf8') as f:
+ app_version = re.search(r'__version__ = \'(.*?)\'', f.read()).group(1)
+ assert isinstance(app_version, str)
# -- General configuration ------------------------------------------------
@@ -59,18 +65,18 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
-project = meta.app
-copyright = '%d, %s' % (time.gmtime()[0], meta.author_name)
-author = meta.author_name
+project = 'browsepy'
+copyright = '2019, Felipe A. Hernandez'
+author = 'Felipe A. Hernandez'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
-version = '{0}.{0}'.format(*meta.version.split('.'))
+version = '{0}.{0}'.format(*app_version.split('.'))
# The full version, including alpha/beta/rc tags.
-release = meta.version
+release = app_version
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/doc/stream.rst b/doc/stream.rst
index 29c38e0..13c40e0 100644
--- a/doc/stream.rst
+++ b/doc/stream.rst
@@ -5,13 +5,16 @@ Stream Module
.. currentmodule:: browsepy.stream
-This module provides a class for streaming directory tarballs. This is used
-by :meth:`browsepy.file.Directory.download` method.
+This module provides a utilities for streaming responses.
.. _tarfilestream-node:
TarFileStream
-----
+-------------
+
+Iterator class for streaming directory tarballs.
+
+This is currently used by :meth:`browsepy.file.Directory.download` method.
.. currentmodule:: browsepy.stream
@@ -19,3 +22,8 @@ TarFileStream
:members:
:inherited-members:
:undoc-members:
+
+Utility functions
+-----------------
+
+.. autofunction:: stream_template
--
GitLab
From 9e4a2b3ca941b4d22c6e9080d7012baa2b5220be Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 4 Jul 2019 22:26:35 +0100
Subject: [PATCH 077/171] fix docstrings, more documentation
---
CHANGELOG | 44 ++++
README.rst | 108 ++++------
browsepy/__init__.py | 8 +-
browsepy/appconfig.py | 16 +-
browsepy/compat.py | 71 +++----
browsepy/exceptions.py | 24 +--
browsepy/file.py | 224 ++++++++++-----------
browsepy/http.py | 12 +-
browsepy/manager.py | 173 ++++++++--------
browsepy/plugin/file_actions/__init__.py | 27 ++-
browsepy/plugin/file_actions/exceptions.py | 36 ++--
browsepy/plugin/file_actions/utils.py | 4 +-
browsepy/plugin/player/__init__.py | 8 +-
browsepy/plugin/player/playable.py | 4 +-
browsepy/stream.py | 60 +++---
browsepy/tests/deprecated/plugin/player.py | 4 +-
browsepy/tests/meta.py | 4 +-
browsepy/tests/test_code_quality.py | 5 +-
browsepy/tests/test_main.py | 42 ++--
browsepy/transform/__init__.py | 28 +--
browsepy/utils.py | 32 +--
browsepy/widget.py | 4 +-
doc/plugins.rst | 57 +++++-
doc/quickstart.rst | 39 ++--
24 files changed, 555 insertions(+), 479 deletions(-)
create mode 100644 CHANGELOG
diff --git a/CHANGELOG b/CHANGELOG
new file mode 100644
index 0000000..e38dc75
--- /dev/null
+++ b/CHANGELOG
@@ -0,0 +1,44 @@
+## [0.6.0] - Upcoming
+### Added
+- Plugin discovery.
+- New plugin `file-actions for copy/cut/paste and directory creation.
+- Smart cookie sessions.
+
+### Changes
+- Faster directory tarball streaming with more options.
+
+### Removed
+- Python 3.3 and 3.4 support have been dropped.
+
+## [0.5.6] - 2017-09-09
+### Added
+- Handle filesystem path size limits.
+
+### Changes
+- Player plugin honoring directory ordering.
+- Windows fixes
+
+## [0.5.3] - 2017-05-12
+### Added
+- New "exclude" option (--exclude and --exclude-from), accepting UNIX globs.
+- Windows support.
+
+### Changed
+- Fixes #12 (Windows compatibility)
+
+## [0.5.0] - 2016-11-24
+### Added
+- Protocol for plugin command line argument handling.
+- Player plugin playlist support (m3u, pls) and directory).
+- Customizable browser file sorting.
+- Multiple-file uploads.
+
+### Changed
+- New plugin and widget API, old one have been deprecated.
+- Use of optimized scandir (faster directory listing).
+- Browsepy HTML responses are now minified.
+- Browsepy command script.
+
+## [0.3.8] - 2015-11-05
+### Added
+- First useable release.
diff --git a/README.rst b/README.rst
index 734c610..8a2ef3f 100644
--- a/README.rst
+++ b/README.rst
@@ -38,7 +38,21 @@ Head to http://ergoithz.github.io/browsepy/ for an online version of current
*master* documentation,
You can also build yourself from sphinx sources using the documentation
-`Makefile` located at `docs` directory.
+`Makefile` located under `docs` directory.
+
+License
+-------
+
+MIT. See `LICENSE`_.
+
+.. _LICENSE: https://raw.githubusercontent.com/ergoithz/browsepy/master/LICENSE
+
+Changelog
+---------
+
+See `CHANGELOG`_.
+
+.. _CHANGELOG: https://raw.githubusercontent.com/ergoithz/browsepy/master/CHANGELOG
Screenshots
-----------
@@ -54,40 +68,9 @@ Features
* **Downloadable directories**, streaming directory tarballs on the fly.
* **Optional remove** for files under given path.
* **Optional upload** for directories under given path.
-* **Player** audio player plugin is provided (without transcoding).
-
-New in 0.6
-----------
-
-* Drop Python 3.3.
-* Plugin discovery.
-* Plugin file-actions providing copy/cut/paste and directory creation.
-* Smarter cookie sessions with session shrinking mechanisms available
- for plugin implementors.
-
-New in 0.5
-----------
-
-* File and plugin APIs have been fully reworked making them more complete and
- extensible, so they can be considered stable now. As a side-effect backward
- compatibility on some edge cases could be broken (please fill an issue if
- your code is affected).
-
- * Old widget API have been deprecated and warnings will be shown if used.
- * Widget registration in a single call (passing a widget instances is still
- available though), no more action-widget duality.
- * Callable-based widget filtering (no longer limited to mimetypes).
- * A raw HTML widget for maximum flexibility.
-
-* Plugins can register command-line arguments now.
-* Player plugin is now able to load `m3u` and `pls` playlists, and optionally
- play everything on a directory (adding a command-line argument).
-* Browsing now takes full advantage of `scandir` (already in Python 3.5 and an
- external dependency for older versions) providing faster directory listing.
-* Custom file ordering while browsing directories.
-* Easy multi-file uploads.
-* Jinja2 template output minification, saving those precious bytes.
-* Setup script now registers a proper `browsepy` command.
+* **Plugins**
+ * **player**, web audio player.
+ * **file-actions**, cut, copy paste and directory creation.
Install
-------
@@ -96,7 +79,7 @@ Install
`pip` and `setuptools` libraries should be upgraded with
`pip install --upgrade pip setuptools`.
-It's on `pypi` so...
+It's on `pypi`_ so...
.. _pypi: https://pypi.python.org/pypi/browsepy/
@@ -105,7 +88,7 @@ It's on `pypi` so...
pip install browsepy
-You can get the same version from our `github repository`.
+You can get the same version from our `github repository`_.
.. _github repository: https://github.com/ergoithz/browsepy
@@ -148,11 +131,13 @@ Command-line arguments
----------------------
This is what is printed when you run `browsepy --help`, keep in mind that
-plugins (loaded with `plugin` argument) could add extra arguments to this list.
+plugins (loaded with `plugin` argument) could add extra arguments to this list
+(you can see all them running `browsepy --help-all` instead).
::
- usage: browsepy [-h] [--help-all] [--directory PATH] [--initial PATH] [--removable PATH] [--upload PATH] [--exclude PATTERN]
+ usage: browsepy [-h] [--help-all] [--directory PATH] [--initial PATH]
+ [--removable PATH] [--upload PATH] [--exclude PATTERN]
[--exclude-from PATH] [--version] [--plugin MODULE]
[host] [port]
@@ -165,7 +150,7 @@ plugins (loaded with `plugin` argument) could add extra arguments to this list.
optional arguments:
-h, --help show this help message and exit
--help-all show help for all available plugins and exit
- --directory PATH serving directory (default: ...)
+ --directory PATH serving directory (default: /my/current/path)
--initial PATH default directory (default: same as --directory)
--removable PATH base directory allowing remove (default: None)
--upload PATH base directory allowing upload (default: None)
@@ -175,9 +160,8 @@ plugins (loaded with `plugin` argument) could add extra arguments to this list.
--plugin MODULE load plugin module (multiple)
available plugins:
- player, browsepy.plugin.player
file-actions, browsepy.plugin.file_actions
-
+ player, browsepy.plugin.player
Using as library
----------------
@@ -193,36 +177,16 @@ url, and mounting **browsepy.app** on the appropriate parent
.. _WSGI: https://www.python.org/dev/peps/pep-0333/
-Browsepy app config (available at :attr:`browsepy.app.config`) uses the
-following configuration options.
-
-* **DIRECTORY_BASE**: anything under this directory will be served,
- defaults to current path.
-* **DIRECTORY_START**: directory will be served when accessing root URL
-* **DIRECTORY_REMOVE**: file removing will be available under this path,
- defaults to **None**.
-* **DIRECTORY_UPLOAD**: file upload will be available under this path,
- defaults to **None**.
-* **DIRECTORY_TAR_BUFFSIZE**, directory tar streaming buffer size,
- defaults to **262144** and must be multiple of 512.
-* **DIRECTORY_DOWNLOADABLE** whether enable directory download or not,
- defaults to **True**.
-* **USE_BINARY_MULTIPLES** whether use binary units (bi-bytes, like KiB)
- instead of common ones (bytes, like KB), defaults to **True**.
-* **PLUGIN_MODULES** list of module names (absolute or relative to
- plugin_namespaces) will be loaded.
-* **PLUGIN_NAMESPACES** prefixes for module names listed at PLUGIN_MODULES
- where relative PLUGIN_MODULES are searched.
-* **EXCLUDE_FNC** function will be used to exclude files from listing and
- directory tarballs. Can be either None or function receiving an absolute
- path and returning a boolean.
-
-After editing `PLUGIN_MODULES` value, plugin manager (available at module
-plugin_manager and app.extensions['plugin_manager']) should be reloaded using
-the `reload` method.
-
-The other way of loading a plugin programmatically is calling plugin manager's
-`load_plugin` method.
+Browsepy app config is available at `browsepy.app.config`.
+
+**Note**: After editing `PLUGIN_MODULES` value, plugin manager (available at
+module plugin_manager and app.extensions['plugin_manager']) should be reloaded
+using `plugin_manager.reload` method.
+
+Alternatively, plugins can be loaded programmatically by calling
+`plugin_manager.load_plugin` method.
+
+More information at http://ergoithz.github.io/browsepy/integrations.html
Extend via plugin API
---------------------
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index c469bb5..9551097 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -69,12 +69,12 @@ def shrink_browse_sort(data, last):
def get_cookie_browse_sorting(path, default):
- '''
+ """
Get sorting-cookie data for path of current request.
:returns: sorting property
:rtype: string
- '''
+ """
if request:
for cpath, cprop in session.get('browse:sort', ()):
if path == cpath:
@@ -83,7 +83,7 @@ def get_cookie_browse_sorting(path, default):
def browse_sortkey_reverse(prop):
- '''
+ """
Get sorting function for directory listing based on given attribute
name, with some caveats:
* Directories will be first.
@@ -94,7 +94,7 @@ def browse_sortkey_reverse(prop):
:type prop: str
:returns: tuple with sorting function and reverse bool
:rtype: tuple of a dict and a bool
- '''
+ """
if prop.startswith('-'):
prop = prop[1:]
reverse = True
diff --git a/browsepy/appconfig.py b/browsepy/appconfig.py
index 3c752fc..edacd92 100644
--- a/browsepy/appconfig.py
+++ b/browsepy/appconfig.py
@@ -9,11 +9,11 @@ from .compat import basestring
class Config(flask.config.Config):
- '''
+ """
Flask-compatible case-insensitive Config class.
See :type:`flask.config.Config` for more info.
- '''
+ """
def __init__(self, root, defaults=None):
self._warned = set()
if defaults:
@@ -21,14 +21,14 @@ class Config(flask.config.Config):
super(Config, self).__init__(root, defaults)
def genkey(self, k):
- '''
+ """
Key translation function.
:param k: key
:type k: str
:returns: uppercase key
;rtype: str
- '''
+ """
if isinstance(k, basestring):
if k not in self._warned and k != k.upper():
self._warned.add(k)
@@ -42,14 +42,14 @@ class Config(flask.config.Config):
return k
def gendict(self, *args, **kwargs):
- '''
+ """
Pre-translated key dictionary constructor.
See :type:`dict` for more info.
:returns: dictionary with uppercase keys
:rtype: dict
- '''
+ """
gk = self.genkey
return dict((gk(k), v) for k, v in dict(*args, **kwargs).items())
@@ -73,9 +73,9 @@ class Config(flask.config.Config):
class Flask(flask.Flask):
- '''
+ """
Flask class using case-insensitive :type:`Config` class.
See :type:`flask.Flask` for more info.
- '''
+ """
config_class = Config
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 109e83b..1021281 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -71,12 +71,12 @@ class HelpFormatter(argparse.RawTextHelpFormatter):
@contextlib.contextmanager
def scandir(path):
- '''
+ """
Backwards-compatible :func:`scandir.scandir` context manager
:param path: path to iterate
:type path: str
- '''
+ """
files = _scandir(path)
try:
yield files
@@ -87,12 +87,12 @@ def scandir(path):
@contextlib.contextmanager
def mkdtemp(suffix='', prefix='', dir=None):
- '''
+ """
Backwards-compatible :class:`tmpfile.TemporaryDirectory` context manager.
:param path: path to iterate
:type path: str
- '''
+ """
path = tempfile.mkdtemp(suffix, prefix, dir)
try:
yield path
@@ -101,19 +101,19 @@ def mkdtemp(suffix='', prefix='', dir=None):
def isexec(path):
- '''
+ """
Check if given path points to an executable file.
:param path: file path
:type path: str
:return: True if executable, False otherwise
:rtype: bool
- '''
+ """
return os.path.isfile(path) and os.access(path, os.X_OK)
def fsdecode(path, os_name=os.name, fs_encoding=FS_ENCODING, errors=None):
- '''
+ """
Decode given path.
:param path: path will be decoded if using bytes
@@ -124,7 +124,7 @@ def fsdecode(path, os_name=os.name, fs_encoding=FS_ENCODING, errors=None):
:type fs_encoding: str
:return: decoded path
:rtype: str
- '''
+ """
if not isinstance(path, bytes):
return path
if not errors:
@@ -134,7 +134,7 @@ def fsdecode(path, os_name=os.name, fs_encoding=FS_ENCODING, errors=None):
def fsencode(path, os_name=os.name, fs_encoding=FS_ENCODING, errors=None):
- '''
+ """
Encode given path.
:param path: path will be encoded if not using bytes
@@ -145,7 +145,7 @@ def fsencode(path, os_name=os.name, fs_encoding=FS_ENCODING, errors=None):
:type fs_encoding: str
:return: encoded path
:rtype: bytes
- '''
+ """
if isinstance(path, bytes):
return path
if not errors:
@@ -155,7 +155,7 @@ def fsencode(path, os_name=os.name, fs_encoding=FS_ENCODING, errors=None):
def getcwd(fs_encoding=FS_ENCODING, cwd_fnc=os.getcwd):
- '''
+ """
Get current work directory's absolute path.
Like os.getcwd but garanteed to return an unicode-str object.
@@ -165,13 +165,13 @@ def getcwd(fs_encoding=FS_ENCODING, cwd_fnc=os.getcwd):
:type cwd_fnc: Callable
:return: path
:rtype: str
- '''
+ """
path = fsdecode(cwd_fnc(), fs_encoding=fs_encoding)
return os.path.abspath(path)
def getdebug(environ=os.environ, true_values=TRUE_VALUES):
- '''
+ """
Get if app is expected to be ran in debug mode looking at environment
variables.
@@ -179,12 +179,12 @@ def getdebug(environ=os.environ, true_values=TRUE_VALUES):
:type environ: collections.abc.Mapping
:returns: True if debug contains a true-like string, False otherwise
:rtype: bool
- '''
+ """
return environ.get('DEBUG', '').lower() in true_values
def deprecated(func_or_text, environ=os.environ):
- '''
+ """
Decorator used to mark functions as deprecated. It will result in a
warning being emmitted hen the function is called.
@@ -206,7 +206,7 @@ def deprecated(func_or_text, environ=os.environ):
:type environ: collections.abc.Mapping
:returns: nested decorator or new decorated function (depending on params)
:rtype: callable
- '''
+ """
def inner(func):
message = (
'Deprecated function {}.'.format(func.__name__)
@@ -226,13 +226,13 @@ def deprecated(func_or_text, environ=os.environ):
def usedoc(other):
- '''
+ """
Decorator which copies __doc__ of given object into decorated one.
Usage:
>>> def fnc1():
- ... """docstring"""
+ ... \"""docstring\"""
... pass
>>> @usedoc(fnc1)
... def fnc2():
@@ -244,7 +244,7 @@ def usedoc(other):
:type other: any
:returns: decorator function
:rtype: callable
- '''
+ """
def inner(fnc):
fnc.__doc__ = fnc.__doc__ or getattr(other, '__doc__')
return fnc
@@ -252,7 +252,7 @@ def usedoc(other):
def pathsplit(value, sep=os.pathsep):
- '''
+ """
Get enviroment PATH elements as list.
This function only cares about spliting across OSes.
@@ -263,7 +263,7 @@ def pathsplit(value, sep=os.pathsep):
:type sep: str
:yields: every path
:ytype: str
- '''
+ """
for part in value.split(sep):
if part[:1] == part[-1:] == '"' or part[:1] == part[-1:] == '\'':
part = part[1:-1]
@@ -271,7 +271,7 @@ def pathsplit(value, sep=os.pathsep):
def pathparse(value, sep=os.pathsep, os_sep=os.sep):
- '''
+ """
Get enviroment PATH directories as list.
This function cares about spliting, escapes and normalization of paths
@@ -285,7 +285,7 @@ def pathparse(value, sep=os.pathsep, os_sep=os.sep):
:type os_sep: str
:yields: every path
:ytype: str
- '''
+ """
escapes = []
normpath = ntpath.normpath if os_sep == '\\' else posixpath.normpath
if '\\' not in (os_sep, sep):
@@ -310,14 +310,14 @@ def pathconf(path,
isdir_fnc=os.path.isdir,
pathconf_fnc=getattr(os, 'pathconf', None),
pathconf_names=getattr(os, 'pathconf_names', ())):
- '''
+ """
Get all pathconf variables for given path.
:param path: absolute fs path
:type path: str
:returns: dictionary containing pathconf keys and their values (both str)
:rtype: dict
- '''
+ """
if pathconf_fnc and pathconf_names:
return {key: pathconf_fnc(path, key) for key in pathconf_names}
@@ -341,7 +341,7 @@ def which(name,
is_executable_fnc=isexec,
path_join_fnc=os.path.join,
os_name=os.name):
- '''
+ """
Get command absolute path.
:param name: name of executable command
@@ -357,7 +357,7 @@ def which(name,
:type os_name: str
:return: absolute path
:rtype: str or None
- '''
+ """
for path in env_path:
for suffix in env_path_ext:
exe_file = path_join_fnc(path, name) + suffix
@@ -367,7 +367,7 @@ def which(name,
def re_escape(pattern, chars=frozenset("()[]{}?*+|^$\\.-#")):
- '''
+ """
Escape all special regex characters in pattern and converts non-ascii
characters into unicode escape sequences.
@@ -377,7 +377,7 @@ def re_escape(pattern, chars=frozenset("()[]{}?*+|^$\\.-#")):
:type patterm: str
:returns: escaped pattern
:rtype: str
- '''
+ """
chr_escape = '\\{}'.format
uni_escape = '\\u{:04d}'.format
return ''.join(
@@ -388,22 +388,13 @@ def re_escape(pattern, chars=frozenset("()[]{}?*+|^$\\.-#")):
)
-@contextlib.contextmanager
-def redirect_stderr(f):
- old = sys.stderr
- sys.stderr = f
- yield
- if sys.stderr is f:
- sys.stderr = old
-
-
class Iterator(BaseIterator):
def next(self):
- '''
+ """
Call :method:`__next__` for compatibility.
:returns: see :method:`__next__`
- '''
+ """
return self.__next__()
diff --git a/browsepy/exceptions.py b/browsepy/exceptions.py
index 2a49f74..e5d362d 100644
--- a/browsepy/exceptions.py
+++ b/browsepy/exceptions.py
@@ -2,27 +2,27 @@
class OutsideDirectoryBase(Exception):
- '''
+ """
Exception raised when trying to access to a file outside path defined on
`DIRECTORY_BASE` config property.
- '''
+ """
pass
class OutsideRemovableBase(Exception):
- '''
+ """
Exception raised when trying to access to a file outside path defined on
`directory_remove` config property.
- '''
+ """
pass
class InvalidPathError(ValueError):
- '''
+ """
Exception raised when a path is not valid.
:property path: value whose length raised this Exception
- '''
+ """
code = 'invalid-path'
template = 'Path {0.path!r} is not valid.'
@@ -33,11 +33,11 @@ class InvalidPathError(ValueError):
class InvalidFilenameError(InvalidPathError):
- '''
+ """
Exception raised when a filename is not valid.
:property filename: value whose length raised this Exception
- '''
+ """
code = 'invalid-filename'
template = 'Filename {0.filename!r} is not valid.'
@@ -47,11 +47,11 @@ class InvalidFilenameError(InvalidPathError):
class PathTooLongError(InvalidPathError):
- '''
+ """
Exception raised when maximum filesystem path length is reached.
:property limit: value length limit
- '''
+ """
code = 'invalid-path-length'
template = 'Path {0.path!r} is too long, max length is {0.limit}'
@@ -61,9 +61,9 @@ class PathTooLongError(InvalidPathError):
class FilenameTooLongError(InvalidFilenameError):
- '''
+ """
Exception raised when maximum filesystem filename length is reached.
- '''
+ """
code = 'invalid-filename-length'
template = 'Filename {0.filename!r} is too long, max length is {0.limit}'
diff --git a/browsepy/file.py b/browsepy/file.py
index 563b49c..eeab4a8 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -52,7 +52,7 @@ fs_safe_characters = string.ascii_uppercase + string.digits
class Node(object):
- '''
+ """
Abstract filesystem node class.
This represents an unspecified entity with a filesystem's path suitable for
@@ -65,7 +65,7 @@ class Node(object):
will be created instead of an instance of this class tself.
* :attr:`directory_class`, class will be used for directory nodes,
* :attr:`file_class`, class will be used for file nodes.
- '''
+ """
generic = True
directory_class = None # set later at import time
file_class = None # set later at import time
@@ -76,22 +76,22 @@ class Node(object):
@cached_property
def is_excluded(self):
- '''
+ """
Get if current node shouldn't be shown, using :attt:`app` config's
exclude_fnc.
:returns: True if excluded, False otherwise
- '''
+ """
return self.plugin_manager.check_excluded(self.path)
@cached_property
def plugin_manager(self):
- '''
+ """
Get current app's plugin manager.
:returns: plugin manager instance
:type: browsepy.manager.PluginManagerBase
- '''
+ """
return (
self.app.extensions.get('plugin_manager') or
manager.PluginManager(self.app)
@@ -99,14 +99,14 @@ class Node(object):
@cached_property
def widgets(self):
- '''
+ """
List widgets with filter return True for this node (or without filter).
Remove button is prepended if :property:can_remove returns true.
:returns: list of widgets
:rtype: list of namedtuple instances
- '''
+ """
widgets = []
if self.can_remove:
widgets.append(
@@ -122,12 +122,12 @@ class Node(object):
@cached_property
def link(self):
- '''
+ """
Get last widget with place `entry-link`.
:returns: widget on entry-link (ideally a link one)
:rtype: namedtuple instance
- '''
+ """
link = None
for widget in self.widgets:
if widget.place == 'entry-link':
@@ -136,45 +136,45 @@ class Node(object):
@cached_property
def can_remove(self):
- '''
+ """
Get if current node can be removed based on app config's
`DIRECTORY_REMOVE`.
:returns: True if current node can be removed, False otherwise.
:rtype: bool
- '''
+ """
dirbase = self.app.config.get('DIRECTORY_REMOVE')
return bool(dirbase and check_under_base(self.path, dirbase))
@cached_property
def stats(self):
- '''
+ """
Get current stats object as returned by `os.stat` function.
:returns: stats object
:rtype: posix.stat_result or nt.stat_result
- '''
+ """
return os.stat(self.path)
@cached_property
def pathconf(self):
- '''
+ """
Get filesystem config for current path.
See :func:`compat.pathconf`.
:returns: fs config
:rtype: dict
- '''
+ """
return compat.pathconf(self.path)
@cached_property
def parent(self):
- '''
+ """
Get parent node if available based on app config's `DIRECTORY_BASE`.
:returns: parent object if available
:rtype: Node instance or None
- '''
+ """
directory_base = self.app.config.get('DIRECTORY_BASE', self.path)
if check_path(self.path, directory_base):
return None
@@ -183,12 +183,12 @@ class Node(object):
@cached_property
def ancestors(self):
- '''
+ """
Get list of ancestors until app config's `DIRECTORY_BASE` is reached.
:returns: list of ancestors starting from nearest.
:rtype: list of Node objects
- '''
+ """
ancestors = []
parent = self.parent
while parent:
@@ -198,12 +198,12 @@ class Node(object):
@property
def modified(self):
- '''
+ """
Get human-readable last modification date-time.
:returns: iso9008-like date-time string (without timezone)
:rtype: str
- '''
+ """
try:
dt = datetime.datetime.fromtimestamp(self.stats.st_mtime)
return dt.strftime('%Y.%m.%d %H:%M:%S')
@@ -212,13 +212,13 @@ class Node(object):
@property
def urlpath(self):
- '''
+ """
Get the url substring corresponding to this node for those endpoints
accepting a 'path' parameter, suitable for :meth:`from_urlpath`.
:returns: relative-url-like for node's path
:rtype: str
- '''
+ """
return abspath_to_urlpath(
self.path,
self.app.config.get('DIRECTORY_BASE', self.path),
@@ -226,27 +226,27 @@ class Node(object):
@property
def name(self):
- '''
+ """
Get the basename portion of node's path.
:returns: filename
:rtype: str
- '''
+ """
return os.path.basename(self.path)
@property
def type(self):
- '''
+ """
Get the mime portion of node's mimetype (without the encoding part).
:returns: mimetype
:rtype: str
- '''
+ """
return self.mimetype.split(";", 1)[0]
@property
def category(self):
- '''
+ """
Get mimetype category (first portion of mimetype before the slash).
:returns: mimetype category
@@ -263,42 +263,42 @@ class Node(object):
* multipart
* text
* video
- '''
+ """
return self.type.split('/', 1)[0]
def __init__(self, path=None, app=None, **defaults):
- '''
+ """
:param path: local path
:type path: str
:param app: app instance (optional inside application context)
:type app: flask.Flask
:param **defaults: initial property values
- '''
+ """
self.path = compat.fsdecode(path) if path else None
self.app = utils.solve_local(app or flask.current_app)
self.__dict__.update(defaults) # only for attr and cached_property
def __repr__(self):
- '''
+ """
Get str representation of instance.
:returns: instance representation
:rtype: str
- '''
+ """
return '<{0.__class__.__name__}({0.path!r})>'.format(self)
def remove(self):
- '''
+ """
Does nothing except raising if can_remove property returns False.
:raises: OutsideRemovableBase if :property:`can_remove` returns false
- '''
+ """
if not self.can_remove:
raise OutsideRemovableBase("File outside removable base")
@classmethod
def from_urlpath(cls, path, app=None):
- '''
+ """
Alternative constructor which accepts a path as taken from URL and uses
the given app or the current app config to get the real path.
@@ -309,7 +309,7 @@ class Node(object):
:param app: optional, flask application
:return: file object pointing to path
:rtype: File
- '''
+ """
app = utils.solve_local(app or flask.current_app)
base = app.config.get('DIRECTORY_BASE', path)
path = urlpath_to_abspath(path, base)
@@ -323,34 +323,34 @@ class Node(object):
@classmethod
def register_file_class(cls, kls):
- '''
+ """
Convenience method for setting current class file_class property.
:param kls: class to set
:type kls: type
:returns: given class (enabling using this as decorator)
:rtype: type
- '''
+ """
cls.file_class = kls
return kls
@classmethod
def register_directory_class(cls, kls):
- '''
+ """
Convenience method for setting current class directory_class property.
:param kls: class to set
:type kls: type
:returns: given class (enabling using this as decorator)
:rtype: type
- '''
+ """
cls.directory_class = kls
return kls
@Node.register_file_class
class File(Node):
- '''
+ """
Filesystem file class.
Some notes:
@@ -363,7 +363,7 @@ class File(Node):
performed.
* :attr:`generic` is set to False, so static method :meth:`from_urlpath`
will always return instances of this class.
- '''
+ """
can_download = True
can_upload = False
is_directory = False
@@ -371,7 +371,7 @@ class File(Node):
@cached_property
def widgets(self):
- '''
+ """
List widgets with filter return True for this file (or without filter).
Entry link is prepended.
@@ -380,7 +380,7 @@ class File(Node):
:returns: list of widgets
:rtype: list of namedtuple instances
- '''
+ """
widgets = [
self.plugin_manager.create_widget(
'entry-link',
@@ -410,33 +410,33 @@ class File(Node):
@cached_property
def mimetype(self):
- '''
+ """
Get full mimetype, with encoding if available.
:returns: mimetype
:rtype: str
- '''
+ """
return self.plugin_manager.get_mimetype(self.path)
@cached_property
def is_file(self):
- '''
+ """
Get if node is file.
:returns: True if file, False otherwise
:rtype: bool
- '''
+ """
return os.path.isfile(self.path)
@property
def size(self):
- '''
+ """
Get human-readable node size in bytes.
If directory, this will corresponds with own inode size.
:returns: fuzzy size with unit
:rtype: str
- '''
+ """
try:
size, unit = fmt_size(
self.stats.st_size,
@@ -450,12 +450,12 @@ class File(Node):
@property
def encoding(self):
- '''
+ """
Get encoding part of mimetype, or "default" if not available.
:returns: file conding as returned by mimetype function or "default"
:rtype: str
- '''
+ """
if ";" in self.mimetype:
match = self.re_charset.search(self.mimetype)
gdict = match.groupdict() if match else {}
@@ -463,27 +463,27 @@ class File(Node):
return "default"
def remove(self):
- '''
+ """
Remove file.
:raises OutsideRemovableBase: when not under removable base directory
- '''
+ """
super(File, self).remove()
os.unlink(self.path)
def download(self):
- '''
+ """
Get a Flask's send_file Response object pointing to this file.
:returns: Response object as returned by flask's send_file
:rtype: flask.Response
- '''
+ """
directory, name = os.path.split(self.path)
return flask.send_from_directory(directory, name, as_attachment=True)
@Node.register_directory_class
class Directory(Node):
- '''
+ """
Filesystem directory class.
Some notes:
@@ -495,7 +495,7 @@ class Directory(Node):
* :attr:`encoding` is fixed to 'default'.
* :attr:`generic` is set to False, so static method :meth:`from_urlpath`
will always return instances of this class.
- '''
+ """
stream_class = stream.TarFileStream
_listdir_cache = None
@@ -507,17 +507,17 @@ class Directory(Node):
@property
def name(self):
- '''
+ """
Get the basename portion of directory's path.
:returns: filename
:rtype: str
- '''
+ """
return super(Directory, self).name or self.path
@cached_property
def widgets(self):
- '''
+ """
List widgets with filter return True for this dir (or without filter).
Entry link is prepended.
@@ -527,7 +527,7 @@ class Directory(Node):
:returns: list of widgets
:rtype: list of namedtuple instances
- '''
+ """
widgets = [
self.plugin_manager.create_widget(
'entry-link',
@@ -581,66 +581,66 @@ class Directory(Node):
@cached_property
def is_directory(self):
- '''
+ """
Get if path points to a real directory.
:returns: True if real directory, False otherwise
:rtype: bool
- '''
+ """
return os.path.isdir(self.path)
@cached_property
def is_root(self):
- '''
+ """
Get if directory is filesystem's root
:returns: True if FS root, False otherwise
:rtype: bool
- '''
+ """
return check_path(os.path.dirname(self.path), self.path)
@cached_property
def can_download(self):
- '''
+ """
Get if path is downloadable (if app's `DIRECTORY_DOWNLOADABLE` config
property is True).
:returns: True if downloadable, False otherwise
:rtype: bool
- '''
+ """
return self.app.config.get('DIRECTORY_DOWNLOADABLE', False)
@cached_property
def can_upload(self):
- '''
+ """
Get if a file can be uploaded to path (if directory path is under app's
`DIRECTORY_UPLOAD` config property).
:returns: True if a file can be upload to directory, False otherwise
:rtype: bool
- '''
+ """
dirbase = self.app.config.get('DIRECTORY_UPLOAD', False)
return dirbase and check_base(self.path, dirbase)
@cached_property
def can_remove(self):
- '''
+ """
Get if current node can be removed based on app config's
directory_remove.
:returns: True if current node can be removed, False otherwise.
:rtype: bool
- '''
+ """
return self.parent and super(Directory, self).can_remove
@cached_property
def is_empty(self):
- '''
+ """
Get if directory is empty (based on :meth:`_listdir`).
:returns: True if this directory has no entries, False otherwise.
:rtype: bool
- '''
+ """
if self._listdir_cache is not None:
return not bool(self._listdir_cache)
for entry in self._listdir():
@@ -648,21 +648,21 @@ class Directory(Node):
return True
def remove(self):
- '''
+ """
Remove directory tree.
:raises OutsideRemovableBase: when not under removable base directory
- '''
+ """
super(Directory, self).remove()
shutil.rmtree(self.path)
def download(self):
- '''
+ """
Get a Flask Response object streaming a tarball of this directory.
:returns: Response object
:rtype: flask.Response
- '''
+ """
stream = self.stream_class(
self.path,
self.app.config.get('DIRECTORY_TAR_BUFFSIZE', 10240),
@@ -688,18 +688,18 @@ class Directory(Node):
)
def contains(self, filename):
- '''
+ """
Check if directory contains an entry with given filename.
:param filename: filename will be check
:type filename: str
:returns: True if exists, False otherwise.
:rtype: bool
- '''
+ """
return os.path.exists(os.path.join(self.path, filename))
def choose_filename(self, filename, attempts=999):
- '''
+ """
Get a new filename which does not colide with any entry on directory,
based on given filename.
@@ -712,7 +712,7 @@ class Directory(Node):
:raises FilenameTooLong: when filesystem filename size limit is reached
:raises PathTooLong: when OS or filesystem path size limit is reached
- '''
+ """
new_filename = filename
for attempt in range(2, attempts + 1):
if not self.contains(new_filename):
@@ -735,12 +735,12 @@ class Directory(Node):
return new_filename
def _listdir(self, precomputed_stats=(os.name == 'nt')):
- '''
+ """
Iter unsorted entries on this directory.
:yields: Directory or File instance for each entry in directory
:rtype: Iterator of browsepy.file.Node
- '''
+ """
directory_class = self.directory_class
file_class = self.file_class
exclude_fnc = self.plugin_manager.check_excluded
@@ -767,12 +767,12 @@ class Directory(Node):
logger.exception(e)
def listdir(self, sortkey=None, reverse=False):
- '''
+ """
Get sorted list (by given sortkey and reverse params) of File objects.
:return: sorted list of File instances
:rtype: list of File instances
- '''
+ """
if self._listdir_cache is None:
self._listdir_cache = tuple(self._listdir())
if sortkey:
@@ -784,7 +784,7 @@ class Directory(Node):
def fmt_size(size, binary=True):
- '''
+ """
Get size and unit.
:param size: size in bytes
@@ -793,7 +793,7 @@ def fmt_size(size, binary=True):
:type binary: bool
:return: size and unit
:rtype: tuple of int and unit as str
- '''
+ """
if binary:
fmt_sizes = binary_units
fmt_divider = 1024.
@@ -808,7 +808,7 @@ def fmt_size(size, binary=True):
def relativize_path(path, base, os_sep=os.sep):
- '''
+ """
Make absolute path relative to an absolute base.
:param path: absolute path
@@ -820,7 +820,7 @@ def relativize_path(path, base, os_sep=os.sep):
:return: relative path
:rtype: str or unicode
:raises OutsideDirectoryBase: if path is not below base
- '''
+ """
if not check_base(path, base, os_sep):
raise OutsideDirectoryBase("%r is not under %r" % (path, base))
prefix_len = len(base)
@@ -830,7 +830,7 @@ def relativize_path(path, base, os_sep=os.sep):
def abspath_to_urlpath(path, base, os_sep=os.sep):
- '''
+ """
Make filesystem absolute path uri relative using given absolute base path.
:param path: absolute path
@@ -839,12 +839,12 @@ def abspath_to_urlpath(path, base, os_sep=os.sep):
:return: relative uri
:rtype: str or unicode
:raises OutsideDirectoryBase: if resulting path is not below base
- '''
+ """
return relativize_path(path, base, os_sep).replace(os_sep, '/')
def urlpath_to_abspath(path, base, os_sep=os.sep):
- '''
+ """
Make uri relative path fs absolute using a given absolute base path.
:param path: relative path
@@ -853,7 +853,7 @@ def urlpath_to_abspath(path, base, os_sep=os.sep):
:return: absolute path
:rtype: str or unicode
:raises OutsideDirectoryBase: if resulting path is not below base
- '''
+ """
prefix = base if base.endswith(os_sep) else base + os_sep
realpath = os.path.abspath(prefix + path.replace('/', os_sep))
if check_path(base, realpath) or check_under_base(realpath, base):
@@ -862,14 +862,14 @@ def urlpath_to_abspath(path, base, os_sep=os.sep):
def generic_filename(path):
- '''
+ """
Extract filename of given path os-indepently, taking care of known path
separators.
:param path: path
:return: filename
:rtype: str or unicode (depending on given path)
- '''
+ """
for sep in common_path_separators:
if sep in path:
@@ -878,13 +878,13 @@ def generic_filename(path):
def clean_restricted_chars(path, restricted_chars=current_restricted_chars):
- '''
+ """
Get path without restricted characters.
:param path: path
:return: path without restricted characters
:rtype: str or unicode (depending on given path)
- '''
+ """
for character in restricted_chars:
path = path.replace(character, '_')
return path
@@ -893,7 +893,7 @@ def clean_restricted_chars(path, restricted_chars=current_restricted_chars):
def check_forbidden_filename(filename,
destiny_os=os.name,
restricted_names=restricted_names):
- '''
+ """
Get if given filename is forbidden for current OS or filesystem.
:param filename:
@@ -901,7 +901,7 @@ def check_forbidden_filename(filename,
:param fs_encoding: destination filesystem filename encoding
:return: wether is forbidden on given OS (or filesystem) or not
:rtype: bool
- '''
+ """
return (
filename in restricted_names or
destiny_os == 'nt' and
@@ -910,7 +910,7 @@ def check_forbidden_filename(filename,
def check_path(path, base, os_sep=os.sep):
- '''
+ """
Check if both given paths are equal.
:param path: absolute path
@@ -921,13 +921,13 @@ def check_path(path, base, os_sep=os.sep):
:type base: str
:return: wether two path are equal or not
:rtype: bool
- '''
+ """
base = base[:-len(os_sep)] if base.endswith(os_sep) else base
return os.path.normcase(path) == os.path.normcase(base)
def check_base(path, base, os_sep=os.sep):
- '''
+ """
Check if given absolute path is under or given base.
:param path: absolute path
@@ -937,7 +937,7 @@ def check_base(path, base, os_sep=os.sep):
:param os_sep: path separator, defaults to os.sep
:return: wether path is under given base or not
:rtype: bool
- '''
+ """
return (
check_path(path, base, os_sep) or
check_under_base(path, base, os_sep)
@@ -945,7 +945,7 @@ def check_base(path, base, os_sep=os.sep):
def check_under_base(path, base, os_sep=os.sep):
- '''
+ """
Check if given absolute path is under given base.
:param path: absolute path
@@ -955,13 +955,13 @@ def check_under_base(path, base, os_sep=os.sep):
:param os_sep: path separator, defaults to os.sep
:return: wether file is under given base or not
:rtype: bool
- '''
+ """
prefix = base if base.endswith(os_sep) else base + os_sep
return os.path.normcase(path).startswith(os.path.normcase(prefix))
def secure_filename(path, destiny_os=os.name, fs_encoding=compat.FS_ENCODING):
- '''
+ """
Get rid of parent path components and special filenames.
If path is invalid or protected, return empty string.
@@ -974,7 +974,7 @@ def secure_filename(path, destiny_os=os.name, fs_encoding=compat.FS_ENCODING):
:type fs_encoding: str
:return: filename or empty string
:rtype: str
- '''
+ """
path = generic_filename(path)
path = clean_restricted_chars(
path,
@@ -1004,7 +1004,7 @@ def secure_filename(path, destiny_os=os.name, fs_encoding=compat.FS_ENCODING):
def alternative_filename(filename, attempt=None):
- '''
+ """
Generates an alternative version of given filename.
If an number attempt parameter is given, will be used on the alternative
@@ -1014,7 +1014,7 @@ def alternative_filename(filename, attempt=None):
:param attempt: optional attempt number, defaults to null
:return: new filename
:rtype: str or unicode
- '''
+ """
filename_parts = filename.rsplit(u'.', 2)
name = filename_parts[0]
ext = ''.join(u'.%s' % ext for ext in filename_parts[1:])
diff --git a/browsepy/http.py b/browsepy/http.py
index cb73cfa..94ad19c 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -11,18 +11,18 @@ logger = logging.getLogger(__name__)
class Headers(BaseHeaders):
- '''
+ """
A wrapper around :class:`werkzeug.datastructures.Headers`, allowing
to specify headers with options on initialization.
Headers are provided as keyword arguments while values can be either
:type:`str` (no options) or tuple of :type:`str` and :type:`dict`.
- '''
+ """
snake_replace = staticmethod(re.compile(r'(^|-)[a-z]').sub)
@classmethod
def genpair(cls, key, value):
- '''
+ """
Extract value and options from values dict based on given key and
options-key.
@@ -32,7 +32,7 @@ class Headers(BaseHeaders):
:type value: str or pair of (str, dict)
:returns: tuple with key and value
:rtype: tuple of (str, str)
- '''
+ """
rkey = cls.snake_replace(
lambda x: x.group(0).upper(),
key.replace('_', '-')
@@ -45,10 +45,10 @@ class Headers(BaseHeaders):
return rkey, rvalue
def __init__(self, **kwargs):
- '''
+ """
:param **kwargs: header and values as keyword arguments
:type **kwargs: str or (str, dict)
- '''
+ """
items = [
self.genpair(key, value)
for key, value in kwargs.items()
diff --git a/browsepy/manager.py b/browsepy/manager.py
index d42bcf3..edecafe 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -23,23 +23,26 @@ from .exceptions import PluginNotFoundError, InvalidArgumentError, \
class PluginManagerBase(object):
- '''
+ """
Base plugin manager for plugin module loading and Flask extension logic.
- '''
+ """
_pyfile_extensions = ('.py', '.pyc', '.pyd', '.pyo')
@property
def namespaces(self):
- '''
+ """
List of plugin namespaces taken from app config.
- '''
+
+ :returns: list of plugin namespaces
+ :rtype: typing.List[str]
+ """
return self.app.config.get('PLUGIN_NAMESPACES', []) if self.app else []
def __init__(self, app=None):
- '''
+ """
:param app: flask application
:type app: flask.Flask
- '''
+ """
self.plugin_filters = []
if app is None:
@@ -48,12 +51,12 @@ class PluginManagerBase(object):
self.init_app(app)
def init_app(self, app):
- '''
+ """
Initialize this Flask extension for given app.
:param app: flask application
:type app: flask.Flask
- '''
+ """
self.app = app
if not hasattr(app, 'extensions'):
app.extensions = {}
@@ -61,35 +64,35 @@ class PluginManagerBase(object):
self.reload()
def reload(self):
- '''
+ """
Clear plugin manager state and reload plugins.
This method will make use of :meth:`clear` and :meth:`load_plugin`,
so all internal state will be cleared, and all plugins defined in
:data:`self.app.config['PLUGIN_MODULES']` will be loaded.
- '''
+ """
self.clear()
for plugin in self.app.config.get('PLUGIN_MODULES', ()):
self.load_plugin(plugin)
def clear(self):
- '''
+ """
Clear plugin manager state.
- '''
+ """
pass
def _iter_modules(self, prefix):
- '''
+ """
Iterate thru all root modules containing given prefix.
- '''
+ """
for finder, name, ispkg in pkgutil.iter_modules():
if name.startswith(prefix):
yield name
def _content_import_name(self, module, item, prefix):
- '''
+ """
Get importable module contnt import name..
- '''
+ """
res = compat.res
name = '%s.%s' % (module, item)
if name.startswith(prefix):
@@ -100,9 +103,9 @@ class PluginManagerBase(object):
return name
def _iter_submodules(self, prefix):
- '''
+ """
Iterate thru all modules which full name contains given prefix.
- '''
+ """
res = compat.res
parent = prefix.rsplit('.', 1)[0]
for base in (prefix, parent):
@@ -115,10 +118,10 @@ class PluginManagerBase(object):
pass
def _iter_plugin_modules(self, get_module_fnc=utils.get_module):
- '''
+ """
Iterate plugin modules, yielding both full qualified name and
short plugin name as tuple.
- '''
+ """
nameset = set()
shortset = set()
filters = self.plugin_filters
@@ -150,19 +153,19 @@ class PluginManagerBase(object):
@cached_property
def available_plugins(self):
- '''
+ """
Iterate through all loadable plugins on default import locations
- '''
+ """
return list(self._iter_plugin_modules())
def import_plugin(self, plugin, get_module_fnc=utils.get_module):
- '''
+ """
Import plugin by given name, looking at :attr:`namespaces`.
:param plugin: plugin module name
:type plugin: str
:raises PluginNotFoundError: if not found on any namespace
- '''
+ """
plugin = plugin.replace('-', '_')
names = [
name
@@ -183,21 +186,21 @@ class PluginManagerBase(object):
plugin, names)
def load_plugin(self, plugin):
- '''
+ """
Import plugin (see :meth:`import_plugin`) and load related data.
:param plugin: plugin module name
:type plugin: str
:raises PluginNotFoundError: if not found on any namespace
- '''
+ """
return self.import_plugin(plugin)
class RegistrablePluginManager(PluginManagerBase):
- '''
+ """
Base plugin manager for plugin registration via :func:`register_plugin`
functions at plugin module level.
- '''
+ """
def __init__(self, app=None):
super(RegistrablePluginManager, self).__init__(app)
self.plugin_filters.append(
@@ -205,7 +208,7 @@ class RegistrablePluginManager(PluginManagerBase):
)
def load_plugin(self, plugin):
- '''
+ """
Import plugin (see :meth:`import_plugin`) and load related data.
If available, plugin's module-level :func:`register_plugin` function
@@ -214,7 +217,7 @@ class RegistrablePluginManager(PluginManagerBase):
:param plugin: plugin module name
:type plugin: str
:raises PluginNotFoundError: if not found on any namespace
- '''
+ """
module = super(RegistrablePluginManager, self).load_plugin(plugin)
if hasattr(module, 'register_plugin'):
module.register_plugin(self)
@@ -222,18 +225,18 @@ class RegistrablePluginManager(PluginManagerBase):
class BlueprintPluginManager(PluginManagerBase):
- '''
+ """
Manager for blueprint registration via :meth:`register_plugin` calls.
Note: blueprints are not removed on `clear` nor reloaded on `reload`
as flask does not allow it.
- '''
+ """
def __init__(self, app=None):
self._blueprint_known = set()
super(BlueprintPluginManager, self).__init__(app=app)
def register_blueprint(self, blueprint):
- '''
+ """
Register given blueprint on curren app.
This method is intended to be used on plugin's module-level
@@ -241,23 +244,23 @@ class BlueprintPluginManager(PluginManagerBase):
:param blueprint: blueprint object with plugin endpoints
:type blueprint: flask.Blueprint
- '''
+ """
if blueprint not in self._blueprint_known:
self.app.register_blueprint(blueprint)
self._blueprint_known.add(blueprint)
class ExcludePluginManager(PluginManagerBase):
- '''
+ """
Manager for exclude-function registration via :meth:`register_exclude_fnc`
calls.
- '''
+ """
def __init__(self, app=None):
self._exclude_functions = set()
super(ExcludePluginManager, self).__init__(app=app)
def register_exclude_function(self, exclude_fnc):
- '''
+ """
Register given exclude-function on curren app.
This method is intended to be used on plugin's module-level
@@ -265,18 +268,18 @@ class ExcludePluginManager(PluginManagerBase):
:param blueprint: blueprint object with plugin endpoints
:type blueprint: flask.Blueprint
- '''
+ """
self._exclude_functions.add(exclude_fnc)
def check_excluded(self, path):
- '''
+ """
Check if given path is excluded.
:param path: absolute path to check against config and plugins
:type path: str
:return: wether if path should be excluded or not
:rtype: bool
- '''
+ """
exclude_fnc = self.app.config.get('EXCLUDE_FNC')
if exclude_fnc and exclude_fnc(path):
return True
@@ -286,17 +289,17 @@ class ExcludePluginManager(PluginManagerBase):
return False
def clear(self):
- '''
+ """
Clear plugin manager state.
Registered exclude functions will be disposed.
- '''
+ """
self._exclude_functions.clear()
super(ExcludePluginManager, self).clear()
class WidgetPluginManager(PluginManagerBase):
- '''
+ """
Plugin manager for widget registration.
This class provides a dictionary of widget types at its
@@ -304,7 +307,7 @@ class WidgetPluginManager(PluginManagerBase):
both :meth:`create_widget` and :meth:`register_widget` methods' `type`
parameter, or instantiated directly and passed to :meth:`register_widget`
via `widget` parameter.
- '''
+ """
widget_types = {
'base': defaultsnamedtuple(
'Widget',
@@ -340,16 +343,16 @@ class WidgetPluginManager(PluginManagerBase):
}
def clear(self):
- '''
+ """
Clear plugin manager state.
Registered widgets will be disposed after calling this method.
- '''
+ """
self._widgets = []
super(WidgetPluginManager, self).clear()
def get_widgets(self, file=None, place=None):
- '''
+ """
List registered widgets, optionally matching given criteria.
:param file: optional file object will be passed to widgets' filter
@@ -359,12 +362,12 @@ class WidgetPluginManager(PluginManagerBase):
:type place: str
:returns: list of widget instances
:rtype: list of objects
- '''
+ """
return list(self.iter_widgets(file, place))
@classmethod
def _resolve_widget(cls, file, widget):
- '''
+ """
Resolve widget callable properties into static ones.
:param file: file will be used to resolve callable properties.
@@ -373,14 +376,14 @@ class WidgetPluginManager(PluginManagerBase):
:type widget: object
:returns: a new widget instance of the same type as widget parameter
:rtype: object
- '''
+ """
return widget.__class__(*[
value(file) if callable(value) else value
for value in widget
])
def iter_widgets(self, file=None, place=None):
- '''
+ """
Iterate registered widgets, optionally matching given criteria.
:param file: optional file object will be passed to widgets' filter
@@ -390,7 +393,7 @@ class WidgetPluginManager(PluginManagerBase):
:type place: str
:yields: widget instances
:ytype: object
- '''
+ """
for filter, dynamic, cwidget in self._widgets:
try:
if (
@@ -411,7 +414,7 @@ class WidgetPluginManager(PluginManagerBase):
yield cwidget
def create_widget(self, place, type, file=None, **kwargs):
- '''
+ """
Create a widget object based on given arguments.
If file object is provided, callable arguments will be resolved:
@@ -429,7 +432,7 @@ class WidgetPluginManager(PluginManagerBase):
:type type: browsepy.files.Node or None
:returns: widget instance
:rtype: object
- '''
+ """
widget_class = self.widget_types.get(type, self.widget_types['base'])
kwargs.update(place=place, type=type)
try:
@@ -451,7 +454,7 @@ class WidgetPluginManager(PluginManagerBase):
def register_widget(self, place=None, type=None, widget=None, filter=None,
**kwargs):
- '''
+ """
Create (see :meth:`create_widget`) or use provided widget and register
it.
@@ -472,7 +475,7 @@ class WidgetPluginManager(PluginManagerBase):
the same time (they're mutually exclusive).
:returns: created or given widget object
:rtype: object
- '''
+ """
if bool(widget) == bool(place or type):
raise InvalidArgumentError(
'register_widget takes either place and type or widget'
@@ -484,23 +487,23 @@ class WidgetPluginManager(PluginManagerBase):
class MimetypePluginManager(RegistrablePluginManager):
- '''
+ """
Plugin manager for mimetype-function registration.
- '''
+ """
_default_mimetype_functions = mimetype.alternatives
def clear(self):
- '''
+ """
Clear plugin manager state.
Registered mimetype functions will be disposed after calling this
method.
- '''
+ """
self._mimetype_functions = list(self._default_mimetype_functions)
super(MimetypePluginManager, self).clear()
def get_mimetype(self, path):
- '''
+ """
Get mimetype of given path calling all registered mime functions (and
default ones).
@@ -508,7 +511,7 @@ class MimetypePluginManager(RegistrablePluginManager):
:type path: str
:returns: mimetype
:rtype: str
- '''
+ """
for fnc in self._mimetype_functions:
mime = fnc(path)
if mime:
@@ -516,7 +519,7 @@ class MimetypePluginManager(RegistrablePluginManager):
return mimetype.by_default(path)
def register_mimetype_function(self, fnc):
- '''
+ """
Register mimetype function.
Given function must accept a filesystem path as string and return
@@ -524,13 +527,13 @@ class MimetypePluginManager(RegistrablePluginManager):
:param fnc: callable accepting a path string
:type fnc: callable
- '''
+ """
self._mimetype_functions.insert(0, fnc)
class SessionPluginManager(PluginManagerBase):
def register_session(self, key_or_keys, shrink_fnc=None):
- '''
+ """
Register session shrink function for specific session key or
keys. Can be used as decorator.
@@ -546,7 +549,7 @@ class SessionPluginManager(PluginManagerBase):
:type shrink_fnc: cookieman.abc.ShrinkFunction
:returns: either original given shrink_fnc or decorator
:rtype: cookieman.abc.ShrinkFunction
- '''
+ """
interface = self.app.session_interface
if isinstance(interface, CookieMan):
return interface.register(key_or_keys, shrink_fnc)
@@ -554,7 +557,7 @@ class SessionPluginManager(PluginManagerBase):
class ArgumentPluginManager(PluginManagerBase):
- '''
+ """
Plugin manager for command-line argument registration.
This function is used by browsepy's :mod:`__main__` module in order
@@ -563,7 +566,7 @@ class ArgumentPluginManager(PluginManagerBase):
This is done by :meth:`load_arguments` which imports all plugin modules
and calls their respective :func:`register_arguments` module-level
function.
- '''
+ """
_argparse_kwargs = {'add_help': False}
_argparse_arguments = argparse.Namespace()
@@ -574,7 +577,7 @@ class ArgumentPluginManager(PluginManagerBase):
)
def extract_plugin_arguments(self, plugin):
- '''
+ """
Given a plugin name, extracts its registered_arguments as an
iterable of (args, kwargs) tuples.
@@ -582,7 +585,7 @@ class ArgumentPluginManager(PluginManagerBase):
:type plugin: str
:returns: iterable if (args, kwargs) tuples.
:rtype: iterable
- '''
+ """
module = self.import_plugin(plugin)
if hasattr(module, 'register_arguments'):
manager = ArgumentPluginManager()
@@ -615,7 +618,7 @@ class ArgumentPluginManager(PluginManagerBase):
)
def load_arguments(self, argv, base=None):
- '''
+ """
Process given argument list based on registered arguments and given
optional base :class:`argparse.ArgumentParser` instance.
@@ -632,7 +635,7 @@ class ArgumentPluginManager(PluginManagerBase):
:returns: argparse.Namespace instance with processed arguments as
given by :meth:`argparse.ArgumentParser.parse_args`.
:rtype: argparse.Namespace
- '''
+ """
parser = self._plugin_argument_parser(base)
options, _ = parser.parse_known_args(argv)
@@ -666,17 +669,17 @@ class ArgumentPluginManager(PluginManagerBase):
return self._argparse_arguments
def clear(self):
- '''
+ """
Clear plugin manager state.
Registered command-line arguments will be disposed after calling this
method.
- '''
+ """
self._argparse_argkwargs = []
super(ArgumentPluginManager, self).clear()
def register_argument(self, *args, **kwargs):
- '''
+ """
Register command-line argument.
All given arguments will be passed directly to
@@ -685,11 +688,11 @@ class ArgumentPluginManager(PluginManagerBase):
See :meth:`argparse.ArgumentParser.add_argument` documentation for
further information.
- '''
+ """
self._argparse_argkwargs.append((args, kwargs))
def get_argument(self, name, default=None):
- '''
+ """
Get argument value from last :meth:`load_arguments` call.
Keep in mind :meth:`argparse.ArgumentParser.parse_args` generates
@@ -700,14 +703,14 @@ class ArgumentPluginManager(PluginManagerBase):
:type name: str
:param default: default value if parameter is not found
:returns: command-line argument or default value
- '''
+ """
return getattr(self._argparse_arguments, name, default)
class MimetypeActionPluginManager(WidgetPluginManager, MimetypePluginManager):
- '''
+ """
Deprecated plugin API
- '''
+ """
_deprecated_places = {
'javascript': 'scripts',
@@ -856,8 +859,8 @@ class PluginManager(MimetypeActionPluginManager,
MimetypePluginManager,
SessionPluginManager,
ArgumentPluginManager):
- '''
- Main plugin manager
+ """
+ Main plugin manager.
Provides:
* Plugin module loading and Flask extension logic.
@@ -878,9 +881,9 @@ class PluginManager(MimetypeActionPluginManager,
both :meth:`create_widget` and :meth:`register_widget` methods' `type`
parameter, or instantiated directly and passed to :meth:`register_widget`
via `widget` parameter.
- '''
+ """
def clear(self):
- '''
+ """
Clear plugin manager state.
Registered widgets will be disposed after calling this method.
@@ -890,5 +893,5 @@ class PluginManager(MimetypeActionPluginManager,
Registered command-line arguments will be disposed after calling this
method.
- '''
+ """
super(PluginManager, self).clear()
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index fc6ca4f..c896133 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -197,26 +197,33 @@ def detect_selection(directory):
current_app.config.get('DIRECTORY_UPLOAD')
-def excluded_clipboard(app, path):
- # TODO: get this working outside requests (for tarfile streaming)
- if not getattr(g, 'file_actions_paste', False) and \
- session.get('clipboard:mode') == 'cut':
- base = app.config['DIRECTORY_BASE']
+def excluded_clipboard(path):
+ """
+ Exclusion function for files in clipboard.
+
+ :param path: path to check
+ :type path: str
+ :return: wether path should be excluded or not
+ :rtype: str
+ """
+ if (
+ not getattr(g, 'file_actions_paste', False) and
+ session.get('clipboard:mode') == 'cut'
+ ):
+ base = current_app.config['DIRECTORY_BASE']
clipboard = session.get('clipboard:items', ())
return abspath_to_urlpath(path, base) in clipboard
return False
def register_plugin(manager):
- '''
+ """
Register blueprints and actions using given plugin manager.
:param manager: plugin manager
:type manager: browsepy.manager.PluginManager
- '''
- manager.register_exclude_function(
- functools.partial(excluded_clipboard, manager.app)
- )
+ """
+ manager.register_exclude_function(excluded_clipboard)
manager.register_blueprint(actions)
manager.register_widget(
place='styles',
diff --git a/browsepy/plugin/file_actions/exceptions.py b/browsepy/plugin/file_actions/exceptions.py
index 8a00a34..d2f0db4 100644
--- a/browsepy/plugin/file_actions/exceptions.py
+++ b/browsepy/plugin/file_actions/exceptions.py
@@ -5,11 +5,11 @@ import errno
class FileActionsException(Exception):
- '''
+ """
Base class for file-actions exceptions
:property path: item path which raised this Exception
- '''
+ """
code = None
template = 'Unhandled error.'
@@ -20,12 +20,12 @@ class FileActionsException(Exception):
class InvalidDirnameError(FileActionsException):
- '''
+ """
Exception raised when a new directory name is invalid.
:property path: item path which raised this Exception
:property name: name which raised this Exception
- '''
+ """
code = 'directory-invalid-name'
template = 'Clipboard item {0.name!r} is not valid.'
@@ -35,12 +35,12 @@ class InvalidDirnameError(FileActionsException):
class DirectoryCreationError(FileActionsException):
- '''
+ """
Exception raised when a new directory creation fails.
:property path: item path which raised this Exception
:property name: name which raised this Exception
- '''
+ """
code = 'directory-mkdir-error'
template = 'Clipboard item {0.name!r} is not valid.'
@@ -64,13 +64,13 @@ class DirectoryCreationError(FileActionsException):
class ClipboardException(FileActionsException):
- '''
+ """
Base class for clipboard exceptions.
:property path: item path which raised this Exception
:property mode: mode which raised this Exception
:property clipboard: :class Clipboard: instance
- '''
+ """
code = 'clipboard-invalid'
template = 'Clipboard is invalid.'
@@ -81,9 +81,9 @@ class ClipboardException(FileActionsException):
class ItemIssue(tuple):
- '''
+ """
Item/error issue
- '''
+ """
@property
def item(self):
return self[0]
@@ -107,14 +107,14 @@ class ItemIssue(tuple):
class InvalidClipboardItemsError(ClipboardException):
- '''
+ """
Exception raised when a clipboard item is not valid.
:property path: item path which raised this Exception
:property mode: mode which raised this Exception
:property clipboard: :class Clipboard: instance
:property issues: iterable of issues
- '''
+ """
pair_class = ItemIssue
code = 'clipboard-invalid-items'
template = 'Clipboard has invalid items.'
@@ -130,13 +130,13 @@ class InvalidClipboardItemsError(ClipboardException):
class InvalidClipboardModeError(ClipboardException):
- '''
+ """
Exception raised when a clipboard mode is not valid.
:property path: item path which raised this Exception
:property mode: mode which raised this Exception
:property clipboard: :class Clipboard: instance
- '''
+ """
code = 'clipboard-invalid-mode'
template = 'Clipboard mode {0.mode!r} is not valid.'
@@ -146,13 +146,13 @@ class InvalidClipboardModeError(ClipboardException):
class InvalidEmptyClipboardError(ClipboardException):
- '''
+ """
Exception raised when an invalid action is requested on an empty clipboard.
:property path: item path which raised this Exception
:property mode: mode which raised this Exception
:property clipboard: :class Clipboard: instance
- '''
+ """
code = 'clipboard-invalid-empty'
template = 'Clipboard action {0.mode!r} cannot be performed without items.'
@@ -162,13 +162,13 @@ class InvalidEmptyClipboardError(ClipboardException):
class InvalidClipboardSizeError(ClipboardException):
- '''
+ """
Exception raised when session manager evicts clipboard data.
:property path: item path which raised this Exception
:property mode: mode which raised this Exception
:property clipboard: :class Clipboard: instance
- '''
+ """
code = 'clipboard-invalid-size'
template = 'Clipboard evicted due session size limit.'
diff --git a/browsepy/plugin/file_actions/utils.py b/browsepy/plugin/file_actions/utils.py
index 694af61..2979c64 100644
--- a/browsepy/plugin/file_actions/utils.py
+++ b/browsepy/plugin/file_actions/utils.py
@@ -46,9 +46,9 @@ def move(target, node, join_fnc=os.path.join):
def paste(target, mode, clipboard):
- '''
+ """
Get pasting function for given directory and keyboard.
- '''
+ """
if mode == 'cut':
paste_fnc = functools.partial(move, target)
elif mode == 'copy':
diff --git a/browsepy/plugin/player/__init__.py b/browsepy/plugin/player/__init__.py
index fbc5e2d..b24625f 100644
--- a/browsepy/plugin/player/__init__.py
+++ b/browsepy/plugin/player/__init__.py
@@ -69,14 +69,14 @@ def directory(path):
def register_arguments(manager):
- '''
+ """
Register arguments using given plugin manager.
This method is called before `register_plugin`.
:param manager: plugin manager
:type manager: browsepy.manager.PluginManager
- '''
+ """
# Arguments are forwarded to argparse:ArgumentParser.add_argument,
# https://docs.python.org/3.7/library/argparse.html#the-add-argument-method
@@ -87,12 +87,12 @@ def register_arguments(manager):
def register_plugin(manager):
- '''
+ """
Register blueprints and actions using given plugin manager.
:param manager: plugin manager
:type manager: browsepy.manager.PluginManager
- '''
+ """
manager.register_blueprint(player)
manager.register_mimetype_function(detect_playable_mimetype)
diff --git a/browsepy/plugin/player/playable.py b/browsepy/plugin/player/playable.py
index c354dcd..1536a02 100644
--- a/browsepy/plugin/player/playable.py
+++ b/browsepy/plugin/player/playable.py
@@ -16,11 +16,11 @@ from browsepy.file import Node, File, Directory, \
class PLSFileParser(object):
- '''
+ """
ConfigParser wrapper accepting fallback on get for convenience.
This wraps instead of inheriting due ConfigParse being classobj on python2.
- '''
+ """
NOT_SET = type('NotSetType', (object,), {})
option_exceptions = (
six.moves.configparser.NoSectionError,
diff --git a/browsepy/stream.py b/browsepy/stream.py
index a5a79b3..50c918a 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -11,13 +11,13 @@ from . import compat
class ByteQueue(compat.Queue):
- '''
+ """
Small synchronized queue storing bytes, with an additional finish method
with turns the queue :method:`get` into non-blocking (returns empty bytes).
On a finished queue all :method:`put` will raise Full exceptions,
regardless of the parameters given.
- '''
+ """
def _init(self, maxsize):
self.queue = []
self.bytes = 0
@@ -42,18 +42,18 @@ class ByteQueue(compat.Queue):
return data
def qsize(self):
- '''
+ """
Return the number of bytes in the queue.
- '''
+ """
with self.mutex:
return self.bytes
def finish(self):
- '''
+ """
Turn queue into finished mode: :method:`get` becomes non-blocking
and returning empty bytes if empty, and :method:`put` raising
:class:`queue.Full` exceptions unconditionally.
- '''
+ """
self.finished = True
with self.not_full:
@@ -61,15 +61,15 @@ class ByteQueue(compat.Queue):
class WriteAbort(Exception):
- '''
+ """
Exception used internally by :class:`TarFileStream`'s default
implementation to stop tarfile compression.
- '''
+ """
pass
class TarFileStream(compat.Iterator):
- '''
+ """
Iterator class which yields tarfile chunks for streaming.
This class implements :class:`collections.abc.Iterator` interface
@@ -85,7 +85,7 @@ class TarFileStream(compat.Iterator):
If your exclude function requires accessing to :data:`flask.app`,
:data:`flask.g`, :data:`flask.request` or any other flask contexts,
ensure is wrapped with :func:`flask.copy_current_request_context`
- '''
+ """
queue_class = ByteQueue
abort_exception = WriteAbort
@@ -102,21 +102,21 @@ class TarFileStream(compat.Iterator):
@property
def name(self):
- '''
+ """
Filename generated from given path and compression method.
- '''
+ """
return '%s.%s' % (os.path.basename(self.path), self._extension)
@property
def encoding(self):
- '''
+ """
Mimetype parameters (such as encoding).
- '''
+ """
return self._compress
def __init__(self, path, buffsize=10240, exclude=None, compress='gzip',
compresslevel=1):
- '''
+ """
Initialize thread and class (thread is not started until interation).
Note that compress parameter will be ignored if buffsize is below 16.
@@ -131,7 +131,7 @@ class TarFileStream(compat.Iterator):
:type compress: str or None
:param compresslevel: compression level [1-9] defaults to 1 (fastest)
:type compresslevel: int
- '''
+ """
self.path = path
self.exclude = exclude
self.closed = False
@@ -148,28 +148,28 @@ class TarFileStream(compat.Iterator):
@property
def _infofilter(self):
- '''
+ """
TarInfo filtering function based on :attr:`exclude`.
- '''
+ """
path = self.path
path_join = os.path.join
exclude = self.exclude
def infofilter(info):
- '''
+ """
Filter TarInfo objects for TarFile.
:param info:
:type info: tarfile.TarInfo
:return: infofile or None if file must be excluded
:rtype: tarfile.TarInfo or None
- '''
+ """
return None if exclude(path_join(path, info.name)) else info
return infofilter if exclude else None
def _fill(self):
- '''
+ """
Writes data on internal tarfile instance, which writes to current
object, using :meth:`write`.
@@ -177,7 +177,7 @@ class TarFileStream(compat.Iterator):
This method is called automatically, on a thread, on initialization,
so there is little need to call it manually.
- '''
+ """
try:
with self.tarfile_class(
fileobj=self,
@@ -194,14 +194,14 @@ class TarFileStream(compat.Iterator):
self._queue.finish()
def __next__(self):
- '''
+ """
Pulls chunk from tarfile (which is processed on its own thread).
:param want: number bytes to read, defaults to 0 (all available)
:type want: int
:returns: tarfile data as bytes
:rtype: bytes
- '''
+ """
if self.closed:
raise StopIteration()
@@ -216,7 +216,7 @@ class TarFileStream(compat.Iterator):
return data
def write(self, data):
- '''
+ """
Put chunk of data into data queue, used on the tarfile thread.
This method blocks when pipe is already, applying backpressure to
@@ -227,7 +227,7 @@ class TarFileStream(compat.Iterator):
:returns: number of bytes written
:rtype: int
:raises WriteAbort: if already closed or closed while blocking
- '''
+ """
if self.closed:
raise self.abort_exception()
@@ -239,9 +239,9 @@ class TarFileStream(compat.Iterator):
return len(data)
def close(self):
- '''
+ """
Closes tarfile pipe and stops further processing.
- '''
+ """
if not self.closed:
self.closed = True
self._queue.finish()
@@ -252,7 +252,7 @@ class TarFileStream(compat.Iterator):
def stream_template(template_name, **context):
- '''
+ """
Some templates can be huge, this function returns an streaming response,
sending the content in chunks and preventing from timeout.
@@ -260,7 +260,7 @@ def stream_template(template_name, **context):
:param **context: parameters for templates.
:yields: HTML strings
:rtype: Iterator of str
- '''
+ """
app = context.get('current_app', flask.current_app)
app.update_template_context(context)
template = app.jinja_env.get_template(template_name)
diff --git a/browsepy/tests/deprecated/plugin/player.py b/browsepy/tests/deprecated/plugin/player.py
index be30474..07e7ab1 100644
--- a/browsepy/tests/deprecated/plugin/player.py
+++ b/browsepy/tests/deprecated/plugin/player.py
@@ -64,12 +64,12 @@ def detect_playable_mimetype(path, os_sep=os.sep):
def register_plugin(manager):
- '''
+ """
Register blueprints and actions using given plugin manager.
:param manager: plugin manager
:type manager: browsepy.manager.PluginManager
- '''
+ """
manager.register_blueprint(player)
manager.register_mimetype_function(detect_playable_mimetype)
diff --git a/browsepy/tests/meta.py b/browsepy/tests/meta.py
index 9ed086a..c9ee86e 100644
--- a/browsepy/tests/meta.py
+++ b/browsepy/tests/meta.py
@@ -2,7 +2,7 @@ from browsepy.compat import res
class TestFileMeta(type):
- '''
+ """
This metaclass generates test methods for every file matching given
rules (as class properties).
@@ -17,7 +17,7 @@ class TestFileMeta(type):
- meta_module: module to inspect for files
- meta_file_extensions: filename extensions will result on test injection.
- meta_prefix: prefix added to injected tests (defaults to `file`)
- '''
+ """
resource_methods = (
'contents',
diff --git a/browsepy/tests/test_code_quality.py b/browsepy/tests/test_code_quality.py
index a0b0b35..f5b3900 100644
--- a/browsepy/tests/test_code_quality.py
+++ b/browsepy/tests/test_code_quality.py
@@ -7,9 +7,8 @@ from . import meta
class TestCodeFormat(six.with_metaclass(meta.TestFileMeta, unittest.TestCase)):
- '''
- pycodestyle unit test
- '''
+ """pycodestyle unit test"""
+
meta_module = 'browsepy'
meta_prefix = 'code'
meta_file_extensions = ('.py',)
diff --git a/browsepy/tests/test_main.py b/browsepy/tests/test_main.py
index cb88d3c..ed747b3 100644
--- a/browsepy/tests/test_main.py
+++ b/browsepy/tests/test_main.py
@@ -1,12 +1,15 @@
-import unittest
+
+import io
import os
+import sys
import os.path
+import unittest
import tempfile
import shutil
+import contextlib
import browsepy
import browsepy.__main__ as main
-import browsepy.compat as compat
class TestMain(unittest.TestCase):
@@ -23,6 +26,16 @@ class TestMain(unittest.TestCase):
def tearDown(self):
shutil.rmtree(self.base)
+ @staticmethod
+ @contextlib.contextmanager
+ def stderr_ctx():
+ with io.StringIO() as f:
+ sys_sderr = sys.stderr
+ sys.stderr = f
+ yield f
+ if sys.stderr is f:
+ sys.stderr = sys_sderr
+
def test_defaults(self):
result = self.parser.parse_args([])
self.assertEqual(result.host, '127.0.0.1')
@@ -83,19 +96,18 @@ class TestMain(unittest.TestCase):
self.assertListEqual(result.exclude_from, [])
self.assertListEqual(result.plugin, [])
- with open(os.devnull, 'w') as f:
- with compat.redirect_stderr(f):
- self.assertRaises(
- SystemExit,
- self.parser.parse_args,
- ['--directory=%s' % __file__]
- )
-
- self.assertRaises(
- SystemExit,
- self.parser.parse_args,
- ['--exclude-from=non-existing']
- )
+ with self.stderr_ctx():
+ self.assertRaises(
+ SystemExit,
+ self.parser.parse_args,
+ ['--directory=%s' % __file__]
+ )
+
+ self.assertRaises(
+ SystemExit,
+ self.parser.parse_args,
+ ['--exclude-from=non-existing']
+ )
def test_exclude(self):
result = self.parser.parse_args([
diff --git a/browsepy/transform/__init__.py b/browsepy/transform/__init__.py
index ec8af0b..409542d 100755
--- a/browsepy/transform/__init__.py
+++ b/browsepy/transform/__init__.py
@@ -1,7 +1,7 @@
class StateMachine(object):
- '''
+ """
Abstract character-driven finite state machine implementation, used to
chop down and transform strings.
@@ -9,7 +9,7 @@ class StateMachine(object):
Important: when implementing this class, you must set the :attr:`current`
attribute to a key defined in :attr:`jumps` dict.
- '''
+ """
jumps = {} # finite state machine jumps
start = '' # character which started current state
current = '' # current state (an initial value must be set)
@@ -18,7 +18,7 @@ class StateMachine(object):
@property
def nearest(self):
- '''
+ """
Get the next state jump.
The next jump is calculated looking at :attr:`current` state
@@ -29,7 +29,7 @@ class StateMachine(object):
:returns: tuple with index, substring and next state label
:rtype: tuple
- '''
+ """
try:
options = self.jumps[self.current]
except KeyError:
@@ -55,14 +55,14 @@ class StateMachine(object):
return result
def __init__(self, data=''):
- '''
+ """
:param data: content will be added to pending data
:type data: str
- '''
+ """
self.pending += data
def __iter__(self):
- '''
+ """
Yield over result chunks, consuming :attr:`pending` data.
On :attr:`streaming` mode, yield only finished states.
@@ -72,7 +72,7 @@ class StateMachine(object):
:yields: transformation result chunka
:ytype: str
- '''
+ """
index, mark, next = self.nearest
while next is not None:
data = self.transform(self.pending[:index], mark, next)
@@ -90,7 +90,7 @@ class StateMachine(object):
yield data
def transform(self, data, mark, next):
- '''
+ """
Apply the appropriate transformation function on current state data,
which is supposed to end at this point.
@@ -107,31 +107,31 @@ class StateMachine(object):
:returns: transformed data
:rtype: str
- '''
+ """
method = getattr(self, 'transform_%s' % self.current, None)
return method(data, mark, next) if method else data
def feed(self, data=''):
- '''
+ """
Optionally add pending data, switch into streaming mode, and yield
result chunks.
:yields: result chunks
:ytype: str
- '''
+ """
self.streaming = True
self.pending += data
for i in self:
yield i
def finish(self, data=''):
- '''
+ """
Optionally add pending data, turn off streaming mode, and yield
result chunks, which implies all pending data will be consumed.
:yields: result chunks
:ytype: str
- '''
+ """
self.pending += data
self.streaming = False
for i in self:
diff --git a/browsepy/utils.py b/browsepy/utils.py
index d6dd38f..f7ffa2b 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -12,12 +12,12 @@ import flask
def ppath(*args, **kwargs):
- '''
+ """
Get joined file path relative to module location.
:param module: Module name
:type module: str
- '''
+ """
module = get_module(kwargs.pop('module', __name__))
if kwargs:
raise TypeError(
@@ -30,28 +30,28 @@ def ppath(*args, **kwargs):
@contextlib.contextmanager
def dummy_context():
- '''
+ """
Context manager which does nothing besides exposing the context
manger interface
- '''
+ """
yield
def get_module(name):
- '''
+ """
Get module object by name.
:param name: module name
:type name: str
:return: module or None if not found
:rtype: module or None
- '''
+ """
__import__(name)
return sys.modules.get(name, None)
def random_string(size, sample=tuple(map(chr, range(256)))):
- '''
+ """
Get random string of given size.
:param size: length of the returned string
@@ -60,34 +60,34 @@ def random_string(size, sample=tuple(map(chr, range(256)))):
:type sample: tuple of str
:returns: random string of specified size
:rtype: str
- '''
+ """
randrange = functools.partial(random.randrange, 0, len(sample))
return ''.join(sample[randrange()] for i in range(size))
def solve_local(context_local):
- '''
+ """
Resolve given context local to its actual value. If given object
it's not a context local nothing happens, just returns the same value.
- '''
+ """
if callable(getattr(context_local, '_get_current_object', None)):
return context_local._get_current_object()
return context_local
def clear_localstack(stack):
- '''
+ """
Clear given werkzeug LocalStack instance.
:param ctx: local stack instance
:type ctx: werkzeug.local.LocalStack
- '''
+ """
while stack.pop():
pass
def clear_flask_context():
- '''
+ """
Clear flask current_app and request globals.
When using :meth:`flask.Flask.test_client`, even as context manager,
@@ -96,13 +96,13 @@ def clear_flask_context():
This function clean said globals, and should be called after testing
with :meth:`flask.Flask.test_client`.
- '''
+ """
clear_localstack(flask._app_ctx_stack)
clear_localstack(flask._request_ctx_stack)
def defaultsnamedtuple(name, fields, defaults=None):
- '''
+ """
Generate namedtuple with default values.
:param name: name
@@ -110,7 +110,7 @@ def defaultsnamedtuple(name, fields, defaults=None):
:param defaults: iterable or mapping with field defaults
:returns: defaultdict with given fields and given defaults
:rtype: collections.defaultdict
- '''
+ """
nt = collections.namedtuple(name, fields)
nt.__new__.__defaults__ = (None, ) * len(nt._fields)
if isinstance(defaults, collections.Mapping):
diff --git a/browsepy/widget.py b/browsepy/widget.py
index 0ced542..bb43f42 100644
--- a/browsepy/widget.py
+++ b/browsepy/widget.py
@@ -1,10 +1,10 @@
# -*- coding: UTF-8 -*-
-'''
+"""
WARNING: deprecated module.
API defined in this module has been deprecated in version 0.5 and it will be
removed at 0.6.
-'''
+"""
import warnings
diff --git a/doc/plugins.rst b/doc/plugins.rst
index ffa34de..7541d40 100644
--- a/doc/plugins.rst
+++ b/doc/plugins.rst
@@ -74,17 +74,17 @@ manager itself (type :class:`PluginManager`) as first parameter.
Plugin manager exposes several methods to register widgets and mimetype
detection functions.
-A *sregister_plugin*s function looks like this (taken from player plugin):
+A *sregister_plugin* function looks like this (taken from player plugin):
.. code-block:: python
def register_plugin(manager):
- '''
+ """
Register blueprints and actions using given plugin manager.
:param manager: plugin manager
:type manager: browsepy.manager.PluginManager
- '''
+ """
manager.register_blueprint(player)
manager.register_mimetype_function(detect_playable_mimetype)
@@ -148,14 +148,14 @@ A simple `register_arguments` example (from player plugin):
.. code-block:: python
def register_arguments(manager):
- '''
+ """
Register arguments using given plugin manager.
This method is called before `register_plugin`.
:param manager: plugin manager
:type manager: browsepy.manager.PluginManager
- '''
+ """
# Arguments are forwarded to argparse:ArgumentParser.add_argument,
# https://docs.python.org/3.7/library/argparse.html#the-add-argument-method
@@ -191,7 +191,7 @@ Here is the "widget_types" for reference.
.. code-block:: python
class WidgetPluginManager(RegistrablePluginManager):
- ''' ... '''
+ # ...
widget_types = {
'base': defaultsnamedtuple(
'Widget',
@@ -219,6 +219,7 @@ Here is the "widget_types" for reference.
'Html',
('place', 'type', 'html')),
}
+ # ...
Function :func:`browsepy.file.defaultsnamedtuple` is a
:func:`collections.namedtuple` which uses a third argument dictionary
@@ -244,6 +245,50 @@ former case it is recommended to use
:meth:`browsepy.file.Node.from_urlpath` static method to create the
appropriate file/directory object (see :mod:`browsepy.file`).
+.. _plugins-exclude:
+
+File exclusions
+---------------
+
+Plugins can define their own file listing exclusion functions, so they can
+effectively control what it hidden when browsing, and excluded from
+directory tarballs.
+
+This is useful as plugins can now define hidden-file behaviors for user
+operations or disk-backed configuration files.
+
+
+.. code-block:: python
+
+ import os.path
+
+
+ def excluded_files(path):
+ """
+ Exclusion function for files in clipboard.
+
+ :param path: path to check
+ :type path: str
+ :return: wether path should be excluded or not
+ :rtype: str
+ """
+ return os.path.basename(path) == '.my-hidden-file':
+
+
+ def register_plugin(manager):
+ """
+ Register blueprints and actions using given plugin manager.
+
+ :param manager: plugin manager
+ :type manager: browsepy.manager.PluginManager
+ """
+ manager.register_exclude_function(excluded_files)
+
+
+Be careful when saving global state from an exclusion function using
+:module:`flask` contexts, as the request context is garanteed to be available
+to the function but not to be processed at this point.
+
.. _plugins-considerations:
Considerations
diff --git a/doc/quickstart.rst b/doc/quickstart.rst
index df20c96..7aed4ef 100644
--- a/doc/quickstart.rst
+++ b/doc/quickstart.rst
@@ -56,10 +56,9 @@ this list.
::
- usage: browsepy [-h] [--directory PATH] [--initial PATH]
- [--removable PATH] [--upload PATH]
- [--exclude PATTERN] [--exclude-from PATH]
- [--plugin MODULE]
+ usage: browsepy [-h] [--help-all] [--directory PATH] [--initial PATH]
+ [--removable PATH] [--upload PATH] [--exclude PATTERN]
+ [--exclude-from PATH] [--version] [--plugin MODULE]
[host] [port]
description: starts a browsepy web file browser
@@ -70,14 +69,20 @@ this list.
optional arguments:
-h, --help show this help message and exit
- --directory PATH serving directory (default: current path)
+ --help-all show help for all available plugins and exit
+ --directory PATH serving directory (default: /my/current/path)
--initial PATH default directory (default: same as --directory)
- --removable PATH base directory allowing remove (default: none)
- --upload PATH base directory allowing upload (default: none)
+ --removable PATH base directory allowing remove (default: None)
+ --upload PATH base directory allowing upload (default: None)
--exclude PATTERN exclude paths by pattern (multiple)
--exclude-from PATH exclude paths by pattern file (multiple)
+ --version show program's version number and exit
--plugin MODULE load plugin module (multiple)
+ available plugins:
+ file-actions, browsepy.plugin.file_actions
+ player, browsepy.plugin.player
+
Showing help including player plugin arguments:
.. code-block:: bash
@@ -89,10 +94,10 @@ Please note the extra parameters below `player arguments`.
::
- usage: browsepy [-h] [--directory PATH] [--initial PATH]
- [--removable PATH] [--upload PATH]
- [--exclude PATTERN] [--exclude-from PATH]
- [--plugin MODULE] [--player-directory-play]
+ usage: browsepy [-h] [--help-all] [--directory PATH] [--initial PATH]
+ [--removable PATH] [--upload PATH] [--exclude PATTERN]
+ [--exclude-from PATH] [--version] [--plugin MODULE]
+ [--player-directory-play]
[host] [port]
description: starts a browsepy web file browser
@@ -103,14 +108,20 @@ Please note the extra parameters below `player arguments`.
optional arguments:
-h, --help show this help message and exit
- --directory PATH serving directory (default: current path)
+ --help-all show help for all available plugins and exit
+ --directory PATH serving directory (default: /my/current/path)
--initial PATH default directory (default: same as --directory)
- --removable PATH base directory allowing remove (default: none)
- --upload PATH base directory allowing upload (default: none)
+ --removable PATH base directory allowing remove (default: None)
+ --upload PATH base directory allowing upload (default: None)
--exclude PATTERN exclude paths by pattern (multiple)
--exclude-from PATH exclude paths by pattern file (multiple)
+ --version show program's version number and exit
--plugin MODULE load plugin module (multiple)
player arguments:
--player-directory-play
enable directories as playlist
+
+ available plugins:
+ file-actions, browsepy.plugin.file_actions
+ player, browsepy.plugin.player
--
GitLab
From 2679ab2122948b13b24b882ff4bef5a9917041f6 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 4 Jul 2019 22:39:29 +0100
Subject: [PATCH 078/171] more cleanup
---
.travis.yml | 4 ----
browsepy/compat.py | 22 +++++++++++-----------
doc/builtin_plugins.rst | 12 ++++++------
doc/conf.py | 1 -
doc/index.rst | 8 ++++----
5 files changed, 21 insertions(+), 26 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 3b448c3..868ec41 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,10 +18,6 @@ matrix:
- sphinx=yes
install:
- - |
- if [ "$legacy" != "yes" ]; then
- pip install --upgrade pip setuptools
- fi
- pip install -r requirements/ci.txt
- pip install .
- |
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 1021281..f7a7f63 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -19,7 +19,7 @@ PY_LEGACY = sys.version_info < (3, )
TRUE_VALUES = frozenset(('true', 'yes', '1', 'enable', 'enabled', True, 1))
try:
- import importlib.resources as res # to support python >= 3.7
+ import importlib.resources as res # python 3.7+
except ImportError:
import importlib_resources as res # noqa
@@ -388,16 +388,6 @@ def re_escape(pattern, chars=frozenset("()[]{}?*+|^$\\.-#")):
)
-class Iterator(BaseIterator):
- def next(self):
- """
- Call :method:`__next__` for compatibility.
-
- :returns: see :method:`__next__`
- """
- return self.__next__()
-
-
if PY_LEGACY:
class FileNotFoundError(BaseException):
__metaclass__ = abc.ABCMeta
@@ -405,6 +395,15 @@ if PY_LEGACY:
FileNotFoundError.register(OSError)
FileNotFoundError.register(IOError)
+ class Iterator(BaseIterator):
+ def next(self):
+ """
+ Call :method:`__next__` for compatibility.
+
+ :returns: see :method:`__next__`
+ """
+ return self.__next__()
+
range = xrange # noqa
filter = itertools.ifilter
map = itertools.imap
@@ -414,6 +413,7 @@ if PY_LEGACY:
bytes = str # noqa
else:
FileNotFoundError = FileNotFoundError
+ Iterator = BaseIterator
range = range
filter = filter
map = map
diff --git a/doc/builtin_plugins.rst b/doc/builtin_plugins.rst
index f5f2224..9ccdd8d 100644
--- a/doc/builtin_plugins.rst
+++ b/doc/builtin_plugins.rst
@@ -29,16 +29,16 @@ Sources are available at browsepy's `plugin.player`_ submodule.
Contributing Builtin Plugins
----------------------------
-Browsepy's team is open to contributions of any kind, even about adding
-built-in plugins, as long as they comply with the following requirements:
+Browsepy's team is open to contributions of any kind, including built-in
+plugins, as long as they comply with the following requirements:
* Plugins must be sufficiently covered by tests to avoid lowering browsepy's
- overall test coverage.
-* Plugins must not add external requirements to browsepy, optional
- requirements are allowed if plugin can work without them, even with
+ global test coverage.
+* Plugins must not add external requirements to browsepy, but optional
+ requirements will be allowed if plugin can work without them, even with
limited functionality.
* Plugins should avoid adding specific logic on browsepy itself, but extending
- browsepy's itself (specially via plugin interface) in a generic and useful
+ browsepy itself (specially on the plugin interface) in a generic and useful
way is definitely welcome.
Said that, feel free to fork, code great stuff and fill pull requests at
diff --git a/doc/conf.py b/doc/conf.py
index e565472..4781e48 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -21,7 +21,6 @@ import os
import io
import re
import sys
-import time
sys.path.insert(0, os.path.abspath('..')) # noqa
diff --git a/doc/index.rst b/doc/index.rst
index 1a2dc88..2bf57db 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -11,8 +11,8 @@ Welcome to browsepy's documentation. It's recommended to start reading both
detailed tutorials about integrating :mod:`browsepy` as module or plugin
development are also available.
-Browsepy has few dependencies: `Flask`_ and `Scandir`_. `Flask`_ is an awesome
-web microframework while `Scandir`_ is a directory listing library which `was
+Browsepy has few dependencies: `Flask`_ and `scandir`_. `Flask`_ is an awesome
+web microframework while `scandir`_ is a directory listing library which `was
included `_ in Python 3.5's
standard library.
@@ -26,11 +26,11 @@ If you want to dive into their documentation, check out the following links:
`_
.. _Flask: http://jinja.pocoo.org/
-.. _Scandir: http://werkzeug.pocoo.org/
+.. _scandir: http://werkzeug.pocoo.org/
User's Guide
============
-Instructions for users, implementers and developers.
+Instructions for users, implementors and developers.
.. toctree::
:maxdepth: 2
--
GitLab
From 2a5e9a3c8204f8bd67e9391209513ce416fbfbe4 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 4 Jul 2019 22:48:01 +0100
Subject: [PATCH 079/171] small README changes
---
README.rst | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/README.rst b/README.rst
index 8a2ef3f..a4768b7 100644
--- a/README.rst
+++ b/README.rst
@@ -25,7 +25,7 @@ browsepy
:target: https://pypi.python.org/pypi/browsepy/
:alt: Version: 0.5.6
-.. image:: https://img.shields.io/badge/python-2.7%2B%2C%203.4%2B-FFC100.svg?style=flat-square
+.. image:: https://img.shields.io/badge/python-2.7%2B%2C%203.5%2B-FFC100.svg?style=flat-square
:target: https://pypi.python.org/pypi/browsepy/
:alt: Python 2.7+, 3.5+
@@ -43,7 +43,7 @@ You can also build yourself from sphinx sources using the documentation
License
-------
-MIT. See `LICENSE`_.
+MIT License. See `LICENSE`_.
.. _LICENSE: https://raw.githubusercontent.com/ergoithz/browsepy/master/LICENSE
@@ -198,7 +198,7 @@ Starting from version 0.6.0, browsepy a new plugin `file-actions` is included
providing copy/cut/paste and directory creation operations.
Plugins can add HTML content to browsepy's browsing view, using some
-convenience abstraction for already used elements like external stylesheet and
-javascript tags, links, buttons and file upload.
+convenience abstraction for frequently used elements like external stylesheets,
+javascript, links, buttons and file upload.
More information at http://ergoithz.github.io/browsepy/plugins.html
--
GitLab
From fdcf2bc2f9f2980b96f8cf4be487f08bf6485a76 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 14 Jul 2019 11:44:41 +0100
Subject: [PATCH 080/171] add gitlab-ci
---
.gitlab-ci.yml | 160 +++++++++++++++++++++++++++++++++++++++++++++++++
.travis.yml | 68 ---------------------
MANIFEST.in | 5 ++
Makefile | 10 ----
4 files changed, 165 insertions(+), 78 deletions(-)
create mode 100644 .gitlab-ci.yml
delete mode 100644 .travis.yml
delete mode 100644 Makefile
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..4193316
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,160 @@
+.test-base:
+ stage: test
+ variables:
+ PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
+ COVERAGE_FILE: "${CI_PROJECT_DIR}/.coverage.${CI_JOB_NAME}"
+ cache:
+ paths:
+ - .cache/pip
+ artifacts:
+ when: on_success
+ expire_in: 1h
+ paths:
+ - .coverage.*
+ before_script:
+ - apk add --no-cache build-base python3-dev
+ - pip install coverage
+ script:
+ - flake8 browsepy
+ - coverage run -p setup.py test
+
+.after-base:
+ image: python:3.7-alpine
+ variables:
+ PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
+ cache:
+ paths:
+ - .cache/pip
+
+.github-base:
+ image: alpine/git
+ before_script:
+ - mkdir -p ~/.ssh
+ - echo "${SSH_PRIVATE_KEY}" | tr -d '\r' > ~/.ssh/id_rsa
+ - chmod 600 ~/.ssh/id_rsa
+ - ssh-keyscan -H $ssh_port "$ssh_host" >> ~/.ssh/known_hosts
+ - git config user.name "${GITLAB_USER_LOGIN}"
+ - git config user.email "${GITLAB_USER_EMAIL}"
+
+stages:
+- test
+- report
+- publish
+
+python27:
+ extends: .test-base
+ image: python:2.7-alpine
+ before_script:
+ - apk add --no-cache build-base python2-dev
+ - pip install coverage
+
+python35:
+ extends: .test-base
+ image: python:3.5-alpine
+
+python36:
+ extends: .test-base
+ image: python:3.6-alpine
+
+python37:
+ extends: .test-base
+ image: python:3.7-alpine
+
+node:
+ image: node:alpine
+ stage: test
+ cache:
+ paths:
+ - node_modules
+ before_script:
+ - npm install eslint
+ script:
+ - node_modules/.bin/eslint browsepy
+
+coverage:
+ extends: .after-base
+ stage: report
+ artifacts:
+ when: always
+ paths:
+ - htmlcov
+ coverage: '/^TOTAL\s+(?:\d+\s+)*(\d+\%)$/'
+ script:
+ - pip install coverage
+ - coverage combine
+ - coverage html --fail-under=0
+ - coverage report
+
+doc:
+ extends: .after-base
+ stage: report
+ dependencies: []
+ artifacts:
+ paths:
+ - doc/.build
+ before_script:
+ - pip install -r requirements/doc.txt
+ script:
+ - make -C doc html
+
+publish-github-mirror:
+ extends: .github-base
+ stage: publish
+ dependencies: []
+ only:
+ refs:
+ - next
+ - master
+ script:
+ - git remote add github git@github.com:ergoithz/browsepy.git
+ - git push github
+
+publish-github-pages:
+ extends: .github-base
+ stage: publish
+ dependencies:
+ - doc
+ only:
+ refs:
+ - master
+ script:
+ - git remote add github git@github.com:ergoithz/browsepy.git
+ - git fetch
+ - git checkout gh-pages
+ - cp -rf doc/.build/* .
+ - rm -rf doc
+ - git add **
+ - git commit -am "doc from ${CI_COMMIT_SHA} ${CI_COMMIT_MESSAGE}"
+ - git push github
+
+publish-alpha:
+ extends: .after-base
+ stage: publish
+ dependencies: []
+ only:
+ refs:
+ - next
+ before_script:
+ - pip install twine
+ script:
+ - |
+ ALPHA=$(date +%s)
+ SED_REGEX="s/^__version__\s*=\s*[\"']([^\"']+)[\"']\$/__version__ = '\\1a${ALPHA}'/g"
+ SED_PATH="unittest_resources/__init__.py"
+ sed -ri "${SED_REGEX}" "${SED_PATH}"
+ python setup.py bdist_wheel sdist
+ twine upload --repository-url=https://test.pypi.org/legacy/ dist/*
+
+publish:
+ extends: .after-base
+ stage: publish
+ dependencies: []
+ only:
+ refs:
+ - master
+ before_script:
+ - pip install twine
+ script:
+ - |
+ python setup.py bdist_wheel sdist
+ twine upload --repository-url=https://upload.pypi.org/legacy/ dist/*
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 868ec41..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,68 +0,0 @@
-language: python
-
-addon:
- apt:
- nodejs
-
-matrix:
- include:
- - python: "2.7"
- - python: "pypy"
- - python: "pypy3"
- - python: "3.5"
- - python: "3.6"
- - python: "3.7"
- dist: xenial
- env:
- - eslint=yes
- - sphinx=yes
-
-install:
- - pip install -r requirements/ci.txt
- - pip install .
- - |
- if [ "$eslint" = "yes" ]; then
- nvm install stable
- nvm use stable
- npm install eslint
- fi
- - |
- if [ "$sphinx" = "yes" ]; then
- pip install -r requirements/doc.txt
- fi
-
-script:
- - flake8 browsepy
- - coverage run setup.py test
- - coverage report
- - |
- if [ "$eslint" = "yes" ]; then
- node_modules/.bin/eslint browsepy
- fi
- - |
- if [ "$sphinx" = "yes" ]; then
- make -C doc html
- fi
-
-after_success:
- - codecov
- - coveralls
- - |
- if [ "$sphinx" = "yes" ] && \
- [ "$TRAVIS_BRANCH" = "master" ] && \
- [ "$TRAVIS_PULL_REQUEST" = "false" ]; then
- ghp-import \
- --push \
- --force \
- --no-jekyll \
- --branch=gh-pages \
- --remote="https://${GH_TOKEN}@github.com${TRAVIS_REPO_SLUG}.git" \
- doc/.build/html
- fi
-
-notifications:
- email: false
-
-cache:
- directories:
- - $HOME/.cache/pip
diff --git a/MANIFEST.in b/MANIFEST.in
index 0545255..78796c2 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,7 +1,12 @@
include README.rst
+include LICENSE
+
graft browsepy/templates
graft browsepy/static
graft browsepy/plugin/player/templates
graft browsepy/plugin/player/static
graft browsepy/plugin/file_actions/templates
graft browsepy/plugin/file_actions/static
+
+global-exclude *.py[co]
+global-exclude __pycache__
diff --git a/Makefile b/Makefile
deleted file mode 100644
index 56f2e1a..0000000
--- a/Makefile
+++ /dev/null
@@ -1,10 +0,0 @@
-.PHONY: clean
-
-clean:
- rm -rf \
- build \
- dist \
- browsepy.egg-info
- find browsepy -type f -name "*.py[co]" -delete
- find browsepy -type d -name "__pycache__" -delete
- $(MAKE) -C doc clean
--
GitLab
From bb93ae723234373919f8aac94006e0ea57a40139 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 14 Jul 2019 11:48:00 +0100
Subject: [PATCH 081/171] add missing ci dependency
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4193316..3e16ba9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -13,7 +13,7 @@
- .coverage.*
before_script:
- apk add --no-cache build-base python3-dev
- - pip install coverage
+ - pip install coverage flake8
script:
- flake8 browsepy
- coverage run -p setup.py test
--
GitLab
From fb74a13a0800fb9b6c8b80a6e8869653682d9596 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 14 Jul 2019 11:52:31 +0100
Subject: [PATCH 082/171] fix eslint
---
.eslintrc.json | 2 +-
browsepy/plugin/file_actions/static/script.js | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/.eslintrc.json b/.eslintrc.json
index 668c8d9..d379fb1 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -5,7 +5,7 @@
},
"extends": "eslint:recommended",
"parserOptions": {
- "sourceType": "module"
+ "sourceType": "script"
},
"rules": {
"indent": [
diff --git a/browsepy/plugin/file_actions/static/script.js b/browsepy/plugin/file_actions/static/script.js
index c26f09c..ffc6bf1 100644
--- a/browsepy/plugin/file_actions/static/script.js
+++ b/browsepy/plugin/file_actions/static/script.js
@@ -1,6 +1,6 @@
(function() {
function every(arr, fnc) {
- for (let i = 0, l = arr.length; i < l; i++) {
+ for (var i = 0, l = arr.length; i < l; i++) {
if (!fnc(arr[i], i, arr)) {
return false;
}
@@ -53,7 +53,7 @@
.forEach(function (tr) {
tr.className += ' clickable';
event(tr, 'click', checkRow, checkbox, tr);
- event(tr, 'selectstart', e => e.preventDefault());
+ event(tr, 'selectstart', function(e) {return e.preventDefault();});
});
check(checkbox);
});
--
GitLab
From 7c8630a145b4b0235b148e9c0626d9fc9547c120 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 14 Jul 2019 11:54:35 +0100
Subject: [PATCH 083/171] add missing flake8
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3e16ba9..9fed602 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -46,7 +46,7 @@ python27:
image: python:2.7-alpine
before_script:
- apk add --no-cache build-base python2-dev
- - pip install coverage
+ - pip install coverage flake8
python35:
extends: .test-base
--
GitLab
From 2e43e2f0cbf59d54dbb40dec9b9cc045c091262d Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 14 Jul 2019 11:57:03 +0100
Subject: [PATCH 084/171] fix unused imports
---
browsepy/file.py | 1 -
browsepy/plugin/file_actions/__init__.py | 2 --
2 files changed, 3 deletions(-)
diff --git a/browsepy/file.py b/browsepy/file.py
index eeab4a8..7c553e6 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -9,7 +9,6 @@ import string
import random
import datetime
import logging
-import functools
import flask
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index c896133..f27b8c5 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -1,7 +1,5 @@
# -*- coding: UTF-8 -*-
-import functools
-
from flask import Blueprint, render_template, request, redirect, url_for, \
session, current_app, g
from werkzeug.exceptions import NotFound
--
GitLab
From 9f8cfe76fb0272fea12f2eaeb0d613fc6c08286f Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 14 Jul 2019 12:58:14 +0100
Subject: [PATCH 085/171] avoid testing responses outside client context
---
.gitlab-ci.yml | 2 +-
browsepy/plugin/player/tests.py | 54 ++++++++++++++++++---------------
2 files changed, 30 insertions(+), 26 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9fed602..517646f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -60,7 +60,7 @@ python37:
extends: .test-base
image: python:3.7-alpine
-node:
+eslint:
image: node:alpine
stage: test
cache:
diff --git a/browsepy/plugin/player/tests.py b/browsepy/plugin/player/tests.py
index a2c7003..b5b9abb 100644
--- a/browsepy/plugin/player/tests.py
+++ b/browsepy/plugin/player/tests.py
@@ -336,12 +336,6 @@ class TestBlueprint(TestPlayerBase):
with self.app.app_context():
return flask.url_for(endpoint, **kwargs)
- def get(self, endpoint, **kwargs):
- with self.app.test_client() as client:
- url = self.url_for(endpoint, **kwargs)
- response = client.get(url)
- return response
-
def file(self, path, data=''):
apath = p(self.app.config['DIRECTORY_BASE'], path)
with open(apath, 'w') as f:
@@ -355,31 +349,41 @@ class TestBlueprint(TestPlayerBase):
def test_playable(self):
name = 'test.mp3'
- result = self.get('player.audio', path=name)
- self.assertEqual(result.status_code, 404)
- self.file(name)
- result = self.get('player.audio', path=name)
- self.assertEqual(result.status_code, 200)
+ url = self.url_for('player.audio', path=name)
+ with self.app.test_client() as client:
+ result = client.get(url)
+ self.assertEqual(result.status_code, 404)
+
+ self.file(name)
+ result = client.get(url)
+ self.assertEqual(result.status_code, 200)
def test_playlist(self):
name = 'test.m3u'
- result = self.get('player.playlist', path=name)
- self.assertEqual(result.status_code, 404)
- self.file(name)
- result = self.get('player.playlist', path=name)
- self.assertEqual(result.status_code, 200)
+ url = self.url_for('player.playlist', path=name)
+ with self.app.test_client() as client:
+ result = client.get(url)
+ self.assertEqual(result.status_code, 404)
+
+ self.file(name)
+ result = client.get(url)
+ self.assertEqual(result.status_code, 200)
def test_directory(self):
name = 'directory'
- result = self.get('player.directory', path=name)
- self.assertEqual(result.status_code, 404)
- self.directory(name)
- result = self.get('player.directory', path=name)
- self.assertEqual(result.status_code, 200)
- self.file('directory/test.mp3')
- result = self.get('player.directory', path=name)
- self.assertEqual(result.status_code, 200)
- self.assertIn(b'test.mp3', result.data)
+ url = self.url_for('player.directory', path=name)
+ with self.app.test_client() as client:
+ result = client.get(url)
+ self.assertEqual(result.status_code, 404)
+
+ self.directory(name)
+ result = client.get(url)
+ self.assertEqual(result.status_code, 200)
+
+ self.file('directory/test.mp3')
+ result = client.get(url)
+ self.assertEqual(result.status_code, 200)
+ self.assertIn(b'test.mp3', result.data)
def test_endpoints(self):
with self.app.app_context():
--
GitLab
From 9b6eb16b9eb269f728b2de99dd1d738457f452a6 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 14 Jul 2019 13:19:46 +0100
Subject: [PATCH 086/171] fix unicode error
---
browsepy/tests/test_main.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/browsepy/tests/test_main.py b/browsepy/tests/test_main.py
index ed747b3..dad9adb 100644
--- a/browsepy/tests/test_main.py
+++ b/browsepy/tests/test_main.py
@@ -29,7 +29,7 @@ class TestMain(unittest.TestCase):
@staticmethod
@contextlib.contextmanager
def stderr_ctx():
- with io.StringIO() as f:
+ with io.BytesIO() as f:
sys_sderr = sys.stderr
sys.stderr = f
yield f
--
GitLab
From 4d893185d5f6ce5bf3b5600bd0728a428ef344b2 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 14 Jul 2019 13:58:17 +0100
Subject: [PATCH 087/171] update tests
---
.gitignore | 22 ++++-----
browsepy/tests/meta.py | 72 -----------------------------
browsepy/tests/test_code.py | 34 ++++++++++++++
browsepy/tests/test_code_quality.py | 28 -----------
browsepy/tests/test_main.py | 8 ++--
setup.py | 19 ++++----
6 files changed, 60 insertions(+), 123 deletions(-)
delete mode 100644 browsepy/tests/meta.py
create mode 100644 browsepy/tests/test_code.py
delete mode 100644 browsepy/tests/test_code_quality.py
diff --git a/.gitignore b/.gitignore
index 82e54ab..8d76b59 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,23 +2,21 @@
.c9
.idea
.coverage
+
htmlcov
dist
build
-doc/.build
-env
-env2
-env3
-wenv2
-wenv3
-penv2
-penv3
-.eggs
-browsepy.build
-browsepy.egg-info
-**/__pycache__
+env*
+wenv*
+package-lock.json
+
+**.build
+**.egg-info
**.egg
**.eggs
**.egg-info
**.pyc
+
**/node_modules
+**/*.py[co]
+**/__pycache__
diff --git a/browsepy/tests/meta.py b/browsepy/tests/meta.py
deleted file mode 100644
index c9ee86e..0000000
--- a/browsepy/tests/meta.py
+++ /dev/null
@@ -1,72 +0,0 @@
-from browsepy.compat import res
-
-
-class TestFileMeta(type):
- """
- This metaclass generates test methods for every file matching given
- rules (as class properties).
-
- The added test methods will call an existing meta_test method passing
- module name and filename as arguments.
-
- Functions from :module:`importlib.resources` (see resource_methods) are
- also injected for convenience.
-
- Honored class properties:
- - meta_test: called with module and filename by injected test methods.
- - meta_module: module to inspect for files
- - meta_file_extensions: filename extensions will result on test injection.
- - meta_prefix: prefix added to injected tests (defaults to `file`)
- """
-
- resource_methods = (
- 'contents',
- 'is_resource',
- 'open_binary',
- 'open_text',
- 'path',
- 'read_binary',
- 'read_text',
- )
-
- @classmethod
- def iter_contents(cls, module, extensions):
- for item in res.contents(module):
- if res.is_resource(module, item):
- if any(map(item.endswith, extensions)):
- yield (module, item)
- continue
- submodule = '%s.%s' % (module, item)
- try:
- for subitem in cls.iter_contents(submodule, extensions):
- yield subitem
- except ImportError:
- pass
-
- @classmethod
- def create_method(cls, module, filename, prefix, extensions):
- def test(self):
- self.meta_test(module, filename)
- strip = max(len(ext) for ext in extensions if filename.endswith(ext))
- test.__name__ = 'test_%s_%s_%s' % (
- prefix,
- module.replace('.', '_').strip('_'),
- filename[:-strip].strip('_'),
- )
- return test
-
- def __init__(self, name, bases, dct):
- super(TestFileMeta, self).__init__(name, bases, dct)
-
- # generate tests from files
- name = self.meta_module
- prefix = getattr(self, 'meta_prefix', 'file')
- extensions = self.meta_file_extensions
- for module, item in self.iter_contents(name, extensions):
- test = self.create_method(module, item, prefix, extensions)
- setattr(self, test.__name__, test)
-
- # add resource methods
- for method in self.resource_methods:
- if not hasattr(self, method):
- setattr(self, method, staticmethod(getattr(res, method)))
diff --git a/browsepy/tests/test_code.py b/browsepy/tests/test_code.py
new file mode 100644
index 0000000..5146df2
--- /dev/null
+++ b/browsepy/tests/test_code.py
@@ -0,0 +1,34 @@
+
+import unittest_resources.testing as bases
+
+
+class TypingTestCase(bases.TypingTestCase):
+ """TestCase checking :module:`mypy`."""
+
+ meta_module = 'browsepy'
+
+
+class CodeStyleTestCase(bases.CodeStyleTestCase):
+ """TestCase checking :module:`pycodestyle`."""
+
+ meta_module = 'browsepy'
+
+
+class DocStyleTestCase(bases.DocStyleTestCase):
+ """TestCase checking :module:`pydocstyle`."""
+
+ meta_module = 'browsepy'
+
+
+class MaintainabilityIndexTestCase(bases.MaintainabilityIndexTestCase):
+ """TestCase checking :module:`radon` maintainability index."""
+
+ meta_module = 'browsepy'
+
+
+class CodeComplexityTestCase(bases.CodeComplexityTestCase):
+ """TestCase checking :module:`radon` code complexity."""
+
+ meta_module = 'browsepy'
+ max_class_complexity = 7
+ max_function_complexity = 7
diff --git a/browsepy/tests/test_code_quality.py b/browsepy/tests/test_code_quality.py
deleted file mode 100644
index f5b3900..0000000
--- a/browsepy/tests/test_code_quality.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import unittest
-
-import six
-import pycodestyle
-
-from . import meta
-
-
-class TestCodeFormat(six.with_metaclass(meta.TestFileMeta, unittest.TestCase)):
- """pycodestyle unit test"""
-
- meta_module = 'browsepy'
- meta_prefix = 'code'
- meta_file_extensions = ('.py',)
-
- def meta_test(self, module, filename):
- style = pycodestyle.StyleGuide(quiet=False)
- with self.path(module, filename) as f:
- result = style.check_files([str(f)])
- self.assertFalse(result.total_errors, (
- 'Found {errors} code style error{s} (or warning{s}) '
- 'on module {module}, file {filename!r}.').format(
- errors=result.total_errors,
- s='s' if result.total_errors > 1 else '',
- module=module,
- filename=filename,
- )
- )
diff --git a/browsepy/tests/test_main.py b/browsepy/tests/test_main.py
index dad9adb..4bf1eaf 100644
--- a/browsepy/tests/test_main.py
+++ b/browsepy/tests/test_main.py
@@ -10,10 +10,12 @@ import contextlib
import browsepy
import browsepy.__main__ as main
+import browsepy.compat as compat
class TestMain(unittest.TestCase):
module = main
+ stream_class = io.BytesIO if compat.PY_LEGACY else io.StringIO
def setUp(self):
self.app = browsepy.app
@@ -26,10 +28,10 @@ class TestMain(unittest.TestCase):
def tearDown(self):
shutil.rmtree(self.base)
- @staticmethod
+ @classmethod
@contextlib.contextmanager
- def stderr_ctx():
- with io.BytesIO() as f:
+ def stderr_ctx(cls):
+ with cls.stream_class() as f:
sys_sderr = sys.stderr
sys.stderr = f
yield f
diff --git a/setup.py b/setup.py
index 7f33865..c812a68 100644
--- a/setup.py
+++ b/setup.py
@@ -57,19 +57,22 @@ setup(
'static/*',
],
},
+ setup_requires=[
+ 'setuptools>36.2',
+ ],
install_requires=[
'flask',
'unicategories',
'cookieman',
- 'backports.shutil_get_terminal_size ; python_version<\'3.3\'',
- 'scandir ; python_version<\'3.5\'',
- 'pathlib2 ; python_version<\'3.5\'',
- 'importlib_resources ; python_version<\'3.7\'',
+ 'backports.shutil_get_terminal_size ; python_version<"3.3"',
+ 'scandir ; python_version<"3.5"',
+ 'pathlib2 ; python_version<"3.5"',
+ 'importlib-resources ; python_version<"3.7"',
+ ],
+ tests_require=[
+ 'beautifulsoup4',
+ 'unittest-resources',
],
- extras_require={
- 'tests': ['beautifulsoup4', 'pycodestyle'],
- },
- tests_require=['beautifulsoup4', 'pycodestyle'],
test_suite='browsepy.tests',
test_runner='browsepy.tests.runner:DebuggerTextTestRunner',
zip_safe=False,
--
GitLab
From bbbd32bf3bed6ffb7e74df7ad0fb90b5c501f2fd Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 31 Jul 2019 15:48:55 +0100
Subject: [PATCH 088/171] fix default argument parser
---
browsepy/manager.py | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/browsepy/manager.py b/browsepy/manager.py
index edecafe..70e8f60 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -570,6 +570,13 @@ class ArgumentPluginManager(PluginManagerBase):
_argparse_kwargs = {'add_help': False}
_argparse_arguments = argparse.Namespace()
+ @cached_property
+ def _default_argument_parser(self):
+ parser = compat.SafeArgumentParser()
+ parser.add_argument('--plugin', action='append', default=[])
+ parser.add_argument('--help-all', action='store_true')
+ return parser
+
def __init__(self, app=None):
super(ArgumentPluginManager, self).__init__(app)
self.plugin_filters.append(
@@ -593,12 +600,6 @@ class ArgumentPluginManager(PluginManagerBase):
return manager._argparse_argkwargs
return ()
- def _default_argument_parser(self):
- parser = compat.SafeArgumentParser()
- parser.add_argument('--plugin', action='append', default=[])
- parser.add_argument('--help-all', action='store_true')
- return parser
-
def _plugin_argument_parser(self, base=None):
plugins = self.available_plugins
parent = base or self._default_argument_parser
--
GitLab
From 80fadb8884ccebd14982854d62d2311b8b5025cd Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 31 Jul 2019 15:58:56 +0100
Subject: [PATCH 089/171] do not use h3 on autosubmit form
---
browsepy/static/base.css | 26 +++++++++-----------------
browsepy/templates/browse.html | 2 +-
2 files changed, 10 insertions(+), 18 deletions(-)
diff --git a/browsepy/static/base.css b/browsepy/static/base.css
index a335628..e13913b 100644
--- a/browsepy/static/base.css
+++ b/browsepy/static/base.css
@@ -60,22 +60,20 @@ a:active {
color: #666;
}
-form.autosubmit h3 {
- font-size: 1em;
- font-weight: normal;
- margin: 0;
- display: none;
-}
-html.autosubmit-support form.autosubmit h3 {
- display: inline-block;
-}
-
form.autosubmit label {
border-radius: 0.25em;
display: inline-block;
vertical-align: middle;
}
+form.autosubmit label span {
+ display: none;
+}
+
+html.autosubmit-support form.autosubmit label span {
+ display: inline-block;
+}
+
form.autosubmit input{
margin: 0;
}
@@ -99,13 +97,7 @@ form.autosubmit input[type=submit] {
html.autosubmit-support form.autosubmit input{
position: fixed;
- top: -500px;
-}
-
-html.autosubmit-support form.autosubmit h2{
- font-size: 1em;
- font-weight: normal;
- padding: 0;
+ top: -10em;
}
table.browser {
diff --git a/browsepy/templates/browse.html b/browsepy/templates/browse.html
index 4710d6b..5773176 100644
--- a/browsepy/templates/browse.html
+++ b/browsepy/templates/browse.html
@@ -24,7 +24,7 @@
action="{{ widget.action or url_for(widget.endpoint, path=file.urlpath) }}"
enctype="multipart/form-data">
--
GitLab
From cae63219a7648924fe161df198561a31dff60049 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 31 Jul 2019 16:09:36 +0100
Subject: [PATCH 090/171] more consistent button spacing
---
browsepy/static/base.css | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/browsepy/static/base.css b/browsepy/static/base.css
index e13913b..68fd186 100644
--- a/browsepy/static/base.css
+++ b/browsepy/static/base.css
@@ -295,7 +295,7 @@ header form.autosubmit,
footer a.button,
footer input.button,
footer form.autosubmit {
- margin: 0 0.5em 0.5em 0;
+ margin: 0.5em 3px 0.5em 0;
}
a.button:last-child,
--
GitLab
From 7c34e94e5dfb78362ed8ce0b0166ce07ba8a820a Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 31 Jul 2019 16:14:02 +0100
Subject: [PATCH 091/171] revert button spacing
---
browsepy/static/base.css | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/browsepy/static/base.css b/browsepy/static/base.css
index 68fd186..cdd7248 100644
--- a/browsepy/static/base.css
+++ b/browsepy/static/base.css
@@ -295,7 +295,7 @@ header form.autosubmit,
footer a.button,
footer input.button,
footer form.autosubmit {
- margin: 0.5em 3px 0.5em 0;
+ margin: 0 0.5em 0.5em 1px;
}
a.button:last-child,
--
GitLab
From c9a8fb4c8b4cb17012f25c2a5f914b9c69042ee8 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 31 Jul 2019 16:42:22 +0100
Subject: [PATCH 092/171] fix missing ci install
---
.gitlab-ci.yml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 517646f..aa18eb2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -79,8 +79,9 @@ coverage:
paths:
- htmlcov
coverage: '/^TOTAL\s+(?:\d+\s+)*(\d+\%)$/'
+ before_script:
+ - pip install coverage
script:
- - pip install coverage
- coverage combine
- coverage html --fail-under=0
- coverage report
@@ -93,6 +94,7 @@ doc:
paths:
- doc/.build
before_script:
+ - apk add --no-cache make
- pip install -r requirements/doc.txt
script:
- make -C doc html
--
GitLab
From 45507888541cf23acf08b02923c247ceb75f2741 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 31 Jul 2019 17:07:54 +0100
Subject: [PATCH 093/171] fix bad ci yml references
---
.gitlab-ci.yml | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index aa18eb2..afbf5d0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,6 +3,7 @@
variables:
PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
COVERAGE_FILE: "${CI_PROJECT_DIR}/.coverage.${CI_JOB_NAME}"
+ MODULE: browsepy
cache:
paths:
- .cache/pip
@@ -15,19 +16,22 @@
- apk add --no-cache build-base python3-dev
- pip install coverage flake8
script:
- - flake8 browsepy
+ - flake8 ${MODULE}
- coverage run -p setup.py test
.after-base:
image: python:3.7-alpine
variables:
PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
+ MODULE: browsepy
cache:
paths:
- .cache/pip
.github-base:
image: alpine/git
+ variables:
+ GITHUB_MIRROR: git@github.com:ergoithz/browsepy.git
before_script:
- mkdir -p ~/.ssh
- echo "${SSH_PRIVATE_KEY}" | tr -d '\r' > ~/.ssh/id_rsa
@@ -69,7 +73,7 @@ eslint:
before_script:
- npm install eslint
script:
- - node_modules/.bin/eslint browsepy
+ - node_modules/.bin/eslint ${MODULE}
coverage:
extends: .after-base
@@ -108,7 +112,7 @@ publish-github-mirror:
- next
- master
script:
- - git remote add github git@github.com:ergoithz/browsepy.git
+ - git remote add github "${GITHUB_MIRROR}"
- git push github
publish-github-pages:
@@ -120,7 +124,7 @@ publish-github-pages:
refs:
- master
script:
- - git remote add github git@github.com:ergoithz/browsepy.git
+ - git remote add github "${GITHUB_MIRROR}"
- git fetch
- git checkout gh-pages
- cp -rf doc/.build/* .
@@ -142,7 +146,7 @@ publish-alpha:
- |
ALPHA=$(date +%s)
SED_REGEX="s/^__version__\s*=\s*[\"']([^\"']+)[\"']\$/__version__ = '\\1a${ALPHA}'/g"
- SED_PATH="unittest_resources/__init__.py"
+ SED_PATH="${MODULE}/__init__.py"
sed -ri "${SED_REGEX}" "${SED_PATH}"
python setup.py bdist_wheel sdist
twine upload --repository-url=https://test.pypi.org/legacy/ dist/*
--
GitLab
From 57dfb44fb0625f4d03dc0fb9ab0c7bc290e6a6f9 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 31 Jul 2019 17:36:05 +0100
Subject: [PATCH 094/171] update requirements, improve ci
---
.gitlab-ci.yml | 13 ++++++++++++-
requirements/base.txt | 2 +-
requirements/ci.txt | 26 --------------------------
requirements/development.txt | 7 ++-----
setup.py | 3 ++-
5 files changed, 17 insertions(+), 34 deletions(-)
delete mode 100644 requirements/ci.txt
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index afbf5d0..bf5f171 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,6 +7,12 @@
cache:
paths:
- .cache/pip
+ only:
+ changes:
+ - "**.py"
+ - "**.html"
+ - "**.cfg"
+ - MANIFEST.in
artifacts:
when: on_success
expire_in: 1h
@@ -31,7 +37,7 @@
.github-base:
image: alpine/git
variables:
- GITHUB_MIRROR: git@github.com:ergoithz/browsepy.git
+ GITHUB_MIRROR: "git@github.com:ergoithz/browsepy.git"
before_script:
- mkdir -p ~/.ssh
- echo "${SSH_PRIVATE_KEY}" | tr -d '\r' > ~/.ssh/id_rsa
@@ -70,6 +76,11 @@ eslint:
cache:
paths:
- node_modules
+ only:
+ changes:
+ - "**.json"
+ - "**.js"
+ - .eslintignore
before_script:
- npm install eslint
script:
diff --git a/requirements/base.txt b/requirements/base.txt
index 5d1f523..e7eddbf 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -12,12 +12,12 @@
# - requirements/development.txt
#
+six
flask
cookieman
unicategories
# compat
-six
backports.shutil_get_terminal_size ; python_version<'3.3'
scandir ; python_version<'3.5'
pathlib2 ; python_version<'3.5'
diff --git a/requirements/ci.txt b/requirements/ci.txt
deleted file mode 100644
index c6aa8f1..0000000
--- a/requirements/ci.txt
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Continuous-integration requirements file
-# ========================================
-#
-# This module includes all requirements for continuous-integration tasks
-# (see .travis.yml).
-#
-# Requirement file inheritance tree
-# ---------------------------------
-#
-# - requirements/ci.txt
-# - requirements/development.txt
-# - requirements/base.txt
-#
-# See also
-# --------
-#
-# - requirements.txt
-# - requirements/development.txt
-#
-
--r development.txt
-
-ghp-import
-codecov
-coveralls
diff --git a/requirements/development.txt b/requirements/development.txt
index cd9a5cd..0d5ffce 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -22,22 +22,19 @@
-r base.txt
-# linting
+# lint and test
flake8
-doc8
yapf
coverage
jedi
-
-# dev
sphinx
pycodestyle
+pydocstyle
importlib_resources # pycodestyle unittest with python < 3.7
six # compatible metaclass with python 2-3
pyyaml
beautifulsoup4
# dist
-setuptools
wheel
twine
diff --git a/setup.py b/setup.py
index c812a68..35441a0 100644
--- a/setup.py
+++ b/setup.py
@@ -61,9 +61,10 @@ setup(
'setuptools>36.2',
],
install_requires=[
+ 'six',
'flask',
- 'unicategories',
'cookieman',
+ 'unicategories',
'backports.shutil_get_terminal_size ; python_version<"3.3"',
'scandir ; python_version<"3.5"',
'pathlib2 ; python_version<"3.5"',
--
GitLab
From 12e76a024b54260183285dcccedd664b8f4bc580 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 31 Jul 2019 17:44:13 +0100
Subject: [PATCH 095/171] fix bad git image usage
---
.gitlab-ci.yml | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bf5f171..17262c6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -35,7 +35,10 @@
- .cache/pip
.github-base:
- image: alpine/git
+ image:
+ name: alpine/git
+ entrypoint:
+ - /bin/sh
variables:
GITHUB_MIRROR: "git@github.com:ergoithz/browsepy.git"
before_script:
--
GitLab
From 82af32122660e43eb4728095de488627ad4e14e1 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 31 Jul 2019 17:48:20 +0100
Subject: [PATCH 096/171] unoptomize ci until coverage issue get solved
---
.gitlab-ci.yml | 6 ------
1 file changed, 6 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 17262c6..775ac3f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,12 +7,6 @@
cache:
paths:
- .cache/pip
- only:
- changes:
- - "**.py"
- - "**.html"
- - "**.cfg"
- - MANIFEST.in
artifacts:
when: on_success
expire_in: 1h
--
GitLab
From 2f46ee8759d618e5920bc580956a612e253baf83 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 31 Jul 2019 17:58:45 +0100
Subject: [PATCH 097/171] appveyor fixes
---
.appveyor.yml | 26 ++++++++------------------
browsepy/plugin/player/tests.py | 2 +-
browsepy/tests/test_app.py | 4 ++--
3 files changed, 11 insertions(+), 21 deletions(-)
diff --git a/.appveyor.yml b/.appveyor.yml
index 54f91e3..8f359aa 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -1,45 +1,35 @@
environment:
matrix:
- PYTHON: "C:\\Python27"
- PYTHON_VERSION: "2.7.x" # currently 2.7.9
+ PYTHON_VERSION: "2.7.16"
PYTHON_ARCH: "32"
- PYTHON: "C:\\Python27-x64"
- PYTHON_VERSION: "2.7.x" # currently 2.7.9
+ PYTHON_VERSION: "2.7.16"
PYTHON_ARCH: "64"
- - PYTHON: "C:\\Python34"
- PYTHON_VERSION: "3.4.x" # currently 3.4.3
- PYTHON_ARCH: "32"
-
- - PYTHON: "C:\\Python34-x64"
- PYTHON_VERSION: "3.4.x" # currently 3.4.3
- PYTHON_ARCH: "64"
-
- # Python versions not pre-installed
-
- PYTHON: "C:\\Python35"
- PYTHON_VERSION: "3.5.3"
+ PYTHON_VERSION: "3.5.7"
PYTHON_ARCH: "32"
- PYTHON: "C:\\Python35-x64"
- PYTHON_VERSION: "3.5.3"
+ PYTHON_VERSION: "3.5.7"
PYTHON_ARCH: "64"
- PYTHON: "C:\\Python36"
- PYTHON_VERSION: "3.6.1"
+ PYTHON_VERSION: "3.6.9"
PYTHON_ARCH: "32"
- PYTHON: "C:\\Python36-x64"
- PYTHON_VERSION: "3.6.1"
+ PYTHON_VERSION: "3.6.9"
PYTHON_ARCH: "64"
- PYTHON: "C:\\Python37"
- PYTHON_VERSION: "3.7.1"
+ PYTHON_VERSION: "3.7.4"
PYTHON_ARCH: "32"
- PYTHON: "C:\\Python37-x64"
- PYTHON_VERSION: "3.7.1"
+ PYTHON_VERSION: "3.7.4"
PYTHON_ARCH: "64"
build: off
diff --git a/browsepy/plugin/player/tests.py b/browsepy/plugin/player/tests.py
index b5b9abb..52f70b5 100644
--- a/browsepy/plugin/player/tests.py
+++ b/browsepy/plugin/player/tests.py
@@ -49,7 +49,7 @@ class TestPLSFileParser(unittest.TestCase):
exceptions = player_playable.PLSFileParser.option_exceptions
def get_parser(self, content=''):
- with tempfile.NamedTemporaryFile(mode='w') as f:
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.pls') as f:
f.write(content)
f.flush()
return self.module.PLSFileParser(f.name)
diff --git a/browsepy/tests/test_app.py b/browsepy/tests/test_app.py
index cf30aee..6916b18 100644
--- a/browsepy/tests/test_app.py
+++ b/browsepy/tests/test_app.py
@@ -16,8 +16,8 @@ class TestApp(unittest.TestCase):
self.app.config._warned.clear()
def test_config(self):
- with tempfile.NamedTemporaryFile() as f:
- f.write(b'DIRECTORY_DOWNLOADABLE = False\n')
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.pls') as f:
+ f.write('DIRECTORY_DOWNLOADABLE = False\n')
f.flush()
os.environ['BROWSEPY_TEST_SETTINGS'] = f.name
--
GitLab
From b03126cd7e54ffee818b555f1437404f8d23d35a Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 31 Jul 2019 17:59:19 +0100
Subject: [PATCH 098/171] fix typo
---
browsepy/tests/test_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/browsepy/tests/test_app.py b/browsepy/tests/test_app.py
index 6916b18..63bb9df 100644
--- a/browsepy/tests/test_app.py
+++ b/browsepy/tests/test_app.py
@@ -16,7 +16,7 @@ class TestApp(unittest.TestCase):
self.app.config._warned.clear()
def test_config(self):
- with tempfile.NamedTemporaryFile(mode='w', suffix='.pls') as f:
+ with tempfile.NamedTemporaryFile(mode='w', suffix='.py') as f:
f.write('DIRECTORY_DOWNLOADABLE = False\n')
f.flush()
--
GitLab
From cf2e11fdd0e4c2fd3116f57325a6cd0a2d6d913a Mon Sep 17 00:00:00 2001
From: "Felipe A. Hernandez"
Date: Thu, 1 Aug 2019 09:20:07 +0000
Subject: [PATCH 099/171] try -again- to get gitlab-ci to behave
---
.gitlab-ci.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 775ac3f..949833e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -33,6 +33,7 @@
name: alpine/git
entrypoint:
- /bin/sh
+ - -c
variables:
GITHUB_MIRROR: "git@github.com:ergoithz/browsepy.git"
before_script:
--
GitLab
From 196ae6a9f32d78188a64d1c145ce39c5a577a98f Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 10:32:49 +0100
Subject: [PATCH 100/171] try to avoid hasefroch filesystem limitations
---
.gitignore | 1 +
.vscode/settings.json | 27 ---------------------------
browsepy/plugin/player/tests.py | 9 +++++----
browsepy/tests/test_app.py | 9 +++++----
4 files changed, 11 insertions(+), 35 deletions(-)
delete mode 100644 .vscode/settings.json
diff --git a/.gitignore b/.gitignore
index 8d76b59..4099e7b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
.c9
.idea
.coverage
+.vscode
htmlcov
dist
diff --git a/.vscode/settings.json b/.vscode/settings.json
deleted file mode 100644
index 6b2ba5b..0000000
--- a/.vscode/settings.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "editor.rulers": [
- 72,
- 79
- ],
- "editor.tabSize": 4,
- "files.trimTrailingWhitespace": true,
- "files.insertFinalNewline": true,
- "python.formatting.provider": "yapf",
- "python.testing.unittestEnabled": true,
- "python.linting.flake8Enabled": true,
- "python.linting.enabled": true,
- "python.linting.pep8Enabled": true,
- "python.linting.pep8Path": "pycodestyle",
- "python.linting.pylintEnabled": false,
- "python.jediEnabled": true,
- "python.pythonPath": "env/bin/python",
- "python.testing.unittestArgs": [
- "-v",
- "-s",
- "./browsepy/tests",
- "-p",
- "test_*.py"
- ],
- "python.testing.pytestEnabled": false,
- "python.testing.nosetestsEnabled": false
-}
diff --git a/browsepy/plugin/player/tests.py b/browsepy/plugin/player/tests.py
index 52f70b5..719d40e 100644
--- a/browsepy/plugin/player/tests.py
+++ b/browsepy/plugin/player/tests.py
@@ -49,10 +49,11 @@ class TestPLSFileParser(unittest.TestCase):
exceptions = player_playable.PLSFileParser.option_exceptions
def get_parser(self, content=''):
- with tempfile.NamedTemporaryFile(mode='w', suffix='.pls') as f:
- f.write(content)
- f.flush()
- return self.module.PLSFileParser(f.name)
+ with tempfile.TemporaryDirectory() as path:
+ name = os.path.join(path, 'file.pls')
+ with open(name, 'w') as f:
+ f.write(content)
+ return self.module.PLSFileParser(name)
def test_getint(self):
parser = self.get_parser()
diff --git a/browsepy/tests/test_app.py b/browsepy/tests/test_app.py
index 63bb9df..73f0134 100644
--- a/browsepy/tests/test_app.py
+++ b/browsepy/tests/test_app.py
@@ -16,11 +16,12 @@ class TestApp(unittest.TestCase):
self.app.config._warned.clear()
def test_config(self):
- with tempfile.NamedTemporaryFile(mode='w', suffix='.py') as f:
- f.write('DIRECTORY_DOWNLOADABLE = False\n')
- f.flush()
+ with tempfile.TemporaryDirectory() as path:
+ name = os.path.join(path, 'file.py')
+ with open(name, 'w') as f:
+ f.write('DIRECTORY_DOWNLOADABLE = False\n')
- os.environ['BROWSEPY_TEST_SETTINGS'] = f.name
+ os.environ['BROWSEPY_TEST_SETTINGS'] = name
with warnings.catch_warnings(record=True) as warns:
warnings.simplefilter('always')
self.app.config['directory_downloadable'] = True
--
GitLab
From c5579f9edf22d208489341690ec88168cb0b1df3 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 10:36:39 +0100
Subject: [PATCH 101/171] set ref on github mirroring
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 949833e..67eae68 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -122,7 +122,7 @@ publish-github-mirror:
- master
script:
- git remote add github "${GITHUB_MIRROR}"
- - git push github
+ - git push -u "${CI_COMMIT_REF_NAME}" github
publish-github-pages:
extends: .github-base
--
GitLab
From 82b0ef14dc4ccee4fe251b0bf14640a947477a1a Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 10:58:17 +0100
Subject: [PATCH 102/171] replace tempdir.TemporaryDirectory by compat
---
browsepy/plugin/player/tests.py | 25 ++++++-------------------
browsepy/tests/test_app.py | 4 ++--
requirements/development.txt | 2 --
3 files changed, 8 insertions(+), 23 deletions(-)
diff --git a/browsepy/plugin/player/tests.py b/browsepy/plugin/player/tests.py
index 719d40e..e358337 100644
--- a/browsepy/plugin/player/tests.py
+++ b/browsepy/plugin/player/tests.py
@@ -3,7 +3,6 @@
import os
import os.path
import unittest
-import shutil
import tempfile
import six
@@ -13,6 +12,7 @@ import flask
from werkzeug.exceptions import NotFound
import browsepy
+import browsepy.compat as compat
import browsepy.utils as utils
import browsepy.file as browsepy_file
import browsepy.manager as browsepy_manager
@@ -49,7 +49,7 @@ class TestPLSFileParser(unittest.TestCase):
exceptions = player_playable.PLSFileParser.option_exceptions
def get_parser(self, content=''):
- with tempfile.TemporaryDirectory() as path:
+ with compat.mkdtemp() as path:
name = os.path.join(path, 'file.pls')
with open(name, 'w') as f:
f.write(content)
@@ -229,8 +229,7 @@ class TestPlayable(TestIntegrationBase):
self.assertEqual(pf.title, 'asdf.%s' % ext)
def test_playabledirectory(self):
- tmpdir = tempfile.mkdtemp()
- try:
+ with compat.mkdtemp() as tmpdir:
file = p(tmpdir, 'playable.mp3')
open(file, 'w').close()
node = browsepy_file.Directory(tmpdir, app=self.app)
@@ -247,9 +246,6 @@ class TestPlayable(TestIntegrationBase):
os.remove(file)
self.assertFalse(self.module.PlayableDirectory.detect(node))
- finally:
- shutil.rmtree(tmpdir)
-
def test_playlistfile(self):
pf = self.module.PlayListFile.from_urlpath(
path='filename.m3u', app=self.app)
@@ -263,8 +259,7 @@ class TestPlayable(TestIntegrationBase):
def test_m3ufile(self):
data = '/base/valid.mp3\n/outside.ogg\n/base/invalid.bin\nrelative.ogg'
- tmpdir = tempfile.mkdtemp()
- try:
+ with compat.mkdtemp() as tmpdir:
file = p(tmpdir, 'playable.m3u')
with open(file, 'w') as f:
f.write(data)
@@ -273,8 +268,6 @@ class TestPlayable(TestIntegrationBase):
[a.path for a in playlist.entries()],
[p(self.base, 'valid.mp3'), p(tmpdir, 'relative.ogg')]
)
- finally:
- shutil.rmtree(tmpdir)
def test_plsfile(self):
data = (
@@ -284,8 +277,7 @@ class TestPlayable(TestIntegrationBase):
'File3=/base/invalid.bin\n'
'File4=relative.ogg'
)
- tmpdir = tempfile.mkdtemp()
- try:
+ with compat.mkdtemp() as tmpdir:
file = p(tmpdir, 'playable.pls')
with open(file, 'w') as f:
f.write(data)
@@ -294,8 +286,6 @@ class TestPlayable(TestIntegrationBase):
[a.path for a in playlist.entries()],
[p(self.base, 'valid.mp3'), p(tmpdir, 'relative.ogg')]
)
- finally:
- shutil.rmtree(tmpdir)
def test_plsfile_with_holes(self):
data = (
@@ -305,8 +295,7 @@ class TestPlayable(TestIntegrationBase):
'File4=relative.ogg\n'
'NumberOfEntries=4'
)
- tmpdir = tempfile.mkdtemp()
- try:
+ with compat.mkdtemp() as tmpdir:
file = p(tmpdir, 'playable.pls')
with open(file, 'w') as f:
f.write(data)
@@ -315,8 +304,6 @@ class TestPlayable(TestIntegrationBase):
[a.path for a in playlist.entries()],
[p(self.base, 'valid.mp3'), p(tmpdir, 'relative.ogg')]
)
- finally:
- shutil.rmtree(tmpdir)
class TestBlueprint(TestPlayerBase):
diff --git a/browsepy/tests/test_app.py b/browsepy/tests/test_app.py
index 73f0134..cda25e5 100644
--- a/browsepy/tests/test_app.py
+++ b/browsepy/tests/test_app.py
@@ -1,10 +1,10 @@
import os
import os.path
import unittest
-import tempfile
import warnings
import browsepy
+import browsepy.compat as compat
import browsepy.appconfig
@@ -16,7 +16,7 @@ class TestApp(unittest.TestCase):
self.app.config._warned.clear()
def test_config(self):
- with tempfile.TemporaryDirectory() as path:
+ with compat.mkdtemp() as path:
name = os.path.join(path, 'file.py')
with open(name, 'w') as f:
f.write('DIRECTORY_DOWNLOADABLE = False\n')
diff --git a/requirements/development.txt b/requirements/development.txt
index 0d5ffce..4c2b38a 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -30,8 +30,6 @@ jedi
sphinx
pycodestyle
pydocstyle
-importlib_resources # pycodestyle unittest with python < 3.7
-six # compatible metaclass with python 2-3
pyyaml
beautifulsoup4
--
GitLab
From a87c93f69094cc59c143a3b8885e2f39aa30697d Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 11:02:14 +0100
Subject: [PATCH 103/171] fix ppath test on hasefroch
---
browsepy/tests/test_utils.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/browsepy/tests/test_utils.py b/browsepy/tests/test_utils.py
index 7be70f6..4d1fc9b 100644
--- a/browsepy/tests/test_utils.py
+++ b/browsepy/tests/test_utils.py
@@ -34,12 +34,12 @@ class TestPPath(unittest.TestCase):
self.assertTrue(
self.module
.ppath('a', 'b', module=__name__)
- .endswith('browsepy/tests/a/b')
+ .endswith(os.path.join('browsepy', 'tests', 'a', 'b'))
)
self.assertTrue(
self.module
.ppath('a', 'b')
- .endswith('browsepy/a/b')
+ .endswith(os.path.join('browsepy', 'a', 'b'))
)
def test_get_module(self):
--
GitLab
From b24fd6cef9215284752488fdd52b39463d5dfb28 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 11:04:35 +0100
Subject: [PATCH 104/171] fix git src refspec on github mirroring
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 67eae68..4fdbfac 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -122,7 +122,7 @@ publish-github-mirror:
- master
script:
- git remote add github "${GITHUB_MIRROR}"
- - git push -u "${CI_COMMIT_REF_NAME}" github
+ - git push -u github "${CI_COMMIT_REF_NAME}"
publish-github-pages:
extends: .github-base
--
GitLab
From 658611df01c2e6d3bc4aa807e9b02483dbd115da Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 11:36:51 +0100
Subject: [PATCH 105/171] workaround rmtree issues on hasefroch
---
browsepy/compat.py | 90 ++++++++++++++++++++-------
browsepy/file.py | 3 +-
browsepy/plugin/file_actions/tests.py | 6 +-
browsepy/tests/test_file.py | 17 +++--
browsepy/tests/test_main.py | 3 +-
browsepy/tests/test_module.py | 13 ++--
browsepy/tests/test_stream.py | 4 +-
7 files changed, 90 insertions(+), 46 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index f7a7f63..a78bd1c 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -4,6 +4,7 @@ import os
import os.path
import sys
import abc
+import time
import shutil
import tempfile
import itertools
@@ -45,12 +46,15 @@ except ImportError:
class SafeArgumentParser(argparse.ArgumentParser):
+ """ArgumentParser based class which safer default behavior."""
+
allow_abbrev_support = sys.version_info >= (3, 5, 0)
def _get_option_tuples(self, option_string):
return []
def __init__(self, **kwargs):
+ """Initialize object."""
if self.allow_abbrev_support:
kwargs.setdefault('allow_abbrev', False)
kwargs.setdefault('add_help', False)
@@ -58,8 +62,11 @@ class SafeArgumentParser(argparse.ArgumentParser):
class HelpFormatter(argparse.RawTextHelpFormatter):
+ """HelpFormatter for argument parsers honoring terminal width."""
+
def __init__(self, prog, indent_increment=2, max_help_position=24,
width=None):
+ """Initialize object."""
if width is None:
try:
width = get_terminal_size().columns - 2
@@ -72,7 +79,11 @@ class HelpFormatter(argparse.RawTextHelpFormatter):
@contextlib.contextmanager
def scandir(path):
"""
- Backwards-compatible :func:`scandir.scandir` context manager
+ Get iterable of :class:`os.DirEntry` as context manager.
+
+ This is just backwards-compatible :func:`scandir.scandir` context
+ manager wrapper, as since `3.6` calling `close` method became
+ mandatory, but it's not available on previous versions.
:param path: path to iterate
:type path: str
@@ -85,10 +96,34 @@ def scandir(path):
files.close()
+def rmtree(path):
+ """
+ Remove directory tree, with platform-specific fixes.
+
+ A simple :func:`shutil.rmtree` wrapper, with some error handling and
+ retry logic, as some filesystems on some platforms does not always
+ behave as they should.
+
+ :param path: path to remove
+ :type path: str
+ """
+ while os.path.exists(path):
+ try:
+ shutil.rmtree(path)
+ except OSError as e:
+ if getattr(e, 'winerror', 145):
+ time.sleep(0.001) # allow dumb filesystems to catch up
+ continue
+ raise
+
+
@contextlib.contextmanager
def mkdtemp(suffix='', prefix='', dir=None):
"""
- Backwards-compatible :class:`tmpfile.TemporaryDirectory` context manager.
+ Create a temporary directory context manager.
+
+ Backwards-compatible :class:`tmpfile.TemporaryDirectory` context
+ manager, as it was added on `3.2`.
:param path: path to iterate
:type path: str
@@ -97,7 +132,7 @@ def mkdtemp(suffix='', prefix='', dir=None):
try:
yield path
finally:
- shutil.rmtree(path)
+ rmtree(path)
def isexec(path):
@@ -114,7 +149,10 @@ def isexec(path):
def fsdecode(path, os_name=os.name, fs_encoding=FS_ENCODING, errors=None):
"""
- Decode given path.
+ Decode given path using filesystem encoding.
+
+ This is necessary as python has pretty bad filesystem support on
+ some platforms.
:param path: path will be decoded if using bytes
:type path: bytes or str
@@ -135,7 +173,10 @@ def fsdecode(path, os_name=os.name, fs_encoding=FS_ENCODING, errors=None):
def fsencode(path, os_name=os.name, fs_encoding=FS_ENCODING, errors=None):
"""
- Encode given path.
+ Encode given path using filesystem encoding.
+
+ This is necessary as python has pretty bad filesystem support on
+ some platforms.
:param path: path will be encoded if not using bytes
:type path: bytes or str
@@ -157,6 +198,7 @@ def fsencode(path, os_name=os.name, fs_encoding=FS_ENCODING, errors=None):
def getcwd(fs_encoding=FS_ENCODING, cwd_fnc=os.getcwd):
"""
Get current work directory's absolute path.
+
Like os.getcwd but garanteed to return an unicode-str object.
:param fs_encoding: filesystem encoding, defaults to autodetected
@@ -172,8 +214,9 @@ def getcwd(fs_encoding=FS_ENCODING, cwd_fnc=os.getcwd):
def getdebug(environ=os.environ, true_values=TRUE_VALUES):
"""
- Get if app is expected to be ran in debug mode looking at environment
- variables.
+ Get if app is running in debug mode.
+
+ This is detected looking at environment variables.
:param environ: environment dict-like object
:type environ: collections.abc.Mapping
@@ -185,8 +228,16 @@ def getdebug(environ=os.environ, true_values=TRUE_VALUES):
def deprecated(func_or_text, environ=os.environ):
"""
- Decorator used to mark functions as deprecated. It will result in a
- warning being emmitted hen the function is called.
+ Decorate function and mark it as deprecated.
+
+ Calling a deprected function will result in a warning message.
+
+ :param func_or_text: message or callable to decorate
+ :type func_or_text: callable
+ :param environ: optional environment mapping
+ :type environ: collections.abc.Mapping
+ :returns: nested decorator or new decorated function (depending on params)
+ :rtype: callable
Usage:
@@ -200,12 +251,6 @@ def deprecated(func_or_text, environ=os.environ):
... def fnc():
... pass
- :param func_or_text: message or callable to decorate
- :type func_or_text: callable
- :param environ: optional environment mapping
- :type environ: collections.abc.Mapping
- :returns: nested decorator or new decorated function (depending on params)
- :rtype: callable
"""
def inner(func):
message = (
@@ -229,6 +274,11 @@ def usedoc(other):
"""
Decorator which copies __doc__ of given object into decorated one.
+ :param other: anything with a __doc__ attribute
+ :type other: any
+ :returns: decorator function
+ :rtype: callable
+
Usage:
>>> def fnc1():
@@ -240,10 +290,6 @@ def usedoc(other):
>>> fnc2.__doc__
'docstring'collections.abc.D
- :param other: anything with a __doc__ attribute
- :type other: any
- :returns: decorator function
- :rtype: callable
"""
def inner(fnc):
fnc.__doc__ = fnc.__doc__ or getattr(other, '__doc__')
@@ -368,8 +414,10 @@ def which(name,
def re_escape(pattern, chars=frozenset("()[]{}?*+|^$\\.-#")):
"""
- Escape all special regex characters in pattern and converts non-ascii
- characters into unicode escape sequences.
+ Escape pattern to include it safely into another regex.
+
+ This function escapes all special regex characters while translating
+ non-ascii characters into unicode escape sequences.
Logic taken from regex module.
diff --git a/browsepy/file.py b/browsepy/file.py
index 7c553e6..2e6ef74 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -3,7 +3,6 @@
import os
import os.path
import re
-import shutil
import codecs
import string
import random
@@ -653,7 +652,7 @@ class Directory(Node):
:raises OutsideRemovableBase: when not under removable base directory
"""
super(Directory, self).remove()
- shutil.rmtree(self.path)
+ compat.rmtree(self.path)
def download(self):
"""
diff --git a/browsepy/plugin/file_actions/tests.py b/browsepy/plugin/file_actions/tests.py
index d1ecf86..8360ed8 100644
--- a/browsepy/plugin/file_actions/tests.py
+++ b/browsepy/plugin/file_actions/tests.py
@@ -2,7 +2,6 @@
import unittest
import tempfile
-import shutil
import os
import os.path
import functools
@@ -12,6 +11,7 @@ import flask
from werkzeug.utils import cached_property
+import browsepy.compat as compat
import browsepy.utils as utils
import browsepy.plugin.file_actions as file_actions
import browsepy.plugin.file_actions.exceptions as file_actions_exceptions
@@ -132,7 +132,7 @@ class TestIntegration(unittest.TestCase):
self.manager.reload()
def tearDown(self):
- shutil.rmtree(self.base)
+ compat.rmtree(self.base)
self.app.config.clear()
self.app.config.update(self.original_config)
self.manager.clear()
@@ -252,7 +252,7 @@ class TestAction(unittest.TestCase):
return '', 400
def tearDown(self):
- shutil.rmtree(self.base)
+ compat.rmtree(self.base)
utils.clear_flask_context()
def mkdir(self, *path):
diff --git a/browsepy/tests/test_file.py b/browsepy/tests/test_file.py
index 08ca9cc..336e15e 100644
--- a/browsepy/tests/test_file.py
+++ b/browsepy/tests/test_file.py
@@ -3,16 +3,15 @@ import os
import os.path
import unittest
import tempfile
-import shutil
import stat
import browsepy
import browsepy.file
-import browsepy.compat
+import browsepy.compat as compat
import browsepy.utils as utils
-PY_LEGACY = browsepy.compat.PY_LEGACY
+PY_LEGACY = compat.PY_LEGACY
class TestFile(unittest.TestCase):
@@ -23,15 +22,15 @@ class TestFile(unittest.TestCase):
self.workbench = tempfile.mkdtemp()
def clear_workbench(self):
- with browsepy.compat.scandir(self.workbench) as files:
+ with compat.scandir(self.workbench) as files:
for entry in files:
if entry.is_dir():
- shutil.rmtree(entry.path)
+ compat.rmtree(entry.path)
else:
os.remove(entry.path)
def tearDown(self):
- shutil.rmtree(self.workbench)
+ compat.rmtree(self.workbench)
utils.clear_flask_context()
def textfile(self, name, text):
@@ -43,7 +42,7 @@ class TestFile(unittest.TestCase):
def test_repr(self):
self.assertIsInstance(
repr(self.module.Node('a', app=self.app)),
- browsepy.compat.basestring
+ compat.basestring
)
def test_iter_listdir(self):
@@ -96,7 +95,7 @@ class TestFile(unittest.TestCase):
tmp_err = os.path.join(self.workbench, 'nonexisting_file')
# test file command
- if browsepy.compat.which('file'):
+ if compat.which('file'):
f = self.module.File(tmp_txt, app=self.app)
self.assertEqual(f.mimetype, 'text/plain; charset=us-ascii')
self.assertEqual(f.type, 'text/plain')
@@ -146,7 +145,7 @@ class TestFile(unittest.TestCase):
f = self.module.File(virtual_file, app=self.app)
self.assertRaises(
- browsepy.compat.FileNotFoundError,
+ compat.FileNotFoundError,
lambda: f.stats
)
self.assertEqual(f.modified, None)
diff --git a/browsepy/tests/test_main.py b/browsepy/tests/test_main.py
index 4bf1eaf..2ba24f9 100644
--- a/browsepy/tests/test_main.py
+++ b/browsepy/tests/test_main.py
@@ -5,7 +5,6 @@ import sys
import os.path
import unittest
import tempfile
-import shutil
import contextlib
import browsepy
@@ -26,7 +25,7 @@ class TestMain(unittest.TestCase):
f.write('.ignore\n')
def tearDown(self):
- shutil.rmtree(self.base)
+ compat.rmtree(self.base)
@classmethod
@contextlib.contextmanager
diff --git a/browsepy/tests/test_module.py b/browsepy/tests/test_module.py
index 2a7d962..46944f3 100644
--- a/browsepy/tests/test_module.py
+++ b/browsepy/tests/test_module.py
@@ -4,7 +4,6 @@
import unittest
import os
import os.path
-import shutil
import tempfile
import tarfile
import io
@@ -21,11 +20,11 @@ import browsepy
import browsepy.file
import browsepy.manager
import browsepy.__main__
-import browsepy.compat
+import browsepy.compat as compat
import browsepy.utils as utils
-PY_LEGACY = browsepy.compat.PY_LEGACY
-range = browsepy.compat.range # noqa
+PY_LEGACY = compat.PY_LEGACY
+range = compat.range # noqa
class AppMock(object):
@@ -219,12 +218,12 @@ class TestApp(unittest.TestCase):
for sub in os.listdir(path):
sub = os.path.join(path, sub)
if os.path.isdir(sub):
- shutil.rmtree(sub)
+ compat.rmtree(sub)
else:
os.remove(sub)
def tearDown(self):
- shutil.rmtree(self.base)
+ compat.rmtree(self.base)
utils.clear_flask_context()
def get(self, endpoint, **kwargs):
@@ -535,7 +534,7 @@ class TestApp(unittest.TestCase):
self.clear(self.upload)
def test_upload_restrictions(self):
- pathconf = browsepy.compat.pathconf(self.upload)
+ pathconf = compat.pathconf(self.upload)
maxname = pathconf['PC_NAME_MAX']
maxpath = pathconf['PC_PATH_MAX']
diff --git a/browsepy/tests/test_stream.py b/browsepy/tests/test_stream.py
index 3935b00..b3bd18e 100644
--- a/browsepy/tests/test_stream.py
+++ b/browsepy/tests/test_stream.py
@@ -4,9 +4,9 @@ import os.path
import codecs
import unittest
import tempfile
-import shutil
import time
+import browsepy.compat as compat
import browsepy.stream
@@ -17,7 +17,7 @@ class StreamTest(unittest.TestCase):
self.base = tempfile.mkdtemp()
def tearDown(self):
- shutil.rmtree(self.base)
+ compat.rmtree(self.base)
def randfile(self, size=1024):
name = codecs.encode(os.urandom(5), 'hex_codec').decode()
--
GitLab
From 93d65796f75298874f4f20f31b85aa491fd2b40e Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 11:45:23 +0100
Subject: [PATCH 106/171] limit rmtree retries
---
browsepy/compat.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index a78bd1c..02f24c6 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -107,12 +107,14 @@ def rmtree(path):
:param path: path to remove
:type path: str
"""
+ attempt = -1
while os.path.exists(path):
+ attempt += 1
try:
shutil.rmtree(path)
- except OSError as e:
- if getattr(e, 'winerror', 145):
- time.sleep(0.001) # allow dumb filesystems to catch up
+ except OSError as error:
+ if getattr(error, 'winerror', 145) and attempt < 50:
+ time.sleep(0.01) # allow dumb filesystems to catch up
continue
raise
--
GitLab
From 1cb74feeccf8c45d46d358aab8c544932b851dd8 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 12:11:33 +0100
Subject: [PATCH 107/171] more hasefroch/appveyor fixes
---
.appveyor.yml | 31 +++++++++++++------------------
browsepy/compat.py | 2 +-
2 files changed, 14 insertions(+), 19 deletions(-)
diff --git a/.appveyor.yml b/.appveyor.yml
index 8f359aa..754966f 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -38,24 +38,19 @@ init:
- "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
install:
- # If there is a newer build queued for the same PR, cancel this one.
- # The AppVeyor 'rollout builds' option is supposed to serve the same
- # purpose but it is problematic because it tends to cancel builds pushed
- # directly to master instead of just PR builds (or the converse).
- # credits: JuliaLang developers.
- - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
- https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
- Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
- throw "There are newer queued builds for this pull request, failing early." }
- - ECHO "Filesystem root:"
- - ps: "ls \"C:/\""
- - ECHO "Installed SDKs:"
- - ps: "ls \"C:/Program Files/Microsoft SDKs/Windows\""
- - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
- - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
- - "python --version"
- - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
- - "pip install --disable-pip-version-check --user --upgrade pip setuptools"
+ - ps: |
+ echo "Available Python versions:"
+ ls "C:/" | Select-String -Pattern "Python*"
+ echo "Installed SDKs:"
+ ls "C:/Program Files/Microsoft SDKs/Windows"
+ if (-not(Test-Path($env:PYTHON))) {
+ & appveyor\install.ps1
+ }
+ - |
+ SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
+ python --version
+ python -c "import struct; print(struct.calcsize('P') * 8)"
+ pip install --disable-pip-version-check --user --upgrade pip setuptools
test_script:
- "python setup.py test"
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 02f24c6..d662861 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -113,7 +113,7 @@ def rmtree(path):
try:
shutil.rmtree(path)
except OSError as error:
- if getattr(error, 'winerror', 145) and attempt < 50:
+ if getattr(error, 'winerror', 0) in (5, 145) and attempt < 50:
time.sleep(0.01) # allow dumb filesystems to catch up
continue
raise
--
GitLab
From b9c66038ede8aab95c21e197e5cb01ee151a79ae Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 12:13:21 +0100
Subject: [PATCH 108/171] push head on github-mirror
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4fdbfac..3de8e3d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -122,7 +122,7 @@ publish-github-mirror:
- master
script:
- git remote add github "${GITHUB_MIRROR}"
- - git push -u github "${CI_COMMIT_REF_NAME}"
+ - git push github "HEAD:${CI_COMMIT_REF_NAME}"
publish-github-pages:
extends: .github-base
--
GitLab
From a7ec49c6ef727b8f5d95a831b01a1b2ee7341e8c Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 13:20:30 +0100
Subject: [PATCH 109/171] ci fixes
---
.appveyor.yml | 1 +
.gitlab-ci.yml | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/.appveyor.yml b/.appveyor.yml
index 754966f..e500f87 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -41,6 +41,7 @@ install:
- ps: |
echo "Available Python versions:"
ls "C:/" | Select-String -Pattern "Python*"
+ echo ""
echo "Installed SDKs:"
ls "C:/Program Files/Microsoft SDKs/Windows"
if (-not(Test-Path($env:PYTHON))) {
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3de8e3d..9ca5bcb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -38,7 +38,7 @@
GITHUB_MIRROR: "git@github.com:ergoithz/browsepy.git"
before_script:
- mkdir -p ~/.ssh
- - echo "${SSH_PRIVATE_KEY}" | tr -d '\r' > ~/.ssh/id_rsa
+ - echo -e "${SSH_PRIVATE_KEY}" | tr -d '\r' > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- ssh-keyscan -H $ssh_port "$ssh_host" >> ~/.ssh/known_hosts
- git config user.name "${GITLAB_USER_LOGIN}"
--
GitLab
From d625cdfa037ae27cd1dd80ed3cbc831812d2d131 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 14:06:55 +0100
Subject: [PATCH 110/171] add gh-pages logic
---
.gitlab-ci.yml | 69 +++++++++++++++++++++-----------------------------
1 file changed, 29 insertions(+), 40 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9ca5bcb..25f92ce 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -28,22 +28,6 @@
paths:
- .cache/pip
-.github-base:
- image:
- name: alpine/git
- entrypoint:
- - /bin/sh
- - -c
- variables:
- GITHUB_MIRROR: "git@github.com:ergoithz/browsepy.git"
- before_script:
- - mkdir -p ~/.ssh
- - echo -e "${SSH_PRIVATE_KEY}" | tr -d '\r' > ~/.ssh/id_rsa
- - chmod 600 ~/.ssh/id_rsa
- - ssh-keyscan -H $ssh_port "$ssh_host" >> ~/.ssh/known_hosts
- - git config user.name "${GITLAB_USER_LOGIN}"
- - git config user.email "${GITLAB_USER_EMAIL}"
-
stages:
- test
- report
@@ -69,8 +53,8 @@ python37:
image: python:3.7-alpine
eslint:
- image: node:alpine
stage: test
+ image: node:alpine
cache:
paths:
- node_modules
@@ -105,42 +89,47 @@ doc:
dependencies: []
artifacts:
paths:
- - doc/.build
+ - doc/.build/html
before_script:
- apk add --no-cache make
- pip install -r requirements/doc.txt
script:
- make -C doc html
-publish-github-mirror:
- extends: .github-base
- stage: publish
- dependencies: []
- only:
- refs:
- - next
- - master
- script:
- - git remote add github "${GITHUB_MIRROR}"
- - git push github "HEAD:${CI_COMMIT_REF_NAME}"
-
-publish-github-pages:
- extends: .github-base
+publish-gh-pages:
stage: publish
+ image:
+ name: alpine/git
+ entrypoint:
+ - /bin/sh
+ - -c
dependencies:
- doc
only:
refs:
- master
+ - next
+ before_script:
+ - mkdir -p ~/.ssh
+ - echo "${SSH_PRIVATE_KEY}" | tr -d '\r' > ~/.ssh/id_rsa
+ - chmod 600 ~/.ssh/id_rsa
+ - ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
+ - git config user.name "${GITLAB_USER_LOGIN}"
+ - git config user.email "${GITLAB_USER_EMAIL}"
script:
- - git remote add github "${GITHUB_MIRROR}"
- - git fetch
- - git checkout gh-pages
- - cp -rf doc/.build/* .
- - rm -rf doc
- - git add **
- - git commit -am "doc from ${CI_COMMIT_SHA} ${CI_COMMIT_MESSAGE}"
- - git push github
+ - |
+ if [ "${CI_COMMIT_REF_NAME}" == "master" ]; then
+ TARGET_BRANCH="gh-pages"
+ else
+ TARGET_BRANCH="gh-pages-${CI_COMMIT_REF_NAME}"
+ fi
+ git checkout "${TARGET_BRANCH}" || git checkout --orphan "${TARGET_BRANCH}"
+ git pull || echo "No remote history."
+ find . -not -path './doc' -delete
+ cp doc/.build/html/** .
+ git add **
+ git commit -am "update ${TARGET_BRANCH}"
+ git push -u origin "${TARGET_BRANCH}"
publish-alpha:
extends: .after-base
--
GitLab
From 9f10750ad75a8cc6f5bfce5251665caa00de0b6e Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 14:42:44 +0100
Subject: [PATCH 111/171] harden rmtree, fix bad ci doc cleaning
---
.gitlab-ci.yml | 2 +-
browsepy/compat.py | 19 +++++++++++++++++--
2 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 25f92ce..e70b252 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -125,7 +125,7 @@ publish-gh-pages:
fi
git checkout "${TARGET_BRANCH}" || git checkout --orphan "${TARGET_BRANCH}"
git pull || echo "No remote history."
- find . -not -path './doc' -delete
+ find . -not -path './doc' -not -path './doc/**' -delete
cp doc/.build/html/** .
git add **
git commit -am "update ${TARGET_BRANCH}"
diff --git a/browsepy/compat.py b/browsepy/compat.py
index d662861..6c17a70 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -3,6 +3,7 @@
import os
import os.path
import sys
+import errno
import abc
import time
import shutil
@@ -113,8 +114,22 @@ def rmtree(path):
try:
shutil.rmtree(path)
except OSError as error:
- if getattr(error, 'winerror', 0) in (5, 145) and attempt < 50:
- time.sleep(0.01) # allow dumb filesystems to catch up
+ if attempt < 50 and (
+ getattr(error, 'winerror', 0) in (5, 145) or
+ error.errno in (
+ errno.ENOENT,
+ errno.EIO,
+ errno.ENXIO,
+ errno.EAGAIN,
+ errno.EBUSY,
+ errno.ENOTDIR,
+ errno.EISDIR,
+ errno.ENOTEMPTY,
+ errno.EALREADY,
+ errno.EINPROGRESS,
+ errno.EREMOTEIO,
+ )):
+ time.sleep(0.01) # allow sluggish filesystems to catch up
continue
raise
--
GitLab
From f906e0777e28836182764158a9044b2183ae8954 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 14:43:40 +0100
Subject: [PATCH 112/171] ci doc cleaning
---
.gitlab-ci.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e70b252..e4b67f1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -127,6 +127,7 @@ publish-gh-pages:
git pull || echo "No remote history."
find . -not -path './doc' -not -path './doc/**' -delete
cp doc/.build/html/** .
+ rm -rf doc
git add **
git commit -am "update ${TARGET_BRANCH}"
git push -u origin "${TARGET_BRANCH}"
--
GitLab
From a0f040b935b1899b0b5845e4b19a36296a239314 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 14:44:56 +0100
Subject: [PATCH 113/171] ci doc cp overwrite
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e4b67f1..8e784a7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -126,7 +126,7 @@ publish-gh-pages:
git checkout "${TARGET_BRANCH}" || git checkout --orphan "${TARGET_BRANCH}"
git pull || echo "No remote history."
find . -not -path './doc' -not -path './doc/**' -delete
- cp doc/.build/html/** .
+ cp -rf doc/.build/html/** .
rm -rf doc
git add **
git commit -am "update ${TARGET_BRANCH}"
--
GitLab
From 7d4eeffe6bf1ffac3ab1bd1c6358b755661d499b Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 15:15:22 +0100
Subject: [PATCH 114/171] retry to support old 3.5 fs retries
---
browsepy/compat.py | 38 +++++++++++++++++++++-----------------
1 file changed, 21 insertions(+), 17 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 6c17a70..5416471 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -19,6 +19,21 @@ import argparse
FS_ENCODING = sys.getfilesystemencoding()
PY_LEGACY = sys.version_info < (3, )
TRUE_VALUES = frozenset(('true', 'yes', '1', 'enable', 'enabled', True, 1))
+RETRYABLE_FS_ERRNO_VALUES = frozenset(
+ ((errno.EPERM,) if os.name == 'nt' else ()) + (
+ errno.ENOENT,
+ errno.EIO,
+ errno.ENXIO,
+ errno.EAGAIN,
+ errno.EBUSY,
+ errno.ENOTDIR,
+ errno.EISDIR,
+ errno.ENOTEMPTY,
+ errno.EALREADY,
+ errno.EINPROGRESS,
+ errno.EREMOTEIO,
+ )
+ )
try:
import importlib.resources as res # python 3.7+
@@ -113,23 +128,12 @@ def rmtree(path):
attempt += 1
try:
shutil.rmtree(path)
- except OSError as error:
- if attempt < 50 and (
- getattr(error, 'winerror', 0) in (5, 145) or
- error.errno in (
- errno.ENOENT,
- errno.EIO,
- errno.ENXIO,
- errno.EAGAIN,
- errno.EBUSY,
- errno.ENOTDIR,
- errno.EISDIR,
- errno.ENOTEMPTY,
- errno.EALREADY,
- errno.EINPROGRESS,
- errno.EREMOTEIO,
- )):
- time.sleep(0.01) # allow sluggish filesystems to catch up
+ except EnvironmentError as error:
+ if (
+ attempt < 5 and
+ getattr(error, 'errno', None) in RETRYABLE_FS_ERRNO_VALUES
+ ):
+ time.sleep(0.1) # allow sluggish filesystems to catch up
continue
raise
--
GitLab
From d382a7905603047fe8f41ddd3b3b53744fc791fc Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 15:20:55 +0100
Subject: [PATCH 115/171] avoid to delete too many files for gh-pages
---
.gitlab-ci.yml | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8e784a7..d16c2a9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -124,8 +124,17 @@ publish-gh-pages:
TARGET_BRANCH="gh-pages-${CI_COMMIT_REF_NAME}"
fi
git checkout "${TARGET_BRANCH}" || git checkout --orphan "${TARGET_BRANCH}"
- git pull || echo "No remote history."
- find . -not -path './doc' -not -path './doc/**' -delete
+ git pull || true
+ find . \
+ -not -path 'doc' \
+ -not -path 'doc/**' \
+ -not -path '.git' \
+ -not -path '.git/**' \
+ -not -path '.gitignore' \
+ -not -path 'README*' \
+ -not -path 'CHANGELOG' \
+ -not -path 'LICENSE' \
+ -delete
cp -rf doc/.build/html/** .
rm -rf doc
git add **
--
GitLab
From 01a7802319487cfa981b3117667df8fa7bbcfa2c Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 15:30:22 +0100
Subject: [PATCH 116/171] cleaner gh-pages environment
---
.gitlab-ci.yml | 19 ++++++-------------
1 file changed, 6 insertions(+), 13 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d16c2a9..daafd68 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -123,20 +123,13 @@ publish-gh-pages:
else
TARGET_BRANCH="gh-pages-${CI_COMMIT_REF_NAME}"
fi
+ git init .gh_pages
+ cd .gh_pages
+ git remote add origin "${CI_REPOSITORY_URL}"
+ git fetch
git checkout "${TARGET_BRANCH}" || git checkout --orphan "${TARGET_BRANCH}"
- git pull || true
- find . \
- -not -path 'doc' \
- -not -path 'doc/**' \
- -not -path '.git' \
- -not -path '.git/**' \
- -not -path '.gitignore' \
- -not -path 'README*' \
- -not -path 'CHANGELOG' \
- -not -path 'LICENSE' \
- -delete
- cp -rf doc/.build/html/** .
- rm -rf doc
+ cp ${CI_PROJECT_DIR}/.gitignore .
+ cp -rf ${CI_PROJECT_DIR}/doc/.build/html/** .
git add **
git commit -am "update ${TARGET_BRANCH}"
git push -u origin "${TARGET_BRANCH}"
--
GitLab
From 7460d7c27f3a0266007aee36b463faeb57b7bcdf Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 15:49:15 +0100
Subject: [PATCH 117/171] another attempt on cover hasefroch fs
---
browsepy/compat.py | 64 ++++++++++++++++++++++++++++++----------------
1 file changed, 42 insertions(+), 22 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 5416471..5157cc6 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -16,24 +16,10 @@ import posixpath
import ntpath
import argparse
-FS_ENCODING = sys.getfilesystemencoding()
-PY_LEGACY = sys.version_info < (3, )
-TRUE_VALUES = frozenset(('true', 'yes', '1', 'enable', 'enabled', True, 1))
-RETRYABLE_FS_ERRNO_VALUES = frozenset(
- ((errno.EPERM,) if os.name == 'nt' else ()) + (
- errno.ENOENT,
- errno.EIO,
- errno.ENXIO,
- errno.EAGAIN,
- errno.EBUSY,
- errno.ENOTDIR,
- errno.EISDIR,
- errno.ENOTEMPTY,
- errno.EALREADY,
- errno.EINPROGRESS,
- errno.EREMOTEIO,
- )
- )
+try:
+ import builtins # python 3+
+except ImportError:
+ import __builtin__ as builtins # noqa
try:
import importlib.resources as res # python 3.7+
@@ -61,6 +47,40 @@ except ImportError:
from collections import Iterator as BaseIterator # noqa
+FS_ENCODING = sys.getfilesystemencoding()
+PY_LEGACY = sys.version_info < (3, )
+TRUE_VALUES = frozenset(
+ # Truthy values
+ ('true', 'yes', '1', 'enable', 'enabled', True, 1)
+ )
+RETRYABLE_FS_EXCEPTIONS = (
+ # Handle non PEP 3151 python versions
+ ((builtins.WindowsError,) if hasattr(builtins, 'WindowsError') else ()) + (
+ EnvironmentError,
+ )
+ )
+RETRYABLE_FS_ERRNO_VALUES = frozenset(
+ # Error codes which could imply a busy filesystem
+ ((errno.EPERM,) if os.name == 'nt' else ()) + (
+ errno.ENOENT,
+ errno.EIO,
+ errno.ENXIO,
+ errno.EAGAIN,
+ errno.EBUSY,
+ errno.ENOTDIR,
+ errno.EISDIR,
+ errno.ENOTEMPTY,
+ errno.EALREADY,
+ errno.EINPROGRESS,
+ errno.EREMOTEIO,
+ )
+ )
+RETRYABLE_FS_WINERROR_VALUES = frozenset(
+ # Handle WindowsError instances without errno
+ (5, 145)
+ )
+
+
class SafeArgumentParser(argparse.ArgumentParser):
"""ArgumentParser based class which safer default behavior."""
@@ -128,10 +148,10 @@ def rmtree(path):
attempt += 1
try:
shutil.rmtree(path)
- except EnvironmentError as error:
- if (
- attempt < 5 and
- getattr(error, 'errno', None) in RETRYABLE_FS_ERRNO_VALUES
+ except RETRYABLE_FS_EXCEPTIONS as error:
+ if attempt < 5 and (
+ getattr(error, 'errno', None) in RETRYABLE_FS_ERRNO_VALUES or
+ getattr(error, 'winerror', None) in RETRYABLE_FS_WINERROR_VALUES
):
time.sleep(0.1) # allow sluggish filesystems to catch up
continue
--
GitLab
From 4d7179a53eebd3ac44a9840f8d7ce8ebd563d602 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 15:51:07 +0100
Subject: [PATCH 118/171] configure ci git user an email
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index daafd68..265fe23 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -114,8 +114,8 @@ publish-gh-pages:
- echo "${SSH_PRIVATE_KEY}" | tr -d '\r' > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
- - git config user.name "${GITLAB_USER_LOGIN}"
- - git config user.email "${GITLAB_USER_EMAIL}"
+ - git config --global user.name "${GITLAB_USER_LOGIN}"
+ - git config --global user.email "${GITLAB_USER_EMAIL}"
script:
- |
if [ "${CI_COMMIT_REF_NAME}" == "master" ]; then
--
GitLab
From 542e267b33f8b0bc01ae7abfcb14f6e6325a6023 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 16:03:20 +0100
Subject: [PATCH 119/171] handle errno inconsistencies across platforms
---
browsepy/compat.py | 38 ++++++++++++++++++++++----------------
1 file changed, 22 insertions(+), 16 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 5157cc6..ac9fcae 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -53,28 +53,34 @@ TRUE_VALUES = frozenset(
# Truthy values
('true', 'yes', '1', 'enable', 'enabled', True, 1)
)
-RETRYABLE_FS_EXCEPTIONS = (
+RETRYABLE_FS_EXCEPTIONS = tuple(
# Handle non PEP 3151 python versions
- ((builtins.WindowsError,) if hasattr(builtins, 'WindowsError') else ()) + (
- EnvironmentError,
+ getattr(builtins, prop)
+ for prop in (
+ 'WindowsError',
+ 'EnvironmentError',
)
+ if hasattr(builtins, prop)
)
RETRYABLE_FS_ERRNO_VALUES = frozenset(
# Error codes which could imply a busy filesystem
- ((errno.EPERM,) if os.name == 'nt' else ()) + (
- errno.ENOENT,
- errno.EIO,
- errno.ENXIO,
- errno.EAGAIN,
- errno.EBUSY,
- errno.ENOTDIR,
- errno.EISDIR,
- errno.ENOTEMPTY,
- errno.EALREADY,
- errno.EINPROGRESS,
- errno.EREMOTEIO,
+ getattr(errno, prop)
+ for prop in (
+ 'ENOENT',
+ 'EIO',
+ 'ENXIO',
+ 'EAGAIN',
+ 'EBUSY',
+ 'ENOTDIR',
+ 'EISDIR',
+ 'ENOTEMPTY',
+ 'EALREADY',
+ 'EINPROGRESS',
+ 'EREMOTEIO',
+ 'EPERM' if os.name == 'nt' else None,
)
- )
+ if prop and hasattr(errno, prop)
+)
RETRYABLE_FS_WINERROR_VALUES = frozenset(
# Handle WindowsError instances without errno
(5, 145)
--
GitLab
From c0fdb3274e0392ab0b2a842958dab3bb23ef87fe Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 16:09:06 +0100
Subject: [PATCH 120/171] define gh-pages remote url manually
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 265fe23..6d77813 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -125,7 +125,7 @@ publish-gh-pages:
fi
git init .gh_pages
cd .gh_pages
- git remote add origin "${CI_REPOSITORY_URL}"
+ git remote add origin "git@gitlab.com:$CI_PROJECT_PATH.git"
git fetch
git checkout "${TARGET_BRANCH}" || git checkout --orphan "${TARGET_BRANCH}"
cp ${CI_PROJECT_DIR}/.gitignore .
--
GitLab
From b18d53fd0c9cb0940e35aae96c1c5bf95f6f8f63 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 16:31:09 +0100
Subject: [PATCH 121/171] try to cover windows permission error
---
.appveyor.yml | 5 +++++
.gitlab-ci.yml | 2 +-
browsepy/compat.py | 8 +++++---
3 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/.appveyor.yml b/.appveyor.yml
index e500f87..f02b561 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -1,3 +1,8 @@
+branches:
+ except:
+ - gh-pages
+ - /gh-pages-.*/
+
environment:
matrix:
- PYTHON: "C:\\Python27"
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6d77813..d4c9a01 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -128,7 +128,7 @@ publish-gh-pages:
git remote add origin "git@gitlab.com:$CI_PROJECT_PATH.git"
git fetch
git checkout "${TARGET_BRANCH}" || git checkout --orphan "${TARGET_BRANCH}"
- cp ${CI_PROJECT_DIR}/.gitignore .
+ cp ../.gitignore ../.appveyor.yml .
cp -rf ${CI_PROJECT_DIR}/doc/.build/html/** .
git add **
git commit -am "update ${TARGET_BRANCH}"
diff --git a/browsepy/compat.py b/browsepy/compat.py
index ac9fcae..30a4891 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -57,15 +57,18 @@ RETRYABLE_FS_EXCEPTIONS = tuple(
# Handle non PEP 3151 python versions
getattr(builtins, prop)
for prop in (
+ 'PermissionError' if os.name == 'nt' else None,
'WindowsError',
'EnvironmentError',
+ 'OSError',
)
- if hasattr(builtins, prop)
+ if prop and hasattr(builtins, prop)
)
RETRYABLE_FS_ERRNO_VALUES = frozenset(
# Error codes which could imply a busy filesystem
getattr(errno, prop)
for prop in (
+ 'EPERM' if os.name == 'nt' else None,
'ENOENT',
'EIO',
'ENXIO',
@@ -77,10 +80,9 @@ RETRYABLE_FS_ERRNO_VALUES = frozenset(
'EALREADY',
'EINPROGRESS',
'EREMOTEIO',
- 'EPERM' if os.name == 'nt' else None,
)
if prop and hasattr(errno, prop)
-)
+ )
RETRYABLE_FS_WINERROR_VALUES = frozenset(
# Handle WindowsError instances without errno
(5, 145)
--
GitLab
From 29bd5fa49f1abae7d1a71b66f60571ebb95b25f5 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 16:34:04 +0100
Subject: [PATCH 122/171] wait for external after publish on ci
---
.gitlab-ci.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d4c9a01..f38d759 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -31,6 +31,7 @@
stages:
- test
- report
+- external
- publish
python27:
--
GitLab
From d5fa3d64ddb1d25d25c3e2c4236a76bad856699a Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 17:53:09 +0100
Subject: [PATCH 123/171] try to improve rmtree error handling
---
.appveyor.yml | 76 +++++++++++----------------------
browsepy/compat.py | 104 +++++++++++++++++++++++++--------------------
2 files changed, 82 insertions(+), 98 deletions(-)
diff --git a/.appveyor.yml b/.appveyor.yml
index f02b561..ede2b90 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -1,62 +1,36 @@
branches:
except:
- - gh-pages
- - /gh-pages-.*/
+ - gh-pages
+ - /gh-pages-.*/
environment:
matrix:
- - PYTHON: "C:\\Python27"
- PYTHON_VERSION: "2.7.16"
- PYTHON_ARCH: "32"
-
- - PYTHON: "C:\\Python27-x64"
- PYTHON_VERSION: "2.7.16"
- PYTHON_ARCH: "64"
-
- - PYTHON: "C:\\Python35"
- PYTHON_VERSION: "3.5.7"
- PYTHON_ARCH: "32"
-
- - PYTHON: "C:\\Python35-x64"
- PYTHON_VERSION: "3.5.7"
- PYTHON_ARCH: "64"
-
- - PYTHON: "C:\\Python36"
- PYTHON_VERSION: "3.6.9"
- PYTHON_ARCH: "32"
-
- - PYTHON: "C:\\Python36-x64"
- PYTHON_VERSION: "3.6.9"
- PYTHON_ARCH: "64"
-
- - PYTHON: "C:\\Python37"
- PYTHON_VERSION: "3.7.4"
- PYTHON_ARCH: "32"
-
- - PYTHON: "C:\\Python37-x64"
- PYTHON_VERSION: "3.7.4"
- PYTHON_ARCH: "64"
+ - PYTHON: "C:\\Python27"
+ - PYTHON: "C:\\Python27-x64"
+ - PYTHON: "C:\\Python35"
+ - PYTHON: "C:\\Python35-x64"
+ - PYTHON: "C:\\Python36"
+ - PYTHON: "C:\\Python36-x64"
+ - PYTHON: "C:\\Python37"
+ - PYTHON: "C:\\Python37-x64"
build: off
-init:
- - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
-
install:
- - ps: |
- echo "Available Python versions:"
- ls "C:/" | Select-String -Pattern "Python*"
- echo ""
- echo "Installed SDKs:"
- ls "C:/Program Files/Microsoft SDKs/Windows"
- if (-not(Test-Path($env:PYTHON))) {
- & appveyor\install.ps1
- }
- - |
- SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
- python --version
- python -c "import struct; print(struct.calcsize('P') * 8)"
- pip install --disable-pip-version-check --user --upgrade pip setuptools
+- ps: |
+ echo "Available Python versions:"
+ ls "C:/" | Select-String -Pattern "Python*"
+ echo ""
+ echo "Installed SDKs:"
+ ls "C:/Program Files/Microsoft SDKs/Windows"
+ if (-not(Test-Path($env:PYTHON))) {
+ & appveyor\install.ps1
+ }
+- |
+ SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
+ python --version
+ python -c "import struct; print(struct.calcsize('P') * 8)"
+ pip install --disable-pip-version-check --user --upgrade pip setuptools
test_script:
- - "python setup.py test"
+- "python setup.py test"
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 30a4891..d2c5f76 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -2,6 +2,7 @@
import os
import os.path
+import stat
import sys
import errno
import abc
@@ -16,11 +17,6 @@ import posixpath
import ntpath
import argparse
-try:
- import builtins # python 3+
-except ImportError:
- import __builtin__ as builtins # noqa
-
try:
import importlib.resources as res # python 3.7+
except ImportError:
@@ -53,40 +49,31 @@ TRUE_VALUES = frozenset(
# Truthy values
('true', 'yes', '1', 'enable', 'enabled', True, 1)
)
-RETRYABLE_FS_EXCEPTIONS = tuple(
- # Handle non PEP 3151 python versions
- getattr(builtins, prop)
- for prop in (
- 'PermissionError' if os.name == 'nt' else None,
- 'WindowsError',
- 'EnvironmentError',
- 'OSError',
- )
- if prop and hasattr(builtins, prop)
- )
-RETRYABLE_FS_ERRNO_VALUES = frozenset(
- # Error codes which could imply a busy filesystem
- getattr(errno, prop)
- for prop in (
- 'EPERM' if os.name == 'nt' else None,
- 'ENOENT',
- 'EIO',
- 'ENXIO',
- 'EAGAIN',
- 'EBUSY',
- 'ENOTDIR',
- 'EISDIR',
- 'ENOTEMPTY',
- 'EALREADY',
- 'EINPROGRESS',
- 'EREMOTEIO',
- )
- if prop and hasattr(errno, prop)
- )
-RETRYABLE_FS_WINERROR_VALUES = frozenset(
- # Handle WindowsError instances without errno
- (5, 145)
- )
+RETRYABLE_OSERROR_PROPERTIES = {
+ 'errno': frozenset(
+ # Error codes which could imply a busy filesystem
+ getattr(errno, prop)
+ for prop in (
+ 'EPERM',
+ 'ENOENT',
+ 'EIO',
+ 'ENXIO',
+ 'EAGAIN',
+ 'EBUSY',
+ 'ENOTDIR',
+ 'EISDIR',
+ 'ENOTEMPTY',
+ 'EALREADY',
+ 'EINPROGRESS',
+ 'EREMOTEIO',
+ )
+ if prop and hasattr(errno, prop)
+ ),
+ 'winerror': frozenset(
+ # Handle WindowsError instances without errno
+ (5, 145)
+ ),
+ }
class SafeArgumentParser(argparse.ArgumentParser):
@@ -151,18 +138,41 @@ def rmtree(path):
:param path: path to remove
:type path: str
"""
- attempt = -1
+ known = set()
+ retries = 5
while os.path.exists(path):
- attempt += 1
try:
shutil.rmtree(path)
- except RETRYABLE_FS_EXCEPTIONS as error:
- if attempt < 5 and (
- getattr(error, 'errno', None) in RETRYABLE_FS_ERRNO_VALUES or
- getattr(error, 'winerror', None) in RETRYABLE_FS_WINERROR_VALUES
+ except EnvironmentError as error:
+ if retries and any(
+ getattr(error, prop, None) in values
+ for prop, values in RETRYABLE_OSERROR_PROPERTIES.items()
):
- time.sleep(0.1) # allow sluggish filesystems to catch up
- continue
+ # files with permission issues (common on some platforms)
+ unreachable = [
+ filename
+ for filename in (
+ getattr(error, 'filename', None),
+ getattr(error, 'filename2', None)
+ )
+ if (
+ filename and
+ filename not in known and
+ not os.access(filename, os.W_OK)
+ )
+ ]
+
+ # allow sluggish filesystems to catch up
+ if not unreachable:
+ retries -= 1
+ time.sleep(0.1)
+ continue
+
+ # try to fix permissions
+ known.update(unreachable)
+ for filename in unreachable:
+ os.chmod(path, stat.S_IWUSR)
+
raise
--
GitLab
From a5e1f43bdda88d59ee52116787b1fd44ee43bbc8 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 18:36:14 +0100
Subject: [PATCH 124/171] retry handle rmtree issues
---
.appveyor.yml | 3 +--
.gitlab-ci.yml | 50 ++++++++++++++++++++++++++++++----------------
browsepy/compat.py | 39 +++++++++++-------------------------
3 files changed, 46 insertions(+), 46 deletions(-)
diff --git a/.appveyor.yml b/.appveyor.yml
index ede2b90..31b2536 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -1,7 +1,6 @@
branches:
except:
- - gh-pages
- - /gh-pages-.*/
+ - /^gh-pages(-.*)?$/
environment:
matrix:
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f38d759..1f3056f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,10 @@
+.base:
+ except:
+ refs:
+ - /^gh-pages(-.*)?$/
+
.test-base:
+ extends: .base
stage: test
variables:
PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
@@ -13,13 +19,16 @@
paths:
- .coverage.*
before_script:
- - apk add --no-cache build-base python3-dev
- - pip install coverage flake8
+ - |
+ apk add --no-cache build-base python3-dev
+ pip install coverage flake8
script:
- - flake8 ${MODULE}
- - coverage run -p setup.py test
+ - |
+ flake8 ${MODULE}
+ coverage run -p setup.py test
.after-base:
+ extends: .base
image: python:3.7-alpine
variables:
PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
@@ -38,8 +47,9 @@ python27:
extends: .test-base
image: python:2.7-alpine
before_script:
- - apk add --no-cache build-base python2-dev
- - pip install coverage flake8
+ - |
+ apk add --no-cache build-base python2-dev
+ pip install coverage flake8
python35:
extends: .test-base
@@ -54,6 +64,7 @@ python37:
image: python:3.7-alpine
eslint:
+ extends: .base
stage: test
image: node:alpine
cache:
@@ -64,6 +75,7 @@ eslint:
- "**.json"
- "**.js"
- .eslintignore
+ - .eslintrc.json
before_script:
- npm install eslint
script:
@@ -80,9 +92,10 @@ coverage:
before_script:
- pip install coverage
script:
- - coverage combine
- - coverage html --fail-under=0
- - coverage report
+ - |
+ coverage combine
+ coverage html --fail-under=0
+ coverage report
doc:
extends: .after-base
@@ -92,12 +105,14 @@ doc:
paths:
- doc/.build/html
before_script:
- - apk add --no-cache make
- - pip install -r requirements/doc.txt
+ - |
+ apk add --no-cache make
+ pip install -r requirements/doc.txt
script:
- make -C doc html
publish-gh-pages:
+ extends: .base
stage: publish
image:
name: alpine/git
@@ -111,12 +126,13 @@ publish-gh-pages:
- master
- next
before_script:
- - mkdir -p ~/.ssh
- - echo "${SSH_PRIVATE_KEY}" | tr -d '\r' > ~/.ssh/id_rsa
- - chmod 600 ~/.ssh/id_rsa
- - ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
- - git config --global user.name "${GITLAB_USER_LOGIN}"
- - git config --global user.email "${GITLAB_USER_EMAIL}"
+ - |
+ mkdir -p ~/.ssh
+ echo "${SSH_PRIVATE_KEY}" | tr -d '\r' > ~/.ssh/id_rsa
+ chmod 600 ~/.ssh/id_rsa
+ ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
+ git config --global user.name "${GITLAB_USER_LOGIN}"
+ git config --global user.email "${GITLAB_USER_EMAIL}"
script:
- |
if [ "${CI_COMMIT_REF_NAME}" == "master" ]; then
diff --git a/browsepy/compat.py b/browsepy/compat.py
index d2c5f76..9d075a7 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -138,40 +138,25 @@ def rmtree(path):
:param path: path to remove
:type path: str
"""
- known = set()
- retries = 5
+ def remove_readonly(operation, name, exc_info):
+ """Clear the readonly bit and reattempt the removal."""
+ exc_type, exc_value, exc_traceback = exc_info
+ if issubclass(exc_type, PermissionError):
+ os.chmod(path, stat.S_IWRITE)
+ raise exc_info
+
+ retries = 10
while os.path.exists(path):
try:
- shutil.rmtree(path)
+ shutil.rmtree(path, onerror=remove_readonly)
except EnvironmentError as error:
if retries and any(
getattr(error, prop, None) in values
for prop, values in RETRYABLE_OSERROR_PROPERTIES.items()
):
- # files with permission issues (common on some platforms)
- unreachable = [
- filename
- for filename in (
- getattr(error, 'filename', None),
- getattr(error, 'filename2', None)
- )
- if (
- filename and
- filename not in known and
- not os.access(filename, os.W_OK)
- )
- ]
-
- # allow sluggish filesystems to catch up
- if not unreachable:
- retries -= 1
- time.sleep(0.1)
- continue
-
- # try to fix permissions
- known.update(unreachable)
- for filename in unreachable:
- os.chmod(path, stat.S_IWUSR)
+ retries -= 1
+ time.sleep(0.1) # allow sluggish filesystems to catch up
+ continue
raise
--
GitLab
From 22d43b654bbedf80b3c6028cdddeec9dcd918aff Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 1 Aug 2019 18:37:44 +0100
Subject: [PATCH 125/171] fix typo
---
browsepy/compat.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 9d075a7..1a885d3 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -138,12 +138,14 @@ def rmtree(path):
:param path: path to remove
:type path: str
"""
- def remove_readonly(operation, name, exc_info):
+ def remove_readonly(action, name, exc_info):
"""Clear the readonly bit and reattempt the removal."""
exc_type, exc_value, exc_traceback = exc_info
if issubclass(exc_type, PermissionError):
- os.chmod(path, stat.S_IWRITE)
- raise exc_info
+ os.chmod(name, stat.S_IWRITE)
+ action(name)
+ else:
+ raise exc_info
retries = 10
while os.path.exists(path):
--
GitLab
From 6e6d73936e94f966bf55f49fd61d6636f41a2a6a Mon Sep 17 00:00:00 2001
From: "Felipe A. Hernandez"
Date: Thu, 1 Aug 2019 17:56:38 +0000
Subject: [PATCH 126/171] handle permission error on py2
---
browsepy/compat.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 1a885d3..6a6ec65 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -17,6 +17,11 @@ import posixpath
import ntpath
import argparse
+try:
+ import builtins # python 3+
+except ImportError:
+ import __builtin__ as builtins # noqa
+
try:
import importlib.resources as res # python 3.7+
except ImportError:
@@ -141,7 +146,10 @@ def rmtree(path):
def remove_readonly(action, name, exc_info):
"""Clear the readonly bit and reattempt the removal."""
exc_type, exc_value, exc_traceback = exc_info
- if issubclass(exc_type, PermissionError):
+ if issubclass(
+ exc_type,
+ getattr(builtins, 'PermissionError', OSError),
+ ) and exc_value.errno == errno.EPERM:
os.chmod(name, stat.S_IWRITE)
action(name)
else:
--
GitLab
From f3195d9f8890bd26478f7f567e6631b233e457f7 Mon Sep 17 00:00:00 2001
From: "Felipe A. Hernandez"
Date: Thu, 1 Aug 2019 18:08:59 +0000
Subject: [PATCH 127/171] better py2 handling
---
browsepy/compat.py | 19 +++++++++++++------
1 file changed, 13 insertions(+), 6 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 6a6ec65..e1b780a 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -146,15 +146,22 @@ def rmtree(path):
def remove_readonly(action, name, exc_info):
"""Clear the readonly bit and reattempt the removal."""
exc_type, exc_value, exc_traceback = exc_info
- if issubclass(
- exc_type,
- getattr(builtins, 'PermissionError', OSError),
- ) and exc_value.errno == errno.EPERM:
+ is_perm = issubclass(
+ exc_type,
+ getattr(
+ builtins,
+ 'PermissionError',
+ EnvironmentError,
+ ),
+ ) and (
+ exc_value == errno.EPERM or
+ getattr(exc_value, 'winerror', None) == 5
+ )
+ if is_perm:
os.chmod(name, stat.S_IWRITE)
action(name)
else:
- raise exc_info
-
+ raise exc_type, exc_value, exc_traceback
retries = 10
while os.path.exists(path):
try:
--
GitLab
From c3310a49b1df72bbc66d880e8b4e87adb538cb72 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 2 Aug 2019 11:14:30 +0100
Subject: [PATCH 128/171] use six reraise
---
browsepy/compat.py | 15 +++++----------
1 file changed, 5 insertions(+), 10 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index e1b780a..b9145d4 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -4,6 +4,7 @@ import os
import os.path
import stat
import sys
+import six
import errno
import abc
import time
@@ -146,22 +147,16 @@ def rmtree(path):
def remove_readonly(action, name, exc_info):
"""Clear the readonly bit and reattempt the removal."""
exc_type, exc_value, exc_traceback = exc_info
- is_perm = issubclass(
- exc_type,
- getattr(
- builtins,
- 'PermissionError',
- EnvironmentError,
- ),
- ) and (
- exc_value == errno.EPERM or
+ is_perm = issubclass(exc_type, EnvironmentError) and (
+ exc_value.errno == errno.EPERM or
getattr(exc_value, 'winerror', None) == 5
)
if is_perm:
os.chmod(name, stat.S_IWRITE)
action(name)
else:
- raise exc_type, exc_value, exc_traceback
+ six.reraise(exc_type, exc_value, exc_traceback)
+
retries = 10
while os.path.exists(path):
try:
--
GitLab
From e8db67d187e2790cb727da6c09cf966f5c649526 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 2 Aug 2019 11:51:08 +0100
Subject: [PATCH 129/171] implement rmtree from scratch
---
browsepy/compat.py | 43 ++++++++++++++++++-------------------------
1 file changed, 18 insertions(+), 25 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index b9145d4..a3d2320 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -8,7 +8,6 @@ import six
import errno
import abc
import time
-import shutil
import tempfile
import itertools
import functools
@@ -137,40 +136,34 @@ def rmtree(path):
"""
Remove directory tree, with platform-specific fixes.
- A simple :func:`shutil.rmtree` wrapper, with some error handling and
- retry logic, as some filesystems on some platforms does not always
- behave as they should.
+ Implemented from scratch as :func:`shutil.rmtree` is broken on some
+ platforms and python version combinations.
:param path: path to remove
:type path: str
"""
- def remove_readonly(action, name, exc_info):
- """Clear the readonly bit and reattempt the removal."""
- exc_type, exc_value, exc_traceback = exc_info
- is_perm = issubclass(exc_type, EnvironmentError) and (
- exc_value.errno == errno.EPERM or
- getattr(exc_value, 'winerror', None) == 5
- )
- if is_perm:
- os.chmod(name, stat.S_IWRITE)
- action(name)
- else:
- six.reraise(exc_type, exc_value, exc_traceback)
-
- retries = 10
- while os.path.exists(path):
+ exc_type, exc_value, exc_traceback = None, None, None
+ for retry in range(10):
try:
- shutil.rmtree(path, onerror=remove_readonly)
- except EnvironmentError as error:
- if retries and any(
- getattr(error, prop, None) in values
+ if os.path.exists(path):
+ for base, dirs, files in os.walk(path, topdown=False):
+ os.chmod(base, stat.S_IRWXU)
+ for filename in files:
+ filename = os.path.join(base, filename)
+ os.chmod(filename, stat.S_IWUSR)
+ os.unlink(filename)
+ os.rmdir(base)
+ return
+ except EnvironmentError:
+ if any(
+ getattr(exc_value, prop, None) in values
for prop, values in RETRYABLE_OSERROR_PROPERTIES.items()
):
- retries -= 1
+ exc_type, exc_value, exc_traceback = sys.exc_info()
time.sleep(0.1) # allow sluggish filesystems to catch up
continue
-
raise
+ six.reraise(exc_type, exc_value, exc_traceback)
@contextlib.contextmanager
--
GitLab
From 44dea1582a614e6e54de3db428f7070d2b7f2b7d Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 2 Aug 2019 12:00:28 +0100
Subject: [PATCH 130/171] reduce complexity
---
browsepy/compat.py | 46 +++++++++++++++++++++++++++-------------------
1 file changed, 27 insertions(+), 19 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index a3d2320..f8e2729 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -132,6 +132,19 @@ def scandir(path):
files.close()
+def is_oserror_retryable(error):
+ """
+ Check if given OSError is retryable for filesystem operations.
+
+ :param error: error to check
+ :type error: OSError
+ """
+ for prop, values in RETRYABLE_OSERROR_PROPERTIES.items():
+ if getattr(error, prop, None) in values:
+ return True
+ return False
+
+
def rmtree(path):
"""
Remove directory tree, with platform-specific fixes.
@@ -142,28 +155,23 @@ def rmtree(path):
:param path: path to remove
:type path: str
"""
- exc_type, exc_value, exc_traceback = None, None, None
+ exc_info = ()
for retry in range(10):
try:
- if os.path.exists(path):
- for base, dirs, files in os.walk(path, topdown=False):
- os.chmod(base, stat.S_IRWXU)
- for filename in files:
- filename = os.path.join(base, filename)
- os.chmod(filename, stat.S_IWUSR)
- os.unlink(filename)
- os.rmdir(base)
+ for base, dirs, files in os.walk(path, topdown=False):
+ os.chmod(base, stat.S_IRWXU)
+ for filename in files:
+ filename = os.path.join(base, filename)
+ os.chmod(filename, stat.S_IWUSR)
+ os.unlink(filename)
+ os.rmdir(base)
return
- except EnvironmentError:
- if any(
- getattr(exc_value, prop, None) in values
- for prop, values in RETRYABLE_OSERROR_PROPERTIES.items()
- ):
- exc_type, exc_value, exc_traceback = sys.exc_info()
- time.sleep(0.1) # allow sluggish filesystems to catch up
- continue
- raise
- six.reraise(exc_type, exc_value, exc_traceback)
+ except EnvironmentError as error:
+ if not is_oserror_retryable(error):
+ raise
+ exc_info = sys.exc_info()
+ time.sleep(0.1) # allow sluggish filesystems to catch up
+ six.reraise(*exc_info)
@contextlib.contextmanager
--
GitLab
From bca3f6f8d41510c3f999fe808b18ba3c77200fd7 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 2 Aug 2019 12:11:21 +0100
Subject: [PATCH 131/171] try again to use shutil.rmtree
---
browsepy/compat.py | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index f8e2729..e5cd52c 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -3,6 +3,7 @@
import os
import os.path
import stat
+import shutil
import sys
import six
import errno
@@ -149,22 +150,18 @@ def rmtree(path):
"""
Remove directory tree, with platform-specific fixes.
- Implemented from scratch as :func:`shutil.rmtree` is broken on some
- platforms and python version combinations.
-
:param path: path to remove
:type path: str
"""
exc_info = ()
+ path_join = os.path.join
for retry in range(10):
try:
for base, dirs, files in os.walk(path, topdown=False):
os.chmod(base, stat.S_IRWXU)
for filename in files:
- filename = os.path.join(base, filename)
- os.chmod(filename, stat.S_IWUSR)
- os.unlink(filename)
- os.rmdir(base)
+ os.chmod(path_join(base, filename), stat.S_IWUSR)
+ shutil.rmtree(path)
return
except EnvironmentError as error:
if not is_oserror_retryable(error):
--
GitLab
From 7c8a85017ada09b54e768f2028c3b87b4c512e4b Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 2 Aug 2019 12:32:33 +0100
Subject: [PATCH 132/171] try to wait for directory content removal
---
.gitlab-ci.yml | 2 +-
browsepy/compat.py | 38 ++++++++++++++++++++++++++++----------
2 files changed, 29 insertions(+), 11 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1f3056f..907e1f9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -147,7 +147,7 @@ publish-gh-pages:
git checkout "${TARGET_BRANCH}" || git checkout --orphan "${TARGET_BRANCH}"
cp ../.gitignore ../.appveyor.yml .
cp -rf ${CI_PROJECT_DIR}/doc/.build/html/** .
- git add **
+ git add .
git commit -am "update ${TARGET_BRANCH}"
git push -u origin "${TARGET_BRANCH}"
diff --git a/browsepy/compat.py b/browsepy/compat.py
index e5cd52c..2f932f5 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -3,7 +3,6 @@
import os
import os.path
import stat
-import shutil
import sys
import six
import errno
@@ -133,7 +132,7 @@ def scandir(path):
files.close()
-def is_oserror_retryable(error):
+def _is_oserror_retryable(error):
"""
Check if given OSError is retryable for filesystem operations.
@@ -146,25 +145,44 @@ def is_oserror_retryable(error):
return False
+def _unsafe_rmtree(path):
+ """
+ Remove directory tree, without error handling.
+
+ :param path: directory path
+ :type path: str
+ """
+ for base, dirs, files in os.walk(path, topdown=False):
+ os.chmod(base, stat.S_IRWXU)
+
+ for filename in files:
+ filename = os.path.join(base, filename)
+ os.chmod(filename, stat.S_IWUSR)
+ os.remove(filename)
+
+ while os.listdir(base):
+ time.sleep(0.1) # wait for emptyness
+
+ os.rmdir(base)
+
+
def rmtree(path):
"""
Remove directory tree, with platform-specific fixes.
+ Implemented from scratch as :func:`shutil.rmtree` is broken on some
+ platforms and python version combinations.
+
:param path: path to remove
:type path: str
"""
+
exc_info = ()
- path_join = os.path.join
for retry in range(10):
try:
- for base, dirs, files in os.walk(path, topdown=False):
- os.chmod(base, stat.S_IRWXU)
- for filename in files:
- os.chmod(path_join(base, filename), stat.S_IWUSR)
- shutil.rmtree(path)
- return
+ return _unsafe_rmtree(path)
except EnvironmentError as error:
- if not is_oserror_retryable(error):
+ if not _is_oserror_retryable(error):
raise
exc_info = sys.exc_info()
time.sleep(0.1) # allow sluggish filesystems to catch up
--
GitLab
From bc443164db870498ccef13f133db3bf0d25fc6d4 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 2 Aug 2019 12:38:16 +0100
Subject: [PATCH 133/171] make rmtree wait and retry recursive
---
browsepy/compat.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 2f932f5..b1a597f 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -160,10 +160,11 @@ def _unsafe_rmtree(path):
os.chmod(filename, stat.S_IWUSR)
os.remove(filename)
- while os.listdir(base):
- time.sleep(0.1) # wait for emptyness
-
- os.rmdir(base)
+ if os.listdir(base):
+ time.sleep(0.5) # wait for emptyness
+ _unsafe_rmtree(base)
+ else:
+ os.rmdir(base)
def rmtree(path):
--
GitLab
From f4d44efbc2a6bbf0d135771921298a6ad56ae99c Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 2 Aug 2019 12:51:03 +0100
Subject: [PATCH 134/171] use scandir for rmtree
---
browsepy/compat.py | 25 +++++++++----------------
1 file changed, 9 insertions(+), 16 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index b1a597f..97859da 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -132,19 +132,6 @@ def scandir(path):
files.close()
-def _is_oserror_retryable(error):
- """
- Check if given OSError is retryable for filesystem operations.
-
- :param error: error to check
- :type error: OSError
- """
- for prop, values in RETRYABLE_OSERROR_PROPERTIES.items():
- if getattr(error, prop, None) in values:
- return True
- return False
-
-
def _unsafe_rmtree(path):
"""
Remove directory tree, without error handling.
@@ -152,7 +139,7 @@ def _unsafe_rmtree(path):
:param path: directory path
:type path: str
"""
- for base, dirs, files in os.walk(path, topdown=False):
+ for base, dirs, files in walk(path, topdown=False):
os.chmod(base, stat.S_IRWXU)
for filename in files:
@@ -160,7 +147,10 @@ def _unsafe_rmtree(path):
os.chmod(filename, stat.S_IWUSR)
os.remove(filename)
- if os.listdir(base):
+ with scandir(base) as remaining:
+ retry = any(remaining)
+
+ if retry:
time.sleep(0.5) # wait for emptyness
_unsafe_rmtree(base)
else:
@@ -183,7 +173,10 @@ def rmtree(path):
try:
return _unsafe_rmtree(path)
except EnvironmentError as error:
- if not _is_oserror_retryable(error):
+ if not any(
+ getattr(error, prop, None) in values
+ for prop, values in RETRYABLE_OSERROR_PROPERTIES.items()
+ ):
raise
exc_info = sys.exc_info()
time.sleep(0.1) # allow sluggish filesystems to catch up
--
GitLab
From f981d7dc1c5da3f0696927579a9b11617c3893d8 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 2 Aug 2019 16:20:16 +0100
Subject: [PATCH 135/171] no longer handle permission errors on rmtree
---
browsepy/compat.py | 12 +++---------
1 file changed, 3 insertions(+), 9 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 97859da..bebe441 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -2,7 +2,6 @@
import os
import os.path
-import stat
import sys
import six
import errno
@@ -59,7 +58,6 @@ RETRYABLE_OSERROR_PROPERTIES = {
# Error codes which could imply a busy filesystem
getattr(errno, prop)
for prop in (
- 'EPERM',
'ENOENT',
'EIO',
'ENXIO',
@@ -140,18 +138,14 @@ def _unsafe_rmtree(path):
:type path: str
"""
for base, dirs, files in walk(path, topdown=False):
- os.chmod(base, stat.S_IRWXU)
-
for filename in files:
- filename = os.path.join(base, filename)
- os.chmod(filename, stat.S_IWUSR)
- os.remove(filename)
+ os.remove(os.path.join(base, filename))
with scandir(base) as remaining:
retry = any(remaining)
if retry:
- time.sleep(0.5) # wait for emptyness
+ time.sleep(0.1) # wait for sluggish filesystems
_unsafe_rmtree(base)
else:
os.rmdir(base)
@@ -179,7 +173,7 @@ def rmtree(path):
):
raise
exc_info = sys.exc_info()
- time.sleep(0.1) # allow sluggish filesystems to catch up
+ time.sleep(0.1)
six.reraise(*exc_info)
--
GitLab
From 1122e71e716699de54d0911b4a3a294ef481661f Mon Sep 17 00:00:00 2001
From: "Felipe A. Hernandez"
Date: Fri, 9 Aug 2019 09:35:34 +0000
Subject: [PATCH 136/171] fix player plugin button sizing
---
browsepy/plugin/player/static/css/base.css | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/browsepy/plugin/player/static/css/base.css b/browsepy/plugin/player/static/css/base.css
index f1199d4..81eeb5f 100644
--- a/browsepy/plugin/player/static/css/base.css
+++ b/browsepy/plugin/player/static/css/base.css
@@ -6,6 +6,10 @@
width: auto;
}
+.jp-controls button {
+ box-sizing: border-box;
+}
+
.jp-current-time, .jp-duration {
width: 4.5em;
}
--
GitLab
From 97ac31318ee0c8343aa5b9bc6724cfdac602a885 Mon Sep 17 00:00:00 2001
From: "Felipe A. Hernandez"
Date: Fri, 9 Aug 2019 09:38:34 +0000
Subject: [PATCH 137/171] generalize player button fix
---
browsepy/plugin/player/static/css/base.css | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/browsepy/plugin/player/static/css/base.css b/browsepy/plugin/player/static/css/base.css
index 81eeb5f..4b92457 100644
--- a/browsepy/plugin/player/static/css/base.css
+++ b/browsepy/plugin/player/static/css/base.css
@@ -6,7 +6,7 @@
width: auto;
}
-.jp-controls button {
+.jp-audio button {
box-sizing: border-box;
}
--
GitLab
From 67cd6581f3928ccfb41f9de92a1b314cf46afc8a Mon Sep 17 00:00:00 2001
From: "Felipe A. Hernandez"
Date: Fri, 9 Aug 2019 09:56:07 +0000
Subject: [PATCH 138/171] generalize page titles
---
.../templates/create_directory.file_actions.html | 6 ++++++
.../file_actions/templates/selection.file_actions.html | 6 ++++++
browsepy/plugin/player/templates/audio.player.html | 6 ++++++
browsepy/templates/browse.html | 6 +++---
4 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/browsepy/plugin/file_actions/templates/create_directory.file_actions.html b/browsepy/plugin/file_actions/templates/create_directory.file_actions.html
index 93811cf..d0db9f2 100644
--- a/browsepy/plugin/file_actions/templates/create_directory.file_actions.html
+++ b/browsepy/plugin/file_actions/templates/create_directory.file_actions.html
@@ -1,5 +1,11 @@
{% extends "base.html" %}
+{% block title %}
+ {{- super() -}}
+ {%- if file and file.urlpath %} - {{ file.urlpath }}{% endif -%}
+ {{- ' - create directory' -}}
+{% endblock %}
+
{% block styles %}
{{ super() }}
diff --git a/browsepy/plugin/file_actions/templates/selection.file_actions.html b/browsepy/plugin/file_actions/templates/selection.file_actions.html
index ac333f9..593e210 100644
--- a/browsepy/plugin/file_actions/templates/selection.file_actions.html
+++ b/browsepy/plugin/file_actions/templates/selection.file_actions.html
@@ -1,5 +1,11 @@
{% extends "base.html" %}
+{% block title %}
+ {{- super() -}}
+ {%- if file and file.urlpath %} - {{ file.urlpath }}{% endif -%}
+ {{- ' - selection' -}}
+{% endblock %}
+
{% block styles %}
{{ super() }}
diff --git a/browsepy/plugin/player/templates/audio.player.html b/browsepy/plugin/player/templates/audio.player.html
index 908ea6c..2019919 100644
--- a/browsepy/plugin/player/templates/audio.player.html
+++ b/browsepy/plugin/player/templates/audio.player.html
@@ -1,5 +1,11 @@
{% extends "base.html" %}
+{% block title %}
+ {{- super() -}}
+ {%- if file and file.urlpath %} - {{ file.urlpath }}{% endif -%}
+ {{- ' - play' -}}
+{% endblock %}
+
{% block styles %}
{{ super() }}
diff --git a/browsepy/templates/browse.html b/browsepy/templates/browse.html
index 5773176..0713611 100644
--- a/browsepy/templates/browse.html
+++ b/browsepy/templates/browse.html
@@ -59,10 +59,10 @@
{%- endmacro %}
-{% block title -%}
- {{ super() }}
+{% block title %}
+ {{- super() -}}
{%- if file and file.urlpath %} - {{ file.urlpath }}{% endif -%}
-{%- endblock %}
+{% endblock %}
{% block styles %}
{{ super() }}
--
GitLab
From 2f2219d7b014d2f85ad72448c4978d84496f949a Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 6 Nov 2019 05:25:25 +0000
Subject: [PATCH 139/171] Add etag to browse endpoint
---
browsepy/__init__.py | 11 ++++++++++-
browsepy/file.py | 41 ++++++++++++++++++++++++++++-------------
browsepy/http.py | 8 ++++++++
3 files changed, 46 insertions(+), 14 deletions(-)
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index 9551097..164abc9 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -17,6 +17,7 @@ from .appconfig import Flask
from .manager import PluginManager
from .file import Node, secure_filename
from .stream import stream_template
+from .http import etag
from .exceptions import OutsideRemovableBase, OutsideDirectoryBase, \
InvalidFilenameError, InvalidPathError
from . import compat
@@ -159,13 +160,21 @@ def browse(path):
try:
directory = Node.from_urlpath(path)
if directory.is_directory and not directory.is_excluded:
- return stream_template(
+ response = stream_template(
'browse.html',
file=directory,
sort_property=sort_property,
sort_fnc=sort_fnc,
sort_reverse=sort_reverse
)
+ response.set_etag(
+ etag(
+ content_mtime=directory.content_mtime,
+ sort_property=sort_property,
+ ),
+ )
+ response.make_conditional(request)
+ return response
except OutsideDirectoryBase:
pass
return NotFound()
diff --git a/browsepy/file.py b/browsepy/file.py
index 2e6ef74..1e5fc4d 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -503,6 +503,21 @@ class Directory(Node):
encoding = 'default'
generic = False
+ @property
+ def content_mtime(self):
+ """
+ Get computed modification file based on its content.
+
+ :returns: modification id
+ :rtype: str
+ """
+ nodes = self.listdir(sortkey=lambda n: n.stats.st_mtime, reverse=True)
+ for node in nodes:
+ if node.stats.st_mtime > self.stats.st_mtime:
+ return node.stats.st_mtime
+ break
+ return self.stats.st_mtime
+
@property
def name(self):
"""
@@ -639,11 +654,11 @@ class Directory(Node):
:returns: True if this directory has no entries, False otherwise.
:rtype: bool
"""
- if self._listdir_cache is not None:
- return not bool(self._listdir_cache)
- for entry in self._listdir():
- return False
- return True
+ if self._listdir_cache is None:
+ for entry in self._listdir():
+ return False
+ return True
+ return not self._listdir_cache
def remove(self):
"""
@@ -689,6 +704,8 @@ class Directory(Node):
"""
Check if directory contains an entry with given filename.
+ This method purposely hits the filesystem to support platform quirks.
+
:param filename: filename will be check
:type filename: str
:returns: True if exists, False otherwise.
@@ -771,14 +788,12 @@ class Directory(Node):
:return: sorted list of File instances
:rtype: list of File instances
"""
- if self._listdir_cache is None:
- self._listdir_cache = tuple(self._listdir())
- if sortkey:
- return sorted(self._listdir_cache, key=sortkey, reverse=reverse)
- data = list(self._listdir_cache)
- if reverse:
- data.reverse()
- return data
+ cache = self._listdir_cache
+ if cache is None:
+ self._listdir_cache = cache = list(self._listdir())
+ if sortkey is None:
+ return cache[::-1 if reverse else 1]
+ return sorted(cache, key=sortkey, reverse=reverse)
def fmt_size(size, binary=True):
diff --git a/browsepy/http.py b/browsepy/http.py
index 94ad19c..ba890cd 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -1,8 +1,11 @@
# -*- coding: UTF-8 -*-
import re
+import binascii
import logging
+import msgpack
+
from werkzeug.http import dump_header, dump_options_header
from werkzeug.datastructures import Headers as BaseHeaders
@@ -54,3 +57,8 @@ class Headers(BaseHeaders):
for key, value in kwargs.items()
]
return super(Headers, self).__init__(items)
+
+
+def etag(*args, **kwargs):
+ """Generate etag identifier from given parameters."""
+ return '{:x}'.format(binascii.crc32(msgpack.dumps((args, kwargs))))
--
GitLab
From 3aa65060c2c5c605e39ea3df7acc9e32ae5c4b61 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 6 Nov 2019 18:22:36 +0000
Subject: [PATCH 140/171] make cached listdir to always precompute stats
---
browsepy/compat.py | 2 ++
browsepy/file.py | 19 +++++++++----------
2 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index bebe441..52038d1 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -515,6 +515,7 @@ if PY_LEGACY:
range = xrange # noqa
filter = itertools.ifilter
map = itertools.imap
+ zip = itertools.izip
basestring = basestring # noqa
unicode = unicode # noqa
chr = unichr # noqa
@@ -525,6 +526,7 @@ else:
range = range
filter = filter
map = map
+ zip = zip
basestring = str
unicode = str
chr = chr
diff --git a/browsepy/file.py b/browsepy/file.py
index 1e5fc4d..e257f9c 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -511,12 +511,9 @@ class Directory(Node):
:returns: modification id
:rtype: str
"""
- nodes = self.listdir(sortkey=lambda n: n.stats.st_mtime, reverse=True)
- for node in nodes:
- if node.stats.st_mtime > self.stats.st_mtime:
- return node.stats.st_mtime
- break
- return self.stats.st_mtime
+ mtimes = [node.stats.st_mtime for node in self.listdir()]
+ mtimes.append(self.stats.st_mtime)
+ return max(mtimes)
@property
def name(self):
@@ -680,7 +677,7 @@ class Directory(Node):
self.path,
self.app.config.get('DIRECTORY_TAR_BUFFSIZE', 10240),
flask.copy_current_request_context(
- self.plugin_manager.check_excluded
+ self.plugin_manager.check_excluded,
),
self.app.config.get('DIRECTORY_TAR_COMPRESS', 'gzip'),
self.app.config.get('DIRECTORY_TAR_COMPRESSLEVEL', 1),
@@ -696,8 +693,8 @@ class Directory(Node):
content_disposition=(
'attachment',
{'filename': stream.name} if stream.name else {},
- )
- )
+ ),
+ ),
)
def contains(self, filename):
@@ -790,7 +787,9 @@ class Directory(Node):
"""
cache = self._listdir_cache
if cache is None:
- self._listdir_cache = cache = list(self._listdir())
+ cache = self._listdir_cache = (
+ list(self._listdir(precomputed_stats=True))
+ )
if sortkey is None:
return cache[::-1 if reverse else 1]
return sorted(cache, key=sortkey, reverse=reverse)
--
GitLab
From 700957f9d8adfbe6965d6145a6c0c065b83b337b Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 6 Nov 2019 19:10:46 +0000
Subject: [PATCH 141/171] cleanup
---
browsepy/__init__.py | 25 ++++++++++---------------
browsepy/file.py | 24 ++++++++++++------------
browsepy/http.py | 5 ++---
browsepy/static/base.css | 12 +++++++++---
4 files changed, 33 insertions(+), 33 deletions(-)
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index 164abc9..20efbc1 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -10,7 +10,7 @@ import cookieman
from flask import request, render_template, redirect, \
url_for, send_from_directory, \
- make_response, session
+ session
from werkzeug.exceptions import NotFound
from .appconfig import Flask
@@ -62,6 +62,7 @@ plugin_manager = PluginManager(app)
@app.session_interface.register('browse:sort')
def shrink_browse_sort(data, last):
+ """Session `browse:short` size reduction logic."""
if data['browse:sort'] and not last:
data['browse:sort'].pop()
else:
@@ -148,7 +149,7 @@ def sort(property, path):
session['browse:sort'] = \
[(path, property)] + session.get('browse:sort', [])
- return redirect(url_for(".browse", path=directory.urlpath))
+ return redirect(url_for('.browse', path=directory.urlpath))
@app.route('/browse', defaults={'path': ''})
@@ -167,6 +168,7 @@ def browse(path):
sort_fnc=sort_fnc,
sort_reverse=sort_reverse
)
+ response.last_modified = directory.content_mtime
response.set_etag(
etag(
content_mtime=directory.content_mtime,
@@ -180,7 +182,7 @@ def browse(path):
return NotFound()
-@app.route('/open/', endpoint="open")
+@app.route('/open/', endpoint='open')
def open_file(path):
try:
file = Node.from_urlpath(path)
@@ -191,7 +193,7 @@ def open_file(path):
return NotFound()
-@app.route("/download/file/")
+@app.route('/download/file/')
def download_file(path):
try:
file = Node.from_urlpath(path)
@@ -202,7 +204,7 @@ def download_file(path):
return NotFound()
-@app.route("/download/directory/.tgz")
+@app.route('/download/directory/.tgz')
def download_directory(path):
try:
directory = Node.from_urlpath(path)
@@ -213,7 +215,7 @@ def download_directory(path):
return NotFound()
-@app.route("/remove/", methods=("GET", "POST"))
+@app.route('/remove/', methods=('GET', 'POST'))
def remove(path):
try:
file = Node.from_urlpath(path)
@@ -230,8 +232,8 @@ def remove(path):
return redirect(url_for(".browse", path=file.parent.urlpath))
-@app.route("/upload", defaults={'path': ''}, methods=("POST",))
-@app.route("/upload/", methods=("POST",))
+@app.route('/upload', defaults={'path': ''}, methods=('POST',))
+@app.route('/upload/', methods=('POST',))
def upload(path):
try:
directory = Node.from_urlpath(path)
@@ -270,13 +272,6 @@ def index():
return browse(urlpath)
-@app.after_request
-def page_not_found(response):
- if response.status_code == 404:
- return make_response((render_template('404.html'), 404))
- return response
-
-
@app.errorhandler(InvalidPathError)
def bad_request_error(e):
file = None
diff --git a/browsepy/file.py b/browsepy/file.py
index e257f9c..409e4b5 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -503,18 +503,6 @@ class Directory(Node):
encoding = 'default'
generic = False
- @property
- def content_mtime(self):
- """
- Get computed modification file based on its content.
-
- :returns: modification id
- :rtype: str
- """
- mtimes = [node.stats.st_mtime for node in self.listdir()]
- mtimes.append(self.stats.st_mtime)
- return max(mtimes)
-
@property
def name(self):
"""
@@ -657,6 +645,18 @@ class Directory(Node):
return True
return not self._listdir_cache
+ @cached_property
+ def content_mtime(self):
+ """
+ Get computed modification file based on its content.
+
+ :returns: modification id
+ :rtype: str
+ """
+ mtimes = [node.stats.st_mtime for node in self.listdir()]
+ mtimes.append(self.stats.st_mtime)
+ return max(mtimes)
+
def remove(self):
"""
Remove directory tree.
diff --git a/browsepy/http.py b/browsepy/http.py
index ba890cd..ec81a9c 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -1,12 +1,11 @@
# -*- coding: UTF-8 -*-
import re
-import binascii
import logging
import msgpack
-from werkzeug.http import dump_header, dump_options_header
+from werkzeug.http import dump_header, dump_options_header, generate_etag
from werkzeug.datastructures import Headers as BaseHeaders
@@ -61,4 +60,4 @@ class Headers(BaseHeaders):
def etag(*args, **kwargs):
"""Generate etag identifier from given parameters."""
- return '{:x}'.format(binascii.crc32(msgpack.dumps((args, kwargs))))
+ return generate_etag(msgpack.dumps((args, kwargs)))
diff --git a/browsepy/static/base.css b/browsepy/static/base.css
index cdd7248..16ef3eb 100644
--- a/browsepy/static/base.css
+++ b/browsepy/static/base.css
@@ -114,6 +114,7 @@ table.browser tr:nth-child(2n) {
table.browser th,
table.browser td {
+ padding: 0;
margin: 0;
text-align: center;
vertical-align: middle;
@@ -122,7 +123,6 @@ table.browser td {
}
table.browser th {
- padding-bottom: 0.75em;
border-bottom: 1px solid black;
}
@@ -230,13 +230,19 @@ ol.path li *.root:after {
a.sorting {
display: block;
- padding: 0 1.4em 0 0;
+}
+
+table.browser th a.sorting {
+ margin: -0.5em;
+ padding: 0.5em 0.5em 0.75em;
}
a.sorting:after,
a.sorting:before {
float: right;
- margin: 0.2em -1.3em -0.2em;
+ line-height: 1.25em;
+ vertical-align: middle;
+ margin-left: -1em;
}
a.sorting:hover:after,
--
GitLab
From 5e4e1ce17ce0b8dbb81cc1127944233391301a30 Mon Sep 17 00:00:00 2001
From: "Felipe A. Hernandez"
Date: Tue, 3 Dec 2019 10:28:05 +0000
Subject: [PATCH 142/171] add py38 to CI
---
.appveyor.yml | 2 ++
.gitlab-ci.yml | 6 ++++++
2 files changed, 8 insertions(+)
diff --git a/.appveyor.yml b/.appveyor.yml
index 31b2536..c3df04d 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -12,6 +12,8 @@ environment:
- PYTHON: "C:\\Python36-x64"
- PYTHON: "C:\\Python37"
- PYTHON: "C:\\Python37-x64"
+ - PYTHON: "C:\\Python38"
+ - PYTHON: "C:\\Python38-x64"
build: off
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 907e1f9..0407b4e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -63,6 +63,10 @@ python37:
extends: .test-base
image: python:3.7-alpine
+python38:
+ extends: .test-base
+ image: python:3.8-alpine
+
eslint:
extends: .base
stage: test
@@ -133,6 +137,8 @@ publish-gh-pages:
ssh-keyscan -H 'gitlab.com' >> ~/.ssh/known_hosts
git config --global user.name "${GITLAB_USER_LOGIN}"
git config --global user.email "${GITLAB_USER_EMAIL}"
+ after_script:
+ - rm ~/.ssh/id_rsa
script:
- |
if [ "${CI_COMMIT_REF_NAME}" == "master" ]; then
--
GitLab
From 8dfb4c16cd648abbcd8e16fb9f520bd6a36baaf9 Mon Sep 17 00:00:00 2001
From: "Felipe A. Hernandez"
Date: Tue, 3 Dec 2019 15:02:01 +0000
Subject: [PATCH 143/171] ensure build deps are installed on ci
---
.appveyor.yml | 2 +-
.gitlab-ci.yml | 58 +++++++++++++++++++++++++-------------------------
2 files changed, 30 insertions(+), 30 deletions(-)
diff --git a/.appveyor.yml b/.appveyor.yml
index c3df04d..85248bb 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -31,7 +31,7 @@ install:
SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
python --version
python -c "import struct; print(struct.calcsize('P') * 8)"
- pip install --disable-pip-version-check --user --upgrade pip setuptools
+ pip install --disable-pip-version-check --user --upgrade pip setuptools wheel
test_script:
- "python setup.py test"
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0407b4e..d9f168f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,17 +2,20 @@
except:
refs:
- /^gh-pages(-.*)?$/
+ cache:
+ paths:
+ - .cache/pip
+ variables:
+ PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
+ MODULE: browsepy
-.test-base:
+.test:
extends: .base
stage: test
variables:
PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
COVERAGE_FILE: "${CI_PROJECT_DIR}/.coverage.${CI_JOB_NAME}"
MODULE: browsepy
- cache:
- paths:
- - .cache/pip
artifacts:
when: on_success
expire_in: 1h
@@ -21,21 +24,26 @@
before_script:
- |
apk add --no-cache build-base python3-dev
- pip install coverage flake8
+ pip install coverage flake8 wheel
script:
- |
flake8 ${MODULE}
coverage run -p setup.py test
-.after-base:
+.report:
extends: .base
+ stage: report
image: python:3.7-alpine
- variables:
- PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
- MODULE: browsepy
- cache:
- paths:
- - .cache/pip
+
+.publish:
+ extends: .base
+ stage: publish
+ dependencies: []
+ before_script:
+ - |
+ apk add --no-cache build-base python3-dev
+ pip install twine
+
stages:
- test
@@ -44,7 +52,7 @@ stages:
- publish
python27:
- extends: .test-base
+ extends: .test
image: python:2.7-alpine
before_script:
- |
@@ -52,19 +60,19 @@ python27:
pip install coverage flake8
python35:
- extends: .test-base
+ extends: .test
image: python:3.5-alpine
python36:
- extends: .test-base
+ extends: .test
image: python:3.6-alpine
python37:
- extends: .test-base
+ extends: .test
image: python:3.7-alpine
python38:
- extends: .test-base
+ extends: .test
image: python:3.8-alpine
eslint:
@@ -86,8 +94,7 @@ eslint:
- node_modules/.bin/eslint ${MODULE}
coverage:
- extends: .after-base
- stage: report
+ extends: .report
artifacts:
when: always
paths:
@@ -102,8 +109,7 @@ coverage:
coverage report
doc:
- extends: .after-base
- stage: report
+ extends: .report
dependencies: []
artifacts:
paths:
@@ -158,14 +164,11 @@ publish-gh-pages:
git push -u origin "${TARGET_BRANCH}"
publish-alpha:
- extends: .after-base
- stage: publish
+ extends: .publish
dependencies: []
only:
refs:
- next
- before_script:
- - pip install twine
script:
- |
ALPHA=$(date +%s)
@@ -176,14 +179,11 @@ publish-alpha:
twine upload --repository-url=https://test.pypi.org/legacy/ dist/*
publish:
- extends: .after-base
- stage: publish
+ extends: .publish
dependencies: []
only:
refs:
- master
- before_script:
- - pip install twine
script:
- |
python setup.py bdist_wheel sdist
--
GitLab
From 3d579e19fe0765ff45af397207ddf8379cda7cba Mon Sep 17 00:00:00 2001
From: "Felipe A. Hernandez"
Date: Tue, 3 Dec 2019 15:12:46 +0000
Subject: [PATCH 144/171] ensure ci got correct image
---
.gitlab-ci.yml | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d9f168f..84a5c34 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,6 +2,10 @@
except:
refs:
- /^gh-pages(-.*)?$/
+
+.python:
+ extends: .base
+ image: python:3.8-alpine
cache:
paths:
- .cache/pip
@@ -10,7 +14,7 @@
MODULE: browsepy
.test:
- extends: .base
+ extends: .python
stage: test
variables:
PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
@@ -31,12 +35,11 @@
coverage run -p setup.py test
.report:
- extends: .base
+ extends: .python
stage: report
- image: python:3.7-alpine
.publish:
- extends: .base
+ extends: .python
stage: publish
dependencies: []
before_script:
--
GitLab
From 36836a03e7bbeb9c5231e8584d877cd385f006ae Mon Sep 17 00:00:00 2001
From: "Felipe A. Hernandez"
Date: Tue, 3 Dec 2019 15:22:51 +0000
Subject: [PATCH 145/171] ci refactor config file for readability
---
.gitlab-ci.yml | 24 ++++++++++++++++--------
1 file changed, 16 insertions(+), 8 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 84a5c34..d487a1a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,3 +1,12 @@
+
+# config
+
+stages:
+- test
+- report
+- external
+- publish
+
.base:
except:
refs:
@@ -47,12 +56,7 @@
apk add --no-cache build-base python3-dev
pip install twine
-
-stages:
-- test
-- report
-- external
-- publish
+# stage: tests
python27:
extends: .test
@@ -96,6 +100,8 @@ eslint:
script:
- node_modules/.bin/eslint ${MODULE}
+# stage: report
+
coverage:
extends: .report
artifacts:
@@ -124,6 +130,8 @@ doc:
script:
- make -C doc html
+# stage: publish
+
publish-gh-pages:
extends: .base
stage: publish
@@ -166,7 +174,7 @@ publish-gh-pages:
git commit -am "update ${TARGET_BRANCH}"
git push -u origin "${TARGET_BRANCH}"
-publish-alpha:
+publish-next:
extends: .publish
dependencies: []
only:
@@ -181,7 +189,7 @@ publish-alpha:
python setup.py bdist_wheel sdist
twine upload --repository-url=https://test.pypi.org/legacy/ dist/*
-publish:
+publish-master:
extends: .publish
dependencies: []
only:
--
GitLab
From 3b906d35dc25cee6b3eea56a508a9299deee1df1 Mon Sep 17 00:00:00 2001
From: "Felipe A. Hernandez"
Date: Tue, 3 Dec 2019 15:34:25 +0000
Subject: [PATCH 146/171] ci add libffi-dev
---
.gitlab-ci.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d487a1a..08f1d91 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -36,7 +36,7 @@ stages:
- .coverage.*
before_script:
- |
- apk add --no-cache build-base python3-dev
+ apk add --no-cache build-base libffi-dev python3-dev
pip install coverage flake8 wheel
script:
- |
@@ -53,7 +53,7 @@ stages:
dependencies: []
before_script:
- |
- apk add --no-cache build-base python3-dev
+ apk add --no-cache build-base libffi-dev python3-dev
pip install twine
# stage: tests
@@ -63,7 +63,7 @@ python27:
image: python:2.7-alpine
before_script:
- |
- apk add --no-cache build-base python2-dev
+ apk add --no-cache build-base libffi-dev python2-dev
pip install coverage flake8
python35:
--
GitLab
From 7b61384167782c8d16567a0347ca8aa99f5b0ee6 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Wed, 4 Dec 2019 13:16:56 +0000
Subject: [PATCH 147/171] fix symlink vulnerabilities, drop deprecated plugin
manager
---
browsepy/compat.py | 2 +-
browsepy/file.py | 48 ++++++++++---
browsepy/manager.py | 168 ++++----------------------------------------
browsepy/stream.py | 79 ++++++++++-----------
4 files changed, 92 insertions(+), 205 deletions(-)
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 52038d1..049727c 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -497,7 +497,7 @@ def re_escape(pattern, chars=frozenset("()[]{}?*+|^$\\.-#")):
if PY_LEGACY:
- class FileNotFoundError(BaseException):
+ class FileNotFoundError(Exception):
__metaclass__ = abc.ABCMeta
FileNotFoundError.register(OSError)
diff --git a/browsepy/file.py b/browsepy/file.py
index 409e4b5..670764f 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -26,11 +26,11 @@ from .exceptions import OutsideDirectoryBase, OutsideRemovableBase, \
logger = logging.getLogger(__name__)
-unicode_underscore = '_'.decode('utf-8') if compat.PY_LEGACY else '_'
+unicode_underscore = compat.unicode('_')
underscore_replace = '%s:underscore' % __name__
codecs.register_error(
underscore_replace,
- lambda error: (unicode_underscore, error.start + 1)
+ lambda error: (unicode_underscore, getattr(error, 'start', 0) + 1),
)
binary_units = ('B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB')
standard_units = ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB')
@@ -80,7 +80,19 @@ class Node(object):
:returns: True if excluded, False otherwise
"""
- return self.plugin_manager.check_excluded(self.path)
+ return self.plugin_manager.check_excluded(
+ self.path,
+ follow_symlinks=self.is_symlink,
+ )
+
+ @cached_property
+ def is_symlink(self):
+ """
+ Get if current node is a symlink.
+
+ :returns: True if symlink, False otherwise
+ """
+ return os.path.islink(self.path)
@cached_property
def plugin_manager(self):
@@ -152,7 +164,12 @@ class Node(object):
:returns: stats object
:rtype: posix.stat_result or nt.stat_result
"""
- return os.stat(self.path)
+ try:
+ return os.stat(self.path)
+ except compat.FileNotFoundError:
+ if self.is_symlink:
+ return os.lstat(self.path)
+ raise
@cached_property
def pathconf(self):
@@ -617,7 +634,7 @@ class Directory(Node):
:returns: True if a file can be upload to directory, False otherwise
:rtype: bool
"""
- dirbase = self.app.config.get('DIRECTORY_UPLOAD', False)
+ dirbase = self.app.config.get('DIRECTORY_UPLOAD')
return dirbase and check_base(self.path, dirbase)
@cached_property
@@ -748,7 +765,9 @@ class Directory(Node):
def _listdir(self, precomputed_stats=(os.name == 'nt')):
"""
- Iter unsorted entries on this directory.
+ Iterate unsorted entries on this directory.
+
+ Symlinks are skipped when pointing outside to base directory.
:yields: Directory or File instance for each entry in directory
:rtype: Iterator of browsepy.file.Node
@@ -756,25 +775,30 @@ class Directory(Node):
directory_class = self.directory_class
file_class = self.file_class
exclude_fnc = self.plugin_manager.check_excluded
-
with compat.scandir(self.path) as files:
for entry in files:
- if exclude_fnc(entry.path):
+ is_symlink = entry.is_symlink()
+ if exclude_fnc(entry.path, follow_symlinks=is_symlink):
continue
kwargs = {
'path': entry.path,
'app': self.app,
'parent': self,
'is_excluded': False,
+ 'is_symlink': is_symlink,
}
try:
- if precomputed_stats and not entry.is_symlink():
- kwargs['stats'] = entry.stat()
+ if precomputed_stats:
+ kwargs['stats'] = entry.stat(
+ follow_symlinks=is_symlink,
+ )
yield (
directory_class(**kwargs)
- if entry.is_dir(follow_symlinks=True) else
+ if entry.is_dir(follow_symlinks=is_symlink) else
file_class(**kwargs)
)
+ except compat.FileNotFoundError:
+ pass
except OSError as e:
logger.exception(e)
@@ -782,6 +806,8 @@ class Directory(Node):
"""
Get sorted list (by given sortkey and reverse params) of File objects.
+ Symlinks are skipped when pointing outside to base directory.
+
:return: sorted list of File instances
:rtype: list of File instances
"""
diff --git a/browsepy/manager.py b/browsepy/manager.py
index 70e8f60..51a71f5 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -1,22 +1,19 @@
# -*- coding: UTF-8 -*-
-import re
+import os.path
import pkgutil
import argparse
import functools
import warnings
-import collections
-
-import flask
from werkzeug.utils import cached_property
from cookieman import CookieMan
from . import mimetype
from . import compat
+from . import file
from . import utils
-from .compat import deprecated, usedoc
from .utils import defaultsnamedtuple
from .exceptions import PluginNotFoundError, InvalidArgumentError, \
WidgetParameterException
@@ -271,12 +268,16 @@ class ExcludePluginManager(PluginManagerBase):
"""
self._exclude_functions.add(exclude_fnc)
- def check_excluded(self, path):
+ def check_excluded(self, path, follow_symlinks=True):
"""
Check if given path is excluded.
+ Followed symlinks are checked against directory base for safety.
+
:param path: absolute path to check against config and plugins
:type path: str
+ :param follow_symlinks: wether or not follow_symlinks
+ :type follow_symlinks: bool
:return: wether if path should be excluded or not
:rtype: bool
"""
@@ -286,6 +287,13 @@ class ExcludePluginManager(PluginManagerBase):
for fnc in self._exclude_functions:
if fnc(path):
return True
+ if follow_symlinks:
+ realpath = os.path.realpath(path)
+ dirbase = self.app.config.get('DIRECTORY_BASE')
+ return realpath != path and (
+ dirbase and not file.check_base(path, dirbase) or
+ self.check_excluded(realpath, follow_symlinks=False)
+ )
return False
def clear(self):
@@ -708,153 +716,7 @@ class ArgumentPluginManager(PluginManagerBase):
return getattr(self._argparse_arguments, name, default)
-class MimetypeActionPluginManager(WidgetPluginManager, MimetypePluginManager):
- """
- Deprecated plugin API
- """
-
- _deprecated_places = {
- 'javascript': 'scripts',
- 'style': 'styles',
- 'button': 'entry-actions',
- 'link': 'entry-link',
- }
-
- @classmethod
- def _mimetype_filter(cls, mimetypes):
- widget_mimetype_re = re.compile(
- '^%s$' % '$|^'.join(
- map(re.escape, mimetypes)
- ).replace('\\*', '[^/]+')
- )
-
- def handler(f):
- return widget_mimetype_re.match(f.type) is not None
-
- return handler
-
- def _widget_attrgetter(self, widget, name):
- def handler(f):
- app = f.app or self.app or flask.current_app
- with app.app_context():
- return getattr(widget.for_file(f), name)
- return handler
-
- def _widget_props(self, widget, endpoint=None, mimetypes=(),
- dynamic=False):
- type = getattr(widget, '_type', 'base')
- fields = self.widget_types[type]._fields
- with self.app.app_context():
- props = {
- name: self._widget_attrgetter(widget, name)
- for name in fields
- if hasattr(widget, name)
- }
- props.update(
- type=type,
- place=self._deprecated_places.get(widget.place),
- )
- if dynamic:
- props['filter'] = self._mimetype_filter(mimetypes)
- if 'endpoint' in fields:
- props['endpoint'] = endpoint
- return props
-
- @usedoc(WidgetPluginManager.__init__)
- def __init__(self, app=None):
- self._action_widgets = []
- super(MimetypeActionPluginManager, self).__init__(app=app)
-
- @usedoc(WidgetPluginManager.clear)
- def clear(self):
- self._action_widgets[:] = ()
- super(MimetypeActionPluginManager, self).clear()
-
- @cached_property
- def _widget(self):
- with warnings.catch_warnings():
- warnings.filterwarnings('ignore', category=DeprecationWarning)
- from . import widget
- return widget
-
- @cached_property
- @deprecated('Deprecated attribute action_class')
- def action_class(self):
- return collections.namedtuple(
- 'MimetypeAction',
- ('endpoint', 'widget')
- )
-
- @cached_property
- @deprecated('Deprecated attribute style_class')
- def style_class(self):
- return self._widget.StyleWidget
-
- @cached_property
- @deprecated('Deprecated attribute button_class')
- def button_class(self):
- return self._widget.ButtonWidget
-
- @cached_property
- @deprecated('Deprecated attribute javascript_class')
- def javascript_class(self):
- return self._widget.JavascriptWidget
-
- @cached_property
- @deprecated('Deprecated attribute link_class')
- def link_class(self):
- return self._widget.LinkWidget
-
- @deprecated('Deprecated method register_action')
- def register_action(self, endpoint, widget, mimetypes=(), **kwargs):
- props = self._widget_props(widget, endpoint, mimetypes, True)
- self.register_widget(**props)
- self._action_widgets.append((widget, props['filter'], endpoint))
-
- @deprecated('Deprecated method get_actions')
- def get_actions(self, file):
- return [
- self.action_class(endpoint, deprecated.for_file(file))
- for deprecated, filter, endpoint in self._action_widgets
- if endpoint and filter(file)
- ]
-
- @usedoc(WidgetPluginManager.register_widget)
- def register_widget(self, place=None, type=None, widget=None, filter=None,
- **kwargs):
- if isinstance(place or widget, self._widget.WidgetBase):
- warnings.warn(
- 'Deprecated use of register_widget',
- DeprecationWarning
- )
- widget = place or widget
- props = self._widget_props(widget)
- self.register_widget(**props)
- self._action_widgets.append((widget, None, None))
- return
- return super(MimetypeActionPluginManager, self).register_widget(
- place=place, type=type, widget=widget, filter=filter, **kwargs)
-
- @usedoc(WidgetPluginManager.get_widgets)
- def get_widgets(self, file=None, place=None):
- if isinstance(file, compat.basestring) or \
- place in self._deprecated_places:
- warnings.warn(
- 'Deprecated use of get_widgets',
- DeprecationWarning
- )
- place = file or place
- return [
- widget
- for widget, filter, endpoint in self._action_widgets
- if not (filter or endpoint) and place == widget.place
- ]
- return super(MimetypeActionPluginManager, self).get_widgets(
- file=file, place=place)
-
-
-class PluginManager(MimetypeActionPluginManager,
- BlueprintPluginManager,
+class PluginManager(BlueprintPluginManager,
ExcludePluginManager,
WidgetPluginManager,
MimetypePluginManager,
diff --git a/browsepy/stream.py b/browsepy/stream.py
index 50c918a..47a9c56 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -90,7 +90,8 @@ class TarFileStream(compat.Iterator):
queue_class = ByteQueue
abort_exception = WriteAbort
thread_class = threading.Thread
- tarfile_class = tarfile.open
+ tarfile_open = tarfile.open
+ tarfile_format = tarfile.PAX_FORMAT
mimetype = 'application/x-tar'
compresion_modes = {
@@ -102,18 +103,19 @@ class TarFileStream(compat.Iterator):
@property
def name(self):
- """
- Filename generated from given path and compression method.
- """
+ """Get filename generated from given path and compression method."""
return '%s.%s' % (os.path.basename(self.path), self._extension)
@property
def encoding(self):
- """
- Mimetype parameters (such as encoding).
- """
+ """Mimetype parameters (such as encoding)."""
return self._compress
+ @property
+ def closed(self):
+ """Get if input stream have been closed with no further writes."""
+ return self._closed
+
def __init__(self, path, buffsize=10240, exclude=None, compress='gzip',
compresslevel=1):
"""
@@ -134,22 +136,23 @@ class TarFileStream(compat.Iterator):
"""
self.path = path
self.exclude = exclude
- self.closed = False
self._started = False
+ self._closed = False
self._buffsize = buffsize
self._compress = compress if compress and buffsize > 15 else None
self._mode, self._extension = self.compresion_modes[self._compress]
self._queue = self.queue_class(buffsize)
- self._th = self.thread_class(target=self._fill)
- self._th_exc = None
+ self._thread = self.thread_class(target=self._fill)
+ self._thread_exception = None
- @property
- def _infofilter(self):
+ def _fill(self):
"""
- TarInfo filtering function based on :attr:`exclude`.
+ Perform compression pushing compressed data to internal queue.
+
+ Used as compression thread target, started on first iteration.
"""
path = self.path
path_join = os.path.join
@@ -164,32 +167,28 @@ class TarFileStream(compat.Iterator):
:return: infofile or None if file must be excluded
:rtype: tarfile.TarInfo or None
"""
- return None if exclude(path_join(path, info.name)) else info
-
- return infofilter if exclude else None
-
- def _fill(self):
- """
- Writes data on internal tarfile instance, which writes to current
- object, using :meth:`write`.
+ return (
+ None
+ if exclude(
+ path_join(path, info.name),
+ follow_symlinks=info.issym(),
+ ) else
+ info
+ )
- As this method is blocking, it is used inside a thread.
-
- This method is called automatically, on a thread, on initialization,
- so there is little need to call it manually.
- """
try:
- with self.tarfile_class(
+ with self.tarfile_open(
fileobj=self,
mode='w|{}'.format(self._mode),
bufsize=self._buffsize,
+ format=self.tarfile_format,
encoding='utf-8',
) as tarfile:
- tarfile.add(self.path, '', filter=self._infofilter)
+ tarfile.add(path, filter=infofilter if exclude else None)
except self.abort_exception:
pass
except Exception as e:
- self._th_exc = e
+ self._thread_exception = e
finally:
self._queue.finish()
@@ -202,16 +201,16 @@ class TarFileStream(compat.Iterator):
:returns: tarfile data as bytes
:rtype: bytes
"""
- if self.closed:
- raise StopIteration()
+ if self._closed:
+ raise StopIteration
if not self._started:
self._started = True
- self._th.start()
+ self._thread.start()
data = self._queue.get()
if not data:
- raise StopIteration()
+ raise StopIteration
return data
@@ -228,7 +227,7 @@ class TarFileStream(compat.Iterator):
:rtype: int
:raises WriteAbort: if already closed or closed while blocking
"""
- if self.closed:
+ if self._closed:
raise self.abort_exception()
try:
@@ -242,13 +241,13 @@ class TarFileStream(compat.Iterator):
"""
Closes tarfile pipe and stops further processing.
"""
- if not self.closed:
- self.closed = True
+ if not self._closed:
+ self._closed = True
self._queue.finish()
- if self._started and self._th.is_alive():
- self._th.join()
- if self._th_exc:
- raise self._th_exc
+ if self._started and self._thread.is_alive():
+ self._thread.join()
+ if self._thread_exception:
+ raise self._thread_exception
def stream_template(template_name, **context):
--
GitLab
From adda9b82d2d626721769e228fe6cecd8d807b195 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Fri, 6 Dec 2019 17:04:36 +0000
Subject: [PATCH 148/171] wip refactor
---
browsepy/__init__.py | 157 ++++------
browsepy/__main__.py | 4 +-
browsepy/file.py | 30 +-
browsepy/manager.py | 6 +-
browsepy/mimetype.py | 16 +-
browsepy/plugin/player/__init__.py | 99 ++----
browsepy/plugin/player/playable.py | 245 +++++++++------
.../plugin/player/templates/audio.player.html | 4 +-
browsepy/stream.py | 58 ++--
browsepy/tests/deprecated/__init__.py | 0
browsepy/tests/deprecated/plugin/__init__.py | 0
browsepy/tests/deprecated/plugin/player.py | 92 ------
browsepy/tests/deprecated/test_plugins.py | 292 ------------------
browsepy/tests/runner.py | 53 ----
browsepy/tests/test_code.py | 28 +-
browsepy/transform/__init__.py | 19 +-
browsepy/transform/glob.py | 19 ++
browsepy/transform/htmlcompress.py | 16 +
browsepy/utils.py | 31 +-
browsepy/widget.py | 89 ------
requirements/development.txt | 3 +
setup.py | 6 +-
22 files changed, 405 insertions(+), 862 deletions(-)
delete mode 100644 browsepy/tests/deprecated/__init__.py
delete mode 100644 browsepy/tests/deprecated/plugin/__init__.py
delete mode 100644 browsepy/tests/deprecated/plugin/player.py
delete mode 100644 browsepy/tests/deprecated/test_plugins.py
delete mode 100644 browsepy/tests/runner.py
delete mode 100644 browsepy/widget.py
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index 20efbc1..bf81e8b 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -10,7 +10,7 @@ import cookieman
from flask import request, render_template, redirect, \
url_for, send_from_directory, \
- session
+ session, abort
from werkzeug.exceptions import NotFound
from .appconfig import Flask
@@ -139,17 +139,12 @@ def template_globals():
@app.route('/sort/', defaults={'path': ''})
@app.route('/sort//')
def sort(property, path):
- try:
- directory = Node.from_urlpath(path)
- except OutsideDirectoryBase:
- return NotFound()
-
- if not directory.is_directory or directory.is_excluded:
- return NotFound()
-
- session['browse:sort'] = \
- [(path, property)] + session.get('browse:sort', [])
- return redirect(url_for('.browse', path=directory.urlpath))
+ directory = Node.from_urlpath(path)
+ if directory.is_directory and not directory.is_excluded:
+ session['browse:sort'] = \
+ [(path, property)] + session.get('browse:sort', [])
+ return redirect(url_for('.browse', path=directory.urlpath))
+ abort(404)
@app.route('/browse', defaults={'path': ''})
@@ -157,119 +152,93 @@ def sort(property, path):
def browse(path):
sort_property = get_cookie_browse_sorting(path, 'text')
sort_fnc, sort_reverse = browse_sortkey_reverse(sort_property)
-
- try:
- directory = Node.from_urlpath(path)
- if directory.is_directory and not directory.is_excluded:
- response = stream_template(
- 'browse.html',
- file=directory,
+ directory = Node.from_urlpath(path)
+ if directory.is_directory and not directory.is_excluded:
+ response = stream_template(
+ 'browse.html',
+ file=directory,
+ sort_property=sort_property,
+ sort_fnc=sort_fnc,
+ sort_reverse=sort_reverse
+ )
+ response.last_modified = directory.content_mtime
+ response.set_etag(
+ etag(
+ content_mtime=directory.content_mtime,
sort_property=sort_property,
- sort_fnc=sort_fnc,
- sort_reverse=sort_reverse
- )
- response.last_modified = directory.content_mtime
- response.set_etag(
- etag(
- content_mtime=directory.content_mtime,
- sort_property=sort_property,
- ),
- )
- response.make_conditional(request)
- return response
- except OutsideDirectoryBase:
- pass
- return NotFound()
+ ),
+ )
+ response.make_conditional(request)
+ return response
+ abort(404)
@app.route('/open/', endpoint='open')
def open_file(path):
- try:
- file = Node.from_urlpath(path)
- if file.is_file and not file.is_excluded:
- return send_from_directory(file.parent.path, file.name)
- except OutsideDirectoryBase:
- pass
- return NotFound()
+ file = Node.from_urlpath(path)
+ if file.is_file and not file.is_excluded:
+ return send_from_directory(file.parent.path, file.name)
+ abort(404)
@app.route('/download/file/')
def download_file(path):
- try:
- file = Node.from_urlpath(path)
- if file.is_file and not file.is_excluded:
- return file.download()
- except OutsideDirectoryBase:
- pass
- return NotFound()
+ file = Node.from_urlpath(path)
+ if file.is_file and not file.is_excluded:
+ return file.download()
+ abort(404)
@app.route('/download/directory/.tgz')
def download_directory(path):
- try:
- directory = Node.from_urlpath(path)
- if directory.is_directory and not directory.is_excluded:
- return directory.download()
- except OutsideDirectoryBase:
- pass
- return NotFound()
+ directory = Node.from_urlpath(path)
+ if directory.is_directory and not directory.is_excluded:
+ return directory.download()
+ abort(404)
@app.route('/remove/', methods=('GET', 'POST'))
def remove(path):
- try:
- file = Node.from_urlpath(path)
- except OutsideDirectoryBase:
- return NotFound()
-
- if not file.can_remove or file.is_excluded:
- return NotFound()
-
- if request.method == 'GET':
- return render_template('remove.html', file=file)
-
- file.remove()
- return redirect(url_for(".browse", path=file.parent.urlpath))
+ file = Node.from_urlpath(path)
+ if file.can_remove and not file.is_excluded:
+ if request.method == 'GET':
+ return render_template('remove.html', file=file)
+ file.remove()
+ return redirect(url_for(".browse", path=file.parent.urlpath))
+ abort(404)
@app.route('/upload', defaults={'path': ''}, methods=('POST',))
@app.route('/upload/', methods=('POST',))
def upload(path):
- try:
- directory = Node.from_urlpath(path)
- except OutsideDirectoryBase:
- return NotFound()
-
+ directory = Node.from_urlpath(path)
if (
- not directory.is_directory or
- not directory.can_upload or
- directory.is_excluded
+ directory.is_directory and
+ directory.can_upload and
+ not directory.is_excluded
):
- return NotFound()
-
- for v in request.files.listvalues():
- for f in v:
- filename = secure_filename(f.filename)
- if filename:
- filename = directory.choose_filename(filename)
- filepath = os.path.join(directory.path, filename)
- f.save(filepath)
- else:
+ files = (
+ (secure_filename(file.filename), file)
+ for values in request.files.listvalues()
+ for file in values
+ )
+ for filename, file in files:
+ if not filename:
raise InvalidFilenameError(
path=directory.path,
- filename=f.filename
+ filename=file.filename,
)
- return redirect(url_for('.browse', path=directory.urlpath))
+ filename = directory.choose_filename(filename)
+ filepath = os.path.join(directory.path, filename)
+ file.save(filepath)
+ return redirect(url_for('.browse', path=directory.urlpath))
+ abort(404)
@app.route('/')
def index():
path = app.config['DIRECTORY_START'] or app.config['DIRECTORY_BASE']
- try:
- urlpath = Node(path).urlpath
- except OutsideDirectoryBase:
- return NotFound()
- return browse(urlpath)
+ return browse(Node(path).urlpath)
@app.errorhandler(InvalidPathError)
@@ -284,11 +253,13 @@ def bad_request_error(e):
@app.errorhandler(OutsideRemovableBase)
+@app.errorhandler(OutsideDirectoryBase)
@app.errorhandler(404)
def page_not_found_error(e):
return render_template('404.html'), 404
+@app.errorhandler(Exception)
@app.errorhandler(500)
def internal_server_error(e): # pragma: no cover
logger.exception(e)
diff --git a/browsepy/__main__.py b/browsepy/__main__.py
index 318234f..987025f 100644
--- a/browsepy/__main__.py
+++ b/browsepy/__main__.py
@@ -39,7 +39,6 @@ class ArgParse(SafeArgumentParser):
default_port = os.getenv('BROWSEPY_PORT', '8080')
defaults = {
- 'add_help': True,
'prog': name,
'formatter_class': HelpFormatter,
'description': 'description: starts a %s web file browser' % name
@@ -55,6 +54,9 @@ class ArgParse(SafeArgumentParser):
'port', nargs='?', type=int,
default=self.default_port,
help='port to listen (default: %(default)s)')
+ self.add_argument(
+ '--help', action='store_true',
+ help='show help and exit (honors --plugin)')
self.add_argument(
'--help-all', action='store_true',
help='show help for all available plugins and exit')
diff --git a/browsepy/file.py b/browsepy/file.py
index 670764f..cda6fc2 100644
--- a/browsepy/file.py
+++ b/browsepy/file.py
@@ -14,7 +14,6 @@ import flask
from werkzeug.utils import cached_property
from . import compat
-from . import manager
from . import utils
from . import stream
@@ -60,13 +59,14 @@ class Node(object):
to specify :meth:`from_urlpath` classmethod behavior:
* :attr:`generic`, if true, an instance of directory_class or file_class
- will be created instead of an instance of this class tself.
+ will be created instead of an instance of this class itself.
* :attr:`directory_class`, class will be used for directory nodes,
* :attr:`file_class`, class will be used for file nodes.
"""
generic = True
- directory_class = None # set later at import time
- file_class = None # set later at import time
+ directory_class = None # set at import time
+ file_class = None # set at import time
+ manager_class = None # set at import time
re_charset = re.compile('; charset=(?P[^;]+)')
can_download = False
@@ -75,8 +75,7 @@ class Node(object):
@cached_property
def is_excluded(self):
"""
- Get if current node shouldn't be shown, using :attt:`app` config's
- exclude_fnc.
+ Get if current node should be avoided based on registered rules.
:returns: True if excluded, False otherwise
"""
@@ -104,7 +103,7 @@ class Node(object):
"""
return (
self.app.extensions.get('plugin_manager') or
- manager.PluginManager(self.app)
+ self.manager_class(self.app)
)
@cached_property
@@ -339,7 +338,7 @@ class Node(object):
@classmethod
def register_file_class(cls, kls):
"""
- Convenience method for setting current class file_class property.
+ Set given type as file_class constructor for current class.
:param kls: class to set
:type kls: type
@@ -352,7 +351,7 @@ class Node(object):
@classmethod
def register_directory_class(cls, kls):
"""
- Convenience method for setting current class directory_class property.
+ Set given type as directory_class constructor for current class.
:param kls: class to set
:type kls: type
@@ -362,6 +361,19 @@ class Node(object):
cls.directory_class = kls
return kls
+ @classmethod
+ def register_manager_class(cls, kls):
+ """
+ Set given type as manager_class constructor for current class.
+
+ :param kls: class to set
+ :type kls: type
+ :returns: given class (enabling using this as decorator)
+ :rtype: type
+ """
+ cls.manager_class = kls
+ return kls
+
@Node.register_file_class
class File(Node):
diff --git a/browsepy/manager.py b/browsepy/manager.py
index 51a71f5..c72bbf5 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -463,8 +463,7 @@ class WidgetPluginManager(PluginManagerBase):
def register_widget(self, place=None, type=None, widget=None, filter=None,
**kwargs):
"""
- Create (see :meth:`create_widget`) or use provided widget and register
- it.
+ Register a widget, optionally creating it with :meth:`create_widget`.
This method provides this dual behavior in order to simplify widget
creation-registration on an functional single step without sacrifycing
@@ -670,7 +669,7 @@ class ArgumentPluginManager(PluginManagerBase):
for argargs, argkwargs in arguments:
group.add_argument(*argargs, **argkwargs)
- if options.help_all:
+ if options.help or options.help_all:
parser.print_help()
parser.exit()
@@ -716,6 +715,7 @@ class ArgumentPluginManager(PluginManagerBase):
return getattr(self._argparse_arguments, name, default)
+@file.Node.register_manager_class
class PluginManager(BlueprintPluginManager,
ExcludePluginManager,
WidgetPluginManager,
diff --git a/browsepy/mimetype.py b/browsepy/mimetype.py
index ef681f0..9201d54 100644
--- a/browsepy/mimetype.py
+++ b/browsepy/mimetype.py
@@ -1,4 +1,9 @@
-# -*- coding: UTF-8 -*-
+"""
+File mimetype detection functions.
+
+This module exposes an :var:`alternatives` tuple containing an ordered
+list of detection strategies, sorted by priority.
+"""
import re
import subprocess
@@ -11,17 +16,19 @@ re_mime_validate = re.compile(r'\w+/\w+(; \w+=[^;]+)*')
def by_python(path):
+ """Get mimetype by file extension using python mimetype database."""
mime, encoding = mimetypes.guess_type(path)
if mime in generic_mimetypes:
return None
return "%s%s%s" % (
- mime or "application/octet-stream", "; "
- if encoding else
- "", encoding or ""
+ mime or "application/octet-stream",
+ "; " if encoding else "",
+ encoding or ""
)
def by_file(path):
+ """Get mimetype by calling file POSIX utility."""
try:
output = subprocess.check_output(
("file", "-ib", path),
@@ -36,6 +43,7 @@ def by_file(path):
def by_default(path):
+ """Get default generic mimetype."""
return "application/octet-stream"
diff --git a/browsepy/plugin/player/__init__.py b/browsepy/plugin/player/__init__.py
index b24625f..5a6a820 100644
--- a/browsepy/plugin/player/__init__.py
+++ b/browsepy/plugin/player/__init__.py
@@ -1,15 +1,11 @@
# -*- coding: UTF-8 -*-
-from flask import Blueprint, render_template
-from werkzeug.exceptions import NotFound
-
+from flask import Blueprint, abort
from browsepy import get_cookie_browse_sorting, browse_sortkey_reverse
-from browsepy.file import OutsideDirectoryBase
from browsepy.utils import ppath
from browsepy.stream import stream_template
-from .playable import PlayableFile, PlayableDirectory, \
- PlayListFile, detect_playable_mimetype
+from .playable import Playable, detect_playable_mimetype
player = Blueprint(
@@ -21,51 +17,29 @@ player = Blueprint(
)
-@player.route('/audio/')
-def audio(path):
- try:
- file = PlayableFile.from_urlpath(path)
- if file.is_file:
- return render_template('audio.player.html', file=file)
- except OutsideDirectoryBase:
- pass
- return NotFound()
-
-
-@player.route('/list/')
-def playlist(path):
- try:
- file = PlayListFile.from_urlpath(path)
- if file.is_file:
- return stream_template(
- 'audio.player.html',
- file=file,
- playlist=True
- )
- except OutsideDirectoryBase:
- pass
- return NotFound()
-
-
-@player.route('/directory', defaults={'path': ''})
-@player.route('/directory/')
-def directory(path):
+@player.route('/', defaults={'path': ''})
+@player.route('/')
+def play(path):
+ """
+ Handle player requests.
+
+ :param path: path to directory
+ :type path: str
+ :returns: flask.Response
+ """
sort_property = get_cookie_browse_sorting(path, 'text')
sort_fnc, sort_reverse = browse_sortkey_reverse(sort_property)
- try:
- file = PlayableDirectory.from_urlpath(path)
- if file.is_directory:
- return stream_template(
- 'audio.player.html',
- file=file,
- sort_property=sort_property,
- sort_fnc=sort_fnc,
- sort_reverse=sort_reverse,
- playlist=True
- )
- except OutsideDirectoryBase:
- pass
- return NotFound()
+ node = Playable.from_urlpath(path)
+ if node.is_playable and not node.is_excluded:
+ return stream_template(
+ 'audio.player.html',
+ file=node,
+ sort_property=sort_property,
+ sort_fnc=sort_fnc,
+ sort_reverse=sort_reverse,
+ playlist=node.playable_list,
+ )
+ abort(404)
def register_arguments(manager):
@@ -77,7 +51,6 @@ def register_arguments(manager):
:param manager: plugin manager
:type manager: browsepy.manager.PluginManager
"""
-
# Arguments are forwarded to argparse:ArgumentParser.add_argument,
# https://docs.python.org/3.7/library/argparse.html#the-add-argument-method
manager.register_argument(
@@ -108,15 +81,8 @@ def register_plugin(manager):
manager.register_widget(
place='entry-link',
type='link',
- endpoint='player.audio',
- filter=PlayableFile.detect
- )
- manager.register_widget(
- place='entry-link',
- icon='playlist',
- type='link',
- endpoint='player.playlist',
- filter=PlayListFile.detect
+ endpoint='player.play',
+ filter=Playable.playable_check,
)
# register action buttons
@@ -124,15 +90,8 @@ def register_plugin(manager):
place='entry-actions',
css='play',
type='button',
- endpoint='player.audio',
- filter=PlayableFile.detect
- )
- manager.register_widget(
- place='entry-actions',
- css='play',
- type='button',
- endpoint='player.playlist',
- filter=PlayListFile.detect
+ endpoint='player.play',
+ filter=Playable.playable_check,
)
# check argument (see `register_arguments`) before registering
@@ -141,7 +100,7 @@ def register_plugin(manager):
manager.register_widget(
place='header',
type='button',
- endpoint='player.directory',
+ endpoint='player.play',
text='Play directory',
- filter=PlayableDirectory.detect
+ filter=Playable.playable_check,
)
diff --git a/browsepy/plugin/player/playable.py b/browsepy/plugin/player/playable.py
index 1536a02..93b75a3 100644
--- a/browsepy/plugin/player/playable.py
+++ b/browsepy/plugin/player/playable.py
@@ -34,6 +34,7 @@ class PLSFileParser(object):
)
def __init__(self, path):
+ """Initialize."""
with warnings.catch_warnings():
# We already know about SafeConfigParser deprecation!
warnings.simplefilter('ignore', category=DeprecationWarning)
@@ -41,6 +42,7 @@ class PLSFileParser(object):
self._parser.read(path)
def getint(self, section, key, fallback=NOT_SET):
+ """Get int from pls file, returning fallback if unable."""
try:
return self._parser.getint(section, key)
except self.option_exceptions:
@@ -49,6 +51,7 @@ class PLSFileParser(object):
return fallback
def get(self, section, key, fallback=NOT_SET):
+ """Get value from pls file, returning fallback if unable."""
try:
return self._parser.get(section, key)
except self.option_exceptions:
@@ -57,74 +60,31 @@ class PLSFileParser(object):
return fallback
-class PlayableBase(File):
- extensions = {
- 'mp3': 'audio/mpeg',
- 'ogg': 'audio/ogg',
- 'wav': 'audio/wav',
- 'm3u': 'audio/x-mpegurl',
- 'm3u8': 'audio/x-mpegurl',
- 'pls': 'audio/x-scpls',
- }
-
- @classmethod
- def extensions_from_mimetypes(cls, mimetypes):
- mimetypes = frozenset(mimetypes)
- return {
- ext: mimetype
- for ext, mimetype in cls.extensions.items()
- if mimetype in mimetypes
- }
-
- @classmethod
- def detect(cls, node, os_sep=os.sep):
- basename = node.path.rsplit(os_sep)[-1]
- if '.' in basename:
- ext = basename.rsplit('.')[-1]
- return cls.extensions.get(ext, None)
- return None
-
-
-class PlayableFile(PlayableBase):
- mimetypes = ['audio/mpeg', 'audio/ogg', 'audio/wav']
- extensions = PlayableBase.extensions_from_mimetypes(mimetypes)
- media_map = {mime: ext for ext, mime in extensions.items()}
-
- def __init__(self, **kwargs):
- self.duration = kwargs.pop('duration', None)
- self.title = kwargs.pop('title', None)
- super(PlayableFile, self).__init__(**kwargs)
-
- @property
- def title(self):
- return self._title or self.name
+class Playable(Node):
+ """Base class for playable nodes."""
- @title.setter
- def title(self, title):
- self._title = title
+ playable_list = False
- @property
- def media_format(self):
- return self.media_map[self.type]
-
-
-class PlayListFile(PlayableBase):
- playable_class = PlayableFile
- mimetypes = ['audio/x-mpegurl', 'audio/x-mpegurl', 'audio/x-scpls']
- extensions = PlayableBase.extensions_from_mimetypes(mimetypes)
+ @cached_property
+ def is_playable(self):
+ """
+ Get if node is playable.
- @classmethod
- def from_urlpath(cls, path, app=None):
- original = Node.from_urlpath(path, app)
- if original.mimetype == PlayableDirectory.mimetype:
- return PlayableDirectory(original.path, original.app)
- elif original.mimetype == M3UFile.mimetype:
- return M3UFile(original.path, original.app)
- if original.mimetype == PLSFile.mimetype:
- return PLSFile(original.path, original.app)
- return original
+ :returns: True if node is playable, False otherwise
+ :rtype: bool
+ """
+ print(self)
+ return self.playable_check(self)
def normalize_playable_path(self, path):
+ """
+ Fixes the path of playable file from a playlist.
+
+ :param path: absolute or relative path or uri
+ :type path: str
+ :returns: absolute path or uri
+ :rtype: str or None
+ """
if '://' in path:
return path
path = os.path.normpath(path)
@@ -141,16 +101,103 @@ class PlayListFile(PlayableBase):
return ()
def entries(self, sortkey=None, reverse=None):
- for file in self._entries():
- if self.playable_class.detect(file):
- yield file
+ """Get playlist file entries."""
+ for node in self._entries():
+ if node.is_playable and not node.is_excluded:
+ yield node
+
+ @classmethod
+ def playable_check(cls, node):
+ """Check if class supports node."""
+ kls = cls.directory_class if node.is_directory else cls.file_class
+ return kls.playable_check(node)
+
+
+@Playable.register_file_class
+class PlayableExtension(Playable):
+ """=Generic node for filenames with extension."""
+
+ title = None
+ duration = None
+
+ playable_extensions = {}
+ playable_extension_classes = [] # deferred
+
+ @cached_property
+ def mimetype(self):
+ """Get mimetype."""
+ print(self)
+ return self.detect_mimetype(self.path)
+
+ @classmethod
+ def detect_extension(cls, path):
+ for extension in cls.playable_extensions:
+ if path.endswith('.%s' % extension):
+ return extension
+ return None
+ @classmethod
+ def detect_mimetype(cls, path):
+ return cls.playable_extensions.get(cls.detect_extension(path))
+
+ @classmethod
+ def playable_check(cls, node):
+ """Get if file is playable."""
+ if cls.generic:
+ return any(
+ kls.playable_check(node)
+ for kls in cls.playable_extension_classes
+ )
+ return cls.detect_extension(node.path) in cls.playable_extensions
+
+ @classmethod
+ def from_node(cls, node, app=None):
+ """Get playable node from given node."""
+ if cls.generic:
+ for kls in cls.playable_classes:
+ playable = kls.from_node(node)
+ if playable:
+ return playable
+ return node
+ return kls(node.path, node.app)
+
+ @classmethod
+ def from_urlpath(cls, path, app=None):
+ kls = cls.get_extension_class(path)
+ if cls.generic:
+ for kls in cls.playable_extension_classes:
+ if kls.detect_extension(path):
+ return kls.from_urlpath(path, app=app)
+ return super(PlayableExtension, cls).from_urlpath(path, app=app)
+
+ @classmethod
+ def register_playable_extension_class(cls, kls):
+ cls.playable_extension_classes.append(kls)
+ return kls
+
+
+@PlayableExtension.register_playable_extension_class
+class PlayableAudioFile(PlayableExtension, File):
+ """Audio file node."""
+
+ playable_extensions = {
+ 'mp3': 'audio/mpeg',
+ 'ogg': 'audio/ogg',
+ 'wav': 'audio/wav',
+ }
+
+
+@PlayableExtension.register_playable_extension_class
+class PLSFile(PlayableExtension, File):
+ """PLS playlist file node."""
+
+ playable_list = True
+ playable_extensions = {
+ 'pls': 'audio/x-scpls',
+ }
-class PLSFile(PlayListFile):
ini_parser_class = PLSFileParser
maxsize = getattr(sys, 'maxint', 0) or getattr(sys, 'maxsize', 0) or 2**32
- mimetype = 'audio/x-scpls'
- extensions = PlayableBase.extensions_from_mimetypes([mimetype])
def _entries(self):
parser = self.ini_parser_class(self.path)
@@ -180,9 +227,15 @@ class PLSFile(PlayListFile):
)
-class M3UFile(PlayListFile):
- mimetype = 'audio/x-mpegurl'
- extensions = PlayableBase.extensions_from_mimetypes([mimetype])
+@PlayableExtension.register_playable_extension_class
+class M3UFile(PlayableExtension, File):
+ """M3U playlist file node."""
+
+ playable_list = True
+ playable_extensions = {
+ 'm3u': 'audio/x-mpegurl',
+ 'm3u8': 'audio/x-mpegurl',
+ }
def _iter_lines(self):
prefix = '#EXTM3U\n'
@@ -214,32 +267,34 @@ class M3UFile(PlayListFile):
data.clear()
-class PlayableDirectory(Directory):
- file_class = PlayableFile
- name = ''
+@Playable.register_directory_class
+class PlayListDirectory(Playable, Directory):
+ """Playable directory node."""
+
+ playable_list = True
@cached_property
def parent(self):
+ """Get standard directory node."""
return self.directory_class(self.path, app=self.app)
- @classmethod
- def detect(cls, node):
- if node.is_directory:
- for file in node._listdir():
- if cls.file_class.detect(file):
- return cls.mimetype
- return None
-
def entries(self, sortkey=None, reverse=None):
- listdir_fnc = super(PlayableDirectory, self).listdir
- for file in listdir_fnc(sortkey=sortkey, reverse=reverse):
- if self.file_class.detect(file):
- yield file
-
-
-def detect_playable_mimetype(path, os_sep=os.sep):
- basename = path.rsplit(os_sep)[-1]
- if '.' in basename:
- ext = basename.rsplit('.')[-1]
- return PlayableBase.extensions.get(ext, None)
- return None
+ """Get playable directory entries."""
+ for node in self.listdir(sortkey=sortkey, reverse=reverse):
+ if (not node.is_directory) and node.is_playable:
+ yield node
+
+ @classmethod
+ def playable_check(cls, node):
+ """Detect if given node contains playable files."""
+ super_check = super(PlayListDirectory, cls).playable_check
+ return node.is_directory and any(
+ super_check(child)
+ for child in node._listdir()
+ if not child.is_directory
+ )
+
+
+def detect_playable_mimetype(path):
+ """Detect if path corresponds to a playable file by its extension."""
+ return PlayableExtension.detect_mimetype(path)
diff --git a/browsepy/plugin/player/templates/audio.player.html b/browsepy/plugin/player/templates/audio.player.html
index 2019919..1dd98b7 100644
--- a/browsepy/plugin/player/templates/audio.player.html
+++ b/browsepy/plugin/player/templates/audio.player.html
@@ -25,14 +25,14 @@
data-player-urls="
{%- for entry in file.entries(sortkey=sort_fnc, reverse=sort_reverse) -%}
{%- if not loop.first -%}|{%- endif -%}
- {{- entry.media_format -}}|
+ {{- entry.mimetype -}}|
{{- entry.name -}}|
{{- url_for('open', path=entry.urlpath) -}}
{%- endfor -%}
"
{% else %}
data-player-title="{{ file.name }}"
- data-player-format="{{ file.media_format }}"
+ data-player-format="{{ file.mimetype }}"
data-player-url="{{ url_for('open', path=file.urlpath) }}"
{% endif %}
>
diff --git a/browsepy/stream.py b/browsepy/stream.py
index 47a9c56..6094230 100644
--- a/browsepy/stream.py
+++ b/browsepy/stream.py
@@ -1,4 +1,4 @@
-# -*- coding: UTF-8 -*-
+"""Streaming functionality with generators and response constructors."""
import os
import os.path
@@ -12,12 +12,12 @@ from . import compat
class ByteQueue(compat.Queue):
"""
- Small synchronized queue storing bytes, with an additional finish method
- with turns the queue :method:`get` into non-blocking (returns empty bytes).
+ Synchronized byte queue, with an additional finish method.
- On a finished queue all :method:`put` will raise Full exceptions,
- regardless of the parameters given.
+ On a finished, :method:`put` will raise queue.Full exceptions and
+ :method:`get` will return empty bytes without blockng.
"""
+
def _init(self, maxsize):
self.queue = []
self.bytes = 0
@@ -42,17 +42,17 @@ class ByteQueue(compat.Queue):
return data
def qsize(self):
- """
- Return the number of bytes in the queue.
- """
+ """Return the number of bytes in the queue."""
with self.mutex:
return self.bytes
def finish(self):
"""
- Turn queue into finished mode: :method:`get` becomes non-blocking
- and returning empty bytes if empty, and :method:`put` raising
- :class:`queue.Full` exceptions unconditionally.
+ Put queue into finished mode.
+
+ On finished mode, :method:`get` becomes non-blocking once empty
+ by returning empty and :method:`put` raises :class:`queue.Full`
+ exceptions unconditionally.
"""
self.finished = True
@@ -61,10 +61,8 @@ class ByteQueue(compat.Queue):
class WriteAbort(Exception):
- """
- Exception used internally by :class:`TarFileStream`'s default
- implementation to stop tarfile compression.
- """
+ """Exception to stop tarfile compression process."""
+
pass
@@ -119,7 +117,7 @@ class TarFileStream(compat.Iterator):
def __init__(self, path, buffsize=10240, exclude=None, compress='gzip',
compresslevel=1):
"""
- Initialize thread and class (thread is not started until interation).
+ Initialize thread and class (thread is not started until iteration).
Note that compress parameter will be ignored if buffsize is below 16.
@@ -150,7 +148,7 @@ class TarFileStream(compat.Iterator):
def _fill(self):
"""
- Perform compression pushing compressed data to internal queue.
+ Compress files in path, pushing compressed data into internal queue.
Used as compression thread target, started on first iteration.
"""
@@ -160,7 +158,7 @@ class TarFileStream(compat.Iterator):
def infofilter(info):
"""
- Filter TarInfo objects for TarFile.
+ Filter TarInfo objects from TarFile.
:param info:
:type info: tarfile.TarInfo
@@ -184,7 +182,12 @@ class TarFileStream(compat.Iterator):
format=self.tarfile_format,
encoding='utf-8',
) as tarfile:
- tarfile.add(path, filter=infofilter if exclude else None)
+ tarfile.add(
+ path,
+ arcname='', # as archive root
+ recursive=True,
+ filter=infofilter if exclude else None,
+ )
except self.abort_exception:
pass
except Exception as e:
@@ -194,7 +197,9 @@ class TarFileStream(compat.Iterator):
def __next__(self):
"""
- Pulls chunk from tarfile (which is processed on its own thread).
+ Get chunk from internal queue.
+
+ Starts compression thread on first call.
:param want: number bytes to read, defaults to 0 (all available)
:type want: int
@@ -216,10 +221,11 @@ class TarFileStream(compat.Iterator):
def write(self, data):
"""
- Put chunk of data into data queue, used on the tarfile thread.
+ Add chunk of data into data queue.
- This method blocks when pipe is already, applying backpressure to
- writers.
+ This method is used inside the compression thread, blocking when
+ the internal queue is already full, propagating backpressure to
+ writer.
:param data: bytes to write to pipe
:type data: bytes
@@ -238,9 +244,7 @@ class TarFileStream(compat.Iterator):
return len(data)
def close(self):
- """
- Closes tarfile pipe and stops further processing.
- """
+ """Close tarfile pipe and stops further processing."""
if not self._closed:
self._closed = True
self._queue.finish()
@@ -252,6 +256,8 @@ class TarFileStream(compat.Iterator):
def stream_template(template_name, **context):
"""
+ Get streaming response rendering a jinja template.
+
Some templates can be huge, this function returns an streaming response,
sending the content in chunks and preventing from timeout.
diff --git a/browsepy/tests/deprecated/__init__.py b/browsepy/tests/deprecated/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/browsepy/tests/deprecated/plugin/__init__.py b/browsepy/tests/deprecated/plugin/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/browsepy/tests/deprecated/plugin/player.py b/browsepy/tests/deprecated/plugin/player.py
deleted file mode 100644
index 07e7ab1..0000000
--- a/browsepy/tests/deprecated/plugin/player.py
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-
-import os.path
-
-from flask import Blueprint, render_template
-from browsepy.file import File
-
-
-mimetypes = {
- 'mp3': 'audio/mpeg',
- 'ogg': 'audio/ogg',
- 'wav': 'audio/wav'
-}
-
-__basedir__ = os.path.dirname(os.path.abspath(__file__))
-
-player = Blueprint(
- 'deprecated_player', __name__,
- url_prefix='/play',
- template_folder=os.path.join(__basedir__, 'templates'),
- static_folder=os.path.join(__basedir__, 'static'),
- )
-
-
-class PlayableFile(File):
- parent_class = File
- media_map = {
- 'audio/mpeg': 'mp3',
- 'audio/ogg': 'ogg',
- 'audio/wav': 'wav',
- }
-
- def __init__(self, duration=None, title=None, **kwargs):
- self.duration = duration
- self.title = title
- super(PlayableFile, self).__init__(**kwargs)
-
- @property
- def title(self):
- return self._title or self.name
-
- @title.setter
- def title(self, title):
- self._title = title
-
- @property
- def media_format(self):
- return self.media_map[self.type]
-
-
-@player.route('/audio/')
-def audio(path):
- f = PlayableFile.from_urlpath(path)
- return render_template('audio.player.html', file=f)
-
-
-def detect_playable_mimetype(path, os_sep=os.sep):
- basename = path.rsplit(os_sep)[-1]
- if '.' in basename:
- ext = basename.rsplit('.')[-1]
- return mimetypes.get(ext, None)
- return None
-
-
-def register_plugin(manager):
- """
- Register blueprints and actions using given plugin manager.
-
- :param manager: plugin manager
- :type manager: browsepy.manager.PluginManager
- """
- manager.register_blueprint(player)
- manager.register_mimetype_function(detect_playable_mimetype)
-
- style = manager.style_class(
- 'deprecated_player.static',
- filename='css/browse.css'
- )
- manager.register_widget(style)
-
- button_widget = manager.button_class(css='play')
- link_widget = manager.link_class()
- for widget in (link_widget, button_widget):
- manager.register_action(
- 'deprecated_player.audio',
- widget,
- mimetypes=(
- 'audio/mpeg',
- 'audio/ogg',
- 'audio/wav',
- ))
diff --git a/browsepy/tests/deprecated/test_plugins.py b/browsepy/tests/deprecated/test_plugins.py
deleted file mode 100644
index 492dfd0..0000000
--- a/browsepy/tests/deprecated/test_plugins.py
+++ /dev/null
@@ -1,292 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-
-import unittest
-
-import flask
-import browsepy
-import browsepy.file as browsepy_file
-import browsepy.widget as browsepy_widget
-import browsepy.manager as browsepy_manager
-
-from browsepy.tests.deprecated.plugin import player as player
-
-
-class ManagerMock(object):
- def __init__(self):
- self.blueprints = []
- self.mimetype_functions = []
- self.actions = []
- self.widgets = []
-
- @staticmethod
- def style_class(endpoint, **kwargs):
- return ('style', endpoint, kwargs)
-
- @staticmethod
- def button_class(*args, **kwargs):
- return ('button', args, kwargs)
-
- @staticmethod
- def javascript_class(endpoint, **kwargs):
- return ('javascript', endpoint, kwargs)
-
- @staticmethod
- def link_class(*args, **kwargs):
- return ('link', args, kwargs)
-
- def register_blueprint(self, blueprint):
- self.blueprints.append(blueprint)
-
- def register_mimetype_function(self, fnc):
- self.mimetype_functions.append(fnc)
-
- def register_widget(self, widget):
- self.widgets.append(widget)
-
- def register_action(self, blueprint, widget, mimetypes=(), **kwargs):
- self.actions.append((blueprint, widget, mimetypes, kwargs))
-
-
-class FileMock(object):
- @property
- def type(self):
- return self.mimetype.split(';')[0]
-
- @property
- def category(self):
- return self.mimetype.split('/')[0]
-
- is_directory = False
- name = 'unnamed'
-
- def __init__(self, **kwargs):
- self.__dict__.update(kwargs)
-
-
-class TestPlugins(unittest.TestCase):
- app_module = browsepy
- manager_module = browsepy_manager
-
- def setUp(self):
- self.app = self.app_module.app
- self.manager = self.manager_module.PluginManager(self.app)
- self.original_namespaces = self.app.config['PLUGIN_NAMESPACES']
- self.plugin_namespace, self.plugin_name = __name__.rsplit('.', 1)
- self.app.config['PLUGIN_NAMESPACES'] = (self.plugin_namespace,)
-
- def tearDown(self):
- self.app.config['PLUGIN_NAMESPACES'] = self.original_namespaces
- self.manager.clear()
-
- def test_manager(self):
- self.manager.load_plugin(self.plugin_name)
- self.assertTrue(self.manager._plugin_loaded)
-
- endpoints = sorted(
- action.endpoint
- for action in self.manager.get_actions(FileMock(mimetype='a/a'))
- )
-
- self.assertEqual(
- endpoints,
- sorted(('test_x_x', 'test_a_x', 'test_x_a', 'test_a_a')))
- self.assertEqual(
- self.app.view_functions['old_test_plugin.root'](),
- 'old_test_plugin')
- self.assertIn('old_test_plugin', self.app.blueprints)
-
- self.assertRaises(
- self.manager_module.PluginNotFoundError,
- self.manager.load_plugin,
- 'non_existent_plugin_module'
- )
-
-
-def register_plugin(manager):
- widget_class = browsepy_widget.WidgetBase
-
- manager._plugin_loaded = True
- manager.register_action('test_x_x', widget_class('test_x_x'), ('*/*',))
- manager.register_action('test_a_x', widget_class('test_a_x'), ('a/*',))
- manager.register_action('test_x_a', widget_class('test_x_a'), ('*/a',))
- manager.register_action('test_a_a', widget_class('test_a_a'), ('a/a',))
- manager.register_action('test_b_x', widget_class('test_b_x'), ('b/*',))
-
- test_plugin_blueprint = flask.Blueprint(
- 'old_test_plugin', __name__, url_prefix='/old_test_plugin_blueprint')
- test_plugin_blueprint.add_url_rule(
- '/', endpoint='root', view_func=lambda: 'old_test_plugin')
-
- manager.register_blueprint(test_plugin_blueprint)
-
-
-class TestPlayerBase(unittest.TestCase):
- module = player
- scheme = 'test'
- hostname = 'localhost'
- urlprefix = '%s://%s' % (scheme, hostname)
-
- def assertUrlEqual(self, a, b):
- self.assertIn(a, (b, '%s%s' % (self.urlprefix, b)))
-
- def setUp(self):
- self.app = flask.Flask(self.__class__.__name__)
- self.app.config['DIRECTORY_REMOVE'] = None
- self.app.config['SERVER_NAME'] = self.hostname
- self.app.config['PREFERRED_URL_SCHEME'] = self.scheme
- self.manager = ManagerMock()
-
-
-class TestPlayer(TestPlayerBase):
- def test_register_plugin(self):
- self.module.register_plugin(self.manager)
-
- self.assertIn(self.module.player, self.manager.blueprints)
- self.assertIn(
- self.module.detect_playable_mimetype,
- self.manager.mimetype_functions)
-
- widgets = [action[1] for action in self.manager.widgets]
- self.assertIn('deprecated_player.static', widgets)
-
- widgets = [action[2] for action in self.manager.widgets]
- self.assertIn({'filename': 'css/browse.css'}, widgets)
-
- actions = [action[0] for action in self.manager.actions]
- self.assertIn('deprecated_player.audio', actions)
-
-
-class TestIntegrationBase(TestPlayerBase):
- player_module = player
- browsepy_module = browsepy
- manager_module = browsepy_manager
- widget_module = browsepy_widget
- file_module = browsepy_file
-
-
-class TestIntegration(TestIntegrationBase):
- def test_register_plugin(self):
- self.app.config.update(self.browsepy_module.app.config)
- self.app.config.update(
- SERVER_NAME=self.hostname,
- PREFERRED_URL_SCHEME=self.scheme,
- PLUGIN_NAMESPACES=('browsepy.tests.deprecated.plugin',)
- )
- manager = self.manager_module.PluginManager(self.app)
- manager.load_plugin('player')
- self.assertIn(self.player_module.player, self.app.blueprints.values())
-
- def test_register_action(self):
- manager = self.manager_module.MimetypeActionPluginManager(self.app)
- widget = self.widget_module.WidgetBase() # empty
- manager.register_action('browse', widget, mimetypes=('*/*',))
- actions = manager.get_actions(FileMock(mimetype='text/plain'))
- self.assertEqual(len(actions), 1)
- self.assertEqual(actions[0].widget, widget)
- manager.register_action('browse', widget, mimetypes=('text/*',))
- actions = manager.get_actions(FileMock(mimetype='text/plain'))
- self.assertEqual(len(actions), 2)
- self.assertEqual(actions[1].widget, widget)
- manager.register_action('browse', widget, mimetypes=('text/plain',))
- actions = manager.get_actions(FileMock(mimetype='text/plain'))
- self.assertEqual(len(actions), 3)
- self.assertEqual(actions[2].widget, widget)
- widget = self.widget_module.ButtonWidget()
- manager.register_action('browse', widget, mimetypes=('text/plain',))
- actions = manager.get_actions(FileMock(mimetype='text/plain'))
- self.assertEqual(len(actions), 4)
- self.assertEqual(actions[3].widget, widget)
- widget = self.widget_module.LinkWidget()
- manager.register_action('browse', widget, mimetypes=('*/plain',))
- actions = manager.get_actions(FileMock(mimetype='text/plain'))
- self.assertEqual(len(actions), 5)
- self.assertNotEqual(actions[4].widget, widget)
- widget = self.widget_module.LinkWidget(icon='file', text='something')
- manager.register_action('browse', widget, mimetypes=('*/plain',))
- actions = manager.get_actions(FileMock(mimetype='text/plain'))
- self.assertEqual(len(actions), 6)
- self.assertEqual(actions[5].widget, widget)
-
- def test_register_widget(self):
- file = self.file_module.Node(app=self.app)
- manager = self.manager_module.MimetypeActionPluginManager(self.app)
- widget = self.widget_module.StyleWidget('static', filename='a.css')
- manager.register_widget(widget)
- widgets = manager.get_widgets('style')
- self.assertEqual(len(widgets), 1)
- self.assertIsInstance(widgets[0], self.widget_module.StyleWidget)
- self.assertEqual(widgets[0], widget)
-
- widgets = manager.get_widgets(place='style')
- self.assertEqual(len(widgets), 1)
- self.assertIsInstance(widgets[0], self.widget_module.StyleWidget)
- self.assertEqual(widgets[0], widget)
-
- widgets = manager.get_widgets(file=file, place='styles')
- self.assertEqual(len(widgets), 1)
- self.assertIsInstance(widgets[0], manager.widget_types['stylesheet'])
- self.assertUrlEqual(widgets[0].href, '/static/a.css')
-
- widget = self.widget_module.JavascriptWidget('static', filename='a.js')
- manager.register_widget(widget)
- widgets = manager.get_widgets('javascript')
- self.assertEqual(len(widgets), 1)
- self.assertIsInstance(widgets[0], self.widget_module.JavascriptWidget)
- self.assertEqual(widgets[0], widget)
-
- widgets = manager.get_widgets(place='javascript')
- self.assertEqual(len(widgets), 1)
- self.assertIsInstance(widgets[0], self.widget_module.JavascriptWidget)
- self.assertEqual(widgets[0], widget)
-
- widgets = manager.get_widgets(file=file, place='scripts')
- self.assertEqual(len(widgets), 1)
- self.assertIsInstance(widgets[0], manager.widget_types['script'])
- self.assertUrlEqual(widgets[0].src, '/static/a.js')
-
- def test_for_file(self):
- manager = self.manager_module.MimetypeActionPluginManager(self.app)
- widget = self.widget_module.LinkWidget(icon='asdf', text='something')
- manager.register_action('browse', widget, mimetypes=('*/plain',))
- file = self.file_module.File('asdf.txt', plugin_manager=manager,
- app=self.app)
- self.assertEqual(file.link.icon, 'asdf')
- self.assertEqual(file.link.text, 'something')
-
- widget = self.widget_module.LinkWidget()
- manager.register_action('browse', widget, mimetypes=('*/plain',))
- file = self.file_module.File('asdf.txt', plugin_manager=manager,
- app=self.app)
- self.assertEqual(file.link.text, 'asdf.txt')
-
- def test_from_file(self):
- with self.app.app_context():
- file = self.file_module.File('asdf.txt')
- widget = self.widget_module.LinkWidget.from_file(file)
- self.assertEqual(widget.text, 'asdf.txt')
-
-
-class TestPlayable(TestIntegrationBase):
- module = player
-
- def setUp(self):
- super(TestPlayable, self).setUp()
- self.manager = self.manager_module.MimetypeActionPluginManager(
- self.app)
- self.manager.register_mimetype_function(
- self.player_module.detect_playable_mimetype)
-
- def test_playablefile(self):
- exts = {
- 'mp3': 'mp3',
- 'wav': 'wav',
- 'ogg': 'ogg'
- }
- for ext, media_format in exts.items():
- pf = self.module.PlayableFile(path='asdf.%s' % ext, app=self.app)
- self.assertEqual(pf.media_format, media_format)
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/browsepy/tests/runner.py b/browsepy/tests/runner.py
deleted file mode 100644
index f5801fa..0000000
--- a/browsepy/tests/runner.py
+++ /dev/null
@@ -1,53 +0,0 @@
-
-import os
-import unittest
-
-
-class DebuggerTextTestResult(unittest._TextTestResult): # pragma: no cover
- def __init__(self, stream, descriptions, verbosity, debugger):
- self.debugger = debugger
- self.shouldStop = True
- supa = super(DebuggerTextTestResult, self)
- supa.__init__(stream, descriptions, verbosity)
-
- def addError(self, test, exc_info):
- self.debugger(exc_info)
- super(DebuggerTextTestResult, self).addError(test, exc_info)
-
- def addFailure(self, test, exc_info):
- self.debugger(exc_info)
- super(DebuggerTextTestResult, self).addFailure(test, exc_info)
-
-
-class DebuggerTextTestRunner(unittest.TextTestRunner): # pragma: no cover
- debugger = os.environ.get('UNITTEST_DEBUG', 'none')
- test_result_class = DebuggerTextTestResult
-
- def __init__(self, *args, **kwargs):
- kwargs.setdefault('verbosity', 2)
- super(DebuggerTextTestRunner, self).__init__(*args, **kwargs)
-
- @staticmethod
- def debug_none(exc_info):
- pass
-
- @staticmethod
- def debug_pdb(exc_info):
- import pdb
- pdb.post_mortem(exc_info[2])
-
- @staticmethod
- def debug_ipdb(exc_info):
- import ipdb
- ipdb.post_mortem(exc_info[2])
-
- @staticmethod
- def debug_pudb(exc_info):
- import pudb
- pudb.post_mortem(exc_info[2], exc_info[1], exc_info[0])
-
- def _makeResult(self):
- return self.test_result_class(
- self.stream, self.descriptions, self.verbosity,
- getattr(self, 'debug_%s' % self.debugger, self.debug_none)
- )
diff --git a/browsepy/tests/test_code.py b/browsepy/tests/test_code.py
index 5146df2..4e43de4 100644
--- a/browsepy/tests/test_code.py
+++ b/browsepy/tests/test_code.py
@@ -1,34 +1,42 @@
+import re
import unittest_resources.testing as bases
-class TypingTestCase(bases.TypingTestCase):
- """TestCase checking :module:`mypy`."""
+class Rules:
+ """Browsepy module mixin."""
meta_module = 'browsepy'
+ meta_module_pattern = re.compile(r'^([^t]*|t(?!ests?))+$')
+ meta_resource_pattern = re.compile(r'^([^t]*|t(?!ests?))+\.py$')
-class CodeStyleTestCase(bases.CodeStyleTestCase):
+# class TypingTestCase(Rules, bases.TypingTestCase):
+# """TestCase checking :module:`mypy`."""
+#
+# pass
+
+
+class CodeStyleTestCase(Rules, bases.CodeStyleTestCase):
"""TestCase checking :module:`pycodestyle`."""
- meta_module = 'browsepy'
+ pass
-class DocStyleTestCase(bases.DocStyleTestCase):
+class DocStyleTestCase(Rules, bases.DocStyleTestCase):
"""TestCase checking :module:`pydocstyle`."""
- meta_module = 'browsepy'
+ pass
-class MaintainabilityIndexTestCase(bases.MaintainabilityIndexTestCase):
+class MaintainabilityIndexTestCase(Rules, bases.MaintainabilityIndexTestCase):
"""TestCase checking :module:`radon` maintainability index."""
- meta_module = 'browsepy'
+ pass
-class CodeComplexityTestCase(bases.CodeComplexityTestCase):
+class CodeComplexityTestCase(Rules, bases.CodeComplexityTestCase):
"""TestCase checking :module:`radon` code complexity."""
- meta_module = 'browsepy'
max_class_complexity = 7
max_function_complexity = 7
diff --git a/browsepy/transform/__init__.py b/browsepy/transform/__init__.py
index 409542d..25f46e0 100755
--- a/browsepy/transform/__init__.py
+++ b/browsepy/transform/__init__.py
@@ -1,15 +1,17 @@
+"""Generic string transform module."""
class StateMachine(object):
"""
- Abstract character-driven finite state machine implementation, used to
- chop down and transform strings.
+ Character-driven finite state machine.
- Useful for implementig simple transpilators, compressors and so on.
+ Useful for implementig simple string transforms, transpilators,
+ compressors and so on.
Important: when implementing this class, you must set the :attr:`current`
attribute to a key defined in :attr:`jumps` dict.
"""
+
jumps = {} # finite state machine jumps
start = '' # character which started current state
current = '' # current state (an initial value must be set)
@@ -56,6 +58,8 @@ class StateMachine(object):
def __init__(self, data=''):
"""
+ Initialize.
+
:param data: content will be added to pending data
:type data: str
"""
@@ -91,8 +95,7 @@ class StateMachine(object):
def transform(self, data, mark, next):
"""
- Apply the appropriate transformation function on current state data,
- which is supposed to end at this point.
+ Apply the appropriate transformation method.
It is expected transformation logic makes use of :attr:`start`,
:attr:`current` and :attr:`streaming` instance attributes to
@@ -113,8 +116,7 @@ class StateMachine(object):
def feed(self, data=''):
"""
- Optionally add pending data, switch into streaming mode, and yield
- result chunks.
+ Add input data and yield partial output results.
:yields: result chunks
:ytype: str
@@ -126,8 +128,7 @@ class StateMachine(object):
def finish(self, data=''):
"""
- Optionally add pending data, turn off streaming mode, and yield
- result chunks, which implies all pending data will be consumed.
+ Add input data, end procesing and yield all remaining output.
:yields: result chunks
:ytype: str
diff --git a/browsepy/transform/glob.py b/browsepy/transform/glob.py
index 428fb5c..a81806f 100644
--- a/browsepy/transform/glob.py
+++ b/browsepy/transform/glob.py
@@ -1,3 +1,4 @@
+"""Module providing transpilation from glob to regexp."""
import os
import warnings
@@ -9,6 +10,8 @@ from . import StateMachine
class GlobTransform(StateMachine):
+ """Glob to regexp string transpiler."""
+
jumps = {
'start': {
'': 'text',
@@ -127,6 +130,7 @@ class GlobTransform(StateMachine):
deferred = False
def __init__(self, data, sep=os.sep, base=None):
+ """Initialize."""
self.sep = sep
self.base = base or ''
self.deferred_data = []
@@ -134,6 +138,7 @@ class GlobTransform(StateMachine):
super(GlobTransform, self).__init__(data)
def transform(self, data, mark, next):
+ """Translate data chunk."""
data = super(GlobTransform, self).transform(data, mark, next)
if self.deferred:
self.deferred_data.append(data)
@@ -144,6 +149,7 @@ class GlobTransform(StateMachine):
return data
def transform_posix_collating_symbol(self, data, mark, next):
+ """Translate POSIX collating symbol (stub)."""
warnings.warn(
'Posix collating symbols (like %s%s) are not supported.'
% (data, mark),
@@ -152,6 +158,7 @@ class GlobTransform(StateMachine):
return None
def transform_posix_character_class(self, data, mark, next):
+ """Translate POSIX character class into REGEX ranges."""
name = data[len(self.start):]
if name not in self.character_classes:
warnings.warn(
@@ -167,6 +174,7 @@ class GlobTransform(StateMachine):
)
def transform_posix_equivalence_class(self, data, mark, next):
+ """Translate POSIX equivalence expression (stub)."""
warnings.warn(
'Posix equivalence class expresions (like %s%s) are not supported.'
% (data, mark),
@@ -175,6 +183,7 @@ class GlobTransform(StateMachine):
return None
def transform_wildcard(self, data, mark, next):
+ """Translate glob wildcard into non-path-separator REGEX."""
if self.start == '**':
return '.*'
if self.start == '*':
@@ -182,17 +191,21 @@ class GlobTransform(StateMachine):
return '[^%s]' % re_escape(self.sep)
def transform_text(self, data, mark, next):
+ """Translate glob text into text, filename or directory REGEX."""
if next is None:
return '%s(%s|$)' % (re_escape(data), re_escape(self.sep))
return re_escape(data)
def transform_sep(self, data, mark, next):
+ """Transform path separator into REGEX."""
return re_escape(self.sep)
def transform_literal(self, data, mark, next):
+ """Transform glob escape into REGEX."""
return data[len(self.start):]
def transform_range(self, data, mark, next):
+ """Transform glob character range into REGEX."""
self.deferred = True
if self.start == '[!':
return '[^%s' % data[2:]
@@ -201,9 +214,11 @@ class GlobTransform(StateMachine):
return data
def transform_range_sep(self, data, mark, next):
+ """Translate range path separator into REGEX."""
return re_escape(self.sep)
def transform_range_close(self, data, mark, next):
+ """Translate range end into REGEX."""
self.deferred = False
if None in self.deferred_data:
self.deferred_data[:] = ()
@@ -211,9 +226,11 @@ class GlobTransform(StateMachine):
return data
def transform_range_ignore(self, data, mark, next):
+ """Translate character to be ignored on REGEX."""
return ''
def transform_group(self, data, mark, next):
+ """Transform glob group into REGEX."""
if self.start == '{':
self.deep += 1
return '('
@@ -225,11 +242,13 @@ class GlobTransform(StateMachine):
return data
def transform_start(self, data, mark, next):
+ """Transform glob start into REGEX."""
if mark == '/':
return '^%s' % re_escape(self.base)
return re_escape(self.sep)
def translate(data, sep=os.sep, base=None):
+ """Transform glob string into REGEX."""
self = GlobTransform(data, sep, base)
return ''.join(self)
diff --git a/browsepy/transform/htmlcompress.py b/browsepy/transform/htmlcompress.py
index f737d85..9f72ad6 100755
--- a/browsepy/transform/htmlcompress.py
+++ b/browsepy/transform/htmlcompress.py
@@ -1,3 +1,5 @@
+"""Module providing HTML compression extension jinja2."""
+
import re
import jinja2
@@ -8,6 +10,8 @@ from . import StateMachine
class CompressContext(StateMachine):
+ """Base jinja2 template token finite state machine."""
+
token_class = jinja2.lexer.Token
block_tokens = {
'variable_begin': 'variable_end',
@@ -17,6 +21,7 @@ class CompressContext(StateMachine):
lineno = 0
def feed(self, token):
+ """Process a single token, yielding processed ones."""
if self.skip_until_token:
yield token
if token.type == self.skip_until_token:
@@ -35,11 +40,14 @@ class CompressContext(StateMachine):
self.lineno = token.lineno
def finish(self):
+ """Set state machine as finished, yielding remaining tokens."""
for data in super(CompressContext, self).finish():
yield self.token_class(self.lineno, 'data', data)
class SGMLCompressContext(CompressContext):
+ """Compression context for jinja2 SGML templates."""
+
re_whitespace = re.compile('[ \\t\\r\\n]+')
block_tags = {} # block content will be treated as literal text
jumps = { # state machine jumps
@@ -63,6 +71,7 @@ class SGMLCompressContext(CompressContext):
@property
def nearest(self):
+ """Get next data chunk to be processed."""
if self.skip_until_text and self.current == 'text':
mark = self.skip_until_text
index = self.pending.find(mark, len(self.start))
@@ -72,6 +81,7 @@ class SGMLCompressContext(CompressContext):
return super(SGMLCompressContext, self).nearest
def transform_tag(self, data, mark, next):
+ """Compress SML tag node."""
tagstart = self.start == '<'
data = self.re_whitespace.sub(' ', data[1:] if tagstart else data)
if tagstart:
@@ -84,6 +94,7 @@ class SGMLCompressContext(CompressContext):
return self.start if data.strip() == self.start else data
def transform_text(self, data, mark, next):
+ """Compress SGML text node."""
if not self.skip_until_text:
return self.start if data.strip() == self.start else data
elif next is not None:
@@ -92,6 +103,8 @@ class SGMLCompressContext(CompressContext):
class HTMLCompressContext(SGMLCompressContext):
+ """Compression context for jinja2 HTML templates."""
+
block_tags = {
'textarea': '',
'pre': '',
@@ -101,9 +114,12 @@ class HTMLCompressContext(SGMLCompressContext):
class HTMLCompress(jinja2.ext.Extension):
+ """Jinja2 HTML template compression extension."""
+
context_class = HTMLCompressContext
def filter_stream(self, stream):
+ """Yield compressed tokens from :class:`~jinja2.lexer.TokenStream`."""
transform = self.context_class()
for token in stream:
diff --git a/browsepy/utils.py b/browsepy/utils.py
index f7ffa2b..42b5ff2 100644
--- a/browsepy/utils.py
+++ b/browsepy/utils.py
@@ -1,4 +1,4 @@
-# -*- coding: UTF-8 -*-
+"""Small utility functions for common tasks."""
import sys
import os
@@ -13,7 +13,7 @@ import flask
def ppath(*args, **kwargs):
"""
- Get joined file path relative to module location.
+ Join given path components relative to a module location.
:param module: Module name
:type module: str
@@ -30,10 +30,7 @@ def ppath(*args, **kwargs):
@contextlib.contextmanager
def dummy_context():
- """
- Context manager which does nothing besides exposing the context
- manger interface
- """
+ """Get a dummy context manager."""
yield
@@ -67,8 +64,10 @@ def random_string(size, sample=tuple(map(chr, range(256)))):
def solve_local(context_local):
"""
- Resolve given context local to its actual value. If given object
- it's not a context local nothing happens, just returns the same value.
+ Resolve given context local to its actual value.
+
+ If given object isn't a context local, nothing happens, return the
+ same object.
"""
if callable(getattr(context_local, '_get_current_object', None)):
return context_local._get_current_object()
@@ -105,6 +104,9 @@ def defaultsnamedtuple(name, fields, defaults=None):
"""
Generate namedtuple with default values.
+ This somewhat tries to mimic py3.7 namedtuple with keyword-based
+ defaults, in a backwards-compatible way.
+
:param name: name
:param fields: iterable with field names
:param defaults: iterable or mapping with field defaults
@@ -112,9 +114,12 @@ def defaultsnamedtuple(name, fields, defaults=None):
:rtype: collections.defaultdict
"""
nt = collections.namedtuple(name, fields)
- nt.__new__.__defaults__ = (None, ) * len(nt._fields)
- if isinstance(defaults, collections.Mapping):
- nt.__new__.__defaults__ = tuple(nt(**defaults))
- elif defaults:
- nt.__new__.__defaults__ = tuple(nt(*defaults))
+ nt.__new__.__defaults__ = (None, ) * len(fields)
+ nt.__module__ = __name__
+ if defaults:
+ nt.__new__.__defaults__ = tuple(
+ nt(**defaults)
+ if isinstance(defaults, collections.Mapping) else
+ nt(*defaults)
+ )
return nt
diff --git a/browsepy/widget.py b/browsepy/widget.py
deleted file mode 100644
index bb43f42..0000000
--- a/browsepy/widget.py
+++ /dev/null
@@ -1,89 +0,0 @@
-# -*- coding: UTF-8 -*-
-"""
-WARNING: deprecated module.
-
-API defined in this module has been deprecated in version 0.5 and it will be
-removed at 0.6.
-"""
-
-import warnings
-
-from markupsafe import Markup
-from flask import url_for
-
-from .compat import deprecated
-
-
-warnings.warn('Deprecated module widget', category=DeprecationWarning)
-
-
-class WidgetBase(object):
- _type = 'base'
- place = None
-
- @deprecated('Deprecated widget API')
- def __new__(cls, *args, **kwargs):
- return super(WidgetBase, cls).__new__(cls)
-
- def __init__(self, *args, **kwargs):
- self.args = args
- self.kwargs = kwargs
-
- def for_file(self, file):
- return self
-
- @classmethod
- def from_file(cls, file):
- if not hasattr(cls, '__empty__'):
- cls.__empty__ = cls()
- return cls.__empty__.for_file(file)
-
-
-class LinkWidget(WidgetBase):
- _type = 'link'
- place = 'link'
-
- def __init__(self, text=None, css=None, icon=None):
- self.text = text
- self.css = css
- self.icon = icon
- super(LinkWidget, self).__init__()
-
- def for_file(self, file):
- if None in (self.text, self.icon):
- return self.__class__(
- file.name if self.text is None else self.text,
- self.css,
- self.icon if self.icon is not None else
- 'dir-icon' if file.is_directory else
- 'file-icon',
- )
- return self
-
-
-class ButtonWidget(WidgetBase):
- _type = 'button'
- place = 'button'
-
- def __init__(self, html='', text='', css=''):
- self.content = Markup(html) if html else text
- self.css = css
- super(ButtonWidget, self).__init__()
-
-
-class StyleWidget(WidgetBase):
- _type = 'stylesheet'
- place = 'style'
-
- @property
- def href(self):
- return url_for(*self.args, **self.kwargs)
-
-
-class JavascriptWidget(WidgetBase):
- _type = 'script'
- place = 'javascript'
-
- @property
- def src(self):
- return url_for(*self.args, **self.kwargs)
diff --git a/requirements/development.txt b/requirements/development.txt
index 4c2b38a..1acbc38 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -30,6 +30,9 @@ jedi
sphinx
pycodestyle
pydocstyle
+mypy
+radon
+unittest-resources[testing]
pyyaml
beautifulsoup4
diff --git a/setup.py b/setup.py
index 35441a0..c04000d 100644
--- a/setup.py
+++ b/setup.py
@@ -73,9 +73,13 @@ setup(
tests_require=[
'beautifulsoup4',
'unittest-resources',
+ 'pycodestyle',
+ 'pydocstyle',
+ 'mypy',
+ 'radon',
+ 'unittest-resources[testing]',
],
test_suite='browsepy.tests',
- test_runner='browsepy.tests.runner:DebuggerTextTestRunner',
zip_safe=False,
platforms='any',
)
--
GitLab
From 6954c4bf97d4bd282d90202507f895ecbaccca95 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Sun, 8 Dec 2019 01:55:32 +0000
Subject: [PATCH 149/171] simplify player logic
---
browsepy/manager.py | 13 +-
browsepy/plugin/player/__init__.py | 12 +-
browsepy/plugin/player/playable.py | 289 ++++--------------
browsepy/plugin/player/playlist.py | 102 +++++++
.../plugin/player/templates/audio.player.html | 6 +-
5 files changed, 179 insertions(+), 243 deletions(-)
create mode 100644 browsepy/plugin/player/playlist.py
diff --git a/browsepy/manager.py b/browsepy/manager.py
index c72bbf5..42a5669 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -23,6 +23,7 @@ class PluginManagerBase(object):
"""
Base plugin manager for plugin module loading and Flask extension logic.
"""
+
_pyfile_extensions = ('.py', '.pyc', '.pyd', '.pyo')
@property
@@ -165,12 +166,12 @@ class PluginManagerBase(object):
"""
plugin = plugin.replace('-', '_')
names = [
- name
- for ns in self.namespaces
- for name in (
- '%s%s' % (ns, plugin),
- '%s.%s' % (ns.rstrip('.'), plugin),
+ '%s%s%s' % (
+ namespace,
+ '.' if namespace and namespace[-1] not in '._' else '',
+ plugin,
)
+ for namespace in self.namespaces
]
names = sorted(frozenset(names), key=names.index)
for name in names:
@@ -258,7 +259,7 @@ class ExcludePluginManager(PluginManagerBase):
def register_exclude_function(self, exclude_fnc):
"""
- Register given exclude-function on curren app.
+ Register given exclude-function on current app.
This method is intended to be used on plugin's module-level
:func:`register_plugin` functions.
diff --git a/browsepy/plugin/player/__init__.py b/browsepy/plugin/player/__init__.py
index 5a6a820..ab0cedb 100644
--- a/browsepy/plugin/player/__init__.py
+++ b/browsepy/plugin/player/__init__.py
@@ -5,7 +5,7 @@ from browsepy import get_cookie_browse_sorting, browse_sortkey_reverse
from browsepy.utils import ppath
from browsepy.stream import stream_template
-from .playable import Playable, detect_playable_mimetype
+from .playable import PlayableNode, PlayableDirectory, PlayableFile
player = Blueprint(
@@ -29,7 +29,7 @@ def play(path):
"""
sort_property = get_cookie_browse_sorting(path, 'text')
sort_fnc, sort_reverse = browse_sortkey_reverse(sort_property)
- node = Playable.from_urlpath(path)
+ node = PlayableNode.from_urlpath(path)
if node.is_playable and not node.is_excluded:
return stream_template(
'audio.player.html',
@@ -67,7 +67,7 @@ def register_plugin(manager):
:type manager: browsepy.manager.PluginManager
"""
manager.register_blueprint(player)
- manager.register_mimetype_function(detect_playable_mimetype)
+ manager.register_mimetype_function(PlayableFile.detect_mimetype)
# add style tag
manager.register_widget(
@@ -82,7 +82,7 @@ def register_plugin(manager):
place='entry-link',
type='link',
endpoint='player.play',
- filter=Playable.playable_check,
+ filter=PlayableFile.detect,
)
# register action buttons
@@ -91,7 +91,7 @@ def register_plugin(manager):
css='play',
type='button',
endpoint='player.play',
- filter=Playable.playable_check,
+ filter=PlayableFile.detect,
)
# check argument (see `register_arguments`) before registering
@@ -102,5 +102,5 @@ def register_plugin(manager):
type='button',
endpoint='player.play',
text='Play directory',
- filter=Playable.playable_check,
+ filter=PlayableDirectory.detect,
)
diff --git a/browsepy/plugin/player/playable.py b/browsepy/plugin/player/playable.py
index 93b75a3..9f90b31 100644
--- a/browsepy/plugin/player/playable.py
+++ b/browsepy/plugin/player/playable.py
@@ -1,6 +1,5 @@
# -*- coding: UTF-8 -*-
-
import sys
import codecs
import os.path
@@ -11,56 +10,13 @@ import six.moves
from werkzeug.utils import cached_property
-from browsepy.file import Node, File, Directory, \
- underscore_replace, check_under_base
-
-
-class PLSFileParser(object):
- """
- ConfigParser wrapper accepting fallback on get for convenience.
-
- This wraps instead of inheriting due ConfigParse being classobj on python2.
- """
- NOT_SET = type('NotSetType', (object,), {})
- option_exceptions = (
- six.moves.configparser.NoSectionError,
- six.moves.configparser.NoOptionError,
- ValueError,
- )
- parser_class = getattr(
- six.moves.configparser,
- 'SafeConfigParser',
- six.moves.configparser.ConfigParser
- )
-
- def __init__(self, path):
- """Initialize."""
- with warnings.catch_warnings():
- # We already know about SafeConfigParser deprecation!
- warnings.simplefilter('ignore', category=DeprecationWarning)
- self._parser = self.parser_class()
- self._parser.read(path)
+from browsepy.file import Node, File, Directory
- def getint(self, section, key, fallback=NOT_SET):
- """Get int from pls file, returning fallback if unable."""
- try:
- return self._parser.getint(section, key)
- except self.option_exceptions:
- if fallback is self.NOT_SET:
- raise
- return fallback
- def get(self, section, key, fallback=NOT_SET):
- """Get value from pls file, returning fallback if unable."""
- try:
- return self._parser.get(section, key)
- except self.option_exceptions:
- if fallback is self.NOT_SET:
- raise
- return fallback
+from .playlist import iter_pls_entries, iter_m3u_entries
-class Playable(Node):
+class PlayableNode(Node):
"""Base class for playable nodes."""
playable_list = False
@@ -73,62 +29,69 @@ class Playable(Node):
:returns: True if node is playable, False otherwise
:rtype: bool
"""
- print(self)
- return self.playable_check(self)
-
- def normalize_playable_path(self, path):
- """
- Fixes the path of playable file from a playlist.
-
- :param path: absolute or relative path or uri
- :type path: str
- :returns: absolute path or uri
- :rtype: str or None
- """
- if '://' in path:
- return path
- path = os.path.normpath(path)
- if not os.path.isabs(path):
- return os.path.join(self.parent.path, path)
- drive = os.path.splitdrive(self.path)[0]
- if drive and not os.path.splitdrive(path)[0]:
- path = drive + path
- if check_under_base(path, self.app.config['DIRECTORY_BASE']):
- return path
- return None
-
- def _entries(self):
- return ()
-
- def entries(self, sortkey=None, reverse=None):
- """Get playlist file entries."""
- for node in self._entries():
- if node.is_playable and not node.is_excluded:
- yield node
+ return self.detect(self)
@classmethod
- def playable_check(cls, node):
+ def detect(cls, node):
"""Check if class supports node."""
kls = cls.directory_class if node.is_directory else cls.file_class
- return kls.playable_check(node)
+ return kls.detect(node)
-@Playable.register_file_class
-class PlayableExtension(Playable):
+@PlayableNode.register_file_class
+class PlayableFile(PlayableNode, File):
"""=Generic node for filenames with extension."""
title = None
duration = None
+ playable_extensions = {
+ 'mp3': 'audio/mpeg',
+ 'ogg': 'audio/ogg',
+ 'oga': 'audio/ogg',
+ 'm4a': 'audio/mp4',
+ 'wav': 'audio/wav',
+ 'm3u': 'audio/x-mpegurl',
+ 'm3u8': 'audio/x-mpegurl',
+ 'pls': 'audio/x-scpls',
+ }
+ playable_list_parsers = {
+ 'pls': iter_pls_entries,
+ 'm3u': iter_m3u_entries,
+ 'm3u8': iter_m3u_entries,
+ }
- playable_extensions = {}
- playable_extension_classes = [] # deferred
+ @cached_property
+ def playable_list(self):
+ """Get whether file is a playlist."""
+ return self.extension in self.playable_list_parsers
@cached_property
def mimetype(self):
"""Get mimetype."""
- print(self)
return self.detect_mimetype(self.path)
+ @cached_property
+ def extension(self):
+ """Get filename extension."""
+ return self.detect_extension(self.path)
+
+ def entries(self):
+ """Iter playlist files."""
+ parser = self.playable_list_parsers.get(self.extension)
+ if parser:
+ for options in parser(self):
+ node = self.file_class(**options, app=self.app)
+ if not node.is_excluded:
+ yield node
+
+ @classmethod
+ def detect(cls, node):
+ """Get whether file is playable."""
+ return (
+ not node.is_directory and
+ cls.detect_extension(node.path) in cls.playable_extensions
+ )
+
@classmethod
def detect_extension(cls, path):
for extension in cls.playable_extensions:
@@ -138,163 +101,33 @@ class PlayableExtension(Playable):
@classmethod
def detect_mimetype(cls, path):
- return cls.playable_extensions.get(cls.detect_extension(path))
-
- @classmethod
- def playable_check(cls, node):
- """Get if file is playable."""
- if cls.generic:
- return any(
- kls.playable_check(node)
- for kls in cls.playable_extension_classes
- )
- return cls.detect_extension(node.path) in cls.playable_extensions
-
- @classmethod
- def from_node(cls, node, app=None):
- """Get playable node from given node."""
- if cls.generic:
- for kls in cls.playable_classes:
- playable = kls.from_node(node)
- if playable:
- return playable
- return node
- return kls(node.path, node.app)
-
- @classmethod
- def from_urlpath(cls, path, app=None):
- kls = cls.get_extension_class(path)
- if cls.generic:
- for kls in cls.playable_extension_classes:
- if kls.detect_extension(path):
- return kls.from_urlpath(path, app=app)
- return super(PlayableExtension, cls).from_urlpath(path, app=app)
-
- @classmethod
- def register_playable_extension_class(cls, kls):
- cls.playable_extension_classes.append(kls)
- return kls
-
-
-@PlayableExtension.register_playable_extension_class
-class PlayableAudioFile(PlayableExtension, File):
- """Audio file node."""
-
- playable_extensions = {
- 'mp3': 'audio/mpeg',
- 'ogg': 'audio/ogg',
- 'wav': 'audio/wav',
- }
-
-
-@PlayableExtension.register_playable_extension_class
-class PLSFile(PlayableExtension, File):
- """PLS playlist file node."""
-
- playable_list = True
- playable_extensions = {
- 'pls': 'audio/x-scpls',
- }
-
- ini_parser_class = PLSFileParser
- maxsize = getattr(sys, 'maxint', 0) or getattr(sys, 'maxsize', 0) or 2**32
-
- def _entries(self):
- parser = self.ini_parser_class(self.path)
- maxsize = parser.getint('playlist', 'NumberOfEntries', None)
- range = six.moves.range
- for i in range(1, self.maxsize if maxsize is None else maxsize + 1):
- path = parser.get('playlist', 'File%d' % i, None)
- if not path:
- if maxsize:
- continue
- break
- path = self.normalize_playable_path(path)
- if not path:
- continue
- yield self.playable_class(
- path=path,
- app=self.app,
- duration=parser.getint(
- 'playlist', 'Length%d' % i,
- None
- ),
- title=parser.get(
- 'playlist',
- 'Title%d' % i,
- None
- ),
- )
-
-
-@PlayableExtension.register_playable_extension_class
-class M3UFile(PlayableExtension, File):
- """M3U playlist file node."""
-
- playable_list = True
- playable_extensions = {
- 'm3u': 'audio/x-mpegurl',
- 'm3u8': 'audio/x-mpegurl',
- }
-
- def _iter_lines(self):
- prefix = '#EXTM3U\n'
- encoding = 'utf-8' if self.path.endswith('.m3u8') else 'ascii'
- with codecs.open(
- self.path, 'r',
- encoding=encoding,
- errors=underscore_replace
- ) as f:
- if f.read(len(prefix)) != prefix:
- f.seek(0)
- for line in f:
- line = line.rstrip()
- if line:
- yield line
-
- def _entries(self):
- data = {}
- for line in self._iter_lines():
- if line.startswith('#EXTINF:'):
- duration, title = line.split(',', 1)
- data['duration'] = None if duration == '-1' else int(duration)
- data['title'] = title
- if not line:
- continue
- path = self.normalize_playable_path(line)
- if path:
- yield self.playable_class(path=path, app=self.app, **data)
- data.clear()
+ """Detect mimetype by its extension."""
+ return cls.playable_extensions.get(
+ cls.detect_extension(path),
+ 'application/octet-stream'
+ )
-@Playable.register_directory_class
-class PlayListDirectory(Playable, Directory):
+@PlayableNode.register_directory_class
+class PlayableDirectory(PlayableNode, Directory):
"""Playable directory node."""
playable_list = True
- @cached_property
- def parent(self):
- """Get standard directory node."""
- return self.directory_class(self.path, app=self.app)
-
def entries(self, sortkey=None, reverse=None):
- """Get playable directory entries."""
+ """Iter playable directory playable files."""
for node in self.listdir(sortkey=sortkey, reverse=reverse):
- if (not node.is_directory) and node.is_playable:
+ if not node.is_directory and node.is_playable:
yield node
@classmethod
- def playable_check(cls, node):
+ def detect(cls, node):
"""Detect if given node contains playable files."""
- super_check = super(PlayListDirectory, cls).playable_check
+ file_detect = cls.file_class.detect
return node.is_directory and any(
- super_check(child)
+ file_detect(child)
for child in node._listdir()
if not child.is_directory
)
-def detect_playable_mimetype(path):
- """Detect if path corresponds to a playable file by its extension."""
- return PlayableExtension.detect_mimetype(path)
diff --git a/browsepy/plugin/player/playlist.py b/browsepy/plugin/player/playlist.py
new file mode 100644
index 0000000..03e70c5
--- /dev/null
+++ b/browsepy/plugin/player/playlist.py
@@ -0,0 +1,102 @@
+# -*- coding: UTF-8 -*-
+
+
+import sys
+import codecs
+import os.path
+import warnings
+
+from six.moves import range, configparser
+
+try:
+ from six.moves.configparser import SafeConfigParser as ConfigParser
+except ImportError:
+ from six.moves.configparser import ConfigParser
+
+from browsepy.file import underscore_replace
+
+
+CONFIGPARSER_OPTION_EXCEPTIONS = (
+ configparser.NoSectionError,
+ configparser.NoOptionError,
+ ValueError,
+ )
+
+def normalize_playable_path(path, node):
+ """
+ Fixes the path of playable file from a playlist.
+
+ :param path: absolute or relative path or uri
+ :type path: str
+ :param node: node where path was found
+ :type node: browsepy.file.Node
+ :returns: absolute path or uri
+ :rtype: str or None
+ """
+ if '://' in path:
+ return path
+ path = os.path.normpath(path)
+ if not os.path.isabs(path):
+ return os.path.join(node.parent.path, path)
+ drive = os.path.splitdrive(self.path)[0]
+ if drive and not os.path.splitdrive(path)[0]:
+ path = drive + path
+ if check_under_base(path, node.app.config['DIRECTORY_BASE']):
+ return path
+ return None
+
+def iter_pls_entries(node):
+ """Iter entries on a PLS playlist file node."""
+ with warnings.catch_warnings():
+ # We already know about SafeConfigParser deprecation!
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ parser = ConfigParser()
+ parser.read(node.path)
+ try:
+ maxsize = parser.getint('playlist', 'NumberOfEntries')
+ except CONFIGPARSER_OPTION_EXCEPTIONS:
+ maxsize = sys.maxsize
+ failures = 0
+ for i in six.moves.range(1, maxsize):
+ if failures > 5:
+ break
+ data = {}
+ for prop, field, extract in (
+ ('path', 'File%d', parser.get),
+ ('duration', 'Length%d', parser.getint),
+ ('title', 'Title%d', parser.get),
+ ):
+ try:
+ data[prop] = extract('playlist', field % i)
+ except CONFIGPARSER_OPTION_EXCEPTIONS:
+ pass
+ if data.get('path'):
+ failures = 0
+ yield data
+ continue
+ failures += 1
+
+def iter_m3u_entries(node):
+ """Iter entries on a M3U playlist file node."""
+ data = {}
+ with codecs.open(
+ node.path,
+ 'r',
+ encoding='utf-8' if node.path.endswith('.m3u8') else 'ascii',
+ errors=underscore_replace
+ ) as f:
+ for line in filter(None, map(str.rstrip, f)):
+ if line.startswith('#EXTINF:'):
+ duration, title = line.split(',', 1)
+ data.update(
+ duration=None if duration == '-1' else int(duration),
+ title=title,
+ )
+ elif not line.startswith('#'):
+ path = normalize_playable_path(line, node)
+ if path:
+ data['path'] = path
+ yield dict(data)
+ data.clear()
+
+
diff --git a/browsepy/plugin/player/templates/audio.player.html b/browsepy/plugin/player/templates/audio.player.html
index 1dd98b7..0c5d5a3 100644
--- a/browsepy/plugin/player/templates/audio.player.html
+++ b/browsepy/plugin/player/templates/audio.player.html
@@ -25,14 +25,14 @@
data-player-urls="
{%- for entry in file.entries(sortkey=sort_fnc, reverse=sort_reverse) -%}
{%- if not loop.first -%}|{%- endif -%}
- {{- entry.mimetype -}}|
+ {{- entry.extension -}}|
{{- entry.name -}}|
{{- url_for('open', path=entry.urlpath) -}}
{%- endfor -%}
"
{% else %}
data-player-title="{{ file.name }}"
- data-player-format="{{ file.mimetype }}"
+ data-player-format="{{ file.extension }}"
data-player-url="{{ url_for('open', path=file.urlpath) }}"
{% endif %}
>
@@ -84,7 +84,7 @@
{% endif %}
Update Required
- In order to get this player working either update your browse or your Flash plugin.
+ In order to get this player working either update your browser or your Flash plugin.
--
GitLab
From a6bd2e29adf1917283595c8dfec0eca8e77729d6 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Mon, 9 Dec 2019 21:17:27 +0000
Subject: [PATCH 150/171] continue refactor
---
browsepy/__init__.py | 3 +-
browsepy/plugin/__init__.py | 2 +-
browsepy/plugin/file_actions/__init__.py | 21 ++++--
browsepy/plugin/file_actions/exceptions.py | 31 +++++++--
browsepy/plugin/file_actions/utils.py | 9 +--
browsepy/plugin/player/__init__.py | 10 ++-
browsepy/plugin/player/playable.py | 22 ++-----
browsepy/plugin/player/playlist.py | 75 ++++++++++++----------
browsepy/plugin/player/tests.py | 28 +-------
9 files changed, 106 insertions(+), 95 deletions(-)
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index bf81e8b..2f95d96 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -1,4 +1,4 @@
-# -*- coding: UTF-8 -*-
+"""Browsepy simple file server."""
__version__ = '0.6.0'
@@ -11,7 +11,6 @@ import cookieman
from flask import request, render_template, redirect, \
url_for, send_from_directory, \
session, abort
-from werkzeug.exceptions import NotFound
from .appconfig import Flask
from .manager import PluginManager
diff --git a/browsepy/plugin/__init__.py b/browsepy/plugin/__init__.py
index 8d98fed..348dc20 100644
--- a/browsepy/plugin/__init__.py
+++ b/browsepy/plugin/__init__.py
@@ -1 +1 @@
-# -*- coding: UTF-8 -*-
+"""Browsepy builtin plugin submodule."""
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index f27b8c5..d375c9b 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -1,4 +1,4 @@
-# -*- coding: UTF-8 -*-
+"""Plugin module with filesystem functionality."""
from flask import Blueprint, render_template, request, redirect, url_for, \
session, current_app, g
@@ -37,6 +37,7 @@ re_basename = '^[^ {0}]([^{0}]*[^ {0}])?$'.format(
defaults={'path': ''})
@actions.route('/create/directory/', methods=('GET', 'POST'))
def create_directory(path):
+ """Handle request to create directory."""
try:
directory = Node.from_urlpath(path)
except OutsideDirectoryBase:
@@ -60,6 +61,7 @@ def create_directory(path):
@actions.route('/selection', methods=('GET', 'POST'), defaults={'path': ''})
@actions.route('/selection/', methods=('GET', 'POST'))
def selection(path):
+ """Handle file selection clipboard request."""
sort_property = get_cookie_browse_sorting(path, 'text')
sort_fnc, sort_reverse = browse_sortkey_reverse(sort_property)
@@ -105,6 +107,7 @@ def selection(path):
@actions.route('/clipboard/paste', defaults={'path': ''})
@actions.route('/clipboard/paste/')
def clipboard_paste(path):
+ """Handle clipboard paste-here request."""
try:
directory = Node.from_urlpath(path)
except OutsideDirectoryBase:
@@ -141,13 +144,15 @@ def clipboard_paste(path):
@actions.route('/clipboard/clear', defaults={'path': ''})
@actions.route('/clipboard/clear/')
def clipboard_clear(path):
+ """Handle clear clipboard request."""
session.pop('clipboard:mode', None)
session.pop('clipboard:items', None)
return redirect(url_for('browse', path=path))
@actions.errorhandler(FileActionsException)
-def clipboard_error(e):
+def file_actions_error(e):
+ """Serve informative error page on plugin errors."""
file = Node(e.path) if hasattr(e, 'path') else None
issues = getattr(e, 'issues', ())
@@ -174,6 +179,7 @@ def clipboard_error(e):
def shrink_session(data, last):
+ """Session shrinking logic (only obeys final attempt)."""
if last:
raise InvalidClipboardSizeError(
mode=data.pop('clipboard:mode', None),
@@ -183,21 +189,26 @@ def shrink_session(data, last):
def detect_upload(directory):
+ """Detect if directory node can be used as clipboard target."""
return directory.is_directory and directory.can_upload
def detect_clipboard(directory):
+ """Detect if clipboard is available on given directory node."""
return directory.is_directory and session.get('clipboard:mode')
def detect_selection(directory):
- return directory.is_directory and \
- current_app.config.get('DIRECTORY_UPLOAD')
+ """Detect if file selection is available on given directory node."""
+ return (
+ directory.is_directory and
+ current_app.config.get('DIRECTORY_UPLOAD')
+ )
def excluded_clipboard(path):
"""
- Exclusion function for files in clipboard.
+ Check if given path should be ignored when pasting clipboard.
:param path: path to check
:type path: str
diff --git a/browsepy/plugin/file_actions/exceptions.py b/browsepy/plugin/file_actions/exceptions.py
index d2f0db4..ac53f7c 100644
--- a/browsepy/plugin/file_actions/exceptions.py
+++ b/browsepy/plugin/file_actions/exceptions.py
@@ -1,4 +1,4 @@
-# -*- coding: UTF-8 -*-
+"""Exception classes for file action errors."""
import os
import errno
@@ -6,14 +6,16 @@ import errno
class FileActionsException(Exception):
"""
- Base class for file-actions exceptions
+ Base class for file-actions exceptions.
:property path: item path which raised this Exception
"""
+
code = None
template = 'Unhandled error.'
def __init__(self, message=None, path=None):
+ """Initialize."""
self.path = path
message = self.template.format(self) if message is None else message
super(FileActionsException, self).__init__(message)
@@ -26,10 +28,12 @@ class InvalidDirnameError(FileActionsException):
:property path: item path which raised this Exception
:property name: name which raised this Exception
"""
+
code = 'directory-invalid-name'
template = 'Clipboard item {0.name!r} is not valid.'
def __init__(self, message=None, path=None, name=None):
+ """Initialize."""
self.name = name
super(InvalidDirnameError, self).__init__(message, path)
@@ -41,19 +45,23 @@ class DirectoryCreationError(FileActionsException):
:property path: item path which raised this Exception
:property name: name which raised this Exception
"""
+
code = 'directory-mkdir-error'
template = 'Clipboard item {0.name!r} is not valid.'
def __init__(self, message=None, path=None, name=None):
+ """Initialize."""
self.name = name
super(DirectoryCreationError, self).__init__(message, path)
@property
def message(self):
+ """Get error message."""
return self.args[0]
@classmethod
def from_exception(cls, exception, *args, **kwargs):
+ """Generate exception from another."""
message = None
if isinstance(exception, OSError):
message = '%s (%s)' % (
@@ -71,29 +79,33 @@ class ClipboardException(FileActionsException):
:property mode: mode which raised this Exception
:property clipboard: :class Clipboard: instance
"""
+
code = 'clipboard-invalid'
template = 'Clipboard is invalid.'
def __init__(self, message=None, path=None, mode=None, clipboard=None):
+ """Initialize."""
self.mode = mode
self.clipboard = clipboard
super(ClipboardException, self).__init__(message, path)
class ItemIssue(tuple):
- """
- Item/error issue
- """
+ """Item/error issue pair."""
+
@property
def item(self):
+ """Get item."""
return self[0]
@property
def error(self):
+ """Get error."""
return self[1]
@property
def message(self):
+ """Get file error message."""
if isinstance(self.error, OSError):
return '%s (%s)' % (
os.strerror(self.error.errno),
@@ -115,17 +127,20 @@ class InvalidClipboardItemsError(ClipboardException):
:property clipboard: :class Clipboard: instance
:property issues: iterable of issues
"""
+
pair_class = ItemIssue
code = 'clipboard-invalid-items'
template = 'Clipboard has invalid items.'
def __init__(self, message=None, path=None, mode=None, clipboard=None,
issues=()):
+ """Initialize."""
self.issues = list(map(self.pair_class, issues))
supa = super(InvalidClipboardItemsError, self)
supa.__init__(message, path, mode, clipboard)
def append(self, item, error):
+ """Add item/error pair to issue list."""
self.issues.append(self.pair_class((item, error)))
@@ -137,10 +152,12 @@ class InvalidClipboardModeError(ClipboardException):
:property mode: mode which raised this Exception
:property clipboard: :class Clipboard: instance
"""
+
code = 'clipboard-invalid-mode'
template = 'Clipboard mode {0.mode!r} is not valid.'
def __init__(self, message=None, path=None, mode=None, clipboard=None):
+ """Initialize."""
supa = super(InvalidClipboardModeError, self)
supa.__init__(message, path, mode, clipboard)
@@ -153,10 +170,12 @@ class InvalidEmptyClipboardError(ClipboardException):
:property mode: mode which raised this Exception
:property clipboard: :class Clipboard: instance
"""
+
code = 'clipboard-invalid-empty'
template = 'Clipboard action {0.mode!r} cannot be performed without items.'
def __init__(self, message=None, path=None, mode=None, clipboard=None):
+ """Initialize."""
supa = super(InvalidEmptyClipboardError, self)
supa.__init__(message, path, mode, clipboard)
@@ -169,9 +188,11 @@ class InvalidClipboardSizeError(ClipboardException):
:property mode: mode which raised this Exception
:property clipboard: :class Clipboard: instance
"""
+
code = 'clipboard-invalid-size'
template = 'Clipboard evicted due session size limit.'
def __init__(self, message=None, path=None, mode=None, clipboard=None):
+ """Initialize."""
supa = super(InvalidClipboardSizeError, self)
supa.__init__(message, path, mode, clipboard)
diff --git a/browsepy/plugin/file_actions/utils.py b/browsepy/plugin/file_actions/utils.py
index 2979c64..c36a74a 100644
--- a/browsepy/plugin/file_actions/utils.py
+++ b/browsepy/plugin/file_actions/utils.py
@@ -1,4 +1,4 @@
-# -*- coding: UTF-8 -*-
+"""Filesystem utility functions."""
import os
import os.path
@@ -15,6 +15,7 @@ from .exceptions import InvalidClipboardModeError, \
def copy(target, node, join_fnc=os.path.join):
+ """Copy node into target path."""
if node.is_excluded:
raise OSError(2, os.strerror(2))
@@ -32,6 +33,7 @@ def copy(target, node, join_fnc=os.path.join):
def move(target, node, join_fnc=os.path.join):
+ """Move node into target path."""
if node.is_excluded or not node.can_remove:
code = 2 if node.is_excluded else 1
raise OSError(code, os.strerror(code))
@@ -46,9 +48,7 @@ def move(target, node, join_fnc=os.path.join):
def paste(target, mode, clipboard):
- """
- Get pasting function for given directory and keyboard.
- """
+ """Get pasting function for given directory and keyboard."""
if mode == 'cut':
paste_fnc = functools.partial(move, target)
elif mode == 'copy':
@@ -78,6 +78,7 @@ def paste(target, mode, clipboard):
def mkdir(path, name):
+ """Create directory under path with given nam."""
if secure_filename(name) != name or not name:
raise InvalidDirnameError(path=path, name=name)
diff --git a/browsepy/plugin/player/__init__.py b/browsepy/plugin/player/__init__.py
index ab0cedb..7f00f11 100644
--- a/browsepy/plugin/player/__init__.py
+++ b/browsepy/plugin/player/__init__.py
@@ -1,5 +1,13 @@
-# -*- coding: UTF-8 -*-
+"""
+Player plugin.
+This plugin is both functional and simple enough to server as example for:
+- Blueprint registration.
+- Widget with filtering function.
+- Custom file nodes.
+- Sorting cookie retrieval and usage.
+
+"""
from flask import Blueprint, abort
from browsepy import get_cookie_browse_sorting, browse_sortkey_reverse
from browsepy.utils import ppath
diff --git a/browsepy/plugin/player/playable.py b/browsepy/plugin/player/playable.py
index 9f90b31..4fd9418 100644
--- a/browsepy/plugin/player/playable.py
+++ b/browsepy/plugin/player/playable.py
@@ -1,12 +1,4 @@
-# -*- coding: UTF-8 -*-
-
-import sys
-import codecs
-import os.path
-import warnings
-
-import six
-import six.moves
+"""Playable file classes."""
from werkzeug.utils import cached_property
@@ -40,7 +32,7 @@ class PlayableNode(Node):
@PlayableNode.register_file_class
class PlayableFile(PlayableNode, File):
- """=Generic node for filenames with extension."""
+ """Generic node for filenames with extension."""
title = None
duration = None
@@ -76,7 +68,7 @@ class PlayableFile(PlayableNode, File):
return self.detect_extension(self.path)
def entries(self):
- """Iter playlist files."""
+ """Iterate playlist files."""
parser = self.playable_list_parsers.get(self.extension)
if parser:
for options in parser(self):
@@ -94,6 +86,7 @@ class PlayableFile(PlayableNode, File):
@classmethod
def detect_extension(cls, path):
+ """Detect extension from given path."""
for extension in cls.playable_extensions:
if path.endswith('.%s' % extension):
return extension
@@ -115,7 +108,7 @@ class PlayableDirectory(PlayableNode, Directory):
playable_list = True
def entries(self, sortkey=None, reverse=None):
- """Iter playable directory playable files."""
+ """Iterate playable directory playable files."""
for node in self.listdir(sortkey=sortkey, reverse=reverse):
if not node.is_directory and node.is_playable:
yield node
@@ -123,11 +116,8 @@ class PlayableDirectory(PlayableNode, Directory):
@classmethod
def detect(cls, node):
"""Detect if given node contains playable files."""
- file_detect = cls.file_class.detect
return node.is_directory and any(
- file_detect(child)
+ child.is_playable
for child in node._listdir()
if not child.is_directory
)
-
-
diff --git a/browsepy/plugin/player/playlist.py b/browsepy/plugin/player/playlist.py
index 03e70c5..e359b49 100644
--- a/browsepy/plugin/player/playlist.py
+++ b/browsepy/plugin/player/playlist.py
@@ -1,30 +1,30 @@
-# -*- coding: UTF-8 -*-
-
+"""Utility functions for playlist files."""
import sys
import codecs
import os.path
import warnings
-from six.moves import range, configparser
-
-try:
- from six.moves.configparser import SafeConfigParser as ConfigParser
-except ImportError:
- from six.moves.configparser import ConfigParser
+from six.moves import configparser, range
-from browsepy.file import underscore_replace
+from browsepy.file import underscore_replace, check_under_base
-CONFIGPARSER_OPTION_EXCEPTIONS = (
+configparser_class = getattr(
+ configparser,
+ 'SafeConfigParser',
+ configparser.ConfigParser
+ )
+configparser_option_exceptions = (
configparser.NoSectionError,
configparser.NoOptionError,
ValueError,
)
+
def normalize_playable_path(path, node):
"""
- Fixes the path of playable file from a playlist.
+ Fix path of playable file from a playlist.
:param path: absolute or relative path or uri
:type path: str
@@ -38,53 +38,61 @@ def normalize_playable_path(path, node):
path = os.path.normpath(path)
if not os.path.isabs(path):
return os.path.join(node.parent.path, path)
- drive = os.path.splitdrive(self.path)[0]
+ drive = os.path.splitdrive(path)[0]
if drive and not os.path.splitdrive(path)[0]:
path = drive + path
if check_under_base(path, node.app.config['DIRECTORY_BASE']):
return path
return None
+
+def iter_pls_fields(parser, index):
+ """Iterate pls entry fields from parser and entry index."""
+ for prop, get_value, entry in (
+ ('path', parser.get, 'File%d' % index),
+ ('duration', parser.getint, 'Length%d' % index),
+ ('title', parser.get, 'Title%d' % index),
+ ):
+ try:
+ yield prop, get_value('playlist', entry)
+ except configparser_option_exceptions:
+ pass
+
+
def iter_pls_entries(node):
- """Iter entries on a PLS playlist file node."""
+ """Iterate entries on a PLS playlist file node."""
with warnings.catch_warnings():
# We already know about SafeConfigParser deprecation!
warnings.simplefilter('ignore', category=DeprecationWarning)
- parser = ConfigParser()
+ parser = configparser_class()
parser.read(node.path)
try:
maxsize = parser.getint('playlist', 'NumberOfEntries')
- except CONFIGPARSER_OPTION_EXCEPTIONS:
+ except configparser_option_exceptions:
maxsize = sys.maxsize
failures = 0
- for i in six.moves.range(1, maxsize):
+ for i in range(1, maxsize):
if failures > 5:
break
- data = {}
- for prop, field, extract in (
- ('path', 'File%d', parser.get),
- ('duration', 'Length%d', parser.getint),
- ('title', 'Title%d', parser.get),
- ):
- try:
- data[prop] = extract('playlist', field % i)
- except CONFIGPARSER_OPTION_EXCEPTIONS:
- pass
- if data.get('path'):
+ data = dict(iter_pls_fields(parser, i))
+ if not data.get('path'):
+ failures += 1
+ continue
+ data['path'] = normalize_playable_path(data['path'])
+ if data['path']:
failures = 0
yield data
- continue
- failures += 1
+
def iter_m3u_entries(node):
- """Iter entries on a M3U playlist file node."""
- data = {}
+ """Iterate entries on a M3U playlist file node."""
with codecs.open(
node.path,
'r',
encoding='utf-8' if node.path.endswith('.m3u8') else 'ascii',
errors=underscore_replace
) as f:
+ data = {}
for line in filter(None, map(str.rstrip, f)):
if line.startswith('#EXTINF:'):
duration, title = line.split(',', 1)
@@ -95,8 +103,5 @@ def iter_m3u_entries(node):
elif not line.startswith('#'):
path = normalize_playable_path(line, node)
if path:
- data['path'] = path
- yield dict(data)
+ yield dict(path=path, **data)
data.clear()
-
-
diff --git a/browsepy/plugin/player/tests.py b/browsepy/plugin/player/tests.py
index e358337..26d2a51 100644
--- a/browsepy/plugin/player/tests.py
+++ b/browsepy/plugin/player/tests.py
@@ -44,30 +44,6 @@ class ManagerMock(object):
return self.argument_values.get(name, default)
-class TestPLSFileParser(unittest.TestCase):
- module = player_playable
- exceptions = player_playable.PLSFileParser.option_exceptions
-
- def get_parser(self, content=''):
- with compat.mkdtemp() as path:
- name = os.path.join(path, 'file.pls')
- with open(name, 'w') as f:
- f.write(content)
- return self.module.PLSFileParser(name)
-
- def test_getint(self):
- parser = self.get_parser()
- self.assertEqual(parser.getint('a', 'a', 2), 2)
- with self.assertRaises(self.exceptions):
- parser.getint('a', 'a')
-
- def test_get(self):
- parser = self.get_parser()
- self.assertEqual(parser.get('a', 'a', 2), 2)
- with self.assertRaises(self.exceptions):
- parser.get('a', 'a')
-
-
class TestPlayerBase(unittest.TestCase):
module = player
@@ -263,7 +239,7 @@ class TestPlayable(TestIntegrationBase):
file = p(tmpdir, 'playable.m3u')
with open(file, 'w') as f:
f.write(data)
- playlist = self.module.M3UFile(path=file, app=self.app)
+ playlist = self.module.PlayableFile(path=file, app=self.app)
self.assertPathListEqual(
[a.path for a in playlist.entries()],
[p(self.base, 'valid.mp3'), p(tmpdir, 'relative.ogg')]
@@ -281,7 +257,7 @@ class TestPlayable(TestIntegrationBase):
file = p(tmpdir, 'playable.pls')
with open(file, 'w') as f:
f.write(data)
- playlist = self.module.PLSFile(path=file, app=self.app)
+ playlist = self.module.PlayableFile(path=file, app=self.app)
self.assertPathListEqual(
[a.path for a in playlist.entries()],
[p(self.base, 'valid.mp3'), p(tmpdir, 'relative.ogg')]
--
GitLab
From 97460e11a1efe086b8e8e424836c94bf19180de0 Mon Sep 17 00:00:00 2001
From: ergoithz
Date: Thu, 12 Dec 2019 16:57:43 +0000
Subject: [PATCH 151/171] more cleanup and fixes
---
browsepy/__init__.py | 54 +++++++++++---
browsepy/compat.py | 31 +++++++-
browsepy/http.py | 29 ++++----
browsepy/manager.py | 55 +++++++--------
browsepy/plugin/file_actions/__init__.py | 5 +-
browsepy/plugin/player/__init__.py | 5 +-
browsepy/static/icon/android-icon-144x144.png | Bin 0 -> 10616 bytes
browsepy/static/icon/android-icon-192x192.png | Bin 0 -> 13594 bytes
browsepy/static/icon/android-icon-36x36.png | Bin 0 -> 2228 bytes
browsepy/static/icon/android-icon-48x48.png | Bin 0 -> 2915 bytes
browsepy/static/icon/android-icon-72x72.png | Bin 0 -> 4423 bytes
browsepy/static/icon/android-icon-96x96.png | Bin 0 -> 6313 bytes
browsepy/static/icon/apple-icon-114x114.png | Bin 0 -> 7865 bytes
browsepy/static/icon/apple-icon-120x120.png | Bin 0 -> 8352 bytes
browsepy/static/icon/apple-icon-144x144.png | Bin 0 -> 10616 bytes
browsepy/static/icon/apple-icon-152x152.png | Bin 0 -> 11226 bytes
browsepy/static/icon/apple-icon-180x180.png | Bin 0 -> 14039 bytes
browsepy/static/icon/apple-icon-57x57.png | Bin 0 -> 3432 bytes
browsepy/static/icon/apple-icon-60x60.png | Bin 0 -> 3594 bytes
browsepy/static/icon/apple-icon-72x72.png | Bin 0 -> 4423 bytes
browsepy/static/icon/apple-icon-76x76.png | Bin 0 -> 4758 bytes
.../static/icon/apple-icon-precomposed.png | Bin 0 -> 14168 bytes
browsepy/static/icon/apple-icon.png | Bin 0 -> 14168 bytes
browsepy/static/icon/favicon-16x16.png | Bin 0 -> 1340 bytes
browsepy/static/icon/favicon-32x32.png | Bin 0 -> 1964 bytes
browsepy/static/icon/favicon-96x96.png | Bin 0 -> 6313 bytes
browsepy/static/icon/favicon.ico | Bin 0 -> 1150 bytes
browsepy/static/icon/ms-icon-144x144.png | Bin 0 -> 10616 bytes
browsepy/static/icon/ms-icon-150x150.png | Bin 0 -> 11052 bytes
browsepy/static/icon/ms-icon-310x310.png | Bin 0 -> 27734 bytes
browsepy/static/icon/ms-icon-70x70.png | Bin 0 -> 4284 bytes
browsepy/stream.py | 13 +++-
browsepy/templates/base.html | 21 +++++-
browsepy/templates/browserconfig.xml | 11 +++
browsepy/templates/manifest.json | 41 +++++++++++
browsepy/tests/test_utils.py | 66 +-----------------
browsepy/utils.py | 57 ---------------
doc/conf.py | 2 +-
requirements/doc.txt | 1 +
setup.cfg | 3 +
40 files changed, 211 insertions(+), 183 deletions(-)
create mode 100644 browsepy/static/icon/android-icon-144x144.png
create mode 100644 browsepy/static/icon/android-icon-192x192.png
create mode 100644 browsepy/static/icon/android-icon-36x36.png
create mode 100644 browsepy/static/icon/android-icon-48x48.png
create mode 100644 browsepy/static/icon/android-icon-72x72.png
create mode 100644 browsepy/static/icon/android-icon-96x96.png
create mode 100644 browsepy/static/icon/apple-icon-114x114.png
create mode 100644 browsepy/static/icon/apple-icon-120x120.png
create mode 100644 browsepy/static/icon/apple-icon-144x144.png
create mode 100644 browsepy/static/icon/apple-icon-152x152.png
create mode 100644 browsepy/static/icon/apple-icon-180x180.png
create mode 100644 browsepy/static/icon/apple-icon-57x57.png
create mode 100644 browsepy/static/icon/apple-icon-60x60.png
create mode 100644 browsepy/static/icon/apple-icon-72x72.png
create mode 100644 browsepy/static/icon/apple-icon-76x76.png
create mode 100644 browsepy/static/icon/apple-icon-precomposed.png
create mode 100644 browsepy/static/icon/apple-icon.png
create mode 100644 browsepy/static/icon/favicon-16x16.png
create mode 100644 browsepy/static/icon/favicon-32x32.png
create mode 100644 browsepy/static/icon/favicon-96x96.png
create mode 100644 browsepy/static/icon/favicon.ico
create mode 100644 browsepy/static/icon/ms-icon-144x144.png
create mode 100644 browsepy/static/icon/ms-icon-150x150.png
create mode 100644 browsepy/static/icon/ms-icon-310x310.png
create mode 100644 browsepy/static/icon/ms-icon-70x70.png
create mode 100644 browsepy/templates/browserconfig.xml
create mode 100644 browsepy/templates/manifest.json
diff --git a/browsepy/__init__.py b/browsepy/__init__.py
index 2f95d96..737cfaf 100644
--- a/browsepy/__init__.py
+++ b/browsepy/__init__.py
@@ -5,40 +5,42 @@ __version__ = '0.6.0'
import logging
import os
import os.path
+import time
import cookieman
-from flask import request, render_template, redirect, \
+from flask import request, render_template, jsonify, redirect, \
url_for, send_from_directory, \
session, abort
from .appconfig import Flask
from .manager import PluginManager
from .file import Node, secure_filename
-from .stream import stream_template
+from .stream import tarfile_extension, stream_template
from .http import etag
from .exceptions import OutsideRemovableBase, OutsideDirectoryBase, \
InvalidFilenameError, InvalidPathError
+
from . import compat
-from . import utils
logger = logging.getLogger(__name__)
app = Flask(
__name__,
- static_url_path='/static',
- static_folder=utils.ppath('static'),
- template_folder=utils.ppath('templates'),
+ template_folder='templates',
+ static_folder='static',
)
app.config.update(
- SECRET_KEY=utils.random_string(4096),
+ SECRET_KEY=os.urandom(4096),
APPLICATION_NAME='browsepy',
+ APPLICATION_TIME=None,
DIRECTORY_BASE=compat.getcwd(),
DIRECTORY_START=None,
DIRECTORY_REMOVE=None,
DIRECTORY_UPLOAD=None,
DIRECTORY_TAR_BUFFSIZE=262144,
DIRECTORY_TAR_COMPRESSION='gzip',
+ DIRECTORY_TAR_EXTENSION=None,
DIRECTORY_TAR_COMPRESSLEVEL=1,
DIRECTORY_DOWNLOADABLE=True,
USE_BINARY_MULTIPLES=True,
@@ -59,6 +61,22 @@ if 'BROWSEPY_SETTINGS' in os.environ:
plugin_manager = PluginManager(app)
+@app.before_first_request
+def prepare():
+ config = app.config
+ if config['APPLICATION_TIME'] is None:
+ config['APPLICATION_TIME'] = time.time()
+
+
+@app.url_defaults
+def default_download_extension(endpoint, values):
+ if endpoint == 'download_directory':
+ values.setdefault(
+ 'extension',
+ tarfile_extension(app.config['DIRECTORY_TAR_EXTENSION']),
+ )
+
+
@app.session_interface.register('browse:sort')
def shrink_browse_sort(data, last):
"""Session `browse:short` size reduction logic."""
@@ -158,9 +176,12 @@ def browse(path):
file=directory,
sort_property=sort_property,
sort_fnc=sort_fnc,
- sort_reverse=sort_reverse
+ sort_reverse=sort_reverse,
+ )
+ response.last_modified = max(
+ directory.content_mtime,
+ app.config['APPLICATION_TIME'],
)
- response.last_modified = directory.content_mtime
response.set_etag(
etag(
content_mtime=directory.content_mtime,
@@ -188,8 +209,11 @@ def download_file(path):
abort(404)
-@app.route('/download/directory/.tgz')
-def download_directory(path):
+@app.route('/download/directory.', defaults={'path': ''})
+@app.route('/download/directory/?.')
+def download_directory(path, extension):
+ if extension != tarfile_extension(app.config['DIRECTORY_TAR_COMPRESSION']):
+ abort(404)
directory = Node.from_urlpath(path)
if directory.is_directory and not directory.is_excluded:
return directory.download()
@@ -234,6 +258,14 @@ def upload(path):
abort(404)
+@app.route('/')
+def metadata(filename):
+ response = app.response_class(render_template(filename))
+ response.last_modified = app.config['APPLICATION_TIME']
+ response.make_conditional(request)
+ return response
+
+
@app.route('/')
def index():
path = app.config['DIRECTORY_START'] or app.config['DIRECTORY_BASE']
diff --git a/browsepy/compat.py b/browsepy/compat.py
index 049727c..d75b0b0 100644
--- a/browsepy/compat.py
+++ b/browsepy/compat.py
@@ -1,4 +1,4 @@
-# -*- coding: UTF-8 -*-
+"""Module providing runtime and platform compatibility workarounds."""
import os
import os.path
@@ -15,6 +15,7 @@ import warnings
import posixpath
import ntpath
import argparse
+import types
try:
import builtins # python 3+
@@ -46,6 +47,16 @@ try:
except ImportError:
from collections import Iterator as BaseIterator # noqa
+try:
+ from importlib import import_module as _import_module # python 3.3+
+except ImportError:
+ _import_module = None # noqa
+
+try:
+ import typing # python 3.5+
+except ImportError:
+ typing = None # noqa
+
FS_ENCODING = sys.getfilesystemencoding()
PY_LEGACY = sys.version_info < (3, )
@@ -112,6 +123,7 @@ class HelpFormatter(argparse.RawTextHelpFormatter):
@contextlib.contextmanager
def scandir(path):
+ # type: (str) -> typing.Generator[typing.Iterator[os.DirEntry], None, None]
"""
Get iterable of :class:`os.DirEntry` as context manager.
@@ -130,7 +142,23 @@ def scandir(path):
files.close()
+def import_module(name):
+ # type: (str) -> types.ModuleType
+ """
+ Import a module by absolute name.
+
+ The 'package' argument is required when performing a relative import. It
+ specifies the package to use as the anchor point from which to resolve the
+ relative import to an absolute import.
+ """
+ if _import_module:
+ return _import_module(name)
+ __import__(name)
+ return sys.modules[name]
+
+
def _unsafe_rmtree(path):
+ # type: (str) -> None
"""
Remove directory tree, without error handling.
@@ -152,6 +180,7 @@ def _unsafe_rmtree(path):
def rmtree(path):
+ # type: (str) -> None
"""
Remove directory tree, with platform-specific fixes.
diff --git a/browsepy/http.py b/browsepy/http.py
index ec81a9c..875e351 100644
--- a/browsepy/http.py
+++ b/browsepy/http.py
@@ -1,4 +1,4 @@
-# -*- coding: UTF-8 -*-
+"""HTTP utility module."""
import re
import logging
@@ -8,32 +8,34 @@ import msgpack
from werkzeug.http import dump_header, dump_options_header, generate_etag
from werkzeug.datastructures import Headers as BaseHeaders
+from .compat import typing
+
logger = logging.getLogger(__name__)
class Headers(BaseHeaders):
"""
- A wrapper around :class:`werkzeug.datastructures.Headers`, allowing
- to specify headers with options on initialization.
+ Covenience :class:`werkzeug.datastructures.Headers` wrapper.
- Headers are provided as keyword arguments while values can be either
- :type:`str` (no options) or tuple of :type:`str` and :type:`dict`.
+ This datastructure allows specifying initial values, as keyword
+ arguments while values can be either :type:`str` (no options)
+ or tuple of :type:`str` and :type:`dict`.
"""
+
snake_replace = staticmethod(re.compile(r'(^|-)[a-z]').sub)
@classmethod
- def genpair(cls, key, value):
+ def genpair(cls,
+ key, # type: str
+ value, # type: typing.Union[str, typing.Mapping]
+ ): # type: (...) -> typing.Tuple[str, str]
"""
- Extract value and options from values dict based on given key and
- options-key.
+ Fix header name and options to be passed to werkzeug.
:param key: value key
- :type key: str
:param value: value or value/options pair
- :type value: str or pair of (str, dict)
:returns: tuple with key and value
- :rtype: tuple of (str, str)
"""
rkey = cls.snake_replace(
lambda x: x.group(0).upper(),
@@ -47,9 +49,11 @@ class Headers(BaseHeaders):
return rkey, rvalue
def __init__(self, **kwargs):
+ # type: (**typing.Union[str, typing.Mapping]) -> None
"""
+ Initialize.
+
:param **kwargs: header and values as keyword arguments
- :type **kwargs: str or (str, dict)
"""
items = [
self.genpair(key, value)
@@ -59,5 +63,6 @@ class Headers(BaseHeaders):
def etag(*args, **kwargs):
+ # type: (*typing.Any, **typing.Any) -> str
"""Generate etag identifier from given parameters."""
return generate_etag(msgpack.dumps((args, kwargs)))
diff --git a/browsepy/manager.py b/browsepy/manager.py
index 42a5669..22c3c9b 100644
--- a/browsepy/manager.py
+++ b/browsepy/manager.py
@@ -12,24 +12,23 @@ from cookieman import CookieMan
from . import mimetype
from . import compat
from . import file
-from . import utils
+from .compat import typing, types
from .utils import defaultsnamedtuple
from .exceptions import PluginNotFoundError, InvalidArgumentError, \
WidgetParameterException
class PluginManagerBase(object):
- """
- Base plugin manager for plugin module loading and Flask extension logic.
- """
+ """Base plugin manager with loading and Flask extension logic."""
_pyfile_extensions = ('.py', '.pyc', '.pyd', '.pyo')
@property
def namespaces(self):
+ # type: () -> typing.Iterable[str]
"""
- List of plugin namespaces taken from app config.
+ List plugin namespaces taken from app config.
:returns: list of plugin namespaces
:rtype: typing.List[str]
@@ -37,9 +36,11 @@ class PluginManagerBase(object):
return self.app.config.get('PLUGIN_NAMESPACES', []) if self.app else []
def __init__(self, app=None):
+ # type app: typing.Optional[flask.Flask]
"""
+ Initialize.
+
:param app: flask application
- :type app: flask.Flask
"""
self.plugin_filters = []
@@ -49,11 +50,11 @@ class PluginManagerBase(object):
self.init_app(app)
def init_app(self, app):
+ # type app: flask.Flask
"""
Initialize this Flask extension for given app.
:param app: flask application
- :type app: flask.Flask
"""
self.app = app
if not hasattr(app, 'extensions'):
@@ -74,23 +75,18 @@ class PluginManagerBase(object):
self.load_plugin(plugin)
def clear(self):
- """
- Clear plugin manager state.
- """
+ """Clear plugin manager state."""
pass
def _iter_modules(self, prefix):
- """
- Iterate thru all root modules containing given prefix.
- """
+ """Iterate thru all root modules containing given prefix."""
for finder, name, ispkg in pkgutil.iter_modules():
if name.startswith(prefix):
yield name
def _content_import_name(self, module, item, prefix):
- """
- Get importable module contnt import name..
- """
+ # type: (str, str, str) -> typing.Optional[str]
+ """Get importable module contnt import name."""
res = compat.res
name = '%s.%s' % (module, item)
if name.startswith(prefix):
@@ -99,11 +95,11 @@ class PluginManagerBase(object):
return name[:-len(ext)]
if not res.is_resource(module, item):
return name
+ return None
def _iter_submodules(self, prefix):
- """
- Iterate thru all modules which full name contains given prefix.
- """
+ # type: (str) -> typing.Generator[str, None, None]
+ """Iterate thru all modules with an absolute prefix."""
res = compat.res
parent = prefix.rsplit('.', 1)[0]
for base in (prefix, parent):
@@ -115,13 +111,15 @@ class PluginManagerBase(object):
except ImportError:
pass
- def _iter_plugin_modules(self, get_module_fnc=utils.get_module):
+ def _iter_plugin_modules(self, get_module_fnc=compat.import_module):
"""
- Iterate plugin modules, yielding both full qualified name and
- short plugin name as tuple.
+ Iterate plugin modules.
+
+ This generator yields both full qualified name and short plugin
+ names.
"""
- nameset = set()
- shortset = set()
+ nameset = set() # type: typing.Set[str]
+ shortset = set() # type: typing.Set[str]
filters = self.plugin_filters
for prefix in filter(None, self.namespaces):
name_iter_fnc = (
@@ -151,12 +149,12 @@ class PluginManagerBase(object):
@cached_property
def available_plugins(self):
- """
- Iterate through all loadable plugins on default import locations
- """
+ # type: () -> typing.List[types.ModuleType]
+ """Iterate through all loadable plugins on typical paths."""
return list(self._iter_plugin_modules())
- def import_plugin(self, plugin, get_module_fnc=utils.get_module):
+ def import_plugin(self, plugin, get_module_fnc=compat.import_module):
+ # type: (str) -> types.ModuleType
"""
Import plugin by given name, looking at :attr:`namespaces`.
@@ -184,6 +182,7 @@ class PluginManagerBase(object):
plugin, names)
def load_plugin(self, plugin):
+ # type: (str) -> types.ModuleType
"""
Import plugin (see :meth:`import_plugin`) and load related data.
diff --git a/browsepy/plugin/file_actions/__init__.py b/browsepy/plugin/file_actions/__init__.py
index d375c9b..9599648 100644
--- a/browsepy/plugin/file_actions/__init__.py
+++ b/browsepy/plugin/file_actions/__init__.py
@@ -9,7 +9,6 @@ from browsepy.file import Node, abspath_to_urlpath, current_restricted_chars, \
common_path_separators
from browsepy.compat import re_escape, FileNotFoundError
from browsepy.exceptions import OutsideDirectoryBase
-from browsepy.utils import ppath
from browsepy.stream import stream_template
from .exceptions import FileActionsException, \
@@ -24,8 +23,8 @@ actions = Blueprint(
'file_actions',
__name__,
url_prefix='/file-actions',
- template_folder=ppath('templates', module=__name__),
- static_folder=ppath('static', module=__name__),
+ template_folder='templates',
+ static_folder='static',
)
re_basename = '^[^ {0}]([^{0}]*[^ {0}])?$'.format(
diff --git a/browsepy/plugin/player/__init__.py b/browsepy/plugin/player/__init__.py
index 7f00f11..8186cb1 100644
--- a/browsepy/plugin/player/__init__.py
+++ b/browsepy/plugin/player/__init__.py
@@ -10,7 +10,6 @@ This plugin is both functional and simple enough to server as example for:
"""
from flask import Blueprint, abort
from browsepy import get_cookie_browse_sorting, browse_sortkey_reverse
-from browsepy.utils import ppath
from browsepy.stream import stream_template
from .playable import PlayableNode, PlayableDirectory, PlayableFile
@@ -20,8 +19,8 @@ player = Blueprint(
'player',
__name__,
url_prefix='/play',
- template_folder=ppath('templates', module=__name__),
- static_folder=ppath('static', module=__name__),
+ template_folder='templates',
+ static_folder='static',
)
diff --git a/browsepy/static/icon/android-icon-144x144.png b/browsepy/static/icon/android-icon-144x144.png
new file mode 100644
index 0000000000000000000000000000000000000000..fb24422d9ac18370f3a878c3f20845ae27737c7d
GIT binary patch
literal 10616
zcmeAS@N?(olHy`uVBq!ia0y~yV3+{H9Bd2>4A0#j?OZ$Gj2)+u#i7W_lB95q`9_1`jbod0DyOYU%)ZaR{(IkomnI(-n%6GR
zn%?1kQSM8|+OO-%*QPr%ar!V{(y3j>yy4u}v$h9HwGLPauw2=>Z-KkuvB)1j+8?SD
zr1`&3S7Nq%ko>@%rBmp~)ET@?GaI}T-pAa(!@Pjqf;(o;YwpSr#q*b*yFLGPOin=S
zP_L?i>zOvzUEF@lCm%ZGG->kWiK&vIbFJ4e)8N{>ckh<%+l4z_p1rYpCY(0cVdu`B
zZ*Fhzzq&d+*=XjoFKNe*AGa)g#Pa+5`}CZg9KZFc`x35kJ}KI{V#SJt&(F?A=uETR
z>ack6;ye3ltIu`X%r|(=GAUJZ^2wH~Sx0lsgnQjM+Yh@e-Yl=?IjQAf!hsAEB~PJ+
z0UD0;B#*5T=5PoN4GC41>V5RJYG>VFtLElrk4rgA7S`D2*tyKl7xu@C6AaINEqCC(N)3Q4lI_5r}eG
zd`4MwCQG9nPuKz5N(;6A5{2|nS51G)oqn>K({TZ(g27KKwl9o#8_GZJcW-b^=y0DB
z!pU@3e%7g3+>W!97VtzE{(P^~^3?F-=BpFc|J%O}ooln`kB+i1|_T<}tPS%~6Yg3hz}3pZ*381h=;rRu+3Mt?B>2{#?Qr7qMH+qwn01(@
zF-JF~Jz!^>oB2=a|2ripW?xCo*iu+OXajmJ6in|)IV@3wMDjabf`v7GnDFGvSi7I
zoSR0|bfeueGc}JMJ(`l9K6%n4CoeCqyzSE6M?X1eDX&^3#MLSxCzqCy(edwJ-L74`
zGP1K5@7y_ab@=*AH*TDG@j|0(QN-el8Jqz&m9e4+G9O4k5L4=_(&*T7fpbbrL1dIb
z&(*A@J9f<2ntgp!+F7ZvH4%=Uo}87Hl`5*LzCT?isU&WUIC18Tim~xzRqtsG55E82
zS^S*OJnzneef#F!x)t^6)vJvBe02$)Qx0}Y8S$+h$NLzb3ow44le|#aBk5A;%^b67
zI+0GPsj9E9txZ1M#ye%ol!T2D6H+6^GXq3gKYlE%v6FXpcIM&dfBn%{Lqti2Z?Q&~
z-kMaSnFh1ZTFmvEGBtFMnKMF(FlQ_g%ltOifi)Rhz$GF30Szo69Y(SMctR
zrC!{g6K`*C?{ra0Yl%MXd?fAXfx;uFa{X9+#jP3*qst-SFe*CRdQ&XEUV}^nZU$TM3hN!g>
zI$}9y(l1}WOi4|hICrkEqa!0LD=R2Y=FasE3lsD7^xP4nr=_i}sjr{@`r6t#HkFgs
z@BcUJ)~zTjYwM>OCa)YK&M++yu;}n$WO?@H=h?5~7U2_AI=5^wId$sPjlI?8*5&V(
ztXRRZ`f8WX>6XKZd*bxpzI|(AY8n_H?_XPMtE;Qq)7STEnp>bq?aFn!cSA$3F3m|&
z&E0Z1#c1N3IX<4AoU6mvFI%*z>FMd|lc!I2kBX8?+dNa%d)kEAvrjMcoxNh!sz)y_
zE;cbWefa&iFNgM*^&KULWb!ElSYSz0F5sW&v%CVy^o7boPXXuJzZT~KkmrevduL=i{@26
zlbl!c$1|0rL*dX%466@HN`2M*8{f!x
ze*YX(z_RIP&YGB=lVq*SHk7}Q>+0$E=nv$t~Nyefq9lR{MTD;x2o0cREv=;g|NhEY
z7PVZv7AE5A$yTP|F@>e3_e9F^+}3RiI8qEGR;*Z|pswED)y4Jc^XG{wocr(l7Z)4H
z*M1c}-Y>s=tc5wJ8*!Z{N0_0SHt6<{(L@P-PCkyQxj8M?bpyXVcPHa
z|F@g27po-0cUw^9QrSO!-+$F*^Iyi;uh{8eAo1ka*Vli3p09tVU;mTaJpW!zMtXY1
z-nc$_d%yU2P(FThcXzly`=4j#`!BZM&d$o}a9ez`!!X*JL&((N^x+gSnMS9D0=&Gu
zHD4~e*ZlkW{L8m*$4sxy>#<
zoSk1TB_SbUo&M`^=RlFXySt9E%hw!O=-mF|9e!5YyNyZUi0l{dd1#*?qz9iDnbvxy}e!Y<6(Qv*Q?
ze0)g;5)qqHIDdb83n~t8ixX$W-|JU=J9YuB!{T>I>G55HFZ
z{B&A><*HRlDJd+?2OT^;IYH5~bm>x$MQW-xJPIR1oS62oHA;4HezHcdy4J
zm#{Fg-R1Aoj&_OOx;>}k)+?^G3tXxkCjJk;R{eZ7JKv)IpUvaPk0W%%_L`Whs=5}Z
zb_;9__~kRVJdXWhaO>5qqi@R|msoAQ{dUILwAzC8+rFC^8aCd4pI%WhW8%b#miN7b
zL#JN57RD!UcV?DpwuY|m(WgZlD?g{LTD|(`!*=-q5mnZWQqheZ%a-5uo2zg^rN>SA
z#H`|Zdq_>=-$YH%}&hB%%GI;
z=SShW`S$+d;o=7q{%jQr30=B;dHeommJZ@3_O^2OdZpp4)vo2n|xFdS;#*K#O=32W42MfEpy3X)X
zJ9X+*#O^ZPuV25mI4LS0;#$j)XM8s4539Gr;)@=cnVS3m|9$`SY<`_{W~QcCcf=eQ
zCBZ*+_IK~zy|c5p{olVjrbdQUt6sTg%JsMZ`BSsETglRL=94EWHgfz&j~=c0bW%Me
zG}O?5
zbZvdSzpE?j%9SgZEL*lI^|aWXJoB{8Gn?7@AASD$sH*<)Z}03whn(J&
zS$j{@N&NZgX-IguFjwoGj@~f-LtMUpBaeMOCnqE#D*EwSbpFRztJgpJSi$h({rl&4
zb{1F3T)uPX%#9m2^y~T#CMek2&NWUy_u)#g|Hka=dSUBgI*%qTT(qcZ`~AAv7cU0(
z$ygrxQSnEm{?J2%?Rj?}9qW}|6T4fi{V=GtH+AaNs7-;z$=w1$M#1Zz&h2GhJHKzTJLbO<$isH1y@Wcgt3->MFlq
zJ3aY$-^Gg;1*KV5|IzUmcQ~;|f*~?A^iQ3AZf9((xpoU
z7BXA5Y`Jjzw)FR;zh}<)fEu4wRaRlEr|#Qlx9|VI>fSS4hkltD_UiSqrZ%n#(|&z@
z{qybn|E|4v>GJjK#~&VUul#Zsmy`
ztCuftKX3P&XaD{4*6;TmKGrL}@%GzaKdr2+GV=5LqisZZ*mlJt?_a!sU*0@itkY#j>1#1rSy>k)!5%kdNl8ha=xsgH=6Mgk|JHTToN)5|q?-Lw
z+-cf}Q;Y%w0}pP?ykQ?w4k1Y3Z%JzZOmxUe+`HXaB+X@wJSD$cpTC
zamO{z2mUkIIlTC$p{%SdV_!FCv3tLfk