8000 [pull] master from sissbruecker:master by pull[bot] · Pull Request #119 · bryanwills/linkding · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,7 @@
!/manage.py
!/package.json
!/package-lock.json
!/postcss.config.js
!/pyproject.toml
!/rollup.config.mjs
!/scripts/setup-ublock.sh
!/supervisord-tasks.conf
!/supervisord-all.conf
Expand Down
12 changes: 12 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
version: 2
updates:
- package-ecosystem: "npm"
directory: "/docs"
schedule:
interval: "weekly"
open-pull-requests-limit: 0
groups:
docs-security:
applies-to: security-updates
patterns:
- "*"
6 changes: 4 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
.PHONY: serve

BIND ?= 127.0.0.1:8000

init:
uv sync
[ -d data ] || mkdir data data/assets data/favicons data/previews
uv run manage.py migrate
npm install

serve:
u 10BC8 v run manage.py runserver
uv run manage.py runserver $(BIND)

tasks:
uv run manage.py run_huey
uv run manage.py run_huey -f

test:
uv run pytest -n auto
Expand Down
13 changes: 2 additions & 11 deletions bookmarks/api/routes.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
import gzip
import logging
import os

from django.conf import settings
from django.http import Http404, StreamingHttpResponse
from django.http import Http404
from rest_framework import mixins, status, viewsets
from rest_framework.decorators import action
from rest_framework.permissions import AllowAny
Expand Down Expand Up @@ -192,14 +190,7 @@ def get_serializer_context(self):
def download(self, request: HttpRequest, bookmark_id, pk):
asset = self.get_object()
try:
file_path = os.path.join(settings.LD_ASSET_FOLDER, asset.file)
content_type = asset.content_type
file_stream = (
gzip.GzipFile(file_path, mode="rb")
if asset.gzip
else open(file_path, "rb") # noqa: SIM115
)
response = StreamingHttpResponse(file_stream, content_type=content_type)
response = assets.stream_asset_file(asset)
response["Content-Disposition"] = (
f'attachment; filename="{asset.download_name}"'
)
Expand Down
3 changes: 3 additions & 0 deletions bookmarks/frontend/dev.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
// Entry point for development builds: includes the dev tool in addition to the regular bundle
import "./index.js";
import "./components/dev-tool.js";
1 change: 0 additions & 1 deletion bookmarks/frontend/index.js
Origin 8 al file 10BC8 line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import "./components/bookmark-page.js";
import "./components/clear-button.js";
import "./components/confirm-dropdown.js";
import "./components/details-modal.js";
import "./components/dev-tool.js";
import "./components/dropdown.js";
import "./components/filter-drawer.js";
import "./components/form.js";
Expand Down
42 changes: 42 additions & 0 deletions bookmarks/services/assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@
import logging
import os
import shutil
from typing import BinaryIO
from wsgiref.util import FileWrapper

import requests
from django.conf import settings
from django.core.files.uploadedfile import UploadedFile
from django.http import StreamingHttpResponse
from django.utils import formats, timezone

from bookmarks.models import Bookmark, BookmarkAsset
Expand Down Expand Up @@ -218,6 +221,45 @@ def upload_asset(bookmark: Bookmark, upload_file: UploadedFile):
raise e


# Chunk size used when streaming asset files to clients
STREAM_CHUNK_SIZE = 64 * 1024


def open_asset_file(asset: BookmarkAsset) -> BinaryIO:
"""
Opens the asset file for reading, transparently decompressing gzipped
files. Raises FileNotFoundError if the file does not exist.
"""
filepath = os.path.join(settings.LD_ASSET_FOLDER, asset.file)

if not os.path.isfile(filepath):
raise FileNotFoundError(filepath)

if asset.gzip:
return gzip.open(filepath, "rb")
return open(filepath, "rb") # noqa: SIM115


def stream_asset_file(asset: BookmarkAsset) -> StreamingHttpResponse:
"""
Creates a response that streams the (decompressed) asset file content in
fixed-size chunks. Callers can add headers to the response as needed.
Raises FileNotFoundError if the file does not exist.

Notes on why this is implemented this way:
- Do not use Django's FileResponse for asset files. Under uWSGI it hands the
file descriptor to sendfile(), which sends the raw compressed bytes for
gzipped assets while announcing the uncompressed content length.
- Do not pass the file object to StreamingHttpResponse directly. Iterating
a file yields lines, which loads files without newlines (e.g. binary
uploads) into memory as a whole.
FileWrapper closes the underlying file when the response is closed.
"""
file = open_asset_file(asset)
content = FileWrapper(file, STREAM_CHUNK_SIZE)
return StreamingHttpResponse(content, content_type=asset.content_type)


def remove_asset(asset: BookmarkAsset):
# If this asset is the latest_snapshot for a bookmark, try to find the next most recent snapshot
bookmark = asset.bookmark
Expand Down
101 changes: 76 additions & 25 deletions bookmarks/services/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from django.contrib.auth.models import User
from django.db.models import Q
from django.utils import timezone
from huey import crontab
from huey.contrib.djhuey import HUEY as huey
from huey.exceptions import TaskLockedException
from waybackpy.exceptions import TooManyRequestsError, WaybackError
Expand All @@ -32,12 +31,6 @@ def inner(*args, **kwargs):
task = kwargs.pop("task")
try:
return fn(*args, **kwargs)
except TaskLockedException as exc:
# Task locks are currently only used as workaround to enforce
# running specific types of tasks (e.g. singlefile snapshots)
# sequentially. In that case don't reduce the number of retries.
task.retries = retries
raise exc
except Exception as exc:
task.retry_delay *= retry_backoff
raise exc
Expand Down Expand Up @@ -261,6 +254,8 @@ def create_html_snapshot(bookmark: Bookmark):
asset = assets.create_snapshot_asset(bookmark)
asset.save()

_trigger_html_snapshot_processing()


def create_html_snapshots(bookmark_list: list[Bookmark]):
if not is_html_snapshot_feature_active():
Expand All @@ -273,30 +268,72 @@ def create_html_snapshots(bookmark_list: list[Bookmark]):

BookmarkAsset.objects.bulk_create(assets_to_create)

_trigger_html_snapshot_processing()

# singe-file does not support running multiple instances in parallel, so we can
# not queue up multiple snapshot tasks at once. Instead, schedule a periodic
# task that grabs a number of pending assets and creates snapshots for them in
# sequence. The task uses a lock to ensure that a new task isn't scheduled
# before the previous one has finished.
@huey.periodic_task(crontab(minute="*"))
@huey.lock_task("schedule-html-snapshots-lock")
def _schedule_html_snapshots_task():
# Get five pending assets
assets = BookmarkAsset.objects.filter(status=BookmarkAsset.STATUS_PENDING).order_by(
"date_created"
)[:5]

for asset in assets:
_create_html_snapshot_task(asset.id)
# single-file does not support running multiple instances in parallel, so we can
# not queue up a task per snapshot. Instead, a single task processes all pending
# assets in sequence, and is triggered whenever new assets are created. The lock
# ensures that only one of these tasks is running at a time.
_html_snapshot_lock = huey.lock_task("html-snapshots-lock")


def _create_html_snapshot_task(asset_id: int):
def _trigger_html_snapshot_processing():
# If a task is already running, it also picks up the assets that were just
# created, either in its processing loop or when checking for remaining
# assets after releasing the lock. Skip queuing a task that would only be
# dropped by the lock anyway.
if _html_snapshot_lock.is_locked():
return

_process_html_snapshots_task()


def _get_next_pending_asset() -> BookmarkAsset | None:
return (
BookmarkAsset.objects.filter(
asset_type=BookmarkAsset.TYPE_SNAPSHOT,
status=BookmarkAsset.STATUS_PENDING,
)
.order_by("date_created", "id")
.first()
)


@task()
def _process_html_snapshots_task():
# Tasks may still be queued after the feature has been disabled
if not is_html_snapshot_feature_active():
return

try:
asset = BookmarkAsset.objects.get(id=asset_id)
except BookmarkAsset.DoesNotExist:
with _html_snapshot_lock:
# Processing an asset always moves it off the pending status, so
# the loop terminates once all pending assets have been processed.
while asset := _get_next_pending_asset():
_create_html_snapshot(asset)
except TaskLockedException:
# Another task is already creating snapshots. As it processes assets
# until there are no pending ones left, it also picks up the assets that
# this task was triggered for.
return

# Assets that were created while holding the lock may have triggered a task
# that was dropped above, in which case they have to be picked up here.
if _get_next_pending_asset():
_process_html_snapshots_task()


@huey.on_startup()
def _process_html_snapshots_on_startup():
# Pick up assets that are still pending, for example because the consumer
# was stopped while creating snapshots. This hook runs once per consumer
# worker, the additional tasks are dropped by the lock.
if is_html_snapshot_feature_active():
_process_html_snapshots_task()


def _create_html_snapshot(asset: BookmarkAsset):
logger.info(f"Create HTML snapshot for bookmark. url={asset.bookmark.url}")

try:
Expand All @@ -307,11 +344,25 @@ def _create_html_snapshot_task(asset_id: int):
)
except Exception as error:
logger.error(
f"Failed to HTML snapshot for bookmark. url={asset.bookmark.url}",
f"Failed to create HTML snapshot for bookmark. url={asset.bookmark.url}",
exc_info=error,
)


@task()
def _schedule_html_snapshots_task():
# Snapshots are no longer created by a periodic task, keeping the task
# function for now to prevent errors when huey tries to run the task
pass


@task()
def _create_html_snapshot_task(asset_id: int):
# Snapshots are now created by _process_html_snapshots_task, keeping the
# task function for now to prevent errors when huey tries to run the task
pass


def create_missing_html_snapshots(user: User) -> int:
if not is_html_snapshot_feature_active():
return 0
Expand Down
1 change: 1 addition & 0 deletions bookmarks/settings/dev.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

INTERNAL_IPS = [
"127.0.0.1",
"172.17.0.0",
]

STATICFILES_DIRS = [
Expand Down
4 changes: 3 additions & 1 deletion bookmarks/templates/bookmarks/read.html
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{% load static %}
{% load shared %}
<!DOCTYPE html>
<html lang="en" class="reader-mode">
<head>
Expand Down Expand Up @@ -34,8 +35,9 @@
media="(prefers-color-scheme: light)"
content="#5856e0">
{% endif %}
{# The page is sandboxed, so requests to the custom CSS view would not include credentials. Embed the CSS as data URL instead. #}
{% if request.user_profile.custom_css %}
<link href="{% url 'linkding:custom_css' %}?hash={{ request.user_profile.custom_css_hash }}"
<link href="{{ request.user_profile.custom_css|css_data_url }}"
rel="stylesheet"
type="text/css" />
{% endif %}
Expand Down
7 changes: 7 additions & 0 deletions bookmarks/templatetags/shared.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import base64
import re

import bleach
Expand Down Expand Up @@ -61,6 +62,12 @@ def humanize_relative_date(value):
return utils.humanize_relative_date(value)


@register.filter(name="css_data_url")
def css_data_url(css):
encoded = base64.b64encode(css.encode("utf-8")).decode("ascii")
return f"data:text/css;charset=utf-8;base64,{encoded}"


@register.filter(name="model_to_dict")
def model_to_dict_filter(value):
result = model_to_dict(value)
Expand Down
Loading
Loading
0