8000 [pull] main from miniflux:main by pull[bot] · Pull Request #234 · suhaoGe/miniflux · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions client/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ type Feed struct {
BlockFilterEntryRules string `json:"block_filter_entry_rules"`
KeepFilterEntryRules string `json:"keep_filter_entry_rules"`
Crawler bool `json:"crawler"`
IgnoreEntryUpdates bool `json:"ignore_entry_updates"`
UserAgent string `json:"user_agent"`
Cookie string `json:"cookie"`
Username string `json:"username"`
Expand All @@ -185,6 +186,7 @@ type FeedCreationRequest struct {
Username string `json:"username"`
Password string `json:"password"`
Crawler bool `json:"crawler"`
IgnoreEntryUpdates bool `json:"ignore_entry_updates"`
Disabled bool `json:"disabled"`
IgnoreHTTPCache bool `json:"ignore_http_cache"`
AllowSelfSignedCertificates bool `json:"allow_self_signed_certificates"`
Expand Down Expand Up @@ -214,6 +216,7 @@ type FeedModificationRequest struct {
BlockFilterEntryRules *string `json:"block_filter_entry_rules"`
KeepFilterEntryRules *string `json:"keep_filter_entry_rules"`
Crawler *bool `json:"crawler"`
IgnoreEntryUpdates *bool `json:"ignore_entry_updates"`
UserAgent *string `json:"user_agent"`
Cookie *string `json:"cookie"`
Username *string `json:"username"`
Expand Down
4 changes: 4 additions & 0 deletions internal/database/migrations.go
Original file line number Diff line number Diff line change
Expand Up @@ -1427,4 +1427,8 @@ var migrations = [...]func(tx *sql.Tx) error{
`)
return err
},
func(tx *sql.Tx) (err error) {
_, err = tx.Exec(`ALTER TABLE feeds ADD COLUMN ignore_entry_updates bool default 'f'`)
return err
},
}
1 change: 1 addition & 0 deletions internal/locale/translations/de_DE.json
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@
"form.feed.label.category": "Kategorie",
"form.feed.label.cookie": "Cookies setzen",
"form.feed.label.crawler": "Originalinhalt herunterladen",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Beschreibung",
"form.feed.label.disable_http2": "HTTP/2 deaktivieren, um Fingerprinting zu verhindern",
"form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/el_EL.json
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@
"form.feed.label.category": "Κατηγορία",
"form.feed.label.cookie": "Ορισμός Cookies",
"form.feed.label.crawler": "Λήψη αρχικού περιεχομένου",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Περιγραφή",
"form.feed.label.disable_http2": "Απενεργοποίηση HTTP/2 για αποφυγή δακτυλικών αποτυπωμάτων",
"form.feed.label.disabled": "Μη ανανέωση αυτής της ροής",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/en_US.json
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@
"form.feed.label.category": "Category",
"form.feed.label.cookie": "Set Cookies",
"form.feed.label.crawler": "Fetch original content",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Description",
"form.feed.label.disable_http2": "Disable HTTP/2 to avoid fingerprinting",
"form.feed.label.disabled": "Do not refresh this feed",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/es_ES.json
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@
"form.feed.label.category": "Categoría",
"form.feed.label.cookie": "Configurar las cookies",
"form.feed.label.crawler": "Obtener rastreador original",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Descripción",
"form.feed.label.disable_http2": "Deshabilite HTTP/2 para evitar huellas digitales",
"form.feed.label.disabled": "No actualice este feed",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/fi_FI.json
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@
"form.feed.label.category": "Kategoria",
"form.feed.label.cookie": "Aseta evästeet",
"form.feed.label.crawler": "Nouda alkuperäinen sisältö",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Kuvaus",
"form.feed.label.disable_http2": "Poista HTTP/2 käytöstä sormenjälkien välttämiseksi",
"form.feed.label.disabled": "Älä päivitä tätä syötettä",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/fr_FR.json
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@
"form.feed.label.category": "Catégorie",
"form.feed.label.cookie": "Définir les cookies",
"form.feed.label.crawler": "Récupérer le contenu original",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Description",
"form.feed.label.disable_http2": "Désactiver HTTP/2",
"form.feed.label.disabled": "Ne pas actualiser ce flux",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/hi_IN.json
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@
"form.feed.label.category": "श्रेणी",
"form.feed.label.cookie": &qu 10BC8 ot;कुकीज़ सेट करें",
"form.feed.label.crawler": "मूल सामग्री प्राप्त करें",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "विवरण",
"form.feed.label.disable_http2": "फिंगरप्रिंटिंग से बचने के लिए HTTP/2 अक्षम करें",
"form.feed.label.disabled": "इस फ़ीड को रीफ़्रेश न करें",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/id_ID.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"form.feed.label.category": "Kategori",
"form.feed.label.cookie": "Atur Kuki",
"form.feed.label.crawler": "Ambil konten asli",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Deskripsi",
"form.feed.label.disable_http2": "Matikan HTTP/2 untuk menghindari pelacakan",
"form.feed.label.disabled": "Jangan perbarui umpan ini",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/it_IT.json
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@
"form.feed.label.category": "Categoria",
"form.feed.label.cookie": "Installare i cookies",
"form.feed.label.crawler": "Scarica il contenuto integrale",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Descrizione",
"form.feed.label.disable_http2": "Disabilita HTTP/2 per evitare il fingerprinting",
"form.feed.label.disabled": "Non aggiornare questo feed",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/ja_JP.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"form.feed.label.category": "カテゴリ",
"form.feed.label.cookie": "Cookie の設定",
"form.feed.label.crawler": "オリジナルの内容を取得",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "説明",
"form.feed.label.disable_http2": "フィンガープリンティング回避のため HTTP/2 を無効化",
"form.feed.label.disabled": "このフィードを更新しない",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/nan_Latn_pehoeji.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"form.feed.label.category": "lūi-pia̍t",
"form.feed.label.cookie": "Siat-tēng Cookies",
"form.feed.label.crawler": "Lia̍h goân-tóe lōe-iông",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Biâu-su̍t",
"form.feed.label.disable_http2": "Thêng iōng HTTP/2 pī-bián chéng-thâu-á-hûn tui-chong",
"form.feed.label.disabled": "Mài tha̍k chit ê siau-sit lâi-goân ê sin siau-sit",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/nl_NL.json
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@
"form.feed.label.category": "Categorie",
"form.feed.label.cookie": "Cookies instellen",
"form.feed.label.crawler": "Download originele inhoud",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Omschrijving",
"form.feed.label.disable_http2": "HTTP/2 uitschakelen om fingerprinting te voorkomen",
"form.feed.label.disabled": "Deze feed niet vernieuwen",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/pl_PL.json
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@
"form.feed.label.category": "Kategoria",
"form.feed.label.cookie": "Ustaw ciasteczka",
"form.feed.label.crawler": "Pobierz oryginalną treść",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Opis",
"form.feed.label.disable_http2": "Wyłącz protokół HTTP/2, aby uniknąć identyfikowania",
"form.feed.label.disabled": "Nie aktualizuj tego kanału",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/pt_BR.json
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@
"form.feed.label.category": "Categoria",
"form.feed.label.cookie": "Definir Cookies",
"form.feed.label.crawler": "Obter conteúdo original",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Descrição",
"form.feed.label.disable_http2": "Desativar HTTP/2 para evitar fingerprinting",
"form.feed.label.disabled": "Não atualizar esta fonte",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/ro_RO.json
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@
"form.feed.label.category": "Categorie",
"form.feed.label.cookie": "Setare Cookie-uri",
"form.feed.label.crawler": "Aduce conținutul original",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Descriere",
"form.feed.label.disable_http2": "Dezactivează HTTP/2 pentru a preveni amprentarea",
"form.feed.label.disabled": "Nu actualiza acest flux",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/ru_RU.json
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@
"form.feed.label.category": "Категория",
"form.feed.label.cookie": "Установить куки",
"form.feed.label.crawler": "Извлечь оригинальное содержимое",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Описание",
"form.f 10BC8 eed.label.disable_http2": "Отключить HTTP/2 для предотвращения фингерпринтинга",
"form.feed.label.disabled": "Не обновлять эту подписку",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/tr_TR.json
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@
"form.feed.label.category": "Kategori",
"form.feed.label.cookie": "Çerezleri Ayarla",
"form.feed.label.crawler": "Orijinal içeriği çek",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Açıklama",
"form.feed.label.disable_http2": "Parmak izini önlemek için HTTP/2'yi devre dışı bırakın",
"form.feed.label.disabled": "Bu beslemeyi yenileme",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/uk_UA.json
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@
"form.feed.label.category": "Категорія",
"form.feed.label.cookie": "Встановити кукі",
"form.feed.label.crawler": "Завантажувати оригінальний вміст",
"form.feed.label.ignore_entry_updates": "Ignore entry updates",
"form.feed.label.description": "Опис",
"form.feed.label.disable_http2": "Вимкнути HTTP/2 для уникнення відбитків",
"form.feed.label.disabled": "Не оновлювати цю стрічку",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/zh_CN.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"form.feed.label.category": "分类",
"form.feed.label.cookie": "设置 Cookie",
"form.feed.label.crawler": "获取原始内容",
"form.feed.label.ignore_entry_updates": "忽略条目更新",
"form.feed.label.description": "描述",
"form.feed.label.disable_http2": "禁用 HTTP/2 以避免指纹识别",
"form.feed.label.disabled": "不刷新此订阅",
Expand Down
1 change: 1 addition & 0 deletions internal/locale/translations/zh_TW.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@
"form.feed.label.category": "類別",
"form.feed.label.cookie": "設定 Cookies",
"form.feed.label.crawler": "下載原文內容",
"form.feed.label.ignore_entry_updates": "忽略條目更新",
"form.feed.label.description": "描述",
"form.feed.label.disable_http2": "停用 HTTP/2 以避免指紋追蹤",
"form.feed.label.disabled": "不要更新此 Feed",
Expand Down
7 changes: 7 additions & 0 deletions internal/model/feed.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ type Feed struct {
PushoverEnabled bool `json:"pushover_enabled"`
NtfyEnabled bool `json:"ntfy_enabled"`
Crawler bool `json:"crawler"`
IgnoreEntryUpdates bool `json:"ignore_entry_updates"`
AppriseServiceURLs string `json:"apprise_service_urls"`
WebhookURL string `json:"webhook_url"`
NtfyPriority int `json:"ntfy_priority"`
Expand Down Expand Up @@ -156,6 +157,7 @@ type FeedCreationRequest struct {
Username string `json:"username"`
Password string `json:"password"`
Crawler bool `json:"crawler"`
IgnoreEntryUpdates bool `json:"ignore_entry_updates"`
Disabled bool `json:"disabled"`
NoMediaPlayer bool `json:"no_media_player"`
IgnoreHTTPCache bool `json:"ignore_http_cache"`
Expand Down Expand Up @@ -195,6 +197,7 @@ type FeedModificationRequest struct {
BlockFilterEntryRules *string `json:"block_filter_entry_rules"`
KeepFilterEntryRules *string `json:"keep_filter_entry_rules"`
Crawler *bool `json:"crawler"`
IgnoreEntryUpdates *bool `json:"ignore_entry_updates"`
UserAgent *string `json:"user_agent"`
Cookie *string `json:"cookie"`
Username *string `json:"username"`
Expand Down Expand Up @@ -260,6 +263,10 @@ func (f *FeedModificationRequest) Patch(feed *Feed) {
feed.Crawler = *f.Crawler
}

if f.IgnoreEntryUpdates != nil {
feed.IgnoreEntryUpdates = *f.IgnoreEntryUpdates
}

if f.UserAgent != nil {
feed.UserAgent = *f.UserAgent
}
Expand Down
8 changes: 6 additions & 2 deletions internal/reader/handler/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f
subscription.Username = feedCreationRequest.Username
subscription.Password = feedCreationRequest.Password
subscription.Crawler = feedCreationRequest.Crawler
subscription.IgnoreEntryUpdates = feedCreationRequest.IgnoreEntryUpdates
subscription.Disabled = feedCreationRequest.Disabled
subscription.IgnoreHTTPCache = feedCreationRequest.IgnoreHTTPCache
subscription.AllowSelfSignedCertificates = feedCreationRequest.AllowSelfSignedCertificates
Expand Down Expand Up @@ -154,6 +155,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model
subscription.Username = feedCreationRequest.Username
subscription.Password = feedCreationRequest.Password
subscription.Crawler = feedCreationRequest.Crawler
subscription.IgnoreEntryUpdates = feedCreationRequest.IgnoreEntryUpdates
subscription.Disabled = feedCreationRequest.Disabled
subscription.IgnoreHTTPCache = feedCreationRequest.IgnoreHTTPCache
subscription.AllowSelfSignedCertificates = feedCreationRequest.AllowSelfSignedCertificates
Expand Down Expand Up @@ -338,8 +340,10 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
originalFeed.Entries = updatedFeed.Entries
processor.ProcessFeedEntries(store, originalFeed, userID, forceRefresh)

// We don't update existing entries when the crawler is enabled (we crawl only inexisting entries). Unless it is forced to refresh
updateExistingEntries := forceRefresh || !originalFeed.Crawler
// We don't update existing entries when the crawler is enabled (we crawl only inexisting entries).
// We also skip updating existing entries if the feed has ignore_entry_updates enabled.
// Unless it is forced to refresh.
updateExistingEntries := forceRefresh || (!originalFeed.Crawler && !originalFeed.IgnoreEntryUpdates)
newEntries, storeErr := store.RefreshFeedEntries(originalFeed.UserID, originalFeed.ID, originalFeed.Entries, updateExistingEntries)
if storeErr != nil {
localizedError := locale.NewLocalizedErrorWrapper(storeErr, "error.database_error", storeErr)
Expand Down
6 changes: 5 additions & 1 deletion internal/reader/parser/format.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ const (
FormatUnknown = "unknown"
)

const maxTokensToConsider = uint(50)

// DetectFeedFormat tries to guess the feed format from input data.
func DetectFeedFormat(r io.ReadSeeker) (string, string) {
r.Seek(0, io.SeekStart)
Expand All @@ -32,11 +34,13 @@ func DetectFeedFormat(r io.ReadSeeker) (string, string) {
r.Seek(0, io.SeekStart)
decoder := rxml.NewXMLDecoder(r)

processedTokens := uint(0)
for {
token, _ := decoder.Token()
if token == nil {
if token == nil || processedTokens == maxTokensToConsider {
break
}
processedTokens += 1

if element, ok := token.(xml.StartElement); ok {
switch element.Name.Local {
Expand Down
Loading
Loading
0