8000 [pull] main from miniflux:main by pull[bot] · Pull Request #340 · suhaoGe/miniflux · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 0 additions & 17 deletions internal/model/language.go

This file was deleted.

26 changes: 0 additions & 26 deletions internal/model/language_test.go

This file was deleted.

12 changes: 10 additions & 2 deletions internal/reader/atom/atom_03.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,11 @@ import (
type atom03Feed struct {
Version string `xml:"version,attr"`

Language string `xml:"lang,attr"`
// Language is the natural language of the feed, declared by an
// xml:lang attribute on the atom:feed element. The tag is
// namespace-qualified so that lang attributes from other namespaces
// cannot override the real xml:lang value.
Language string `xml:"http://www.w3.org/XML/1998/namespace lang,attr"`

// The "atom:id" element's content conveys a permanent, globally unique identifier for the feed.
// It MUST NOT change over time, even if the feed is relocated. atom:feed elements MAY contain an atom:id element,
Expand Down Expand Up @@ -49,7 +53,11 @@ type atom03Entry struct {
// If the same entry is syndicated in two atom:feeds published by the same entity, the entry's atom:id MUST be the same in both feeds.
ID string `xml:"id"`

Language string `xml:"lang,attr"`
// Language is the natural language of the entry, declared by an
// xml:lang attribute on the atom:entry element. The tag is
// namespace-qualified so that lang attributes from other namespaces
// cannot override the real xml:lang value.
Language string `xml:"http://www.w3.org/XML/1998/namespace lang,attr"`

// The "atom:title" element is a Content construct that conveys a human-readable title for the entry.
// atom:entry elements MUST have exactly one "atom:title" element.
Expand Down
11 changes: 9 additions & 2 deletions internal/reader/atom/atom_03_adapter.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/reader/date"
"miniflux.app/v2/internal/reader/language"
"miniflux.app/v2/internal/reader/sanitizer"
"miniflux.app/v2/internal/urllib"
)
Expand Down Expand Up @@ -47,12 +48,18 @@ func (a *atom03Adapter) buildFeed(baseURL string) *model.Feed {
feed.Title = feed.SiteURL
}

feed.Language = model.NormalizeLanguage(a.atomFeed.Language)
feed.Language = language.Normalize(a.atomFeed.Language)

for _, atomEntry := range a.atomFeed.Entries {
entry := model.NewEntry()

entry.Language = model.NormalizeLanguage(atomEntry.Language)
// Populate the entry language. xml:lang applies to the whole
// subtree it is declared on, so an entry without its own
// xml:lang inherits the feed-level value.
entry.Language = language.Normalize(atomEntry.Language)
if entry.Language == "" {
entry.Language = language.Normalize(a.atomFeed.Language)
}

// Populate the entry URL.
entry.URL = atomEntry.Links.originalLink()
Expand Down
44 changes: 44 additions & 0 deletions internal/reader/atom/atom_03_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -294,3 +294,47 @@ func TestParseAtom03WithBase64Content(t *testing.T) {
t.Errorf("Incorrect entry content, got: %s", feed.Entries[0].Content)
}
}

func TestParseAtom03WithLanguage(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed version="0.3" xmlns="http://purl.org/atom/ns#" xmlns:foo="http://example.org/ns" xml:lang="fr-CA" foo:lang="zz">
<title>dive into mark</title>
<link rel="alternate" type="text/html" href="http://diveintomark.org/"/>
<modified>2003-12-13T18:30:02Z</modified>
<entry xml:lang="pt-BR" foo:lang="zz">
<title>Atom 0.3 snapshot</title>
<link rel="alternate" type="text/html" href="http://diveintomark.org/2003/12/13/atom03"/>
<id>tag:diveintomark.org,2003:3.2397</id>
<issued>2003-12-13T08:29:29-04:00</issued>
<modified>2003-12-13T18:30:02Z</modified>
</entry>
<entry>
<title>Atom 0.3 second snapshot</title>
<link rel="alternate" type="text/html" href="http://diveintomark.org/2003/12/14/atom03"/>
<id>tag:diveintomark.org,2003:3.2398</id>
<issued>2003-12-14T08:29:29-04:00</issued>
<modified>2003-12-14T18:30:02Z</modified>
</entry>
</feed>`

feed, err := Parse("http://diveintomark.org/atom.xml", bytes.NewReader([]byte(data)), "0.3")
if err != nil {
t.Fatal(err)
}

if feed.Language != "fr-ca" {
t.Errorf("Incorrect language, got: %q", feed.Language)
}

if len(feed.Entries) != 2 {
t.Fatalf("Expected 2 entries, got: %d", len(feed.Entries))
}

if feed.Entries[0].Language != "pt-br" {
t.Errorf("Incorrect entry language, got: %q", feed.Entries[0].Language)
}

if feed.Entries[1].Language != "fr-ca" {
t.Errorf("Expected entry to inherit feed language, got: %q", feed.Entries[1].Language)
}
}
15 changes: 10 additions & 5 deletions internal/reader/atom/atom_10.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,11 @@ import (
type atom10Feed struct {
XMLName xml.Name `xml:"http://www.w3.org/2005/Atom feed"`

// xml:lang declares the natural language of the feed and, by
// inheritance, of its entries. Persisted on the feed row and emitted
// as an HTML lang attribute by the web reader.
Language string `xml:"lang,attr"`
// Language is the natural language of the feed, declared by an
// xml:lang attribute on the atom:feed element. The tag is
// namespace-qualified so that lang attributes from other namespaces
// cannot override the real xml:lang value.
Language string `xml:"http://www.w3.org/XML/1998/namespace lang,attr"`

// The "atom:id" element conveys a permanent, universally unique
// identifier for an entry or feed.
Expand Down Expand Up @@ -101,7 +102,11 @@ type atom10Entry struct {
// atom:entry elements MUST contain exactly one atom:id element.
ID string `xml:"http://www.w3.org/2005/Atom id"`

Language string `xml:"lang,attr"`
// Language is the natural language of the entry, declared by an
// xml:lang attribute on the atom:entry element. The tag is
// namespace-qualified so that lang attributes from other namespaces
// cannot override the real xml:lang value.
Language string `xml:"http://www.w3.org/XML/1998/namespace lang,attr"`

// The "atom:title" element is a Text construct that conveys a human-
// readable title for an entry or feed.
Expand Down
11 changes: 9 additions & 2 deletions internal/reader/atom/atom_10_adapter.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/reader/date"
"miniflux.app/v2/internal/reader/language"
"miniflux.app/v2/internal/reader/sanitizer"
"miniflux.app/v2/internal/urllib"
)
Expand Down Expand Up @@ -51,7 +52,7 @@ func (a *atom10Adapter) buildFeed(baseURL string) *model.Feed {
// Populate the feed description.
feed.Description = a.atomFeed.Subtitle.body()

feed.Language = model.NormalizeLanguage(a.atomFeed.Language)
feed.Language = language.Normalize(a.atomFeed.Language)

// Populate the feed icon.
for _, value := range []string{a.atomFeed.Icon, a.atomFeed.Logo} {
Expand Down Expand Up @@ -111,7 +112,13 @@ func (a *atom10Adapter) populateEntries(siteURL string) model.Entries {
}
}

entry.Language = model.NormalizeLanguage(atomEntry.Language)
// Populate the entry language. xml:lang applies to the whole
// subtree it is declared on, so an entry without its own
// xml:lang inherits the feed-level value.
entry.Language = language.Normalize(atomEntry.Language)
if entry.Language == "" {
entry.Language = language.Normalize(a.atomFeed.Language)
}

// Populate the entry author.
authors := atomEntry.Authors.personNames()
Expand Down
83 changes: 82 additions & 1 deletion internal/reader/atom/atom_10_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1933,7 +1933,7 @@ func TestParseEntryWithLanguage(t *testing.T) {
}
}

func TestParseEntryWithoutLanguage(t *testing.T) {
func TestParseEntryWithoutLanguageInheritsFeedLanguage(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en">
<title>Example Feed</title>
Expand All @@ -1957,7 +1957,88 @@ func TestParseEntryWithoutLanguage(t *testing.T) {
t.Fatalf("Expected 1 entry, got: %d", len(feed.Entries))
}

if feed.Entries[0].Language != "en" {
t.Errorf("Expected entry to inherit feed language, got: %q", feed.Entries[0].Language)
}
}

func TestParseEntryWithoutAnyLanguage(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<updated>2003-12-13T18:30:02Z</updated>
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
<entry>
<title>Hello</title>
<link href="http://example.org/2003/12/13/hello"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
</entry>
</feed>`

feed, err := Parse("http://example.org/feed.xml", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}

if len(feed.Entries) != 1 {
t.Fatalf("Expected 1 entry, got: %d", len(feed.Entries))
}

if feed.Entries[0].Language != "" {
t.Errorf("Expected empty entry language, got: %q", feed.Entries[0].Language)
}
}

func TestParseFeedWithForeignLangAttribute(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom" xmlns:foo="http://example.org/ns" xml:lang="fr-CA" foo:lang="zz">
<title>Example Feed</title>
<link href="http://example.org/"/>
<updated>2003-12-13T18:30:02Z</updated>
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
<entry xml:lang="fr-CA" foo:lang="zz">
<title>Bonjour</title>
<link href="http://example.org/2003/12/13/bonjour"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
</entry>
</feed>`

feed, err := Parse("http://example.org/feed.xml", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}

if feed.Language != "fr-ca" {
t.Errorf("Incorrect language, got: %q", feed.Language)
}

if len(feed.Entries) != 1 {
t.Fatalf("Expected 1 entry, got: %d", len(feed.Entries))
}

if feed.Entries[0].Language != "fr-ca" {
t.Errorf("Incorrect entry language, got: %q", feed.Entries[0].Language)
}
}

func TestParseFeedWithUnqualifiedLangAttribute(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom" lang="de">
<title>Example Feed</title>
<link href="http://example.org/"/>
<updated>2003-12-13T18:30:02Z</updated>
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
</feed>`

feed, err := Parse("http://example.org/feed.xml", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}

if feed.Language != "" {
t.Errorf("Expected empty language for unqualified lang attribute, got: %q", feed.Language)
}
}
5 changes: 3 additions & 2 deletions internal/reader/json/adapter.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/reader/date"
"miniflux.app/v2/internal/reader/language"
"miniflux.app/v2/internal/reader/sanitizer"
"miniflux.app/v2/internal/urllib"
)
Expand All @@ -31,7 +32,7 @@ func (j *JSONAdapter) BuildFeed(baseURL string) *model.Feed {
FeedURL: strings.TrimSpace(j.jsonFeed.FeedURL),
SiteURL: strings.TrimSpace(j.jsonFeed.HomePageURL),
Description: strings.TrimSpace(j.jsonFeed.Description),
Language: model.NormalizeLanguage(j.jsonFeed.Language),
Language: language.Normalize(j.jsonFeed.Language),
}

if feed.FeedURL == "" {
Expand Down Expand Up @@ -70,7 +71,7 @@ func (j *JSONAdapter) BuildFeed(baseURL string) *model.Feed {

for _, item := range j.jsonFeed.Items {
entry := model.NewEntry()
entry.Language = model.NormalizeLanguage(item.Language)
entry.Language = language.Normalize(item.Language)

for _, itemURL := range []string{item.URL, item.ExternalURL} {
if itemURL = strings.TrimSpace(itemURL); itemURL == "" {
Expand Down
46 changes: 46 additions & 0 deletions internal/reader/language/language.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package language // import "miniflux.app/v2/internal/reader/language"

import "strings"

// maxLength bounds accepted language tags. RFC 5646 recommends supporting
// tags of at least 35 characters; anything much longer is garbage.
const maxLength = 50

// Normalize cleans up a language tag declared by a feed so it is
// suitable for use as an HTML lang attribute. It trims surrounding
// whitespace, lower-cases the value, and replaces underscores with hyphens
// (e.g. "en_US" -> "en-us"). No strict BCP-47 validation is performed:
// many real feeds use loose values and silently dropping them yields worse
// downstream behaviour than passing them through.
//
// The value is feed-controlled and is persisted and rendered as-is, so
// anything outside the BCP-47 tag alphabet ([a-z0-9-]) or longer than
// maxLength is rejected: such a value carries no usable language
// information, and stripping bad characters could assemble a wrong tag.
func Normalize(s string) string {
s = strings.TrimSpace(s)
if len(s) > maxLength {
return ""
}

// Lower-case ASCII-only, in the same pass as the charset check.
// Unicode case folding (strings.ToLower) would map some non-ASCII
// characters to ASCII (e.g. the Kelvin sign U+212A to "k"), turning
// input the filter should reject into an apparently valid tag.
b := []byte(s)
for i, c := range b {
switch {
case c >= 'A' && c <= 'Z':
b[i] = c + 'a' - 'A'
case c == '_':
b[i] = '-'
case (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-':
default:
return ""
}
}
return string(b)
}
Loading
Loading
0