8000 [pull] main from miniflux:main by pull[bot] · Pull Request #307 · suhaoGe/miniflux · GitHub
[go: up one dir, main page]

Skip to content
8000
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions internal/locale/catalog_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ func TestTranslationFilePluralForms(t *testing.T) {
"id_ID": 1,
"it_IT": 2,
"ja_JP": 1,
"ko_KR": 1,
"nan_Latn_pehoeji": 1,
"nl_NL": 2,
"pl_PL": 3,
Expand Down
1 change: 1 addition & 0 deletions internal/locale/locale.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ var AvailableLanguages = map[string]string{
"id_ID": "Bahasa Indonesia",
"it_IT": "Italiano",
"ja_JP": "日本語",
"ko_KR": "한국어",
"nan_Latn_pehoeji": "Pe̍h-ōe-jī",
"nl_NL": "Nederlands",
"pl_PL": "Polski",
Expand Down
2 changes: 1 addition & 1 deletion internal/locale/plural.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func getPluralForm(lang string, n int) int {
return 1
}
return 0
case "id_ID", "ja_JP":
case "id_ID", "ja_JP", "ko_KR":
return 0
case "pl_PL":
switch {
Expand Down
8 changes: 8 additions & 0 deletions internal/locale/plural_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,14 @@ func TestPluralRules(t *testing.T) {
5: 0,
100: 0,
},
// Korean (ko_KR) - always form 0
"ko_KR": {
0: 0,
1: 0,
2: 0,
5: 0,
100: 0,
},
// Polish (pl_PL) - 3 forms
"pl_PL": {
1: 0, // n == 1
Expand Down
620 changes: 620 additions & 0 deletions internal/locale/translations/ko_KR.json

Large diffs are not rendered by default.

41 changes: 29 additions & 12 deletions internal/reader/sanitizer/strip_tags.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,39 @@ import (
// StripTags removes all HTML/XML tags from the input string.
// This function must *only* be used for cosmetic purposes, not to prevent code injections like XSS.
func StripTags(input string) string {
tokenizer := html.NewTokenizer(strings.NewReader(input))
var buffer strings.Builder
dst := &strings.Builder{}
src := strings.NewReader(input)

for {
if tokenizer.Next() == html.ErrorToken {
err := tokenizer.Err()
if errors.Is(err, io.EOF) {
return buffer.String()
}
err := stripIter(src, func(text string) bool {
dst.WriteString(text)
return true
})
if err != nil {
return ""
}

return ""
}
return dst.String()
}

// stripIter iterates over the input [io.Reader] and calls the yield function for each [html.TextToken].
// Other kinds of [html.TokenType] are skipped.
func stripIter(src io.Reader, yield func(string) bool) error {
tokenizer := html.NewTokenizer(src)

for tokenizer.Next() != html.ErrorToken {
token := tokenizer.Token()
if token.Type == html.TextToken {
buffer.WriteString(token.Data)
if token.Type != html.TextToken {
continue
}

if !yield(token.Data) {
break
}
}

if err := tokenizer.Err(); !errors.Is(err, io.EOF) {
return err
}

return nil
}
87 changes: 77 additions & 10 deletions internal/reader/sanitizer/truncate.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,86 @@

package sanitizer

import "strings"
import (
"strings"
"unicode"
"unicode/utf8"
)

func TruncateHTML(input string, max int) string {
text := StripTags(input)
// TruncateHTML returns cleaned up and shortened version of input.
// - HTML tags are removed
// - Consecutive whitespace characters replaced with single SPACE (0x20) character
// - If input has more runes than limit, it's truncated
func TruncateHTML(input string, limit int) string {
dst := &strings.Builder{}
src := strings.NewReader(input)

// Collapse multiple spaces into a single space
text = strings.Join(strings.Fields(text), " ")
words := 0
count := 0
needspace := false

// Convert to runes to be safe with unicode
runes := []rune(text)
if len(runes) > max {
return strings.TrimSpace(string(runes[:max])) + "…"
err := stripIter(src, func(token string) bool {
// Skip leading space.
if words > 0 {
// Add a space between tokens if there's one before HTML tag.
r, _ := utf8.DecodeRuneInString(token)
needspace = needspace || unicode.IsSpace(r)
}

for word := range strings.FieldsSeq(token) {
if needspace {
if count += 1; count > limit {
return false
}
}

// Compute how much of the word we can use later.
wordlen := 0
for wordlen < len(word) {
if count += 1; count > limit {
break
}

r, w := utf8.DecodeRuneInString(word[wordlen:])
if r == utf8.RuneError {
wordlen += 1
continue
}

wordlen += w
}

// This is the only place where space being placed.
// That way any sequence of space characters ends up as a singular SPACE (0x20) character.
//
// wordlen > 0 skips spaces if no printable characters left.
if needspace && wordlen > 0 {
dst.WriteByte(' ')
}

dst.WriteString(word[:wordlen])

if count > limit {
return false
}

needspace = true // To insert spaces in-between words in a token.
words++
}

// Add a space between tokens if there's one after HTML tag.
r, _ := utf8.DecodeLastRuneInString(token)
needspace = unicode.IsSpace(r) && words > 0

return true
})
if err != nil {
return ""
}

if count > limit {
dst.WriteRune('…')
}

return text
return dst.String()
}
Loading
Loading
0