8000 [pull] main from miniflux:main by pull[bot] · Pull Request #309 · suhaoGe/miniflux · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 21 additions & 13 deletions internal/reader/readingtime/readingtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@ import (

// EstimateReadingTime returns the estimated reading time of an article in minute.
func EstimateReadingTime(content string, defaultReadingSpeed, cjkReadingSpeed int) int {
const truncationPoint = 100

sanitizedContent := sanitizer.StripTags(content)
truncationPoint := min(len(sanitizedContent), 50)

if isCJK(sanitizedContent[:truncationPoint]) {
if isCJK(sanitizedContent, truncationPoint) {
return (utf8.RuneCountInString(sanitizedContent) + cjkReadingSpeed - 1) / cjkReadingSpeed
}

return (countWords(sanitizedContent) + defaultReadingSpeed - 1) / defaultReadingSpeed
}

Expand All @@ -31,20 +33,26 @@ func countWords(s string) int {
return n
}

func isCJK(text string) bool {
totalCJK := 0
func isCJK(text string, limit int) bool {
var letters, totalCJK int
for _, r := range text {
// Numbers and control characters often used in CJK too.
// Counting them makes detection less reliable.
if !unicode.In(r, unicode.Letter) {
continue
}

if letters++; letters == limit {
break
}

for _, r := range text[:min(len(text), 50)] {
if unicode.Is(unicode.Han, r) ||
unicode.Is(unicode.Hangul, r) ||
unicode.Is(unicode.Hiragana, r) ||
unicode.Is(unicode.Katakana, r) ||
unicode.Is(unicode.Yi, r) ||
unicode.Is(unicode.Bopomofo, r) {
if unicode.In(r, unicode.Han, unicode.Hangul, unicode.Hiragana, unicode.Katakana, unicode.Yi, unicode.Bopomofo) {
totalCJK++
}
}

// if at least 50% of the text is CJK, odds are that the text is in CJK.
return totalCJK > len(text)/50
// If at least half of the letters is CJK, odds are that the text is CJK.
midpoint := letters / 2

return totalCJK > midpoint
}
65 changes: 65 additions & 0 deletions internal/reader/readingtime/readingtime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,3 +114,68 @@ func TestCountWordsZeroAllocs(t *testing.T) {
t.Errorf("countWords allocated %v times, expected 0", allocs)
}
}

func Test_isCJK(t *testing.T) {
t.Parallel()

tests := []struct {
name string
input string
limit int
expected bool
}{
{
name: "latin short",
input: "Lorem ipsum dolor sit amet",
limit: 100,
expected: false,
},
{
name: "latin long",
input: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce fermentum id sem sed commodo. Ut eget mauris eu lectus mollis aliquam.",
limit: 100,
expected: false,
},
{
name: "cyrillic",
input: "Съешь ещё этих мягких французских булок, да выпей же чаю",
limit: 50,
expected: false,
},
{
name: "japanese",
input: "政盤が力和文ぱのら仕2以第ミ討漢年レ毎開スヤ間度を時深読じどが画気握てぐゆぶ留海え転5航調ヤ運旬クヌロレ探憲御裕こぎびさ。能ど加開ゆぶ軍43地伐2施ヤ実58共験それ合氾み歳熊つごゅま手柴りらし認整ラカ死感は界後えい警立ゃがよト訪奏エシサミ携勝ヘヒコカ軽通年クタ公無せはぐ読階齢間て。",
limit: 100,
expected: true,
},
{
name: "hangul",
input: "대통령은 국가의 안위에 관계되는 중대한 교전상태에 있어서 국가를 보위하기 위하여 긴급한 조치가 필요하고 국회의 집회가 불가능한 때에 한하여 법률의 효력을 가지는 명령을 발할 수 있다. 국가는 사회적·경제적 방법으로 근로자의 고용의 증진과 적정임금의 보장에 노력하여야 하며. 국교는 인정되지 아니하며. 법률이 정한 국무위원의 순서로 그 권한을 대행한다.",
limit: 100,
expected: true,
},
{
name: "mixed but mostly non-CJK",
input: "Tofu (Japanese: 豆腐, Hepburn: Tōfu; Chinese: 豆腐; pinyin: dòufu; Korean: 두부; RR: dubu)",
limit: 100,
expected: false,
},
{
name: "mixed but just enough hangul",
input: "Korean: 국한문 혼용체; Hanja: 國漢文混用體",
limit: 100,
expected: true,
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()

got := isCJK(test.input, test.limit)
if got != test.expected {
t.Errorf("isCJK(%q, %d) = %v, want %v", test.input, test.limit, got, test.expected)
}
})
}
}
2 changes: 1 addition & 1 deletion internal/reader/subscription/finder.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ func (f *subscriptionFinder) findSubscriptionsFromWebPage(websiteURL string, doc
}

var subscriptions Subscriptions
subscriptionURLs := make(map[string]bool)
subscriptionURLs := make(map[string]bool, len(queries))
for feedQuerySelector, feedFormat := range queries {
doc.Find(feedQuerySelector).Each(func(i int, s *goquery.Selection) {
subscription := new(subscription)
Expand Down
Loading
0