8000 [pull] main from miniflux:main by pull[bot] · Pull Request #185 · suhaoGe/miniflux · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions internal/reader/rewrite/content_rewrite_functions.go
Original file line number Diff line number Diff line change
Expand Up @@ -286,9 +286,7 @@ func getYoutubVideoIDFromURL(entryURL string) string {
}

func buildVideoPlayerIframe(absoluteVideoURL string) string {
// Note: the referrerpolicy seems to be required to avoid YouTube error 153 video player configuration error
// See https://developers.google.com/youtube/terms/required-minimum-functionality#embedded-player-api-client-identity
return `<iframe width="650" height="350" frameborder="0" src="` + absoluteVideoURL + `" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe>`
return `<iframe width="650" height="350" frameborder="0" src="` + absoluteVideoURL + `" allowfullscreen></iframe>`
}

func addVideoPlayerIframe(absoluteVideoURL, entryContent string) string {
Expand Down
36 changes: 18 additions & 18 deletions internal/reader/rewrite/content_rewrite_test.go
8000
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func TestRewriteYoutubeVideoLink(t *testing.T) {
controlEntry := &model.Entry{
URL: "https://www.youtube.com/watch?v=1234",
Title: `A title`,
Content: `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/1234" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Video Description`,
Content: `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/1234" allowfullscreen></iframe><br>Video Description`,
}
testEntry := &model.Entry{
URL: "https://www.youtube.com/watch?v=1234",
Expand All @@ -92,7 +92,7 @@ func TestRewriteYoutubeShortLink(t *testing.T) {
controlEntry := &model.Entry{
URL: "https://www.youtube.com/shorts/1LUWKWZkPjo",
Title: `A title`,
Content: `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/1LUWKWZkPjo" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Video Description`,
Content: `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/1LUWKWZkPjo" allowfullscreen></iframe><br>Video Description`,
}
testEntry := &model.Entry{
URL: "https://www.youtube.com/shorts/1LUWKWZkPjo",
Expand Down Expand Up @@ -140,7 +140,7 @@ func TestRewriteYoutubeLinkAndCustomEmbedURL(t *testing.T) {
controlEntry := &model.Entry{
URL: "https://www.youtube.com/watch?v=1234",
Title: `A title`,
Content: `<iframe width="650" height="350" frameborder="0" src="https://invidious.custom/embed/1234" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Video Description`,
Content: `<iframe width="650" height="350" frameborder="0" src="https://invidious.custom/embed/1234" allowfullscreen></iframe><br>Video Description`,
}
testEntry := &model.Entry{
URL: "https://www.youtube.com/watch?v=1234",
Expand All @@ -159,7 +159,7 @@ func TestRewriteYoutubeVideoLinkUsingInvidious(t *testing.T) {
controlEntry := &model.Entry{
URL: "https://www.youtube.com/watch?v=1234",
Title: `A title`,
Content: `<iframe width="650" height="350" frameborder="0" src="https://yewtu.be/embed/1234" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Video Description`,
Content: `<iframe width="650" height="350" frameborder="0" src="https://yewtu.be/embed/1234" allowfullscreen></iframe><br>Video Description`,
}
testEntry := &model.Entry{
URL: "https://www.youtube.com/watch?v=1234",
Expand All @@ -179,7 +179,7 @@ func TestRewriteYoutubeShortLinkUsingInvidious(t *testing.T) {
controlEntry := &model.Entry{
URL: "https://www.youtube.com/shorts/1LUWKWZkPjo",
Title: `A title`,
Content: `<iframe width="650" height="350" frameborder="0" src="https://yewtu.be/embed/1LUWKWZkPjo" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Video Description`,
Content: `<iframe width="650" height="350" frameborder="0" src="https://yewtu.be/embed/1LUWKWZkPjo" allowfullscreen></iframe><br>Video Description`,
}
testEntry := &model.Entry{
URL: "https://www.youtube.com/shorts/1LUWKWZkPjo",
Expand All @@ -199,16 +199,16 @@ func TestAddYoutubeVideoFromId(t *testing.T) {

scenarios := map[string]string{
// Test with single YouTube ID
`Some content with youtube ID <script type="text/javascript" data-reactid="6">window.__APOLLO_STATE__ = {youtube_id: "9uASADiYe_8"}</script>`: `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/9uASADiYe_8" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Some content with youtube ID <script type="text/javascript" data-reactid="6">window.__APOLLO_STATE__ = {youtube_id: "9uASADiYe_8"}</script>`,
`Some content with youtube ID <script type="text/javascript" data-reactid="6">window.__APOLLO_STATE__ = {youtube_id: "9uASADiYe_8"}</script>`: `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/9uASADiYe_8" allowfullscreen></iframe><br>Some content with youtube ID <script type="text/javascript" data-reactid="6">window.__APOLLO_STATE__ = {youtube_id: "9uASADiYe_8"}</script>`,

// Test with multiple YouTube IDs
`Content with youtube_id: "dQw4w9WgXcQ" and youtube_id: "jNQXAC9IVRw"`: `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br><iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/jNQXAC9IVRw" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Content with youtube_id: "dQw4w9WgXcQ" and youtube_id: "jNQXAC9IVRw"`,
`Content with youtube_id: "dQw4w9WgXcQ" and youtube_id: "jNQXAC9IVRw"`: `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ" allowfullscreen></iframe><br><iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/jNQXAC9IVRw" allowfullscreen></iframe><br>Content with youtube_id: "dQw4w9WgXcQ" and youtube_id: "jNQXAC9IVRw"`,

// Test with YouTube ID using equals sign
`Some content with youtube_id = "dQw4w9WgXcQ"`: `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Some content with youtube_id = "dQw4w9WgXcQ"`,
`Some content with youtube_id = "dQw4w9WgXcQ"`: `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ" allowfullscreen></iframe><br>Some content with youtube_id = "dQw4w9WgXcQ"`,

// Test with spaces around delimiters
`Some content with youtube_id : "dQw4w9WgXcQ"`: `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Some content with youtube_id : "dQw4w9WgXcQ"`,
`Some content with youtube_id : "dQw4w9WgXcQ"`: `<iframe width="650" height="350" frameborder="0" src="https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ" allowfullscreen></iframe><br>Some content with youtube_id : "dQw4w9WgXcQ"`,

// Test with YouTube ID without quotes (regex requires quotes)
`Some content with youtube_id: dQw4w9WgXcQ and more`: `Some content with youtube_id: dQw4w9WgXcQ and more`,
Expand Down Expand Up @@ -245,7 +245,7 @@ func TestAddYoutubeVideoFromIdWithCustomEmbedURL(t *testing.T) {
}

input := `Some content with youtube_id: "dQw4w9WgXcQ"`
expected := `<iframe width="650" height="350" frameborder="0" src="https://invidious.custom/embed/dQw4w9WgXcQ" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Some content with youtube_id: "dQw4w9WgXcQ"`
expected := `<iframe width="650" height="350" frameborder="0" src="https://invidious.custom/embed/dQw4w9WgXcQ" allowfullscreen></iframe><br>Some content with youtube_id: "dQw4w9WgXcQ"`

actual := addYoutubeVideoFromId(input)
if actual != expected {
Expand All @@ -260,35 +260,35 @@ func TestAddInvidiousVideo(t *testing.T) {
// Test with various Invidious instances
"https://invidious.io/watch?v=dQw4w9WgXcQ": {
"Some video content",
`<iframe width="650" height="350" frameborder="0" src="https://invidious.io/embed/dQw4w9WgXcQ" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Some video content`,
`<iframe width="650" height="350" frameborder="0" src="https://invidious.io/embed/dQw4w9WgXcQ" allowfullscreen></iframe><br>Some video content`,
},
"https://yewtu.be/watch?v=jNQXAC9IVRw": {
"Another video description",
`<iframe width="650" height="350" frameborder="0" src="https://yewtu.be/embed/jNQXAC9IVRw" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Another video description`,
`<iframe width="650" height="350" frameborder="0" src="https://yewtu.be/embed/jNQXAC9IVRw" allowfullscreen></iframe><br>Another video description`,
},
"http://invidious.snopyta.org/watch?v=dQw4w9WgXcQ": {
"HTTP instance test",
`<iframe width="650" height="350" frameborder="0" src="https://invidious.snopyta.org/embed/dQw4w9WgXcQ" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>HTTP instance test`,
`<iframe width="650" height="350" frameborder="0" src="https://invidious.snopyta.org/embed/dQw4w9WgXcQ" allowfullscreen></iframe><br>HTTP instance test`,
},
"https://youtube.com/watch?v=dQw4w9WgXcQ": {
"YouTube URL (also matches regex)",
`<iframe width="650" height="350" frameborder="0" src="https://youtube.com/embed/dQw4w9WgXcQ" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>YouTube URL (also matches regex)`,
`<iframe width="650" height="350" frameborder="0" src="https://youtube.com/embed/dQw4w9WgXcQ" allowfullscreen></iframe><br>YouTube URL (also matches regex)`,
},
"https://example.org/watch?v=dQw4w9WgXcQ": {
"Any domain with watch pattern",
`<iframe width="650" height="350" frameborder="0" src="https://example.org/embed/dQw4w9WgXcQ" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Any domain with watch pattern`,
`<iframe width="650" height="350" frameborder="0" src="https://example.org/embed/dQw4w9WgXcQ" allowfullscreen></iframe><br>Any domain with watch pattern`,
},

// Test with query parameters
"https://invidious.io/watch?v=dQw4w9WgXcQ&t=30s": {
"Video with timestamp",
`<iframe width="650" height="350" frameborder="0" src="https://invidious.io/embed/dQw4w9WgXcQ?t=30s" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Video with timestamp`,
`<iframe width="650" height="350" frameborder="0" src="https://invidious.io/embed/dQw4w9WgXcQ?t=30s" allowfullscreen></iframe><br>Video with timestamp`,
},

// Test with more complex query parameters
"https://invidious.io/watch?v=dQw4w9WgXcQ&t=30s&autoplay=1": {
"Video with multiple parameters",
`<iframe width="650" height="350" frameborder="0" src="https://invidious.io/embed/dQw4w9WgXcQ?autoplay=1&t=30s" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>Video with multiple parameters`,
`<iframe width="650" height="350" frameborder="0" src="https://invidious.io/embed/dQw4w9WgXcQ?autoplay=1&t=30s" allowfullscreen></iframe><br>Video with multiple parameters`,
},

// Test with non-matching URLs (should return content unchanged)
Expand All @@ -308,7 +308,7 @@ func TestAddInvidiousVideo(t *testing.T) {
// Test with empty content
"https://empty.invidious.io/watch?v=dQw4w9WgXcQ": {
"",
`<iframe width="650" height="350" frameborder="0" src="https://empty.invidious.io/embed/dQw4w9WgXcQ" allowfullscreen referrerpolicy="strict-origin-when-cross-origin"></iframe><br>`,
`<iframe width="650" height="350" frameborder="0" src="https://empty.invidious.io/embed/dQw4w9WgXcQ" allowfullscreen></iframe><br>`,
},
}

Expand Down
40 changes: 30 additions & 10 deletions internal/reader/sanitizer/sanitizer.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ var (
"h5": {"id"},
"h6": {"id"},
"hr": {},
"iframe": {"width", "height", "frameborder", "src", "allowfullscreen", "referrerpolicy"},
"iframe": {"width", "height", "frameborder", "src", "allowfullscreen"},
"img": {"alt", "title", "src", "srcset", "sizes", "width", "height", "fetchpriority", "decoding"},
"ins": {},
"kbd": {},
Expand Down Expand Up @@ -309,6 +309,7 @@ func sanitizeAttributes(parsedBaseUrl *url.URL, tagName string, attributes []htm
attrNames := make([]string, 0, len(attributes))
var err error
var isAnchorLink bool
var isYouTubeEmbed bool

for _, attribute := range attributes {
if !isValidAttribute(tagName, attribute.Key) {
Expand Down Expand Up @@ -355,10 +356,16 @@ func sanitizeAttributes(parsedBaseUrl *url.URL, tagName string, attributes []htm
if isExternalResourceAttribute(attribute.Key) {
switch {
case tagName == "iframe":
if !isValidIframeSource(attribute.Val) {
iframeSourceDomain, trustedIframeDomain := findAllowedIframeSourceDomain(attribute.Val)
if !trustedIframeDomain {
continue
}

value = rewriteIframeURL(attribute.Val)

if iframeSourceDomain == "youtube.com" || iframeSourceDomain == "youtube-nocookie.com" {
isYouTubeEmbed = true
}
case tagName == "img" && attribute.Key == "src" && isValidDataAttribute(attribute.Val):
value = attribute.Val
case tagName == "a" && attribute.Key == "href" && strings.HasPrefix(attribute.Val, "#"):
Expand Down Expand Up @@ -387,7 +394,7 @@ func sanitizeAttributes(parsedBaseUrl *url.URL, tagName string, attributes []htm
}

if !isAnchorLink {
extraAttrNames, extraHTMLAttributes := getExtraAttributes(tagName, sanitizerOptions)
extraAttrNames, extraHTMLAttributes := getExtraAttributes(tagName, isYouTubeEmbed, sanitizerOptions)
if len(extraAttrNames) > 0 {
attrNames = append(attrNames, extraAttrNames...)
htmlAttrs = append(htmlAttrs, extraHTMLAttributes...)
Expand All @@ -397,7 +404,7 @@ func sanitizeAttributes(parsedBaseUrl *url.URL, tagName string, attributes []htm
return attrNames, strings.Join(htmlAttrs, " ")
}

func getExtraAttributes(tagName string, sanitizerOptions *SanitizerOptions) ([]string, []string) {
func getExtraAttributes(tagName string, isYouTubeEmbed bool, sanitizerOptions *SanitizerOptions) ([]string, []string) {
switch tagName {
case "a":
attributeNames := []string{"rel", "referrerpolicy"}
Expand All @@ -410,7 +417,20 @@ func getExtraAttributes(tagName string, sanitizerOptions *SanitizerOptions) ([]s
case "video", "audio":
return []string{"controls"}, []string{"controls"}
case "iframe":
return []string{"sandbox", "loading"}, []string{`sandbox="allow-scripts allow-same-origin allow-popups allow-popups-to-escape-sandbox"`, `loading="lazy"`}
extraAttrNames := []string{}
extraHTMLAttributes := []string{}

// Note: the referrerpolicy seems to be required to avoid YouTube error 153 video player configuration error
// See https://developers.google.com/youtube/terms/required-minimum-functionality#embedded-player-api-client-identity
if isYouTubeEmbed {
extraAttrNames = append(extraAttrNames, "referrerpolicy")
extraHTMLAttributes = append(extraHTMLAttributes, `referrerpolicy="strict-origin-when-cross-origin"`)
}

extraAttrNames = append(extraAttrNames, "sandbox", "loading")
extraHTMLAttributes = append(extraHTMLAttributes, `sandbox="allow-scripts allow-same-origin allow-popups allow-popups-to-escape-sandbox"`, `loading="lazy"`)

return extraAttrNames, extraHTMLAttributes
case "img":
return []string{"loading"}, []string{`loading="lazy"`}
default:
Expand Down Expand Up @@ -496,22 +516,22 @@ func isBlockedResource(absoluteURL string) bool {
return false
}

func isValidIframeSource(iframeSourceURL string) bool {
func findAllowedIframeSourceDomain(iframeSourceURL string) (string, bool) {
iframeSourceDomain := urllib.DomainWithoutWWW(iframeSourceURL)

if _, ok := iframeAllowList[iframeSourceDomain]; ok {
return true
return iframeSourceDomain, true
}

if ytDomain := config.Opts.YouTubeEmbedDomain(); ytDomain != "" && iframeSourceDomain == strings.TrimPrefix(ytDomain, "www.") {
return true
return iframeSourceDomain, true
}

if invidiousInstance := config.Opts.InvidiousInstance(); invidiousInstance != "" && iframeSourceDomain == strings.TrimPrefix(invidiousInstance, "www.") {
return true
return iframeSourceDomain, true
}

return false
return "", false
}

func rewriteIframeURL(link string) string {
Expand Down
Loading
0