8000 [pull] main from miniflux:main by pull[bot] · Pull Request #243 · suhaoGe/miniflux · GitHub
[go: up one dir, main page]

Skip to content
Merged 8000
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ integration-test:
RUN_MIGRATIONS=1 \
LOG_LEVEL=debug \
FETCHER_ALLOW_PRIVATE_NETWORKS=1 \
INTEGRATION_ALLOW_PRIVATE_NETWORKS=1 \
go run main.go >/tmp/miniflux.log 2>&1 & echo "$$!" > "/tmp/miniflux.pid"

while ! nc -z localhost 8080; do sleep 1; done
Expand Down
12 changes: 12 additions & 0 deletions internal/config/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,11 @@ func NewConfigOptions() *configOptions {
rawValue: "0",
valueType: boolType,
},
"INTEGRATION_ALLOW_PRIVATE_NETWORKS": {
parsedBoolValue: false,
rawValue: "0",
valueType: boolType,
},
"INVIDIOUS_INSTANCE": {
parsedStringValue: "yewtu.be",
rawValue: "yewtu.be",
Expand Down Expand Up @@ -790,6 +795,13 @@ func (c *configOptions) FetcherAllowPrivateNetworks() bool {
return c.options["FETCHER_ALLOW_PRIVATE_NETWORKS"].parsedBoolValue
}

func (c *configOptions) IntegrationAllowPrivateNetworks() bool {
if c == nil {
return false
}
return c.options["INTEGRATION_ALLOW_PRIVATE_NETWORKS"].parsedBoolValue
}

func (c *configOptions) InvidiousInstance() string {
return c.options["INVIDIOUS_INSTANCE"].parsedStringValue
}
Expand Down
24 changes: 24 additions & 0 deletions internal/config/options_parsing_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1375,6 +1375,30 @@ func TestFetcherAllowPrivateNetworksOptionParsing(t *testing.T) {
}
}

func TestIntegrationAllowPrivateNetworksOptionParsing(t *testing.T) {
configParser := NewConfigParser()

if configParser.options.IntegrationAllowPrivateNetworks() {
t.Fatalf("Expected INTEGRATION_ALLOW_PRIVATE_NETWORKS to be disabled by default")
}

if err := configParser.parseLines([]string{"INTEGRATION_ALLOW_PRIVATE_NETWORKS=1"}); err != nil {
t.Fatalf("Unexpected error: %v", err)
}

if !configParser.options.IntegrationAllowPrivateNetworks() {
t.Fatalf("Expected INTEGRATION_ALLOW_PRIVATE_NETWORKS to be enabled")
}

if err := configParser.parseLines([]string{"INTEGRATION_ALLOW_PRIVATE_NETWORKS=0"}); err != nil {
t.Fatalf("Unexpected error: %v", err)
}

if configParser.options.IntegrationAllowPrivateNetworks() {
t.Fatalf("Expected INTEGRATION_ALLOW_PRIVATE_NETWORKS to be disabled")
}
}

func TestHTTPServerTimeoutOptionParsing(t *testing.T) {
configParser := NewConfigParser()

Expand Down
70 changes: 70 additions & 0 deletions internal/http/client/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package client // import "miniflux.app/v2/internal/http/client"

import (
"context"
"errors"
"fmt"
"net"
"net/http"
"time"

"miniflux.app/v2/internal/urllib"
)

// ErrPrivateNetwork is returned when a connection to a private network is blocked.
var ErrPrivateNetwork = errors.New("client: connection to private network is blocked")

// Options holds configuration for creating an HTTP client.
type Options struct {
Timeout time.Duration
BlockPrivateNetworks bool
}

// NewClientWithOptions creates a new HTTP client with the specified options.
func NewClientWithOptions(opts Options) *http.Client {
if !opts.BlockPrivateNetworks {
return &http.Client{Timeout: opts.Timeout}
}

dialer := &net.Dialer{
Timeout: opts.Timeout,
}

transport := &http.Transport{
// The check is performed at connect time on the actual resolved IP, which eliminates TOCTOU / DNS-rebinding vulnerabilities.
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, fmt.Errorf("client: unable to parse address %q: %w", addr, err)
}

ips, err := net.LookupIP(host)
if err != nil {
return nil, fmt.Errorf("client: unable to resolve host %q: %w", host, err)
}

var safeIP net.IP
for _, ip := range ips {
if !urllib.IsNonPublicIP(ip) {
safeIP = ip
break
}
}

if safeIP == nil {
return nil, fmt.Errorf("%w: host %q resolves to a non-public IP address", ErrPrivateNetwork, host)
}

safeAddr := net.JoinHostPort(safeIP.String(), port)
return dialer.DialContext(ctx, network, safeAddr)
},
}

return &http.Client{
Timeout: opts.Timeout,
Transport: transport,
}
}
113 changes: 113 additions & 0 deletions internal/http/client/client_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package client

import (
"errors"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
)

func TestNewClientWithoutBlockingPrivateNetworks(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer server.Close()

client := NewClientWithOptions(Options{Timeout: 5 * time.Second})
resp, err := client.Get(server.URL)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
t.Fatalf("Expected status 200, got %d", resp.StatusCode)
}
}

func TestBlockPrivateNetworksBlocksLoopback(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer server.Close()

client := NewClientWithOptions(Options{Timeout: 5 * time.Second, BlockPrivateNetworks: true})
_, err := client.Get(server.URL)
if err == nil {
t.Fatal("Expected an error when connecting to loopback address, got nil")
}

if !errors.Is(err, ErrPrivateNetwork) {
t.Fatalf("Expected ErrPrivateNetwork, got %v", err)
}
}

func TestBlockPrivateNetworksAllowsPublicIPs(t *testing.T) {
client := NewClientWithOptions(Options{Timeout: 5 * time.Second, BlockPrivateNetworks: true})
if client == nil {
t.Fatal("Expected non-nil client")
}

transport, ok := client.Transport.(*http.Transport)
if !ok {
t.Fatal("Expected custom http.Transport when blockPrivateNetworks is true")
}
if transport.DialContext == nil {
t.Fatal("Expected custom DialContext when blockPrivateNetworks is true")
}
}

func TestNoCustomTransportWhenNotBlocking(t *testing.T) {
client := NewClientWithOptions(Options{Timeout: 5 * time.Second})
if client.Transport != nil {
t.Fatal("Expected nil transport when blockPrivateNetworks is false")
}
}

func TestBlockPrivateNetworksBlocksPrivateIP(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("Failed to create listener: %v", err)
}
defer listener.Close()

server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
server.Listener = listener
server.Start()
defer server.Close()

client := NewClientWithOptions(Options{Timeout: 5 * time.Second, BlockPrivateNetworks: true})
_, err = client.Get(server.URL)
if err == nil {
t.Fatal("Expected error when connecting to private IP")
}

if !errors.Is(err, ErrPrivateNetwork) {
t.Fatalf("Expected ErrPrivateNetwork, got: %v", err)
}
}

func TestBlockPrivateNetworksAllowsLoopbackWhenDisabled(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer server.Close()

client := NewClientWithOptions(Options{Timeout: 5 * time.Second})
resp, err := client.Get(server.URL)
if err != nil {
t.Fatalf("Expected no error when blockPrivateNetworks is false, got %v", err)
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
t.Fatalf("Expected status 200, got %d", resp.StatusCode)
}
}
8 changes: 5 additions & 3 deletions internal/integration/apprise/apprise.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import (
"net/http"
"time"

"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/http/client"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/urllib"
"miniflux.app/v2/internal/version"
Expand All @@ -25,7 +27,7 @@ type Client struct {
}

func NewClient(serviceURL, baseURL string) *Client {
return &Client{serviceURL, baseURL}
return &Client{servicesURL: serviceURL, baseURL: baseURL}
}

func (c *Client) SendNotification(feed *model.Feed, entries model.Entries) error {
Expand Down Expand Up @@ -65,12 +67,12 @@ func (c *Client) SendNotification(feed *model.Feed, entries model.Entries) error
slog.String("entry_url", entry.URL),
)

httpClient := &http.Client{Timeout: defaultClientTimeout}
httpClient := client.NewClientWithOptions(client.Options{Timeout: defaultClientTimeout, BlockPrivateNetworks: !config.Opts.IntegrationAllowPrivateNetworks()})
response, err := httpClient.Do(request)
if err != nil {
return fmt.Errorf("apprise: unable to send request: %v", err)
}
response.Body.Close()
defer response.Body.Close()

if response.StatusCode >= 400 {
return fmt.Errorf("apprise: unable to send a notification: url=%s status=%d", apiEndpoint, response.StatusCode)
Expand Down
51 changes: 28 additions & 23 deletions internal/integration/archiveorg/archiveorg.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,17 @@
package archiveorg

import (
"log/slog"
"fmt"
"net/http"
"net/url"
"time"

"miniflux.app/v2/internal/http/client"
"miniflux.app/v2/internal/version"
)

const defaultClientTimeout = 30 * time.Second

// See https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA/edit?tab=t.0
const options = "delay_wb_availability=1&if_not_archived_within=15d"

Expand All @@ -18,26 +24,25 @@ func NewClient() *Client {
return &Client{}
}

func (c *Client) SendURL(entryURL, title string) {
// We're using a goroutine here as submissions to archive.org might take a long time
// and trigger a timeout on miniflux' side.
go func(entryURL string) {
res, err := http.Get("https://web.archive.org/save/" + url.QueryEscape(entryURL) + "?" + options)
if err != nil {
slog.Error("archiveorg: unable to send request: %v",
slog.Any("err", err),
slog.String("title", title),
slog.String("url", entryURL),
)
return
}
if res.StatusCode > 299 {
slog.Error("archiveorg: failed with status code",
slog.String("title", title),
slog.String("url", entryURL),
slog.Int("code", res.StatusCode),
)
}
res.Body.Close()
}(entryURL)
func (c *Client) SendURL(entryURL string) error {
requestURL := "https://web.archive.org/save/" + url.QueryEscape(entryURL) + "?" + options
request, err := http.NewRequest(http.MethodGet, requestURL, nil)
if err != nil {
return fmt.Errorf("archiveorg: unable to create request: %v", err)
}

request.Header.Set("User-Agent", "Miniflux/"+version.Version)

httpClient := client.NewClientWithOptions(client.Options{Timeout: defaultClientTimeout})
response, err := httpClient.Do(request)
if err != nil {
return fmt.Errorf("archiveorg: unable to send request: %v", err)
}
defer response.Body.Close()

if response.StatusCode >= 400 {
return fmt.Errorf("archiveorg: unexpected status code: url=%s status=%d", requestURL, response.StatusCode)
}

return nil
}
5 changes: 3 additions & 2 deletions internal/integration/betula/betula.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
"strings"
"time"

"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/http/client"
"miniflux.app/v2/internal/urllib"
"miniflux.app/v2/internal/version"
)
Expand Down Expand Up @@ -41,11 +43,10 @@ func (c *Client) CreateBookmark(entryURL, entryTitle string, tags []string) erro
return fmt.Errorf("betula: unable to create request: %v", err)
}

request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
request.Header.Set("User-Agent", "Miniflux/"+version.Version)
request.AddCookie(&http.Cookie{Name: "betula-token", Value: c.token})

httpClient := &http.Client{Timeout: defaultClientTimeout}
httpClient := client.NewClientWithOptions(client.Options{Timeout: defaultClientTimeout, BlockPrivateNetworks: !config.Opts.IntegrationAllowPrivateNetworks()})
response, err := httpClient.Do(request)
if err != nil {
return fmt.Errorf("betula: unable to send request: %v", err)
Expand Down
4 changes: 3 additions & 1 deletion internal/integration/cubox/cubox.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import (
"net/http"
"time"

"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/http/client"
"miniflux.app/v2/internal/version"
)

Expand Down Expand Up @@ -51,7 +53,7 @@ func (c *Client) SaveLink(entryURL string) error {
request.Header.Set("Content-Type", "application/json")
request.Header.Set("User-Agent", "Miniflux/"+version.Version)

response, err := http.DefaultClient.Do(request)
response, err := client.NewClientWithOptions(client.Options{Timeout: defaultClientTimeout, BlockPrivateNetworks: !config.Opts.IntegrationAllowPrivateNetworks()}).Do(request)
if err != nil {
return fmt.Errorf("cubox: unable to send request: %w", err)
}
Expand Down
Loading
0