~ttt/minifluxlite

bb720c87c191efe36a328d95a918f75df51d4976 — Frédéric Guillot 4 years ago 228862f
Make HTTP Client timeout and max body size configurable
5 files changed, 117 insertions(+), 33 deletions(-)

M config/config_test.go
M config/options.go
M config/parser.go
M http/client/client.go
M miniflux.1
M config/config_test.go => config/config_test.go +66 -0
@@ 984,3 984,69 @@ func TestHTTPSOn(t *testing.T) {
		t.Fatalf(`Unexpected HTTPS value, got "%v"`, opts.HTTPS)
	}
}

func TestHTTPClientTimeout(t *testing.T) {
	os.Clearenv()
	os.Setenv("HTTP_CLIENT_TIMEOUT", "42")

	opts, err := parse()
	if err != nil {
		t.Fatalf(`Parsing failure: %q`, err)
	}

	expected := 42
	result := opts.HTTPClientTimeout()

	if result != expected {
		t.Fatalf(`Unexpected HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected)
	}
}

func TestDefaultHTTPClientTimeoutValue(t *testing.T) {
	os.Clearenv()

	opts, err := parse()
	if err != nil {
		t.Fatalf(`Parsing failure: %q`, err)
	}

	expected := defaultHTTPClientTimeout
	result := opts.HTTPClientTimeout()

	if result != expected {
		t.Fatalf(`Unexpected HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected)
	}
}

func TestHTTPClientMaxBodySize(t *testing.T) {
	os.Clearenv()
	os.Setenv("HTTP_CLIENT_MAX_BODY_SIZE", "42")

	opts, err := parse()
	if err != nil {
		t.Fatalf(`Parsing failure: %q`, err)
	}

	expected := int64(42 * 1024 * 1024)
	result := opts.HTTPClientMaxBodySize()

	if result != expected {
		t.Fatalf(`Unexpected HTTP_CLIENT_MAX_BODY_SIZE value, got %d instead of %d`, result, expected)
	}
}

func TestDefaultHTTPClientMaxBodySizeValue(t *testing.T) {
	os.Clearenv()

	opts, err := parse()
	if err != nil {
		t.Fatalf(`Parsing failure: %q`, err)
	}

	expected := int64(defaultHTTPClientMaxBodySize * 1024 * 1024)
	result := opts.HTTPClientMaxBodySize()

	if result != expected {
		t.Fatalf(`Unexpected HTTP_CLIENT_MAX_BODY_SIZE value, got %d instead of %d`, result, expected)
	}
}

M config/options.go => config/options.go +33 -19
@@ 5,25 5,27 @@
package config // import "miniflux.app/config"

const (
	defaultBaseURL            = "http://localhost"
	defaultWorkerPoolSize     = 5
	defaultPollingFrequency   = 60
	defaultBatchSize          = 10
	defaultDatabaseURL        = "user=postgres password=postgres dbname=miniflux2 sslmode=disable"
	defaultDatabaseMaxConns   = 20
	defaultDatabaseMinConns   = 1
	defaultArchiveReadDays    = 60
	defaultListenAddr         = "127.0.0.1:8080"
	defaultCertFile           = ""
	defaultKeyFile            = ""
	defaultCertDomain         = ""
	defaultCertCache          = "/tmp/cert_cache"
	defaultCleanupFrequency   = 24
	defaultProxyImages        = "http-only"
	defaultOAuth2ClientID     = ""
	defaultOAuth2ClientSecret = ""
	defaultOAuth2RedirectURL  = ""
	defaultOAuth2Provider     = ""
	defaultBaseURL               = "http://localhost"
	defaultWorkerPoolSize        = 5
	defaultPollingFrequency      = 60
	defaultBatchSize             = 10
	defaultDatabaseURL           = "user=postgres password=postgres dbname=miniflux2 sslmode=disable"
	defaultDatabaseMaxConns      = 20
	defaultDatabaseMinConns      = 1
	defaultArchiveReadDays       = 60
	defaultListenAddr            = "127.0.0.1:8080"
	defaultCertFile              = ""
	defaultKeyFile               = ""
	defaultCertDomain            = ""
	defaultCertCache             = "/tmp/cert_cache"
	defaultCleanupFrequency      = 24
	defaultProxyImages           = "http-only"
	defaultOAuth2ClientID        = ""
	defaultOAuth2ClientSecret    = ""
	defaultOAuth2RedirectURL     = ""
	defaultOAuth2Provider        = ""
	defaultHTTPClientTimeout     = 20
	defaultHTTPClientMaxBodySize = 15
)

// Options contains configuration options.


@@ 58,6 60,8 @@ type Options struct {
	oauth2RedirectURL         string
	oauth2Provider            string
	pocketConsumerKey         string
	httpClientTimeout         int
	httpClientMaxBodySize     int64
}

// HasDebugMode returns true if debug mode is enabled.


@@ 212,3 216,13 @@ func (o *Options) PocketConsumerKey(defaultValue string) string {
	}
	return defaultValue
}

// HTTPClientTimeout returns the time limit in seconds before the HTTP client cancel the request.
func (o *Options) HTTPClientTimeout() int {
	return o.httpClientTimeout
}

// HTTPClientMaxBodySize returns the number of bytes allowed for the HTTP client to transfer.
func (o *Options) HTTPClientMaxBodySize() int64 {
	return o.httpClientMaxBodySize
}

M config/parser.go => config/parser.go +4 -3
@@ 45,6 45,8 @@ func parse() (opts *Options, err error) {
	opts.batchSize = getIntValue("BATCH_SIZE", defaultBatchSize)
	opts.archiveReadDays = getIntValue("ARCHIVE_READ_DAYS", defaultArchiveReadDays)
	opts.proxyImages = getStringValue("PROXY_IMAGES", defaultProxyImages)
	opts.createAdmin = getBooleanValue("CREATE_ADMIN")
	opts.pocketConsumerKey = getStringValue("POCKET_CONSUMER_KEY", "")

	opts.oauth2UserCreationAllowed = getBooleanValue("OAUTH2_USER_CREATION")
	opts.oauth2ClientID = getStringValue("OAUTH2_CLIENT_ID", defaultOAuth2ClientID)


@@ 52,9 54,8 @@ func parse() (opts *Options, err error) {
	opts.oauth2RedirectURL = getStringValue("OAUTH2_REDIRECT_URL", defaultOAuth2RedirectURL)
	opts.oauth2Provider = getStringValue("OAUTH2_PROVIDER", defaultOAuth2Provider)

	opts.pocketConsumerKey = getStringValue("POCKET_CONSUMER_KEY", "")

	opts.createAdmin = getBooleanValue("CREATE_ADMIN")
	opts.httpClientTimeout = getIntValue("HTTP_CLIENT_TIMEOUT", defaultHTTPClientTimeout)
	opts.httpClientMaxBodySize = int64(getIntValue("HTTP_CLIENT_MAX_BODY_SIZE", defaultHTTPClientMaxBodySize) * 1024 * 1024)

	return opts, nil
}

M http/client/client.go => http/client/client.go +4 -11
@@ 18,20 18,13 @@ import (
	"strings"
	"time"

	"miniflux.app/config"
	"miniflux.app/errors"
	"miniflux.app/logger"
	"miniflux.app/timer"
	"miniflux.app/version"
)

const (
	// 20 seconds max.
	requestTimeout = 20

	// 15MB max.
	maxBodySize = 1024 * 1024 * 15
)

var (
	// DefaultUserAgent sets the User-Agent header used for any requests by miniflux.
	DefaultUserAgent = "Mozilla/5.0 (compatible; Miniflux/" + version.Version + "; +https://miniflux.app)"


@@ 144,7 137,7 @@ func (c *Client) executeRequest(request *http.Request) (*Response, error) {
			case net.Error:
				nerr := uerr.Err.(net.Error)
				if nerr.Timeout() {
					err = errors.NewLocalizedError(errRequestTimeout, requestTimeout)
					err = errors.NewLocalizedError(errRequestTimeout, config.Opts.HTTPClientTimeout())
				} else if nerr.Temporary() {
					err = errors.NewLocalizedError(errTemporaryNetworkOperation, nerr)
				}


@@ 154,7 147,7 @@ func (c *Client) executeRequest(request *http.Request) (*Response, error) {
		return nil, err
	}

	if resp.ContentLength > maxBodySize {
	if resp.ContentLength > config.Opts.HTTPClientMaxBodySize() {
		return nil, fmt.Errorf("client: response too large (%d bytes)", resp.ContentLength)
	}



@@ 212,7 205,7 @@ func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, err
}

func (c *Client) buildClient() http.Client {
	client := http.Client{Timeout: time.Duration(requestTimeout * time.Second)}
	client := http.Client{Timeout: time.Duration(config.Opts.HTTPClientTimeout()) * time.Second}
	if c.Insecure {
		client.Transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},

M miniflux.1 => miniflux.1 +10 -0
@@ 169,6 169,16 @@ Pocket consumer API key for all users\&.
Avoids mixed content warnings for external images: http-only, all, or none\&.
.br
Default is http-only\&.
.TP
.B HTTP_CLIENT_TIMEOUT
Time limit in seconds before the HTTP client cancel the request\&.
.br
Default is 20 seconds\&.
.TP
.B HTTP_CLIENT_MAX_BODY_SIZE
Maximum body size for HTTP requests in Mebibyte (MiB)\&.
.br
Default is 20 MiB\&.

.SH AUTHORS
.sp