Initial commit: Open sourcing all of the Maple Open Technologies code.
This commit is contained in:
commit
755d54a99d
2010 changed files with 448675 additions and 0 deletions
|
|
@ -0,0 +1,199 @@
|
|||
package httpclient
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Service provides an HTTP client with proper timeouts.
|
||||
// This addresses OWASP security concern B1: using http.DefaultClient which has
|
||||
// no timeouts and can be vulnerable to slowloris attacks and resource exhaustion.
|
||||
//
|
||||
// Note: TLS/SSL is handled by Caddy reverse proxy in production (see OWASP report
|
||||
// A04-4.1 "Certificate Pinning Not Required" - BY DESIGN). This service focuses
|
||||
// on adding timeouts, not TLS configuration.
|
||||
//
|
||||
// For large file downloads, use DoDownloadNoTimeout() which relies on the request's
|
||||
// context for cancellation instead of a fixed timeout. This allows multi-gigabyte
|
||||
// files to download without timeout issues while still being cancellable.
|
||||
type Service struct {
|
||||
// client is the configured HTTP client for API requests
|
||||
client *http.Client
|
||||
|
||||
// downloadClient is a separate client for file downloads with longer timeouts
|
||||
downloadClient *http.Client
|
||||
|
||||
// noTimeoutClient is for large file downloads where context controls cancellation
|
||||
noTimeoutClient *http.Client
|
||||
}
|
||||
|
||||
// Config holds configuration options for the HTTP client service
|
||||
type Config struct {
|
||||
// RequestTimeout is the overall timeout for API requests (default: 30s)
|
||||
RequestTimeout time.Duration
|
||||
|
||||
// DownloadTimeout is the overall timeout for file downloads (default: 10m)
|
||||
DownloadTimeout time.Duration
|
||||
|
||||
// ConnectTimeout is the timeout for establishing connections (default: 10s)
|
||||
ConnectTimeout time.Duration
|
||||
|
||||
// TLSHandshakeTimeout is the timeout for TLS handshake (default: 10s)
|
||||
TLSHandshakeTimeout time.Duration
|
||||
|
||||
// IdleConnTimeout is how long idle connections stay in the pool (default: 90s)
|
||||
IdleConnTimeout time.Duration
|
||||
|
||||
// MaxIdleConns is the max number of idle connections (default: 100)
|
||||
MaxIdleConns int
|
||||
|
||||
// MaxIdleConnsPerHost is the max idle connections per host (default: 10)
|
||||
MaxIdleConnsPerHost int
|
||||
}
|
||||
|
||||
// DefaultConfig returns sensible default configuration values
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
RequestTimeout: 30 * time.Second,
|
||||
DownloadTimeout: 10 * time.Minute,
|
||||
ConnectTimeout: 10 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
MaxIdleConns: 100,
|
||||
MaxIdleConnsPerHost: 10,
|
||||
}
|
||||
}
|
||||
|
||||
// ProvideService creates a new HTTP client service with secure defaults
|
||||
func ProvideService() *Service {
|
||||
return NewService(DefaultConfig())
|
||||
}
|
||||
|
||||
// NewService creates a new HTTP client service with the given configuration
|
||||
func NewService(cfg Config) *Service {
|
||||
// Create transport with timeouts and connection pooling
|
||||
// Note: We don't set TLSClientConfig - Go's defaults are secure and
|
||||
// production uses Caddy for TLS termination anyway
|
||||
transport := &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: cfg.ConnectTimeout,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
TLSHandshakeTimeout: cfg.TLSHandshakeTimeout,
|
||||
IdleConnTimeout: cfg.IdleConnTimeout,
|
||||
MaxIdleConns: cfg.MaxIdleConns,
|
||||
MaxIdleConnsPerHost: cfg.MaxIdleConnsPerHost,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
ForceAttemptHTTP2: true,
|
||||
}
|
||||
|
||||
// Create the main client for API requests
|
||||
client := &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: cfg.RequestTimeout,
|
||||
}
|
||||
|
||||
// Create a separate transport for downloads with longer timeouts
|
||||
downloadTransport := &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: cfg.ConnectTimeout,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
TLSHandshakeTimeout: cfg.TLSHandshakeTimeout,
|
||||
IdleConnTimeout: cfg.IdleConnTimeout,
|
||||
MaxIdleConns: cfg.MaxIdleConns,
|
||||
MaxIdleConnsPerHost: cfg.MaxIdleConnsPerHost,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
ForceAttemptHTTP2: true,
|
||||
// Disable compression for downloads to avoid decompression overhead
|
||||
DisableCompression: true,
|
||||
}
|
||||
|
||||
// Create the download client with longer timeout
|
||||
downloadClient := &http.Client{
|
||||
Transport: downloadTransport,
|
||||
Timeout: cfg.DownloadTimeout,
|
||||
}
|
||||
|
||||
// Create a no-timeout transport for large file downloads
|
||||
// This client has no overall timeout - cancellation is controlled via request context
|
||||
// Connection and TLS handshake still have timeouts to prevent hanging on initial connect
|
||||
noTimeoutTransport := &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: cfg.ConnectTimeout,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
TLSHandshakeTimeout: cfg.TLSHandshakeTimeout,
|
||||
IdleConnTimeout: cfg.IdleConnTimeout,
|
||||
MaxIdleConns: cfg.MaxIdleConns,
|
||||
MaxIdleConnsPerHost: cfg.MaxIdleConnsPerHost,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
ForceAttemptHTTP2: true,
|
||||
DisableCompression: true,
|
||||
}
|
||||
|
||||
// No timeout - relies on context cancellation for large file downloads
|
||||
noTimeoutClient := &http.Client{
|
||||
Transport: noTimeoutTransport,
|
||||
Timeout: 0, // No timeout
|
||||
}
|
||||
|
||||
return &Service{
|
||||
client: client,
|
||||
downloadClient: downloadClient,
|
||||
noTimeoutClient: noTimeoutClient,
|
||||
}
|
||||
}
|
||||
|
||||
// Client returns the HTTP client for API requests (30s timeout)
|
||||
func (s *Service) Client() *http.Client {
|
||||
return s.client
|
||||
}
|
||||
|
||||
// DownloadClient returns the HTTP client for file downloads (10m timeout)
|
||||
func (s *Service) DownloadClient() *http.Client {
|
||||
return s.downloadClient
|
||||
}
|
||||
|
||||
// Do executes an HTTP request using the API client
|
||||
func (s *Service) Do(req *http.Request) (*http.Response, error) {
|
||||
return s.client.Do(req)
|
||||
}
|
||||
|
||||
// DoDownload executes an HTTP request using the download client (longer timeout)
|
||||
func (s *Service) DoDownload(req *http.Request) (*http.Response, error) {
|
||||
return s.downloadClient.Do(req)
|
||||
}
|
||||
|
||||
// Get performs an HTTP GET request using the API client
|
||||
func (s *Service) Get(url string) (*http.Response, error) {
|
||||
return s.client.Get(url)
|
||||
}
|
||||
|
||||
// GetDownload performs an HTTP GET request using the download client (longer timeout)
|
||||
func (s *Service) GetDownload(url string) (*http.Response, error) {
|
||||
return s.downloadClient.Get(url)
|
||||
}
|
||||
|
||||
// DoLargeDownload executes an HTTP request for large file downloads.
|
||||
// This client has NO overall timeout - cancellation must be handled via the request's context.
|
||||
// Use this for multi-gigabyte files that may take hours to download.
|
||||
// The connection establishment and TLS handshake still have timeouts.
|
||||
//
|
||||
// Example usage:
|
||||
//
|
||||
// ctx, cancel := context.WithCancel(context.Background())
|
||||
// defer cancel() // Call cancel() to abort the download
|
||||
// req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
// resp, err := httpClient.DoLargeDownload(req)
|
||||
func (s *Service) DoLargeDownload(req *http.Request) (*http.Response, error) {
|
||||
return s.noTimeoutClient.Do(req)
|
||||
}
|
||||
|
||||
// GetLargeDownload performs an HTTP GET request for large file downloads.
|
||||
// This client has NO overall timeout - the download can run indefinitely.
|
||||
// Use this for multi-gigabyte files. To cancel, use DoLargeDownload with a context.
|
||||
func (s *Service) GetLargeDownload(url string) (*http.Response, error) {
|
||||
return s.noTimeoutClient.Get(url)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue