Tweak HTTP client handling

sources/almalinux-http.go

@@ -212,7 +212,7 @@ func (s *almalinux) getRelease(URL, release, variant, arch string) (string, erro
 	)
 
 	err = shared.Retry(func() error {
-		resp, err = http.Get(fullURL)
+		resp, err = s.client.Get(fullURL)
 		if err != nil {
 			return fmt.Errorf("Failed to GET %q: %w", fullURL, err)
 		}

sources/alpine-http.go

@@ -159,7 +159,7 @@ func (s *alpineLinux) getLatestRelease(baseURL, release string, arch string) (st
 	)
 
 	err = shared.Retry(func() error {
-		resp, err = http.Get(baseURL)
+		resp, err = s.client.Get(baseURL)
 		if err != nil {
 			return fmt.Errorf("Failed to GET %q: %w", baseURL, err)
 		}

sources/apertis-http.go

@@ -32,7 +32,7 @@ func (s *apertis) Run() error {
 	)
 
 	err = shared.Retry(func() error {
-		resp, err = http.Head(baseURL)
+		resp, err = s.client.Head(baseURL)
 		if err != nil {
 			return fmt.Errorf("Failed to HEAD %q: %w", baseURL, err)
 		}
@@ -94,7 +94,7 @@ func (s *apertis) getLatestRelease(baseURL, release string) (string, error) {
 	)
 
 	err = shared.Retry(func() error {
-		resp, err = http.Get(baseURL)
+		resp, err = s.client.Get(baseURL)
 		if err != nil {
 			return fmt.Errorf("Failed to GET %q: %w", baseURL, err)
 		}

sources/apertis-http_test.go

@@ -2,13 +2,15 @@ package sources
 
 import (
 	"fmt"
+	"net/http"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
 func TestApertisHTTP_getLatestRelease(t *testing.T) {
 	s := &apertis{}
+	s.client = http.DefaultClient
 
 	tests := []struct {
 		release string

sources/archlinux-http_test.go

@@ -1,14 +1,16 @@
 package sources
 
 import (
+	"net/http"
 	"regexp"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
 func TestArchLinuxGetLatestRelease(t *testing.T) {
-	var src archlinux
+	src := &archlinux{}
+	src.client = http.DefaultClient
 
 	release, err := src.getLatestRelease("https://archive.archlinux.org/iso/", "x86_64")
 	require.NoError(t, err)

sources/centos-http.go

@@ -414,7 +414,7 @@ func (s *centOS) getRelease(URL, release, variant, arch string) (string, error)
 	)
 
 	err = shared.Retry(func() error {
-		resp, err = http.Get(u)
+		resp, err = s.client.Get(u)
 		if err != nil {
 			return fmt.Errorf("Failed to get URL %q: %w", u, err)
 		}

sources/common.go

@@ -30,6 +30,19 @@ type common struct {
 	client     *http.Client
 }
 
+type httpCustomTransport struct{}
+
+func (ct *httpCustomTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if req.Header.Get("Accept") == "" {
+		req.Header.Set("Accept", "*/*")
+	}
+
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.TLSHandshakeTimeout = 60 * time.Second
+
+	return transport.RoundTrip(req)
+}
+
 func (s *common) init(ctx context.Context, logger *logrus.Logger, definition shared.Definition, rootfsDir string, cacheDir string, sourcesDir string) {
 	s.logger = logger
 	s.definition = definition
@@ -38,10 +51,7 @@ func (s *common) init(ctx context.Context, logger *logrus.Logger, definition sha
 	s.sourcesDir = sourcesDir
 	s.ctx = ctx
 
-	transport := http.DefaultTransport.(*http.Transport).Clone()
-	// Increase TLS handshake timeout for mirrors which need a bit more time.
-	transport.TLSHandshakeTimeout = 60 * time.Second
-
+	transport := &httpCustomTransport{}
 	s.client = &http.Client{
 		Transport: transport,
 	}

sources/fedora-http.go

@@ -9,7 +9,6 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
-	"slices"
 	"sort"
 
 	"github.com/lxc/incus/v6/shared/subprocess"
@@ -25,10 +24,6 @@ type fedora struct {
 func (s *fedora) Run() error {
 	base := "Fedora-Container-Base-Generic"
 	extension := "oci.tar.xz"
-	if slices.Contains([]string{"39", "40"}, s.definition.Image.Release) {
-		base = "Fedora-Container-Base"
-		extension = "tar.xz"
-	}
 
 	baseURL := fmt.Sprintf("%s/packages/%s", s.definition.Source.URL, base)
 
@@ -186,7 +181,7 @@ func (s *fedora) getLatestBuild(URL, release string) (string, error) {
 	)
 
 	err = shared.Retry(func() error {
-		resp, err = http.Get(fmt.Sprintf("%s/%s", URL, release))
+		resp, err = s.client.Get(fmt.Sprintf("%s/%s", URL, release))
 		if err != nil {
 			return fmt.Errorf("Failed to GET %q: %w", fmt.Sprintf("%s/%s", URL, release), err)
 		}

sources/gentoo.go

@@ -158,7 +158,7 @@ func (s *gentoo) getLatestBuild(baseURL, arch, variant string) (string, error) {
 	)
 
 	err = shared.Retry(func() error {
-		resp, err = http.Get(baseURL)
+		resp, err = s.client.Get(baseURL)
 		if err != nil {
 			return fmt.Errorf("Failed to GET %q: %w", baseURL, err)
 		}

sources/openeuler-http.go

@@ -37,7 +37,7 @@ func (s *openEuler) getLatestRelease(baseURL, release string) (string, error) {
 		return "", fmt.Errorf("Failed to parse URL %s: %w", baseURL, err)
 	}
 
-	resp, err = http.Get(baseURL)
+	resp, err = s.client.Get(baseURL)
 	if err != nil {
 		return "", fmt.Errorf("Failed to read url: %w", err)
 	}

sources/openeuler-http_test.go

@@ -1,13 +1,15 @@
 package sources
 
 import (
+	"net/http"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
 func TestGetLatestRelease(t *testing.T) {
 	s := &openEuler{}
+	s.client = http.DefaultClient
 
 	tests := []struct {
 		url        string

sources/opensuse-http.go

@@ -41,7 +41,7 @@ func (s *opensuse) Run() error {
 	var resp *http.Response
 
 	err = shared.Retry(func() error {
-		resp, err = http.Head(tarballPath)
+		resp, err = s.client.Head(tarballPath)
 		if err != nil {
 			return fmt.Errorf("Failed to HEAD %q: %w", tarballPath, err)
 		}
@@ -226,7 +226,7 @@ func (s *opensuse) getTarballName(u *url.URL, release, arch string) (string, err
 func (s *opensuse) validateURL(u url.URL, tarball string) bool {
 	u.Path = path.Join(u.Path, tarball)
 
-	resp, err := http.Head(u.String())
+	resp, err := s.client.Head(u.String())
 	if err != nil {
 		return false
 	}

sources/openwrt-http.go

@@ -104,7 +104,7 @@ func (s *openwrt) Run() error {
 	)
 
 	err = shared.Retry(func() error {
-		resp, err = http.Head(baseURL)
+		resp, err = s.client.Head(baseURL)
 		if err != nil {
 			return fmt.Errorf("Failed to HEAD %q: %w", baseURL, err)
 		}
@@ -183,7 +183,7 @@ func (s *openwrt) getLatestServiceRelease(baseURL, release string) (string, erro
 	)
 
 	err = shared.Retry(func() error {
-		resp, err = http.Get(baseURL)
+		resp, err = s.client.Get(baseURL)
 		if err != nil {
 			return fmt.Errorf("Failed to GET %q: %w", baseURL, err)
 		}

sources/openwrt-http_test.go

@@ -1,6 +1,7 @@
 package sources
 
 import (
+	"net/http"
 	"regexp"
 	"testing"
 
@@ -9,6 +10,7 @@ import (
 
 func TestOpenWrtHTTP_getLatestServiceRelease(t *testing.T) {
 	s := &openwrt{}
+	s.client = http.DefaultClient
 
 	tests := []struct {
 		release string

sources/oraclelinux-http.go

@@ -65,7 +65,7 @@ func (s *oraclelinux) Run() error {
 			)
 
 			err = shared.Retry(func() error {
-				resp, err = http.Head(fullURL)
+				resp, err = s.client.Head(fullURL)
 				if err != nil {
 					return errors.New("")
 				}
@@ -100,7 +100,7 @@ func (s *oraclelinux) Run() error {
 		)
 
 		err = shared.Retry(func() error {
-			resp, err = http.Head(fullURL)
+			resp, err = s.client.Head(fullURL)
 			if err != nil {
 				return errors.New("")
 			}

sources/rocky-http.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net/http"
 	"net/url"
 	"path"
 	"path/filepath"
@@ -251,7 +250,7 @@ rm -rf /rootfs/var/cache/yum
 func (s *rockylinux) getRelease(URL, release, variant, arch string) (string, error) {
 	u := URL + path.Join("/", strings.ToLower(release), "isos", arch)
 
-	resp, err := http.Get(u)
+	resp, err := s.client.Get(u)
 	if err != nil {
 		return "", fmt.Errorf("Failed to GET %q: %w", u, err)
 	}

sources/ubuntu-http.go

@@ -46,7 +46,7 @@ func (s *ubuntu) downloadImage(definition shared.Definition) error {
 				s.definition.Image.Release, s.definition.Image.ArchitectureMapped)
 		} else {
 			// if release is non-numerical, find the latest release
-			s.fname, err = getLatestRelease(baseURL,
+			s.fname, err = s.getLatestRelease(baseURL,
 				s.definition.Image.Release, s.definition.Image.ArchitectureMapped)
 			if err != nil {
 				return fmt.Errorf("Failed to get latest release: %w", err)
@@ -125,14 +125,14 @@ func (s ubuntu) unpack(filePath, rootDir string) error {
 	return nil
 }
 
-func getLatestRelease(baseURL, release, arch string) (string, error) {
+func (s ubuntu) getLatestRelease(baseURL, release, arch string) (string, error) {
 	var (
 		resp *http.Response
 		err  error
 	)
 
 	err = shared.Retry(func() error {
-		resp, err = http.Get(baseURL)
+		resp, err = s.client.Get(baseURL)
 		if err != nil {
 			return fmt.Errorf("Failed to GET %q: %w", baseURL, err)
 		}

sources/voidlinux-http.go

@@ -98,7 +98,7 @@ func (s *voidlinux) getLatestBuild(baseURL, arch, variant string) (string, error
 	)
 
 	err = shared.Retry(func() error {
-		resp, err = http.Get(baseURL)
+		resp, err = s.client.Get(baseURL)
 		if err != nil {
 			return fmt.Errorf("Failed to GET %q: %w", baseURL, err)
 		}