Add support for .spec.proxySecretRef for gcp provider of Bucket API

Signed-off-by: Matheus Pimenta <matheuscscp@gmail.com>

Author: matheuscscp

internal/controller/bucket_controller.go

@@ -448,7 +448,7 @@ func (r *BucketReconciler) reconcileSource(ctx context.Context, sp *patch.Serial
 			conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Error())
 			return sreconcile.ResultEmpty, e
 		}
-		if provider, err = gcp.NewClient(ctx, secret); err != nil {
+		if provider, err = gcp.NewClient(ctx, obj, secret, proxyURL); err != nil {
 			e := serror.NewGeneric(err, "ClientError")
 			conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Error())
 			return sreconcile.ResultEmpty, e

pkg/gcp/gcp.go

@@ -21,6 +21,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 
@@ -30,6 +32,8 @@ import (
 	"google.golang.org/api/option"
 	corev1 "k8s.io/api/core/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
+
+	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 )
 
 var (
@@ -48,24 +52,28 @@ type GCSClient struct {
 	*gcpstorage.Client
 }
 
-// NewClient creates a new GCP storage client. The Client will automatically look for  the Google Application
+// NewClient creates a new GCP storage client. The Client will automatically look for the Google Application
 // Credential environment variable or look for the Google Application Credential file.
-func NewClient(ctx context.Context, secret *corev1.Secret) (*GCSClient, error) {
-	c := &GCSClient{}
+func NewClient(ctx context.Context, bucket *sourcev1.Bucket,
+	secret *corev1.Secret, proxyURL *url.URL) (*GCSClient, error) {
+	var opts []option.ClientOption
+
 	if secret != nil {
-		client, err := gcpstorage.NewClient(ctx, option.WithCredentialsJSON(secret.Data["serviceaccount"]))
-		if err != nil {
-			return nil, err
-		}
-		c.Client = client
-	} else {
-		client, err := gcpstorage.NewClient(ctx)
-		if err != nil {
-			return nil, err
-		}
-		c.Client = client
+		opts = append(opts, option.WithCredentialsJSON(secret.Data["serviceaccount"]))
 	}
-	return c, nil
+
+	if proxyURL != nil {
+		transport := http.DefaultTransport.(*http.Transport).Clone()
+		transport.Proxy = http.ProxyURL(proxyURL)
+		opts = append(opts, option.WithHTTPClient(&http.Client{Transport: transport}))
+	}
+
+	client, err := gcpstorage.NewClient(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &GCSClient{Client: client}, nil
 }
 
 // ValidateSecret validates the credential secret. The provided Secret may

pkg/gcp/gcp_test.go

@@ -33,12 +33,13 @@ import (
 
 	gcpstorage "cloud.google.com/go/storage"
 	"google.golang.org/api/googleapi"
+	"google.golang.org/api/option"
 	raw "google.golang.org/api/storage/v1"
 	"gotest.tools/assert"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	"google.golang.org/api/option"
+	pkgtesting "github.com/fluxcd/source-controller/pkg/testing"
 )
 
 const (
@@ -49,11 +50,12 @@ const (
 )
 
 var (
-	hc     *http.Client
-	client *gcpstorage.Client
-	close  func()
-	err    error
-	secret = corev1.Secret{
+	hc         *http.Client
+	serverAddr string
+	client     *gcpstorage.Client
+	close      func()
+	err        error
+	secret     = corev1.Secret{
 		ObjectMeta: v1.ObjectMeta{
 			Name:      "gcp-secret",
 			Namespace: "default",
@@ -76,7 +78,7 @@ var (
 )
 
 func TestMain(m *testing.M) {
-	hc, close = newTestServer(func(w http.ResponseWriter, r *http.Request) {
+	hc, serverAddr, close = newTestServer(func(w http.ResponseWriter, r *http.Request) {
 		io.Copy(io.Discard, r.Body)
 		switch r.RequestURI {
 		case fmt.Sprintf("/storage/v1/b/%s?alt=json&prettyPrint=false&projection=full", bucketName):
@@ -140,7 +142,7 @@ func TestMain(m *testing.M) {
 }
 
 func TestNewClientWithSecretErr(t *testing.T) {
-	gcpClient, err := NewClient(context.Background(), secret.DeepCopy())
+	gcpClient, err := NewClient(context.Background(), nil, secret.DeepCopy(), nil)
 	t.Log(err)
 	assert.Error(t, err, "dialing: invalid character 'e' looking for beginning of value")
 	assert.Assert(t, gcpClient == nil)
@@ -216,6 +218,27 @@ func TestFGetObject(t *testing.T) {
 	assert.Equal(t, etag, objectEtag)
 }
 
+func TestNewClientAndFGetObjectWithProxy(t *testing.T) {
+	proxyURL, closeProxy := pkgtesting.NewHTTPProxy(t, serverAddr)
+	defer closeProxy()
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.Proxy = http.ProxyURL(proxyURL)
+	transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // needed because the server is a mock
+	httpClient := &http.Client{Transport: transport}
+	client, err := gcpstorage.NewClient(context.Background(), option.WithHTTPClient(httpClient))
+	assert.NilError(t, err)
+	gcpClient := &GCSClient{
+		Client: client,
+	}
+	tempDir := t.TempDir()
+	localPath := filepath.Join(tempDir, objectName)
+	etag, err := gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath)
+	if err != io.EOF {
+		assert.NilError(t, err)
+	}
+	assert.Equal(t, etag, objectEtag)
+}
+
 func TestFGetObjectNotExists(t *testing.T) {
 	object := "notexists.txt"
 	tempDir := t.TempDir()
@@ -272,16 +295,17 @@ func TestValidateSecret(t *testing.T) {
 	}
 }
 
-func newTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*http.Client, func()) {
-	ts := httptest.NewTLSServer(http.HandlerFunc(handler))
+func newTestServer(handler http.HandlerFunc) (*http.Client, string, func()) {
+	ts := httptest.NewTLSServer(handler)
+	serverAddr := ts.Listener.Addr().String()
 	tlsConf := &tls.Config{InsecureSkipVerify: true}
 	tr := &http.Transport{
 		TLSClientConfig: tlsConf,
 		DialTLS: func(netw, addr string) (net.Conn, error) {
-			return tls.Dial("tcp", ts.Listener.Addr().String(), tlsConf)
+			return tls.Dial("tcp", serverAddr, tlsConf)
 		},
 	}
-	return &http.Client{Transport: tr}, func() {
+	return &http.Client{Transport: tr}, serverAddr, func() {
 		tr.CloseIdleConnections()
 		ts.Close()
 	}

pkg/minio/minio_test.go

@@ -236,7 +236,7 @@ func TestFGetObject(t *testing.T) {
 }
 
 func TestNewClientAndFGetObjectWithProxy(t *testing.T) {
-	proxyURL, closeProxy := pkgtesting.NewHTTPProxy(t)
+	proxyURL, closeProxy := pkgtesting.NewHTTPProxy(t, "")
 	defer closeProxy()
 	minioClient, err := NewClient(bucketStub(bucket, testMinioAddress), secret.DeepCopy(), testTLSConfig, proxyURL)
 	assert.NilError(t, err)

pkg/testing/http_proxy.go

@@ -13,13 +13,20 @@ import (
 
 // NewHTTPProxy starts an HTTP proxy server in a random port and returns the
 // URL of the proxy server and a function to stop the server.
-func NewHTTPProxy(t *testing.T) (*url.URL, func()) {
+// If httpsTargetAddr is not empty, the proxy server will handle CONNECT
+// HTTPS requests to that address.
+func NewHTTPProxy(t *testing.T, httpsTargetAddr string) (*url.URL, func()) {
 	listener, err := net.Listen("tcp", ":0")
 	assert.NilError(t, err, "could not start proxy server")
 
 	addr := listener.Addr().String()
 	proxy := goproxy.NewProxyHttpServer()
 	proxy.Verbose = true
+	if httpsTargetAddr != "" {
+		proxy.OnRequest().HandleConnectFunc(func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
+			return goproxy.OkConnect, httpsTargetAddr
+		})
+	}
 	server := &http.Server{
 		Addr:    addr,
 		Handler: proxy,