Skip to content

Commit 02fea49

Browse files
committed
Add support for .spec.proxySecretRef for generic provider of Bucket API
Signed-off-by: Matheus Pimenta <matheuscscp@gmail.com>
1 parent 59ad5a7 commit 02fea49

File tree

11 files changed

+221
-10
lines changed

11 files changed

+221
-10
lines changed

api/v1beta2/bucket_types.go

+5
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,11 @@ type BucketSpec struct {
100100
// +optional
101101
CertSecretRef *meta.LocalObjectReference `json:"certSecretRef,omitempty"`
102102

103+
// ProxySecretRef specifies the Secret containing the proxy configuration
104+
// to use while communicating with the Bucket server.
105+
// +optional
106+
ProxySecretRef *meta.LocalObjectReference `json:"proxySecretRef,omitempty"`
107+
103108
// Interval at which the Bucket Endpoint is checked for updates.
104109
// This interval is approximate and may be subject to jitter to ensure
105110
// efficient use of resources.

api/v1beta2/zz_generated.deepcopy.go

+5
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml

+11
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,17 @@ spec:
391391
- gcp
392392
- azure
393393
type: string
394+
proxySecretRef:
395+
description: |-
396+
ProxySecretRef specifies the Secret containing the proxy configuration
397+
to use while communicating with the Bucket server.
398+
properties:
399+
name:
400+
description: Name of the referent.
401+
type: string
402+
required:
403+
- name
404+
type: object
394405
region:
395406
description: Region of the Endpoint where the BucketName is located
396407
in.

docs/api/v1beta2/source.md

+30
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,21 @@ be of type <code>Opaque</code> or <code>kubernetes.io/tls</code>.</p>
191191
</tr>
192192
<tr>
193193
<td>
194+
<code>proxySecretRef</code><br>
195+
<em>
196+
<a href="https://pkg.go.dev/github.com/fluxcd/pkg/apis/meta#LocalObjectReference">
197+
github.com/fluxcd/pkg/apis/meta.LocalObjectReference
198+
</a>
199+
</em>
200+
</td>
201+
<td>
202+
<em>(Optional)</em>
203+
<p>ProxySecretRef specifies the Secret containing the proxy configuration
204+
to use while communicating with the Bucket server.</p>
205+
</td>
206+
</tr>
207+
<tr>
208+
<td>
194209
<code>interval</code><br>
195210
<em>
196211
<a href="https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Duration">
@@ -1541,6 +1556,21 @@ be of type <code>Opaque</code> or <code>kubernetes.io/tls</code>.</p>
15411556
</tr>
15421557
<tr>
15431558
<td>
1559+
<code>proxySecretRef</code><br>
1560+
<em>
1561+
<a href="https://pkg.go.dev/github.com/fluxcd/pkg/apis/meta#LocalObjectReference">
1562+
github.com/fluxcd/pkg/apis/meta.LocalObjectReference
1563+
</a>
1564+
</em>
1565+
</td>
1566+
<td>
1567+
<em>(Optional)</em>
1568+
<p>ProxySecretRef specifies the Secret containing the proxy configuration
1569+
to use while communicating with the Bucket server.</p>
1570+
</td>
1571+
</tr>
1572+
<tr>
1573+
<td>
15441574
<code>interval</code><br>
15451575
<em>
15461576
<a href="https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Duration">

docs/spec/v1beta2/buckets.md

+35
Original file line numberDiff line numberDiff line change
@@ -824,6 +824,41 @@ stringData:
824824
ca.crt: <PEM-encoded cert>
825825
```
826826

827+
### Proxy secret reference
828+
829+
`.spec.proxySecretRef.name` is an optional field used to specify the name of a
830+
Secret that contains the proxy settings for the object. These settings are used
831+
for all the remote operations related to the Bucket.
832+
The Secret can contain three keys:
833+
834+
- `address`, to specify the address of the proxy server. This is a required key.
835+
- `username`, to specify the username to use if the proxy server is protected by
836+
basic authentication. This is an optional key.
837+
- `password`, to specify the password to use if the proxy server is protected by
838+
basic authentication. This is an optional key.
839+
840+
The proxy server must be HTTP/S.
841+
842+
Example:
843+
844+
```yaml
845+
---
846+
apiVersion: v1
847+
kind: Secret
848+
metadata:
849+
name: http-proxy
850+
type: Opaque
851+
stringData:
852+
address: http://proxy.com
853+
username: mandalorian
854+
password: grogu
855+
```
856+
857+
Proxying can also be configured in the source-controller Deployment directly by
858+
using the standard environment variables such as `HTTPS_PROXY`, `ALL_PROXY`, etc.
859+
860+
`.spec.proxySecretRef.name` takes precedence over all environment variables.
861+
827862
### Insecure
828863

829864
`.spec.insecure` is an optional field to allow connecting to an insecure (HTTP)

go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ require (
1919
github.com/distribution/distribution/v3 v3.0.0-alpha.1
2020
github.com/docker/cli v24.0.9+incompatible
2121
github.com/docker/go-units v0.5.0
22+
github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5
2223
github.com/fluxcd/cli-utils v0.36.0-flux.7
2324
github.com/fluxcd/pkg/apis/event v0.9.0
2425
github.com/fluxcd/pkg/apis/meta v1.5.0

go.sum

+3
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
311311
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
312312
github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5 h1:m62nsMU279qRD9PQSWD1l66kmkXzuYcnVJqL4XLeV2M=
313313
github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
314+
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
315+
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
314316
github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk=
315317
github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
316318
github.com/emicklei/proto v1.12.1 h1:6n/Z2pZAnBwuhU66Gs8160B8rrrYKo7h2F2sCOnNceE=
@@ -831,6 +833,7 @@ github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0
831833
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
832834
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
833835
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
836+
github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
834837
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
835838
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
836839
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=

internal/controller/bucket_controller.go

+32-1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
stdtls "crypto/tls"
2222
"errors"
2323
"fmt"
24+
"net/url"
2425
"os"
2526
"path/filepath"
2627
"strings"
@@ -468,7 +469,13 @@ func (r *BucketReconciler) reconcileSource(ctx context.Context, sp *patch.Serial
468469
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Error())
469470
return sreconcile.ResultEmpty, e
470471
}
471-
if provider, err = minio.NewClient(obj, secret, tlsConfig); err != nil {
472+
proxyURL, err := r.getProxyURL(ctx, obj)
473+
if err != nil {
474+
e := serror.NewGeneric(err, sourcev1.AuthenticationFailedReason)
475+
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Error())
476+
return sreconcile.ResultEmpty, e
477+
}
478+
if provider, err = minio.NewClient(obj, secret, tlsConfig, proxyURL); err != nil {
472479
e := serror.NewGeneric(err, "ClientError")
473480
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Error())
474481
return sreconcile.ResultEmpty, e
@@ -703,6 +710,30 @@ func (r *BucketReconciler) getTLSConfig(ctx context.Context, obj *bucketv1.Bucke
703710
return tlsConfig, nil
704711
}
705712

713+
func (r *BucketReconciler) getProxyURL(ctx context.Context, obj *bucketv1.Bucket) (*url.URL, error) {
714+
namespace := obj.GetNamespace()
715+
proxySecret, err := r.getSecret(ctx, obj.Spec.ProxySecretRef, namespace)
716+
if err != nil || proxySecret == nil {
717+
return nil, err
718+
}
719+
proxyData := proxySecret.Data
720+
address, ok := proxyData["address"]
721+
if !ok {
722+
return nil, fmt.Errorf("invalid proxy secret '%s/%s': key 'address' is missing",
723+
obj.Spec.ProxySecretRef.Name, namespace)
724+
}
725+
proxyURL, err := url.Parse(string(address))
726+
if err != nil {
727+
return nil, fmt.Errorf("failed to parse proxy address '%s': %w", address, err)
728+
}
729+
user, hasUser := proxyData["username"]
730+
password, hasPassword := proxyData["password"]
731+
if hasUser || hasPassword {
732+
proxyURL.User = url.UserPassword(string(user), string(password))
733+
}
734+
return proxyURL, nil
735+
}
736+
706737
// eventLogf records events, and logs at the same time.
707738
//
708739
// This log is different from the debug log in the EventRecorder, in the sense

internal/controller/bucket_controller_test.go

+41
Original file line numberDiff line numberDiff line change
@@ -551,6 +551,47 @@ func TestBucketReconciler_reconcileSource_generic(t *testing.T) {
551551
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.AuthenticationFailedReason, "certificate secret does not contain any TLS configuration"),
552552
},
553553
},
554+
{
555+
name: "Observes non-existing proxySecretRef",
556+
bucketName: "dummy",
557+
beforeFunc: func(obj *bucketv1.Bucket) {
558+
obj.Spec.ProxySecretRef = &meta.LocalObjectReference{
559+
Name: "dummy",
560+
}
561+
conditions.MarkReconciling(obj, meta.ProgressingReason, "foo")
562+
conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar")
563+
},
564+
wantErr: true,
565+
assertIndex: index.NewDigester(),
566+
assertConditions: []metav1.Condition{
567+
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.AuthenticationFailedReason, "failed to get secret '/dummy': secrets \"dummy\" not found"),
568+
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"),
569+
*conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"),
570+
},
571+
},
572+
{
573+
name: "Observes invalid proxySecretRef",
574+
bucketName: "dummy",
575+
secret: &corev1.Secret{
576+
ObjectMeta: metav1.ObjectMeta{
577+
Name: "dummy",
578+
},
579+
},
580+
beforeFunc: func(obj *bucketv1.Bucket) {
581+
obj.Spec.ProxySecretRef = &meta.LocalObjectReference{
582+
Name: "dummy",
583+
}
584+
conditions.MarkReconciling(obj, meta.ProgressingReason, "foo")
585+
conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar")
586+
},
587+
wantErr: true,
588+
assertIndex: index.NewDigester(),
589+
assertConditions: []metav1.Condition{
590+
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"),
591+
*conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"),
592+
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.AuthenticationFailedReason, "invalid proxy secret 'dummy/': key 'address' is missing"),
593+
},
594+
},
554595
{
555596
name: "Observes non-existing bucket name",
556597
bucketName: "dummy",

pkg/minio/minio.go

+19-5
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ import (
2121
"crypto/tls"
2222
"errors"
2323
"fmt"
24+
"net/http"
25+
"net/url"
2426

2527
"github.com/minio/minio-go/v7"
2628
"github.com/minio/minio-go/v7/pkg/credentials"
@@ -37,7 +39,9 @@ type MinioClient struct {
3739
}
3840

3941
// NewClient creates a new Minio storage client.
40-
func NewClient(bucket *sourcev1.Bucket, secret *corev1.Secret, tlsConfig *tls.Config) (*MinioClient, error) {
42+
func NewClient(bucket *sourcev1.Bucket, secret *corev1.Secret,
43+
tlsConfig *tls.Config, proxyURL *url.URL) (*MinioClient, error) {
44+
4145
opt := minio.Options{
4246
Region: bucket.Spec.Region,
4347
Secure: !bucket.Spec.Insecure,
@@ -61,15 +65,25 @@ func NewClient(bucket *sourcev1.Bucket, secret *corev1.Secret, tlsConfig *tls.Co
6165
opt.Creds = credentials.NewIAM("")
6266
}
6367

64-
if opt.Secure && tlsConfig != nil {
68+
secure := opt.Secure && tlsConfig != nil
69+
proxy := proxyURL != nil
70+
if secure || proxy {
6571
// Use the default minio transport, but override the TLS config.
66-
secure := false // true causes the TLS config to be defined internally, but here we have our own so we just pass false.
67-
transport, err := minio.DefaultTransport(secure)
72+
minioSecure := false // true causes the TLS config to be defined internally, but here we have our own so we just pass false.
73+
transport, err := minio.DefaultTransport(minioSecure)
6874
if err != nil {
6975
// The error returned here is always nil, but we keep the check for future compatibility.
7076
return nil, fmt.Errorf("failed to create default minio transport: %w", err)
7177
}
72-
transport.TLSClientConfig = tlsConfig.Clone()
78+
79+
if secure {
80+
transport.TLSClientConfig = tlsConfig.Clone()
81+
}
82+
83+
if proxy {
84+
transport.Proxy = http.ProxyURL(proxyURL)
85+
}
86+
7387
opt.Transport = transport
7488
}
7589

0 commit comments

Comments
 (0)