Skip to content

Commit cdc0753

Browse files
tmihalactmihalac
authored
fix: Made fixes to Go Operator DB persistence (feast-dev#4830)
* Made fixes to Go Operator DB persistence Signed-off-by: Theodor Mihalache <tmihalac@redhat.com> * Fixes following review Signed-off-by: Theodor Mihalache <tmihalac@redhat.com> --------- Signed-off-by: Theodor Mihalache <tmihalac@redhat.com>
1 parent 732865f commit cdc0753

File tree

7 files changed

+128
-94
lines changed

7 files changed

+128
-94
lines changed

‎infra/feast-operator/api/v1alpha1/featurestore_types.go

+11-4
Original file line numberDiff line numberDiff line change
@@ -99,19 +99,20 @@ type OfflineStorePersistence struct {
9999

100100
// OfflineStoreFilePersistence configures the file-based persistence for the offline store service
101101
type OfflineStoreFilePersistence struct {
102-
// +kubebuilder:validation:Enum=dask;duckdb
102+
// +kubebuilder:validation:Enum=file;dask;duckdb
103103
Type string `json:"type,omitempty"`
104104
PvcConfig *PvcConfig `json:"pvc,omitempty"`
105105
}
106106

107107
var ValidOfflineStoreFilePersistenceTypes = []string{
108108
"dask",
109109
"duckdb",
110+
"file",
110111
}
111112

112113
// OfflineStoreDBStorePersistence configures the DB store persistence for the offline store service
113114
type OfflineStoreDBStorePersistence struct {
114-
// +kubebuilder:validation:Enum=snowflake.offline;bigquery;redshift;spark;postgres;feast_trino.trino.TrinoOfflineStore;redis
115+
// +kubebuilder:validation:Enum=snowflake.offline;bigquery;redshift;spark;postgres;trino;redis;athena;mssql
115116
Type string `json:"type"`
116117
// Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed.
117118
SecretRef corev1.LocalObjectReference `json:"secretRef"`
@@ -125,8 +126,10 @@ var ValidOfflineStoreDBStorePersistenceTypes = []string{
125126
"redshift",
126127
"spark",
127128
"postgres",
128-
"feast_trino.trino.TrinoOfflineStore",
129+
"trino",
129130
"redis",
131+
"athena",
132+
"mssql",
130133
}
131134

132135
// OnlineStore configures the deployed online store service
@@ -158,7 +161,7 @@ type OnlineStoreFilePersistence struct {
158161

159162
// OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service
160163
type OnlineStoreDBStorePersistence struct {
161-
// +kubebuilder:validation:Enum=snowflake.online;redis;ikv;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore
164+
// +kubebuilder:validation:Enum=snowflake.online;redis;ikv;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase
162165
Type string `json:"type"`
163166
// Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed.
164167
SecretRef corev1.LocalObjectReference `json:"secretRef"`
@@ -178,6 +181,10 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{
178181
"mysql",
179182
"hazelcast",
180183
"singlestore",
184+
"hbase",
185+
"elasticsearch",
186+
"qdrant",
187+
"couchbase",
181188
}
182189

183190
// LocalRegistryConfig configures the deployed registry service

‎infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml

+16-2
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,7 @@ spec:
323323
rule: self.mountPath.matches('^/[^:]*$')
324324
type:
325325
enum:
326+
- file
326327
- dask
327328
- duckdb
328329
type: string
@@ -355,8 +356,10 @@ spec:
355356
- redshift
356357
- spark
357358
- postgres
358-
- feast_trino.trino.TrinoOfflineStore
359+
- trino
359360
- redis
361+
- athena
362+
- mssql
360363
type: string
361364
required:
362365
- secretRef
@@ -729,6 +732,10 @@ spec:
729732
- mysql
730733
- hazelcast
731734
- singlestore
735+
- hbase
736+
- elasticsearch
737+
- qdrant
738+
- couchbase
732739
type: string
733740
required:
734741
- secretRef
@@ -1559,6 +1566,7 @@ spec:
15591566
rule: self.mountPath.matches('^/[^:]*$')
15601567
type:
15611568
enum:
1569+
- file
15621570
- dask
15631571
- duckdb
15641572
type: string
@@ -1592,8 +1600,10 @@ spec:
15921600
- redshift
15931601
- spark
15941602
- postgres
1595-
- feast_trino.trino.TrinoOfflineStore
1603+
- trino
15961604
- redis
1605+
- athena
1606+
- mssql
15971607
type: string
15981608
required:
15991609
- secretRef
@@ -1973,6 +1983,10 @@ spec:
19731983
- mysql
19741984
- hazelcast
19751985
- singlestore
1986+
- hbase
1987+
- elasticsearch
1988+
- qdrant
1989+
- couchbase
19761990
type: string
19771991
required:
19781992
- secretRef

‎infra/feast-operator/dist/install.yaml

+16-2
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,7 @@ spec:
331331
rule: self.mountPath.matches('^/[^:]*$')
332332
type:
333333
enum:
334+
- file
334335
- dask
335336
- duckdb
336337
type: string
@@ -363,8 +364,10 @@ spec:
363364
- redshift
364365
- spark
365366
- postgres
366-
- feast_trino.trino.TrinoOfflineStore
367+
- trino
367368
- redis
369+
- athena
370+
- mssql
368371
type: string
369372
required:
370373
- secretRef
@@ -737,6 +740,10 @@ spec:
737740
- mysql
738741
- hazelcast
739742
- singlestore
743+
- hbase
744+
- elasticsearch
745+
- qdrant
746+
- couchbase
740747
type: string
741748
required:
742749
- secretRef
@@ -1567,6 +1574,7 @@ spec:
15671574
rule: self.mountPath.matches('^/[^:]*$')
15681575
type:
15691576
enum:
1577+
- file
15701578
- dask
15711579
- duckdb
15721580
type: string
@@ -1600,8 +1608,10 @@ spec:
16001608
- redshift
16011609
- spark
16021610
- postgres
1603-
- feast_trino.trino.TrinoOfflineStore
1611+
- trino
16041612
- redis
1613+
- athena
1614+
- mssql
16051615
type: string
16061616
required:
16071617
- secretRef
@@ -1981,6 +1991,10 @@ spec:
19811991
- mysql
19821992
- hazelcast
19831993
- singlestore
1994+
- hbase
1995+
- elasticsearch
1996+
- qdrant
1997+
- couchbase
19841998
type: string
19851999
required:
19862000
- secretRef

‎infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go

+57-60
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ sqlalchemy_config_kwargs:
7878
pool_pre_ping: true
7979
`
8080

81-
var invalidSecretContainingTypeYamlString = `
81+
var secretContainingValidTypeYamlString = `
8282
type: cassandra
8383
hosts:
8484
- 192.168.1.1
@@ -305,37 +305,12 @@ var _ = Describe("FeatureStore Controller - db storage services", func() {
305305

306306
Expect(err.Error()).To(Equal("secret key invalid.secret.key doesn't exist in secret online-store-secret"))
307307

308-
By("Referring to a secret that contains parameter named type")
309-
resource = &feastdevv1alpha1.FeatureStore{}
310-
err = k8sClient.Get(ctx, typeNamespacedName, resource)
311-
Expect(err).NotTo(HaveOccurred())
312-
313-
secret := &corev1.Secret{}
314-
err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret)
315-
Expect(err).NotTo(HaveOccurred())
316-
secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(invalidSecretContainingTypeYamlString)
317-
Expect(k8sClient.Update(ctx, secret)).To(Succeed())
318-
319-
resource.Spec.Services.OnlineStore.Persistence.DBPersistence.SecretRef = corev1.LocalObjectReference{Name: "online-store-secret"}
320-
resource.Spec.Services.OnlineStore.Persistence.DBPersistence.SecretKeyName = ""
321-
Expect(k8sClient.Update(ctx, resource)).To(Succeed())
322-
resource = &feastdevv1alpha1.FeatureStore{}
323-
err = k8sClient.Get(ctx, typeNamespacedName, resource)
324-
Expect(err).NotTo(HaveOccurred())
325-
326-
_, err = controllerReconciler.Reconcile(ctx, reconcile.Request{
327-
NamespacedName: typeNamespacedName,
328-
})
329-
Expect(err).To(HaveOccurred())
330-
331-
Expect(err.Error()).To(Equal("secret key cassandra in secret online-store-secret contains invalid tag named type"))
332-
333308
By("Referring to a secret that contains parameter named type with invalid value")
334309
resource = &feastdevv1alpha1.FeatureStore{}
335310
err = k8sClient.Get(ctx, typeNamespacedName, resource)
336311
Expect(err).NotTo(HaveOccurred())
337312

338-
secret = &corev1.Secret{}
313+
secret := &corev1.Secret{}
339314
err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret)
340315
Expect(err).NotTo(HaveOccurred())
341316
secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(invalidSecretTypeYamlString)
@@ -353,39 +328,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() {
353328
})
354329
Expect(err).To(HaveOccurred())
355330

356-
Expect(err.Error()).To(Equal("secret key cassandra in secret online-store-secret contains invalid tag named type"))
357-
358-
By("Referring to a secret that contains parameter named registry_type")
359-
resource = &feastdevv1alpha1.FeatureStore{}
360-
err = k8sClient.Get(ctx, typeNamespacedName, resource)
361-
Expect(err).NotTo(HaveOccurred())
362-
363-
secret = &corev1.Secret{}
364-
err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret)
365-
Expect(err).NotTo(HaveOccurred())
366-
secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(cassandraYamlString)
367-
Expect(k8sClient.Update(ctx, secret)).To(Succeed())
368-
369-
secret = &corev1.Secret{}
370-
err = k8sClient.Get(ctx, registrySecretNamespacedName, secret)
371-
Expect(err).NotTo(HaveOccurred())
372-
secret.Data["sql_custom_registry_key"] = nil
373-
secret.Data[string(services.RegistryDBPersistenceSQLConfigType)] = []byte(invalidSecretRegistryTypeYamlString)
374-
Expect(k8sClient.Update(ctx, secret)).To(Succeed())
375-
376-
resource.Spec.Services.Registry.Local.Persistence.DBPersistence.SecretRef = corev1.LocalObjectReference{Name: "registry-store-secret"}
377-
resource.Spec.Services.Registry.Local.Persistence.DBPersistence.SecretKeyName = ""
378-
Expect(k8sClient.Update(ctx, resource)).To(Succeed())
379-
resource = &feastdevv1alpha1.FeatureStore{}
380-
err = k8sClient.Get(ctx, typeNamespacedName, resource)
381-
Expect(err).NotTo(HaveOccurred())
382-
383-
_, err = controllerReconciler.Reconcile(ctx, reconcile.Request{
384-
NamespacedName: typeNamespacedName,
385-
})
386-
Expect(err).To(HaveOccurred())
387-
388-
Expect(err.Error()).To(Equal("secret key sql in secret registry-store-secret contains invalid tag named registry_type"))
331+
Expect(err.Error()).To(Equal("secret key cassandra in secret online-store-secret contains tag named type with value wrong"))
389332
})
390333

391334
It("should successfully reconcile the resource", func() {
@@ -506,6 +449,60 @@ var _ = Describe("FeatureStore Controller - db storage services", func() {
506449
Expect(err).NotTo(HaveOccurred())
507450
Expect(controllerutil.HasControllerReference(svc)).To(BeTrue())
508451
Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(services.FeastServiceConstants[services.RegistryFeastType].TargetHttpPort))))
452+
453+
By("Referring to a secret that contains parameter named type")
454+
resource = &feastdevv1alpha1.FeatureStore{}
455+
err = k8sClient.Get(ctx, typeNamespacedName, resource)
456+
Expect(err).NotTo(HaveOccurred())
457+
458+
secret := &corev1.Secret{}
459+
err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret)
460+
Expect(err).NotTo(HaveOccurred())
461+
secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(secretContainingValidTypeYamlString)
462+
Expect(k8sClient.Update(ctx, secret)).To(Succeed())
463+
464+
resource.Spec.Services.OnlineStore.Persistence.DBPersistence.SecretRef = corev1.LocalObjectReference{Name: "online-store-secret"}
465+
resource.Spec.Services.OnlineStore.Persistence.DBPersistence.SecretKeyName = ""
466+
Expect(k8sClient.Update(ctx, resource)).To(Succeed())
467+
resource = &feastdevv1alpha1.FeatureStore{}
468+
err = k8sClient.Get(ctx, typeNamespacedName, resource)
469+
Expect(err).NotTo(HaveOccurred())
470+
471+
_, err = controllerReconciler.Reconcile(ctx, reconcile.Request{
472+
NamespacedName: typeNamespacedName,
473+
})
474+
475+
Expect(err).To(Not(HaveOccurred()))
476+
477+
By("Referring to a secret that contains parameter named registry_type")
478+
resource = &feastdevv1alpha1.FeatureStore{}
479+
err = k8sClient.Get(ctx, typeNamespacedName, resource)
480+
Expect(err).NotTo(HaveOccurred())
481+
482+
secret = &corev1.Secret{}
483+
err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret)
484+
Expect(err).NotTo(HaveOccurred())
485+
secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(cassandraYamlString)
486+
Expect(k8sClient.Update(ctx, secret)).To(Succeed())
487+
488+
secret = &corev1.Secret{}
489+
err = k8sClient.Get(ctx, registrySecretNamespacedName, secret)
490+
Expect(err).NotTo(HaveOccurred())
491+
secret.Data["sql_custom_registry_key"] = nil
492+
secret.Data[string(services.RegistryDBPersistenceSQLConfigType)] = []byte(invalidSecretRegistryTypeYamlString)
493+
Expect(k8sClient.Update(ctx, secret)).To(Succeed())
494+
495+
resource.Spec.Services.Registry.Local.Persistence.DBPersistence.SecretRef = corev1.LocalObjectReference{Name: "registry-store-secret"}
496+
resource.Spec.Services.Registry.Local.Persistence.DBPersistence.SecretKeyName = ""
497+
Expect(k8sClient.Update(ctx, resource)).To(Succeed())
498+
resource = &feastdevv1alpha1.FeatureStore{}
499+
err = k8sClient.Get(ctx, typeNamespacedName, resource)
500+
Expect(err).NotTo(HaveOccurred())
501+
502+
_, err = controllerReconciler.Reconcile(ctx, reconcile.Request{
503+
NamespacedName: typeNamespacedName,
504+
})
505+
Expect(err).To(Not(HaveOccurred()))
509506
})
510507

511508
It("should properly encode a feature_store.yaml config", func() {

0 commit comments

Comments
 (0)