Skip to content

Commit 4b64dd5

Browse files
somtochiamasomtochiama
committed
store helm indexes in JSON
Signed-off-by: Somtochi Onyekwere <somtochionyekwere@gmail.com>
1 parent 7f40be7 commit 4b64dd5

File tree

3 files changed

+216
-16
lines changed

3 files changed

+216
-16
lines changed

‎internal/helm/repository/chart_repository.go

+59-7
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,11 @@ type ChartRepository struct {
109109
// URL the ChartRepository's index.yaml can be found at,
110110
// without the index.yaml suffix.
111111
URL string
112-
// Path is the absolute path to the Index file.
112+
// Path is the absolute path to the Index file in JSON format.
113113
Path string
114+
// OriginalPath is the absolute path to the Index file
115+
// as retrieved from the server
116+
OriginalPath string
114117
// Index of the ChartRepository.
115118
Index *repo.IndexFile
116119

@@ -281,27 +284,57 @@ func (r *ChartRepository) DownloadChart(chart *repo.ChartVersion) (*bytes.Buffer
281284
}
282285

283286
// CacheIndex attempts to write the index from the remote into a new temporary file
284-
// using DownloadIndex, and sets Path and cached.
285-
// The caller is expected to handle the garbage collection of Path, and to
287+
// using DownloadIndex. The original file from the server is written to OriginalPath
288+
// and the index is converted to JSON and written to Path. The cached field is also set.
289+
// The caller is expected to handle the garbage collection of Path and OriginalPath, and to
286290
// load the Index separately using LoadFromPath if required.
287291
func (r *ChartRepository) CacheIndex() error {
288292
f, err := os.CreateTemp("", "chart-index-*.yaml")
289293
if err != nil {
290294
return fmt.Errorf("failed to create temp file to cache index to: %w", err)
291295
}
292296

293-
if err = r.DownloadIndex(f); err != nil {
297+
serverIndexFile, err := os.CreateTemp("", "server-chart-index-*.yaml")
298+
if err != nil {
299+
return fmt.Errorf("failed to create temp file to cache index to: %w", err)
300+
}
301+
302+
var b bytes.Buffer
303+
mw := io.MultiWriter(&b, serverIndexFile)
304+
305+
if err = r.DownloadIndex(mw); err != nil {
294306
f.Close()
307+
serverIndexFile.Close()
295308
os.Remove(f.Name())
309+
os.Remove(serverIndexFile.Name())
296310
return fmt.Errorf("failed to cache index to temporary file: %w", err)
297311
}
312+
if err = serverIndexFile.Close(); err != nil {
313+
os.Remove(serverIndexFile.Name())
314+
f.Close()
315+
os.Remove(f.Name())
316+
return fmt.Errorf("failed to close cached index file '%s': %w", f.Name(), err)
317+
}
318+
319+
// convert yaml to json before writing to disk
320+
jsonBytes, err := yaml.YAMLToJSON(b.Bytes())
321+
if err != nil {
322+
return fmt.Errorf("error converting json to yaml: %w", err)
323+
}
324+
325+
if _, err = f.Write(jsonBytes); err != nil {
326+
f.Close()
327+
os.Remove(f.Name())
328+
return fmt.Errorf("error writing index to file: %w", err)
329+
}
298330
if err = f.Close(); err != nil {
299331
os.Remove(f.Name())
300332
return fmt.Errorf("failed to close cached index file '%s': %w", f.Name(), err)
301333
}
302334

303335
r.Lock()
304336
r.Path = f.Name()
337+
r.OriginalPath = serverIndexFile.Name()
305338
r.Index = nil
306339
r.cached = true
307340
r.invalidate()
@@ -380,17 +413,17 @@ func (r *ChartRepository) DownloadIndex(w io.Writer) (err error) {
380413
return nil
381414
}
382415

383-
// Digest returns the digest of the file at the ChartRepository's Path.
416+
// Digest returns the digest of the file at the ChartRepository's OriginalPath.
384417
func (r *ChartRepository) Digest(algorithm digest.Algorithm) digest.Digest {
385-
if !r.HasFile() {
418+
if !r.HasOriginalFile() {
386419
return ""
387420
}
388421

389422
r.Lock()
390423
defer r.Unlock()
391424

392425
if _, ok := r.digests[algorithm]; !ok {
393-
if f, err := os.Open(r.Path); err == nil {
426+
if f, err := os.Open(r.OriginalPath); err == nil {
394427
defer f.Close()
395428
rd := io.LimitReader(f, helm.MaxIndexSize)
396429
if d, err := algorithm.FromReader(rd); err == nil {
@@ -422,6 +455,19 @@ func (r *ChartRepository) HasFile() bool {
422455
return false
423456
}
424457

458+
// HasOriginalFile returns true if OriginalPath exists and is a regular file.
459+
func (r *ChartRepository) HasOriginalFile() bool {
460+
r.RLock()
461+
defer r.RUnlock()
462+
463+
if r.OriginalPath != "" {
464+
if stat, err := os.Lstat(r.OriginalPath); err == nil {
465+
return stat.Mode().IsRegular()
466+
}
467+
}
468+
return false
469+
}
470+
425471
// Clear clears the Index and removes the file at Path, if cached.
426472
func (r *ChartRepository) Clear() error {
427473
r.Lock()
@@ -433,7 +479,13 @@ func (r *ChartRepository) Clear() error {
433479
if err := os.Remove(r.Path); err != nil {
434480
return fmt.Errorf("failed to remove cached index: %w", err)
435481
}
482+
483+
if err := os.Remove(r.OriginalPath); err != nil {
484+
return fmt.Errorf("failed to remove cached server index: %w", err)
485+
}
486+
436487
r.Path = ""
488+
r.OriginalPath = ""
437489
r.cached = false
438490
}
439491

‎internal/helm/repository/chart_repository_test.go

+85-9
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,10 @@ import (
3939
var now = time.Now()
4040

4141
const (
42-
testFile = "../testdata/local-index.yaml"
43-
chartmuseumTestFile = "../testdata/chartmuseum-index.yaml"
44-
unorderedTestFile = "../testdata/local-index-unordered.yaml"
42+
testFile = "../testdata/local-index.yaml"
43+
chartmuseumTestFile = "../testdata/chartmuseum-index.yaml"
44+
chartmuseumTestFileJSON = "../testdata/chartmuseum-index.json"
45+
unorderedTestFile = "../testdata/local-index-unordered.yaml"
4546
)
4647

4748
// mockGetter is a simple mocking getter.Getter implementation, returning
@@ -81,6 +82,10 @@ func TestIndexFromFile(t *testing.T) {
8182
name: "chartmuseum index file",
8283
filename: chartmuseumTestFile,
8384
},
85+
{
86+
name: "chartmuseum index file",
87+
filename: chartmuseumTestFileJSON,
88+
},
8489
{
8590
name: "error if index size exceeds max size",
8691
filename: bigIndexFile,
@@ -151,6 +156,29 @@ entries:
151156
- name: nginx`),
152157
wantErr: "key \"nginx\" already set in map",
153158
},
159+
{
160+
name: "json index",
161+
b: []byte(`{
162+
"apiVersion": "v1",
163+
"entries": {
164+
"nginx": [
165+
{
166+
"urls": [
167+
"https://kubernetes-charts.storage.googleapis.com/nginx-0.2.0.tgz"
168+
],
169+
"name": "nginx",
170+
"description": "string",
171+
"version": "0.2.0",
172+
"home": "https://github.com/something/else",
173+
"digest": "sha256:1234567890abcdef"
174+
}
175+
]
176+
}
177+
}`),
178+
wantName: "nginx",
179+
wantVersion: "0.2.0",
180+
wantDigest: "sha256:1234567890abcdef",
181+
},
154182
}
155183
for _, tt := range tests {
156184
tt := tt
@@ -387,7 +415,8 @@ func TestChartRepository_DownloadChart(t *testing.T) {
387415
func TestChartRepository_CacheIndex(t *testing.T) {
388416
g := NewWithT(t)
389417

390-
mg := mockGetter{Response: []byte("foo")}
418+
mg := mockGetter{Response: []byte("foo: bar")}
419+
jsonResp := []byte(`{"foo":"bar"}`)
391420

392421
r := newChartRepository()
393422
r.URL = "https://example.com"
@@ -397,12 +426,19 @@ func TestChartRepository_CacheIndex(t *testing.T) {
397426
err := r.CacheIndex()
398427
g.Expect(err).To(Not(HaveOccurred()))
399428

429+
g.Expect(r.OriginalPath).ToNot(BeEmpty())
430+
t.Cleanup(func() { _ = os.Remove(r.OriginalPath) })
431+
432+
g.Expect(r.OriginalPath).To(BeARegularFile())
433+
b, _ := os.ReadFile(r.OriginalPath)
434+
g.Expect(b).To(Equal(mg.Response))
435+
400436
g.Expect(r.Path).ToNot(BeEmpty())
401437
t.Cleanup(func() { _ = os.Remove(r.Path) })
402438

403439
g.Expect(r.Path).To(BeARegularFile())
404-
b, _ := os.ReadFile(r.Path)
405-
g.Expect(b).To(Equal(mg.Response))
440+
b, _ = os.ReadFile(r.Path)
441+
g.Expect(b).To(Equal(jsonResp))
406442

407443
g.Expect(r.digests).To(BeEmpty())
408444
}
@@ -442,6 +478,20 @@ func TestChartRepository_StrategicallyLoadIndex(t *testing.T) {
442478
g.Expect(r.Index).ToNot(BeNil())
443479
})
444480

481+
t.Run("loads from path with json format", func(t *testing.T) {
482+
g := NewWithT(t)
483+
484+
i := filepath.Join(t.TempDir(), "index.yaml")
485+
g.Expect(os.WriteFile(i, []byte(`{"apiVersion":"v1"}`), 0o644)).To(Succeed())
486+
487+
r := newChartRepository()
488+
r.Path = i
489+
490+
err := r.StrategicallyLoadIndex()
491+
g.Expect(err).To(Succeed())
492+
g.Expect(r.Index).ToNot(BeNil())
493+
})
494+
445495
t.Run("loads from client", func(t *testing.T) {
446496
g := NewWithT(t)
447497

@@ -456,6 +506,7 @@ func TestChartRepository_StrategicallyLoadIndex(t *testing.T) {
456506
err := r.StrategicallyLoadIndex()
457507
g.Expect(err).To(Succeed())
458508
g.Expect(r.Path).ToNot(BeEmpty())
509+
g.Expect(r.OriginalPath).ToNot(BeEmpty())
459510
g.Expect(r.Index).ToNot(BeNil())
460511
})
461512

@@ -511,7 +562,7 @@ func TestChartRepository_Digest(t *testing.T) {
511562
g.Expect(repo.NewIndexFile().WriteFile(p, 0o644)).To(Succeed())
512563

513564
r := newChartRepository()
514-
r.Path = p
565+
r.OriginalPath = p
515566

516567
for _, algo := range []digest.Algorithm{digest.SHA256, digest.SHA512} {
517568
t.Run(algo.String(), func(t *testing.T) {
@@ -542,7 +593,7 @@ func TestChartRepository_Digest(t *testing.T) {
542593
g.Expect(os.WriteFile(i, []byte(`apiVersion: v1`), 0o644)).To(Succeed())
543594

544595
r := newChartRepository()
545-
r.Path = i
596+
r.OriginalPath = i
546597
r.digests[algo] = expect
547598

548599
g.Expect(r.Digest(algo)).To(Equal(expect))
@@ -565,11 +616,23 @@ func TestChartRepository_HasFile(t *testing.T) {
565616
g.Expect(r.HasFile()).To(BeFalse())
566617

567618
i := filepath.Join(t.TempDir(), "index.yaml")
568-
g.Expect(os.WriteFile(i, []byte(`apiVersion: v1`), 0o644)).To(Succeed())
619+
g.Expect(os.WriteFile(i, []byte(`{"apiVersion":"v1"}`), 0o644)).To(Succeed())
569620
r.Path = i
570621
g.Expect(r.HasFile()).To(BeTrue())
571622
}
572623

624+
func TestChartRepository_HasOriginalFile(t *testing.T) {
625+
g := NewWithT(t)
626+
627+
r := newChartRepository()
628+
g.Expect(r.HasOriginalFile()).To(BeFalse())
629+
630+
i := filepath.Join(t.TempDir(), "index.yaml")
631+
g.Expect(os.WriteFile(i, []byte(`apiVersion: v1`), 0o644)).To(Succeed())
632+
r.OriginalPath = i
633+
g.Expect(r.HasOriginalFile()).To(BeTrue())
634+
}
635+
573636
func TestChartRepository_Clear(t *testing.T) {
574637
t.Run("without index", func(t *testing.T) {
575638
g := NewWithT(t)
@@ -595,15 +658,21 @@ func TestChartRepository_Clear(t *testing.T) {
595658
g.Expect(err).ToNot(HaveOccurred())
596659
g.Expect(f.Close()).To(Succeed())
597660

661+
serverFile, err := os.CreateTemp(t.TempDir(), "chart-index-*.yaml")
662+
g.Expect(err).ToNot(HaveOccurred())
663+
g.Expect(serverFile.Close()).To(Succeed())
664+
598665
r := newChartRepository()
599666
r.Path = f.Name()
667+
r.OriginalPath = serverFile.Name()
600668
r.Index = repo.NewIndexFile()
601669
r.digests["key"] = "value"
602670
r.cached = true
603671

604672
g.Expect(r.Clear()).To(Succeed())
605673
g.Expect(r.Index).To(BeNil())
606674
g.Expect(r.Path).To(BeEmpty())
675+
g.Expect(r.OriginalPath).To(BeEmpty())
607676
g.Expect(r.digests).To(BeEmpty())
608677
g.Expect(r.cached).To(BeFalse())
609678
})
@@ -615,13 +684,20 @@ func TestChartRepository_Clear(t *testing.T) {
615684
g.Expect(err).ToNot(HaveOccurred())
616685
g.Expect(f.Close()).To(Succeed())
617686

687+
serverFile, err := os.CreateTemp(t.TempDir(), "server-index-*.yaml")
688+
g.Expect(err).ToNot(HaveOccurred())
689+
g.Expect(serverFile.Close()).To(Succeed())
690+
618691
r := newChartRepository()
619692
r.Path = f.Name()
693+
r.OriginalPath = serverFile.Name()
620694
r.digests["key"] = "value"
621695

622696
g.Expect(r.Clear()).To(Succeed())
623697
g.Expect(r.Path).ToNot(BeEmpty())
624698
g.Expect(r.Path).To(BeARegularFile())
699+
g.Expect(r.OriginalPath).ToNot(BeEmpty())
700+
g.Expect(r.OriginalPath).To(BeARegularFile())
625701
g.Expect(r.digests).To(BeEmpty())
626702
})
627703
}

0 commit comments

Comments
 (0)