Skip to content

Commit c741805

Browse files
darkowlzzdarkowlzz
authored
Merge pull request #992 from fluxcd/gc-lock-tests
Garbage collection lock file ignore tests
2 parents 5ccf2fd + b115dda commit c741805

File tree

1 file changed

+120
-67
lines changed

1 file changed

+120
-67
lines changed

‎controllers/storage_test.go

Lines changed: 120 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
"fmt"
2424
"io"
2525
"os"
26-
"path"
2726
"path/filepath"
2827
"strings"
2928
"testing"
@@ -268,7 +267,7 @@ func TestStorageRemoveAllButCurrent(t *testing.T) {
268267
t.Fatalf("Valid path did not successfully return: %v", err)
269268
}
270269

271-
if _, err := s.RemoveAllButCurrent(sourcev1.Artifact{Path: path.Join(dir, "really", "nonexistent")}); err == nil {
270+
if _, err := s.RemoveAllButCurrent(sourcev1.Artifact{Path: filepath.Join(dir, "really", "nonexistent")}); err == nil {
272271
t.Fatal("Did not error while pruning non-existent path")
273272
}
274273
})
@@ -281,18 +280,18 @@ func TestStorageRemoveAllButCurrent(t *testing.T) {
281280
g.Expect(err).ToNot(HaveOccurred(), "failed to create new storage")
282281

283282
artifact := sourcev1.Artifact{
284-
Path: path.Join("foo", "bar", "artifact1.tar.gz"),
283+
Path: filepath.Join("foo", "bar", "artifact1.tar.gz"),
285284
}
286285

287286
// Create artifact dir and artifacts.
288-
artifactDir := path.Join(dir, "foo", "bar")
287+
artifactDir := filepath.Join(dir, "foo", "bar")
289288
g.Expect(os.MkdirAll(artifactDir, 0o750)).NotTo(HaveOccurred())
290289
current := []string{
291-
path.Join(artifactDir, "artifact1.tar.gz"),
290+
filepath.Join(artifactDir, "artifact1.tar.gz"),
292291
}
293292
wantDeleted := []string{
294-
path.Join(artifactDir, "file1.txt"),
295-
path.Join(artifactDir, "file2.txt"),
293+
filepath.Join(artifactDir, "file1.txt"),
294+
filepath.Join(artifactDir, "file2.txt"),
296295
}
297296
createFile := func(files []string) {
298297
for _, c := range files {
@@ -321,15 +320,15 @@ func TestStorageRemoveAll(t *testing.T) {
321320
}{
322321
{
323322
name: "delete non-existent path",
324-
artifactPath: path.Join("foo", "bar", "artifact1.tar.gz"),
323+
artifactPath: filepath.Join("foo", "bar", "artifact1.tar.gz"),
325324
createArtifactPath: false,
326325
wantDeleted: "",
327326
},
328327
{
329328
name: "delete existing path",
330-
artifactPath: path.Join("foo", "bar", "artifact1.tar.gz"),
329+
artifactPath: filepath.Join("foo", "bar", "artifact1.tar.gz"),
331330
createArtifactPath: true,
332-
wantDeleted: path.Join("foo", "bar"),
331+
wantDeleted: filepath.Join("foo", "bar"),
333332
},
334333
}
335334

@@ -346,7 +345,7 @@ func TestStorageRemoveAll(t *testing.T) {
346345
}
347346

348347
if tt.createArtifactPath {
349-
g.Expect(os.MkdirAll(path.Join(dir, tt.artifactPath), 0o750)).ToNot(HaveOccurred())
348+
g.Expect(os.MkdirAll(filepath.Join(dir, tt.artifactPath), 0o750)).ToNot(HaveOccurred())
350349
}
351350

352351
deleted, err := s.RemoveAll(artifact)
@@ -449,7 +448,7 @@ func TestStorageCopyFromPath(t *testing.T) {
449448
}
450449

451450
func TestStorage_getGarbageFiles(t *testing.T) {
452-
artifactFolder := path.Join("foo", "bar")
451+
artifactFolder := filepath.Join("foo", "bar")
453452
tests := []struct {
454453
name string
455454
artifactPaths []string
@@ -462,77 +461,119 @@ func TestStorage_getGarbageFiles(t *testing.T) {
462461
{
463462
name: "delete files based on maxItemsToBeRetained",
464463
artifactPaths: []string{
465-
path.Join(artifactFolder, "artifact1.tar.gz"),
466-
path.Join(artifactFolder, "artifact2.tar.gz"),
467-
path.Join(artifactFolder, "artifact3.tar.gz"),
468-
path.Join(artifactFolder, "artifact4.tar.gz"),
469-
path.Join(artifactFolder, "artifact5.tar.gz"),
464+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
465+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
466+
filepath.Join(artifactFolder, "artifact3.tar.gz"),
467+
filepath.Join(artifactFolder, "artifact4.tar.gz"),
468+
filepath.Join(artifactFolder, "artifact5.tar.gz"),
470469
},
471470
createPause: time.Millisecond * 10,
472471
ttl: time.Minute * 2,
473472
totalCountLimit: 10,
474473
maxItemsToBeRetained: 2,
475474
wantDeleted: []string{
476-
path.Join(artifactFolder, "artifact1.tar.gz"),
477-
path.Join(artifactFolder, "artifact2.tar.gz"),
478-
path.Join(artifactFolder, "artifact3.tar.gz"),
475+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
476+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
477+
filepath.Join(artifactFolder, "artifact3.tar.gz"),
478+
},
479+
},
480+
{
481+
name: "delete files based on maxItemsToBeRetained, ignore lock files",
482+
artifactPaths: []string{
483+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
484+
filepath.Join(artifactFolder, "artifact1.tar.gz.lock"),
485+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
486+
filepath.Join(artifactFolder, "artifact2.tar.gz.lock"),
487+
filepath.Join(artifactFolder, "artifact3.tar.gz"),
488+
filepath.Join(artifactFolder, "artifact3.tar.gz.lock"),
489+
filepath.Join(artifactFolder, "artifact4.tar.gz"),
490+
filepath.Join(artifactFolder, "artifact5.tar.gz"),
491+
},
492+
createPause: time.Millisecond * 10,
493+
ttl: time.Minute * 2,
494+
totalCountLimit: 10,
495+
maxItemsToBeRetained: 2,
496+
wantDeleted: []string{
497+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
498+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
499+
filepath.Join(artifactFolder, "artifact3.tar.gz"),
479500
},
480501
},
481502
{
482503
name: "delete files based on ttl",
483504
artifactPaths: []string{
484-
path.Join(artifactFolder, "artifact1.tar.gz"),
485-
path.Join(artifactFolder, "artifact2.tar.gz"),
486-
path.Join(artifactFolder, "artifact3.tar.gz"),
487-
path.Join(artifactFolder, "artifact4.tar.gz"),
488-
path.Join(artifactFolder, "artifact5.tar.gz"),
505+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
506+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
507+
filepath.Join(artifactFolder, "artifact3.tar.gz"),
508+
filepath.Join(artifactFolder, "artifact4.tar.gz"),
509+
filepath.Join(artifactFolder, "artifact5.tar.gz"),
510+
},
511+
createPause: time.Second * 1,
512+
ttl: time.Second*3 + time.Millisecond*500,
513+
totalCountLimit: 10,
514+
maxItemsToBeRetained: 4,
515+
wantDeleted: []string{
516+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
517+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
518+
},
519+
},
520+
{
521+
name: "delete files based on ttl, ignore lock files",
522+
artifactPaths: []string{
523+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
524+
filepath.Join(artifactFolder, "artifact1.tar.gz.lock"),
525+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
526+
filepath.Join(artifactFolder, "artifact2.tar.gz.lock"),
527+
filepath.Join(artifactFolder, "artifact3.tar.gz"),
528+
filepath.Join(artifactFolder, "artifact4.tar.gz"),
529+
filepath.Join(artifactFolder, "artifact5.tar.gz"),
489530
},
490531
createPause: time.Second * 1,
491532
ttl: time.Second*3 + time.Millisecond*500,
492533
totalCountLimit: 10,
493534
maxItemsToBeRetained: 4,
494535
wantDeleted: []string{
495-
path.Join(artifactFolder, "artifact1.tar.gz"),
496-
path.Join(artifactFolder, "artifact2.tar.gz"),
536+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
537+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
497538
},
498539
},
499540
{
500541
name: "delete files based on ttl and maxItemsToBeRetained",
501542
artifactPaths: []string{
502-
path.Join(artifactFolder, "artifact1.tar.gz"),
503-
path.Join(artifactFolder, "artifact2.tar.gz"),
504-
path.Join(artifactFolder, "artifact3.tar.gz"),
505-
path.Join(artifactFolder, "artifact4.tar.gz"),
506-
path.Join(artifactFolder, "artifact5.tar.gz"),
507-
path.Join(artifactFolder, "artifact6.tar.gz"),
543+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
544+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
545+
filepath.Join(artifactFolder, "artifact3.tar.gz"),
546+
filepath.Join(artifactFolder, "artifact4.tar.gz"),
547+
filepath.Join(artifactFolder, "artifact5.tar.gz"),
548+
filepath.Join(artifactFolder, "artifact6.tar.gz"),
508549
},
509550
createPause: time.Second * 1,
510551
ttl: time.Second*5 + time.Millisecond*500,
511552
totalCountLimit: 10,
512553
maxItemsToBeRetained: 4,
513554
wantDeleted: []string{
514-
path.Join(artifactFolder, "artifact1.tar.gz"),
515-
path.Join(artifactFolder, "artifact2.tar.gz"),
555+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
556+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
516557
},
517558
},
518559
{
519560
name: "delete files based on ttl and maxItemsToBeRetained and totalCountLimit",
520561
artifactPaths: []string{
521-
path.Join(artifactFolder, "artifact1.tar.gz"),
522-
path.Join(artifactFolder, "artifact2.tar.gz"),
523-
path.Join(artifactFolder, "artifact3.tar.gz"),
524-
path.Join(artifactFolder, "artifact4.tar.gz"),
525-
path.Join(artifactFolder, "artifact5.tar.gz"),
526-
path.Join(artifactFolder, "artifact6.tar.gz"),
562+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
563+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
564+
filepath.Join(artifactFolder, "artifact3.tar.gz"),
565+
filepath.Join(artifactFolder, "artifact4.tar.gz"),
566+
filepath.Join(artifactFolder, "artifact5.tar.gz"),
567+
filepath.Join(artifactFolder, "artifact6.tar.gz"),
527568
},
528569
createPause: time.Millisecond * 500,
529570
ttl: time.Millisecond * 500,
530571
totalCountLimit: 3,
531572
maxItemsToBeRetained: 2,
532573
wantDeleted: []string{
533-
path.Join(artifactFolder, "artifact1.tar.gz"),
534-
path.Join(artifactFolder, "artifact2.tar.gz"),
535-
path.Join(artifactFolder, "artifact3.tar.gz"),
574+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
575+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
576+
filepath.Join(artifactFolder, "artifact3.tar.gz"),
536577
},
537578
},
538579
}
@@ -548,9 +589,9 @@ func TestStorage_getGarbageFiles(t *testing.T) {
548589
artifact := sourcev1.Artifact{
549590
Path: tt.artifactPaths[len(tt.artifactPaths)-1],
550591
}
551-
g.Expect(os.MkdirAll(path.Join(dir, artifactFolder), 0o750)).ToNot(HaveOccurred())
592+
g.Expect(os.MkdirAll(filepath.Join(dir, artifactFolder), 0o750)).ToNot(HaveOccurred())
552593
for _, artifactPath := range tt.artifactPaths {
553-
f, err := os.Create(path.Join(dir, artifactPath))
594+
f, err := os.Create(filepath.Join(dir, artifactPath))
554595
g.Expect(err).ToNot(HaveOccurred())
555596
g.Expect(f.Close()).ToNot(HaveOccurred())
556597
time.Sleep(tt.createPause)
@@ -576,35 +617,44 @@ func TestStorage_getGarbageFiles(t *testing.T) {
576617
}
577618

578619
func TestStorage_GarbageCollect(t *testing.T) {
579-
artifactFolder := path.Join("foo", "bar")
620+
artifactFolder := filepath.Join("foo", "bar")
580621
tests := []struct {
581622
name string
582623
artifactPaths []string
624+
wantCollected []string
583625
wantDeleted []string
584626
wantErr string
585627
ctxTimeout time.Duration
586628
}{
587629
{
588630
name: "garbage collects",
589631
artifactPaths: []string{
590-
path.Join(artifactFolder, "artifact1.tar.gz"),
591-
path.Join(artifactFolder, "artifact2.tar.gz"),
592-
path.Join(artifactFolder, "artifact3.tar.gz"),
593-
path.Join(artifactFolder, "artifact4.tar.gz"),
632+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
633+
filepath.Join(artifactFolder, "artifact1.tar.gz.lock"),
634+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
635+
filepath.Join(artifactFolder, "artifact2.tar.gz.lock"),
636+
filepath.Join(artifactFolder, "artifact3.tar.gz"),
637+
filepath.Join(artifactFolder, "artifact4.tar.gz"),
638+
},
639+
wantCollected: []string{
640+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
641+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
594642
},
595643
wantDeleted: []string{
596-
path.Join(artifactFolder, "artifact1.tar.gz"),
597-
path.Join(artifactFolder, "artifact2.tar.gz"),
644+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
645+
filepath.Join(artifactFolder, "artifact1.tar.gz.lock"),
646+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
647+
filepath.Join(artifactFolder, "artifact2.tar.gz.lock"),
598648
},
599649
ctxTimeout: time.Second * 1,
600650
},
601651
{
602652
name: "garbage collection fails with context timeout",
603653
artifactPaths: []string{
604-
path.Join(artifactFolder, "artifact1.tar.gz"),
605-
path.Join(artifactFolder, "artifact2.tar.gz"),
606-
path.Join(artifactFolder, "artifact3.tar.gz"),
607-
path.Join(artifactFolder, "artifact4.tar.gz"),
654+
filepath.Join(artifactFolder, "artifact1.tar.gz"),
655+
filepath.Join(artifactFolder, "artifact2.tar.gz"),
656+
filepath.Join(artifactFolder, "artifact3.tar.gz"),
657+
filepath.Join(artifactFolder, "artifact4.tar.gz"),
608658
},
609659
wantErr: "context deadline exceeded",
610660
ctxTimeout: time.Nanosecond * 1,
@@ -622,39 +672,42 @@ func TestStorage_GarbageCollect(t *testing.T) {
622672
artifact := sourcev1.Artifact{
623673
Path: tt.artifactPaths[len(tt.artifactPaths)-1],
624674
}
625-
g.Expect(os.MkdirAll(path.Join(dir, artifactFolder), 0o750)).ToNot(HaveOccurred())
675+
g.Expect(os.MkdirAll(filepath.Join(dir, artifactFolder), 0o750)).ToNot(HaveOccurred())
626676
for i, artifactPath := range tt.artifactPaths {
627-
f, err := os.Create(path.Join(dir, artifactPath))
677+
f, err := os.Create(filepath.Join(dir, artifactPath))
628678
g.Expect(err).ToNot(HaveOccurred())
629679
g.Expect(f.Close()).ToNot(HaveOccurred())
630680
if i != len(tt.artifactPaths)-1 {
631681
time.Sleep(time.Second * 1)
632682
}
633683
}
634684

635-
deletedPaths, err := s.GarbageCollect(context.TODO(), artifact, tt.ctxTimeout)
685+
collectedPaths, err := s.GarbageCollect(context.TODO(), artifact, tt.ctxTimeout)
636686
if tt.wantErr == "" {
637687
g.Expect(err).ToNot(HaveOccurred(), "failed to collect garbage files")
638688
} else {
639689
g.Expect(err).To(HaveOccurred())
640690
g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
641691
}
642-
if len(tt.wantDeleted) > 0 {
643-
g.Expect(len(tt.wantDeleted)).To(Equal(len(deletedPaths)))
644-
for _, wantDeletedPath := range tt.wantDeleted {
692+
if len(tt.wantCollected) > 0 {
693+
g.Expect(len(tt.wantCollected)).To(Equal(len(collectedPaths)))
694+
for _, wantCollectedPath := range tt.wantCollected {
645695
present := false
646-
for _, deletedPath := range deletedPaths {
647-
if strings.Contains(deletedPath, wantDeletedPath) {
648-
g.Expect(deletedPath).ToNot(BeAnExistingFile())
696+
for _, collectedPath := range collectedPaths {
697+
if strings.Contains(collectedPath, wantCollectedPath) {
698+
g.Expect(collectedPath).ToNot(BeAnExistingFile())
649699
present = true
650700
break
651701
}
652702
}
653703
if present == false {
654-
g.Fail(fmt.Sprintf("expected file to be deleted, still exists: %s", wantDeletedPath))
704+
g.Fail(fmt.Sprintf("expected file to be garbage collected, still exists: %s", wantCollectedPath))
655705
}
656706
}
657707
}
708+
for _, delFile := range tt.wantDeleted {
709+
g.Expect(filepath.Join(dir, delFile)).ToNot(BeAnExistingFile())
710+
}
658711
})
659712
}
660713
}

0 commit comments

Comments
 (0)