Removed resumable downloads

Author: pa250194

pkg/gcp/gcp.go

@@ -45,12 +45,6 @@ type GCPClient struct {
 	// client for interacting with the Google Cloud
 	// Storage APIs.
 	*gcpStorage.Client
-	// startRange is the starting read value for
-	// reading the object from bucket.
-	StartRange int64
-	// endRange is the ending read value for
-	// reading the object from bucket.
-	EndRange int64
 }
 
 // NewClient creates a new GCP storage client
@@ -63,7 +57,7 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*GCPClient, er
 		return nil, err
 	}
 
-	return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil
+	return &GCPClient{Client: client}, nil
 }
 
 // ValidateSecret validates the credential secrets
@@ -76,18 +70,11 @@ func ValidateSecret(secret map[string][]byte, name string) error {
 	return nil
 }
 
-// SetRange sets the startRange and endRange used to read the Object from
-// the bucket. It is a helper method for resumable downloads.
-func (c *GCPClient) SetRange(start, end int64) {
-	c.StartRange = start
-	c.EndRange = end
-}
-
 // BucketExists checks if the bucket with the provided name exists.
 func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, error) {
 	_, err := c.Client.Bucket(bucketName).Attrs(ctx)
 	if err == gcpStorage.ErrBucketNotExist {
-		return false, nil
+		return false, err
 	}
 	if err != nil {
 		return false, err
@@ -97,16 +84,16 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool,
 
 // ObjectAttributes checks if the object with the provided name exists.
 // If it exists the Object attributes are returned.
-func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) {
-	attrs, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx)
+func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, error) {
+	_, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx)
 	// ErrObjectNotExist is returned if the object does not exist
 	if err == gcpStorage.ErrObjectNotExist {
-		return false, nil, err
+		return false, err
 	}
 	if err != nil {
-		return false, nil, err
+		return false, err
 	}
-	return true, attrs, nil
+	return true, nil
 }
 
 // FGetObject gets the object from the bucket and downloads the object locally
@@ -140,65 +127,33 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca
 	// ObjectExists verifies if object exists and you have permission to access.
 	// Check if the object exists and if you have permission to access it
 	// The Object attributes are returned if the Object exists.
-	exists, attrs, err := c.ObjectAttributes(ctx, bucketName, objectName)
+	exists, err := c.ObjectAttributes(ctx, bucketName, objectName)
 	if err != nil {
 		return err
 	}
 	if !exists {
 		return ErrorObjectDoesNotExist
 	}
 
-	// Write to a temporary file "filename.part.gcp" before saving.
-	filePartPath := localPath + ".part.gcp"
-	// If exists, open in append mode. If not create it as a part file.
-	filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	objectFile, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY, 0600)
 	if err != nil {
 		return err
 	}
 
-	// If we return early with an error, be sure to close and delete
-	// filePart.  If we have an error along the way there is a chance
-	// that filePart is somehow damaged, and we should discard it.
-	closeAndRemove := true
-	defer func() {
-		if closeAndRemove {
-			_ = filePart.Close()
-			_ = os.Remove(filePartPath)
-		}
-	}()
-
-	// Issue Stat to get the current offset.
-	partFileStat, err := filePart.Stat()
-	if err != nil {
-		return err
-	}
-
-	// Set the File size request range
-	// If the part file exists
-	if partFileStat.Size() > 0 {
-		c.SetRange(partFileStat.Size(), 0)
-	}
-
 	// Get Object from GCP Bucket
-	objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.StartRange, c.EndRange)
+	objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewReader(ctx)
 	if err != nil {
 		return err
 	}
 	defer objectReader.Close()
 
-	// Write to the part file.
-	if _, err := io.CopyN(filePart, objectReader, attrs.Size); err != nil {
-		return err
-	}
-
-	// Close the file before rename, this is specifically needed for Windows users.
-	closeAndRemove = false
-	if err := filePart.Close(); err != nil {
+	// Write Object to file.
+	if _, err := io.Copy(objectFile, objectReader); err != nil {
 		return err
 	}
 
-	// Safely completed. Now commit by renaming to actual filename.
-	if err := os.Rename(filePartPath, localPath); err != nil {
+	// Close the file.
+	if err := objectFile.Close(); err != nil {
 		return err
 	}
 

pkg/gcp/gcp_test.go

@@ -118,9 +118,7 @@ func TestNewClient(t *testing.T) {
 
 func TestBucketExists(t *testing.T) {
 	gcpClient := &gcp.GCPClient{
-		Client:     client,
-		StartRange: 0,
-		EndRange:   -1,
+		Client: client,
 	}
 	exists, err := gcpClient.BucketExists(context.Background(), bucketName)
 	assert.NilError(t, err)
@@ -130,35 +128,28 @@ func TestBucketExists(t *testing.T) {
 func TestBucketNotExists(t *testing.T) {
 	bucket := "notexistsbucket"
 	gcpClient := &gcp.GCPClient{
-		Client:     client,
-		StartRange: 0,
-		EndRange:   -1,
+		Client: client,
 	}
 	exists, err := gcpClient.BucketExists(context.Background(), bucket)
-	assert.NilError(t, err)
+	assert.Error(t, err, "storage: bucket doesn't exist")
 	assert.Assert(t, !exists)
 }
 
 func TestObjectAttributes(t *testing.T) {
 	gcpClient := &gcp.GCPClient{
-		Client:     client,
-		StartRange: 0,
-		EndRange:   -1,
+		Client: client,
 	}
-	exists, objectAttrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName)
+	exists, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName)
 	if err == gcpStorage.ErrObjectNotExist {
 		assert.NilError(t, err)
 	}
 	assert.NilError(t, err)
 	assert.Assert(t, exists)
-	assert.Assert(t, objectAttrs != nil)
 }
 
 func TestListObjects(t *testing.T) {
 	gcpClient := &gcp.GCPClient{
-		Client:     client,
-		StartRange: 0,
-		EndRange:   -1,
+		Client: client,
 	}
 	objectInterator := gcpClient.ListObjects(context.Background(), bucketName, nil)
 	for {
@@ -176,9 +167,7 @@ func TestFGetObject(t *testing.T) {
 	assert.NilError(t, err)
 	defer os.RemoveAll(tempDir)
 	gcpClient := &gcp.GCPClient{
-		Client:     client,
-		StartRange: 0,
-		EndRange:   -1,
+		Client: client,
 	}
 	localPath := filepath.Join(tempDir, objectName)
 	err = gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath)
@@ -193,9 +182,7 @@ func TestFGetObjectNotExists(t *testing.T) {
 	assert.NilError(t, err)
 	defer os.RemoveAll(tempDir)
 	gcpClient := &gcp.GCPClient{
-		Client:     client,
-		StartRange: 0,
-		EndRange:   -1,
+		Client: client,
 	}
 	localPath := filepath.Join(tempDir, object)
 	err = gcpClient.FGetObject(context.Background(), bucketName, object, localPath)
@@ -209,27 +196,14 @@ func TestFGetObjectDirectoryIsFileName(t *testing.T) {
 	defer os.RemoveAll(tempDir)
 	assert.NilError(t, err)
 	gcpClient := &gcp.GCPClient{
-		Client:     client,
-		StartRange: 0,
-		EndRange:   -1,
+		Client: client,
 	}
 	err = gcpClient.FGetObject(context.Background(), bucketName, objectName, tempDir)
 	if err != io.EOF {
 		assert.Error(t, err, "filename is a directory")
 	}
 }
 
-func TestSetRange(t *testing.T) {
-	gcpClient := &gcp.GCPClient{
-		Client:     client,
-		StartRange: 0,
-		EndRange:   -1,
-	}
-	gcpClient.SetRange(2, 5)
-	assert.Equal(t, gcpClient.StartRange, int64(2))
-	assert.Equal(t, gcpClient.EndRange, int64(5))
-}
-
 func TestValidateSecret(t *testing.T) {
 	t.Parallel()
 	testCases := []struct {