Merge pull request #10591 from neumayer/retry

Add retry strategies to oci calls

builder/oracle/oci/driver_oci.go

@@ -4,9 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
+	"math/rand"
+	"net/http"
 	"regexp"
+	"sync/atomic"
 	"time"
 
+	"github.com/oracle/oci-go-sdk/common"
 	core "github.com/oracle/oci-go-sdk/core"
 )
 
@@ -19,6 +24,31 @@ type driverOCI struct {
 	context       context.Context
 }
 
+var retryPolicy = &common.RetryPolicy{
+	MaximumNumberAttempts: 10,
+	ShouldRetryOperation: func(res common.OCIOperationResponse) bool {
+		var e common.ServiceError
+		if errors.As(res.Error, &e) {
+			switch e.GetHTTPStatusCode() {
+			case http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusServiceUnavailable:
+				return true
+			}
+		}
+		return false
+	},
+	NextDuration: func(res common.OCIOperationResponse) time.Duration {
+		x := uint64(res.AttemptNumber)
+		d := time.Duration(math.Pow(2, float64(atomic.LoadUint64(&x)))) * time.Second
+		j := time.Duration(rand.Float64()*(2000)) * time.Millisecond
+		w := d + j
+		return w
+	},
+}
+
+var requestMetadata = common.RequestMetadata{
+	RetryPolicy: retryPolicy,
+}
+
 // NewDriverOCI Creates a new driverOCI with a connected compute client and a connected vcn client.
 func NewDriverOCI(cfg *Config) (Driver, error) {
 	coreClient, err := core.NewComputeClientWithConfigurationProvider(cfg.configProvider)
@@ -80,6 +110,7 @@ func (d *driverOCI) CreateInstance(ctx context.Context, publicKey string) (strin
 			LifecycleState:         "AVAILABLE",
 			SortBy:                 "TIMECREATED",
 			SortOrder:              "DESC",
+			RequestMetadata:        requestMetadata,
 		})
 		if err != nil {
 			return "", err
@@ -127,7 +158,10 @@ func (d *driverOCI) CreateInstance(ctx context.Context, publicKey string) (strin
 		Metadata:           metadata,
 	}
 
-	instance, err := d.computeClient.LaunchInstance(context.TODO(), core.LaunchInstanceRequest{LaunchInstanceDetails: instanceDetails})
+	instance, err := d.computeClient.LaunchInstance(context.TODO(), core.LaunchInstanceRequest{
+		LaunchInstanceDetails: instanceDetails,
+		RequestMetadata:       requestMetadata,
+	})
 
 	if err != nil {
 		return "", err
@@ -145,7 +179,9 @@ func (d *driverOCI) CreateImage(ctx context.Context, id string) (core.Image, err
 		FreeformTags:  d.cfg.Tags,
 		DefinedTags:   d.cfg.DefinedTags,
 		LaunchMode:    core.CreateImageDetailsLaunchModeEnum(d.cfg.LaunchMode),
-	}})
+	},
+		RequestMetadata: requestMetadata,
+	})
 
 	if err != nil {
 		return core.Image{}, err
@@ -156,15 +192,19 @@ func (d *driverOCI) CreateImage(ctx context.Context, id string) (core.Image, err
 
 // DeleteImage deletes a custom image.
 func (d *driverOCI) DeleteImage(ctx context.Context, id string) error {
-	_, err := d.computeClient.DeleteImage(ctx, core.DeleteImageRequest{ImageId: &id})
+	_, err := d.computeClient.DeleteImage(ctx, core.DeleteImageRequest{
+		ImageId:         &id,
+		RequestMetadata: requestMetadata,
+	})
 	return err
 }
 
 // GetInstanceIP returns the public or private IP corresponding to the given instance id.
 func (d *driverOCI) GetInstanceIP(ctx context.Context, id string) (string, error) {
 	vnics, err := d.computeClient.ListVnicAttachments(ctx, core.ListVnicAttachmentsRequest{
-		InstanceId:    &id,
-		CompartmentId: &d.cfg.CompartmentID,
+		InstanceId:      &id,
+		CompartmentId:   &d.cfg.CompartmentID,
+		RequestMetadata: requestMetadata,
 	})
 	if err != nil {
 		return "", err
@@ -174,7 +214,10 @@ func (d *driverOCI) GetInstanceIP(ctx context.Context, id string) (string, error
 		return "", errors.New("instance has zero VNICs")
 	}
 
-	vnic, err := d.vcnClient.GetVnic(ctx, core.GetVnicRequest{VnicId: vnics.Items[0].VnicId})
+	vnic, err := d.vcnClient.GetVnic(ctx, core.GetVnicRequest{
+		VnicId:          vnics.Items[0].VnicId,
+		RequestMetadata: requestMetadata,
+	})
 	if err != nil {
 		return "", fmt.Errorf("Error getting VNIC details: %s", err)
 	}
@@ -192,7 +235,8 @@ func (d *driverOCI) GetInstanceIP(ctx context.Context, id string) (string, error
 
 func (d *driverOCI) GetInstanceInitialCredentials(ctx context.Context, id string) (string, string, error) {
 	credentials, err := d.computeClient.GetWindowsInstanceInitialCredentials(ctx, core.GetWindowsInstanceInitialCredentialsRequest{
-		InstanceId: &id,
+		InstanceId:      &id,
+		RequestMetadata: requestMetadata,
 	})
 	if err != nil {
 		return "", "", err
@@ -204,7 +248,8 @@ func (d *driverOCI) GetInstanceInitialCredentials(ctx context.Context, id string
 // TerminateInstance terminates a compute instance.
 func (d *driverOCI) TerminateInstance(ctx context.Context, id string) error {
 	_, err := d.computeClient.TerminateInstance(ctx, core.TerminateInstanceRequest{
-		InstanceId: &id,
+		InstanceId:      &id,
+		RequestMetadata: requestMetadata,
 	})
 	return err
 }
@@ -214,7 +259,10 @@ func (d *driverOCI) TerminateInstance(ctx context.Context, id string) error {
 func (d *driverOCI) WaitForImageCreation(ctx context.Context, id string) error {
 	return waitForResourceToReachState(
 		func(string) (string, error) {
-			image, err := d.computeClient.GetImage(ctx, core.GetImageRequest{ImageId: &id})
+			image, err := d.computeClient.GetImage(ctx, core.GetImageRequest{
+				ImageId:         &id,
+				RequestMetadata: requestMetadata,
+			})
 			if err != nil {
 				return "", err
 			}
@@ -233,7 +281,10 @@ func (d *driverOCI) WaitForImageCreation(ctx context.Context, id string) error {
 func (d *driverOCI) WaitForInstanceState(ctx context.Context, id string, waitStates []string, terminalState string) error {
 	return waitForResourceToReachState(
 		func(string) (string, error) {
-			instance, err := d.computeClient.GetInstance(ctx, core.GetInstanceRequest{InstanceId: &id})
+			instance, err := d.computeClient.GetInstance(ctx, core.GetInstanceRequest{
+				InstanceId:      &id,
+				RequestMetadata: requestMetadata,
+			})
 			if err != nil {
 				return "", err
 			}