|
|
|
|
@ -70,22 +70,17 @@ func (d *SSMDriver) StartSession(ctx context.Context, input ssm.StartSessionInpu
|
|
|
|
|
d.retryConnection = make(chan bool, 1)
|
|
|
|
|
// Starts go routine that will keep listening to a retry channel and retry the session creation when needed.
|
|
|
|
|
// The log loop will add data to the retry channel whenever a retryable error happens to session.
|
|
|
|
|
// TODO @sylviamoss add max retry attempts
|
|
|
|
|
// TODO @sylviamoss zero retry times
|
|
|
|
|
go func(ctx context.Context, driver *SSMDriver, input ssm.StartSessionInput) {
|
|
|
|
|
retryTimes := 0
|
|
|
|
|
for {
|
|
|
|
|
select {
|
|
|
|
|
case <-ctx.Done():
|
|
|
|
|
return
|
|
|
|
|
case <-driver.retryConnection:
|
|
|
|
|
if retryTimes <= 11 {
|
|
|
|
|
retryTimes++
|
|
|
|
|
d.wg.Wait()
|
|
|
|
|
_, err := driver.StartSession(ctx, input)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
d.wg.Wait()
|
|
|
|
|
log.Printf("[DEBUG] Retrying to establish SSM connection")
|
|
|
|
|
_, err := driver.StartSession(ctx, input)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
@ -148,9 +143,11 @@ func (d *SSMDriver) openTunnelForSession(ctx context.Context) error {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if output != "" {
|
|
|
|
|
log.Printf("[ERROR] %s: %s", prefix, output)
|
|
|
|
|
if isRetryableError(output) {
|
|
|
|
|
log.Printf("[ERROR] Retryable error - %s: %s", prefix, output)
|
|
|
|
|
d.retryConnection <- true
|
|
|
|
|
} else {
|
|
|
|
|
log.Printf("[ERROR] %s: %s", prefix, output)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
case output, ok := <-stdoutCh:
|
|
|
|
|
@ -186,9 +183,8 @@ func (d *SSMDriver) openTunnelForSession(ctx context.Context) error {
|
|
|
|
|
|
|
|
|
|
func isRetryableError(output string) bool {
|
|
|
|
|
retryableError := []string{
|
|
|
|
|
"Unable to connect to specified port",
|
|
|
|
|
"connection refused",
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for _, err := range retryableError {
|
|
|
|
|
if strings.Contains(output, err) {
|
|
|
|
|
return true
|
|
|
|
|
|