verbose logging + retries for http calls that unexpected EOF

Author: NorseGaud

internal/ankacloud/client.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/veertuinc/anka-cloud-gitlab-executor/internal/gitlab"
@@ -77,7 +78,25 @@ func (c *APIClient) Post(ctx context.Context, endpoint string, payload interface
 
 	bodyBytes, err := io.ReadAll(r.Body)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read response body: %w", err)
+		if strings.Contains(err.Error(), "unexpected EOF") {
+			// Retry once on unexpected EOF
+			r2, err2 := c.HttpClient.Do(req)
+			if err2 != nil {
+				if e, ok := err2.(*url.Error); ok && e.Timeout() {
+					return nil, gitlab.TransientError(fmt.Errorf("failed to send POST request to %s with payload %+v (retry): %w", endpointUrl, payload, e))
+				}
+				return nil, fmt.Errorf("failed to send POST request to %s with payload %+v (retry): %w", endpoint, payload, err2)
+			}
+			defer r2.Body.Close()
+
+			bodyBytes, err2 = io.ReadAll(r2.Body)
+			if err2 != nil {
+				return nil, fmt.Errorf("failed to read response body (retry): %w", err2)
+			}
+			r = r2
+		} else {
+			return nil, fmt.Errorf("failed to read response body: %w", err)
+		}
 	}
 
 	baseResponse, err := c.parse(bodyBytes)
@@ -123,7 +142,25 @@ func (c *APIClient) Delete(ctx context.Context, endpoint string, payload interfa
 
 	bodyBytes, err := io.ReadAll(r.Body)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read response body: %w", err)
+		if strings.Contains(err.Error(), "unexpected EOF") {
+			// Retry once on unexpected EOF
+			r2, err2 := c.HttpClient.Do(req)
+			if err2 != nil {
+				if e, ok := err2.(*url.Error); ok && e.Timeout() {
+					return nil, gitlab.TransientError(fmt.Errorf("failed to send DELETE request to %s with payload %+v (retry): %w", endpointUrl, payload, e))
+				}
+				return nil, fmt.Errorf("failed to send DELETE request to %s with payload %+v (retry): %w", endpoint, payload, err2)
+			}
+			defer r2.Body.Close()
+
+			bodyBytes, err2 = io.ReadAll(r2.Body)
+			if err2 != nil {
+				return nil, fmt.Errorf("failed to read response body (retry): %w", err2)
+			}
+			r = r2
+		} else {
+			return nil, fmt.Errorf("failed to read response body: %w", err)
+		}
 	}
 
 	baseResponse, err := c.parse(bodyBytes)
@@ -166,7 +203,25 @@ func (c *APIClient) Get(ctx context.Context, endpoint string, queryParams map[st
 
 	bodyBytes, err := io.ReadAll(r.Body)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read response body: %w", err)
+		if strings.Contains(err.Error(), "unexpected EOF") {
+			// Retry once on unexpected EOF
+			r2, err2 := c.HttpClient.Do(req)
+			if err2 != nil {
+				if e, ok := err2.(*url.Error); ok && e.Timeout() {
+					return nil, gitlab.TransientError(fmt.Errorf("failed to send GET request to %s (retry): %w", endpointUrl, e))
+				}
+				return nil, fmt.Errorf("failed to send GET request to %s (retry): %w", endpointUrl, err2)
+			}
+			defer r2.Body.Close()
+
+			bodyBytes, err2 = io.ReadAll(r2.Body)
+			if err2 != nil {
+				return nil, fmt.Errorf("failed to read response body (retry): %w", err2)
+			}
+			r = r2
+		} else {
+			return nil, fmt.Errorf("failed to read response body: %w", err)
+		}
 	}
 
 	baseResponse, err := c.parse(bodyBytes)

internal/ankacloud/controller.go

@@ -176,7 +176,7 @@ func (c *controller) GetAllInstances(ctx context.Context) ([]Instance, error) {
 
 	body, err := c.APIClient.Get(ctx, "/api/v1/vm", nil)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get instances: %w", err)
+		return nil, fmt.Errorf("failed to get all instances: %w", err)
 	}
 
 	var response getAllInstancesResponse
@@ -199,7 +199,7 @@ func (c *controller) GetInstanceByExternalId(ctx context.Context, externalId str
 		return nil, fmt.Errorf("no instances returned from controller")
 	}
 	if err != nil {
-		return nil, fmt.Errorf("failed to get instances: %w", err)
+		return nil, fmt.Errorf("failed to get instance by external id %s: %w", externalId, err)
 	}
 
 	for _, instance := range instances {

internal/command/cleanup.go

@@ -46,9 +46,9 @@ func executeCleanup(ctx context.Context, env gitlab.Environment) error {
 		log.Errorf("cleanup: failed to get instance by external id %q: %v", env.GitlabJobUrl, err)
 		return fmt.Errorf("cleanup: failed to get instance by external id %q: %v", env.GitlabJobUrl, err)
 	}
-	log.Debugf("instance id: %s\n", instance.Id)
+	log.Printf("instance id: %s\n", instance.Id)
 
-	log.Debugf("Issuing termination request for instance %s\n", instance.Id)
+	log.Printf("Issuing termination request for instance %s\n", instance.Id)
 	err = controller.TerminateInstance(ctx, ankacloud.TerminateInstanceRequest{
 		Id: instance.Id,
 	})
@@ -57,6 +57,6 @@ func executeCleanup(ctx context.Context, env gitlab.Environment) error {
 		return fmt.Errorf("cleanup: failed to terminate instance %q: %v", instance.Id, err)
 	}
 
-	log.Colorf("cleanup stage completed for job: %s", env.GitlabJobUrl)
+	log.Println("cleanup stage completed for job: ", env.GitlabJobUrl)
 	return nil
 }