Skip to content

Add logic for pending pods timeout #482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
kstrenkova wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Add logic for pending pods timeout #482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions api/bases/test.openstack.org_ansibletests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1276,6 +1276,11 @@ spec:
description: OpenStackConfigSecret is the name of the Secret containing
the secure.yaml
type: string
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
privileged:
default: false
description: |-
Expand Down Expand Up @@ -1507,6 +1512,11 @@ spec:
description: OpenStackConfigSecret is the name of the Secret
containing the secure.yaml
type: string
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
privileged:
description: |-
Use with caution! This parameter specifies whether test-operator should spawn test
Expand Down
5 changes: 5 additions & 0 deletions api/bases/test.openstack.org_horizontests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1304,6 +1304,11 @@ spec:
tests.
maxLength: 253
type: string
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
privileged:
default: false
description: |-
Expand Down
10 changes: 10 additions & 0 deletions api/bases/test.openstack.org_tempests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1275,6 +1275,11 @@ spec:
instances of test-operator related CRs exist. If you want to turn off this
behaviour then set this option to true.
type: boolean
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
privileged:
default: false
description: |-
Expand Down Expand Up @@ -1848,6 +1853,11 @@ spec:
instances of test-operator related CRs exist. If you want to turn off this
behaviour then set this option to true.
type: boolean
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
privileged:
description: |-
Use with caution! This parameter specifies whether test-operator should spawn test
Expand Down
10 changes: 10 additions & 0 deletions api/bases/test.openstack.org_tobikoes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1281,6 +1281,11 @@ spec:
format: uri
type: string
type: object
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
preventCreate:
default: false
description: Boolean specifying whether tobiko tests create new resources
Expand Down Expand Up @@ -1541,6 +1546,11 @@ spec:
format: uri
type: string
type: object
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
preventCreate:
description: Boolean specifying whether tobiko tests create
new resources or re-use those previously created
Expand Down
5 changes: 5 additions & 0 deletions api/v1beta1/ansibletest_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,11 @@ func (instance *AnsibleTest) GetConditions() *condition.Conditions {
return &instance.Status.Conditions
}

// GetPendingTimeout - return the pending timeout
func (instance *AnsibleTest) GetPendingTimeout() int {
return instance.Spec.PendingTimeout
}

// GetStorageClass - return the storage class name
func (instance *AnsibleTest) GetStorageClass() string {
return instance.Spec.StorageClass
Expand Down
12 changes: 12 additions & 0 deletions api/v1beta1/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,12 @@ type CommonOptions struct {
// ExtraMounts containing conf files, credentials and storage volumes
ExtraMounts []ExtraVolMounts `json:"extraMounts,omitempty"`

// +operator-sdk:csv:customresourcedefinitions:type=spec
// +kubebuilder:validation:Optional
// Timeout after which a test pod in the Pending state is considered
// stuck and its execution is terminated.
PendingTimeout int `json:"pendingTimeout"`

// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec
// This value contains a nodeSelector value that is applied to test pods
Expand Down Expand Up @@ -186,6 +192,12 @@ type WorkflowCommonOptions struct {
// Please use ExtraMounts parameter instead!
ExtraConfigmapsMounts *[]ExtraConfigmapsMounts `json:"extraConfigmapsMounts,omitempty"`

// +operator-sdk:csv:customresourcedefinitions:type=spec
// +kubebuilder:validation:Optional
// Timeout after which a test pod in the Pending state is considered
// stuck and its execution is terminated.
PendingTimeout *int `json:"pendingTimeout,omitempty"`

// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec
// This value contains a nodeSelector value that is applied to test pods
Expand Down
5 changes: 5 additions & 0 deletions api/v1beta1/horizontest_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,11 @@ func (instance *HorizonTest) GetConditions() *condition.Conditions {
return &instance.Status.Conditions
}

// GetPendingTimeout - return the pending timeout
func (instance *HorizonTest) GetPendingTimeout() int {
return instance.Spec.PendingTimeout
}

// GetStorageClass - return the storage class name
func (instance *HorizonTest) GetStorageClass() string {
return instance.Spec.StorageClass
Expand Down
5 changes: 5 additions & 0 deletions api/v1beta1/tempest_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,11 @@ func (instance *Tempest) GetConditions() *condition.Conditions {
return &instance.Status.Conditions
}

// GetPendingTimeout - return the pending timeout
func (instance *Tempest) GetPendingTimeout() int {
return instance.Spec.PendingTimeout
}

// GetStorageClass - return the storage class name
func (instance *Tempest) GetStorageClass() string {
return instance.Spec.StorageClass
Expand Down
5 changes: 5 additions & 0 deletions api/v1beta1/tobiko_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,11 @@ func (instance *Tobiko) GetConditions() *condition.Conditions {
return &instance.Status.Conditions
}

// GetPendingTimeout - return the pending timeout
func (instance *Tobiko) GetPendingTimeout() int {
return instance.Spec.PendingTimeout
}

// GetStorageClass - return the storage class name
func (instance *Tobiko) GetStorageClass() string {
return instance.Spec.StorageClass
Expand Down
5 changes: 5 additions & 0 deletions api/v1beta1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 10 additions & 0 deletions config/crd/bases/test.openstack.org_ansibletests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1276,6 +1276,11 @@ spec:
description: OpenStackConfigSecret is the name of the Secret containing
the secure.yaml
type: string
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
privileged:
default: false
description: |-
Expand Down Expand Up @@ -1507,6 +1512,11 @@ spec:
description: OpenStackConfigSecret is the name of the Secret
containing the secure.yaml
type: string
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
privileged:
description: |-
Use with caution! This parameter specifies whether test-operator should spawn test
Expand Down
5 changes: 5 additions & 0 deletions config/crd/bases/test.openstack.org_horizontests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1304,6 +1304,11 @@ spec:
tests.
maxLength: 253
type: string
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
privileged:
default: false
description: |-
Expand Down
10 changes: 10 additions & 0 deletions config/crd/bases/test.openstack.org_tempests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1275,6 +1275,11 @@ spec:
instances of test-operator related CRs exist. If you want to turn off this
behaviour then set this option to true.
type: boolean
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
privileged:
default: false
description: |-
Expand Down Expand Up @@ -1848,6 +1853,11 @@ spec:
instances of test-operator related CRs exist. If you want to turn off this
behaviour then set this option to true.
type: boolean
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
privileged:
description: |-
Use with caution! This parameter specifies whether test-operator should spawn test
Expand Down
10 changes: 10 additions & 0 deletions config/crd/bases/test.openstack.org_tobikoes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1281,6 +1281,11 @@ spec:
format: uri
type: string
type: object
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
preventCreate:
default: false
description: Boolean specifying whether tobiko tests create new resources
Expand Down Expand Up @@ -1541,6 +1546,11 @@ spec:
format: uri
type: string
type: object
pendingTimeout:
description: |-
Timeout after which a test pod in the Pending state is considered
stuck and its execution is terminated.
type: integer
preventCreate:
description: Boolean specifying whether tobiko tests create
new resources or re-use those previously created
Expand Down
12 changes: 12 additions & 0 deletions internal/controller/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ const (
workflowStepLabel = "workflowStep"
instanceNameLabel = "instanceName"
operatorNameLabel = "operator"
pendingTimeoutAnnotation = "test.openstack.org/pending-timeout"
testOperatorLockName = "test-operator-lock"
testOperatorLockOwnerField = "owner"
testOperatorBaseDir = "/etc/test_operator/"
Expand All @@ -58,6 +59,8 @@ const (
InfoWaitingOnPod = "Waiting on either pod to finish or release of the lock."
// InfoPendingPod is the info message when waiting for a pending pod to start
InfoPendingPod = "Waiting for pending pod to start running."
// InfoPendingPodTimeout is the info message when a pending pod exceeds its timeout
InfoPendingPodTimeout = "Pod exceeded pending timeout."
// InfoTestingCompleted is the info message when all testing is completed
InfoTestingCompleted = "Testing completed. All pods spawned by the test-operator finished."
// InfoCreatingFirstPod is the info message when creating the first test pod
Expand Down Expand Up @@ -203,6 +206,15 @@ func (r *Reconciler) NextAction(
return Failure, workflowStepIdx, err
}

// if the last pod has exceeded pending timeout
if lastPod.Annotations[pendingTimeoutAnnotation] == "true" {
if !isLastPodIndex(workflowStepIdx, workflowLength) {
workflowStepIdx++
return CreateNextPod, workflowStepIdx, nil
}
return EndTesting, workflowStepIdx, nil
}

switch lastPod.Status.Phase {
case corev1.PodPending:
// If the last pod is in Pending state -> CheckPending
Expand Down
31 changes: 29 additions & 2 deletions internal/controller/common_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"fmt"
"strconv"
"time"

"github.com/go-logr/logr"
"github.com/openstack-k8s-operators/lib-common/modules/common"
Expand All @@ -36,6 +37,7 @@ import (
type TestResource interface {
client.Object
GetConditions() *condition.Conditions
GetPendingTimeout() int
GetStorageClass() string
SetObservedGeneration()
}
Expand Down Expand Up @@ -211,8 +213,33 @@ func CommonReconcile[T TestResource](

switch nextAction {
case CheckPending:
Log.Info(InfoPendingPod)
return ctrl.Result{RequeueAfter: RequeueAfterValue}, nil
pendingTimeout := time.Duration(instance.GetPendingTimeout()) * time.Second

lastPod, err := r.GetLastPod(ctx, instance)
if err != nil {
return ctrl.Result{}, err
}

if pendingTimeout <= 0 || time.Since(lastPod.CreationTimestamp.Time) <= pendingTimeout {
Log.Info(InfoPendingPod)
return ctrl.Result{RequeueAfter: RequeueAfterValue}, nil
}

Log.Info(InfoPendingPodTimeout)

if lastPod.Annotations == nil {
lastPod.Annotations = make(map[string]string)
}
lastPod.Annotations[pendingTimeoutAnnotation] = "true"

var deadline int64 = 1
lastPod.Spec.ActiveDeadlineSeconds = &deadline

if err := r.Client.Update(ctx, lastPod); err != nil {
return ctrl.Result{}, err
}

return ctrl.Result{Requeue: true}, nil

case Wait:
Log.Info(InfoWaitingOnPod)
Expand Down