Merge pull request #1451 from zhifei92/support-total-pods-limit

The descheduler supports limiting the total number of pods evicted per rescheduling cycle

pkg/descheduler/descheduler.go

@@ -102,6 +102,7 @@ func newDescheduler(rs *options.DeschedulerServer, deschedulerPolicy *api.Desche
 			WithPolicyGroupVersion(evictionPolicyGroupVersion).
 			WithMaxPodsToEvictPerNode(deschedulerPolicy.MaxNoOfPodsToEvictPerNode).
 			WithMaxPodsToEvictPerNamespace(deschedulerPolicy.MaxNoOfPodsToEvictPerNamespace).
+			WithMaxPodsToEvictTotal(deschedulerPolicy.MaxNoOfPodsToEvictTotal).
 			WithDryRun(rs.DryRun).
 			WithMetricsEnabled(!rs.DisableMetrics),
 	)

pkg/descheduler/evictions/errors.go

@@ -31,3 +31,15 @@ func NewEvictionNamespaceLimitError(namespace string) *EvictionNamespaceLimitErr
 }
 
 var _ error = &EvictionNamespaceLimitError{}
+
+type EvictionTotalLimitError struct{}
+
+func (e EvictionTotalLimitError) Error() string {
+	return "maximum number of evicted pods per a descheduling cycle reached"
+}
+
+func NewEvictionTotalLimitError() *EvictionTotalLimitError {
+	return &EvictionTotalLimitError{}
+}
+
+var _ error = &EvictionTotalLimitError{}

pkg/descheduler/evictions/evictions.go

@@ -47,8 +47,10 @@ type PodEvictor struct {
 	dryRun                     bool
 	maxPodsToEvictPerNode      *uint
 	maxPodsToEvictPerNamespace *uint
-	nodepodCount               nodePodEvictedCount
+	maxPodsToEvictTotal        *uint
+	nodePodCount               nodePodEvictedCount
 	namespacePodCount          namespacePodEvictCount
+	totalPodCount              uint
 	metricsEnabled             bool
 	eventRecorder              events.EventRecorder
 }
@@ -69,29 +71,27 @@ func NewPodEvictor(
 		dryRun:                     options.dryRun,
 		maxPodsToEvictPerNode:      options.maxPodsToEvictPerNode,
 		maxPodsToEvictPerNamespace: options.maxPodsToEvictPerNamespace,
+		maxPodsToEvictTotal:        options.maxPodsToEvictTotal,
 		metricsEnabled:             options.metricsEnabled,
-		nodepodCount:               make(nodePodEvictedCount),
+		nodePodCount:               make(nodePodEvictedCount),
 		namespacePodCount:          make(namespacePodEvictCount),
 	}
 }
 
 // NodeEvicted gives a number of pods evicted for node
 func (pe *PodEvictor) NodeEvicted(node *v1.Node) uint {
-	return pe.nodepodCount[node.Name]
+	return pe.nodePodCount[node.Name]
 }
 
 // TotalEvicted gives a number of pods evicted through all nodes
 func (pe *PodEvictor) TotalEvicted() uint {
-	var total uint
-	for _, count := range pe.nodepodCount {
-		total += count
-	}
-	return total
+	return pe.totalPodCount
 }
 
 func (pe *PodEvictor) ResetCounters() {
-	pe.nodepodCount = make(nodePodEvictedCount)
+	pe.nodePodCount = make(nodePodEvictedCount)
 	pe.namespacePodCount = make(namespacePodEvictCount)
+	pe.totalPodCount = 0
 }
 
 func (pe *PodEvictor) SetClient(client clientset.Interface) {
@@ -115,8 +115,18 @@ func (pe *PodEvictor) EvictPod(ctx context.Context, pod *v1.Pod, opts EvictOptio
 	ctx, span = tracing.Tracer().Start(ctx, "EvictPod", trace.WithAttributes(attribute.String("podName", pod.Name), attribute.String("podNamespace", pod.Namespace), attribute.String("reason", opts.Reason), attribute.String("operation", tracing.EvictOperation)))
 	defer span.End()
 
+	if pe.maxPodsToEvictTotal != nil && pe.totalPodCount+1 > *pe.maxPodsToEvictTotal {
+		err := NewEvictionTotalLimitError()
+		if pe.metricsEnabled {
+			metrics.PodsEvicted.With(map[string]string{"result": err.Error(), "strategy": opts.StrategyName, "namespace": pod.Namespace, "node": pod.Spec.NodeName, "profile": opts.ProfileName}).Inc()
+		}
+		span.AddEvent("Eviction Failed", trace.WithAttributes(attribute.String("node", pod.Spec.NodeName), attribute.String("err", err.Error())))
+		klog.ErrorS(err, "Error evicting pod", "limit", *pe.maxPodsToEvictTotal)
+		return err
+	}
+
 	if pod.Spec.NodeName != "" {
-		if pe.maxPodsToEvictPerNode != nil && pe.nodepodCount[pod.Spec.NodeName]+1 > *pe.maxPodsToEvictPerNode {
+		if pe.maxPodsToEvictPerNode != nil && pe.nodePodCount[pod.Spec.NodeName]+1 > *pe.maxPodsToEvictPerNode {
 			err := NewEvictionNodeLimitError(pod.Spec.NodeName)
 			if pe.metricsEnabled {
 				metrics.PodsEvicted.With(map[string]string{"result": err.Error(), "strategy": opts.StrategyName, "namespace": pod.Namespace, "node": pod.Spec.NodeName, "profile": opts.ProfileName}).Inc()
@@ -149,9 +159,10 @@ func (pe *PodEvictor) EvictPod(ctx context.Context, pod *v1.Pod, opts EvictOptio
 	}
 
 	if pod.Spec.NodeName != "" {
-		pe.nodepodCount[pod.Spec.NodeName]++
+		pe.nodePodCount[pod.Spec.NodeName]++
 	}
 	pe.namespacePodCount[pod.Namespace]++
+	pe.totalPodCount++
 
 	if pe.metricsEnabled {
 		metrics.PodsEvicted.With(map[string]string{"result": "success", "strategy": opts.StrategyName, "namespace": pod.Namespace, "node": pod.Spec.NodeName, "profile": opts.ProfileName}).Inc()

pkg/descheduler/evictions/options.go

@@ -9,6 +9,7 @@ type Options struct {
 	dryRun                     bool
 	maxPodsToEvictPerNode      *uint
 	maxPodsToEvictPerNamespace *uint
+	maxPodsToEvictTotal        *uint
 	metricsEnabled             bool
 }
 
@@ -39,6 +40,11 @@ func (o *Options) WithMaxPodsToEvictPerNamespace(maxPodsToEvictPerNamespace *uin
 	return o
 }
 
+func (o *Options) WithMaxPodsToEvictTotal(maxPodsToEvictTotal *uint) *Options {
+	o.maxPodsToEvictTotal = maxPodsToEvictTotal
+	return o
+}
+
 func (o *Options) WithMetricsEnabled(metricsEnabled bool) *Options {
 	o.metricsEnabled = metricsEnabled
 	return o