Merge pull request #5252 from Shelley-BaoYue/kubeletconfig

modification for edged config

5 files changed, 94 insertions, 74 deletions

diff --git a/pkg/apis/componentconfig/edgecore/v1alpha2/default_kubelet_configuration.go b/pkg/apis/componentconfig/edgecore/v1alpha2/default_kubelet_configuration.go
index 11bf0eb55..dabaf5556 100644
--- a/pkg/apis/componentconfig/edgecore/v1alpha2/default_kubelet_configuration.go
+++ b/pkg/apis/componentconfig/edgecore/v1alpha2/default_kubelet_configuration.go
@@ -39,22 +39,25 @@ import (
 
 // SetDefaultsKubeletConfiguration sets defaults for tailored kubelet configuration
 func SetDefaultsKubeletConfiguration(obj *TailoredKubeletConfiguration) {
+	obj.StaticPodPath = constants.DefaultManifestsDir
 	obj.SyncFrequency = metav1.Duration{Duration: 1 * time.Minute}
 	obj.Address = constants.ServerAddress
 	obj.ReadOnlyPort = constants.ServerPort
 	obj.ClusterDomain = constants.DefaultClusterDomain
-	obj.RegistryPullQPS = utilpointer.Int32Ptr(5)
+	obj.RegistryPullQPS = utilpointer.Int32(5)
 	obj.RegistryBurst = 10
-	obj.EnableDebuggingHandlers = utilpointer.BoolPtr(true)
-	obj.OOMScoreAdj = utilpointer.Int32Ptr(int32(qos.KubeletOOMScoreAdj))
+	obj.EventRecordQPS = utilpointer.Int32(50)
+	obj.EventBurst = 100
+	obj.EnableDebuggingHandlers = utilpointer.Bool(true)
+	obj.OOMScoreAdj = utilpointer.Int32(int32(qos.KubeletOOMScoreAdj))
 	obj.StreamingConnectionIdleTimeout = metav1.Duration{Duration: 4 * time.Hour}
 	obj.NodeStatusReportFrequency = metav1.Duration{Duration: 5 * time.Minute}
 	obj.NodeStatusUpdateFrequency = metav1.Duration{Duration: 10 * time.Second}
 	obj.NodeLeaseDurationSeconds = 40
 	obj.ImageMinimumGCAge = metav1.Duration{Duration: 2 * time.Minute}
 	// default is below docker's default dm.min_free_space of 90%
-	obj.ImageGCHighThresholdPercent = utilpointer.Int32Ptr(constants.DefaultImageGCHighThreshold)
-	obj.ImageGCLowThresholdPercent = utilpointer.Int32Ptr(constants.DefaultImageGCLowThreshold)
+	obj.ImageGCHighThresholdPercent = utilpointer.Int32(85)
+	obj.ImageGCLowThresholdPercent = utilpointer.Int32(80)
 	obj.VolumeStatsAggPeriod = metav1.Duration{Duration: time.Minute}
 	obj.CPUManagerPolicy = "none"
 	// Keep the same as default NodeStatusUpdateFrequency
@@ -68,38 +71,37 @@ func SetDefaultsKubeletConfiguration(obj *TailoredKubeletConfiguration) {
 	// default nil or negative value to -1 (implies node allocatable pid limit)
 	obj.PodPidsLimit = utilpointer.Int64(-1)
 	obj.CPUCFSQuotaPeriod = &metav1.Duration{Duration: 100 * time.Millisecond}
-	obj.NodeStatusMaxImages = utilpointer.Int32Ptr(0)
+	obj.NodeStatusMaxImages = utilpointer.Int32(0)
 	obj.MaxOpenFiles = 1000000
 	obj.ContentType = "application/json"
-	obj.SerializeImagePulls = utilpointer.BoolPtr(true)
+	obj.SerializeImagePulls = utilpointer.Bool(true)
 	obj.EvictionHard = eviction.DefaultEvictionHard
 	obj.EvictionPressureTransitionPeriod = metav1.Duration{Duration: 5 * time.Minute}
 	obj.EnableControllerAttachDetach = utilpointer.Bool(true)
-	obj.MakeIPTablesUtilChains = utilpointer.BoolPtr(true)
-	obj.IPTablesMasqueradeBit = utilpointer.Int32Ptr(configv1beta1.DefaultIPTablesMasqueradeBit)
-	obj.IPTablesDropBit = utilpointer.Int32Ptr(configv1beta1.DefaultIPTablesDropBit)
-	obj.FailSwapOn = utilpointer.BoolPtr(false)
+	obj.MakeIPTablesUtilChains = utilpointer.Bool(true)
+	obj.IPTablesMasqueradeBit = utilpointer.Int32(configv1beta1.DefaultIPTablesMasqueradeBit)
+	obj.IPTablesDropBit = utilpointer.Int32(configv1beta1.DefaultIPTablesDropBit)
+	obj.FailSwapOn = utilpointer.Bool(false)
 	obj.ContainerLogMaxSize = "10Mi"
-	obj.ContainerLogMaxFiles = utilpointer.Int32Ptr(5)
+	obj.ContainerLogMaxFiles = utilpointer.Int32(5)
 	obj.ConfigMapAndSecretChangeDetectionStrategy = kubeletconfigv1beta1.GetChangeDetectionStrategy
 	obj.EnforceNodeAllocatable = DefaultNodeAllocatableEnforcement
 	obj.VolumePluginDir = constants.DefaultVolumePluginDir
 	// Use the Default LoggingConfiguration option
 	logsapi.SetRecommendedLoggingConfiguration(&obj.Logging)
-	obj.EnableSystemLogHandler = utilpointer.BoolPtr(true)
-	obj.EnableProfilingHandler = utilpointer.BoolPtr(true)
-	obj.EnableDebugFlagsHandler = utilpointer.BoolPtr(true)
-	obj.SeccompDefault = utilpointer.BoolPtr(false)
-	obj.MemoryThrottlingFactor = utilpointer.Float64Ptr(configv1beta1.DefaultMemoryThrottlingFactor)
-	obj.RegisterNode = utilpointer.BoolPtr(true)
+	obj.EnableSystemLogHandler = utilpointer.Bool(true)
+	obj.EnableProfilingHandler = utilpointer.Bool(true)
+	obj.EnableDebugFlagsHandler = utilpointer.Bool(true)
+	obj.SeccompDefault = utilpointer.Bool(false)
+	obj.MemoryThrottlingFactor = utilpointer.Float64(configv1beta1.DefaultMemoryThrottlingFactor)
+	obj.RegisterNode = utilpointer.Bool(true)
 
 	obj.EnforceNodeAllocatable = DefaultNodeAllocatableEnforcement
 	obj.CgroupDriver = DefaultCgroupDriver
 	obj.CgroupsPerQOS = utilpointer.Bool(DefaultCgroupsPerQOS)
 	obj.ResolverConfig = utilpointer.String(DefaultResolverConfig)
 	obj.CPUCFSQuota = utilpointer.Bool(DefaultCPUCFSQuota)
-	// Add static pod default path
-	obj.StaticPodPath = constants.DefaultManifestsDir
+	obj.LocalStorageCapacityIsolation = utilpointer.Bool(true)
 	obj.ContainerRuntimeEndpoint = constants.DefaultRemoteRuntimeEndpoint
 	obj.ImageServiceEndpoint = constants.DefaultRemoteImageEndpoint
 }
diff --git a/pkg/apis/componentconfig/edgecore/v1alpha2/types.go b/pkg/apis/componentconfig/edgecore/v1alpha2/types.go
index 10083a981..624070a55 100644
--- a/pkg/apis/componentconfig/edgecore/v1alpha2/types.go
+++ b/pkg/apis/componentconfig/edgecore/v1alpha2/types.go
@@ -117,6 +117,11 @@ type Edged struct {
 // TailoredKubeletConfiguration indicates the tailored kubelet configuration.
 // It is derived from Kubernetes code `KubeletConfiguration` in package `k8s.io/kubelet/config/v1beta1` and made some variant.
 type TailoredKubeletConfiguration struct {
+	// staticPodPath is the path to the directory containing local (static) pods to
+	// run, or the path to a single static pod file.
+	// Default: "/etc/kubeedge/manifests"
+	// +optional
+	StaticPodPath string `json:"staticPodPath,omitempty"`
 	// syncFrequency is the max period between synchronizing running
 	// containers and config.
 	// Default: "1m"
@@ -149,14 +154,14 @@ type TailoredKubeletConfiguration struct {
 	RegistryBurst int32 `json:"registryBurst,omitempty"`
 	// eventRecordQPS is the maximum event creations per second. If 0, there
 	// is no limit enforced. The value cannot be a negative number.
-	// Default: 0
+	// Default: 50
 	// +optional
 	EventRecordQPS *int32 `json:"eventRecordQPS,omitempty"`
 	// eventBurst is the maximum size of a burst of event creations, temporarily
 	// allows event creations to burst to this number, while still not exceeding
 	// eventRecordQPS. This field cannot be a negative number and it is only used
 	// when eventRecordQPS > 0.
-	// Default: 10
+	// Default: 100
 	// +optional
 	EventBurst int32 `json:"eventBurst,omitempty"`
 	// enableDebuggingHandlers enables server endpoints for log access
@@ -216,7 +221,6 @@ type TailoredKubeletConfiguration struct {
 	// The lease is currently renewed every 10s, per KEP-0009. In the future, the lease renewal interval
 	// may be set based on the lease duration.
 	// The field value must be greater than 0.
-	// Requires the NodeLease feature gate to be enabled.
 	// Default: 40
 	// +optional
 	NodeLeaseDurationSeconds int32 `json:"nodeLeaseDurationSeconds,omitempty"`
@@ -259,7 +263,6 @@ type TailoredKubeletConfiguration struct {
 	SystemCgroups string `json:"systemCgroups,omitempty"`
 	// cgroupRoot is the root cgroup to use for pods. This is handled by the
 	// container runtime on a best effort basis.
-	// Default: ""
 	// +optional
 	CgroupRoot string `json:"cgroupRoot,omitempty"`
 	// cgroupsPerQOS enable QoS based CGroup hierarchy: top level CGroups for QoS classes
@@ -319,6 +322,12 @@ type TailoredKubeletConfiguration struct {
 	// Default: "container"
 	// +optional
 	TopologyManagerScope string `json:"topologyManagerScope,omitempty"`
+	// TopologyManagerPolicyOptions is a set of key=value which allows to set extra options
+	// to fine tune the behaviour of the topology manager policies.
+	// Requires  both the "TopologyManager" and "TopologyManagerPolicyOptions" feature gates to be enabled.
+	// Default: nil
+	// +optional
+	TopologyManagerPolicyOptions map[string]string `json:"topologyManagerPolicyOptions,omitempty"`
 	// qosReserved is a set of resource name to percentage pairs that specify
 	// the minimum percentage of a resource reserved for exclusive use by the
 	// guaranteed QoS tier.
@@ -362,6 +371,7 @@ type TailoredKubeletConfiguration struct {
 	PodPidsLimit *int64 `json:"podPidsLimit,omitempty"`
 	// resolvConf is the resolver configuration file used as the basis
 	// for the container DNS resolution configuration.
+	// If set to the empty string, will override the default and effectively disable DNS lookups.
 	// Default: "/etc/resolv.conf"
 	// +optional
 	ResolverConfig *string `json:"resolvConf,omitempty"`
@@ -398,6 +408,12 @@ type TailoredKubeletConfiguration struct {
 	// Default: true
 	// +optional
 	SerializeImagePulls *bool `json:"serializeImagePulls,omitempty"`
+	// MaxParallelImagePulls sets the maximum number of image pulls in parallel.
+	// This field cannot be set if SerializeImagePulls is true.
+	// Setting it to nil means no limit.
+	// Default: nil
+	// +optional
+	MaxParallelImagePulls *int32 `json:"maxParallelImagePulls,omitempty"`
 	// evictionHard is a map of signal names to quantities that defines hard eviction
 	// thresholds. For example: `{"memory.available": "300Mi"}`.
 	// To explicitly disable, pass a 0% or 100% threshold on an arbitrary resource.
@@ -597,6 +613,12 @@ type TailoredKubeletConfiguration struct {
 	// Default: true
 	// +optional
 	EnableSystemLogHandler *bool `json:"enableSystemLogHandler,omitempty"`
+	// enableSystemLogQuery enables the node log query feature on the /logs endpoint.
+	// EnableSystemLogHandler has to be enabled in addition for this feature to work.
+	// Default: false
+	// +featureGate=NodeLogQuery
+	// +optional
+	EnableSystemLogQuery *bool `json:"enableSystemLogQuery,omitempty"`
 	// shutdownGracePeriod specifies the total duration that the node should delay the
 	// shutdown and total grace period for pod termination during a node shutdown.
 	// Default: "0s"
@@ -696,16 +718,20 @@ type TailoredKubeletConfiguration struct {
 	// Default: true
 	// +optional
 	RegisterNode *bool `json:"registerNode,omitempty"`
-	// staticPodPath is the path to the directory containing local (static) pods to
-	// run, or the path to a single static pod file.
-	// Default: "/etc/kubeedge/manifests"
+	// LocalStorageCapacityIsolation enables local ephemeral storage isolation feature. The default setting is true.
+	// This feature allows users to set request/limit for container's ephemeral storage and manage it in a similar way
+	// as cpu and memory. It also allows setting sizeLimit for emptyDir volume, which will trigger pod eviction if disk
+	// usage from the volume exceeds the limit.
+	// This feature depends on the capability of detecting correct root file system disk usage. For certain systems,
+	// such as kind rootless, if this capability cannot be supported, the feature LocalStorageCapacityIsolation should be
+	// disabled. Once disabled, user should not set request/limit for container's ephemeral storage, or sizeLimit for emptyDir.
+	// Default: true
 	// +optional
-	StaticPodPath string `json:"staticPodPath,omitempty"`
+	LocalStorageCapacityIsolation *bool `json:"localStorageCapacityIsolation,omitempty"`
 	// ContainerRuntimeEndpoint is the endpoint of container runtime.
 	// Unix Domain Sockets are supported on Linux, while npipe and tcp endpoints are supported on Windows.
 	// Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'
 	ContainerRuntimeEndpoint string `json:"containerRuntimeEndpoint"`
-
 	// ImageServiceEndpoint is the endpoint of container image service.
 	// Unix Domain Socket are supported on Linux, while npipe and tcp endpoints are supported on Windows.
 	// Examples:'unix:///path/to/runtime.sock', 'npipe:////./pipe/runtime'.
@@ -731,12 +757,12 @@ type TailoredKubeletFlag struct {
 	RootDirectory string `json:"rootDirectory,omitempty"`
 	// registerNode enables automatic registration with the apiserver.
 	// default true
-	// DEPRECATED: This parameter should be set via the TailoredKubeletConfig
+	// DEPRECATED: This parameter will be removed at KubeEdge v1.17 and should be set via the TailoredKubeletConfig
 	RegisterNode bool `json:"registerNode,omitempty"`
 	// registerWithTaints are an array of taints to add to a node object when
 	// the edgecore registers itself. This only takes effect when registerNode
 	// is true and upon the initial registration of the node.
-	// DEPRECATED: This parameter should be set via the TailoredKubeletConfig
+	// DEPRECATED: This parameter will be removed at KubeEdge v1.17 and should be set via the TailoredKubeletConfig
 	RegisterWithTaints []core.Taint `json:"registerWithTaints,omitempty"`
 	// WindowsService should be set to true if kubelet is running as a service on Windows.
 	// Its corresponding flag only gets registered in Windows builds.
@@ -749,11 +775,11 @@ type TailoredKubeletFlag struct {
 	WindowsPriorityClass string `json:"windowsPriorityClass,omitempty"`
 	// remoteRuntimeEndpoint is the endpoint of remote runtime service
 	// default "unix:///run/containerd/containerd.sock"
-	// DEPRECATED and will be remove in KubeEdge v1.17
+	// DEPRECATED: This parameter will be removed in KubeEdge v1.17 and should be set via the ContainerRuntimeEndpoint on TailoredKubeletConfig
 	RemoteRuntimeEndpoint string `json:"remoteRuntimeEndpoint,omitempty"`
 	// remoteImageEndpoint is the endpoint of remote image service
 	// default "unix:///run/containerd/containerd.sock"
-	// DEPRECATED and will be remove in KubeEdge v1.17
+	// DEPRECATED: This parameter will be removed in KubeEdge v1.17 and should be set via the ImageServiceEndpoint on TailoredKubeletConfig
 	RemoteImageEndpoint string `json:"remoteImageEndpoint,omitempty"`
 	// experimentalMounterPath is the path of mounter binary. Leave empty to use the default mount path
 	ExperimentalMounterPath string `json:"experimentalMounterPath,omitempty"`
@@ -774,6 +800,7 @@ type TailoredKubeletFlag struct {
 	MaxContainerCount int32 `json:"maxContainerCount,omitempty"`
 	// masterServiceNamespace is The namespace from which the kubernetes
 	// master services should be injected into pods.
+	// DEPRECATED: will be removed in KubeEdge v1.17
 	MasterServiceNamespace string `json:"masterServiceNamespace,omitempty"`
 	// registerSchedulable tells the edgecore to register the node as
 	// schedulable. Won't have any effect if register-node is false.
@@ -793,7 +820,7 @@ type ContainerRuntimeOptions struct {
 
 	// ContainerRuntime is the container runtime to use.
 	// only valid value "remote"
-	// ContainerRuntime is deprecated and will be removed at v1.17
+	// ContainerRuntime is deprecated and will be removed at KubeEdge v1.17
 	ContainerRuntime string `json:"containerRuntime,omitempty"`
 	// RuntimeCgroups that container runtime is expected to be isolated in.
 	RuntimeCgroups string `json:"runtimeCgroups,omitempty"`
diff --git a/pkg/apis/componentconfig/edgecore/v1alpha2/validation/validation.go b/pkg/apis/componentconfig/edgecore/v1alpha2/validation/validation.go
index 76ef74a2e..6ed385136 100644
--- a/pkg/apis/componentconfig/edgecore/v1alpha2/validation/validation.go
+++ b/pkg/apis/componentconfig/edgecore/v1alpha2/validation/validation.go
@@ -22,6 +22,8 @@ import (
 	"path"
 
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/apis/core/validation"
 
 	"github.com/kubeedge/kubeedge/pkg/apis/componentconfig/edgecore/v1alpha2"
 	utilvalidation "github.com/kubeedge/kubeedge/pkg/util/validation"
@@ -55,6 +57,25 @@ func ValidateDataBase(db v1alpha2.DataBase) field.ErrorList {
 	return allErrs
 }
 
+// ValidateModuleEdged validates `e` and returns an errorList if it is invalid
+func ValidateModuleEdged(e v1alpha2.Edged) field.ErrorList {
+	if !e.Enable {
+		return field.ErrorList{}
+	}
+	allErrs := field.ErrorList{}
+	messages := validation.ValidateNodeName(e.HostnameOverride, false)
+	for _, msg := range messages {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("HostnameOverride"), e.HostnameOverride, msg))
+	}
+	if e.NodeIP == "" {
+		klog.Warningf("NodeIP is empty , use default ip which can connect to cloud.")
+	}
+	if err := ValidateCgroupDriver(e.TailoredKubeletConfig.CgroupDriver); err != nil {
+		allErrs = append(allErrs, err)
+	}
+	return allErrs
+}
+
 // ValidateModuleEdgeHub validates `h` and returns an errorList if it is invalid
 func ValidateModuleEdgeHub(h v1alpha2.EdgeHub) field.ErrorList {
 	if !h.Enable {
diff --git a/pkg/apis/componentconfig/edgecore/v1alpha2/validation/validation_others.go b/pkg/apis/componentconfig/edgecore/v1alpha2/validation/validation_others.go
index 9c88befe9..4a30bbe81 100644
--- a/pkg/apis/componentconfig/edgecore/v1alpha2/validation/validation_others.go
+++ b/pkg/apis/componentconfig/edgecore/v1alpha2/validation/validation_others.go
@@ -4,30 +4,17 @@ package validation
 
 import (
 	"k8s.io/apimachinery/pkg/util/validation/field"
-	"k8s.io/klog/v2"
-	"k8s.io/kubernetes/pkg/apis/core/validation"
 
 	"github.com/kubeedge/kubeedge/pkg/apis/componentconfig/edgecore/v1alpha2"
 )
 
-// ValidateModuleEdged validates `e` and returns an errorList if it is invalid
-func ValidateModuleEdged(e v1alpha2.Edged) field.ErrorList {
-	if !e.Enable {
-		return field.ErrorList{}
-	}
-	allErrs := field.ErrorList{}
-	messages := validation.ValidateNodeName(e.HostnameOverride, false)
-	for _, msg := range messages {
-		allErrs = append(allErrs, field.Invalid(field.NewPath("HostnameOverride"), e.HostnameOverride, msg))
-	}
-	if e.NodeIP == "" {
-		klog.Warningf("NodeIP is empty , use default ip which can connect to cloud.")
-	}
-	switch e.TailoredKubeletConfig.CgroupDriver {
+// ValidateCgroupDriver validates `edged.TailoredKubeletConfig.CgroupDriver` and returns an errorList if it is invalid
+func ValidateCgroupDriver(cgroupDriver string) *field.Error {
+	switch cgroupDriver {
 	case v1alpha2.CGroupDriverCGroupFS, v1alpha2.CGroupDriverSystemd:
 	default:
-		allErrs = append(allErrs, field.Invalid(field.NewPath("CGroupDriver"), e.TailoredKubeletConfig.CgroupDriver,
-			"CGroupDriver value error"))
+		return field.Invalid(field.NewPath("CGroupDriver"), cgroupDriver,
+			"CGroupDriver value error")
 	}
-	return allErrs
+	return nil
 }
diff --git a/pkg/apis/componentconfig/edgecore/v1alpha2/validation/validation_windows.go b/pkg/apis/componentconfig/edgecore/v1alpha2/validation/validation_windows.go
index ba5d7dee5..749cd5bd2 100644
--- a/pkg/apis/componentconfig/edgecore/v1alpha2/validation/validation_windows.go
+++ b/pkg/apis/componentconfig/edgecore/v1alpha2/validation/validation_windows.go
@@ -2,26 +2,9 @@
 
 package validation
 
-import (
-	"k8s.io/apimachinery/pkg/util/validation/field"
-	"k8s.io/klog/v2"
-	"k8s.io/kubernetes/pkg/apis/core/validation"
+import "k8s.io/apimachinery/pkg/util/validation/field"
 
-	"github.com/kubeedge/kubeedge/pkg/apis/componentconfig/edgecore/v1alpha2"
-)
-
-// ValidateModuleEdged validates `e` and returns an errorList if it is invalid
-func ValidateModuleEdged(e v1alpha2.Edged) field.ErrorList {
-	if !e.Enable {
-		return field.ErrorList{}
-	}
-	allErrs := field.ErrorList{}
-	messages := validation.ValidateNodeName(e.HostnameOverride, false)
-	for _, msg := range messages {
-		allErrs = append(allErrs, field.Invalid(field.NewPath("HostnameOverride"), e.HostnameOverride, msg))
-	}
-	if e.NodeIP == "" {
-		klog.Warningf("NodeIP is empty , use default ip which can connect to cloud.")
-	}
-	return allErrs
+// ValidateCgroupDriver validates `e` and returns an errorList if it is invalid
+func ValidateCgroupDriver(cgroupDriver string) *field.Error {
+	return nil
 }