diff options
| -rw-r--r-- | Makefile | 3 | ||||
| -rw-r--r-- | build/conformance/Dockerfile | 45 | ||||
| -rw-r--r-- | build/conformance/e2e-runner/run.go | 406 | ||||
| -rw-r--r-- | build/conformance/kubernetes/edge_skip_case.yaml | 66 | ||||
| -rw-r--r-- | build/conformance/kubernetes/kube_conformance_test.go | 31 | ||||
| -rwxr-xr-x | hack/make-rules/crossbuildimage.sh | 1 | ||||
| -rwxr-xr-x | hack/make-rules/image.sh | 1 |
7 files changed, 552 insertions, 1 deletions
@@ -14,7 +14,8 @@ BINARIES=cloudcore \ csidriver \ iptablesmanager \ edgemark \ - controllermanager + controllermanager \ + conformance COMPONENTS=cloud \ edge diff --git a/build/conformance/Dockerfile b/build/conformance/Dockerfile new file mode 100644 index 000000000..5525416c3 --- /dev/null +++ b/build/conformance/Dockerfile @@ -0,0 +1,45 @@ +# Copyright 2022 The KubeEdge Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang:1.16-alpine3.13 AS builder
+
+ARG GO_LDFLAGS
+
+RUN go install github.com/onsi/ginkgo/ginkgo@v1.16.5
+
+COPY . /go/src/github.com/kubeedge/kubeedge
+
+RUN cp /go/src/github.com/kubeedge/kubeedge/build/conformance/kubernetes/kube_conformance_test.go \
+ /go/src/github.com/kubeedge/kubeedge/tests/e2e/
+
+RUN cd /go/src/github.com/kubeedge/kubeedge && go mod vendor
+
+RUN CGO_ENABLED=0 GO111MODULE=off ginkgo build -ldflags "-w -s -extldflags -static" -r /go/src/github.com/kubeedge/kubeedge/tests/e2e
+
+RUN CGO_ENABLED=0 GO111MODULE=off go build -v -o /usr/local/bin/e2e-runner -ldflags "$GO_LDFLAGS -w -s" \
+ /go/src/github.com/kubeedge/kubeedge/build/conformance/e2e-runner
+
+FROM alpine:3.13
+
+COPY --from=builder /go/bin/ginkgo /usr/local/bin/ginkgo
+
+COPY --from=builder /usr/local/bin/e2e-runner /usr/local/bin/e2e-runner
+
+COPY --from=builder /go/src/github.com/kubeedge/kubeedge/tests/e2e/e2e.test /usr/local/bin/e2e.test
+
+COPY --from=builder /go/src/github.com/kubeedge/kubeedge/build/conformance/kubernetes/edge_skip_case.yaml /testdata/edge_skip_case.yaml
+
+RUN mkdir -p /tmp/results
+
+ENTRYPOINT ["e2e-runner"]
\ No newline at end of file diff --git a/build/conformance/e2e-runner/run.go b/build/conformance/e2e-runner/run.go new file mode 100644 index 000000000..d23f9f125 --- /dev/null +++ b/build/conformance/e2e-runner/run.go @@ -0,0 +1,406 @@ +/* +Copyright 2022 The KubeEdge Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/pkg/errors" + "gopkg.in/yaml.v3" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/retry" +) + +const ( + dryRunEnvKey = "E2E_DRYRUN" + skipEnvKey = "E2E_SKIP" + ginkgoEnvKey = "GINKGO_BIN" + testBinEnvKey = "TEST_BIN" + resultsDirEnvKey = "RESULTS_DIR" + kubeConfigEnvKey = "KUBECONFIG" + logFileName = "e2e.log" + defaultFocus = "\\[Conformance\\]" + extraArgsEnvKey = "E2E_EXTRA_ARGS" + defaultResultsDir = "/tmp/results" + defaultGinkgoBinary = "/usr/local/bin/ginkgo" + defaultTestBinary = "/usr/local/bin/e2e.test" + + edgeNodeLabelKey = "node-role.kubernetes.io/edge" +) + +func main() { + c := make(chan os.Signal) + signal.Notify(c, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) + go func() { + select { + case _ = <-c: + err := afterRunConformance() + if err != nil { + log.Printf("failed to cleanup after conformance, err: %v\n", err) + } + } + }() + + if err := RunE2E(); err != nil { + log.Fatal(err) + } +} + +func RunE2E() error { + err := beforeRunConformance() + if err != nil { + return fmt.Errorf("failed to prepare for run conformance, err: %v", err) + } + + defer func() { + err := afterRunConformance() + if err != nil { + log.Printf("failed to cleanup after conformance, err: %v\n", err) + } + }() + + resultsDir := getEnvWithDefault(resultsDirEnvKey, defaultResultsDir) + + // Print the output to stdout and a logfile which will be returned + // as part of the results tarball. + logFilePath := filepath.Join(resultsDir, logFileName) + logFile, err := os.Create(logFilePath) + if err != nil { + return fmt.Errorf("failed to create log file %v, err: %v", logFilePath, err) + } + + mw := io.MultiWriter(os.Stdout, logFile) + + cmd, err := makeCmd(mw) + if err != nil { + return err + } + + log.Printf("Running command:\n%v\n", cmdInfo(cmd)) + + err = cmd.Start() + if err != nil { + return errors.Wrap(err, "starting command") + } + + return errors.Wrap(cmd.Wait(), "running command") +} + +func makeCmd(w io.Writer) (*exec.Cmd, error) { + var ginkgoArgs []string + + skipCommands, err := skipCommands() + if err != nil { + return nil, err + } + + skipped := strings.Join(skipCommands, "|") + + ginkgoArgs = append(ginkgoArgs, "--skip="+skipped) + + skipEnvValue := getEnvWithDefault(skipEnvKey, "") + if len(skipEnvValue) > 0 { + ginkgoArgs = append(ginkgoArgs, "--skip="+skipEnvValue) + } + + ginkgoArgs = append(ginkgoArgs, "--focus="+defaultFocus) + ginkgoArgs = append(ginkgoArgs, "--noColor=true") + + if len(getEnvWithDefault(dryRunEnvKey, "")) > 0 { + ginkgoArgs = append(ginkgoArgs, "--dryRun=true") + } + + extraArgs := []string{ + "--report-dir=" + getEnvWithDefault(resultsDirEnvKey, ""), + "--kubeconfig=" + getEnvWithDefault(kubeConfigEnvKey, ""), + "--image-url=nginx", + "--image-url=nginx", + "--test-device=false", + } + + if len(getEnvWithDefault(extraArgsEnvKey, "")) > 0 { + extraArgs = append(extraArgs, strings.Split(getEnvWithDefault(extraArgsEnvKey, ""), ",")...) + } + + var args []string + args = append(args, ginkgoArgs...) + args = append(args, getEnvWithDefault(testBinEnvKey, defaultTestBinary)) + args = append(args, "--") + args = append(args, extraArgs...) + + cmd := exec.Command(getEnvWithDefault(ginkgoEnvKey, defaultGinkgoBinary), args...) + cmd.Stdout = w + cmd.Stderr = w + return cmd, nil +} + +func getEnvWithDefault(envKey, defaultValue string) string { + value := os.Getenv(envKey) + if len(value) == 0 { + return defaultValue + } + return value +} + +type Tests struct { + TestName string `yaml:"testname"` + CodeName string `yaml:"codename"` + Description string `yaml:"description"` + Release string `yaml:"release"` + File string `yaml:"file"` +} + +func skipCommands() ([]string, error) { + tests, err := skipCases() + if err != nil { + return nil, err + } + + var skipCommands []string + skipCommands = append(skipCommands, "\\[sig-api-machinery\\]") + for _, test := range tests { + skipCommands = append(skipCommands, test.CodeName) + } + + return skipCommands, nil +} + +func skipCases() ([]Tests, error) { + data, err := Read("/testdata/edge_skip_case.yaml") + if err != nil { + return nil, fmt.Errorf("read skip test case err: %v", err) + } + + var skipTests []Tests + + if err := yaml.Unmarshal(data, &skipTests); err != nil { + return nil, fmt.Errorf("unmarshal skip test case err: %v", err) + } + + return skipTests, err +} + +func Read(filePath string) ([]byte, error) { + data, err := ioutil.ReadFile(filePath) + if os.IsNotExist(err) { + // Not an error (yet), some other provider may have the file. + return nil, nil + } + return data, err +} + +func cmdInfo(cmd *exec.Cmd) string { + return fmt.Sprintf( + `Command env: %v +Run from directory: %v +Executable path: %v +Args (comma-delimited): %v`, cmd.Env, cmd.Dir, cmd.Path, strings.Join(cmd.Args, ","), + ) +} + +// tempTaints is temporarily added to center node when run kubeEdge conformance +// to make sure that all the pod created by conformance to run on the edge node +var tempTaints = &v1.Taint{ + Key: "node.kubeedge.io/conformance", + Value: "remove-when-completed", + Effect: v1.TaintEffectNoSchedule, +} + +var updateTaintBackoff = wait.Backoff{ + Steps: 5, + Duration: 100 * time.Millisecond, + Jitter: 1.0, +} + +// beforeRunConformance do prepare work before run conformance +func beforeRunConformance() error { + kubeClient, err := getKubeClient() + if err != nil { + return err + } + + nodeList, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return err + } + + for _, node := range nodeList.Items { + if isEdgeNode(node) { + continue + } + + err = addConformanceTaintOnNode(kubeClient, &node) + if err != nil { + return err + } + } + + return nil +} + +// afterRunConformance do clean work after conformance done +func afterRunConformance() error { + kubeClient, err := getKubeClient() + if err != nil { + return err + } + + nodeList, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return err + } + + for _, node := range nodeList.Items { + if isEdgeNode(node) { + continue + } + + err = deleteConformanceTaintOnNode(kubeClient, &node) + if err != nil { + log.Printf("failed delete taint for node:%v\n", node.Name) + } + } + + return nil +} + +func addConformanceTaintOnNode(c kubernetes.Interface, node *v1.Node) error { + newNode, updated := addTaint(node, tempTaints) + if !updated { + return nil + } + + return retry.RetryOnConflict(updateTaintBackoff, func() error { + return patchNodeTaints(c, node, newNode) + }) +} + +func deleteConformanceTaintOnNode(c kubernetes.Interface, node *v1.Node) error { + newNode, updated := removeTaint(node, tempTaints) + if !updated { + return nil + } + + return retry.RetryOnConflict(updateTaintBackoff, func() error { + return patchNodeTaints(c, node, newNode) + }) +} + +func isEdgeNode(node v1.Node) bool { + if node.Labels == nil { + return false + } + + _, ok := node.Labels[edgeNodeLabelKey] + return ok +} + +func getKubeClient() (kubernetes.Interface, error) { + configPath := getEnvWithDefault(kubeConfigEnvKey, "") + kubeConfig, err := clientcmd.BuildConfigFromFlags("", configPath) + if err != nil { + return nil, err + } + + kubeConfig.ContentType = runtime.ContentTypeProtobuf + kubeClient := kubernetes.NewForConfigOrDie(kubeConfig) + return kubeClient, nil +} + +func addTaint(node *v1.Node, taint *v1.Taint) (*v1.Node, bool) { + newNode := node.DeepCopy() + nodeTaints := newNode.Spec.Taints + + var newTaints []v1.Taint + for i := range nodeTaints { + if taint.MatchTaint(&nodeTaints[i]) { + log.Printf("taint already exist for node:%v\n", node.Name) + return node, false + } + + newTaints = append(newTaints, nodeTaints[i]) + } + + newTaints = append(newTaints, *taint) + newNode.Spec.Taints = newTaints + + return newNode, true +} + +func removeTaint(node *v1.Node, taintToDelete *v1.Taint) (*v1.Node, bool) { + newNode := node.DeepCopy() + nodeTaints := newNode.Spec.Taints + if len(nodeTaints) == 0 { + return newNode, false + } + + var newTaints []v1.Taint + deleted := false + for i := range nodeTaints { + if taintToDelete.MatchTaint(&nodeTaints[i]) { + deleted = true + continue + } + newTaints = append(newTaints, nodeTaints[i]) + } + + newNode.Spec.Taints = newTaints + + return newNode, deleted +} + +func patchNodeTaints(c kubernetes.Interface, oldNode *v1.Node, newNode *v1.Node) error { + oldData, err := json.Marshal(oldNode) + if err != nil { + return fmt.Errorf("failed to marshal old node %#v for node %q: %v", oldNode, oldNode.Name, err) + } + + newTaints := newNode.Spec.Taints + newNodeClone := oldNode.DeepCopy() + newNodeClone.Spec.Taints = newTaints + newData, err := json.Marshal(newNodeClone) + if err != nil { + return fmt.Errorf("failed to marshal new node %#v for node %q: %v", newNodeClone, oldNode.Name, err) + } + + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) + if err != nil { + return fmt.Errorf("failed to create patch for node %q: %v", oldNode.Name, err) + } + + _, err = c.CoreV1().Nodes().Patch(context.TODO(), oldNode.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}) + return err +} diff --git a/build/conformance/kubernetes/edge_skip_case.yaml b/build/conformance/kubernetes/edge_skip_case.yaml new file mode 100644 index 000000000..91effb16f --- /dev/null +++ b/build/conformance/kubernetes/edge_skip_case.yaml @@ -0,0 +1,66 @@ +- codename: should be able to switch session affinity for service with type clusterIP
+- codename: should provide DNS for pods for Subdomain
+- codename: should be able to change the type from NodePort to ExternalName
+- codename: optional updates should be reflected in volume
+- codename: runs ReplicaSets to verify preemption running path
+- codename: should provide DNS for ExternalName services
+- codename: should function for intra-pod communication
+- codename: optional updates should be reflected in volume
+- codename: Should recreate evicted statefulset
+- codename: should function for node-pod communication
+- codename: should perform rolling updates and roll backs of template modifications
+- codename: should create and stop a working application
+- codename: should provide DNS for the cluster
+- codename: should serve a basic image on each replica with a public image
+- codename: should unconditionally reject operations on fail closed webhook
+- codename: should not be able to pull image from invalid registry
+- codename: should perform canary updates and phased rolling updates of template modifications
+- codename: should scale a replication controller
+- codename: Scaling should happen in predictable order and halt if any stateful pod is unhealthy
+- codename: validates lower priority pod preemption by critical pod
+- codename: should succeed in writing subpaths in container
+
+- codename: should have session affinity work for NodePort service
+- codename: should be able to change the type from ExternalName to NodePort
+- codename: should proxy through a service and a pod
+- codename: with readiness probe should not be ready before initial delay and never restart
+- codename: should provide DNS for services
+- codename: should have session affinity work for service with type clusterIP
+- codename: optional updates should be reflected in volume
+- codename: should verify that a failing subpath expansion can be modified during the lifecycle of a container
+- codename: should have a working scale subresource
+- codename: optional updates should be reflected in volume
+- codename: should have session affinity timeout work for service with type clusterIP
+- codename: should be able to switch session affinity for NodePort service
+- codename: validates that there is no conflict between pods with same hostPort but different hostIP and protocol
+- codename: should write entries to /etc/hosts
+- codename: should serve a basic image on each replica with a public image
+- codename: should have session affinity timeout work for NodePort service
+- codename: should provide /etc/hosts entries for the cluster
+
+- codename: should not be able to pull from private registry without secret
+- codename: should be sent by kubelets and the scheduler about pods scheduling and running
+- codename: should provide DNS for pods for Hostname
+- codename: should resolve DNS of partial qualified names for services
+- codename: should be able to change the type from ExternalName to ClusterIP
+- codename: Should be able to support the 1.17 Sample API Server using the current Aggregator
+
+- codename: ServiceAccountIssuerDiscovery should support OIDC discovery of service account issuer
+- codename: validates basic preemption works
+- codename: should be able to create a functioning NodePort service
+- codename: Burst scaling should run to completion even with unhealthy pods
+- codename: should be able to change the type from ClusterIP to ExternalName
+- codename: should support sysctls
+- codename: should provide container's limits.cpu/memory and requests.cpu/memory as env vars
+- codename: should provide default limits.cpu/memory from node allocatable
+- codename: should allow activeDeadlineSeconds to be updated
+- codename: should contain environment variables for services
+- codename: should run through the lifecycle of Pods and PodStatus
+- codename: should call prestop when killing a pod
+- codename: should fail substituting values in a volume subpath with absolute path
+- codename: should fail substituting values in a volume subpath with backticks
+- codename: should allow substituting values in a volume subpath
+- codename: should provide node allocatable \(memory\) as default memory limit if the limit is not set
+- codename: should provide node allocatable \(cpu\) as default cpu limit if the limit is not set
+- codename: should support unsafe sysctls which are actually allowed
+- codename: should create and stop a replication controller
\ No newline at end of file diff --git a/build/conformance/kubernetes/kube_conformance_test.go b/build/conformance/kubernetes/kube_conformance_test.go new file mode 100644 index 000000000..77f118502 --- /dev/null +++ b/build/conformance/kubernetes/kube_conformance_test.go @@ -0,0 +1,31 @@ +/* +Copyright 2022 The KubeEdge Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + + +import ( + _ "k8s.io/kubernetes/test/e2e/apps" + //_ "k8s.io/kubernetes/test/e2e/autoscaling" + //_ "k8s.io/kubernetes/test/e2e/common" + //_ "k8s.io/kubernetes/test/e2e/instrumentation" + //_ "k8s.io/kubernetes/test/e2e/lifecycle" + //_ "k8s.io/kubernetes/test/e2e/lifecycle/bootstrap" + //_ "k8s.io/kubernetes/test/e2e/node" + //_ "k8s.io/kubernetes/test/e2e/scheduling" + //_ "k8s.io/kubernetes/test/e2e/storage" +) + diff --git a/hack/make-rules/crossbuildimage.sh b/hack/make-rules/crossbuildimage.sh index e823d92cf..9b3a9830e 100755 --- a/hack/make-rules/crossbuildimage.sh +++ b/hack/make-rules/crossbuildimage.sh @@ -35,6 +35,7 @@ ALL_IMAGES_AND_TARGETS=( csidriver:csidriver:build/csidriver/Dockerfile iptablesmanager:iptables-manager:build/iptablesmanager/Dockerfile edgemark:edgemark:build/edgemark/Dockerfile + conformance:conformance:build/conformance/Dockerfile installation-package:installation-package:build/docker/installation-package/installation-package.dockerfile controllermanager:controller-manager:build/controllermanager/Dockerfile ) diff --git a/hack/make-rules/image.sh b/hack/make-rules/image.sh index 275a55dbc..9f6c2782d 100755 --- a/hack/make-rules/image.sh +++ b/hack/make-rules/image.sh @@ -32,6 +32,7 @@ ALL_IMAGES_AND_TARGETS=( csidriver:csidriver:build/csidriver/Dockerfile iptablesmanager:iptables-manager:build/iptablesmanager/Dockerfile edgemark:edgemark:build/edgemark/Dockerfile + conformance:conformance:build/conformance/Dockerfile controllermanager:controller-manager:build/controllermanager/Dockerfile installation-package:installation-package:build/docker/installation-package/installation-package.dockerfile ) |