diff options
| author | wackxu <xushiwei5@huawei.com> | 2022-06-20 21:29:48 +0800 |
|---|---|---|
| committer | wackxu <xushiwei5@huawei.com> | 2022-10-11 20:47:14 +0800 |
| commit | f30b1abf5be496bb252af8cbb562b1c5c6645125 (patch) | |
| tree | 61301bce461f3a33ca90ccbb9179e560654437df /tests | |
| parent | Merge pull request #4125 from wackxu/fixdeletetermin (diff) | |
| download | kubeedge-f30b1abf5be496bb252af8cbb562b1c5c6645125.tar.gz | |
fix e2e use fixed edge node for test and cleanup
Signed-off-by: wackxu <xushiwei5@huawei.com>
fix
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/e2e/constants/constants.go | 13 | ||||
| -rw-r--r-- | tests/e2e/deployment/deployment_test.go | 159 | ||||
| -rw-r--r-- | tests/e2e/deployment/device_crd_test.go | 12 | ||||
| -rw-r--r-- | tests/e2e/deployment/e2e_test.go | 4 | ||||
| -rw-r--r-- | tests/e2e/edgesite/edgesite_suite_test.go | 2 | ||||
| -rw-r--r-- | tests/e2e/edgesite/edgesite_test.go | 140 | ||||
| -rw-r--r-- | tests/e2e/keadm/keadm_suite_test.go | 3 | ||||
| -rw-r--r-- | tests/e2e/keadm/keadm_test.go | 53 | ||||
| -rw-r--r-- | tests/e2e/testsuite/testsuite.go | 63 | ||||
| -rw-r--r-- | tests/e2e/utils/common.go | 451 | ||||
| -rw-r--r-- | tests/e2e/utils/context.go | 13 | ||||
| -rw-r--r-- | tests/e2e/utils/node.go | 17 | ||||
| -rw-r--r-- | tests/e2e/utils/pod.go | 218 |
13 files changed, 366 insertions, 782 deletions
diff --git a/tests/e2e/constants/constants.go b/tests/e2e/constants/constants.go index 0f5952490..6ae2a4378 100644 --- a/tests/e2e/constants/constants.go +++ b/tests/e2e/constants/constants.go @@ -3,9 +3,16 @@ package constants import "time" const ( - AppHandler = "/api/v1/namespaces/default/pods" - DeploymentHandler = "/apis/apps/v1/namespaces/default/deployments" - Interval = 5 * time.Second Timeout = 10 * time.Minute + + E2ELabelKey = "kubeedge" + E2ELabelValue = "e2e-test" +) + +var ( + // KubeEdgeE2ELabel labels resources created during e2e testing + KubeEdgeE2ELabel = map[string]string{ + E2ELabelKey: E2ELabelValue, + } ) diff --git a/tests/e2e/deployment/deployment_test.go b/tests/e2e/deployment/deployment_test.go index 6fd386a78..f9f20745a 100644 --- a/tests/e2e/deployment/deployment_test.go +++ b/tests/e2e/deployment/deployment_test.go @@ -19,13 +19,12 @@ package deployment import ( "context" "fmt" - "net/http" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" @@ -38,7 +37,7 @@ import ( var DeploymentTestTimerGroup = utils.NewTestTimerGroup() -//Run Test cases +// Run Test cases var _ = Describe("Application deployment test in E2E scenario", func() { var UID string var testTimer *utils.TestTimer @@ -57,59 +56,68 @@ var _ = Describe("Application deployment test in E2E scenario", func() { // Start test timer testTimer = DeploymentTestTimerGroup.NewTestTimer(testSpecReport.LeafNodeText) }) + AfterEach(func() { // End test timer testTimer.End() // Print result testTimer.PrintResult() - var podlist corev1.PodList - var deploymentList appsv1.DeploymentList - err := utils.GetDeployments(&deploymentList, ctx.Cfg.K8SMasterForKubeEdge+constants.DeploymentHandler) + + By(fmt.Sprintf("get deployment %s", UID)) + deployment, err := utils.GetDeployment(clientSet, metav1.NamespaceDefault, UID) Expect(err).To(BeNil()) - for _, deployment := range deploymentList.Items { - if deployment.Name == UID { - label := nodeName - podlist, err = utils.GetPods(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, label) - Expect(err).To(BeNil()) - StatusCode := utils.DeleteDeployment(ctx.Cfg.K8SMasterForKubeEdge+constants.DeploymentHandler, deployment.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) - } - } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) + + By(fmt.Sprintf("list pod for deploy %s", UID)) + labelSelector := labels.SelectorFromSet(map[string]string{"app": UID}) + _, err = utils.GetPods(clientSet, metav1.NamespaceDefault, labelSelector, nil) + Expect(err).To(BeNil()) + + By(fmt.Sprintf("delete deploy %s", UID)) + err = utils.DeleteDeployment(clientSet, deployment.Namespace, deployment.Name) + Expect(err).To(BeNil()) + + By(fmt.Sprintf("wait for pod of deploy %s to disappear", UID)) + err = utils.WaitForPodsToDisappear(clientSet, metav1.NamespaceDefault, labelSelector, constants.Interval, constants.Timeout) + Expect(err).To(BeNil()) + utils.PrintTestcaseNameandStatus() }) It("E2E_APP_DEPLOYMENT_1: Create deployment and check the pods are coming up correctly", func() { - replica := 1 + replica := int32(1) //Generate the random string and assign as a UID UID = "edgecore-depl-app-" + utils.GetRandomString(5) - CreateDeploymentTest(replica, UID, nodeName, nodeSelector, ctx) + CreateDeploymentTest(clientSet, replica, UID, ctx) }) + It("E2E_APP_DEPLOYMENT_2: Create deployment with replicas and check the pods are coming up correctly", func() { - replica := 3 + replica := int32(3) //Generate the random string and assign as a UID UID = "edgecore-depl-app-" + utils.GetRandomString(5) - CreateDeploymentTest(replica, UID, nodeName, nodeSelector, ctx) + CreateDeploymentTest(clientSet, replica, UID, ctx) }) It("E2E_APP_DEPLOYMENT_3: Create deployment and check deployment ctrler re-creating pods when user deletes the pods manually", func() { - replica := 3 + replica := int32(3) //Generate the random string and assign as a UID UID = "edgecore-depl-app-" + utils.GetRandomString(5) - podlist := CreateDeploymentTest(replica, UID, nodeName, nodeSelector, ctx) - for _, pod := range podlist.Items { - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + pod.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) + podList := CreateDeploymentTest(clientSet, replica, UID, ctx) + for _, pod := range podList.Items { + err := utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(err).To(BeNil()) } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) - label := nodeName - podlist, err := utils.GetPods(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, label) + utils.CheckPodDeleteState(clientSet, podList) + + labelSelector := labels.SelectorFromSet(map[string]string{"app": UID}) + podList, err := utils.GetPods(clientSet, metav1.NamespaceDefault, labelSelector, nil) Expect(err).To(BeNil()) - Expect(len(podlist.Items)).Should(Equal(replica)) - utils.WaitforPodsRunning(ctx.Cfg.KubeConfigPath, podlist, 240*time.Second) + Expect(len(podList.Items)).Should(Equal(int(replica))) + + utils.WaitForPodsRunning(clientSet, podList, 240*time.Second) }) }) + Context("Test application deployment using Pod spec", func() { BeforeEach(func() { // Get current test SpecReport @@ -122,71 +130,80 @@ var _ = Describe("Application deployment test in E2E scenario", func() { testTimer.End() // Print result testTimer.PrintResult() - var podlist corev1.PodList - label := nodeName - podlist, err := utils.GetPods(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, label) + + labelSelector := labels.SelectorFromSet(constants.KubeEdgeE2ELabel) + podList, err := utils.GetPods(clientSet, metav1.NamespaceDefault, labelSelector, nil) Expect(err).To(BeNil()) - for _, pod := range podlist.Items { - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + pod.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) + + for _, pod := range podList.Items { + err = utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(err).To(BeNil()) } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) + + utils.CheckPodDeleteState(clientSet, podList) + utils.PrintTestcaseNameandStatus() }) It("E2E_POD_DEPLOYMENT_1: Create a pod and check the pod is coming up correctly", func() { //Generate the random string and assign as podName podName := "pod-app-" + utils.GetRandomString(5) - pod := utils.NewPodObj(podName, ctx.Cfg.AppImageURL[0], nodeSelector) + pod := utils.NewPod(podName, ctx.Cfg.AppImageURL[0]) - CreatePodTest(nodeName, podName, ctx, pod) + CreatePodTest(clientSet, pod) }) It("E2E_POD_DEPLOYMENT_2: Create the pod and delete pod happening successfully", func() { //Generate the random string and assign as podName podName := "pod-app-" + utils.GetRandomString(5) - pod := utils.NewPodObj(podName, ctx.Cfg.AppImageURL[0], nodeSelector) + pod := utils.NewPod(podName, ctx.Cfg.AppImageURL[0]) - podlist := CreatePodTest(nodeName, podName, ctx, pod) - for _, pod := range podlist.Items { - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + pod.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) + podList := CreatePodTest(clientSet, pod) + for _, pod := range podList.Items { + err := utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(err).To(BeNil()) } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) + + utils.CheckPodDeleteState(clientSet, podList) }) + It("E2E_POD_DEPLOYMENT_3: Create pod and delete the pod successfully, and delete already deleted pod and check the behaviour", func() { //Generate the random string and assign as podName podName := "pod-app-" + utils.GetRandomString(5) - pod := utils.NewPodObj(podName, ctx.Cfg.AppImageURL[0], nodeSelector) + pod := utils.NewPod(podName, ctx.Cfg.AppImageURL[0]) - podlist := CreatePodTest(nodeName, podName, ctx, pod) - for _, pod := range podlist.Items { - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + pod.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) + podList := CreatePodTest(clientSet, pod) + for _, pod := range podList.Items { + err := utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(err).To(BeNil()) } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + UID) - Expect(StatusCode).Should(Equal(http.StatusNotFound)) + + utils.CheckPodDeleteState(clientSet, podList) + + err := utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(errors.IsNotFound(err)).To(BeTrue()) }) + It("E2E_POD_DEPLOYMENT_4: Create and delete pod multiple times and check all the Pod created and deleted successfully", func() { //Generate the random string and assign as a UID for i := 0; i < 10; i++ { //Generate the random string and assign as podName podName := "pod-app-" + utils.GetRandomString(5) - pod := utils.NewPodObj(podName, ctx.Cfg.AppImageURL[0], nodeSelector) + pod := utils.NewPod(podName, ctx.Cfg.AppImageURL[0]) - podlist := CreatePodTest(nodeName, podName, ctx, pod) - for _, pod := range podlist.Items { - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + pod.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) + podList := CreatePodTest(clientSet, pod) + for _, pod := range podList.Items { + err := utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(err).To(BeNil()) } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) + utils.CheckPodDeleteState(clientSet, podList) } }) + It("E2E_POD_DEPLOYMENT_5: Create pod with hostpath volume successfully", func() { //Generate the random string and assign as podName podName := "pod-app-" + utils.GetRandomString(5) - pod := utils.NewPodObj(podName, ctx.Cfg.AppImageURL[0], nodeSelector) + pod := utils.NewPod(podName, ctx.Cfg.AppImageURL[0]) pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{ Name: "hp", @@ -199,12 +216,12 @@ var _ = Describe("Application deployment test in E2E scenario", func() { }, }} - podlist := CreatePodTest(nodeName, podName, ctx, pod) - for _, pod := range podlist.Items { - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + pod.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) + podList := CreatePodTest(clientSet, pod) + for _, pod := range podList.Items { + err := utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(err).To(BeNil()) } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) + utils.CheckPodDeleteState(clientSet, podList) }) }) @@ -228,7 +245,7 @@ var _ = Describe("Application deployment test in E2E scenario", func() { By(fmt.Sprintf("list pod for StatefulSet %s", UID)) labelSelector := labels.SelectorFromSet(map[string]string{"app": UID}) - _, err = utils.ListPods(clientSet, metav1.NamespaceDefault, labelSelector, nil) + _, err = utils.GetPods(clientSet, metav1.NamespaceDefault, labelSelector, nil) Expect(err).To(BeNil()) By(fmt.Sprintf("delete StatefulSet %s", UID)) @@ -256,12 +273,12 @@ var _ = Describe("Application deployment test in E2E scenario", func() { By(fmt.Sprintf("get pod for StatefulSet %s", UID)) labelSelector := labels.SelectorFromSet(map[string]string{"app": UID}) - podList, err := utils.ListPods(clientSet, corev1.NamespaceDefault, labelSelector, nil) + podList, err := utils.GetPods(clientSet, corev1.NamespaceDefault, labelSelector, nil) Expect(err).To(BeNil()) Expect(len(podList.Items)).ShouldNot(Equal(0)) By(fmt.Sprintf("wait for pod of StatefulSet %s running", UID)) - utils.WaitforPodsRunning(ctx.Cfg.KubeConfigPath, *podList, 240*time.Second) + utils.WaitForPodsRunning(clientSet, podList, 240*time.Second) }) It("Delete statefulSet pod multi times", func() { @@ -278,17 +295,17 @@ var _ = Describe("Application deployment test in E2E scenario", func() { By(fmt.Sprintf("get pod for StatefulSet %s", UID)) labelSelector := labels.SelectorFromSet(map[string]string{"app": UID}) - podList, err := utils.ListPods(clientSet, corev1.NamespaceDefault, labelSelector, nil) + podList, err := utils.GetPods(clientSet, corev1.NamespaceDefault, labelSelector, nil) Expect(err).To(BeNil()) Expect(len(podList.Items)).ShouldNot(Equal(0)) By(fmt.Sprintf("wait for pod of StatefulSet %s running", UID)) - utils.WaitforPodsRunning(ctx.Cfg.KubeConfigPath, *podList, 240*time.Second) + utils.WaitForPodsRunning(clientSet, podList, 240*time.Second) deletePodName := fmt.Sprintf("%s-1", UID) for i := 0; i < 5; i++ { By(fmt.Sprintf("delete pod %s", deletePodName)) - err = utils.DeletePod(clientSet, deletePodName, "default") + err = utils.DeletePod(clientSet, "default", deletePodName) Expect(err).To(BeNil()) By(fmt.Sprintf("wait for pod %s running again", fmt.Sprintf("%s-1", UID))) diff --git a/tests/e2e/deployment/device_crd_test.go b/tests/e2e/deployment/device_crd_test.go index 90e3c3919..e155a05cd 100644 --- a/tests/e2e/deployment/device_crd_test.go +++ b/tests/e2e/deployment/device_crd_test.go @@ -24,6 +24,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" "github.com/kubeedge/kubeedge/pkg/apis/devices/v1alpha2" "github.com/kubeedge/kubeedge/tests/e2e/utils" @@ -43,6 +45,12 @@ var CRDTestTimerGroup = utils.NewTestTimerGroup() var _ = Describe("Device Management test in E2E scenario", func() { var testTimer *utils.TestTimer var testSpecReport SpecReport + var clientSet clientset.Interface + + BeforeEach(func() { + clientSet = utils.NewKubeClient(ctx.Cfg.KubeConfigPath) + }) + Context("Test Device Model Creation, Updation and deletion", func() { BeforeEach(func() { // Delete any pre-existing device models @@ -319,8 +327,8 @@ var _ = Describe("Device Management test in E2E scenario", func() { Expect(isEqual).Should(Equal(true)) }) It("E2E_CREATE_DEVICE_4: Create device instance for incorrect device instance", func() { - statusCode := utils.DeleteConfigmap(ctx.Cfg.K8SMasterForKubeEdge + ConfigmapHandler + "/" + "device-profile-config-" + nodeName) - Expect(statusCode == http.StatusOK || statusCode == http.StatusNotFound).Should(Equal(true)) + err := utils.DeleteConfigMap(clientSet, metav1.NamespaceDefault, "device-profile-config-"+nodeName) + Expect(err).To(BeNil()) IsDeviceModelCreated, statusCode := utils.HandleDeviceModel(http.MethodPost, ctx.Cfg.K8SMasterForKubeEdge+DeviceModelHandler, "", "led") Expect(IsDeviceModelCreated).Should(BeTrue()) Expect(statusCode).Should(Equal(http.StatusCreated)) diff --git a/tests/e2e/deployment/e2e_test.go b/tests/e2e/deployment/e2e_test.go index 24e346676..883f2d5bf 100644 --- a/tests/e2e/deployment/e2e_test.go +++ b/tests/e2e/deployment/e2e_test.go @@ -29,8 +29,7 @@ import ( ) var ( - nodeName string - nodeSelector string + nodeName string // context to load config and access across the package ctx *utils.TestContext ) @@ -50,7 +49,6 @@ func TestE2E(t *testing.T) { utils.Infof("Before Suite Execution") ctx = utils.NewTestContext(utils.LoadConfig()) nodeName = "edge-node" - nodeSelector = "test" err := utils.MqttConnect() Expect(err).To(BeNil()) diff --git a/tests/e2e/edgesite/edgesite_suite_test.go b/tests/e2e/edgesite/edgesite_suite_test.go index e37064f27..298a4a765 100644 --- a/tests/e2e/edgesite/edgesite_suite_test.go +++ b/tests/e2e/edgesite/edgesite_suite_test.go @@ -21,8 +21,6 @@ import ( ) var ( - nodeName string - nodeSelector string //context to load config and access across the package ctx *utils.TestContext ) diff --git a/tests/e2e/edgesite/edgesite_test.go b/tests/e2e/edgesite/edgesite_test.go index bdaee6389..270d62d4e 100644 --- a/tests/e2e/edgesite/edgesite_test.go +++ b/tests/e2e/edgesite/edgesite_test.go @@ -17,13 +17,15 @@ limitations under the License. package edgesite import ( - "net/http" + "fmt" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - v1 "k8s.io/api/apps/v1" metav1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + clientset "k8s.io/client-go/kubernetes" "github.com/kubeedge/kubeedge/tests/e2e/constants" . "github.com/kubeedge/kubeedge/tests/e2e/testsuite" @@ -37,6 +39,13 @@ var _ = Describe("Application deployment test in E2E scenario using EdgeSite", f var UID string var testTimer *utils.TestTimer var testSpecReport SpecReport + + var clientSet clientset.Interface + + BeforeEach(func() { + clientSet = utils.NewKubeClient(ctx.Cfg.KubeConfigPath) + }) + Context("Test application deployment and delete deployment using deployment spec", func() { BeforeEach(func() { // Get current test SpecReport @@ -44,56 +53,64 @@ var _ = Describe("Application deployment test in E2E scenario using EdgeSite", f // Start test timer testTimer = DeploymentTestTimerGroup.NewTestTimer(testSpecReport.LeafNodeText) }) + AfterEach(func() { // End test timer testTimer.End() // Print result testTimer.PrintResult() - var podlist metav1.PodList - var deploymentList v1.DeploymentList - err := utils.GetDeployments(&deploymentList, ctx.Cfg.K8SMasterForKubeEdge+constants.DeploymentHandler) + + By(fmt.Sprintf("get deployment %s", UID)) + deployment, err := utils.GetDeployment(clientSet, metav1.NamespaceDefault, UID) Expect(err).To(BeNil()) - for _, deployment := range deploymentList.Items { - if deployment.Name == UID { - label := nodeName - podlist, err = utils.GetPods(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, label) - Expect(err).To(BeNil()) - StatusCode := utils.DeleteDeployment(ctx.Cfg.K8SMasterForKubeEdge+constants.DeploymentHandler, deployment.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) - } - } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) + + By(fmt.Sprintf("list pod for deploy %s", UID)) + labelSelector := labels.SelectorFromSet(map[string]string{"app": UID}) + _, err = utils.GetPods(clientSet, metav1.NamespaceDefault, labelSelector, nil) + Expect(err).To(BeNil()) + + By(fmt.Sprintf("delete deploy %s", UID)) + err = utils.DeleteDeployment(clientSet, deployment.Namespace, deployment.Name) + Expect(err).To(BeNil()) + + By(fmt.Sprintf("wait for pod of deploy %s to disappear", UID)) + err = utils.WaitForPodsToDisappear(clientSet, metav1.NamespaceDefault, labelSelector, constants.Interval, constants.Timeout) + Expect(err).To(BeNil()) + utils.PrintTestcaseNameandStatus() }) It("E2E_ES_APP_DEPLOYMENT_1: Create deployment and check the pods are coming up correctly", func() { - replica := 1 + replica := int32(1) //Generate the random string and assign as a UID UID = "edgecore-depl-app-" + utils.GetRandomString(5) - CreateDeploymentTest(replica, UID, nodeName, nodeSelector, ctx) + CreateDeploymentTest(clientSet, replica, UID, ctx) }) + It("E2E_ES_APP_DEPLOYMENT_2: Create deployment with replicas and check the pods are coming up correctly", func() { - replica := 3 + replica := int32(3) //Generate the random string and assign as a UID UID = "edgecore-depl-app-" + utils.GetRandomString(5) - CreateDeploymentTest(replica, UID, nodeName, nodeSelector, ctx) + CreateDeploymentTest(clientSet, replica, UID, ctx) }) It("E2E_ES_APP_DEPLOYMENT_3: Create deployment and check deployment ctrler re-creating pods when user deletes the pods manually", func() { - replica := 3 + replica := int32(3) //Generate the random string and assign as a UID UID = "edgecore-depl-app-" + utils.GetRandomString(5) - podlist := CreateDeploymentTest(replica, UID, nodeName, nodeSelector, ctx) - for _, pod := range podlist.Items { - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + pod.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) + podList := CreateDeploymentTest(clientSet, replica, UID, ctx) + for _, pod := range podList.Items { + err := utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(err).To(BeNil()) } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) - label := nodeName - podlist, err := utils.GetPods(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, label) + utils.CheckPodDeleteState(clientSet, podList) + + labelSelector := labels.SelectorFromSet(map[string]string{"app": UID}) + podList, err := utils.GetPods(clientSet, metav1.NamespaceDefault, labelSelector, nil) Expect(err).To(BeNil()) - Expect(len(podlist.Items)).Should(Equal(replica)) - utils.WaitforPodsRunning(ctx.Cfg.KubeConfigPath, podlist, 240*time.Second) + Expect(len(podList.Items)).Should(Equal(replica)) + + utils.WaitForPodsRunning(clientSet, podList, 240*time.Second) }) }) @@ -104,70 +121,77 @@ var _ = Describe("Application deployment test in E2E scenario using EdgeSite", f // Start test timer testTimer = DeploymentTestTimerGroup.NewTestTimer(testSpecReport.LeafNodeText) }) + AfterEach(func() { // End test timer testTimer.End() // Print result testTimer.PrintResult() - var podlist metav1.PodList - label := nodeName - podlist, err := utils.GetPods(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, label) + + labelSelector := labels.SelectorFromSet(constants.KubeEdgeE2ELabel) + podList, err := utils.GetPods(clientSet, metav1.NamespaceDefault, labelSelector, nil) Expect(err).To(BeNil()) - for _, pod := range podlist.Items { - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + pod.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) + + for _, pod := range podList.Items { + err = utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(err).To(BeNil()) } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) + + utils.CheckPodDeleteState(clientSet, podList) + utils.PrintTestcaseNameandStatus() }) It("E2E_ES_POD_DEPLOYMENT_1: Create a pod and check the pod is coming up correctly", func() { //Generate the random string and assign as podName podName := "pod-app-" + utils.GetRandomString(5) - pod := utils.NewPodObj(podName, ctx.Cfg.AppImageURL[0], nodeSelector) + pod := utils.NewPod(podName, ctx.Cfg.AppImageURL[0]) - CreatePodTest(nodeName, podName, ctx, pod) + CreatePodTest(clientSet, pod) }) It("E2E_ES_POD_DEPLOYMENT_2: Create the pod and delete pod happening successfully", func() { //Generate the random string and assign as podName podName := "pod-app-" + utils.GetRandomString(5) - pod := utils.NewPodObj(podName, ctx.Cfg.AppImageURL[0], nodeSelector) + pod := utils.NewPod(podName, ctx.Cfg.AppImageURL[0]) - podlist := CreatePodTest(nodeName, podName, ctx, pod) - for _, pod := range podlist.Items { - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + pod.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) + podList := CreatePodTest(clientSet, pod) + for _, pod := range podList.Items { + err := utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(err).To(BeNil()) } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) + utils.CheckPodDeleteState(clientSet, podList) }) + It("E2E_ES_POD_DEPLOYMENT_3: Create pod and delete the pod successfully, and delete already deleted pod and check the behaviour", func() { //Generate the random string and assign as podName podName := "pod-app-" + utils.GetRandomString(5) - pod := utils.NewPodObj(podName, ctx.Cfg.AppImageURL[0], nodeSelector) + pod := utils.NewPod(podName, ctx.Cfg.AppImageURL[0]) - podlist := CreatePodTest(nodeName, podName, ctx, pod) - for _, pod := range podlist.Items { - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + pod.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) + podList := CreatePodTest(clientSet, pod) + for _, pod := range podList.Items { + err := utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(err).To(BeNil()) } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + UID) - Expect(StatusCode).Should(Equal(http.StatusNotFound)) + utils.CheckPodDeleteState(clientSet, podList) + + err := utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) + It("E2E_ES_POD_DEPLOYMENT_4: Create and delete pod multiple times and check all the Pod created and deleted successfully", func() { //Generate the random string and assign as a UID for i := 0; i < 10; i++ { //Generate the random string and assign as podName podName := "pod-app-" + utils.GetRandomString(5) - pod := utils.NewPodObj(podName, ctx.Cfg.AppImageURL[0], nodeSelector) + pod := utils.NewPod(podName, ctx.Cfg.AppImageURL[0]) - podlist := CreatePodTest(nodeName, podName, ctx, pod) - for _, pod := range podlist.Items { - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + pod.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) + podList := CreatePodTest(clientSet, pod) + for _, pod := range podList.Items { + err := utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(err).To(BeNil()) } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) + utils.CheckPodDeleteState(clientSet, podList) } }) }) diff --git a/tests/e2e/keadm/keadm_suite_test.go b/tests/e2e/keadm/keadm_suite_test.go index bfab30e54..7b9b9aa12 100644 --- a/tests/e2e/keadm/keadm_suite_test.go +++ b/tests/e2e/keadm/keadm_suite_test.go @@ -29,8 +29,6 @@ import ( ) var ( - nodeName string - //context to load config and access across the package ctx *utils.TestContext ) @@ -49,7 +47,6 @@ func TestKeadmAppDeployment(t *testing.T) { var _ = BeforeSuite(func() { utils.Infof("Before Suite Execution") ctx = utils.NewTestContext(utils.LoadConfig()) - nodeName = "edge-node" }) AfterSuite(func() { By("After Suite Execution....!") diff --git a/tests/e2e/keadm/keadm_test.go b/tests/e2e/keadm/keadm_test.go index 1037ccc81..022e336a4 100644 --- a/tests/e2e/keadm/keadm_test.go +++ b/tests/e2e/keadm/keadm_test.go @@ -17,12 +17,11 @@ limitations under the License. package keadm import ( - "net/http" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + clientset "k8s.io/client-go/kubernetes" "github.com/kubeedge/kubeedge/tests/e2e/constants" . "github.com/kubeedge/kubeedge/tests/e2e/testsuite" @@ -35,6 +34,13 @@ var DeploymentTestTimerGroup = utils.NewTestTimerGroup() var _ = Describe("Application deployment test in keadm E2E scenario", func() { var testTimer *utils.TestTimer var testSpecReport SpecReport + + var clientSet clientset.Interface + + BeforeEach(func() { + clientSet = utils.NewKubeClient(ctx.Cfg.KubeConfigPath) + }) + Context("Test application deployment using Pod spec", func() { BeforeEach(func() { // Get current test SpecReport @@ -47,44 +53,27 @@ var _ = Describe("Application deployment test in keadm E2E scenario", func() { testTimer.End() // Print result testTimer.PrintResult() - var podlist corev1.PodList - label := nodeName - podlist, err := utils.GetPods(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, label) + + labelSelector := labels.SelectorFromSet(constants.KubeEdgeE2ELabel) + podList, err := utils.GetPods(clientSet, metav1.NamespaceDefault, labelSelector, nil) Expect(err).To(BeNil()) - for _, pod := range podlist.Items { - _, StatusCode := utils.DeletePods(ctx.Cfg.K8SMasterForKubeEdge + constants.AppHandler + "/" + pod.Name) - Expect(StatusCode).Should(Equal(http.StatusOK)) + + for _, pod := range podList.Items { + err = utils.DeletePod(clientSet, pod.Namespace, pod.Name) + Expect(err).To(BeNil()) } - utils.CheckPodDeleteState(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podlist) + + utils.CheckPodDeleteState(clientSet, podList) + utils.PrintTestcaseNameandStatus() }) It("E2E_POD_DEPLOYMENT: Create a pod and check the pod is coming up correctly", func() { //Generate the random string and assign as podName podName := "pod-app-" + utils.GetRandomString(5) - pod := NewPodObj(podName, ctx.Cfg.AppImageURL[0], nodeName) + pod := utils.NewPod(podName, ctx.Cfg.AppImageURL[0]) - CreatePodTest(nodeName, podName, ctx, pod) + CreatePodTest(clientSet, pod) }) }) }) - -func NewPodObj(podName, imgURL, nodeName string) *corev1.Pod { - pod := corev1.Pod{ - TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: map[string]string{"app": "nginx"}, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "nginx", - Image: imgURL, - }, - }, - NodeName: nodeName, - }, - } - return &pod -} diff --git a/tests/e2e/testsuite/testsuite.go b/tests/e2e/testsuite/testsuite.go index ce158a8f8..c8a1a7bcf 100644 --- a/tests/e2e/testsuite/testsuite.go +++ b/tests/e2e/testsuite/testsuite.go @@ -17,50 +17,57 @@ limitations under the License. package testsuite import ( - "net/http" + "fmt" "time" + "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" - v1 "k8s.io/api/apps/v1" - metav1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + clientset "k8s.io/client-go/kubernetes" - "github.com/kubeedge/kubeedge/tests/e2e/constants" "github.com/kubeedge/kubeedge/tests/e2e/utils" ) -func CreateDeploymentTest(replica int, deplName, nodeName, nodeSelector string, ctx *utils.TestContext) metav1.PodList { - var deploymentList v1.DeploymentList - var podlist metav1.PodList - IsAppDeployed := utils.HandleDeployment(false, false, http.MethodPost, ctx.Cfg.K8SMasterForKubeEdge+constants.DeploymentHandler, deplName, ctx.Cfg.AppImageURL[1], nodeSelector, "", replica) - gomega.Expect(IsAppDeployed).Should(gomega.BeTrue()) - err := utils.GetDeployments(&deploymentList, ctx.Cfg.K8SMasterForKubeEdge+constants.DeploymentHandler) +func CreateDeploymentTest(c clientset.Interface, replica int32, deplName string, ctx *utils.TestContext) *v1.PodList { + ginkgo.By(fmt.Sprintf("create deployment %s", deplName)) + d := utils.NewDeployment(deplName, ctx.Cfg.AppImageURL[1], replica) + _, err := utils.CreateDeployment(c, d) + gomega.Expect(err).To(gomega.BeNil()) + + ginkgo.By(fmt.Sprintf("get deployment %s", deplName)) + _, err = utils.GetDeployment(c, v1.NamespaceDefault, deplName) gomega.Expect(err).To(gomega.BeNil()) time.Sleep(time.Second * 1) - for _, deployment := range deploymentList.Items { - if deployment.Name == deplName { - label := nodeName - podlist, err = utils.GetPods(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, label) - gomega.Expect(err).To(gomega.BeNil()) - break - } - } - utils.WaitforPodsRunning(ctx.Cfg.KubeConfigPath, podlist, 240*time.Second) + ginkgo.By(fmt.Sprintf("get pod for deployment %s", deplName)) + labelSelector := labels.SelectorFromSet(map[string]string{"app": deplName}) + podList, err := utils.GetPods(c, v1.NamespaceDefault, labelSelector, nil) + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(podList).NotTo(gomega.BeNil()) + + ginkgo.By(fmt.Sprintf("wait for pod of deployment %s running", deplName)) + utils.WaitForPodsRunning(c, podList, 240*time.Second) - return podlist + return podList } -func CreatePodTest(nodeName, podName string, ctx *utils.TestContext, pod *metav1.Pod) metav1.PodList { - var podlist metav1.PodList - IsAppDeployed := utils.HandlePod(http.MethodPost, ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, podName, pod) - gomega.Expect(IsAppDeployed).Should(gomega.BeTrue()) - label := nodeName +func CreatePodTest(c clientset.Interface, pod *v1.Pod) *v1.PodList { + ginkgo.By(fmt.Sprintf("create pod %s/%s", pod.Namespace, pod.Name)) + _, err := utils.CreatePod(c, pod) + gomega.Expect(err).To(gomega.BeNil()) time.Sleep(time.Second * 1) - podlist, err := utils.GetPods(ctx.Cfg.K8SMasterForKubeEdge+constants.AppHandler, label) + ginkgo.By("get pods") + labelSelector := labels.SelectorFromSet(map[string]string{"app": pod.Name}) + podList, err := utils.GetPods(c, v1.NamespaceDefault, labelSelector, nil) gomega.Expect(err).To(gomega.BeNil()) - utils.WaitforPodsRunning(ctx.Cfg.KubeConfigPath, podlist, 240*time.Second) - return podlist + gomega.Expect(podList).NotTo(gomega.BeNil()) + + ginkgo.By(fmt.Sprintf("wait pod %s/%s running", pod.Namespace, pod.Name)) + utils.WaitForPodsRunning(c, podList, 240*time.Second) + + return podList } diff --git a/tests/e2e/utils/common.go b/tests/e2e/utils/common.go index 4370785b6..a4ffda3ad 100644 --- a/tests/e2e/utils/common.go +++ b/tests/e2e/utils/common.go @@ -24,27 +24,21 @@ import ( "fmt" "io" "net/http" - "os/exec" "reflect" "strings" "time" MQTT "github.com/eclipse/paho.mqtt.golang" - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - "github.com/kubeedge/kubeedge/common/constants" "github.com/kubeedge/kubeedge/pkg/apis/devices/v1alpha2" - "github.com/kubeedge/viaduct/pkg/api" + "github.com/kubeedge/kubeedge/tests/e2e/constants" ) const ( @@ -134,424 +128,86 @@ type ServicebusResponse struct { Body string `json:"body"` } -// Function to get nginx deployment spec -func nginxDeploymentSpec(imgURL, selector string, replicas int) *apps.DeploymentSpec { - var nodeselector map[string]string - if selector == "" { - nodeselector = map[string]string{} - } else { - nodeselector = map[string]string{"disktype": selector} - } - deplObj := apps.DeploymentSpec{ - Replicas: func() *int32 { i := int32(replicas); return &i }(), - Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "nginx"}}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "nginx"}, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "nginx", - Image: imgURL, - }, - }, - NodeSelector: nodeselector, - }, +func NewDeployment(name, imgURL string, replicas int32) *apps.Deployment { + deployment := apps.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{"app": name}, + Namespace: Namespace, }, - } - - return &deplObj -} - -// Function to get edgecore deploymentspec object -func edgecoreDeploymentSpec(imgURL, configmap string, replicas int) *apps.DeploymentSpec { - IsSecureCtx := true - deplObj := apps.DeploymentSpec{ - Replicas: func() *int32 { i := int32(replicas); return &i }(), - Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "edgecore"}}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "edgecore"}, + Spec: apps.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": name, + constants.E2ELabelKey: constants.E2ELabelValue, + }, }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "edgecore", - Image: imgURL, - SecurityContext: &v1.SecurityContext{Privileged: &IsSecureCtx}, - ImagePullPolicy: v1.PullPolicy("IfNotPresent"), - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - }, - Limits: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - }, - }, - Env: []v1.EnvVar{{Name: "DOCKER_HOST", Value: "tcp://localhost:2375"}}, - VolumeMounts: []v1.VolumeMount{{Name: "cert", MountPath: "/etc/kubeedge/certs"}, {Name: "conf", MountPath: "/etc/kubeedge/edge/conf"}}, - }, { - Name: "dind-daemon", - SecurityContext: &v1.SecurityContext{Privileged: &IsSecureCtx}, - Image: "docker:dind", - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("20m"), - v1.ResourceMemory: resource.MustParse("256Mi"), - }, - }, - VolumeMounts: []v1.VolumeMount{{Name: "docker-graph-storage", MountPath: "/var/lib/docker"}}, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": name, + constants.E2ELabelKey: constants.E2ELabelValue, }, }, - NodeSelector: map[string]string{"k8snode": "kb-perf-node"}, - Volumes: []v1.Volume{ - {Name: "cert", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubeedge/certs"}}}, - {Name: "conf", VolumeSource: v1.VolumeSource{ConfigMap: &v1.ConfigMapVolumeSource{LocalObjectReference: v1.LocalObjectReference{Name: configmap}}}}, - {Name: "docker-graph-storage", VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}}}, - }, - }, - }, - } - return &deplObj -} - -// Function to create cloudcore deploymentspec object -func cloudcoreDeploymentSpec(imgURL, configmap string, replicas int) *apps.DeploymentSpec { - portInfo := []v1.ContainerPort{{ContainerPort: 10000, Protocol: "TCP", Name: "websocket"}, {ContainerPort: 10001, Protocol: "UDP", Name: "quic"}} - - deplObj := apps.DeploymentSpec{ - Replicas: func() *int32 { i := int32(replicas); return &i }(), - Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "cloudcore"}}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "cloudcore"}, - }, - Spec: v1.PodSpec{ - HostNetwork: true, - RestartPolicy: "Always", - Containers: []v1.Container{ - { - Name: "cloudcore", - Image: imgURL, - ImagePullPolicy: v1.PullPolicy("IfNotPresent"), - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100m"), - v1.ResourceMemory: resource.MustParse("512Mi"), - }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: name, + Image: imgURL, }, - Ports: portInfo, - VolumeMounts: []v1.VolumeMount{{Name: "cert", MountPath: "/etc/kubeedge/certs"}, {Name: "conf", MountPath: "/etc/kubeedge/cloud/conf"}}, }, - }, - Volumes: []v1.Volume{ - {Name: "cert", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubeedge/certs"}}}, - {Name: "conf", VolumeSource: v1.VolumeSource{ConfigMap: &v1.ConfigMapVolumeSource{LocalObjectReference: v1.LocalObjectReference{Name: configmap}}}}, + NodeSelector: map[string]string{ + "node-role.kubernetes.io/edge": "", + }, }, }, }, } - return &deplObj -} - -func newDeployment(cloudcore, edgecore bool, name, imgURL, nodeselector, configmap string, replicas int) *apps.Deployment { - var depObj *apps.DeploymentSpec - var namespace string - - if edgecore { - depObj = edgecoreDeploymentSpec(imgURL, configmap, replicas) - namespace = Namespace - } else if cloudcore { - depObj = cloudcoreDeploymentSpec(imgURL, configmap, replicas) - namespace = Namespace - } else { - depObj = nginxDeploymentSpec(imgURL, nodeselector, replicas) - namespace = Namespace - } - - deployment := apps.Deployment{ - TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{"app": constants.SystemName}, - Namespace: namespace, - }, - Spec: *depObj, - } return &deployment } -func NewPodObj(podName, imgURL, nodeselector string) *v1.Pod { +func NewPod(podName, imgURL string) *v1.Pod { pod := v1.Pod{ - TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: map[string]string{"app": "nginx"}, + Name: podName, + Namespace: v1.NamespaceDefault, + Labels: map[string]string{ + "app": podName, + constants.E2ELabelKey: constants.E2ELabelValue, + }, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "nginx", + Name: podName, Image: imgURL, }, }, - NodeSelector: map[string]string{"disktype": nodeselector}, + NodeSelector: map[string]string{ + "node-role.kubernetes.io/edge": "", + }, }, } return &pod } -// GetDeployments to get the deployments list -func GetDeployments(list *apps.DeploymentList, getDeploymentAPI string) error { - resp, err := SendHTTPRequest(http.MethodGet, getDeploymentAPI) - if err != nil { - Fatalf("HTTP Response reading has failed: %v", err) - return err - } - defer resp.Body.Close() - contents, err := io.ReadAll(resp.Body) - if err != nil { - Fatalf("HTTP Response reading has failed: %v", err) - return err - } - err = json.Unmarshal(contents, &list) - if err != nil { - Fatalf("Unmarshal HTTP Response has failed: %v", err) - return err - } - return nil -} -func VerifyDeleteDeployment(getDeploymentAPI string) int { - resp, err := SendHTTPRequest(http.MethodGet, getDeploymentAPI) - if err != nil { - Fatalf("Send HTTP Request failed: %v", err) - return -1 - } - defer resp.Body.Close() - return resp.StatusCode -} - -// HandlePod to handle app deployment/delete using pod spec. -func HandlePod(operation string, apiserver string, UID string, pod *v1.Pod) bool { - var req *http.Request - var err error - var body io.Reader - - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client := &http.Client{ - Transport: tr, - } - switch operation { - case http.MethodPost: - body := pod - respBytes, err := json.Marshal(body) - if err != nil { - Fatalf("Marshalling body failed: %v", err) - } - req, err = http.NewRequest(http.MethodPost, apiserver, bytes.NewBuffer(respBytes)) - case http.MethodDelete: - req, err = http.NewRequest(http.MethodDelete, apiserver+UID, body) - } - if err != nil { - // handle error - Fatalf("Frame HTTP request failed: %v", err) - return false - } - req.Header.Set("Content-Type", "application/json") - t := time.Now() - resp, err := client.Do(req) - if err != nil { - // handle error - Fatalf("HTTP request is failed :%v", err) - return false - } - defer resp.Body.Close() - Infof("%s %s %v in %v", req.Method, req.URL, resp.Status, time.Since(t)) - return true +func GetDeployment(c clientset.Interface, ns, name string) (*apps.Deployment, error) { + return c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{}) } -// HandleDeployment to handle app deployment/delete deployment. -func HandleDeployment(IsCloudCore, IsEdgeCore bool, operation, apiserver, UID, ImageURL, nodeselector, configmapname string, replica int) bool { - var req *http.Request - var err error - var body io.Reader - - defer ginkgo.GinkgoRecover() - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client := &http.Client{ - Transport: tr, - } - - switch operation { - case http.MethodPost: - depObj := newDeployment(IsCloudCore, IsEdgeCore, UID, ImageURL, nodeselector, configmapname, replica) - if err != nil { - Fatalf("GenerateDeploymentBody marshalling failed: %v", err) - } - respBytes, err := json.Marshal(depObj) - if err != nil { - Fatalf("Marshalling body failed: %v", err) - } - req, err = http.NewRequest(http.MethodPost, apiserver, bytes.NewBuffer(respBytes)) - case http.MethodDelete: - req, err = http.NewRequest(http.MethodDelete, apiserver+UID, body) - } - if err != nil { - // handle error - Fatalf("Frame HTTP request failed: %v", err) - return false - } - req.Header.Set("Content-Type", "application/json") - t := time.Now() - resp, err := client.Do(req) - if err != nil { - // handle error - Fatalf("HTTP request is failed :%v", err) - return false - } - defer resp.Body.Close() - Infof("%s %s %v in %v", req.Method, req.URL, resp.Status, time.Since(t)) - return true +func CreateDeployment(c clientset.Interface, deployment *apps.Deployment) (*apps.Deployment, error) { + return c.AppsV1().Deployments(deployment.Namespace).Create(context.TODO(), deployment, metav1.CreateOptions{}) } // DeleteDeployment to delete deployment -func DeleteDeployment(DeploymentAPI, deploymentname string) int { - resp, err := SendHTTPRequest(http.MethodDelete, DeploymentAPI+"/"+deploymentname) - if err != nil { - // handle error - Fatalf("HTTP request is failed :%v", err) - return -1 - } - - defer resp.Body.Close() - - return resp.StatusCode -} - -// PrintCombinedOutput to show the os command injuction in combined format -func PrintCombinedOutput(cmd *exec.Cmd) error { - Infof("===========> Executing: %s\n", strings.Join(cmd.Args, " ")) - output, err := cmd.CombinedOutput() - if err != nil { - Infof("CombinedOutput failed %v", err) - return err - } - if len(output) > 0 { - Infof("=====> Output: %s\n", string(output)) - } - return nil -} - -// ExposeCloudService function to expose the service for cloud deployment -func ExposeCloudService(name, serviceHandler string) error { - ServiceObj := CreateServiceObject(name) - respBytes, err := json.Marshal(ServiceObj) - if err != nil { - Fatalf("Marshalling body failed: %v", err) - } - req, err := http.NewRequest(http.MethodPost, serviceHandler, bytes.NewBuffer(respBytes)) - if err != nil { - // handle error - Fatalf("Frame HTTP request failed: %v", err) - return err - } - client := &http.Client{} - req.Header.Set("Content-Type", "application/json") - t := time.Now() - resp, err := client.Do(req) - if err != nil { - // handle error - Fatalf("HTTP request is failed :%v", err) - return err - } - defer resp.Body.Close() - Infof("%s %s %v in %v", req.Method, req.URL, resp.Status, time.Since(t)) - gomega.Expect(resp.StatusCode).Should(gomega.Equal(http.StatusCreated)) - return nil -} - -// CreateServiceObject function to create a servcice object -func CreateServiceObject(name string) *v1.Service { - portInfo := []v1.ServicePort{ - { - Name: "websocket", Protocol: "TCP", Port: 10000, TargetPort: intstr.FromInt(10000), - }, { - Name: "quic", Protocol: "UDP", Port: 10001, TargetPort: intstr.FromInt(10001), - }, - } - - Service := v1.Service{ - TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, - ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"app": constants.SystemName}}, - - Spec: v1.ServiceSpec{ - Ports: portInfo, - Selector: map[string]string{"app": "cloudcore"}, - Type: "NodePort", - }, - } - return &Service -} - -// GetServicePort function to get the service port created for deployment. -func GetServicePort(cloudName, serviceHandler string) (int32, int32) { - var svc v1.ServiceList - var wssport, quicport int32 - resp, err := SendHTTPRequest(http.MethodGet, serviceHandler) - if err != nil { - // handle error - Fatalf("HTTP request is failed :%v", err) - return -1, -1 - } - defer resp.Body.Close() - - contents, err := io.ReadAll(resp.Body) - if err != nil { - Fatalf("HTTP Response reading has failed: %v", err) - return -1, -1 - } - - err = json.Unmarshal(contents, &svc) - if err != nil { - Fatalf("Unmarshal HTTP Response has failed: %v", err) - return -1, -1 - } - - for _, svcs := range svc.Items { - if svcs.Name == cloudName { - for _, nodePort := range svcs.Spec.Ports { - if nodePort.Name == api.ProtocolTypeQuic { - quicport = nodePort.NodePort - } - if nodePort.Name == api.ProtocolTypeWS { - wssport = nodePort.NodePort - } - } - break - } - } - return wssport, quicport -} - -// DeleteSvc function to delete service -func DeleteSvc(svcname string) int { - resp, err := SendHTTPRequest(http.MethodDelete, svcname) - if err != nil { - // handle error - Fatalf("HTTP request is failed :%v", err) - return -1 +func DeleteDeployment(c clientset.Interface, ns, name string) error { + err := c.AppsV1().Deployments(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && apierrors.IsNotFound(err) { + return nil } - defer resp.Body.Close() - - return resp.StatusCode + return err } // HandleDeviceModel to handle app deployment/delete using pod spec. @@ -1064,13 +720,22 @@ func NewTestStatefulSet(name, imgURL string, replicas int32) *apps.StatefulSet { Spec: apps.StatefulSetSpec{ Replicas: &replicas, Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": name}, + MatchLabels: map[string]string{ + "app": name, + constants.E2ELabelKey: constants.E2ELabelValue, + }, }, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": name}, + Labels: map[string]string{ + "app": name, + constants.E2ELabelKey: constants.E2ELabelValue, + }, }, Spec: v1.PodSpec{ + NodeSelector: map[string]string{ + "node-role.kubernetes.io/edge": "", + }, Containers: []v1.Container{ { Name: "nginx", diff --git a/tests/e2e/utils/context.go b/tests/e2e/utils/context.go index ed0eef9a3..121becca7 100644 --- a/tests/e2e/utils/context.go +++ b/tests/e2e/utils/context.go @@ -19,9 +19,6 @@ import ( "crypto/tls" "io" "net/http" - "net/url" - "sort" - "strings" "time" ) @@ -65,13 +62,3 @@ func SendHTTPRequest(method, reqAPI string) (*http.Response, error) { Infof("%s %s %v in %v", req.Method, req.URL, resp.Status, time.Since(t)) return resp, nil } - -//MapLabels function add label selector -func MapLabels(ls map[string]string) string { - selector := make([]string, 0, len(ls)) - for key, value := range ls { - selector = append(selector, key+"="+value) - } - sort.StringSlice(selector).Sort() - return url.QueryEscape(strings.Join(selector, ",")) -} diff --git a/tests/e2e/utils/node.go b/tests/e2e/utils/node.go index ab9795a29..4762be2e3 100644 --- a/tests/e2e/utils/node.go +++ b/tests/e2e/utils/node.go @@ -18,6 +18,7 @@ package utils import ( "bytes" + "context" "crypto/tls" "encoding/json" "fmt" @@ -31,6 +32,9 @@ import ( "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" "sigs.k8s.io/yaml" ) @@ -214,15 +218,12 @@ func GetConfigmap(apiConfigMap string) (int, []byte) { return resp.StatusCode, body } -//DeleteConfigmap function to delete configmaps -func DeleteConfigmap(apiConfigMap string) int { - resp, err := SendHTTPRequest(http.MethodDelete, apiConfigMap) - if err != nil { - Fatalf("Sending SenHttpRequest failed: %v", err) - return -1 +func DeleteConfigMap(client clientset.Interface, ns, name string) error { + err := client.CoreV1().ConfigMaps(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && apierrors.IsNotFound(err) { + return nil } - defer resp.Body.Close() - return resp.StatusCode + return err } func TaintEdgeDeployedNode(toTaint bool, taintHandler string) error { diff --git a/tests/e2e/utils/pod.go b/tests/e2e/utils/pod.go index ac791199f..5908ff619 100644 --- a/tests/e2e/utils/pod.go +++ b/tests/e2e/utils/pod.go @@ -18,14 +18,11 @@ package utils import ( "context" - "encoding/json" - "io" - "net/http" - "strings" "time" "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" @@ -33,13 +30,10 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" ) -const ( - podLabelSelector = "?fieldSelector=spec.nodeName=" -) - -func ListPods(c clientset.Interface, ns string, labelSelector labels.Selector, fieldSelector fields.Selector) (*v1.PodList, error) { +func GetPods(c clientset.Interface, ns string, labelSelector labels.Selector, fieldSelector fields.Selector) (*v1.PodList, error) { options := metav1.ListOptions{} if fieldSelector != nil { @@ -53,6 +47,18 @@ func ListPods(c clientset.Interface, ns string, labelSelector labels.Selector, f return c.CoreV1().Pods(ns).List(context.TODO(), options) } +func GetPod(c clientset.Interface, ns, name string) (*v1.Pod, error) { + return c.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{}) +} + +func DeletePod(c clientset.Interface, ns, name string) error { + return c.CoreV1().Pods(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}) +} + +func CreatePod(c clientset.Interface, pod *v1.Pod) (*v1.Pod, error) { + return c.CoreV1().Pods(pod.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{}) +} + func WaitForPodsToDisappear(c clientset.Interface, ns string, label labels.Selector, interval, timeout time.Duration) error { return wait.PollImmediate(interval, timeout, func() (bool, error) { Infof("Waiting for pod with label %s to disappear", label.String()) @@ -71,151 +77,35 @@ func WaitForPodsToDisappear(c clientset.Interface, ns string, label labels.Selec }) } -func DeletePod(c clientset.Interface, name, ns string) error { - return c.CoreV1().Pods(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}) -} - -//GetPods function to get the pods from Edged -func GetPods(apiserver, label string) (v1.PodList, error) { - var pods v1.PodList - var resp *http.Response - var err error - - if len(label) > 0 { - resp, err = SendHTTPRequest(http.MethodGet, apiserver+podLabelSelector+label) - } else { - resp, err = SendHTTPRequest(http.MethodGet, apiserver) - } - if err != nil { - Fatalf("Frame HTTP request failed: %v", err) - return pods, nil - } - defer resp.Body.Close() - contents, err := io.ReadAll(resp.Body) - if err != nil { - Fatalf("HTTP Response reading has failed: %v", err) - return pods, nil - } - err = json.Unmarshal(contents, &pods) - if err != nil { - Fatalf("Unmarshal HTTP Response has failed: %v", err) - return pods, nil - } - return pods, nil -} - -//GetPodState function to get the pod status and response code -func GetPodState(apiserver string) (string, int) { - var pod v1.Pod - - resp, err := SendHTTPRequest(http.MethodGet, apiserver) - if err != nil { - Fatalf("GetPodState :SenHttpRequest failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusNotFound { - contents, err := io.ReadAll(resp.Body) - if err != nil { - Fatalf("HTTP Response reading has failed: %v", err) - } - err = json.Unmarshal(contents, &pod) - if err != nil { - Fatalf("Unmarshal HTTP Response has failed: %v", err) - } - return string(pod.Status.Phase), resp.StatusCode - } - - return "", resp.StatusCode -} - -//DeletePods function to get the pod status and response code -func DeletePods(apiserver string) (string, int) { - var pod v1.Pod - resp, err := SendHTTPRequest(http.MethodDelete, apiserver) - if err != nil { - Fatalf("GetPodState :SenHttpRequest failed: %v", err) - } - defer resp.Body.Close() +// CheckPodDeleteState check whether the given pod list is deleted successfully +func CheckPodDeleteState(c clientset.Interface, podList *v1.PodList) { + podCount := len(podList.Items) - if resp.StatusCode != http.StatusNotFound { - contents, err := io.ReadAll(resp.Body) - if err != nil { - Fatalf("HTTP Response reading has failed: %v", err) - } - err = json.Unmarshal(contents, &pod) - if err != nil { - Fatalf("Unmarshal HTTP Response has failed: %v", err) - } - return string(pod.Status.Phase), resp.StatusCode - } + errInfo := "Pods of deploy are not deleted within the time" - return "", resp.StatusCode -} - -//CheckPodRunningState function to check the Pod state -func CheckPodRunningState(apiserver string, podlist v1.PodList) { gomega.Eventually(func() int { var count int - for _, pod := range podlist.Items { - state, _ := GetPodState(apiserver + "/" + pod.Name) - Infof("PodName: %s PodStatus: %s", pod.Name, state) - if state == "Running" { + for _, pod := range podList.Items { + _, err := GetPod(c, pod.Namespace, pod.Name) + if err != nil && apierrors.IsNotFound(err) { count++ + continue } - } - return count - }, "600s", "2s").Should(gomega.Equal(len(podlist.Items)), "Application deployment is Unsuccessful, Pod has not come to Running State") -} -//CheckPodDeleteState function to check the Pod state -func CheckPodDeleteState(apiserver string, podlist v1.PodList) { - var count int - //skip the edgecore/cloudcore deployment pods and count only application pods deployed on KubeEdge edgen node - for _, pod := range podlist.Items { - if strings.Contains(pod.Name, "deployment-") { - count++ - } - } - podCount := len(podlist.Items) - count - gomega.Eventually(func() int { - var count int - for _, pod := range podlist.Items { - status, statusCode := GetPodState(apiserver + "/" + pod.Name) - Infof("PodName: %s status: %s StatusCode: %d", pod.Name, status, statusCode) - if statusCode == 404 { - count++ + if err != nil { + klog.Errorf("get pod %s/%s error", pod.Namespace, pod.Name) + continue } - } - return count - }, "600s", "4s").Should(gomega.Equal(podCount), "Delete Application deployment is Unsuccessful, Pods are not deleted within the time") -} -//CheckDeploymentPodDeleteState function to check the Pod state -func CheckDeploymentPodDeleteState(apiserver string, podlist v1.PodList) { - var count int - //count the edgecore/cloudcore deployment pods and count only application pods deployed on KubeEdge edgen node - for _, pod := range podlist.Items { - if strings.Contains(pod.Name, "deployment-") { - count++ - } - } - //podCount := len(podlist.Items) - count - gomega.Eventually(func() int { - var count int - for _, pod := range podlist.Items { - status, statusCode := GetPodState(apiserver + "/" + pod.Name) - Infof("PodName: %s status: %s StatusCode: %d", pod.Name, status, statusCode) - if statusCode == 404 { - count++ - } + Infof("Pod %s/%s still exist", pod.Namespace, pod.Name) } + return count - }, "240s", "4s").Should(gomega.Equal(count), "Delete Application deployment is Unsuccessful, Pods are not deleted within the time") + }, "240s", "4s").Should(gomega.Equal(podCount), errInfo) } // NewKubeClient creates kube client from config -func NewKubeClient(kubeConfigPath string) *clientset.Clientset { +func NewKubeClient(kubeConfigPath string) clientset.Interface { kubeConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) if err != nil { Fatalf("Get kube config failed with error: %v", err) @@ -232,38 +122,32 @@ func NewKubeClient(kubeConfigPath string) *clientset.Clientset { return kubeClient } -// WaitforPodsRunning waits util all pods are in running status or timeout -func WaitforPodsRunning(kubeConfigPath string, podlist v1.PodList, timout time.Duration) { - if len(podlist.Items) == 0 { - Fatalf("podlist should not be empty") +// WaitForPodsRunning waits util all pods are in running status or timeout +func WaitForPodsRunning(c clientset.Interface, podList *v1.PodList, timeout time.Duration) { + if len(podList.Items) == 0 { + Fatalf("podList should not be empty") } podRunningCount := 0 - for _, pod := range podlist.Items { + for _, pod := range podList.Items { if pod.Status.Phase == v1.PodRunning { podRunningCount++ } } - if podRunningCount == len(podlist.Items) { + + if podRunningCount == len(podList.Items) { Infof("All pods come into running status") return } - // new kube client - kubeClient := NewKubeClient(kubeConfigPath) // define signal signal := make(chan struct{}) + // define list watcher - listWatcher := cache.NewListWatchFromClient( - kubeClient.CoreV1().RESTClient(), - "pods", - v1.NamespaceAll, - fields.Everything()) + listWatcher := cache.NewListWatchFromClient(c.CoreV1().RESTClient(), "pods", v1.NamespaceAll, fields.Everything()) + // new controller - _, controller := cache.NewInformer( - listWatcher, - &v1.Pod{}, - time.Second*0, + _, controller := cache.NewInformer(listWatcher, &v1.Pod{}, 0, cache.ResourceEventHandlerFuncs{ // receive update events UpdateFunc: func(oldObj, newObj interface{}) { @@ -272,28 +156,30 @@ func WaitforPodsRunning(kubeConfigPath string, podlist v1.PodList, timout time.D if !ok { Fatalf("Failed to cast observed object to pod") } + // calculate the pods in running status count := 0 - for i := range podlist.Items { - // update pod status in podlist - if podlist.Items[i].Name == p.Name { + for i := range podList.Items { + // update pod status in podList + if podList.Items[i].Name == p.Name { Infof("PodName: %s PodStatus: %s", p.Name, p.Status.Phase) - podlist.Items[i].Status = p.Status + podList.Items[i].Status = p.Status } // check if the pod is in running status - if podlist.Items[i].Status.Phase == v1.PodRunning { + if podList.Items[i].Status.Phase == v1.PodRunning { count++ } } + // send an end signal when all pods are in running status - if len(podlist.Items) == count { + if len(podList.Items) == count { signal <- struct{}{} } }, }, ) - // run controoler + // run controller podChan := make(chan struct{}) go controller.Run(podChan) defer close(podChan) @@ -302,7 +188,7 @@ func WaitforPodsRunning(kubeConfigPath string, podlist v1.PodList, timout time.D select { case <-signal: Infof("All pods come into running status") - case <-time.After(timout): - Fatalf("Wait for pods come into running status timeout: %v", timout) + case <-time.After(timeout): + Fatalf("Wait for pods come into running status timeout: %v", timeout) } } |
