Add keadm compatible e2e test

Signed-off-by: liqing.wu <liqing.wu@daoclod.io>

3 files changed, 223 insertions, 0 deletions

diff --git a/.github/workflows/keadm_compatibility.yaml b/.github/workflows/keadm_compatibility.yaml
new file mode 100644
index 000000000..2cb45e757
--- /dev/null
+++ b/.github/workflows/keadm_compatibility.yaml
@@ -0,0 +1,62 @@
+name: KEAMD COMPATIBILITY
+on:
+  workflow_dispatch:
+  schedule:
+    #  Exectues keamd compatibility test every Wednesday at 3:00 AM and if you need to convert to UTC+8 timezone, you need to -8 hours for cron expression
+    - cron: "0 17 * * 3"
+
+env:
+  CONTAINER_RUN_OPTIONS: " "
+  GINKGO_VERSION: "v2.9.5"
+
+jobs:
+
+  keadm_compatibility_e2e_test:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        version:
+          - v1.15.1
+          - v1.14.4
+    timeout-minutes: 90
+    name: Keadm compatibility e2e test
+    env:
+      GO111MODULE: on
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.x
+
+      - uses: actions/cache@v3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+
+      - name: Install dependences
+        run: |
+          command -v ginkgo || go install github.com/onsi/ginkgo/v2/ginkgo@${{ env.GINKGO_VERSION }}
+          go install sigs.k8s.io/kind@v0.19.0
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.27.7/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: v3.4.0
+
+      - name: enable cri config in containerd service
+        run: |
+          containerd config default | sudo tee /etc/containerd/config.toml && sudo systemctl restart containerd.service
+
+      - name: cleanup images
+        run: docker system prune -a -f
+
+      - run: |
+          export CLOUD_EDGE_VERSION=${{ matrix.version }}
+          make keadm_compatibility_e2e
+
diff --git a/Makefile b/Makefile
index 395feba50..548a93fe6 100644
--- a/Makefile
+++ b/Makefile
@@ -449,3 +449,20 @@ else
 release:
 	hack/make-rules/release.sh $(WHAT) $(ARM_VERSION) $(OS)
 endif
+
+define KEADM_COMPATIBILITY_E2E_HELP_INFO
+# keadm compatibility e2e test
+#
+# Example:
+#   make keadm_compatibility_e2e
+#   make keadm_compatibility_e2e HELP=y
+#
+endef
+.PHONY: keadm_compatibility_e2e
+ifeq ($(HELP),y)
+keadm_compatibility_e2e:
+	@echo "KEADM_COMPATIBILITY_E2E_HELP_INFO"
+else
+keadm_compatibility_e2e:
+	tests/scripts/keadm_compatibility_e2e.sh ${CLOUD_EDGE_VERSION}
+endif
diff --git a/tests/scripts/keadm_compatibility_e2e.sh b/tests/scripts/keadm_compatibility_e2e.sh
new file mode 100755
index 000000000..beeada334
--- /dev/null
+++ b/tests/scripts/keadm_compatibility_e2e.sh
@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 The KubeEdge Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+KUBEEDGE_ROOT=$PWD
+WORKDIR=$(dirname $0)
+E2E_DIR=$(realpath $(dirname $0)/..)
+IMAGE_TAG=$(git describe --tags)
+CLOUD_EDGE_VERSION=${1:-"v1.15.1"}
+
+
+source "${KUBEEDGE_ROOT}/hack/lib/install.sh"
+
+function cleanup() {
+  sudo pkill edgecore || true
+  helm uninstall cloudcore -n kubeedge && kubectl delete ns kubeedge  || true
+  kind delete cluster --name test
+  sudo rm -rf /var/log/kubeedge /etc/kubeedge /etc/systemd/system/edgecore.service $E2E_DIR/e2e_keadm/e2e_keadm.test $E2E_DIR/config.json
+}
+
+function build_ginkgo() {
+  cd $E2E_DIR
+  ginkgo build -r e2e_keadm/
+}
+
+function prepare_cluster() {
+  kind create cluster --name test
+
+  echo "wait the control-plane ready..."
+  kubectl wait --for=condition=Ready node/test-control-plane --timeout=60s
+
+  kubectl create clusterrolebinding system:anonymous --clusterrole=cluster-admin --user=system:anonymous
+
+  # edge side don't support kind cni now, delete kind cni plugin for workaround
+  kubectl delete daemonset kindnet -nkube-system
+}
+
+function build_image() {
+  cd $KUBEEDGE_ROOT
+  make image WHAT=installation-package -f $KUBEEDGE_ROOT/Makefile
+  docker save kubeedge/installation-package:$IMAGE_TAG > installation-package.tar
+  sudo ctr -n=k8s.io image import installation-package.tar
+  kind load docker-image docker.io/kubeedge/installation-package:$IMAGE_TAG --name test
+
+  set +e
+  docker rmi $(docker images -f "dangling=true" -q)
+  set -Ee
+}
+
+function get_cloudcore_image() {
+   docker pull kubeedge/cloudcore:$CLOUD_EDGE_VERSION
+   docker tag kubeedge/cloudcore:$CLOUD_EDGE_VERSION docker.io/kubeedge/cloudcore:$CLOUD_EDGE_VERSION
+   kind load docker-image docker.io/kubeedge/cloudcore:$CLOUD_EDGE_VERSION --name test
+
+   set +e
+   docker rmi $(docker images -f "dangling=true" -q)
+   docker system prune -f
+   set -Ee
+
+}
+
+function start_kubeedge() {
+  sudo mkdir -p /var/lib/kubeedge
+  cd $KUBEEDGE_ROOT
+  export MASTER_IP=`kubectl get node test-control-plane -o jsonpath={.status.addresses[0].address}`
+  export KUBECONFIG=$HOME/.kube/config
+  docker run --rm kubeedge/installation-package:$IMAGE_TAG cat /usr/local/bin/keadm > /usr/local/bin/keadm && chmod +x /usr/local/bin/keadm
+  /usr/local/bin/keadm init --advertise-address=$MASTER_IP --profile version=$CLOUD_EDGE_VERSION --set cloudCore.service.enable=false --kube-config=$KUBECONFIG --force
+
+  # ensure tokensecret is generated
+  while true; do
+      sleep 3
+      kubectl get secret -nkubeedge 2>/dev/null | grep -q tokensecret && break
+  done
+
+  cd $KUBEEDGE_ROOT
+  export TOKEN=$(sudo /usr/local/bin/keadm gettoken --kube-config=$KUBECONFIG)
+  sudo systemctl set-environment CHECK_EDGECORE_ENVIRONMENT="false"
+  sudo /usr/local/bin/keadm join --token=$TOKEN --cloudcore-ipport=$MASTER_IP:10000 --edgenode-name=edge-node --kubeedge-version=$CLOUD_EDGE_VERSION
+
+  # ensure edgenode is ready
+  while true; do
+      sleep 3
+      kubectl get node | grep edge-node | grep -q -w Ready && break
+  done
+}
+
+function run_test() {
+  :> /tmp/testcase.log
+  cd $E2E_DIR
+
+  export ACK_GINKGO_RC=true
+
+  ginkgo -v ./e2e_keadm/e2e_keadm.test -- \
+  --image-url=nginx \
+  --image-url=nginx \
+  --kube-master="https://$MASTER_IP:6443" \
+  --kubeconfig=$KUBECONFIG \
+  --test.v
+  if [[ $? != 0 ]]; then
+      echo "Integration suite has failures, Please check !!"
+      exit 1
+  else
+      echo "Integration suite successfully passed all the tests !!"
+      exit 0
+  fi
+}
+
+set -Ee
+trap cleanup EXIT
+
+echo -e "\nBuilding ginkgo test cases..."
+build_ginkgo
+
+export KUBECONFIG=$HOME/.kube/config
+
+echo -e "\nPreparing cluster..."
+prepare_cluster
+
+echo -e "\nBuilding keadm image..."
+build_image
+
+echo -e "\nGet cloudcore image..."
+get_cloudcore_image
+
+install_cni_plugins
+
+echo -e "\nStarting kubeedge..."
+start_kubeedge
+
+echo -e "\nRunning test..."
+run_test