Merge pull request #5321 from luomengY/ci-cri

add cri-ci

2 files changed, 232 insertions, 43 deletions

diff --git a/hack/lib/install.sh b/hack/lib/install.sh
index 3850f063e..be83a53a6 100755
--- a/hack/lib/install.sh
+++ b/hack/lib/install.sh
@@ -94,6 +94,30 @@ verify_docker_installed() {
   }
 }
 
+verify_cridockerd_installed() {
+  # verify the cri-dockerd installed
+  command -v cri-dockerd >/dev/null || {
+    echo "must install the cri-dockerd first"
+    exit 1
+  }
+}
+
+verify_crio_installed() {
+  # verify the cri-o installed
+  command -v crio >/dev/null || {
+    echo "must install the cri-o first"
+    exit 1
+  }
+}
+
+verify_isulad_installed() {
+  # verify the isulad installed
+  command -v isulad >/dev/null || {
+    echo "must install the isulad first"
+    exit 1
+  }
+}
+
 # install CNI plugins
 function install_cni_plugins() {
   CNI_DOWNLOAD_ADDR=${CNI_DOWNLOAD_ADDR:-"https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz"}
@@ -111,7 +135,7 @@ function install_cni_plugins() {
       exit 1
     fi
     sudo tar Cxzvf /opt/cni/bin ${CNI_PKG}
-    rm -rf ${CNI_PKG}
+    sudo rm -rf ${CNI_PKG}
     if [ ! -f "/opt/cni/bin/loopback" ]; then
       echo -e "the ${CNI_PKG} package does not contain a loopback file."
       exit 1
@@ -162,9 +186,132 @@ function install_cni_plugins() {
   ]
 }
 EOF'
-    sudo systemctl restart containerd
-    sleep 2
   else
     echo "CNI plugins already installed and no need to install"
   fi
 }
+
+function install_docker() {
+  CRIDOCKERD_VERSION="v0.3.8"
+  sudo apt-get update
+  sudo apt-get install \
+    apt-transport-https \
+    ca-certificates \
+    curl \
+    gnupg \
+    lsb-release
+  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+  echo \
+    "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
+                 $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
+  sudo apt-get update
+  sudo apt-get install docker-ce docker-ce-cli containerd.io
+  git clone https://github.com/Mirantis/cri-dockerd.git -b ${CRIDOCKERD_VERSION}
+  cd cri-dockerd
+  make cri-dockerd
+  sudo install -o root -g root -m 0755 cri-dockerd /usr/local/bin/cri-dockerd
+  sudo install packaging/systemd/* /etc/systemd/system
+  sudo sed -i -e 's,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,' /etc/systemd/system/cri-docker.service
+  sudo systemctl daemon-reload
+  sudo systemctl enable --now cri-docker.socket
+  sudo systemctl restart cri-docker
+  cd .. && sudo rm -rf cri-dockerd
+}
+
+function install_crio() {
+  CRIO_VERSION="1.28.2"
+  sudo rm -rf cri-o.amd64.v${CRIO_VERSION}.tar.gz && sudo rm -rf cri-o
+  sudo wget https://storage.googleapis.com/cri-o/artifacts/cri-o.amd64.v${CRIO_VERSION}.tar.gz
+  sudo tar -zxvf cri-o.amd64.v${CRIO_VERSION}.tar.gz
+  sudo sed -i 's/\/usr\/bin\/env sh/!\/bin\/bash/' cri-o/install
+  sudo sed -i 's/ExecStart=.*/ExecStart=\/usr\/local\/bin\/crio --selinux=false \\/' cri-o/contrib/crio.service
+  cd cri-o
+  sudo /bin/bash ./install
+  sudo systemctl daemon-reload
+  sudo systemctl enable --now crio
+  sudo systemctl restart crio
+  cd .. && sudo rm -rf cri-o.amd64.v${CRIO_VERSION}.tar.gz && sudo rm -rf cri-o
+}
+
+install_isulad() {
+  # export LDFLAGS
+  export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH
+  export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH
+  sudo sh -c "echo '/usr/local/lib' >>/etc/ld.so.conf"
+  CURRENT_PATH=$(
+    cd $(dirname $0)
+    pwd
+  )
+  sudo apt-get install -y g++ libprotobuf-dev protobuf-compiler protobuf-compiler-grpc libgrpc++-dev libgrpc-dev libtool automake autoconf cmake make pkg-config libyajl-dev zlib1g-dev libselinux1-dev libseccomp-dev libcap-dev libsystemd-dev git libarchive-dev libcurl4-gnutls-dev openssl libdevmapper-dev python3 libtar0 libtar-dev libhttp-parser-dev libwebsockets-dev
+  BUILD_DIR=/tmp/build_isulad
+
+  sudo rm -rf $BUILD_DIR
+  sudo mkdir -p $BUILD_DIR
+
+  sudo git config --global --add safe.directory /tmp/build_isulad/lxc/lxc-4.0.3
+  # build lxc
+  cd $BUILD_DIR
+  sudo git clone https://gitee.com/src-openeuler/lxc.git -b openEuler-22.03-LTS-Next
+  cd lxc
+  sudo ./apply-patches
+  cd lxc-4.0.3
+  sudo ./autogen.sh
+  sudo ./configure
+  sudo make CFLAGS="-Wno-error=strict-prototypes -Wno-error=old-style-definition" -j $(nproc)
+  sudo make install CFLAGS="-Wno-error=strict-prototypes -Wno-error=old-style-definition"
+
+  # build lcr
+  cd $BUILD_DIR
+  sudo git clone https://gitee.com/openeuler/lcr.git
+  cd lcr
+  sudo mkdir build
+  cd build
+  sudo cmake ..
+  sudo make -j $(nproc)
+  sudo make install
+
+  # build and install clibcni
+  cd $BUILD_DIR
+  sudo git clone https://gitee.com/openeuler/clibcni.git
+  cd clibcni
+  sudo mkdir build
+  cd build
+  sudo cmake ..
+  sudo make -j $(nproc)
+  sudo make install
+
+  # build and install iSulad
+  cd $BUILD_DIR
+  sudo git clone https://gitee.com/openeuler/iSulad.git
+  cd iSulad
+  sudo mkdir build
+  cd build
+  sudo cmake -DENABLE_CRI_API_V1=ON ..
+  sudo make -j $(nproc)
+  sudo make install
+
+  sudo apt-get install -y jq
+  sudo sed -i 's#/usr/bin/isulad#/usr/local/bin/isulad#g' ../src/contrib/init/isulad.service
+  sudo sed -i 's#-/etc/sysconfig/iSulad#/etc/isulad/daemon.json#g' ../src/contrib/init/isulad.service
+  TMP_FILE=/home/runner/tmp.json
+  ISULAD_CONF_FILE=/etc/isulad/daemon.json
+  sudo cat ${ISULAD_CONF_FILE} | sudo jq '.["websocket-server-listening-port"]=10355' >${TMP_FILE} && sudo mv -f ${TMP_FILE} ${ISULAD_CONF_FILE}
+  sudo cat ${ISULAD_CONF_FILE} | sudo jq '.["cni-bin-dir"]="/opt/cni/bin"' >${TMP_FILE} && sudo mv -f ${TMP_FILE} ${ISULAD_CONF_FILE}
+  sudo cat ${ISULAD_CONF_FILE} | sudo jq '.["cni-conf-dir"]="/etc/cni/net.d"' >${TMP_FILE} && sudo mv -f ${TMP_FILE} ${ISULAD_CONF_FILE}
+  sudo cat ${ISULAD_CONF_FILE} | sudo jq '.["network-plugin"]="cni"' >${TMP_FILE} && sudo mv -f ${TMP_FILE} ${ISULAD_CONF_FILE}
+  sudo cat ${ISULAD_CONF_FILE} | sudo jq '.["enable-cri-v1"]=true' >${TMP_FILE} && sudo mv -f ${TMP_FILE} ${ISULAD_CONF_FILE}
+  sudo cat ${ISULAD_CONF_FILE} | sudo jq '.["pod-sandbox-image"]="kubeedge/pause:3.6"' >${TMP_FILE} && sudo mv -f ${TMP_FILE} ${ISULAD_CONF_FILE}
+  sudo cat ${ISULAD_CONF_FILE} | sudo jq '.["registry-mirrors"]=["docker.io"]' >${TMP_FILE} && sudo mv -f ${TMP_FILE} ${ISULAD_CONF_FILE}
+  sudo cat ${ISULAD_CONF_FILE} | sudo jq '.["insecure-registries"]=["k8s.gcr.io"]' >${TMP_FILE} && sudo mv -f ${TMP_FILE} ${ISULAD_CONF_FILE}
+  sudo cat /etc/isulad/daemon.json
+
+  sudo cp ../src/contrib/init/isulad.service /usr/lib/systemd/system/
+  sudo ldconfig
+  sudo systemctl daemon-reload
+  sudo systemctl enable isulad
+  sudo systemctl restart isulad
+  cd $CURRENT_PATH
+  # clean
+  sudo rm -rf $BUILD_DIR
+  sudo apt autoremove
+}
diff --git a/hack/local-up-kubeedge.sh b/hack/local-up-kubeedge.sh
index d008a082c..8ffe0c9f9 100755
--- a/hack/local-up-kubeedge.sh
+++ b/hack/local-up-kubeedge.sh
@@ -14,22 +14,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-KUBEEDGE_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/..
+KUBEEDGE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/..
 ENABLE_DAEMON=${ENABLE_DAEMON:-false}
 LOG_DIR=${LOG_DIR:-"/tmp"}
 LOG_LEVEL=${LOG_LEVEL:-2}
 TIMEOUT=${TIMEOUT:-60}s
 PROTOCOL=${PROTOCOL:-"WebSocket"}
-CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-"remote"}
+CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-"containerd"}
 KIND_IMAGE=${1:-"kindest/node:v1.27.0"}
 echo -e "The installation of the cni plugin will overwrite the cni config file. Use export CNI_CONF_OVERWRITE=false to disable it."
 
-if [[ "${CLUSTER_NAME}x" == "x" ]];then
-    CLUSTER_NAME="test"
+if [[ "${CLUSTER_NAME}x" == "x" ]]; then
+  CLUSTER_NAME="test"
 fi
 
 export CLUSTER_CONTEXT="--name ${CLUSTER_NAME}"
 
+function install_cr() {
+  if [[ "${CONTAINER_RUNTIME}" = "docker" ]]; then
+    install_docker
+  elif [[ "${CONTAINER_RUNTIME}" = "cri-o" ]]; then
+    install_crio
+  elif [[ "${CONTAINER_RUNTIME}" = "isulad" ]]; then
+    install_isulad
+  fi
+}
+
 function check_prerequisites {
   kubeedge::golang::verify_golang_version
   check_kubectl
@@ -37,7 +47,14 @@ function check_prerequisites {
   if [[ "${CONTAINER_RUNTIME}" = "docker" ]]; then
     # if we will use docker as edgecore container runtime, we need to verify whether docker already installed
     verify_docker_installed
-  elif [[ "${CONTAINER_RUNTIME}" = "remote" ]]; then
+    verify_cridockerd_installed
+  elif [[ "${CONTAINER_RUNTIME}" = "cri-o" ]]; then
+    # function to verify if cri-o is installed
+    verify_crio_installed
+  elif [[ "${CONTAINER_RUNTIME}" = "isulad" ]]; then
+    # function to verify if isulad is installed
+    verify_isulad_installed
+  elif [[ "${CONTAINER_RUNTIME}" = "containerd" ]]; then
     # we will use containerd as cri runtime, so need to verify whether containerd already installed
     verify_containerd_installed
   else
@@ -60,7 +77,7 @@ function uninstall_kubeedge {
   [[ -n "${EDGECORE_PID-}" ]] && sudo kill "${EDGECORE_PID}" 2>/dev/null
 
   # delete data
-  rm -rf /tmp/etc/kubeedge /tmp/var/lib/kubeedge
+  sudo rm -rf /tmp/etc/kubeedge /tmp/var/lib/kubeedge
 
 }
 
@@ -123,7 +140,7 @@ function build_edgecore {
 function start_cloudcore {
   CLOUD_CONFIGFILE=${KUBEEDGE_ROOT}/_output/local/bin/cloudcore.yaml
   CLOUD_BIN=${KUBEEDGE_ROOT}/_output/local/bin/cloudcore
-  ${CLOUD_BIN} --defaultconfig >  ${CLOUD_CONFIGFILE}
+  ${CLOUD_BIN} --defaultconfig >${CLOUD_CONFIGFILE}
   sed -i '/cloudStream:/{n;s/false/true/;}' ${CLOUD_CONFIGFILE}
   if [[ "${PROTOCOL}" = "QUIC" ]]; then
     sed -i '/quic:/{n;N;s/false/true/;}' ${CLOUD_CONFIGFILE}
@@ -142,20 +159,20 @@ function start_cloudcore {
     -e '/router:/{n;N;N;N;N;s/false/true/}' ${CLOUD_CONFIGFILE}
   CLOUDCORE_LOG=${LOG_DIR}/cloudcore.log
   echo "start cloudcore..."
-  nohup sudo ${CLOUD_BIN} --config=${CLOUD_CONFIGFILE} --v=${LOG_LEVEL} > "${CLOUDCORE_LOG}" 2>&1 &
+  nohup sudo ${CLOUD_BIN} --config=${CLOUD_CONFIGFILE} --v=${LOG_LEVEL} >"${CLOUDCORE_LOG}" 2>&1 &
   CLOUDCORE_PID=$!
 
   # ensure tokensecret is generated
   while true; do
-      sleep 3
-      kubectl get secret -nkubeedge| grep -q tokensecret && break
+    sleep 3
+    kubectl get secret -nkubeedge | grep -q tokensecret && break
   done
 }
 
 function start_edgecore {
   EDGE_CONFIGFILE=${KUBEEDGE_ROOT}/_output/local/bin/edgecore.yaml
   EDGE_BIN=${KUBEEDGE_ROOT}/_output/local/bin/edgecore
-  ${EDGE_BIN} --defaultconfig >  ${EDGE_CONFIGFILE}
+  ${EDGE_BIN} --defaultconfig >${EDGE_CONFIGFILE}
 
   sed -i '/edgeStream:/{n;s/false/true/;}' ${EDGE_CONFIGFILE}
   sed -i '/metaServer:/{n;s/false/true/;}' ${EDGE_CONFIGFILE}
@@ -168,27 +185,36 @@ function start_edgecore {
   # if we will use docker as edgecore container runtime
   # we need to change edgecore container runtime from default containerd to docker
   if [[ "${CONTAINER_RUNTIME}" = "docker" ]]; then
-    sed -i 's|containerRuntime: .*|containerRuntime: docker|' ${EDGE_CONFIGFILE}
-    sed -i 's|remoteImageEndpoint: .*|remoteImageEndpoint: unix:///var/run/dockershim.sock|' ${EDGE_CONFIGFILE}
-    sed -i 's|remoteRuntimeEndpoint: .*|remoteRuntimeEndpoint: unix:///var/run/dockershim.sock|' ${EDGE_CONFIGFILE}
+    sed -i 's|imageServiceEndpoint: .*|imageServiceEndpoint: unix:///var/run/cri-dockerd.sock|' ${EDGE_CONFIGFILE}
+    sed -i 's|containerRuntimeEndpoint: .*|containerRuntimeEndpoint: unix:///var/run/cri-dockerd.sock|' ${EDGE_CONFIGFILE}
   fi
 
-  token=`kubectl get secret -nkubeedge tokensecret -o=jsonpath='{.data.tokendata}' | base64 -d`
+  if [[ "${CONTAINER_RUNTIME}" = "cri-o" ]]; then
+    sed -i 's|imageServiceEndpoint: .*|imageServiceEndpoint: unix:///var/run/crio/crio.sock|' ${EDGE_CONFIGFILE}
+    sed -i 's|containerRuntimeEndpoint: .*|containerRuntimeEndpoint: unix:///var/run/crio/crio.sock|' ${EDGE_CONFIGFILE}
+    sed -i 's|cgroupDriver: .*|cgroupDriver: systemd|' ${EDGE_CONFIGFILE}
+  fi
+
+  if [[ "${CONTAINER_RUNTIME}" = "isulad" ]]; then
+    sed -i 's|imageServiceEndpoint: .*|imageServiceEndpoint: unix:///var/run/isulad.sock|' ${EDGE_CONFIGFILE}
+    sed -i 's|containerRuntimeEndpoint: .*|containerRuntimeEndpoint: unix:///var/run/isulad.sock|' ${EDGE_CONFIGFILE}
+  fi
+
+  token=$(kubectl get secret -nkubeedge tokensecret -o=jsonpath='{.data.tokendata}' | base64 -d)
 
   sed -i -e "s|token: .*|token: ${token}|g" \
-      -e "s|hostnameOverride: .*|hostnameOverride: edge-node|g" \
-      -e "s|/etc/|/tmp/etc/|g" \
-      -e "s|/var/lib/kubeedge/|/tmp&|g" \
-      -e "s|mqttMode: .*|mqttMode: 0|g" \
-      -e '/serviceBus:/{n;s/false/true/;}' ${EDGE_CONFIGFILE}
+    -e "s|hostnameOverride: .*|hostnameOverride: edge-node|g" \
+    -e "s|/etc/|/tmp/etc/|g" \
+    -e "s|/var/lib/kubeedge/|/tmp&|g" \
+    -e "s|mqttMode: .*|mqttMode: 0|g" \
+    -e '/serviceBus:/{n;s/false/true/;}' ${EDGE_CONFIGFILE}
 
   sed -i -e "s|/tmp/etc/resolv|/etc/resolv|g" ${EDGE_CONFIGFILE}
-
   EDGECORE_LOG=${LOG_DIR}/edgecore.log
 
   echo "start edgecore..."
   export CHECK_EDGECORE_ENVIRONMENT="false"
-  nohup sudo -E ${EDGE_BIN} --config=${EDGE_CONFIGFILE} --v=${LOG_LEVEL} > "${EDGECORE_LOG}" 2>&1 &
+  nohup sudo -E ${EDGE_BIN} --config=${EDGE_CONFIGFILE} --v=${LOG_LEVEL} >"${EDGECORE_LOG}" 2>&1 &
   EDGECORE_PID=$!
 }
 
@@ -220,14 +246,14 @@ function generate_streamserver_cert {
   K8SCA_KEY_FILE=/tmp/etc/kubernetes/pki/ca.key
   streamsubject=${SUBJECT:-/C=CN/ST=Zhejiang/L=Hangzhou/O=KubeEdge}
 
-  if [[ ! -d /tmp/etc/kubernetes/pki ]] ; then
+  if [[ ! -d /tmp/etc/kubernetes/pki ]]; then
     mkdir -p /tmp/etc/kubernetes/pki
   fi
-  if [[ ! -d $CA_PATH ]] ; then
-	mkdir -p $CA_PATH
+  if [[ ! -d $CA_PATH ]]; then
+    mkdir -p $CA_PATH
   fi
-  if [[ ! -d $CERT_PATH ]] ; then
-	mkdir -p $CERT_PATH
+  if [[ ! -d $CERT_PATH ]]; then
+    mkdir -p $CERT_PATH
   fi
 
   docker cp ${CLUSTER_NAME}-control-plane:/etc/kubernetes/pki/ca.crt $K8SCA_FILE
@@ -235,11 +261,11 @@ function generate_streamserver_cert {
   cp /tmp/etc/kubernetes/pki/ca.crt /tmp/etc/kubeedge/ca/streamCA.crt
 
   SUBJECTALTNAME="subjectAltName = IP.1:127.0.0.1"
-  echo $SUBJECTALTNAME > /tmp/server-extfile.cnf
+  echo $SUBJECTALTNAME >/tmp/server-extfile.cnf
 
   touch ~/.rnd
 
-  openssl genrsa -out ${STREAM_KEY_FILE}  2048
+  openssl genrsa -out ${STREAM_KEY_FILE} 2048
   openssl req -new -key ${STREAM_KEY_FILE} -subj ${streamsubject} -out ${STREAM_CSR_FILE}
   openssl x509 -req -in ${STREAM_CSR_FILE} -CA ${K8SCA_FILE} -CAkey ${K8SCA_KEY_FILE} -CAcreateserial -out ${STREAM_CRT_FILE} -days 5000 -sha256 -extfile /tmp/server-extfile.cnf
 }
@@ -249,6 +275,8 @@ cleanup
 source "${KUBEEDGE_ROOT}/hack/lib/golang.sh"
 source "${KUBEEDGE_ROOT}/hack/lib/install.sh"
 
+install_cr
+
 check_prerequisites
 
 # Stop right away if there's an error
@@ -278,9 +306,20 @@ generate_streamserver_cert
 start_cloudcore
 
 # install CNI plugins
-if [[ "${CONTAINER_RUNTIME}" = "remote" ]]; then
-# we need to install CNI plugins only when we use remote(containerd) as edgecore container runtime
-install_cni_plugins
+if [[ "${CONTAINER_RUNTIME}" = "containerd" || "${CONTAINER_RUNTIME}" = "docker" || "${CONTAINER_RUNTIME}" = "isulad" ]]; then
+  # we need to install CNI plugins only when we use remote(containerd) as edgecore container runtime
+  install_cni_plugins
+  if [[ "${CONTAINER_RUNTIME}" = "docker" ]]; then
+    sudo systemctl restart docker
+    sudo systemctl restart cri-docker
+    sleep 2
+  elif [[ "${CONTAINER_RUNTIME}" = "containerd" ]]; then
+    sudo systemctl restart containerd
+    sleep 2
+  elif [[ "${CONTAINER_RUNTIME}" = "isulad" ]]; then
+    sudo systemctl restart isulad
+    sleep 2
+  fi
 fi
 
 sleep 2
@@ -288,9 +327,9 @@ sleep 2
 start_edgecore
 
 if [[ "${ENABLE_DAEMON}" = false ]]; then
-    echo "Local KubeEdge cluster is running. Press Ctrl-C to shut it down."
+  echo "Local KubeEdge cluster is running. Press Ctrl-C to shut it down."
 else
-    echo "Local KubeEdge cluster is running. Use \"kill $BASHPID\" to shut it down."
+  echo "Local KubeEdge cluster is running. Use \"kill $BASHPID\" to shut it down."
 fi
 
 echo "Logs:
@@ -305,11 +344,14 @@ To start using your kubeedge, you can run:
 "
 
 if [[ "${ENABLE_DAEMON}" = false ]]; then
-  while true; do sleep 1; healthcheck; done
+  while true; do
+    sleep 1
+    healthcheck
+  done
 else
-    while true; do
-        sleep 3
-        kubectl get nodes | grep edge-node | grep -q -w Ready && break
-    done
-    kubectl label node edge-node disktype=test
+  while true; do
+    sleep 3
+    kubectl get nodes | grep edge-node | grep -q -w Ready && break
+  done
+  kubectl label node edge-node disktype=test
 fi