add edgemark performance testing tool

Signed-off-by: wackxu <xushiwei5@huawei.com>

11 files changed, 386 insertions, 7 deletions

diff --git a/Makefile b/Makefile
index 38d8654a8..5ffe8c3ea 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,8 @@ BINARIES=cloudcore \
 	edgesite-server \
 	keadm \
 	csidriver \
-	iptablesmanager
+	iptablesmanager \
+	edgemark
 
 COMPONENTS=cloud \
 	edge
@@ -46,7 +47,7 @@ ifeq ($(HELP),y)
 all: clean
 	@echo "$$ALL_HELP_INFO"
 else
-all: 
+all:
 	KUBEEDGE_OUTPUT_SUBPATH=$(OUT_DIR) hack/make-rules/build.sh $(WHAT)
 endif
 
@@ -174,7 +175,7 @@ ifeq ($(HELP),y)
 crossbuild:
 	@echo "$$CROSSBUILD_HELP_INFO"
 else
-crossbuild: 
+crossbuild:
 	hack/make-rules/crossbuild.sh $(WHAT) $(ARM_VERSION)
 endif
 
@@ -194,7 +195,7 @@ define GENERATE_CRDS_HELP_INFO
 #     RELIABLESYNCS_VERSION, default: v1alpha1
 #
 # Example:
-#     make generate 
+#     make generate
 #     make generate -e CRD_VERSIONS=v1 -e CRD_OUTPUTS=build/crds
 #
 endef
@@ -227,7 +228,7 @@ ifeq ($(HELP),y)
 smallbuild:
 	@echo "$$SMALLBUILD_HELP_INFO"
 else
-smallbuild: 
+smallbuild:
 	hack/make-rules/smallbuild.sh $(WHAT)
 endif
 
diff --git a/build/edgemark/Dockerfile b/build/edgemark/Dockerfile
new file mode 100644
index 000000000..8f4b85884
--- /dev/null
+++ b/build/edgemark/Dockerfile
@@ -0,0 +1,19 @@
+ARG BUILD_FROM=golang:1.16-alpine3.13

+

+FROM ${BUILD_FROM} AS builder

+

+ARG GO_LDFLAGS

+

+COPY . /go/src/github.com/kubeedge/kubeedge

+

+RUN apk --no-cache update && \

+apk --no-cache upgrade && \

+apk --no-cache add build-base linux-headers sqlite-dev binutils-gold && \

+CGO_ENABLED=1 GO111MODULE=off go build -v -o /usr/local/bin/edgemark -ldflags="${GO_LDFLAGS} -w -s -extldflags -static" \

+github.com/kubeedge/kubeedge/edge/cmd/edgemark

+

+FROM alpine:3.13

+

+COPY --from=builder /usr/local/bin/edgemark /usr/local/bin/edgemark

+

+ENTRYPOINT ["edgemark"]
\ No newline at end of file
diff --git a/build/edgemark/README.md b/build/edgemark/README.md
new file mode 100644
index 000000000..d74d5becf
--- /dev/null
+++ b/build/edgemark/README.md
@@ -0,0 +1,48 @@
+# Edgemark User Guide
+
+## Introduction
+
+Edgemark is a performance testing tool that inspired by
+[Kubemark](https://github.com/kubernetes/kubernetes/tree/master/cmd/kubemark)
+which allows users to run experiments on
+simulated clusters. The primary use case is scalability testing, as simulated
+clusters can be much bigger than the real ones. The objective is to expose
+problems with the KubeEdge cloud components CloudCore that appear only on bigger clusters.
+
+This document serves as a primer to understand what Edgemark is, what it is not,
+and how to use it.
+
+## Architecture
+
+On a very high level, Edgemark cluster consists of three parts: a real kubernetes master,
+KubeEdge CloudCore component and a set of “Hollow” Edge Nodes. Hollow Edge Node is registered
+with `HollowEdgeCore`, which pretends to be an ordinary EdgeCore, but does not create any real containers.
+`HollowEdgeCore` mocks runtime manager with Kubernetes `k8s.io/kubernetes/pkg/kubelet/cri/remote/fake/fake_runtime.go`,
+where most logic sits. Except for simulating runtime manager, other behaviors is the same as edgecore.
+
+![img](images/edgemark.jpg)
+
+Currently, Kubernetes master components run on a dedicated machine as pods that are
+created/managed by kubelet, which itself runs as either a systemd or a supervisord
+service on the master VM depending on the VM distro. Having a dedicated machine for the master
+has a slight advantage over running the master components on an external cluster,
+which is being able to completely isolate master resources from everything else.
+The CloudCore has multiple instances and requires multiple dedicated nodes to maintain high reliability.
+Usually In highly available configurations, a load balancer must sit in front of the CloudCore
+to correctly route requests to healthy CloudCore servers. And The HollowEdgeNodes
+on the other hand are run on an ‘external’ Kubernetes cluster
+as pods in an isolated namespace (named edgemark). This idea of using pods on a
+real cluster behave (or act) as nodes on the edgemark cluster lies at the heart of
+edgemark's design.
+
+## Requirements
+
+To run Edgemark, you need:
+
+1. A Kubernetes cluster (called `external cluster`) for running all your HollowEdgeNodes
+2. A Kubernetes cluster (called `edgemark cluster`)  the master for HollowEdgeNodes.
+3. Several dedicated nodes in `edgemark cluster` for deploy CloudCore, a load balancer
+   that expose CloudCore service for HollowEdgeNodes and the load balancer has to be directly
+   routable from HollowEdgeNodes.
+4. You also need access to a Docker repository that has the
+   container images for CloudCore, hollow-edge-node and node-problem-detector.
diff --git a/build/edgemark/edgemark_setup_guide.md b/build/edgemark/edgemark_setup_guide.md
new file mode 100644
index 000000000..fd0a48885
--- /dev/null
+++ b/build/edgemark/edgemark_setup_guide.md
@@ -0,0 +1,97 @@
+## Introduction
+This document serves to understand how to set up edgemark cluster given
+that a base cluster (to run hollow-edge-node pods) and
+separate master (to act as master for the hollow edge nodes) are already present.
+
+## Precondition
+You need edgemark master and external cluster to set up a edgemark cluster.
+
+The functions are as follows:
+
+- edgemark master: can be StandAlone or HA, used to be the edgemark cluster's master
+- external cluster: used to create hollow edge nodes for the edgemark cluster
+
+## Steps:
+1. Deploy CloudCore in edgemark cluster, refer to [CloudCore deploy](https://kubeedge.io/en/docs/setup/ha/) for more details.
+
+2. Build edgemark image
+
+If you want to build/use your own edgemark image, do as follows.
+
+- i. pull kubeedge code
+
+```
+cd $GOPATH/src/github.com/kubeedge
+git clone git@github.com:kubeedge/kubeedge.git
+```
+
+- ii. build edgemark image
+
+```
+cd $GOPATH/src/github.com/kubeedge/kubeedge
+make image WHAT=edgemark
+```
+
+Then you can get the image named `kubeedge/edgemark:{tag}` locally.
+
+3. Create hollow nodes in external cluster
+
+- i. create namespace and secret
+
+Copy edgemark master's `tokensecret` which is generated by CloudCore which is used to access CloudCore for edge node,
+and create it in namespace that will deploy hollow edge node in external cluster.
+
+first, save `tokensecret` in file from edgemark cluster,
+
+```
+kubectl get secret -nkubeedge tokensecret -oyaml > tokensecret.yaml
+```
+
+modify namespace in `tokensecret.yaml`
+```
+sed -i "s|namespace: .*|namespace: {ns}|g" tokensecret.yaml
+```
+
+create ns and tokensecret in external cluster
+
+```
+kubectl create ns edgemark
+
+kubectl create -f tokensecret.yaml
+```
+
+- ii. apply yaml to create hollow nodes
+
+You can use `hollow-edge-node_template.yaml` in the current directory.
+
+Note:
+
+- the parameters `{{numreplicas}}` means the number of hollow nodes in the edgemark cluster
+- the parameters `{{server}}` means the server address exposed for hollow nodes join in
+- the parameters `{{server}}`, `{{numreplicas}}`, `{{edgemark_image_registry}}` and `{{edgemark_image_tag}}` need to be filled in the template
+- your external cluster should have enough resources to be able to run `{{numreplicas}}` no. of hollow-node pods
+
+```
+kubectl create -f hollow-edge-node_template.yaml
+```
+
+Waiting for these hollow-node pods to be running. Then you can see these pods register as edgemark master's nodes.
+
+Finally, edgemark master and external cluster set up the edgemark cluster.
+
+
+4. Run performance testing with ClusterLoader2
+
+After set up the edgemark cluster, we can do our performance testing with ClusterLoader2.
+[ClusterLoader2](https://github.com/kubernetes/perf-tests/tree/master/clusterloader2) is
+an official K8s scalability and performance testing framework.
+refer to [ClusterLoader2 Getting started](https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/docs/GETTING_STARTED.md) for more details.
+we just need config `--provider=kubemark` when we run performance test, for example:
+
+```
+./clusterloader  --testconfig=config.yaml --provider=kubemark --kubeconfig=${HOME}/.kube/config --v=2
+```
+
+
+
+
diff --git a/build/edgemark/hollow_edge_node_template.yaml b/build/edgemark/hollow_edge_node_template.yaml
new file mode 100644
index 000000000..aa677ad94
--- /dev/null
+++ b/build/edgemark/hollow_edge_node_template.yaml
@@ -0,0 +1,50 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: hollow-edge-node
+spec:
+  replicas: {{numreplicas}}
+  selector:
+    matchLabels:
+      app: hollow-edge-node
+  template:
+    metadata:
+      labels:
+        app: hollow-edge-node
+    spec:
+      containers:
+        - name: hollow-edgecore
+          image: {{edgemark_image_registry}}/edgemark:{{edgemark_image_tag}}
+          command:
+            - edgemark
+          args:
+            - --token=$(TOKEN)
+            - --name=$(NODE_NAME)
+            - --http-server=https://{{server}}:10002
+            - --websocket-server={{server}}:10000
+            - --alsologtostderr
+            - --v=2
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
+            - name: TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: tokensecret
+                  key: tokendata
+          resources:
+            requests:
+              cpu: 20m
+              memory: 50M
+          securityContext:
+            privileged: true
+      tolerations:
+        - effect: NoExecute
+          key: node.kubernetes.io/unreachable
+          operator: Exists
+        - effect: NoExecute
+          key: node.kubernetes.io/not-ready
+          operator: Exists
diff --git a/build/edgemark/images/edgemark.jpg b/build/edgemark/images/edgemark.jpg
new file mode 100644
index 000000000..2dfe3dadf
--- /dev/null
+++ b/build/edgemark/images/edgemark.jpg
diff --git a/edge/cmd/edgemark/hollow_edgecore.go b/edge/cmd/edgemark/hollow_edgecore.go
new file mode 100644
index 000000000..c76ff63f8
--- /dev/null
+++ b/edge/cmd/edgemark/hollow_edgecore.go
@@ -0,0 +1,153 @@
+/*
+Copyright 2022 The KubeEdge Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	cliflag "k8s.io/component-base/cli/flag"
+	internalapi "k8s.io/cri-api/pkg/apis"
+	"k8s.io/kubernetes/pkg/kubelet/cri/remote"
+	fakeremote "k8s.io/kubernetes/pkg/kubelet/cri/remote/fake"
+
+	"github.com/kubeedge/beehive/pkg/core"
+	"github.com/kubeedge/kubeedge/edge/pkg/common/dbm"
+	"github.com/kubeedge/kubeedge/edge/pkg/edged"
+	"github.com/kubeedge/kubeedge/edge/pkg/edgehub"
+	"github.com/kubeedge/kubeedge/edge/pkg/metamanager"
+	"github.com/kubeedge/kubeedge/pkg/apis/componentconfig/edgecore/v1alpha1"
+	"github.com/kubeedge/kubeedge/pkg/version/verflag"
+)
+
+type hollowEdgeNodeConfig struct {
+	Token           string
+	NodeName        string
+	HTTPServer      string
+	WebsocketServer string
+	NodeLabels      map[string]string
+}
+
+func main() {
+	command := newHollowEdgeNodeCommand()
+	if err := command.Execute(); err != nil {
+		os.Exit(1)
+	}
+}
+
+// newHollowEdgeNodeCommand creates a *cobra.Command object with default parameters
+func newHollowEdgeNodeCommand() *cobra.Command {
+	s := &hollowEdgeNodeConfig{
+		NodeLabels: make(map[string]string),
+	}
+
+	cmd := &cobra.Command{
+		Use:  "edgemark",
+		Long: "edgemark",
+		Run: func(cmd *cobra.Command, args []string) {
+			verflag.PrintAndExitIfRequested()
+			run(s)
+		},
+		Args: func(cmd *cobra.Command, args []string) error {
+			for _, arg := range args {
+				if len(arg) > 0 {
+					return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args)
+				}
+			}
+			return nil
+		},
+	}
+
+	fs := cmd.Flags()
+	fs.AddGoFlagSet(flag.CommandLine) // for flags like --docker-only
+	s.addFlags(fs)
+
+	return cmd
+}
+
+func run(config *hollowEdgeNodeConfig) {
+	c := EdgeCoreConfig(config)
+
+	// use fake runtime service
+	edged.DefaultGetRuntimeService = GetFakeRuntimeAndImageServices
+
+	edged.Register(c.Modules.Edged)
+	edgehub.Register(c.Modules.EdgeHub, c.Modules.Edged.HostnameOverride)
+	metamanager.Register(c.Modules.MetaManager)
+
+	dbm.InitDBConfig(c.DataBase.DriverName, c.DataBase.AliasName, c.DataBase.DataSource)
+
+	// start all modules
+	core.Run()
+}
+
+func (c *hollowEdgeNodeConfig) addFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&c.Token, "token", "", "Token indicates the priority of joining the cluster for the edge.")
+	fs.StringVar(&c.NodeName, "name", "fake-node", "Name of this Hollow Node.")
+	fs.StringVar(&c.WebsocketServer, "websocket-server", "", "Server indicates websocket server address.")
+	fs.StringVar(&c.HTTPServer, "http-server", "", "HTTPServer indicates the server for edge to apply for the certificate.")
+	bindableNodeLabels := cliflag.ConfigurationMap(c.NodeLabels)
+	fs.Var(&bindableNodeLabels, "node-labels", "Additional node labels")
+}
+
+func EdgeCoreConfig(config *hollowEdgeNodeConfig) *v1alpha1.EdgeCoreConfig {
+	edgeCoreConfig := v1alpha1.NewDefaultEdgeCoreConfig()
+
+	// overWrite config
+	edgeCoreConfig.DataBase.DataSource = "/edgecore.db"
+	edgeCoreConfig.Modules.EdgeHub.Token = config.Token
+	edgeCoreConfig.Modules.EdgeHub.HTTPServer = config.HTTPServer
+	edgeCoreConfig.Modules.EdgeHub.WebSocket.Server = config.WebsocketServer
+
+	// use fake runtime for test
+	edgeCoreConfig.Modules.Edged.RuntimeType = "fake"
+	edgeCoreConfig.Modules.Edged.RemoteRuntimeEndpoint = "/run/fake/fake.sock"
+	edgeCoreConfig.Modules.Edged.RemoteImageEndpoint = "/run/fake/fake.sock"
+	edgeCoreConfig.Modules.Edged.EnableMetrics = false
+
+	edgeCoreConfig.Modules.Edged.HostnameOverride = config.NodeName
+	edgeCoreConfig.Modules.Edged.Labels = config.NodeLabels
+
+	return edgeCoreConfig
+}
+
+func GetFakeRuntimeAndImageServices(
+	remoteRuntimeEndpoint,
+	remoteImageEndpoint string,
+	runtimeRequestTimeout metav1.Duration) (internalapi.RuntimeService, internalapi.ImageManagerService, error) {
+	endpoint, err := fakeremote.GenerateEndpoint()
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to generate fake endpoint %v", err)
+	}
+
+	fakeRemoteRuntime := fakeremote.NewFakeRemoteRuntime()
+	if err = fakeRemoteRuntime.Start(endpoint); err != nil {
+		return nil, nil, fmt.Errorf("failed to start fake runtime %v", err)
+	}
+
+	runtimeService, err := remote.NewRemoteRuntimeService(endpoint, 15*time.Second)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to init runtime service %v", err)
+	}
+
+	return runtimeService, fakeRemoteRuntime.ImageService, err
+}
diff --git a/edge/pkg/edged/edged.go b/edge/pkg/edged/edged.go
index 974674645..6039c1b8f 100644
--- a/edge/pkg/edged/edged.go
+++ b/edge/pkg/edged/edged.go
@@ -163,6 +163,13 @@ const (
 	ResolvConfDefault = "/etc/resolv.conf"
 )
 
+type GetRuntimeServiceFunc func(
+	remoteRuntimeEndpoint,
+	remoteImageEndpoint string,
+	runtimeRequestTimeout metav1.Duration) (internalapi.RuntimeService, internalapi.ImageManagerService, error)
+
+var DefaultGetRuntimeService GetRuntimeServiceFunc = getRuntimeAndImageServices
+
 // podReady holds the initPodReady flag and its lock
 type podReady struct {
 	// initPodReady is flag to check Pod ready status
@@ -509,7 +516,7 @@ func newEdged(enable bool) (*edged, error) {
 		ResolvConfDefault)
 
 	httpClient := &http.Client{}
-	runtimeService, imageService, err := getRuntimeAndImageServices(
+	runtimeService, imageService, err := DefaultGetRuntimeService(
 		edgedconfig.Config.RemoteRuntimeEndpoint,
 		edgedconfig.Config.RemoteImageEndpoint,
 		metav1.Duration{
@@ -518,6 +525,7 @@ func newEdged(enable bool) (*edged, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	if ed.os == nil {
 		ed.os = kubecontainer.RealOS{}
 	}
diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh
index eaee5d42b..fe9288506 100755
--- a/hack/lib/golang.sh
+++ b/hack/lib/golang.sh
@@ -177,6 +177,7 @@ ALL_BINARIES_AND_TARGETS=(
   edgesite-server:edgesite/cmd/edgesite-server
   csidriver:cloud/cmd/csidriver
   iptablesmanager:cloud/cmd/iptablesmanager
+  edgemark:edge/cmd/edgemark
 )
 
 kubeedge::golang::get_target_by_binary() {
diff --git a/hack/make-rules/crossbuildimage.sh b/hack/make-rules/crossbuildimage.sh
index 67f8d4e25..5d65b48b2 100755
--- a/hack/make-rules/crossbuildimage.sh
+++ b/hack/make-rules/crossbuildimage.sh
@@ -34,6 +34,7 @@ ALL_IMAGES_AND_TARGETS=(
   edgesite-server:edgesite-server:build/edgesite/server-build.Dockerfile
   csidriver:csidriver:build/csidriver/Dockerfile
   iptablesmanager:iptables-manager:build/iptablesmanager/Dockerfile
+  edgemark:edgemark:build/edgemark/Dockerfile
 )
 
 function get_imagename_by_target() {
@@ -82,7 +83,7 @@ function build_multi_arch_images() {
     DOCKERFILE_PATH="$(get_dockerfile_by_target ${arg})"
 
     set -x
-    
+
     # If there's any issues when using buildx, can refer to the issue below
     # https://github.com/docker/buildx/issues/495
     # https://github.com/multiarch/qemu-user-static/issues/100
diff --git a/hack/make-rules/image.sh b/hack/make-rules/image.sh
index 49642be8b..f2067deed 100755
--- a/hack/make-rules/image.sh
+++ b/hack/make-rules/image.sh
@@ -31,6 +31,7 @@ ALL_IMAGES_AND_TARGETS=(
   edgesite-server:edgesite-server:build/edgesite/server-build.Dockerfile
   csidriver:csidriver:build/csidriver/Dockerfile
   iptablesmanager:iptables-manager:build/iptablesmanager/Dockerfile
+  edgemark:edgemark:build/edgemark/Dockerfile
 )
 
 GO_LDFLAGS="$(${KUBEEDGE_ROOT}/hack/make-rules/version.sh)"