add k8s deployment for edge (#367)

17 files changed, 255 insertions, 39 deletions

diff --git a/README.md b/README.md
index 3f09fa60c..b5fb20232 100644
--- a/README.md
+++ b/README.md
@@ -162,6 +162,8 @@ cd $GOPATH/src/github.com/kubeedge/kubeedge
 
 ### Run Edge
 
+#### [Run as Kubernetes deployment](./build/edge/kubernetes/README.md)
+
 #### Deploy the Edge node
 We have provided a sample node.json to add a node in kubernetes. Please make sure edge-node is added in kubernetes. Run below steps to add edge-node.
 
diff --git a/README_zh.md b/README_zh.md
index 947a03fa9..0741dcd13 100644
--- a/README_zh.md
+++ b/README_zh.md
@@ -137,6 +137,8 @@ cd $GOPATH/src/github.com/kubeedge/kubeedge
 
 ### 运行 Edge
 
+#### [以 k8s deployment 方式运行](./build/edge/kubernetes/README_zh.md)
+
 #### 部署 Edge node
 我们提供了一个示例 node.json 来在 Kubernetes 中添加一个节点。
 请确保在 Kubernetes 中添加了边缘节点 edge-node。运行以下步骤以添加边缘节点 edge-node。
diff --git a/build/edge/Dockerfile b/build/edge/Dockerfile
index 5c2cc83ff..83a695cb3 100644
--- a/build/edge/Dockerfile
+++ b/build/edge/Dockerfile
@@ -1,5 +1,5 @@
 ARG BUILD_FROM=golang:1.12-alpine3.9
-ARG RUN_FROM=alpine:3.9
+ARG RUN_FROM=docker
 
 FROM ${BUILD_FROM} AS builder
 
@@ -22,7 +22,7 @@ COPY --from=builder /usr/bin/qemu* /usr/bin/
 ENV GOARCHAIUS_CONFIG_PATH /etc/kubeedge/edge
 ENV database.source /var/lib/kubeedge/edge.db
 
-VOLUME ["/etc/kubeedge/edge/certs", "/var/lib/edged", "/var/lib/kubeedge", "/var/run/docker.sock"]
+VOLUME ["/etc/kubeedge/certs", "/var/lib/edged", "/var/lib/kubeedge", "/var/run/docker.sock"]
 
 COPY --from=builder /usr/local/bin/edge_core /usr/local/bin/edge_core
 COPY --from=builder /go/src/github.com/kubeedge/kubeedge/edge/conf /etc/kubeedge/edge/conf
diff --git a/build/edge/README.md b/build/edge/README.md
index b66c31aa5..fc532ff06 100644
--- a/build/edge/README.md
+++ b/build/edge/README.md
@@ -14,27 +14,27 @@ container and MQTT Broker, so make sure that docker engine listening on
 
   The following parameters do not need to be set if they are not modified.
 
-  | name            | default                           | note                      |
-  | --------------- | --------------------------------- | ------------------------- |
-  | cloudhub        | 0.0.0.0:10000                     |                           |
-  | edgename        | edge-node                         |                           |
-  | edge_core_image | kubeedge/edgecore:latest          |                           |
-  | arch            | amd64                             | Optional: amd64 \|arm64v8 |
-  | qemu_arch       | x86_64                            | Optional: x86_64 \| aarch |
-  | certpath        | /etc/kubeedge/edge/certs          |                           |
-  | certfile        | /etc/kubeedge/edge/certs/edge.crt |                           |
-  | keyfile         | /etc/kubeedge/edge/certs/edge.key |                           |
+  | name            | default                           | note                       |
+  | --------------- | --------------------------------- | -------------------------- |
+  | cloudhub        | 0.0.0.0:10000                     |                            |
+  | edgename        | edge-node                         |                            |
+  | edge_core_image | kubeedge/edgecore:latest          |                            |
+  | arch            | amd64                             | Optional: amd64 \| arm64v8 |
+  | qemu_arch       | x86_64                            | Optional: x86_64 \| aarch  |
+  | certpath        | /etc/kubeedge/certs               |                            |
+  | certfile        | /etc/kubeedge/certs/edge.crt      |                            |
+  | keyfile         | /etc/kubeedge/certs/edge.key      |                            |
 
   ```shell
   ./run_daemon.sh set \
-  		    cloudhub=0.0.0.0:10000 \
-          edgename=edgeNode \
+          cloudhub=0.0.0.0:10000 \
+          edgename=edge-node \
           edge_core_image="kubeedge/edgecore:latest" \
           arch=amd64 \
           qemu_arch=x86_64 \
-          certpath=/etc/kubeedge/edge/certs \
-          certfile=/etc/kubeedge/edge/certs/edge.crt \
-          keyfile=/etc/kubeedge/edge/certs/edge.key
+          certpath=/etc/kubeedge/certs \
+          certfile=/etc/kubeedge/certs/edge.crt \
+          keyfile=/etc/kubeedge/certs/edge.key
   ```
 
 + Build image
diff --git a/build/edge/README_zh.md b/build/edge/README_zh.md
index 62e9cde87..1bfcd2605 100644
--- a/build/edge/README_zh.md
+++ b/build/edge/README_zh.md
@@ -20,46 +20,46 @@
   | edge_core_image | kubeedge/edgecore:latest          |                          |
   | arch            | amd64                             | 可选值：amd64 \| arm64v8 |
   | qemu_arch       | x86_64                            | 可选值：x86_64 \| aarch  |
-  | certpath        | /etc/kubeedge/edge/certs          |                          |
-  | certfile        | /etc/kubeedge/edge/certs/edge.crt |                          |
-  | keyfile         | /etc/kubeedge/edge/certs/edge.key |                          |
+  | certpath        | /etc/kubeedge/certs               |                          |
+  | certfile        | /etc/kubeedge/certs/edge.crt      |                          |
+  | keyfile         | /etc/kubeedge/certs/edge.key      |                          |
 
   ```shell
-  ./build/edge/run_daemon.sh set \
-  		    cloudhub=0.0.0.0:10000 \
-          edgename=edgeNode \
+  ./run_daemon.sh set \
+          cloudhub=0.0.0.0:10000 \
+          edgename=edge-node \
           edge_core_image="kubeedge/edgecore:latest" \
           arch=amd64 \
           qemu_arch=x86_64 \
-          certpath=/etc/kubeedge/edge/certs \
-          certfile=/etc/kubeedge/edge/certs/edge.crt \
-          keyfile=/etc/kubeedge/edge/certs/edge.ke
+          certpath=/etc/kubeedge/certs \
+          certfile=/etc/kubeedge/certs/edge.crt \
+          keyfile=/etc/kubeedge/certs/edge.key
   ````
 
 + 编译容器镜像
 
   ```
-  ./build/edge/run_daemon.sh build
+  ./run_daemon.sh build
   ```
 
 + **(可选)** 如果edge的性能不够，可以在cloud上交叉编译edge的镜像，在edge端加载镜像
   - 设置CPU类型
 
     ```
-    ./build/edge/run_daemon.sh set arch=arm64v8 qemu_arch=aarch
+    ./run_daemon.sh set arch=arm64v8 qemu_arch=aarch
     ```
 
   - 编译镜像
     ```
-    ./build/edge/run_daemon.sh build
+    ./run_daemon.sh build
     ```
 
   - 保存镜像
     ```
-    ./build/edge/run_daemon.sh save 
+    ./run_daemon.sh save
     ```
 
 + 启动容器
   ```
-  ./build/edge/run_daemon.sh up
-  ```
\ No newline at end of file
+  ./run_daemon.sh up
+  ```
diff --git a/build/edge/docker-compose.yaml b/build/edge/docker-compose.yaml
index 6ca102176..2071212ee 100644
--- a/build/edge/docker-compose.yaml
+++ b/build/edge/docker-compose.yaml
@@ -30,7 +30,7 @@ services:
       edgehub.websocket.certfile: ${CERTFILE} 
       edgehub.websocket.keyfile: ${KEYFILE}
     volumes:
-      - ${CERTPATH}:/etc/kubeedge/edge/certs:ro
+      - ${CERTPATH}:/etc/kubeedge/certs:ro
       - /var/lib/edged:/var/lib/edged 
       - /var/lib/kubeedge:/var/lib/kubeedge
       - /var/run/docker.sock:/var/run/docker.sock
@@ -41,4 +41,4 @@ services:
           cpus: '1.00'
           memory: 1g
     networks:
-      - kubeedge-bridge
\ No newline at end of file
+      - kubeedge-bridge
diff --git a/build/edge/kubernetes/01-namespace.yaml b/build/edge/kubernetes/01-namespace.yaml
new file mode 100644
index 000000000..de122b6e8
--- /dev/null
+++ b/build/edge/kubernetes/01-namespace.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kubeedge
diff --git a/build/edge/kubernetes/02-edgenode.yaml b/build/edge/kubernetes/02-edgenode.yaml
new file mode 100644
index 000000000..b00b81348
--- /dev/null
+++ b/build/edge/kubernetes/02-edgenode.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1

+kind: Node

+metadata:

+  name: edgenode1

+  labels:

+    name: edge-node

diff --git a/build/edge/kubernetes/03-configmap-edgenodeconf.yaml b/build/edge/kubernetes/03-configmap-edgenodeconf.yaml
new file mode 100644
index 000000000..0d95c7463
--- /dev/null
+++ b/build/edge/kubernetes/03-configmap-edgenodeconf.yaml
@@ -0,0 +1,53 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: edgenodeconf
+  namespace: kubeedge
+  labels:
+    k8s-app: kubeedge
+    kubeedge: edgenode
+data:
+  edge.yaml: |
+    mqtt:
+        server: tcp://127.0.0.1:1883 # external mqtt broker url.
+        internal-server: tcp://127.0.0.1:1884 # internal mqtt broker url.
+        mode: 0 # 0: internal mqtt broker enable only. 1: internal and external mqtt broker enable. 2: external mqtt broker enable only.
+        qos: 0 # 0: QOSAtMostOnce, 1: QOSAtLeastOnce, 2: QOSExactlyOnce.
+        retain: false # if the flag set true, server will store the message and can be delivered to future subscribers.
+        session-queue-size: 100 # A size of how many sessions will be handled. default to 100.
+    edgehub:
+        websocket:
+            url: wss://0.0.0.0:10000/e632aba927ea4ac2b575ec1603d56f10/edgenode1/events
+            certfile: /etc/kubeedge/certs/edge.crt
+            keyfile: /etc/kubeedge/certs/edge.key
+            handshake-timeout: 30 #second
+            write-deadline: 15 # second
+            read-deadline: 15 # second
+        controller:
+            placement: false
+            heartbeat: 15  # second
+            refresh-ak-sk-interval: 10 # minute
+            auth-info-files-path: /var/IEF/secret
+            placement-url: https://10.154.193.32:7444/v1/placement_external/message_queue
+            project-id: e632aba927ea4ac2b575ec1603d56f10
+            node-id: edgenode1
+    edged:
+        register-node-namespace: default
+        hostname-override: edgenode1
+        interface-name: eth0
+        node-status-update-frequency: 10 # second
+        device-plugin-enabled: false
+        gpu-plugin-enabled: false
+        image-gc-high-threshold: 80 # percent
+        image-gc-low-threshold: 40 # percent
+        maximum-dead-containers-per-container: 1
+        docker-address: tcp://localhost:2375
+        version: 2.0.0
+  logging.yaml: |
+    loggerLevel: "DEBUG"
+    enableRsyslog: false
+    logFormatText: true
+    writers: [stdout]
+  modules.yaml: |
+    modules:
+        enabled: [eventbus, servicebus, websocket, metaManager, edged, twin, dbTest]
diff --git a/build/edge/kubernetes/04-deployment-edgenode.yaml b/build/edge/kubernetes/04-deployment-edgenode.yaml
new file mode 100644
index 000000000..ad3cba4b0
--- /dev/null
+++ b/build/edge/kubernetes/04-deployment-edgenode.yaml
@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: edgenode1
+  namespace: kubeedge
+  labels:
+    k8s-app: kubeedge
+    kubeedge: edgenode
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kubeedge
+      kubeedge: edgenode
+  template:
+    metadata:
+      labels:
+        k8s-app: kubeedge
+        kubeedge: edgenode
+    spec:
+      containers:
+      - name: edgenode
+        securityContext:
+          privileged: true
+        image: kubeedge/edgecore:latest
+        imagePullPolicy: IfNotPresent
+        resources:
+          limits:
+            cpu: 200m
+            memory: 1Gi
+          requests:
+            cpu: 100m
+            memory: 512Mi
+        env: 
+          - name: DOCKER_HOST
+            value: tcp://localhost:2375
+        volumeMounts:
+        - name: certs
+          mountPath: /etc/kubeedge/certs
+        - name: conf
+          mountPath: /etc/kubeedge/edge/conf
+      - name: dind-daemon
+        securityContext: 
+            privileged: true
+        image: docker:dind 
+        resources: 
+            requests: 
+                cpu: 20m 
+                memory: 512Mi 
+        volumeMounts: 
+          - name: docker-graph-storage 
+            mountPath: /var/lib/docker
+      volumes:
+      - name: certs
+        hostPath:
+          path: /etc/kubeedge/certs
+      - name: conf
+        configMap:
+          name: edgenodeconf
+      - name: docker-graph-storage 
+        emptyDir: {}
diff --git a/build/edge/kubernetes/README.md b/build/edge/kubernetes/README.md
new file mode 100644
index 000000000..5dafef7f5
--- /dev/null
+++ b/build/edge/kubernetes/README.md
@@ -0,0 +1,40 @@
+##  Deploy the edge part into a k8s cluster
+
+This method will guide you to deploy the edge part into a k8s cluster,
+so you need to login to the k8s master node (or where else if you can
+operate the cluster with `kubectl`).
+
+The manifests and scripts in `github.com/kubeedge/kubeedge/build/edge/kubernetes`
+will be used, so place these files to somewhere you can kubectl with.
+
+First, ensure your k8s cluster can pull edge core image. If the
+image not exist. We can make one, and push to your registry.
+
+```bash
+cd $GOPATH/src/github.com/kubeedge/kubeedge
+make edgeimage
+```
+
+We create k8s resources from the manifests in name order. Before
+creating, check the content of each manifest to make sure it meets your
+environment.
+
+Firstly you need to copy edge certs including `edge.crt` and `edge.key` into the folder
+`/etc/kubeedge/certs/` on the k8s nodes where you want to deploy edge part.
+
+On the other side, you need to replace `0.0.0.0:10000` with your kubeedge cloud web socket url.
+* [url](04-configmap-edgenodeconf.yaml#L20)
+
+The default edge node name is `edgenode1`,
+if you want to change edge node name or add new edge node,
+you need to replace the following places with your new edge node name.
+
+* [name in 02-edgenode.yaml](02-edgenode.yaml#L4)
+* [url in 03-configmap-edgenodeconf.yaml](03-configmap-edgenodeconf.yaml#L20)
+* [node-id in 03-configmap-edgenodeconf.yaml](03-configmap-edgenodeconf.yaml#L33)
+* [hostname-override in 03-configmap-edgenodeconf.yaml](03-configmap-edgenodeconf.yaml#L36)
+* [name in 04-deployment-edgenode.yaml](04-deployment-edgenode.yaml#L4)
+
+```bash
+for resource in $(ls *.yaml); do kubectl create -f $resource; done
+```
diff --git a/build/edge/kubernetes/README_zh.md b/build/edge/kubernetes/README_zh.md
new file mode 100644
index 000000000..25a31cc26
--- /dev/null
+++ b/build/edge/kubernetes/README_zh.md
@@ -0,0 +1,30 @@
+## 部署 edge 端到 k8s 集群
+
+此方式将部署 edge 端到 k8s 集群，所以需要登录到 k8s 的 master 节点上（或者其他可以用 `kubectl` 操作集群的机器）。
+
+存放在 `github.com/kubeedge/kubeedge/build/edge/kubernetes` 里的各个编排文件和脚本会被用到。所以需要先将这些文件放到可以用 kubectl 操作的地方。
+
+首先， 确保 k8s 集群可以拉到 edge core 镜像。如果没有， 可以构建一个，然后推到集群能拉到的 registry 上。
+
+```bash
+cd $GOPATH/src/github.com/kubeedge/kubeedge
+make edgeimage
+```
+
+我们按照编排文件的文件名顺序创建各个 k8s 资源。在创建之前，应该检查每个编排文件内容，以确保符合特定的集群环境。
+
+首先您需要去拷贝 edge certs 文件包括`edge.crt`和`edge.key`到您想要部署 edge part 的 k8s 节点上的`/etc/kubeedge/certs/`文件夹中。
+
+另一方面，您需要替换`0.0.0.0:10000`成您的 kubeedge cloud web socket url。
+* [url](04-configmap-edgenodeconf.yaml#L20)
+
+默认的边缘节点名称是`edgenode1`，如果您想要改变节点名称或者是创建新的边缘节点，您需要用新的边缘节点名称替换如下几个地方。
+* [name in 02-edgenode.yaml](02-edgenode.yaml#L4)
+* [url in 03-configmap-edgenodeconf.yaml](03-configmap-edgenodeconf.yaml#L20)
+* [node-id in 03-configmap-edgenodeconf.yaml](03-configmap-edgenodeconf.yaml#L33)
+* [hostname-override in 03-configmap-edgenodeconf.yaml](03-configmap-edgenodeconf.yaml#L36)
+* [name in 04-deployment-edgenode.yaml](04-deployment-edgenode.yaml#L4)
+
+```bash
+for resource in $(ls *.yaml); do kubectl create -f $resource; done
+```
diff --git a/build/edge/run_daemon.sh b/build/edge/run_daemon.sh
index d01622102..c89181b8c 100755
--- a/build/edge/run_daemon.sh
+++ b/build/edge/run_daemon.sh
@@ -90,13 +90,13 @@ docker_set(){
     # 
     #  ./build.sh set \
     #    cloudhub=0.0.0.0:10000 \
-    #    edgename=edgeNode \
+    #    edgename=edge-node \
     #    edge_core_image="kubeedge/edgecore:latest" \
     #    arch=amd64 \
     #    qemu_arch=x86_64 \
     #    certpath=/etc/kubeedge/certs \
     #    certfile=/etc/kubeedge/certs/edge.crt \
-    #    keyfile=/etc/kubeedge/certs/edge.ke
+    #    keyfile=/etc/kubeedge/certs/edge.key
 
     ARGS=$@
 
@@ -148,7 +148,7 @@ docker_only_run_edge(){
     # 
     # Example
     # 
-    # ./run_daemon.sh only_run_edge mqtt=0.0.0.0:1883 cloudhub=0.0.0.0:10000 edgename=edgeNode image="kubeedge/edgecore:latest"
+    # ./run_daemon.sh only_run_edge mqtt=0.0.0.0:1883 cloudhub=0.0.0.0:10000 edgename=edge-node image="kubeedge/edgecore:latest"
 
     ARGS=$@
 
diff --git a/edge/conf/edge.yaml b/edge/conf/edge.yaml
index 90854e3a6..6aad91f61 100644
--- a/edge/conf/edge.yaml
+++ b/edge/conf/edge.yaml
@@ -33,4 +33,5 @@ edged:
     image-gc-high-threshold: 80 # percent
     image-gc-low-threshold: 40 # percent
     maximum-dead-containers-per-container: 1
+    docker-address: unix:///var/run/docker.sock
     version: 2.0.0
diff --git a/edge/pkg/edged/dockertools/docker_manager.go b/edge/pkg/edged/dockertools/docker_manager.go
index d1d98c504..db82b191a 100644
--- a/edge/pkg/edged/dockertools/docker_manager.go
+++ b/edge/pkg/edged/dockertools/docker_manager.go
@@ -81,8 +81,18 @@ const PROJECTNAME = "southchina"
 var (
 	// ErrImageNeverPull : Required Image is absent on host and PullPolicy is NeverPullImage
 	ErrImageNeverPull = errors.New("ErrImageNeverPull")
+	// DockerAddress is address for docker
+	DockerAddress = DockerDefaultAddress
 )
 
+// InitDockerAddress inits docker address
+func InitDockerAddress(dockerAddress string) {
+	DockerAddress = dockerAddress
+	if DockerAddress == "" {
+		DockerAddress = DockerDefaultAddress
+	}
+}
+
 //DockerManager defines object structure of docker manager
 type DockerManager struct {
 	containers.ContainerManager
@@ -104,7 +114,7 @@ func NewDockerClient(dockerEndpoint string) (dockerapi.CommonAPIClient, error) {
 func NewDockerManager(livenessManager proberesults.Manager, qps float32, burst int, backOff *flowcontrol.Backoff, serializeImagePulls bool, devicePluginEnabled bool, gpuManager gpu.GPUManager, interfaceName string) (*DockerManager, error) {
 	var err error
 	dm := &DockerManager{}
-	client, err := NewDockerClient(DockerDefaultAddress)
+	client, err := NewDockerClient(DockerAddress)
 	if err != nil {
 		return nil, fmt.Errorf("new docker client failed: %s", err)
 	}
@@ -590,6 +600,6 @@ func convertTime(stringTime string) time.Time {
 // TODO: fillup other field in this struct
 func NewDockerConfig() *dockershim.ClientConfig {
 	return &dockershim.ClientConfig{
-		DockerEndpoint: DockerDefaultAddress,
+		DockerEndpoint: DockerAddress,
 	}
 }
diff --git a/edge/pkg/edged/edged.go b/edge/pkg/edged/edged.go
index 4d7ed2c20..f18856087 100644
--- a/edge/pkg/edged/edged.go
+++ b/edge/pkg/edged/edged.go
@@ -144,6 +144,7 @@ type Config struct {
 	imageGCHighThreshold     int
 	imageGCLowThreshold      int
 	MaxPerPodContainerCount  int
+	DockerAddress            string
 	version                  string
 }
 
@@ -248,6 +249,7 @@ func getConfig() *Config {
 	conf.imageGCLowThreshold = config.CONFIG.GetConfigurationByKey("edged.image-gc-low-threshold").(int)
 	conf.MaxPerPodContainerCount = config.CONFIG.GetConfigurationByKey("edged.maximum-dead-containers-per-container").(int)
 	conf.version = config.CONFIG.GetConfigurationByKey("edged.version").(string)
+	conf.DockerAddress = config.CONFIG.GetConfigurationByKey("edged.docker-address").(string)
 	return &conf
 }
 
@@ -287,6 +289,11 @@ func newEdged() (*edged, error) {
 		workQueue:                 queue.NewBasicWorkQueue(clock.RealClock{}),
 	}
 
+	// Set docker address if it is set in the conf
+	if conf.DockerAddress != "" {
+		dockertools.InitDockerAddress(conf.DockerAddress)
+	}
+
 	if conf.gpuPluginEnabled {
 		gpuManager, _ = nvidia.NewNvidiaGPUManager(ed, dockertools.NewDockerConfig())
 	} else {
diff --git a/edge/test/README.md b/edge/test/README.md
index 4c6adbfa8..eccf6558f 100644
--- a/edge/test/README.md
+++ b/edge/test/README.md
@@ -95,6 +95,7 @@ edged:
     image-gc-high-threshold: 80 # percent
     image-gc-low-threshold: 40 # percent
     maximum-dead-containers-per-container: 1
+    docker-address: unix:///var/run/docker.sock
     version: 2.0.0
 ```