Pod running on edge node has invalid serviceaccount token

Signed-off-by: vincentgoat <linguohui1@huawei.com>

3 files changed, 137 insertions, 3 deletions

diff --git a/edge/pkg/edged/volume_host.go b/edge/pkg/edged/volume_host.go
index be5280747..e5cdca36f 100644
--- a/edge/pkg/edged/volume_host.go
+++ b/edge/pkg/edged/volume_host.go
@@ -158,7 +158,9 @@ func (evh *edgedVolumeHost) GetSecretFunc() func(namespace, name string) (*api.S
 func (evh *edgedVolumeHost) GetVolumeDevicePluginDir(pluginName string) string { return "" }
 
 func (evh *edgedVolumeHost) DeleteServiceAccountTokenFunc() func(podUID types.UID) {
-	return func(types.UID) {}
+	return func(podUID types.UID) {
+		evh.edge.metaClient.ServiceAccountToken().DeleteServiceAccountToken(podUID)
+	}
 }
 
 func (evh *edgedVolumeHost) GetEventRecorder() recordtools.EventRecorder {
diff --git a/edge/pkg/metamanager/client/serviceaccount.go b/edge/pkg/metamanager/client/serviceaccount.go
index 0a840db57..30a8a23c4 100644
--- a/edge/pkg/metamanager/client/serviceaccount.go
+++ b/edge/pkg/metamanager/client/serviceaccount.go
@@ -3,14 +3,17 @@ package client
 import (
 	"encoding/json"
 	"fmt"
+	"time"
 
 	authenticationv1 "k8s.io/api/authentication/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
 
 	"github.com/kubeedge/beehive/pkg/core/model"
 	"github.com/kubeedge/kubeedge/edge/pkg/common/message"
 	"github.com/kubeedge/kubeedge/edge/pkg/common/modules"
 	"github.com/kubeedge/kubeedge/edge/pkg/metamanager"
+	"github.com/kubeedge/kubeedge/edge/pkg/metamanager/dao"
 )
 
 // ServiceAccountTokenGetter is interface to get client service account token
@@ -21,20 +24,97 @@ type ServiceAccountTokenGetter interface {
 // ServiceAccountTokenInterface is interface for client service account token
 type ServiceAccountTokenInterface interface {
 	GetServiceAccountToken(namespace string, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error)
+	DeleteServiceAccountToken(podUID types.UID)
 }
 
 type serviceAccountToken struct {
 	send SendInterface
 }
 
+const maxTTL = 24 * time.Hour
+
 func newServiceAccountToken(s SendInterface) *serviceAccountToken {
 	return &serviceAccountToken{
 		send: s,
 	}
 }
 
-func (c *serviceAccountToken) GetServiceAccountToken(namespace string, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) {
-	resource := fmt.Sprintf("%s/%s/%s", namespace, model.ResourceTypeServiceAccountToken, name)
+func (c *serviceAccountToken) DeleteServiceAccountToken(podUID types.UID) {
+	svcAccounts, err := dao.QueryAllMeta("type", model.ResourceTypeServiceAccountToken)
+	if err != nil {
+		klog.Errorf("query meta failed: %v", err)
+		return
+	}
+	for _, sa := range *svcAccounts {
+		var tr authenticationv1.TokenRequest
+		err = json.Unmarshal([]byte(sa.Value), &tr)
+		if err != nil {
+			klog.Errorf("unmarshal resource %s token request failed: %v", sa.Key, err)
+			continue
+		}
+		if podUID == tr.Spec.BoundObjectRef.UID {
+			err := dao.DeleteMetaByKey(sa.Key)
+			if err != nil {
+				klog.Errorf("delete meta %s failed: %v", sa.Key, err)
+				return
+			}
+		}
+	}
+}
+
+// requiresRefresh returns true if the token is older than 80% of its total
+// ttl, or if the token is older than 24 hours.
+func requiresRefresh(tr *authenticationv1.TokenRequest) bool {
+	if tr.Spec.ExpirationSeconds == nil {
+		cpy := tr.DeepCopy()
+		cpy.Status.Token = ""
+		klog.Errorf("expiration seconds was nil for tr: %#v", cpy)
+		return false
+	}
+	now := time.Now()
+	exp := tr.Status.ExpirationTimestamp.Time
+	iat := exp.Add(-1 * time.Duration(*tr.Spec.ExpirationSeconds) * time.Second)
+
+	if now.After(iat.Add(maxTTL)) {
+		return true
+	}
+	// Require a refresh if within 20% of the TTL from the expiration time.
+	if now.After(exp.Add(-1 * time.Duration((*tr.Spec.ExpirationSeconds*20)/100) * time.Second)) {
+		return true
+	}
+	return false
+}
+
+func getTokenLocally(name, namespace string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) {
+	resKey := metamanager.KeyFunc(name, namespace, tr)
+	metas, err := dao.QueryMeta("key", resKey)
+	if err != nil {
+		klog.Errorf("query meta %s failed: %v", resKey, err)
+		return nil, err
+	}
+	if len(*metas) != 1 {
+		klog.Errorf("query meta %s length error", resKey)
+		return nil, fmt.Errorf("query meta %s length error", resKey)
+	}
+	var tokenRequest authenticationv1.TokenRequest
+	err = json.Unmarshal([]byte((*metas)[0]), &tokenRequest)
+	if err != nil {
+		klog.Errorf("unmarshal resource %s token request failed: %v", resKey, err)
+		return nil, err
+	}
+	if requiresRefresh(&tokenRequest) {
+		err := dao.DeleteMetaByKey(resKey)
+		if err != nil {
+			klog.Errorf("delete meta %s failed: %v", resKey, err)
+			return nil, err
+		}
+		klog.Errorf("resource %s token expired", resKey)
+		return nil, fmt.Errorf("resource %s token expired", resKey)
+	}
+	return &tokenRequest, nil
+}
+
+func getTokenRemotely(resource string, tr *authenticationv1.TokenRequest, c *serviceAccountToken) (*authenticationv1.TokenRequest, error) {
 	tokenMsg := message.BuildMsg(modules.MetaGroup, "", modules.EdgedModuleName, resource, model.QueryOperation, tr)
 	msg, err := c.send.SendSync(tokenMsg)
 	if err != nil {
@@ -54,6 +134,15 @@ func (c *serviceAccountToken) GetServiceAccountToken(namespace string, name stri
 	return handleServiceAccountTokenFromMetaManager(content)
 }
 
+func (c *serviceAccountToken) GetServiceAccountToken(namespace string, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) {
+	tokenReq, err := getTokenLocally(name, namespace, tr)
+	if err != nil {
+		resource := fmt.Sprintf("%s/%s/%s", namespace, model.ResourceTypeServiceAccountToken, name)
+		return getTokenRemotely(resource, tr, c)
+	}
+	return tokenReq, nil
+}
+
 func handleServiceAccountTokenFromMetaDB(content []byte) (*authenticationv1.TokenRequest, error) {
 	var lists []string
 	err := json.Unmarshal(content, &lists)
diff --git a/edge/pkg/metamanager/process.go b/edge/pkg/metamanager/process.go
index 40af4071e..12f149165 100644
--- a/edge/pkg/metamanager/process.go
+++ b/edge/pkg/metamanager/process.go
@@ -6,6 +6,7 @@ import (
 	"strings"
 	"time"
 
+	authenticationv1 "k8s.io/api/authentication/v1"
 	"k8s.io/klog/v2"
 
 	beehiveContext "github.com/kubeedge/beehive/pkg/core/context"
@@ -238,11 +239,47 @@ func (m *metaManager) processDelete(message model.Message) {
 	sendToCloud(resp)
 }
 
+// KeyFunc keys should be nonconfidential and safe to log
+func KeyFunc(name, namespace string, tr *authenticationv1.TokenRequest) string {
+	var exp int64
+	if tr.Spec.ExpirationSeconds != nil {
+		exp = *tr.Spec.ExpirationSeconds
+	}
+
+	var ref authenticationv1.BoundObjectReference
+	if tr.Spec.BoundObjectRef != nil {
+		ref = *tr.Spec.BoundObjectRef
+	}
+
+	return fmt.Sprintf("%q/%q/%#v/%#v/%#v", name, namespace, tr.Spec.Audiences, exp, ref)
+}
+
+// getSpecialResourceKey get service account db key
+func getSpecialResourceKey(resType, resKey string, message model.Message) (string, error) {
+	if resType != model.ResourceTypeServiceAccountToken {
+		return resKey, nil
+	}
+	tokenReq, ok := message.GetContent().(*authenticationv1.TokenRequest)
+	if !ok {
+		return "", fmt.Errorf("failed to get resource %s name and namespace", resKey)
+	}
+	tokens := strings.Split(resKey, constants.ResourceSep)
+	if len(tokens) != 3 {
+		return "", fmt.Errorf("failed to get resource %s name and namespace", resKey)
+	}
+	return KeyFunc(tokens[2], tokens[0], tokenReq), nil
+}
+
 func (m *metaManager) processQuery(message model.Message) {
 	resKey, resType, resID := parseResource(message.GetResource())
 	var metas *[]string
 	var err error
 	if requireRemoteQuery(resType) && isConnected() {
+		resKey, err = getSpecialResourceKey(resType, resKey, message)
+		if err != nil {
+			klog.Errorf("failed to get special resource %s key", resKey)
+			return
+		}
 		metas, err = dao.QueryMeta("key", resKey)
 		if err != nil || len(*metas) == 0 || resType == model.ResourceTypeNode || resType == constants.ResourceTypeVolumeAttachment {
 			m.processRemoteQuery(message)
@@ -294,6 +331,12 @@ func (m *metaManager) processRemoteQuery(message model.Message) {
 		}
 
 		resKey, resType, _ := parseResource(message.GetResource())
+		resKey, err = getSpecialResourceKey(resType, resKey, message)
+		if err != nil {
+			klog.Errorf("get remote query response content data failed, %s", msgDebugInfo(&resp))
+			feedbackError(err, "Error to get remote query response message content data", message)
+			return
+		}
 		meta := &dao.Meta{
 			Key:   resKey,
 			Type:  resType,