騰訊云：一文讀懂Kubernetes APIServer原理（上）

前言

整個(gè)Kubernetes技術(shù)體系由聲明式API以及Controller構(gòu)成，而kube-apiserver是Kubernetes的聲明式api server，并為其它組件交互提供了橋梁。因此加深對(duì)kube-apiserver的理解就顯得至關(guān)重要了。

整體組件功能

kube-apiserver作為整個(gè)Kubernetes集群操作etcd的唯一入口，負(fù)責(zé)Kubernetes各資源的認(rèn)證&鑒權(quán)，校驗(yàn)以及CRUD等操作，提供RESTful APIs，供其它組件調(diào)用：

kube-apiserver包含三種APIServer：

·aggregatorServer：負(fù)責(zé)處理apiregistration.k8s.io組下的APIService資源請(qǐng)求，同時(shí)將來(lái)自用戶的請(qǐng)求攔截轉(zhuǎn)發(fā)給aggregated server(AA)

·kubeAPIServer：負(fù)責(zé)對(duì)請(qǐng)求的一些通用處理，包括：認(rèn)證、鑒權(quán)以及各個(gè)內(nèi)建資源(pod,deployment，service and etc)的REST服務(wù)等

·apiExtensionsServer：負(fù)責(zé)CustomResourceDefinition（CRD）apiResources以及apiVersions的注冊(cè)，同時(shí)處理CRD以及相應(yīng)CustomResource（CR）的REST請(qǐng)求(如果對(duì)應(yīng)CR不能被處理的話則會(huì)返回404)，也是apiserver Delegation的最后一環(huán)

另外還包括bootstrap-controller，主要負(fù)責(zé)Kubernetes default apiserver service的創(chuàng)建以及管理。

接下來(lái)將對(duì)上述組件進(jìn)行概覽性總結(jié)。

bootstrap-controller

·apiserver bootstrap-controller創(chuàng)建&運(yùn)行邏輯在k8s.io/kubernetes/pkg/master目錄

·bootstrap-controller主要用于創(chuàng)建以及維護(hù)內(nèi)部kubernetes default apiserver service

·kubernetes default apiserver service spec.selector為空，這是default apiserver service與其它正常service的最大區(qū)別，表明了這個(gè)特殊的service對(duì)應(yīng)的endpoints不由endpoints controller控制，而是直接受kube-apiserver bootstrap-controller管理(maintained by this code,not by the pod selector)

·bootstrap-controller的幾個(gè)主要功能如下：

·創(chuàng)建default、kube-system和kube-public以及kube-node-lease命名空間

·創(chuàng)建&維護(hù)kubernetes default apiserver service以及對(duì)應(yīng)的endpoint

·提供基于Service ClusterIP的檢查及修復(fù)功能(--service-cluster-ip-range指定范圍)

·提供基于Service NodePort的檢查及修復(fù)功能(--service-node-port-range指定范圍)

// k8s.io/kubernetes/pkg/master/controller.go:142

// Start begins the core controller loops that must exist for bootstrapping

// a cluster.

func (c *Controller) Start() {

    if c.runner != nil {

        return

    }

    // Reconcile during first run removing itself until server is ready.

    endpointPorts := createEndpointPortSpec(c.PublicServicePort, "https", c.ExtraEndpointPorts)

    if err := c.EndpointReconciler.RemoveEndpoints(kubernetesServiceName, c.PublicIP, endpointPorts); err != nil {

        klog.Errorf("Unable to remove old endpoints from kubernetes service: %v", err)

    }

    repairClusterIPs := servicecontroller.NewRepair(c.ServiceClusterIPInterval, c.ServiceClient, c.EventClient, &c.ServiceClusterIPRange, c.ServiceClusterIPRegistry, &c.SecondaryServiceClusterIPRange, c.SecondaryServiceClusterIPRegistry)

    repairNodePorts := portallocatorcontroller.NewRepair(c.ServiceNodePortInterval, c.ServiceClient, c.EventClient, c.ServiceNodePortRange, c.ServiceNodePortRegistry)

    // run all of the controllers once prior to returning from Start.

    if err := repairClusterIPs.RunOnce(); err != nil {

        // If we fail to repair cluster IPs apiserver is useless. We should restart and retry.

        klog.Fatalf("Unable to perform initial IP allocation check: %v", err)

    }

    if err := repairNodePorts.RunOnce(); err != nil {

        // If we fail to repair node ports apiserver is useless. We should restart and retry.

        klog.Fatalf("Unable to perform initial service nodePort check: %v", err)

    }

    // 定期執(zhí)行bootstrap controller主要的四個(gè)功能(reconciliation)  

    c.runner = async.NewRunner(c.RunKubernetesNamespaces, c.RunKubernetesService, repairClusterIPs.RunUntil, repairNodePorts.RunUntil)

    c.runner.Start()

}

更多代碼原理詳情，參考kubernetes-reading-notes[1]。

kubeAPIServer

KubeAPIServer主要提供對(duì)內(nèi)建API Resources的操作請(qǐng)求，為Kubernetes中各API Resources注冊(cè)路由信息，同時(shí)暴露RESTful API，使集群中以及集群外的服務(wù)都可以通過(guò)RESTful API操作Kubernetes中的資源

另外，kubeAPIServer是整個(gè)Kubernetes apiserver的核心，下面將要講述的aggregatorServer以及apiExtensionsServer都是建立在kubeAPIServer基礎(chǔ)上進(jìn)行擴(kuò)展的(補(bǔ)充了Kubernetes對(duì)用戶自定義資源的能力支持)

kubeAPIServer最核心的功能是為Kubernetes內(nèi)置資源添加路由，如下：

·調(diào)用m.InstallLegacyAPI將核心API Resources添加到路由中，在apiserver中即是以/api開(kāi)頭的resource；

·調(diào)用m.InstallAPIs將擴(kuò)展的API Resources添加到路由中，在apiserver中即是以/apis開(kāi)頭的resource；

// k8s.io/kubernetes/pkg/master/master.go:332

// New returns a new instance of Master from the given config.

// Certain config fields will be set to a default value if unset.

// Certain config fields must be specified, including:

//   KubeletClientConfig

func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Master, error) {

    ...

    // 安裝 LegacyAPI(core API)

    // install legacy rest storage

    if c.ExtraConfig.APIResourceConfigSource.VersionEnabled(apiv1.SchemeGroupVersion) {

        legacyRESTStorageProvider := corerest.LegacyRESTStorageProvider{

            StorageFactory:              c.ExtraConfig.StorageFactory,

            ProxyTransport:              c.ExtraConfig.ProxyTransport,

            KubeletClientConfig:         c.ExtraConfig.KubeletClientConfig,

            EventTTL:                    c.ExtraConfig.EventTTL,

            ServiceIPRange:              c.ExtraConfig.ServiceIPRange,

            SecondaryServiceIPRange:     c.ExtraConfig.SecondaryServiceIPRange,

            ServiceNodePortRange:        c.ExtraConfig.ServiceNodePortRange,

            LoopbackClientConfig:        c.GenericConfig.LoopbackClientConfig,

            ServiceAccountIssuer:        c.ExtraConfig.ServiceAccountIssuer,

            ServiceAccountMaxExpiration: c.ExtraConfig.ServiceAccountMaxExpiration,

            APIAudiences:                c.GenericConfig.Authentication.APIAudiences,

        }

        if err := m.InstallLegacyAPI(&c, c.GenericConfig.RESTOptionsGetter, legacyRESTStorageProvider); err != nil {

            return nil, err

        }

    }

    ...

    // 安裝 APIs(named groups apis)

    if err := m.InstallAPIs(c.ExtraConfig.APIResourceConfigSource, c.GenericConfig.RESTOptionsGetter, restStorageProviders...); err != nil {

        return nil, err

    }

    ...

    return m, nil

}

整個(gè)kubeAPIServer提供了三類API Resource接口：

·core group：主要在/api/v1下；

·named groups：其path為/apis/$GROUP/$VERSION；

·系統(tǒng)狀態(tài)的一些API：如/metrics、/version等；

而API的URL大致以/apis/{group}/{version}/namespaces/{namespace}/resource/{name}組成，結(jié)構(gòu)如下圖所示：

kubeAPIServer會(huì)為每種API資源創(chuàng)建對(duì)應(yīng)的RESTStorage，RESTStorage的目的是將每種資源的訪問(wèn)路徑及其后端存儲(chǔ)的操作對(duì)應(yīng)起來(lái)：通過(guò)構(gòu)造的REST Storage實(shí)現(xiàn)的接口判斷該資源可以執(zhí)行哪些操作（如：create、update等），將其對(duì)應(yīng)的操作存入到action中，每一個(gè)操作對(duì)應(yīng)一個(gè)標(biāo)準(zhǔn)的REST method，如create對(duì)應(yīng)REST method為POST，而update對(duì)應(yīng)REST method為PUT。最終根據(jù)actions數(shù)組依次遍歷，對(duì)每一個(gè)操作添加一個(gè)handler(handler對(duì)應(yīng)REST Storage實(shí)現(xiàn)的相關(guān)接口)，并注冊(cè)到route，最終對(duì)外提供RESTful API，如下：

// m.GenericAPIServer.InstallLegacyAPIGroup --> s.installAPIResources --> apiGroupVersion.InstallREST --> installer.Install --> a.registerResourceHandlers

// k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go:181

func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storage, ws *restful.WebService) (*metav1.APIResource, error) {

    ...

    // 1、判斷該 resource 實(shí)現(xiàn)了哪些 REST 操作接口，以此來(lái)判斷其支持的 verbs 以便為其添加路由

    // what verbs are supported by the storage, used to know what verbs we support per path

    creater, isCreater := storage.(rest.Creater)

    namedCreater, isNamedCreater := storage.(rest.NamedCreater)

    lister, isLister := storage.(rest.Lister)

    getter, isGetter := storage.(rest.Getter)

    ...

    // 2、為 resource 添加對(duì)應(yīng)的 actions(+根據(jù)是否支持 namespace)

    // Get the list of actions for the given scope.

    switch {

    case !namespaceScoped:

        // Handle non-namespace scoped resources like nodes.

        resourcePath := resource

        resourceParams := params

        itemPath := resourcePath + "/{name}"

        nameParams := append(params, nameParam)

        proxyParams := append(nameParams, pathParam)

        ...

        // Handler for standard REST verbs (GET, PUT, POST and DELETE).

        // Add actions at the resource path: /api/apiVersion/resource

        actions = appendIf(actions, action{"LIST", resourcePath, resourceParams, namer, false}, isLister)

        actions = appendIf(actions, action{"POST", resourcePath, resourceParams, namer, false}, isCreater)

        ...

    }

    ...

    // 3、從 rest.Storage 到 restful.Route 映射

    // 為每個(gè)操作添加對(duì)應(yīng)的 handler

    for _, action := range actions {

        ...

        switch action.Verb {

        ...

        case "POST": // Create a resource.

            var handler restful.RouteFunction

            // 4、初始化 handler

            if isNamedCreater {

                handler = restfulCreateNamedResource(namedCreater, reqScope, admit)

            } else {

                handler = restfulCreateResource(creater, reqScope, admit)

            }

            handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, handler)

            ...

            // 5、route 與 handler 進(jìn)行綁定    

            route := ws.POST(action.Path).To(handler).

                Doc(doc).

                Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).

                Operation("create"+namespaced+kind+strings.Title(subresource)+operationSuffix).

                Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).

                Returns(http.StatusOK, "OK", producedObject).

                // TODO: in some cases, the API may return a v1.Status instead of the versioned object

                // but currently go-restful can't handle multiple different objects being returned.

                Returns(http.StatusCreated, "Created", producedObject).

                Returns(http.StatusAccepted, "Accepted", producedObject).

                Reads(defaultVersionedObject).

                Writes(producedObject)

            if err := AddObjectParams(ws, route, versionedCreateOptions); err != nil {

                return nil, err

            }

            addParams(route, action.Params)

            // 6、添加到路由中    

            routes = append(routes, route)

        case "DELETE": // Delete a resource.

        ...

        default:

            return nil, fmt.Errorf("unrecognized action verb: %s", action.Verb)

        }

        for _, route := range routes {

            route.Metadata(ROUTE_META_GVK, metav1.GroupVersionKind{

                Group:   reqScope.Kind.Group,

                Version: reqScope.Kind.Version,

                Kind:    reqScope.Kind.Kind,

            })

            route.Metadata(ROUTE_META_ACTION, strings.ToLower(action.Verb))

            ws.Route(route)

        }

        // Note: update GetAuthorizerAttributes() when adding a custom handler.

    }

    ...

}