Kubernetes 垃圾回收机制分析

为了回收系统上的资源kubelet有ImageGC和ContainerGC等功能对image和container进行回收；

根据kubelet代码对ContainerGC部分进行分析

相关的参数主要有:

• minimum-container-ttl-duration

• maximum-dead-containers-per-container

• minimum-container-ttl-duration

相对应的代码是:

1. //pkg/kubelet/container_gc.go

2. 
   

3. type ContainerGCPolicy struct {

4.    // 已经死掉的容器在机器上存留的时间

5.    MinAge time.Duration

6. 
   

7.    // 每个pod可以保留的死掉的容器

8.    MaxPerPodContainer int

9. 
   

10.    // 机器上最大的可以保留的死亡容器数量

11.    MaxContainers int

12. }

13. ...

14. func NewContainerGC(runtime Runtime, policy ContainerGCPolicy, sourcesReadyProvider SourcesReadyProvider) (ContainerGC, error) {

15.    if policy.MinAge <>0 {

16.        return nil, fmt.Errorf('invalid minimum garbage collection age: %v', policy.MinAge)

17.    }

18. 
    

19.    return &realContainerGC{

20.        runtime:              runtime,

21.        policy:               policy,

22.        sourcesReadyProvider: sourcesReadyProvider,

23.    }, nil

24. }

25. 
    

26. func (cgc *realContainerGC) GarbageCollect() error {

27.    return cgc.runtime.GarbageCollect(cgc.policy, cgc.sourcesReadyProvider.AllReady(), false)

28. }

29. 
    

30. func (cgc *realContainerGC) DeleteAllUnusedContainers() error {

31.    return cgc.runtime.GarbageCollect(cgc.policy, cgc.sourcesReadyProvider.AllReady(), true)

32. }

GarbageCollect方法里面的调用就是ContainerGC真正的逻辑所在, GarbageCollect函数是在pkg/kubelet/kubelet.go里面调用的，每隔一分钟会执行一次。GarbageCollect里面所调用的runtime的GarbageCollect函数是在pkg/kubelet/kuberuntime/kuberuntime_gc.go里面。

//pkg/kubelet/kuberuntime/kuberuntime_gc.go
func (cgc *containerGC) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool, evictNonDeletedPods bool) error {
    // Remove evictable containers
    if err := cgc.evictContainers(gcPolicy, allSourcesReady, evictNonDeletedPods); err != nil {
        return err
    }

    // Remove sandboxes with zero containers
    if err := cgc.evictSandboxes(evictNonDeletedPods); err != nil {
        return err
    }

    // Remove pod sandbox log directory
    return cgc.evictPodLogsDirectories(allSourcesReady)
}

第一步是驱逐容器

1. func (cgc *containerGC) evictContainers(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool, evictNonDeletedPods bool) error {

2.    // Separate containers by evict units.

3.    evictUnits, err := cgc.evictableContainers(gcPolicy.MinAge)

4.    if err != nil {

5.        return err

6.    }

7. 
   

8.    // Remove deleted pod containers if all sources are ready.

9.    if allSourcesReady {

10.        for key, unit := range evictUnits {

11.            if cgc.isPodDeleted(key.uid) || evictNonDeletedPods {

12.                cgc.removeOldestN(unit, len(unit)) // Remove all.

13.                delete(evictUnits, key)

14.            }

15.        }

16.    }

17. 
    

18.    // Enforce max containers per evict unit.

19.    if gcPolicy.MaxPerPodContainer >= 0 {

20.        cgc.enforceMaxContainersPerEvictUnit(evictUnits, gcPolicy.MaxPerPodContainer)

21.    }

22. 
    

23.    // Enforce max total number of containers.

24.    if gcPolicy.MaxContainers >= 0 && evictUnits.NumContainers() > gcPolicy.MaxContainers {

25.        // Leave an equal number of containers per evict unit (min: 1).

26.        numContainersPerEvictUnit := gcPolicy.MaxContainers / evictUnits.NumEvictUnits()

27.        if numContainersPerEvictUnit <>1 {

28.            numContainersPerEvictUnit = 1

29.        }

30.        cgc.enforceMaxContainersPerEvictUnit(evictUnits, numContainersPerEvictUnit)

31. 
    

32.        // If we still need to evict, evict oldest first.

33.        numContainers := evictUnits.NumContainers()

34.        if numContainers > gcPolicy.MaxContainers {

35.            flattened := make([]containerGCInfo, 0, numContainers)

36.            for key := range evictUnits {

37.                flattened = append(flattened, evictUnits[key]...)

38.            }

39.            sort.Sort(byCreated(flattened))

40. 
    

41.            cgc.removeOldestN(flattened, numContainers-gcPolicy.MaxContainers)

42.        }

43.    }

44.    return nil

45. }

1.首先获取已经死掉的并且创建时间大于minage的容器

2.如果pod已经delete那么把属于这个pod的容器全部删除

3.如果设置了MaxPerPodContainer那么把MaxPerPodContainer之外数量的容器删除，这个值默认是1

4.如果设置MaxContainers那么再次对容器进行清理，这个值默认是-1也就是不清理的。首先会拿所有容器的数量除以pod的数量，这样会得到一个平均值，然后按照这个值对3进行再次处理。这个时候如果机器上的死亡容器的数量还大于MaxContainer那么直接按照时间对容器进行排序然后删除大于MaxContainer数量之外的容器。

下一步是清理机器上的sandbox

func (cgc *containerGC) evictSandboxes(evictNonDeletedPods bool) error {
    containers, err := cgc.manager.getKubeletContainers(true)
    if err != nil {
        return err
    }

    sandboxes, err := cgc.manager.getKubeletSandboxes(true)
    if err != nil {
        return err
    }

    sandboxesByPod := make(sandboxesByPodUID)
    for _, sandbox := range sandboxes {
        podUID := types.UID(sandbox.Metadata.Uid)
        sandboxInfo := sandboxGCInfo{
            id:         sandbox.Id,
            createTime: time.Unix(0, sandbox.CreatedAt),
        }

        // Set ready sandboxes to be active.
        if sandbox.State == runtimeapi.PodSandboxState_SANDBOX_READY {
            sandboxInfo.active = true
        }

        // Set sandboxes that still have containers to be active.
        hasContainers := false
        sandboxID := sandbox.Id
        for _, container := range containers {
            if container.PodSandboxId == sandboxID {
                hasContainers = true
                break
            }
        }
        if hasContainers {
            sandboxInfo.active = true
        }

        sandboxesByPod[podUID] = append(sandboxesByPod[podUID], sandboxInfo)
    }

    // Sort the sandboxes by age.
    for uid := range sandboxesByPod {
        sort.Sort(sandboxByCreated(sandboxesByPod[uid]))
    }

    for podUID, sandboxes := range sandboxesByPod {
        if cgc.isPodDeleted(podUID) || evictNonDeletedPods {
            // Remove all evictable sandboxes if the pod has been removed.
            // Note that the latest dead sandbox is also removed if there is
            // already an active one.
            cgc.removeOldestNSandboxes(sandboxes, len(sandboxes))
        } else {
            // Keep latest one if the pod still exists.
            cgc.removeOldestNSandboxes(sandboxes, len(sandboxes)-1)
        }
    }
    return nil
}

先获取机器上所有的容器和sandbox,如果pod的状态是0则致为active状态，如果此sandbox还有运行的container也认为是active状态，接着对sandbox进行排序，如果sandbox所属的pod的已经被删除那么删除所有的sandbox，如果pod还存在那么就留下最新的一个sandbox其他的都删除.

最后一步是清除container和pod日志

1. // evictPodLogsDirectories evicts all evictable pod logs directories. Pod logs directories

2. // are evictable if there are no corresponding pods.

3. func (cgc *containerGC) evictPodLogsDirectories(allSourcesReady bool) error {

4.    osInterface := cgc.manager.osInterface

5.    if allSourcesReady {

6.        // Only remove pod logs directories when all sources are ready.

7.        dirs, err := osInterface.ReadDir(podLogsRootDirectory)

8.        if err != nil {

9.            return fmt.Errorf('failed to read podLogsRootDirectory %q: %v', podLogsRootDirectory, err)

10.        }

11.        for _, dir := range dirs {

12.            name := dir.Name()

13.            podUID := types.UID(name)

14.            if !cgc.isPodDeleted(podUID) {

15.                continue

16.            }

17.            err := osInterface.RemoveAll(filepath.Join(podLogsRootDirectory, name))

18.            if err != nil {

19.                glog.Errorf('Failed to remove pod logs directory %q: %v', name, err)

20.            }

21.        }

22.    }

23. 
    

24.    // Remove dead container log symlinks.

25.    // TODO(random-liu): Remove this after cluster logging supports CRI container log path.

26.    logSymlinks, _ := osInterface.Glob(filepath.Join(legacyContainerLogsDir, fmt.Sprintf('*.%s', legacyLogSuffix)))

27.    for _, logSymlink := range logSymlinks {

28.        if _, err := osInterface.Stat(logSymlink); os.IsNotExist(err) {

29.            err := osInterface.Remove(logSymlink)

30.            if err != nil {

31.                glog.Errorf('Failed to remove container log dead symlink %q: %v', logSymlink, err)

32.            }

33.        }

34.    }

35.    return nil

36. }

首先会读取/var/log/pods目录下面的子目录，下面的目录名称都是pod的uid，如果pod已经删除那么直接把pod所属的目录删除，然后删除/var/log/containers目录下的软连接。

至此单次ContainerGC的流程结束。

k8s为了回收机器上的存储资源同时有ImageGC对image资源进行回收

ImageGC同样定义了image gc manager和gc policy。

//pkg/kubelet/image/image_gc_manager.go
type ImageGCPolicy struct {
    // Any usage above this threshold will always trigger garbage collection.
    // This is the highest usage we will allow.
    HighThresholdPercent int

    // Any usage below this threshold will never trigger garbage collection.
    // This is the lowest threshold we will try to garbage collect to.
    LowThresholdPercent int

    // Minimum age at which an image can be garbage collected.
    MinAge time.Duration
}

type realImageGCManager struct {
    // Container runtime
    runtime container.Runtime

    // Records of images and their use.
    imageRecords     map[string]*imageRecord
    imageRecordsLock sync.Mutex

    // The image garbage collection policy in use.
    policy ImageGCPolicy

    // cAdvisor instance.
    cadvisor cadvisor.Interface

    // Recorder for Kubernetes events.
    recorder record.EventRecorder

    // Reference to this node.
    nodeRef *v1.ObjectReference

    // Track initialization
    initialized bool

    // imageCache is the cache of latest image list.
    imageCache imageCache
}

策略也主要有三个参数:

• HighThresholdPercent 高于此阈值将进行回收

• LowThresholdPercent 低于此阈值将不会触发回收

• MinAge 回收image的最小年龄

在这个文件里同样有一个GarbageCollect方法

1. func (im *realImageGCManager) detectImages(detectTime time.Time) error {

2.    images, err := im.runtime.ListImages()

3.    if err != nil {

4.        return err

5.    }

6.    pods, err := im.runtime.GetPods(true)

7.    if err != nil {

8.        return err

9.    }

10. 
    

11.    // Make a set of images in use by containers.

12.    imagesInUse := sets.NewString()

13.    for _, pod := range pods {

14.        for _, container := range pod.Containers {

15.            glog.V(5).Infof('Pod %s/%s, container %s uses image %s(%s)', pod.Namespace, pod.Name, container.Name, container.Image, container.ImageID)

16.            imagesInUse.Insert(container.ImageID)

17.        }

18.    }

19. 
    

20.    // Add new images and record those being used.

21.    now := time.Now()

22.    currentImages := sets.NewString()

23.    im.imageRecordsLock.Lock()

24.    defer im.imageRecordsLock.Unlock()

25.    for _, image := range images {

26.        glog.V(5).Infof('Adding image ID %s to currentImages', image.ID)

27.        currentImages.Insert(image.ID)

28. 
    

29.        // New image, set it as detected now.

30.        if _, ok := im.imageRecords[image.ID]; !ok {

31.            glog.V(5).Infof('Image ID %s is new', image.ID)

32.            im.imageRecords[image.ID] = &imageRecord{

33.                firstDetected: detectTime,

34.            }

35.        }

36. 
    

37.        // Set last used time to now if the image is being used.

38.        if isImageUsed(image, imagesInUse) {

39.            glog.V(5).Infof('Setting Image ID %s lastUsed to %v', image.ID, now)

40.            im.imageRecords[image.ID].lastUsed = now

41.        }

42. 
    

43.        glog.V(5).Infof('Image ID %s has size %d', image.ID, image.Size)

44.        im.imageRecords[image.ID].size = image.Size

45.    }

46. 
    

47.    // Remove old images from our records.

48.    for image := range im.imageRecords {

49.        if !currentImages.Has(image) {

50.            glog.V(5).Infof('Image ID %s is no longer present; removing from imageRecords', image)

51.            delete(im.imageRecords, image)

52.        }

53.    }

54. 
    

55.    return nil

56. }

57. 
    

58. func (im *realImageGCManager) GarbageCollect() error {

59.    // Get disk usage on disk holding images.

60.    fsInfo, err := im.cadvisor.ImagesFsInfo()

61.    if err != nil {

62.        return err

63.    }

64.    capacity := int64(fsInfo.Capacity)

65.    available := int64(fsInfo.Available)

66.    if available > capacity {

67.        glog.Warningf('available %d is larger than capacity %d', available, capacity)

68.        available = capacity

69.    }

70. 
    

71.    // Check valid capacity.

72.    if capacity == 0 {

73.        err := fmt.Errorf('invalid capacity %d on device %q at mount point %q', capacity, fsInfo.Device, fsInfo.Mountpoint)

74.        im.recorder.Eventf(im.nodeRef, v1.EventTypeWarning, events.InvalidDiskCapacity, err.Error())

75.        return err

76.    }

77. 
    

78.    // If over the max threshold, free enough to place us at the lower threshold.

79.    usagePercent := 100 - int(available*100/capacity)

80.    if usagePercent >= im.policy.HighThresholdPercent {

81.        amountToFree := capacity*int64(100-im.policy.LowThresholdPercent)/100 - available

82.        glog.Infof('[imageGCManager]: Disk usage on %q (%s) is at %d%% which is over the high threshold (%d%%). Trying to free %d bytes', fsInfo.Device, fsInfo.Mountpoint, usagePercent, im.policy.HighThresholdPercent, amountToFree)

83.        freed, err := im.freeSpace(amountToFree, time.Now())

84.        if err != nil {

85.            return err

86.        }

87. 
    

88.        if freed < amounttofree="">

89.            err := fmt.Errorf('failed to garbage collect required amount of images. Wanted to free %d bytes, but freed %d bytes', amountToFree, freed)

90.            im.recorder.Eventf(im.nodeRef, v1.EventTypeWarning, events.FreeDiskSpaceFailed, err.Error())

91.            return err

92.        }

93.    }

94. 
    

95.    return nil

96. }

97. 
    

98. func (im *realImageGCManager) DeleteUnusedImages() (int64, error) {

99.    return im.freeSpace(math.MaxInt64, time.Now())

100. }

101. 
     

102. // Tries to free bytesToFree worth of images on the disk.

103. //

104. // Returns the number of bytes free and an error if any occurred. The number of

105. // bytes freed is always returned.

106. // Note that error may be nil and the number of bytes free may be less

107. // than bytesToFree.

108. func (im *realImageGCManager) freeSpace(bytesToFree int64, freeTime time.Time) (int64, error) {

109.    err := im.detectImages(freeTime)

110.    if err != nil {

111.        return 0, err

112.    }

113. 
     

114.    im.imageRecordsLock.Lock()

115.    defer im.imageRecordsLock.Unlock()

116. 
     

117.    // Get all images in eviction order.

118.    images := make([]evictionInfo, 0, len(im.imageRecords))

119.    for image, record := range im.imageRecords {

120.        images = append(images, evictionInfo{

121.            id:          image,

122.            imageRecord: *record,

123.        })

124.    }

125.    sort.Sort(byLastUsedAndDetected(images))

126. 
     

127.    // Delete unused images until we've freed up enough space.

128.    var deletionErrors []error

129.    spaceFreed := int64(0)

130.    for _, image := range images {

131.        glog.V(5).Infof('Evaluating image ID %s for possible garbage collection', image.id)

132.        // Images that are currently in used were given a newer lastUsed.

133.        if image.lastUsed.Equal(freeTime) || image.lastUsed.After(freeTime) {

134.            glog.V(5).Infof('Image ID %s has lastUsed=%v which is >= freeTime=%v, not eligible for garbage collection', image.id, image.lastUsed, freeTime)

135.            break

136.        }

137. 
     

138.        // Avoid garbage collect the image if the image is not old enough.

139.        // In such a case, the image may have just been pulled down, and will be used by a container right away.

140. 
     

141.        if freeTime.Sub(image.firstDetected) <>MinAge {

142.            glog.V(5).Infof('Image ID %s has age %v which is less than the policy's minAge of %v, not eligible for garbage collection', image.id, freeTime.Sub(image.firstDetected), im.policy.MinAge)

143.            continue

144.        }

145. 
     

146.        // Remove image. Continue despite errors.

147.        glog.Infof('[imageGCManager]: Removing image %q to free %d bytes', image.id, image.size)

148.        err := im.runtime.RemoveImage(container.ImageSpec{Image: image.id})

149.        if err != nil {

150.            deletionErrors = append(deletionErrors, err)

151.            continue

152.        }

153.        delete(im.imageRecords, image.id)

154.        spaceFreed += image.size

155. 
     

156.        if spaceFreed >= bytesToFree {

157.            break

158.        }

159.    }

160. 
     

161.    if len(deletionErrors) > 0 {

162.        return spaceFreed, fmt.Errorf('wanted to free %d bytes, but freed %d bytes space with errors in image deletion: %v', bytesToFree, spaceFreed, errors.NewAggregate(deletionErrors))

163.    }

164.    return spaceFreed, nil

165. }

GarbageCollect方法也是在pkg/kubelet/kubelet.go里面进行调用，每调用一次执行一次回收流程，每次调用间隔是5分钟。 GarbageCollect首先会调用cadvisor获取监控机器上的资源总量可用量，当发现使用的资源已经大于设置的最高阈值将会触发回收行为，并且计算出需要回收的空间大小。

freeSpace方法则是真正的回收过程，过程分两步。

• 探测机器上的image

• 删除老的image

探测镜像: 

探测机器上的image是获取机器上的image列表并如果是正在使用的image标记最后使用时间和image大小和第一次探测到的时间，并把这个列表的image放到imageRecords里面进行缓存，imageRecords是个map结结构的数据类型。

回收镜像： 

回收镜像首先遍历imageRecords里面的镜像，放到一个数组里面排序，按照最后使用时间或者第一次探测到的时间进行排序。 下一步则对数组里面的镜像进行回收，如果镜像还在使用中或者最后探测到的时间小于设置的MinAge则不进行回收，否则删除镜像。回收过程中如果发现现在机器上的资源已经小于设置的LowThresholdPercent那么跳出回收流程。

原文：

https://www.jianshu.com/p/a6a6f6bab4a1

https://www.jianshu.com/p/2531c043cd70