GCS: periodically update cgroup limits

cmd/gcs/main.go

@@ -113,6 +113,92 @@ func readMemoryEvents(startTime time.Time, efdFile *os.File, cgName string, thre
 	}
 }
 
+type cgroupUpdater interface {
+	Update(resources *oci.LinuxResources) error
+}
+
+// Sanity checks the memory limit value
+func calculateContainersMemoryLimit(totalMemoryBytes, rootMemReserveBytes uint64) (int64, error) {
+	if totalMemoryBytes <= rootMemReserveBytes {
+		return 0, errors.Errorf("total memory %d must be greater than root memory reserve %d", totalMemoryBytes, rootMemReserveBytes)
+	}
+
+	limit := totalMemoryBytes - rootMemReserveBytes
+	const maxInt64 = uint64(1<<63 - 1)
+	if limit > maxInt64 {
+		return 0, errors.Errorf("containers cgroup memory limit %d exceeds maximum supported value %d", limit, maxInt64)
+	}
+	return int64(limit), nil
+}
+
+// Returns the total system memory in bytes.
+func currentTotalMemoryBytes() (uint64, error) {
+	sinfo := syscall.Sysinfo_t{}
+	if err := syscall.Sysinfo(&sinfo); err != nil {
+		return 0, errors.Wrap(err, "failed to get sys info")
+	}
+
+	totalMemoryBytes := uint64(sinfo.Totalram)
+	unit := uint64(sinfo.Unit)
+
+	// If the unit is non-zero, multiply the total memory by the unit to get the actual memory in bytes.
+	if unit != 0 {
+		const maxUint64 = ^uint64(0)
+		if totalMemoryBytes > maxUint64/unit {
+			return 0, errors.Errorf("total memory %d with unit %d exceeds maximum supported value", totalMemoryBytes, unit)
+		}
+		totalMemoryBytes *= unit
+	}
+	return totalMemoryBytes, nil
+}
+
+// Set the cgroup memory limits for the containers and virtual-pods cgroups (pass-through to setCGroupMemoryLimits)
+func updateCgroupMemoryLimits(podControl cgroupUpdater, rootMemReserveBytes uint64, lastAppliedLimit int64) (int64, bool, error) {
+	totalMemoryBytes, err := currentTotalMemoryBytes()
+	if err != nil {
+		return lastAppliedLimit, false, err
+	}
+	return setCgroupMemoryLimits(podControl, totalMemoryBytes, rootMemReserveBytes, lastAppliedLimit)
+}
+
+// Set the cgroup memory limits for the containers and virtual-pods cgroups if they've changed
+func setCgroupMemoryLimits(podControl cgroupUpdater, totalMemoryBytes, rootMemReserveBytes uint64, lastAppliedLimit int64) (int64, bool, error) {
+	podLimit, err := calculateContainersMemoryLimit(totalMemoryBytes, rootMemReserveBytes)
+	if err != nil {
+		return lastAppliedLimit, false, err
+	}
+	if podLimit == lastAppliedLimit {
+		return lastAppliedLimit, false, nil
+	}
+	resources := &oci.LinuxResources{
+		Memory: &oci.LinuxMemory{Limit: &podLimit},
+	}
+
+	if err := podControl.Update(resources); err != nil {
+		return lastAppliedLimit, false, errors.Wrap(err, "failed to update containers cgroup memory limit")
+	}
+	return podLimit, true, nil
+}
+
+// Start a periodic goroutine to monitor and update the cgroup memory limits for the containers and virtual-pods cgroups
+func monitorCgroupMemoryLimits(containersControl cgroupUpdater, rootMemReserveBytes uint64, initialLimit int64, interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	lastAppliedLimit := initialLimit
+	for range ticker.C {
+		podLimit, updated, err := updateCgroupMemoryLimits(containersControl, rootMemReserveBytes, lastAppliedLimit)
+		if err != nil {
+			logrus.WithError(err).Error("failed to refresh cgroup memory limits")
+			continue
+		}
+		if !updated {
+			continue
+		}
+		lastAppliedLimit = podLimit
+		logrus.WithField("memoryLimitBytes", podLimit).Debug("refreshed cgroup memory limits")
+	}
+}
+
 // runWithRestartMonitor starts a command with given args and waits for it to exit. If the
 // command exit code is non-zero the command is restarted with with some back off delay.
 // Any stdout or stderr of the command will be split into lines and written as a log with
@@ -351,11 +437,14 @@ func main() {
 	//
 	// The gcs cgroup is not limited but an event will get logged if memory
 	// usage exceeds 50 MB.
-	sinfo := syscall.Sysinfo_t{}
-	if err := syscall.Sysinfo(&sinfo); err != nil {
-		logrus.WithError(err).Fatal("failed to get sys info")
+	totalMemoryBytes, err := currentTotalMemoryBytes()
+	if err != nil {
+		logrus.WithError(err).Fatal("failed to get total system memory")
+	}
+	podsLimit, err := calculateContainersMemoryLimit(totalMemoryBytes, *rootMemReserveBytes)
+	if err != nil {
+		logrus.WithError(err).Fatal("failed to calculate pods cgroup memory limit")
 	}
-	podsLimit := int64(sinfo.Totalram - *rootMemReserveBytes)
 	podsControl, err := cgroup.NewManager("/pods", &oci.LinuxResources{
 		Memory: &oci.LinuxMemory{
 			Limit: &podsLimit,
@@ -383,6 +472,9 @@ func main() {
 
 	h := hcsv2.NewHost(rtime, tport, initialEnforcer, logWriter)
 
+	// Start periodic task to update cgroup memory limits for containers and virtual-pods cgroups
+	go monitorCgroupMemoryLimits(podsControl, *rootMemReserveBytes, podsLimit, 60*time.Second)
+
 	// During live migration the VM is frozen and only wakes up when the host
 	// shim is ready, so the vsock port should be immediately available. We
 	// use a tight retry interval instead of exponential backoff.

cmd/gcs/main_test.go

@@ -0,0 +1,105 @@
+//go:build linux
+// +build linux
+
+package main
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	oci "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+func TestCalculateContainersMemoryLimit(t *testing.T) {
+	tests := []struct {
+		name        string
+		total       uint64
+		reserve     uint64
+		want        int64
+		wantErrText string
+	}{
+		{name: "subtracts reserve", total: 4096, reserve: 1024, want: 3072},
+		{name: "rejects equal reserve", total: 1024, reserve: 1024, wantErrText: "must be greater"},
+		{name: "rejects below reserve", total: 512, reserve: 1024, wantErrText: "must be greater"},
+		{name: "rejects int64 overflow", total: uint64(1 << 63), wantErrText: "exceeds maximum"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := calculateContainersMemoryLimit(test.total, test.reserve)
+			if test.wantErrText != "" {
+				if err == nil || !strings.Contains(err.Error(), test.wantErrText) {
+					t.Fatalf("expected error containing %q, got %v", test.wantErrText, err)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("calculateContainersMemoryLimit returned error: %v", err)
+			}
+			if got != test.want {
+				t.Fatalf("calculateContainersMemoryLimit returned %d, want %d", got, test.want)
+			}
+		})
+	}
+}
+
+func TestSetCgroupMemoryLimitsUpdatesParentBeforeChild(t *testing.T) {
+	var updateOrder []string
+	podsControl := &testCgroupUpdater{name: "pods", updateOrder: &updateOrder}
+
+	limit, updated, err := setCgroupMemoryLimits(podsControl, 4096, 1024, 2048)
+	if err != nil {
+		t.Fatalf("setCgroupMemoryLimits returned error: %v", err)
+	}
+	if !updated {
+		t.Fatal("setCgroupMemoryLimits did not report an update")
+	}
+	if limit != 3072 {
+		t.Fatalf("setCgroupMemoryLimits returned limit %d, want 3072", limit)
+	}
+	if want := []string{"pods"}; !reflect.DeepEqual(updateOrder, want) {
+		t.Fatalf("cgroups updated in order %v, want %v", updateOrder, want)
+	}
+	if podsControl.limit != 3072 {
+		t.Fatalf("cgroup limits are pods=%d, want 3072", podsControl.limit)
+	}
+}
+
+func TestSetCgroupMemoryLimitsSkipsUnchangedLimit(t *testing.T) {
+	podsControl := &testCgroupUpdater{}
+
+	limit, updated, err := setCgroupMemoryLimits(podsControl, 4096, 1024, 3072)
+	if err != nil {
+		t.Fatalf("setCgroupMemoryLimits returned error: %v", err)
+	}
+	if updated {
+		t.Fatal("setCgroupMemoryLimits reported an update for an unchanged limit")
+	}
+	if limit != 3072 {
+		t.Fatalf("setCgroupMemoryLimits returned limit %d, want 3072", limit)
+	}
+	if podsControl.updates != 0 {
+		t.Fatalf("unchanged limit updated cgroups: pods=%d", podsControl.updates)
+	}
+}
+
+type testCgroupUpdater struct {
+	name        string
+	limit       int64
+	updates     int
+	updateErr   error
+	updateOrder *[]string
+}
+
+func (cgroup *testCgroupUpdater) Update(resources *oci.LinuxResources) error {
+	cgroup.updates++
+	if cgroup.updateErr != nil {
+		return cgroup.updateErr
+	}
+	cgroup.limit = *resources.Memory.Limit
+	if cgroup.updateOrder != nil {
+		*cgroup.updateOrder = append(*cgroup.updateOrder, cgroup.name)
+	}
+	return nil
+}