Skip to content

Commit 8d70ec6

Browse files
bpareesbparees
committed
set cgroup parent on build child containers
1 parent bf857bd commit 8d70ec6

File tree

8 files changed

+155
-52
lines changed

8 files changed

+155
-52
lines changed

‎pkg/build/builder/common.go

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,10 @@ func execPostCommitHook(client DockerClient, postCommitSpec api.BuildPostCommitS
130130
if err != nil {
131131
return fmt.Errorf("read cgroup limits: %v", err)
132132
}
133+
parent, err := getCgroupParent()
134+
if err != nil {
135+
return fmt.Errorf("read cgroup parent: %v", err)
136+
}
133137

134138
return dockerRun(client, docker.CreateContainerOptions{
135139
Name: containerName,
@@ -140,11 +144,13 @@ func execPostCommitHook(client DockerClient, postCommitSpec api.BuildPostCommitS
140144
},
141145
HostConfig: &docker.HostConfig{
142146
// Limit container's resource allocation.
143-
CPUShares: limits.CPUShares,
144-
CPUPeriod: limits.CPUPeriod,
145-
CPUQuota: limits.CPUQuota,
146-
Memory: limits.MemoryLimitBytes,
147-
MemorySwap: limits.MemorySwap,
147+
// Though we are capped on memory and cpu at the cgroup parent level,
148+
// some build containers care what their memory limit is so they can
149+
// adapt, thus we need to set the memory limit at the container level
150+
// too, so that information is available to them.
151+
Memory: limits.MemoryLimitBytes,
152+
MemorySwap: limits.MemorySwap,
153+
CgroupParent: parent,
148154
},
149155
}, docker.AttachToContainerOptions{
150156
// Stream logs to stdout and stderr.

‎pkg/build/builder/docker.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -426,13 +426,16 @@ func (d *DockerBuilder) dockerBuild(dir string, tag string, secrets []api.Secret
426426
NetworkMode: string(getDockerNetworkMode()),
427427
}
428428

429+
// Though we are capped on memory and cpu at the cgroup parent level,
430+
// some build containers care what their memory limit is so they can
431+
// adapt, thus we need to set the memory limit at the container level
432+
// too, so that information is available to them.
429433
if d.cgLimits != nil {
430434
opts.Memory = d.cgLimits.MemoryLimitBytes
431435
opts.Memswap = d.cgLimits.MemorySwap
432-
opts.CPUShares = d.cgLimits.CPUShares
433-
opts.CPUPeriod = d.cgLimits.CPUPeriod
434-
opts.CPUQuota = d.cgLimits.CPUQuota
436+
opts.CgroupParent = d.cgLimits.Parent
435437
}
438+
436439
if auth != nil {
437440
opts.AuthConfigs = *auth
438441
}

‎pkg/build/builder/util.go

Lines changed: 44 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@ import (
77
"io/ioutil"
88
"math"
99
"os"
10-
"path/filepath"
1110
"regexp"
1211
"strconv"
1312
"strings"
1413

1514
docker "github.com/fsouza/go-dockerclient"
15+
"github.com/opencontainers/runc/libcontainer/cgroups"
1616

1717
s2iapi "github.com/openshift/source-to-image/pkg/api"
1818

@@ -88,49 +88,21 @@ func GetCGroupLimits() (*s2iapi.CGroupLimits, error) {
8888
byteLimit = 92233720368547
8989
}
9090

91-
// different docker versions seem to use different cgroup directories,
92-
// check for all of them.
93-
94-
// seen on rhel systems
95-
cpuDir := "/sys/fs/cgroup/cpuacct,cpu"
96-
97-
// seen on fedora systems with docker 1.9
98-
// note that in this case there is also a /sys/fs/cgroup/cpu that symlinks
99-
// to /sys/fs/cgroup/cpu,cpuacct, so technically the next check
100-
// would be sufficient, but it seems better to rely on the real directory
101-
// rather than a symlink.
102-
if _, err := os.Stat("/sys/fs/cgroup/cpu,cpuacct"); err == nil {
103-
cpuDir = "/sys/fs/cgroup/cpu,cpuacct"
104-
}
105-
106-
// seen on debian systems with docker 1.10
107-
if _, err := os.Stat("/sys/fs/cgroup/cpu"); err == nil {
108-
cpuDir = "/sys/fs/cgroup/cpu"
109-
}
110-
111-
cpuQuota, err := readInt64(filepath.Join(cpuDir, "cpu.cfs_quota_us"))
112-
if err != nil {
113-
return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
114-
}
115-
116-
cpuPeriod, err := readInt64(filepath.Join(cpuDir, "cpu.cfs_period_us"))
117-
if err != nil {
118-
return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
119-
}
120-
121-
cpuShares, err := readInt64(filepath.Join(cpuDir, "cpu.shares"))
91+
parent, err := getCgroupParent()
12292
if err != nil {
123-
return nil, fmt.Errorf("cannot determine cgroup limits: %v", err)
93+
return nil, fmt.Errorf("read cgroup parent: %v", err)
12494
}
12595

12696
return &s2iapi.CGroupLimits{
127-
CPUShares: cpuShares,
128-
CPUPeriod: cpuPeriod,
129-
CPUQuota: cpuQuota,
97+
// Though we are capped on memory and cpu at the cgroup parent level,
98+
// some build containers care what their memory limit is so they can
99+
// adapt, thus we need to set the memory limit at the container level
100+
// too, so that information is available to them.
130101
MemoryLimitBytes: byteLimit,
131102
// Set memoryswap==memorylimit, this ensures no swapping occurs.
132103
// see: https://docs.docker.com/engine/reference/run/#runtime-constraints-on-cpu-and-memory
133104
MemorySwap: byteLimit,
105+
Parent: parent,
134106
}, nil
135107
}
136108

@@ -201,3 +173,39 @@ func addBuildLabels(labels map[string]string, build *buildapi.Build) {
201173
labels[buildapi.DefaultDockerLabelNamespace+"build.name"] = build.Name
202174
labels[buildapi.DefaultDockerLabelNamespace+"build.namespace"] = build.Namespace
203175
}
176+
177+
// getCgroupParent determines the parent cgroup for a container from
178+
// within that container.
179+
func getCgroupParent() (string, error) {
180+
cgMap, err := cgroups.ParseCgroupFile("/proc/self/cgroup")
181+
if err != nil {
182+
return "", err
183+
}
184+
glog.V(6).Infof("found cgroup values map: %v", cgMap)
185+
return extractParentFromCgroupMap(cgMap)
186+
}
187+
188+
// extractParentFromCgroupMap finds the cgroup parent in the cgroup map
189+
func extractParentFromCgroupMap(cgMap map[string]string) (string, error) {
190+
memory, ok := cgMap["memory"]
191+
if !ok {
192+
return "", fmt.Errorf("could not find memory cgroup subsystem in map %v", cgMap)
193+
}
194+
glog.V(6).Infof("cgroup memory subsystem value: %s", memory)
195+
196+
parts := strings.Split(memory, "/")
197+
if len(parts) < 2 {
198+
return "", fmt.Errorf("unprocessable cgroup memory value: %s", memory)
199+
}
200+
201+
var cgroupParent string
202+
if strings.HasSuffix(memory, ".scope") {
203+
// systemd system, take the second to last segment.
204+
cgroupParent = parts[len(parts)-2]
205+
} else {
206+
// non-systemd, take everything except the last segment.
207+
cgroupParent = strings.Join(parts[:len(parts)-1], "/")
208+
}
209+
glog.V(5).Infof("running docker build under cgroup parent %v", cgroupParent)
210+
return cgroupParent, nil
211+
}

‎pkg/build/builder/util_test.go

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,3 +97,97 @@ func TestMergeEnv(t *testing.T) {
9797
}
9898
}
9999
}
100+
101+
type testcase struct {
102+
name string
103+
input map[string]string
104+
expect string
105+
fail bool
106+
}
107+
108+
func TestCGroupParentExtraction(t *testing.T) {
109+
tcs := []testcase{
110+
// with systemd
111+
{
112+
name: "systemd",
113+
input: map[string]string{
114+
"cpu": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
115+
"cpuacct": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
116+
"name=systemd": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
117+
"net_prio": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
118+
"freezer": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
119+
"blkio": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
120+
"net_cls": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
121+
"memory": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
122+
"hugetlb": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
123+
"perf_event": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
124+
"cpuset": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
125+
"devices": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
126+
"pids": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344.scope",
127+
},
128+
expect: "kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice",
129+
},
130+
// without systemd
131+
{
132+
name: "nosystemd",
133+
input: map[string]string{
134+
"cpu": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
135+
"cpuacct": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
136+
"name=systemd": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
137+
"net_prio": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
138+
"freezer": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
139+
"blkio": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
140+
"net_cls": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
141+
"memory": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
142+
"hugetlb": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
143+
"perf_event": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
144+
"cpuset": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
145+
"devices": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
146+
"pids": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
147+
},
148+
expect: "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice",
149+
},
150+
// no memory subsystem
151+
{
152+
name: "nomemory",
153+
input: map[string]string{
154+
"cpu": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
155+
"cpuacct": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
156+
"name=systemd": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
157+
"net_prio": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
158+
"freezer": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
159+
"blkio": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
160+
"net_cls": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
161+
"hugetlb": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
162+
"perf_event": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
163+
"cpuset": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
164+
"devices": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
165+
"pids": "/kubepods.slice/kubepods-podd0d034ed_5204_11e7_9710_507b9d27b5d9.slice/docker-a91b1981b6a4ae463fccd273e3f8665fd911e9abcaa1af27de773afb17ec4344",
166+
},
167+
expect: "",
168+
fail: true,
169+
},
170+
// unparseable
171+
{
172+
name: "unparseable",
173+
input: map[string]string{
174+
"memory": "kubepods.slice",
175+
},
176+
expect: "",
177+
fail: true,
178+
},
179+
}
180+
181+
for _, tc := range tcs {
182+
parent, err := extractParentFromCgroupMap(tc.input)
183+
if err != nil && !tc.fail {
184+
t.Errorf("[%s] unexpected exception: %v", tc.name, err)
185+
}
186+
if tc.fail && err == nil {
187+
t.Errorf("[%s] expected failure, did not get one and got cgroup parent=%s", tc.name, parent)
188+
}
189+
if parent != tc.expect {
190+
t.Errorf("[%s] expected cgroup parent= %s, got %s", tc.name, tc.expect, parent)
191+
}
192+
}
193+
}

‎test/extended/builds/docker_quota.go

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,6 @@ var _ = g.Describe("[builds][quota][Slow] docker build with a quota", func() {
5050
o.Expect(err).NotTo(o.HaveOccurred())
5151
o.Expect(out).To(o.ContainSubstring("MEMORY=209715200"))
5252
o.Expect(out).To(o.ContainSubstring("MEMORYSWAP=209715200"))
53-
o.Expect(out).To(o.ContainSubstring("SHARES=61"))
54-
o.Expect(out).To(o.ContainSubstring("PERIOD=100000"))
55-
o.Expect(out).To(o.ContainSubstring("QUOTA=6000"))
5653
})
5754
})
5855
})

‎test/extended/builds/s2i_quota.go

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,6 @@ var _ = g.Describe("[builds][Conformance] s2i build with a quota", func() {
5252
o.Expect(err).NotTo(o.HaveOccurred())
5353
o.Expect(buildLog).To(o.ContainSubstring("MEMORY=209715200"))
5454
o.Expect(buildLog).To(o.ContainSubstring("MEMORYSWAP=209715200"))
55-
o.Expect(buildLog).To(o.ContainSubstring("SHARES=61"))
56-
o.Expect(buildLog).To(o.ContainSubstring("PERIOD=100000"))
57-
o.Expect(buildLog).To(o.ContainSubstring("QUOTA=6000"))
5855

5956
events, err := oc.KubeClient().Core().Events(oc.Namespace()).Search(kapi.Scheme, br.Build)
6057
o.Expect(err).NotTo(o.HaveOccurred(), "Should be able to get events from the build")

‎test/extended/testdata/bindata.go

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎test/extended/testdata/test-docker-build-quota.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
"spec": {
1212
"resources": {
1313
"limits": {
14-
"cpu": "60m",
1514
"memory": "200Mi"
1615
}
1716
},

0 commit comments

Comments
 (0)