diff --git a/cgroup/cgroup.go b/cgroup/cgroup.go index 4938886..2786d45 100644 --- a/cgroup/cgroup.go +++ b/cgroup/cgroup.go @@ -187,6 +187,14 @@ func containerByCgroup(cgroupPath string) (ContainerType, string, error) { return ContainerTypeSandbox, "", nil } return ContainerTypeDocker, matches[1], nil + case prefix == "ecs" || prefix == "ecstasks.slice": + // ECS on EC2: /ecs// (cgroup v1) or + // /ecstasks.slice/ecstasks-.slice/docker-.scope (cgroup v2) + matches := dockerIdRegexp.FindStringSubmatch(cgroupPath) + if matches == nil { + return ContainerTypeUnknown, "", fmt.Errorf("invalid ecs cgroup %s", cgroupPath) + } + return ContainerTypeDocker, matches[1], nil case prefix == "system" || prefix == "podruntime": matches := talosIdRegexp.FindStringSubmatch(cgroupPath) if matches == nil { diff --git a/cgroup/cgroup_test.go b/cgroup/cgroup_test.go index 4ac82cd..66185e1 100644 --- a/cgroup/cgroup_test.go +++ b/cgroup/cgroup_test.go @@ -82,6 +82,18 @@ func TestNewFromProcessCgroupFile(t *testing.T) { assert.Equal(t, "/lxc/first", cg.ContainerId) assert.Equal(t, ContainerTypeLxc, cg.ContainerType) + cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/1100/cgroup")) // ECS on EC2, cgroup v1 + require.Nil(t, err) + assert.Equal(t, "/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b", cg.Id) + assert.Equal(t, "cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b", cg.ContainerId) + assert.Equal(t, ContainerTypeDocker, cg.ContainerType) + + cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/1200/cgroup")) // ECS on EC2, cgroup v2 + require.Nil(t, err) + assert.Equal(t, "/ecstasks.slice/ecstasks-9d7549331be24a6284b12238167e58b7.slice/docker-b2754197021709db2022ff6970befad8d0ee42553611290a00a7fd9ae65d3cc8.scope", cg.Id) + assert.Equal(t, "b2754197021709db2022ff6970befad8d0ee42553611290a00a7fd9ae65d3cc8", cg.ContainerId) + assert.Equal(t, ContainerTypeDocker, cg.ContainerType) + baseCgroupPath = "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podc83d0428_58af_41eb_8dba_b9e6eddffe7b.slice/docker-0e612005fd07e7f47e2cd07df99a2b4e909446814d71d0b5e4efc7159dd51252.scope" defer func() { baseCgroupPath = "" diff --git a/cgroup/fixtures/proc/1100/cgroup b/cgroup/fixtures/proc/1100/cgroup new file mode 100644 index 0000000..0084632 --- /dev/null +++ b/cgroup/fixtures/proc/1100/cgroup @@ -0,0 +1,11 @@ +11:memory:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b +10:blkio:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b +9:pids:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b +8:hugetlb:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b +7:freezer:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b +6:cpu,cpuacct:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b +5:perf_event:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b +4:devices:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b +3:net_cls,net_prio:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b +2:cpuset:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b +1:name=systemd:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b diff --git a/cgroup/fixtures/proc/1200/cgroup b/cgroup/fixtures/proc/1200/cgroup new file mode 100644 index 0000000..0176a16 --- /dev/null +++ b/cgroup/fixtures/proc/1200/cgroup @@ -0,0 +1 @@ +0::/ecstasks.slice/ecstasks-9d7549331be24a6284b12238167e58b7.slice/docker-b2754197021709db2022ff6970befad8d0ee42553611290a00a7fd9ae65d3cc8.scope diff --git a/containers/container.go b/containers/container.go index 13e7a47..6b50680 100644 --- a/containers/container.go +++ b/containers/container.go @@ -52,6 +52,7 @@ type ContainerMetadata struct { networks map[string]ContainerNetwork env map[string]string systemd SystemdProperties + ecsService string } type Delays struct { diff --git a/containers/ecs.go b/containers/ecs.go new file mode 100644 index 0000000..8c5b04e --- /dev/null +++ b/containers/ecs.go @@ -0,0 +1,79 @@ +package containers + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +const ( + ecsLabelCluster = "com.amazonaws.ecs.cluster" + ecsLabelContainerName = "com.amazonaws.ecs.container-name" + ecsLabelTaskArn = "com.amazonaws.ecs.task-arn" + ecsLabelTaskDefinitionFamily = "com.amazonaws.ecs.task-definition-family" + + ecsPauseContainerName = "~internal~ecs~pause" + + ecsMetadataURIEnv = "ECS_CONTAINER_METADATA_URI_V4" + ecsMetadataTimeout = 2 * time.Second + ecsMetadataRetryInterval = 30 * time.Second +) + +var ( + ecsMetadataClient = &http.Client{Timeout: ecsMetadataTimeout} + ecsMetadataRetryAfter time.Time +) + +func ecsEnrichMetadata(md *ContainerMetadata) error { + if md.labels[ecsLabelTaskArn] == "" || md.labels[ecsLabelContainerName] == ecsPauseContainerName { + return nil + } + uri := md.env[ecsMetadataURIEnv] + if uri == "" { + return nil + } + if now := time.Now(); now.Before(ecsMetadataRetryAfter) { + return fmt.Errorf("ECS task metadata endpoint is unavailable, next attempt in %s", ecsMetadataRetryAfter.Sub(now).Truncate(time.Second)) + } + resp, err := ecsMetadataClient.Get(uri + "/task") + if err != nil { + ecsMetadataRetryAfter = time.Now().Add(ecsMetadataRetryInterval) + return fmt.Errorf("failed to get ECS task metadata: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to get ECS task metadata: %s", resp.Status) + } + var task struct { + ServiceName string `json:"ServiceName"` + } + if err = json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&task); err != nil { + return fmt.Errorf("failed to decode ECS task metadata: %w", err) + } + md.ecsService = task.ServiceName + return nil +} + +func ecsContainerId(md *ContainerMetadata) ContainerID { + taskArn := md.labels[ecsLabelTaskArn] // arn:aws:ecs:::task// + name := md.labels[ecsLabelContainerName] + cluster := md.labels[ecsLabelCluster] + if taskArn == "" || name == "" || cluster == "" || name == ecsPauseContainerName { + return "" + } + taskId := taskArn[strings.LastIndex(taskArn, "/")+1:] + if taskId == "" { + return "" + } + service := md.ecsService + if service == "" { + service = md.labels[ecsLabelTaskDefinitionFamily] + } + if service == "" { + return "" + } + return ContainerID(fmt.Sprintf("/ecs/%s/%s/%s/%s", cluster, service, taskId, name)) +} diff --git a/containers/registry.go b/containers/registry.go index fbf1cd7..a664fa5 100644 --- a/containers/registry.go +++ b/containers/registry.go @@ -571,6 +571,9 @@ func calcId(cg *cgroup.Cgroup, md *ContainerMetadata) ContainerID { } return ContainerID(fmt.Sprintf("/k8s/%s/%s/%s", namespace, pod, name)) } + if md.labels[ecsLabelTaskArn] != "" { + return ecsContainerId(md) + } if taskNameParts := strings.SplitN(md.labels["com.docker.swarm.task.name"], ".", 3); len(taskNameParts) == 3 { namespace := md.labels["com.docker.stack.namespace"] service := md.labels["com.docker.swarm.service.name"] @@ -620,6 +623,9 @@ func getContainerMetadata(cg *cgroup.Cgroup) (*ContainerMetadata, error) { if dockerdClient != nil { md, err := DockerdInspect(cg.ContainerId) if err == nil { + if err = ecsEnrichMetadata(md); err != nil { + return nil, err + } return md, nil } dockerdErr = err