containers: add support for Amazon ECS on EC2

cgroup/cgroup.go

@@ -187,6 +187,14 @@ func containerByCgroup(cgroupPath string) (ContainerType, string, error) {
 			return ContainerTypeSandbox, "", nil
 		}
 		return ContainerTypeDocker, matches[1], nil
+	case prefix == "ecs" || prefix == "ecstasks.slice":
+		// ECS on EC2: /ecs/<task-id>/<container-id> (cgroup v1) or
+		// /ecstasks.slice/ecstasks-<task-id>.slice/docker-<container-id>.scope (cgroup v2)
+		matches := dockerIdRegexp.FindStringSubmatch(cgroupPath)
+		if matches == nil {
+			return ContainerTypeUnknown, "", fmt.Errorf("invalid ecs cgroup %s", cgroupPath)
+		}
+		return ContainerTypeDocker, matches[1], nil
 	case prefix == "system" || prefix == "podruntime":
 		matches := talosIdRegexp.FindStringSubmatch(cgroupPath)
 		if matches == nil {

cgroup/cgroup_test.go

@@ -82,6 +82,18 @@ func TestNewFromProcessCgroupFile(t *testing.T) {
 	assert.Equal(t, "/lxc/first", cg.ContainerId)
 	assert.Equal(t, ContainerTypeLxc, cg.ContainerType)
 
+	cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/1100/cgroup")) // ECS on EC2, cgroup v1
+	require.Nil(t, err)
+	assert.Equal(t, "/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b", cg.Id)
+	assert.Equal(t, "cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b", cg.ContainerId)
+	assert.Equal(t, ContainerTypeDocker, cg.ContainerType)
+
+	cg, err = NewFromProcessCgroupFile(path.Join("fixtures/proc/1200/cgroup")) // ECS on EC2, cgroup v2
+	require.Nil(t, err)
+	assert.Equal(t, "/ecstasks.slice/ecstasks-9d7549331be24a6284b12238167e58b7.slice/docker-b2754197021709db2022ff6970befad8d0ee42553611290a00a7fd9ae65d3cc8.scope", cg.Id)
+	assert.Equal(t, "b2754197021709db2022ff6970befad8d0ee42553611290a00a7fd9ae65d3cc8", cg.ContainerId)
+	assert.Equal(t, ContainerTypeDocker, cg.ContainerType)
+
 	baseCgroupPath = "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podc83d0428_58af_41eb_8dba_b9e6eddffe7b.slice/docker-0e612005fd07e7f47e2cd07df99a2b4e909446814d71d0b5e4efc7159dd51252.scope"
 	defer func() {
 		baseCgroupPath = ""

cgroup/fixtures/proc/1100/cgroup

@@ -0,0 +1,11 @@
+11:memory:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b
+10:blkio:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b
+9:pids:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b
+8:hugetlb:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b
+7:freezer:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b
+6:cpu,cpuacct:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b
+5:perf_event:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b
+4:devices:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b
+3:net_cls,net_prio:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b
+2:cpuset:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b
+1:name=systemd:/ecs/6b70638e857c4df284d52028bcdd6232/cc7924a92b43f3e453b9edf00b7a7350c7b620ec323350e755d8d15565c8ec2b

cgroup/fixtures/proc/1200/cgroup

@@ -0,0 +1 @@
+0::/ecstasks.slice/ecstasks-9d7549331be24a6284b12238167e58b7.slice/docker-b2754197021709db2022ff6970befad8d0ee42553611290a00a7fd9ae65d3cc8.scope

containers/container.go

@@ -52,6 +52,7 @@ type ContainerMetadata struct {
 	networks    map[string]ContainerNetwork
 	env         map[string]string
 	systemd     SystemdProperties
+	ecsService  string
 }
 
 type Delays struct {

containers/ecs.go

@@ -0,0 +1,79 @@
+package containers
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+)
+
+const (
+	ecsLabelCluster              = "com.amazonaws.ecs.cluster"
+	ecsLabelContainerName        = "com.amazonaws.ecs.container-name"
+	ecsLabelTaskArn              = "com.amazonaws.ecs.task-arn"
+	ecsLabelTaskDefinitionFamily = "com.amazonaws.ecs.task-definition-family"
+
+	ecsPauseContainerName = "~internal~ecs~pause"
+
+	ecsMetadataURIEnv        = "ECS_CONTAINER_METADATA_URI_V4"
+	ecsMetadataTimeout       = 2 * time.Second
+	ecsMetadataRetryInterval = 30 * time.Second
+)
+
+var (
+	ecsMetadataClient     = &http.Client{Timeout: ecsMetadataTimeout}
+	ecsMetadataRetryAfter time.Time
+)
+
+func ecsEnrichMetadata(md *ContainerMetadata) error {
+	if md.labels[ecsLabelTaskArn] == "" || md.labels[ecsLabelContainerName] == ecsPauseContainerName {
+		return nil
+	}
+	uri := md.env[ecsMetadataURIEnv]
+	if uri == "" {
+		return nil
+	}
+	if now := time.Now(); now.Before(ecsMetadataRetryAfter) {
+		return fmt.Errorf("ECS task metadata endpoint is unavailable, next attempt in %s", ecsMetadataRetryAfter.Sub(now).Truncate(time.Second))
+	}
+	resp, err := ecsMetadataClient.Get(uri + "/task")
+	if err != nil {
+		ecsMetadataRetryAfter = time.Now().Add(ecsMetadataRetryInterval)
+		return fmt.Errorf("failed to get ECS task metadata: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to get ECS task metadata: %s", resp.Status)
+	}
+	var task struct {
+		ServiceName string `json:"ServiceName"`
+	}
+	if err = json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&task); err != nil {
+		return fmt.Errorf("failed to decode ECS task metadata: %w", err)
+	}
+	md.ecsService = task.ServiceName
+	return nil
+}
+
+func ecsContainerId(md *ContainerMetadata) ContainerID {
+	taskArn := md.labels[ecsLabelTaskArn] // arn:aws:ecs:<region>:<account>:task/<cluster>/<task-id>
+	name := md.labels[ecsLabelContainerName]
+	cluster := md.labels[ecsLabelCluster]
+	if taskArn == "" || name == "" || cluster == "" || name == ecsPauseContainerName {
+		return ""
+	}
+	taskId := taskArn[strings.LastIndex(taskArn, "/")+1:]
+	if taskId == "" {
+		return ""
+	}
+	service := md.ecsService
+	if service == "" {
+		service = md.labels[ecsLabelTaskDefinitionFamily]
+	}
+	if service == "" {
+		return ""
+	}
+	return ContainerID(fmt.Sprintf("/ecs/%s/%s/%s/%s", cluster, service, taskId, name))
+}

containers/registry.go

@@ -571,6 +571,9 @@ func calcId(cg *cgroup.Cgroup, md *ContainerMetadata) ContainerID {
 		}
 		return ContainerID(fmt.Sprintf("/k8s/%s/%s/%s", namespace, pod, name))
 	}
+	if md.labels[ecsLabelTaskArn] != "" {
+		return ecsContainerId(md)
+	}
 	if taskNameParts := strings.SplitN(md.labels["com.docker.swarm.task.name"], ".", 3); len(taskNameParts) == 3 {
 		namespace := md.labels["com.docker.stack.namespace"]
 		service := md.labels["com.docker.swarm.service.name"]
@@ -620,6 +623,9 @@ func getContainerMetadata(cg *cgroup.Cgroup) (*ContainerMetadata, error) {
 	if dockerdClient != nil {
 		md, err := DockerdInspect(cg.ContainerId)
 		if err == nil {
+			if err = ecsEnrichMetadata(md); err != nil {
+				return nil, err
+			}
 			return md, nil
 		}
 		dockerdErr = err