Day 3 - CloudWatch Agent: 내부 동작, StatsD/collectd 통합, procstat 플러그인

{
  "agent": {
    "metrics_collection_interval": 60,
    "run_as_user": "cwagent",
    "logfile": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log",
    "debug": false
  },
  "metrics": {
    "namespace": "MyApp/EC2",
    "append_dimensions": {
      "InstanceId": "${aws:InstanceId}",
      "AutoScalingGroupName": "${aws:AutoScalingGroupName}",
      "ImageId": "${aws:ImageId}"
    },
    "aggregation_dimensions": [["AutoScalingGroupName"]],
    "metrics_collected": {
      "mem": {
        "measurement": ["mem_used_percent", "mem_available_percent", "mem_cached"],
        "metrics_collection_interval": 60
      },
      "disk": {
        "resources": ["/", "/var", "/home"],
        "measurement": ["used_percent", "inodes_free", "disk_free"],
        "ignore_file_system_types": ["sysfs", "devtmpfs", "tmpfs"],
        "metrics_collection_interval": 60
      },
      "cpu": {
        "totalcpu": true,
        "per_cpu": false,
        "measurement": ["usage_user", "usage_system", "usage_iowait", "usage_steal"],
        "metrics_collection_interval": 60
      },
      "procstat": [
        {
          "exe": "nginx",
          "measurement": ["cpu_usage", "memory_rss", "memory_vms",
                          "num_threads", "read_bytes", "write_bytes"],
          "metrics_collection_interval": 60
        },
        {
          "exe": "java",
          "measurement": ["cpu_usage", "memory_rss", "num_fds"],
          "metrics_collection_interval": 60
        }
      ],
      "statsd": {
        "service_address": ":8125",
        "metrics_collection_interval": 10,
        "metrics_aggregation_interval": 60
      },
      "collectd": {
        "service_address": "udp://127.0.0.1:25826",
        "name_prefix": "collectd_"
      }
    }
  },
  "logs": {
    "logs_collected": {
      "files": {
        "collect_list": [
          {
            "file_path": "/var/log/myapp/app.log",
            "log_group_name": "/prod/myapp/app",
            "log_stream_name": "{instance_id}",
            "timezone": "UTC",
            "timestamp_format": "%Y-%m-%dT%H:%M:%S.%f",
            "multi_line_start_pattern": "^\\d{4}-\\d{2}-\\d{2}",
            "retention_in_days": 90
          },
          {
            "file_path": "/var/log/nginx/error.log",
            "log_group_name": "/prod/nginx/error",
            "log_stream_name": "{instance_id}",
            "retention_in_days": 30
          }
        ]
      },
      "windows_events": {
        "collect_list": [
          {
            "event_name": "System",
            "event_levels": ["ERROR", "CRITICAL"],
            "log_group_name": "/windows/system-events",
            "log_stream_name": "{instance_id}"
          }
        ]
      }
    }
  }
}

"procstat": [
  {
    "exe": "nginx",
    "measurement": ["cpu_usage", "memory_rss", "num_threads", "read_bytes", "write_bytes"]
  },
  {
    "pid_file": "/var/run/myapp.pid",
    "measurement": ["cpu_usage", "memory_rss", "num_fds", "involuntary_context_switches"]
  },
  {
    "pattern": "python.*worker",
    "measurement": ["cpu_usage", "memory_rss"]
  }
]

# Python 애플리케이션에서 StatsD 사용 예시
import statsd
c = statsd.StatsClient('localhost', 8125, prefix='payment')
 
def process_payment(amount):
    with c.timer('latency'):  # 처리 시간 측정
        result = do_payment(amount)
    if result.success:
        c.incr('success')       # 성공 카운터
        c.gauge('amount', amount)  # 최근 결제 금액
    else:
        c.incr('failure')
    return result

# 1단계: 설정을 SSM Parameter Store에 저장
aws ssm put-parameter \
  --name "/cloudwatch/agent/prod-web" \
  --type String \
  --tier Advanced \
  --value file://amazon-cloudwatch-agent.json \
  --description "Web tier CloudWatch Agent configuration"
 
# 2단계: 초기 설치 (Run Command)
aws ssm send-command \
  --document-name "AWS-ConfigureAWSPackage" \
  --parameters 'action=Install,name=AmazonCloudWatchAgent,version=latest' \
  --targets "Key=tag:Role,Values=web-prod" \
  --comment "Install CloudWatch Agent on web-prod"
 
# 3단계: 설정 적용 (Run Command)
aws ssm send-command \
  --document-name "AmazonCloudWatch-ManageAgent" \
  --parameters '{
    "action": ["configure"],
    "mode": ["ec2"],
    "optionalConfigurationSource": ["ssm"],
    "optionalConfigurationLocation": ["/cloudwatch/agent/prod-web"],
    "optionalRestart": ["yes"]
  }' \
  --targets "Key=tag:Role,Values=web-prod"
 
# 4단계: State Manager Association으로 drift 자동 교정
aws ssm create-association \
  --name "AmazonCloudWatch-ManageAgent" \
  --association-name "CWAgent-WebProd-Maintain" \
  --targets "Key=tag:Role,Values=web-prod" \
  --parameters '{
    "action": ["configure"],
    "mode": ["ec2"],
    "optionalConfigurationSource": ["ssm"],
    "optionalConfigurationLocation": ["/cloudwatch/agent/prod-web"],
    "optionalRestart": ["yes"]
  }' \
  --schedule-expression "rate(24 hours)" \
  --compliance-severity "MEDIUM"

# EC2 IAM Role에 부여할 관리형 정책
arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy
arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore

# Agent 상태 확인
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a status
 
# Agent 로그 (오류 원인의 90%가 여기에 있음)
sudo tail -f /opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log
 
# 시간 동기화 상태 (시간이 틀리면 메트릭 거부됨)
chronyc tracking
timedatectl show
 
# IAM 권한 테스트 (인스턴스에서 직접 실행)
aws cloudwatch put-metric-data \
  --namespace "Test/Agent" \
  --metric-name "TestMetric" \
  --value 1 \
  --region ap-northeast-2

# 메모리 85% 초과 3 of 5 알람
aws cloudwatch put-metric-alarm \
  --alarm-name "EC2-Memory-High-WebProd" \
  --metric-name mem_used_percent \
  --namespace CWAgent \
  --dimensions \
    Name=InstanceId,Value=i-0123456789abcdef0 \
    Name=AutoScalingGroupName,Value=web-prod-asg \
  --period 60 \
  --statistic Average \
  --threshold 85 \
  --comparison-operator GreaterThanThreshold \
  --evaluation-periods 5 \
  --datapoints-to-alarm 3 \
  --treat-missing-data notBreaching \
  --alarm-actions arn:aws:sns:ap-northeast-2:123456789012:ops-critical