AWS Chatbot + Terraform で実装する CloudWatch エラーログの Slack 通知

{
  "level_name": "ERROR",
  "message": "request_error",
  "context": {
    "status": 500,
    "exception": {
      "class": "RuntimeException",
      "message": "Database connection failed",
      "trace": "..."
    }
  },
  "datetime": "2025-10-21T03:07:31.263879+00:00"
}

{ $.level_name = "ERROR" }

resource "aws_cloudwatch_log_metric_filter" "api_errors" {
  name           = "${var.name_prefix}-api-errors"
  log_group_name = var.api_log_group_name
  pattern        = "{ $.level_name = \"ERROR\" }"

  metric_transformation {
    name      = "APIErrorCount"
    namespace = "${var.name_prefix}/Logs"
    value     = "1"
    unit      = "Count"
  }
}

{
  "level_name": "ERROR",
  "message": "request_error",
  "context": {
    "status": 500,
    "exception": {
      "class": "RuntimeException",
      "message": "Database connection failed"
    }
  },
  "datetime": "2025-10-21T03:07:31.263879+00:00"
}


{
  "level_name": "ERROR",
  "message": "job_error",
  "context": {
    "job": "sample_batch_job",
    "exception": {
      "class": "ApiException",
      "message": "API rate limit exceeded"
    }
  },
  "datetime": "2025-10-21T03:05:37.791542+00:00"
}


{
  "level_name": "WARNING",
  "message": "slow_query",
  "context": {
    "query_time": 2.5
  },
  "datetime": "2025-10-21T03:08:15.123456+00:00"
}

{
  "level_name": "WARNING",
  "message": "ERROR",
  "context": {
    "query_time": 2.5
  },
  "datetime": "2025-10-21T03:08:15.123456+00:00"
}

# 厳密一致
{ $.level_name = "ERROR" }

# OR 条件
{ $.level_name = "ERROR" || $.level_name = "CRITICAL" }

# ワイルドカード
{ $.message = "database*" }

resource "aws_cloudwatch_metric_alarm" "api_errors" {
  alarm_name          = "${var.name_prefix}-api-high-error-rate"
  comparison_operator = "GreaterThanOrEqualToThreshold"
  evaluation_periods  = var.evaluation_periods
  metric_name         = "APIErrorCount"
  namespace           = "${var.name_prefix}/Logs"
  period              = var.period_seconds
  statistic           = "Sum"
  threshold           = var.error_threshold
  alarm_description   = "API error count exceeded ${var.error_threshold} in ${var.period_seconds} seconds"
  alarm_actions       = [aws_sns_topic.alerts.arn]
  ok_actions          = var.notify_on_ok ? [aws_sns_topic.alerts.arn] : []
  treat_missing_data  = "notBreaching"

  tags = {
    Name        = "${var.name_prefix}-api-high-error-rate"
    Environment = var.env
    Component   = "API"
  }
}

resource "aws_sns_topic" "alerts" {
  name         = "${var.name_prefix}-alerts"
  display_name = "CloudWatch Alerts for ${var.env}"

  tags = {
    Name        = "${var.name_prefix}-alerts"
    Environment = var.env
  }
}

resource "aws_sns_topic_policy" "alerts" {
  arn = aws_sns_topic.alerts.arn

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Sid    = "AllowCloudWatchAlarmsPublish"
        Effect = "Allow"
        Principal = {
          Service = "cloudwatch.amazonaws.com"
        }
        Action   = "SNS:Publish"
        Resource = aws_sns_topic.alerts.arn
      },
      {
        Sid    = "AllowChatbotSubscribe"
        Effect = "Allow"
        Principal = {
          Service = "chatbot.amazonaws.com"
        }
        Action = [
          "SNS:Subscribe",
        ]
        Resource = aws_sns_topic.alerts.arn
      }
    ]
  })
}

resource "aws_iam_role" "chatbot" {
  name = "${var.name_prefix}-chatbot-role"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Principal = {
          Service = "chatbot.amazonaws.com"
        }
        Action = "sts:AssumeRole"
      }
    ]
  })

  managed_policy_arns = [
    "arn:aws:iam::aws:policy/CloudWatchReadOnlyAccess"
  ]

  tags = {
    Name        = "${var.name_prefix}-chatbot-role"
    Environment = var.env
  }
}

resource "aws_iam_role_policy" "chatbot" {
  name = "${var.name_prefix}-chatbot-policy"
  role = aws_iam_role.chatbot.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Action = [
          "cloudwatch:Describe*",
          "cloudwatch:Get*",
          "cloudwatch:List*"
        ]
        Resource = "*"
      },
      {
        Effect = "Allow"
        Action = [
          "logs:DescribeLogGroups",
          "logs:DescribeLogStreams",
          "logs:GetLogEvents",
          "logs:FilterLogEvents"
        ]
        Resource = [
          "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:${var.api_log_group_name}:*",
          "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:${var.batch_log_group_name}:*"
        ]
      }
    ]
  })
}

resource "aws_chatbot_slack_channel_configuration" "alerts" {
  configuration_name = "${var.name_prefix}-slack-alerts"
  iam_role_arn       = aws_iam_role.chatbot.arn
  slack_channel_id   = var.slack_channel_id
  slack_team_id      = var.slack_workspace_id

  sns_topic_arns = [
    aws_sns_topic.alerts.arn
  ]

  logging_level = "INFO"

  tags = {
    Name        = "${var.name_prefix}-slack-alerts"
    Environment = var.env
  }
}

variable "name_prefix" {
  type        = string
  description = "Prefix for resource names"
}

variable "env" {
  type        = string
  description = "Environment name (dev/prd)"
}

variable "api_log_group_name" {
  type        = string
  description = "CloudWatch Log Group name for API"
}

variable "batch_log_group_name" {
  type        = string
  description = "CloudWatch Log Group name for Batch"
}

variable "error_threshold" {
  type        = number
  description = "Error count threshold for alarms"
  default     = 3
}

variable "evaluation_periods" {
  type        = number
  description = "Number of periods to evaluate"
  default     = 1
}

variable "period_seconds" {
  type        = number
  description = "Period in seconds for metric evaluation"
  default     = 60
}

variable "slack_workspace_id" {
  type        = string
  description = "Slack Workspace ID (Team ID) for AWS Chatbot integration"
  sensitive   = true

  validation {
    condition     = can(regex("^T[A-Z0-9]{8,}$", var.slack_workspace_id))
    error_message = "Slack Workspace ID must start with 'T' followed by at least 8 alphanumeric characters"
  }
}

variable "slack_channel_id" {
  type        = string
  description = "Slack Channel ID for notifications"
  sensitive   = true

  validation {
    condition     = can(regex("^C[A-Z0-9]{8,}$", var.slack_channel_id))
    error_message = "Slack Channel ID must start with 'C' followed by at least 8 alphanumeric characters"
  }
}

variable "notify_on_ok" {
  type        = bool
  description = "Send Slack notifications when alarms return to OK state"
  default     = true
}

name_prefix = "myapp-dev"
env         = "dev"


api_log_group_name   = "/ecs/api"
batch_log_group_name = "/ecs/batch"


error_threshold    = 3
evaluation_periods = 1
period_seconds     = 60





notify_on_ok = true

terraform init -upgrade


terraform plan

















terraform apply

🚨 CloudWatch Alarm | {作成したアラーム名} | ap-northeast-1 | Account: {アカウントID}
Threshold Crossed: no datapoints were received for 1 period and 1 missing datapoint was treated as [Breaching].

Namespace
sentinel-dev/Logs
Metric
BatchErrorCount
Timestamp
Tue, 28 Oct 2025 06:29:47 UTC
Alarm State
ALARM

✅ CloudWatch Alarm | {作成したアラーム名} | ap-northeast-1 | Account: {アカウントID}
Threshold Crossed: no datapoints were received for 1 period and 1 missing datapoint was treated as [NonBreaching].

Namespace
myapp/Logs
Metric
hErrorCount
Timestamp
Tue, 28 Oct 2025 06:56:36 UTC
Alarm State
OK

Error: Invalid resource type
│
│   on main.tf line 67, in resource "aws_chatbot_slack_channel_configuration" "alerts":
│   67: resource "aws_chatbot_slack_channel_configuration" "alerts" {
│
│ The provider hashicorp/aws does not support resource type "aws_chatbot_slack_channel_configuration".

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.61.0"  
    }
  }
}

Error: error creating Chatbot Slack Channel Configuration: InvalidParameterException:
Slack workspace T0123456789 is not authorized

pattern = "{ $.level_name = ERROR }"  

pattern = "{ $.level_name = \"ERROR\" }"  

aws logs filter-log-events \
  --log-group-name /ecs/api \
  --filter-pattern '{ $.level_name = "ERROR" }' \
  --limit 5

{
    "events": [
        {
            "logStreamName": "ecs/api/container-1",
            "timestamp": 1729568851263,
            "message": "{\"level_name\":\"ERROR\",\"message\":\"test_error\"}",
            "ingestionTime": 1729568851500,
            "eventId": "12345678901234567890"
        }
    ],
    "searchedLogStreams": [...]
}