Send windows server 2016's metrics by cloudwatch agent

0

Hello mates,

I am working on observability. I have a Windows Server 2016, I installed a web application that produces logs. To do the observability, I turned to CloudWatch, to visualize logs and metrics. I installed a CloudWatch agent on the Windows server.

Here is the configuration file:

{
  "agent": {
    "metrics_collection_interval": 5,
    "logfile": "C:\\ProgramData\\Amazon\\AmazonCloudWatchAgent\\Logs\\amazon-cloudwatch-agent.log",
    "region": "eu-central-1",
    "debug": true
  },
  "logs": {
    "logs_collected": {
      "files": {
        "collect_list": [
          {
            "file_path": "C:\\ProgramData\\Amazon\\AmazonCloudWatchAgent\\Logs\\amazon-cloudwatch-agent.log",
            "log_group_name": "amazon-cloudwatch-agent-group-log.log",
            "log_stream_name": "amazon-cloudwatch-agent-stream-log.log",
            "timezone": "UTC"
          },
          {
            "file_path": "C:\\Users\\michael.ranivo\\Docuements\\Monitoring\\Middleware\\questions.txt",
            "log_group_name": "test-middleware-group-logs",
            "log_stream_name": "test-middleware-stream-logs",
            "timezone":"Local"
          }
        ]
      }
    },
    "force_flush_interval" : 5
  },
  "metrics": {
    "metrics_collected": {
      "namespace": "test-middleware-metrics",
      "statsd": {},
      "Processor": {
        "measurement": [
          {"name": "% Idle Time", "rename": "CPU_IDLE", "unit": "Percent"},
          "% Interrupt Time",
          "% User Time",
          "% Processor Time"
        ],
        "resources": [
          "*"
        ],
        "append_dimensions": {
          "d1": "win_foo",
          "d2": "win_bar"
        }
      },
      "LogicalDisk": {
        "measurement": [
          {"name": "% Idle Time", "unit": "Percent"},
          {"name": "% Disk Read Time", "rename": "DISK_READ"},
          "% Disk Write Time"
        ],
        "resources": [
          "*"
        ]
      },
      "Memory": {
        "metrics_collection_interval": 5,
        "measurement": [
          "Available Bytes",
          "Cache Faults/sec",
          "Page Faults/sec",
          "Pages/sec"
        ],
        "append_dimensions": {
          "d3": "win_bo"
        }
      },
      "Network Interface": {
        "metrics_collection_interval": 5,
        "measurement": [
          "Bytes Received/sec",
          "Bytes Sent/sec",
          "Packets Received/sec",
          "Packets Sent/sec"
        ],
        "resources": [
          "*"
        ],
        "append_dimensions": {
          "d3": "win_bo"
        }
      },
      "System": {
        "measurement": [
          "Context Switches/sec",
          "System Calls/sec",
          "Processor Queue Length"
        ],
        "append_dimensions": {
          "d1": "win_foo",
          "d2": "win_bar"
        }
      }
    },
    "append_dimensions": {
      "ImageId": "${aws:ImageId}",
      "InstanceId": "${aws:InstanceId}",
      "InstanceType": "${aws:InstanceType}",
      "AutoScalingGroupName": "${aws:AutoScalingGroupName}"
    },
    "aggregation_dimensions" : [["ImageId"], ["InstanceId", "InstanceType"], ["d1"],[]]
  }
}

When I launch the agent, with this command:

& "C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1" -a fetch-config -m onPremise -s -c file:"C:\\ProgramData\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent.json"

The agent launches well but when I look at the agent’s logs I have this:

2023/01/19 16:56:29 I! Config has been translated into TOML C:\ProgramData\Amazon\AmazonCloudWatchAgent\\amazon-cloudwatch-agent.toml 
2023/01/19 16:56:29 D! toml config [agent]
  collection_jitter = "0s"
  debug = true
  flush_interval = "1s"
  flush_jitter = "0s"
  hostname = ""
  interval = "5s"
  logfile = "C:\\ProgramData\\Amazon\\AmazonCloudWatchAgent\\Logs\\amazon-cloudwatch-agent.log"
  logtarget = "lumberjack"
  metric_batch_size = 1000
  metric_buffer_limit = 10000
  omit_hostname = false
  precision = ""
  quiet = false
  round_interval = false

[inputs]

  [[inputs.logfile]]
    destination = "cloudwatchlogs"
    file_state_folder = "C:\\ProgramData\\Amazon\\AmazonCloudWatchAgent\\Logs\\state"

    [[inputs.logfile.file_config]]
      file_path = "C:\\ProgramData\\Amazon\\AmazonCloudWatchAgent\\Logs\\amazon-cloudwatch-agent.log"
      from_beginning = true
      log_group_name = "server-perso-amazon-cloudwatch-agent-group-log.log"
      log_stream_name = "server-perso-amazon-cloudwatch-agent-stream-log.log"
      pipe = false
      retention_in_days = -1
      timezone = "UTC"

    [[inputs.logfile.file_config]]
      file_path = "C:\\Users\\leka\\Documents\\tests.txt"
      from_beginning = true
      log_group_name = "server-perso-test-middleware-group-logs"
      log_stream_name = "server-perso-test-middleware-stream-logs"
      pipe = false
      retention_in_days = -1
      timezone = "LOCAL"
    [inputs.logfile.tags]
      metricPath = "logs"

  [[inputs.statsd]]
    interval = "10s"
    parse_data_dog_tags = true
    service_address = ":8125"
    [inputs.statsd.tags]
      "aws:AggregationInterval" = "60s"
      metricPath = "metrics"

  [[inputs.win_perf_counters]]
    DisableReplacer = true

    [[inputs.win_perf_counters.object]]
      Counters = ["% Idle Time", "% Disk Read Time", "% Disk Write Time"]
      Instances = ["*"]
      Measurement = "LogicalDisk"
      ObjectName = "LogicalDisk"
      WarnOnMissing = true
    [inputs.win_perf_counters.tags]
      "aws:StorageResolution" = "true"
      metricPath = "metrics"

  [[inputs.win_perf_counters]]
    DisableReplacer = true
    interval = "5s"

    [[inputs.win_perf_counters.object]]
      Counters = ["Available Bytes", "Cache Faults/sec", "Page Faults/sec", "Pages/sec"]
      Instances = ["------"]
      Measurement = "Memory"
      ObjectName = "Memory"
      WarnOnMissing = true

    [[inputs.win_perf_counters.object]]
      Counters = ["Bytes Received/sec", "Bytes Sent/sec", "Packets Received/sec", "Packets Sent/sec"]
      Instances = ["*"]
      Measurement = "Network Interface"
      ObjectName = "Network Interface"
      WarnOnMissing = true
    [inputs.win_perf_counters.tags]
      "aws:StorageResolution" = "true"
      d3 = "win_bo"
      metricPath = "metrics"

  [[inputs.win_perf_counters]]
    DisableReplacer = true

    [[inputs.win_perf_counters.object]]
      Counters = ["% Idle Time", "% Interrupt Time", "% User Time", "% Processor Time"]
      Instances = ["*"]
      Measurement = "Processor"
      ObjectName = "Processor"
      WarnOnMissing = true

    [[inputs.win_perf_counters.object]]
      Counters = ["Context Switches/sec", "System Calls/sec", "Processor Queue Length"]
      Instances = ["------"]
      Measurement = "System"
      ObjectName = "System"
      WarnOnMissing = true
    [inputs.win_perf_counters.tags]
      "aws:StorageResolution" = "true"
      d1 = "win_foo"
      d2 = "win_bar"
      metricPath = "metrics"

[outputs]

  [[outputs.cloudwatch]]
    force_flush_interval = "60s"
    namespace = "server-perso-test-middleware-metrics"
    profile = "default"
    region = "eu-central-1"
    rollup_dimensions = [["ImageId"], ["InstanceId", "InstanceType"], ["d1"], []]
    shared_credential_file = "C:\\Users\\leka\\.aws\\credentials"
    tagexclude = ["host", "metricPath"]

    [[outputs.cloudwatch.metric_decoration]]
      category = "LogicalDisk"
      name = "% Idle Time"
      unit = "Percent"

    [[outputs.cloudwatch.metric_decoration]]
      category = "LogicalDisk"
      name = "% Disk Read Time"
      rename = "DISK_READ"

    [[outputs.cloudwatch.metric_decoration]]
      category = "Processor"
      name = "% Idle Time"
      rename = "CPU_IDLE"
      unit = "Percent"
    [outputs.cloudwatch.tagpass]
      metricPath = ["metrics"]

  [[outputs.cloudwatchlogs]]
    force_flush_interval = "5s"
    log_stream_name = "wind"
    profile = "default"
    region = "eu-central-1"
    shared_credential_file = "C:\\Users\\leka\\.aws\\credentials"
    tagexclude = ["metricPath"]
    [outputs.cloudwatchlogs.tagpass]
      metricPath = ["logs"]

[processors]

  [[processors.ec2tagger]]
    ec2_instance_tag_keys = ["aws:autoscaling:groupName"]
    ec2_metadata_tags = ["ImageId", "InstanceId", "InstanceType"]
    profile = "default"
    refresh_interval_seconds = "0s"
    shared_credential_file = "C:\\Users\\leka\\.aws\\credentials"
    [processors.ec2tagger.tagpass]
      metricPath = ["metrics"]
2023-01-19T15:56:29Z I! Starting AmazonCloudWatchAgent 1.247357.0
2023-01-19T15:56:29Z I! AWS SDK log level not set
2023-01-19T15:56:29Z I! Loaded inputs: logfile statsd win_perf_counters (3x)
2023-01-19T15:56:29Z I! Loaded aggregators: 
2023-01-19T15:56:29Z I! Loaded processors: ec2tagger
2023-01-19T15:56:29Z I! Loaded outputs: cloudwatch cloudwatchlogs
2023-01-19T15:56:29Z I! Tags enabled: host=wind
2023-01-19T15:56:29Z I! [agent] Config: Interval:5s, Quiet:false, Hostname:"wind", Flush Interval:1s
2023-01-19T15:56:29Z D! [agent] Initializing plugins
2023-01-19T15:56:29Z I! [processors.ec2tagger] ec2tagger: Check EC2 Metadata.
2023-01-19T15:56:29Z D! Successfully created credential sessions
2023-01-19T15:56:29Z I! [logagent] starting
2023-01-19T15:56:29Z I! [logagent] found plugin cloudwatchlogs is a log backend
2023-01-19T15:56:29Z I! [logagent] found plugin logfile is a log collection
2023-01-19T15:56:30Z D! [logagent] open file count, 0
2023-01-19T15:56:31Z D! [logagent] open file count, 0
2023-01-19T15:56:32Z D! [logagent] open file count, 0
2023-01-19T15:56:33Z D! [logagent] open file count, 0
2023-01-19T15:56:34Z D! [logagent] open file count, 0
2023-01-19T15:56:35Z D! [logagent] open file count, 0
2023-01-19T15:56:36Z D! [logagent] open file count, 0
2023-01-19T15:56:37Z D! [logagent] open file count, 0
2023-01-19T15:56:38Z D! [logagent] open file count, 0
2023-01-19T15:56:39Z D! [logagent] open file count, 0
2023-01-19T15:56:40Z D! [logagent] open file count, 0
2023-01-19T15:56:41Z D! [logagent] open file count, 0
2023-01-19T15:56:42Z D! [logagent] open file count, 0
2023-01-19T15:56:43Z D! [logagent] open file count, 0
2023-01-19T15:56:44Z D! [logagent] open file count, 0
2023-01-19T15:56:45Z D! [logagent] open file count, 0
2023-01-19T15:56:46Z D! [logagent] open file count, 0
2023-01-19T15:56:47Z D! [logagent] open file count, 0
2023-01-19T15:56:48Z D! [logagent] open file count, 0
2023-01-19T15:56:49Z D! [logagent] open file count, 0
2023-01-19T15:56:50Z D! [logagent] open file count, 0
2023-01-19T15:56:51Z D! [logagent] open file count, 0
2023-01-19T15:56:52Z D! [logagent] open file count, 0
2023-01-19T15:56:54Z D! [logagent] open file count, 0
2023-01-19T15:56:54Z D! [logagent] open file count, 0
2023-01-19T15:56:55Z D! [logagent] open file count, 0
2023-01-19T15:56:56Z D! [logagent] open file count, 0
2023-01-19T15:56:57Z D! [logagent] open file count, 0
2023-01-19T15:56:58Z D! [logagent] open file count, 0
2023-01-19T15:56:59Z D! [logagent] open file count, 0
2023-01-19T15:56:59Z I! CWAGENT_LOG_LEVEL is set to "DEBUG"
2023-01-19T15:57:00Z D! [logagent] open file count, 0
2023-01-19T15:57:01Z D! [logagent] open file count, 0
2023-01-19T15:57:02Z D! [logagent] open file count, 0
2023-01-19T15:57:03Z D! [logagent] open file count, 0
2023-01-19T15:57:04Z D! [logagent] open file count, 0
2023-01-19T15:57:05Z D! [logagent] open file count, 0
2023-01-19T15:57:06Z D! [logagent] open file count, 0
2023-01-19T15:57:07Z D! [logagent] open file count, 0
2023-01-19T15:59:18Z E! Failed to get credential from session: NoCredentialProviders: no valid providers in chain
caused by: EnvAccessKeyNotFound: failed to find credentials in the environment.
SharedCredsLoad: failed to load profile, .
EC2RoleRequestError: no EC2 instance role found
caused by: RequestError: send request failed
caused by: Get "http://169.254.169.254/latest/meta-data/iam/security-credentials/": dial tcp 169.254.169.254:80: connectex: Une tentative de connexion a échoué car le parti connecté n’a pas répondu convenablement au-delà d’une certaine durée ou une connexion établie a échoué car l’hôte de connexion n’a pas répondu.
2023-01-19T15:59:18Z D! [logagent] open file count, 0
2023-01-19T15:59:19Z D! [logagent] open file count, 0
2023-01-19T15:59:20Z D! [logagent] open file count, 0
2023-01-19T15:59:21Z D! [logagent] open file count, 0
2023-01-19T15:59:22Z D! [logagent] open file count, 0
2023-01-19T15:59:23Z D! [logagent] open file count, 0
2023-01-19T15:59:24Z D! [logagent] open file count, 0
2023-01-19T15:59:25Z E! [processors.ec2tagger] ec2tagger: Unable to retrieve EC2 Metadata. This plugin must only be used on an EC2 instance.
2023-01-19T15:59:25Z E! [telegraf] Error running agent: could not initialize processor processors.ec2tagger: EC2MetadataRequestError: failed to get EC2 instance identity document
caused by: RequestError: send request failed
caused by: Get "http://169.254.169.254/latest/dynamic/instance-identity/document": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
2023/01/19 16:59:25 E! Error when starting Agent, Error is exit status 1 

I removed the metric part from the configuration file and this worked.

I have no idea how my agent cloudwatch can send the metrics to cloudwatch.

Do you have any idea?

Thank you in advance for what I read.

asked a year ago563 views
1 Answer
0

From the error message in the log file, it is clear that the CloudWatch Agent was unable to resolve the AWS credentials. The original error message in French also suggests that the Agent was trying to access the metadata service (IMDS) that should only be the case for an EC2 instance, not for an on-premises server.

You should specify the IAM credentials and the AWS Region the logs should be sent to in a configuration file. Please refer to the CloudWatch Agent documentation.

You can also use temporary credentials to improve security. Refer to this Knowledge Base article.

Most likely, your Agent config enables the ec2tagger plugin which can only be used on an EC2 instance, not on an on-premises server. Please make sure you configure your Agent properly according to the guidance referenced above.

profile pictureAWS
answered a year ago

You are not logged in. Log in to post an answer.

A good answer clearly answers the question and provides constructive feedback and encourages professional growth in the question asker.

Guidelines for Answering Questions