I am trying to configure Datadog agent on AKS Cluster and to read logs from file location at /var/log/datadog/messages.log in each service pod.
It is streaming all the metrics except logs from file and after checking the logs from Datadog agent it shows following message
2023-11-07 19:41:30 UTC | CORE | DEBUG | (pkg/logs/internal/launchers/kubernetes/launcher.go:200 in getSource) | annotation found: [{ "type": "file", "path": "/var/log/datadog/messages.log", "source": "camp-api", "service": "camp-api"}], for pod camp-api-b6758948f-wjfn8, container camp-api, but no config was usable for container log collection
Not sure what config I am missing here.
Here's my -- env output from datadog agent pod
Defaulted container "agent" out of: agent, trace-agent, process-agent, init-volume (init), init-config (init)
PATH=/opt/datadog-agent/bin/agent/:/opt/datadog-agent/embedded/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
HOSTNAME=datadog-6dg6l
DEBIAN_FRONTEND=noninteractive
DOCKER_DD_AGENT=true
DD_PYTHON_VERSION=3
CURL_CA_BUNDLE=/opt/datadog-agent/embedded/ssl/certs/cacert.pem
S6_KEEP_ENV=1
S6_LOGGING=0
S6_BEHAVIOUR_IF_STAGE2_FAILS=2
S6_READ_ONLY_ROOT=1
DD_SECRET_BACKEND_COMMAND_ALLOW_GROUP_EXEC_PERM=true
LD_PRELOAD=/lib/x86_64-linux-gnu/nosys.so
DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL=true
DD_LOGS_CONFIG_K8S_CONTAINER_USE_FILE=true
GODEBUG=x509ignoreCN=0
DD_CLUSTER_NAME=aks-test
DD_DOGSTATSD_PORT=8125
DD_LOGS_ENABLED=true
DD_ENV=aks-test
DD_LOG_LEVEL=DEBUG
DD_APM_ENABLED=false
DD_EXTRA_CONFIG_PROVIDERS=clusterchecks endpointschecks
KUBERNETES=yes
DD_KUBELET_CLIENT_CA=/etc/kubernetes/certs/kubeletserver.crt
DD_CLUSTER_AGENT_KUBERNETES_SERVICE_NAME=datadog-cluster-agent
DD_HEALTH_PORT=5555
DD_AUTH_TOKEN_FILE_PATH=/etc/datadog-agent/auth/token
DD_CONTAINER_EXCLUDE_LOGS=name:citadel name:galley name:cc-spring-boot-admin-server name:liveness-probe
DD_APM_IGNORE_RESOURCES=POST /instances,POST /api/v2/spans,GET /actuator,GET /.*/.*/actuator/.*
DD_PROCFS_PATH=/proc
DD_DOGSTATSD_NON_LOCAL_TRAFFIC=true
DD_CLUSTER_AGENT_ENABLED=true
DD_API_KEY=744657120b4aea61984f3870f1ee51eb
DD_KUBERNETES_KUBELET_HOST=xxx.xxx.0.10
DD_CLUSTER_AGENT_AUTH_TOKEN=5EEi5uT4wv00LcXVjq8MIvD5JLECUVwu
DD_IGNORE_AUTOCONF=kubernetes_state
DD_KUBELET_TLS_VERIFY=false
DD_LOGS_CONFIG_AUTO_MULTI_LINE_DETECTION=false
DD_DOGSTATSD_SOCKET=/var/run/datadog/dsd.socket
DD_EXPVAR_PORT=6000
DATADOG_SERVICE_PORT_DOGSTATSDPORT=8125
DATADOG_CLUSTER_AGENT_ADMISSION_CONTROLLER_SERVICE_PORT=443
DATADOG_CLUSTER_AGENT_ADMISSION_CONTROLLER_SERVICE_HOST=xxx.xxx.xxx.191
DATADOG_CLUSTER_AGENT_ADMISSION_CONTROLLER_PORT_443_TCP_PORT=443
KUBERNETES_PORT_443_TCP=tcp://xxx.xxx.0.1:443
KUBERNETES_PORT_443_TCP_PORT=443
KUBERNETES_PORT_443_TCP_ADDR=xxx.xxx.0.1
DATADOG_CLUSTER_AGENT_PORT_5005_TCP_PORT=5005
DATADOG_SERVICE_PORT_TRACEPORT=8126
DATADOG_CLUSTER_AGENT_ADMISSION_CONTROLLER_PORT_443_TCP=tcp://xxx.xxx.xxx.191:443
DATADOG_SERVICE_HOST=xxx.xxx.228.81
DATADOG_PORT_8125_UDP_PORT=8125
DATADOG_PORT_8125_UDP_ADDR=xxx.xxx.228.81
DATADOG_PORT_8126_TCP_PORT=8126
DATADOG_CLUSTER_AGENT_ADMISSION_CONTROLLER_PORT=tcp://xxx.xxx.xxx.191:443
DATADOG_CLUSTER_AGENT_PORT=tcp://xxx.xxx.60.205:5005
DATADOG_CLUSTER_AGENT_PORT_5005_TCP_PROTO=tcp
DATADOG_CLUSTER_AGENT_PORT_5005_TCP_ADDR=xxx.xxx.60.205
DATADOG_CLUSTER_AGENT_SERVICE_HOST=xxx.xxx.60.205
DATADOG_PORT_8125_UDP=udp://xxx.xxx.228.81:8125
DATADOG_PORT_8126_TCP=tcp://xxx.xxx.228.81:8126
DATADOG_CLUSTER_AGENT_ADMISSION_CONTROLLER_PORT_443_TCP_PROTO=tcp
KUBERNETES_SERVICE_HOST=xxx.xxx.0.1
KUBERNETES_SERVICE_PORT=443
KUBERNETES_PORT=tcp://xxx.xxx.0.1:443
DATADOG_PORT_8126_TCP_ADDR=xxx.xxx.228.81
DATADOG_CLUSTER_AGENT_ADMISSION_CONTROLLER_PORT_443_TCP_ADDR=xxx.xxx.xxx.191
KUBERNETES_SERVICE_PORT_HTTPS=443
KUBERNETES_PORT_443_TCP_PROTO=tcp
DATADOG_PORT_8125_UDP_PROTO=udp
DATADOG_SERVICE_PORT=8125
DATADOG_PORT=udp://xxx.xxx.228.81:8125
DATADOG_PORT_8126_TCP_PROTO=tcp
DATADOG_CLUSTER_AGENT_SERVICE_PORT=5005
DATADOG_CLUSTER_AGENT_SERVICE_PORT_AGENTPORT=5005
DATADOG_CLUSTER_AGENT_PORT_5005_TCP=tcp://xxx.xxx.60.205:5005
HOME=/root
Any help would be much appreciated!
