otel collector pod is not up "Permanent error: HTTP 400 \"Bad Request\": entry with timestamp

[prashantnitt07]

"/var/log/pods/cortex_cortex-alertmanager-8549bc4cff-nmqdm_f4f096c4-cb6b-42ed-9d99-95cfc4e83c4c/alertmanager/0.log"}
2023-09-25T09:25:34.722Z error exporterhelper/queued_retry.go:391 Exporting failed. The error is not retryable. Dropping data. {"kind": "exporter", "data_type": "logs", "name": "loki", "error": "Permanent error: HTTP 400 "Bad Request": entry with timestamp 2023-09-25 07:52:43.865344266 +0000 UTC ignored, reason: 'entry too far behind, oldest acceptable timestamp is: 2023-09-25T08:24:02Z',", "dropped_items": 3666}

[prashantnitt07]

default:
envOverrides:

• name: 'OTEL_COLLECTOR_NAME'
  value: $(OTEL_K8S_NODE_NAME)
  opentelemetry-collector:
  mode: daemonset
  presets:
  logsCollection:
  enabled: true
  includeCollectorLogs: false
  config:
  extensions:
  basicauth/traces:
  client_auth:
  username: "${TRACES_USER_ID}"
  password: "${API_KEY}"
  basicauth/metrics:
  client_auth:
  username: "${METRICS_USER_ID}"
  password: "${API_KEY}"
  basicauth/logs:
  client_auth:
  username: "${LOGS_USER_ID}"
  password: "${API_KEY}"

  exporters:
  otlp:
  endpoint: ${TRACES_URL}
  tls:
  insecure: false
  auth:
  authenticator: basicauth/traces

  prometheusremotewrite:
  endpoint: ${METRICS_URL}
  auth:
  authenticator: basicauth/metrics

  loki:
  endpoint: ${LOGS_URL}
  auth:
  authenticator: basicauth/logs

  processors:
  spanmetrics:
  metrics_exporter: prometheusremotewrite

  service:
  extensions: [ basicauth/traces, basicauth/metrics, basicauth/logs, health_check , memory_ballast ]
  pipelines:
  metrics:
  exporters: [prometheusremotewrite]
  logs:
  receivers: [filelog]
  exporters: [loki]