[chore] add an error log for backpressure from upstream (open-telemet…

…ry#32790)

Adds an error log on error reported by the backend indicating that
backpressure is being applied.

exporter/splunkhecexporter/client.go

@@ -631,7 +631,7 @@ func (c *client) start(ctx context.Context, host component.Host) (err error) {
  }
  }
  url, _ := c.config.getURL()
- c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(c.config, c.buildInfo)}
+ c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(c.config, c.buildInfo), c.logger}
  c.heartbeater = newHeartbeater(c.config, c.buildInfo, getPushLogFn(c))
  if c.config.Heartbeat.Startup {
  if err := c.heartbeater.sendHeartbeat(c.config, c.buildInfo, getPushLogFn(c)); err != nil {

exporter/splunkhecexporter/client_test.go

@@ -1560,7 +1560,7 @@ func Test_pushLogData_InvalidLog(t *testing.T) {
 
 func Test_pushLogData_PostError(t *testing.T) {
  c := newLogsClient(exportertest.NewNopCreateSettings(), NewFactory().CreateDefaultConfig().(*Config))
- c.hecWorker = &defaultHecWorker{url: &url.URL{Host: "in va lid"}}
+ c.hecWorker = &defaultHecWorker{url: &url.URL{Host: "in va lid"}, logger: zap.NewNop()}
 
  // 2000 log records -> ~371888 bytes when JSON encoded.
  logs := createLogData(1, 1, 2000)
@@ -1605,7 +1605,7 @@ func Test_pushLogData_ShouldAddResponseTo400Error(t *testing.T) {
 
  // An HTTP client that returns status code 400 and response body responseBody.
  httpClient, _ := newTestClient(400, responseBody)
- splunkClient.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
+ splunkClient.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()}
  // Sending logs using the client.
  err := splunkClient.pushLogData(context.Background(), logs)
  require.True(t, consumererror.IsPermanent(err), "Expecting permanent error")
@@ -1615,7 +1615,7 @@ func Test_pushLogData_ShouldAddResponseTo400Error(t *testing.T) {
 
  // An HTTP client that returns some other status code other than 400 and response body responseBody.
  httpClient, _ = newTestClient(500, responseBody)
- splunkClient.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
+ splunkClient.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()}
  // Sending logs using the client.
  err = splunkClient.pushLogData(context.Background(), logs)
  require.False(t, consumererror.IsPermanent(err), "Expecting non-permanent error")
@@ -1638,7 +1638,7 @@ func Test_pushLogData_ShouldReturnUnsentLogsOnly(t *testing.T) {
 
  // The first record is to be sent successfully, the second one should not
  httpClient, _ := newTestClientWithPresetResponses([]int{200, 400}, []string{"OK", "NOK"})
- c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
+ c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()}
 
  err := c.pushLogData(context.Background(), logs)
  require.Error(t, err)
@@ -1665,7 +1665,7 @@ func Test_pushLogData_ShouldAddHeadersForProfilingData(t *testing.T) {
 
  httpClient, headers := newTestClient(200, "OK")
  url := &url.URL{Scheme: "http", Host: "splunk"}
- c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
+ c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()}
 
  err := c.pushLogData(context.Background(), logs)
  require.NoError(t, err)
@@ -1948,7 +1948,7 @@ func Test_pushLogData_Small_MaxContentLength(t *testing.T) {
  config.DisableCompression = disable
 
  c := newLogsClient(exportertest.NewNopCreateSettings(), config)
- c.hecWorker = &defaultHecWorker{&url.URL{Scheme: "http", Host: "splunk"}, http.DefaultClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
+ c.hecWorker = &defaultHecWorker{&url.URL{Scheme: "http", Host: "splunk"}, http.DefaultClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), zap.NewNop()}
 
  err := c.pushLogData(context.Background(), logs)
  require.Error(t, err)
@@ -2060,7 +2060,7 @@ func TestPushLogsPartialSuccess(t *testing.T) {
  // The first request succeeds, the second fails.
  httpClient, _ := newTestClientWithPresetResponses([]int{200, 503}, []string{"OK", "NOK"})
  url := &url.URL{Scheme: "http", Host: "splunk"}
- c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(cfg, component.NewDefaultBuildInfo())}
+ c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(cfg, component.NewDefaultBuildInfo()), zap.NewNop()}
 
  logs := plog.NewLogs()
  logRecords := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords()
@@ -2081,7 +2081,7 @@ func TestPushLogsRetryableFailureMultipleResources(t *testing.T) {
 
  httpClient, _ := newTestClientWithPresetResponses([]int{503}, []string{"NOK"})
  url := &url.URL{Scheme: "http", Host: "splunk"}
- c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(c.config, component.NewDefaultBuildInfo())}
+ c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(c.config, component.NewDefaultBuildInfo()), zap.NewNop()}
 
  logs := plog.NewLogs()
  logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty().Body().SetStr("log-1")

exporter/splunkhecexporter/hec_worker.go

@@ -10,6 +10,7 @@ import (
  "net/url"
 
  "go.opentelemetry.io/collector/consumer/consumererror"
+ "go.uber.org/zap"
 
  "github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk"
 )
@@ -22,6 +23,7 @@ type defaultHecWorker struct {
  url *url.URL
  client *http.Client
  headers map[string]string
+ logger *zap.Logger
 }
 
 func (hec *defaultHecWorker) send(ctx context.Context, buf buffer, headers map[string]string) error {
@@ -51,6 +53,10 @@ func (hec *defaultHecWorker) send(ctx context.Context, buf buffer, headers map[s
  }
  defer resp.Body.Close()
 
+ if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable {
+ hec.logger.Error("Splunk is unable to receive data. Please investigate the health of the cluster", zap.Int("status", resp.StatusCode), zap.String("host", hec.url.String()))
+ }
+
  err = splunk.HandleHTTPCode(resp)
  if err != nil {
  return err

exporter/splunkhecexporter/integration_test.go

@@ -245,7 +245,7 @@ func logsTest(t *testing.T, config *Config, url *url.URL, test testCfg) {
  }
 
  httpClient := createInsecureClient()
- c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
+ c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), settings.Logger}
 
  err := c.pushLogData(context.Background(), logs)
  require.NoError(t, err, "Must not error while sending Logs data")
@@ -268,7 +268,7 @@ func metricsTest(t *testing.T, config *Config, url *url.URL, test testCfg) {
  metricData := prepareMetricsData(test.config.event)
 
  httpClient := createInsecureClient()
- c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
+ c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), settings.Logger}
 
  err := c.pushMetricsData(context.Background(), metricData)
  require.NoError(t, err, "Must not error while sending Metrics data")
@@ -284,7 +284,7 @@ func tracesTest(t *testing.T, config *Config, url *url.URL, test testCfg) {
  tracesData := prepareTracesData(test.config.index, test.config.source, test.config.sourcetype)
 
  httpClient := createInsecureClient()
- c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo())}
+ c.hecWorker = &defaultHecWorker{url, httpClient, buildHTTPHeaders(config, component.NewDefaultBuildInfo()), settings.Logger}
 
  err := c.pushTraceData(context.Background(), tracesData)
  require.NoError(t, err, "Must not error while sending Trace data")