diff --git a/cmd/configschema/go.sum b/cmd/configschema/go.sum index a25a7b9bc86c9..a5df0d26bcc15 100644 --- a/cmd/configschema/go.sum +++ b/cmd/configschema/go.sum @@ -1261,8 +1261,6 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oapi-codegen/runtime v1.0.0 h1:P4rqFX5fMFWqRzY9M/3YF9+aPSPPB06IzP2P7oOxrWo= github.com/oapi-codegen/runtime v1.0.0/go.mod h1:LmCUMQuPB4M/nLXilQXhHw+BLZdDb18B34OO356yJ/A= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc h1:49ewVBwLcy+eYqI4R0ICilCI4dPjddpFXWv3liXzUxM= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc/go.mod h1:WXIHwGy+c7/IK2PiJ4oxuTHkpnkSut7TNFpKnI5llPU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= diff --git a/cmd/otelcontribcol/go.sum b/cmd/otelcontribcol/go.sum index 3351f940e7607..72cb49aa8030d 100644 --- a/cmd/otelcontribcol/go.sum +++ b/cmd/otelcontribcol/go.sum @@ -1256,8 +1256,6 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oapi-codegen/runtime v1.0.0 h1:P4rqFX5fMFWqRzY9M/3YF9+aPSPPB06IzP2P7oOxrWo= github.com/oapi-codegen/runtime v1.0.0/go.mod h1:LmCUMQuPB4M/nLXilQXhHw+BLZdDb18B34OO356yJ/A= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc h1:49ewVBwLcy+eYqI4R0ICilCI4dPjddpFXWv3liXzUxM= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc/go.mod h1:WXIHwGy+c7/IK2PiJ4oxuTHkpnkSut7TNFpKnI5llPU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= diff --git a/cmd/oteltestbedcol/go.sum b/cmd/oteltestbedcol/go.sum index f7aaa4d18b3f4..e16f8585e18a4 100644 --- a/cmd/oteltestbedcol/go.sum +++ b/cmd/oteltestbedcol/go.sum @@ -515,8 +515,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc h1:49ewVBwLcy+eYqI4R0ICilCI4dPjddpFXWv3liXzUxM= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc/go.mod h1:WXIHwGy+c7/IK2PiJ4oxuTHkpnkSut7TNFpKnI5llPU= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= diff --git a/exporter/datadogexporter/go.sum b/exporter/datadogexporter/go.sum index 08bfb3b54d02c..43fc133a97680 100644 --- a/exporter/datadogexporter/go.sum +++ b/exporter/datadogexporter/go.sum @@ -595,8 +595,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc h1:49ewVBwLcy+eYqI4R0ICilCI4dPjddpFXWv3liXzUxM= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc/go.mod h1:WXIHwGy+c7/IK2PiJ4oxuTHkpnkSut7TNFpKnI5llPU= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/go.sum b/go.sum index b596ba352c45a..39069f7a20d1e 100644 --- a/go.sum +++ b/go.sum @@ -1261,8 +1261,6 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oapi-codegen/runtime v1.0.0 h1:P4rqFX5fMFWqRzY9M/3YF9+aPSPPB06IzP2P7oOxrWo= github.com/oapi-codegen/runtime v1.0.0/go.mod h1:LmCUMQuPB4M/nLXilQXhHw+BLZdDb18B34OO356yJ/A= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc h1:49ewVBwLcy+eYqI4R0ICilCI4dPjddpFXWv3liXzUxM= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc/go.mod h1:WXIHwGy+c7/IK2PiJ4oxuTHkpnkSut7TNFpKnI5llPU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= diff --git a/pkg/stanza/fileconsumer/internal/emittest/sink.go b/pkg/stanza/fileconsumer/internal/emittest/sink.go index 1785bfb8bfb17..7f5fd77d367b6 100644 --- a/pkg/stanza/fileconsumer/internal/emittest/sink.go +++ b/pkg/stanza/fileconsumer/internal/emittest/sink.go @@ -111,7 +111,7 @@ func (s *Sink) ExpectTokens(t *testing.T, expected ...[]byte) { case call := <-s.emitChan: actual = append(actual, call.Token) case <-time.After(s.timeout): - assert.Fail(t, "Timed out waiting for message") + assert.Fail(t, fmt.Sprintf("timeout: expected: %d, actual: %d", len(expected), i)) return } } @@ -135,7 +135,7 @@ func (s *Sink) ExpectCalls(t *testing.T, expected ...*Call) { case call := <-s.emitChan: actual = append(actual, call) case <-time.After(s.timeout): - assert.Fail(t, "Timed out waiting for message") + assert.Fail(t, fmt.Sprintf("timeout: expected: %d, actual: %d", len(expected), i)) return } } diff --git a/pkg/stanza/fileconsumer/rotation_test.go b/pkg/stanza/fileconsumer/rotation_test.go index 288ad58a1fbec..c302f8e276177 100644 --- a/pkg/stanza/fileconsumer/rotation_test.go +++ b/pkg/stanza/fileconsumer/rotation_test.go @@ -7,7 +7,6 @@ import ( "context" "fmt" "io" - "log" "os" "path/filepath" "runtime" @@ -15,85 +14,22 @@ import ( "testing" "time" - "github.com/observiq/nanojack" "github.com/stretchr/testify/require" - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/emittest" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/internal/filetest" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/testutil" ) const windowsOS = "windows" -func TestMultiFileRotate(t *testing.T) { +func TestCopyTruncate(t *testing.T) { if runtime.GOOS == windowsOS { - // Windows has very poor support for moving active files, so rotation is less commonly used - t.Skip() - } - t.Parallel() - - getMessage := func(f, k, m int) string { return fmt.Sprintf("file %d-%d, message %d", f, k, m) } - - tempDir := t.TempDir() - cfg := NewConfig().includeDir(tempDir) - cfg.StartAt = "beginning" - operator, sink := testManager(t, cfg) - - numFiles := 3 - numMessages := 3 - numRotations := 3 - - expected := make([][]byte, 0, numFiles*numMessages*numRotations) - for i := 0; i < numFiles; i++ { - for j := 0; j < numMessages; j++ { - for k := 0; k < numRotations; k++ { - expected = append(expected, []byte(getMessage(i, k, j))) - } - } - } - - require.NoError(t, operator.Start(testutil.NewUnscopedMockPersister())) - defer func() { - require.NoError(t, operator.Stop()) - }() - - temps := make([]*os.File, 0, numFiles) - for i := 0; i < numFiles; i++ { - temps = append(temps, filetest.OpenTemp(t, tempDir)) - } - - var wg sync.WaitGroup - for i, temp := range temps { - wg.Add(1) - go func(tf *os.File, f int) { - defer wg.Done() - for k := 0; k < numRotations; k++ { - for j := 0; j < numMessages; j++ { - filetest.WriteString(t, tf, getMessage(f, k, j)+"\n") - } - - require.NoError(t, tf.Close()) - require.NoError(t, os.Rename(tf.Name(), fmt.Sprintf("%s.%d", tf.Name(), k))) - tf = filetest.ReopenTemp(t, tf.Name()) - } - }(temp, i) - } - - sink.ExpectTokens(t, expected...) - wg.Wait() -} - -func TestMultiFileRotateSlow(t *testing.T) { - if runtime.GOOS == windowsOS { - // Windows has very poor support for moving active files, so rotation is less commonly used - t.Skip() + t.Skip("Rotation tests have been flaky on Windows. See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16331") } - - t.Parallel() - tempDir := t.TempDir() cfg := NewConfig().includeDir(tempDir) cfg.StartAt = "beginning" + cfg.PollInterval = 10 * time.Millisecond operator, sink := testManager(t, cfg) getMessage := func(f, k, m int) string { return fmt.Sprintf("file %d-%d, message %d", f, k, m) } @@ -101,7 +37,7 @@ func TestMultiFileRotateSlow(t *testing.T) { baseFileName := func(f int) string { return filepath.Join(tempDir, fmt.Sprintf("file%d.log", f)) } numFiles := 3 - numMessages := 30 + numMessages := 300 numRotations := 3 expected := make([][]byte, 0, numFiles*numMessages*numRotations) @@ -124,15 +60,22 @@ func TestMultiFileRotateSlow(t *testing.T) { go func(fn int) { defer wg.Done() + file := filetest.OpenFile(t, baseFileName(fn)) for rotationNum := 0; rotationNum < numRotations; rotationNum++ { - file := filetest.OpenFile(t, baseFileName(fn)) for messageNum := 0; messageNum < numMessages; messageNum++ { filetest.WriteString(t, file, getMessage(fn, rotationNum, messageNum)+"\n") time.Sleep(5 * time.Millisecond) } - - require.NoError(t, file.Close()) - require.NoError(t, os.Rename(baseFileName(fn), fileName(fn, rotationNum))) + require.NoError(t, file.Sync()) + _, err := file.Seek(0, 0) + require.NoError(t, err) + dst := filetest.OpenFile(t, fileName(fn, rotationNum)) + _, err = io.Copy(dst, file) + require.NoError(t, err) + require.NoError(t, dst.Close()) + require.NoError(t, file.Truncate(0)) + _, err = file.Seek(0, 0) + require.NoError(t, err) } }(fileNum) } @@ -141,7 +84,7 @@ func TestMultiFileRotateSlow(t *testing.T) { wg.Wait() } -func TestMultiCopyTruncateSlow(t *testing.T) { +func TestMoveCreate(t *testing.T) { if runtime.GOOS == windowsOS { t.Skip("Rotation tests have been flaky on Windows. See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16331") } @@ -182,19 +125,10 @@ func TestMultiCopyTruncateSlow(t *testing.T) { file := filetest.OpenFile(t, baseFileName(fn)) for messageNum := 0; messageNum < numMessages; messageNum++ { filetest.WriteString(t, file, getMessage(fn, rotationNum, messageNum)+"\n") - time.Sleep(5 * time.Millisecond) + time.Sleep(10 * time.Millisecond) } - - _, err := file.Seek(0, 0) - require.NoError(t, err) - dst := filetest.OpenFile(t, fileName(fn, rotationNum)) - _, err = io.Copy(dst, file) - require.NoError(t, err) - require.NoError(t, dst.Close()) - require.NoError(t, file.Truncate(0)) - _, err = file.Seek(0, 0) - require.NoError(t, err) - file.Close() + require.NoError(t, file.Close()) + require.NoError(t, os.Rename(baseFileName(fn), fileName(fn, rotationNum))) } }(fileNum) } @@ -203,162 +137,6 @@ func TestMultiCopyTruncateSlow(t *testing.T) { wg.Wait() } -type rotationTest struct { - name string - totalLines int - maxLinesPerFile int - maxBackupFiles int - writeInterval time.Duration - pollInterval time.Duration - ephemeralLines bool -} - -/* -When log files are rotated at extreme speeds, it is possible to miss some log entries. -This can happen when an individual log entry is written and deleted within the duration -of a single poll interval. For example, consider the following scenario: - - A log file may have up to 9 backups (10 total log files) - - Each log file may contain up to 10 entries - - Log entries are written at an interval of 10µs - - Log files are polled at an interval of 100ms - -In this scenario, a log entry that is written may only exist on disk for about 1ms. -A polling interval of 100ms will most likely never produce a chance to read the log file. - -In production settings, this consideration is not very likely to be a problem, but it is -easy to encounter the issue in tests, and difficult to deterministically simulate edge cases. -However, the above understanding does allow for some consistent expectations. - 1. Cases that do not require deletion of old log entries should always pass. - 2. Cases where the polling interval is sufficiently rapid should always pass. - 3. When neither 1 nor 2 is true, there may be missing entries, but still no duplicates. - -The following method is provided largely as documentation of how this is expected to behave. -In practice, timing is largely dependent on the responsiveness of system calls. -*/ -func (rt rotationTest) expectEphemeralLines() bool { - // primary + backups - maxLinesInAllFiles := rt.maxLinesPerFile + rt.maxLinesPerFile*rt.maxBackupFiles - - // Will the test write enough lines to result in deletion of oldest backups? - maxBackupsExceeded := rt.totalLines > maxLinesInAllFiles - - // last line written in primary file will exist for l*b more writes - minTimeToLive := time.Duration(int(rt.writeInterval) * rt.maxLinesPerFile * rt.maxBackupFiles) - - // can a line be written and then rotated to deletion before ever observed? - return maxBackupsExceeded && rt.pollInterval > minTimeToLive -} - -func (rt rotationTest) run(tc rotationTest, copyTruncate, sequential bool) func(t *testing.T) { - return func(t *testing.T) { - - tempDir := t.TempDir() - cfg := NewConfig().includeDir(tempDir) - cfg.StartAt = "beginning" - cfg.PollInterval = tc.pollInterval - sink := emittest.NewSink(emittest.WithCallBuffer(tc.totalLines)) - operator := testManagerWithSink(t, cfg, sink) - - file, err := os.CreateTemp(tempDir, "") - require.NoError(t, err) - require.NoError(t, file.Close()) // will be managed by rotator - - rotator := nanojack.Logger{ - Filename: file.Name(), - MaxLines: tc.maxLinesPerFile, - MaxBackups: tc.maxBackupFiles, - CopyTruncate: copyTruncate, - Sequential: sequential, - } - t.Cleanup(func() { _ = rotator.Close() }) - - logger := log.New(&rotator, "", 0) - - expected := make([][]byte, 0, tc.totalLines) - baseStr := string(filetest.TokenWithLength(46)) // + ' 123' - for i := 0; i < tc.totalLines; i++ { - expected = append(expected, []byte(fmt.Sprintf("%s %3d", baseStr, i))) - } - - require.NoError(t, operator.Start(testutil.NewUnscopedMockPersister())) - defer func() { - require.NoError(t, operator.Stop()) - }() - - for _, message := range expected { - logger.Println(string(message)) - time.Sleep(tc.writeInterval) - } - - received := make([][]byte, 0, tc.totalLines) - for i := 0; i < tc.totalLines; i++ { - received = append(received, sink.NextToken(t)) - } - - if tc.ephemeralLines { - if !tc.expectEphemeralLines() { - // This is helpful for test development, and ensures the sample computation is used - t.Logf("Potentially unstable ephemerality expectation for test: %s", tc.name) - } - require.Subset(t, expected, received) - } else { - require.ElementsMatch(t, expected, received) - } - } -} - -func TestRotation(t *testing.T) { - if runtime.GOOS == windowsOS { - t.Skip("Rotation tests have been flaky on Windows. See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16331") - } - cases := []rotationTest{ - { - name: "NoRotation", - totalLines: 10, - maxLinesPerFile: 10, - maxBackupFiles: 1, - writeInterval: time.Millisecond, - pollInterval: 10 * time.Millisecond, - }, - { - name: "NoDeletion", - totalLines: 20, - maxLinesPerFile: 10, - maxBackupFiles: 1, - writeInterval: time.Millisecond, - pollInterval: 10 * time.Millisecond, - }, - { - name: "Deletion", - totalLines: 30, - maxLinesPerFile: 10, - maxBackupFiles: 1, - writeInterval: time.Millisecond, - pollInterval: 10 * time.Millisecond, - ephemeralLines: true, - }, - { - name: "Deletion/ExceedFingerprint", - totalLines: 300, - maxLinesPerFile: 100, - maxBackupFiles: 1, - writeInterval: time.Millisecond, - pollInterval: 10 * time.Millisecond, - ephemeralLines: true, - }, - } - - for _, tc := range cases { - if runtime.GOOS != windowsOS { - // Windows has very poor support for moving active files, so rotation is less commonly used - t.Run(fmt.Sprintf("%s/MoveCreateTimestamped", tc.name), tc.run(tc, false, false)) - t.Run(fmt.Sprintf("%s/MoveCreateSequential", tc.name), tc.run(tc, false, true)) - } - t.Run(fmt.Sprintf("%s/CopyTruncateTimestamped", tc.name), tc.run(tc, true, false)) - t.Run(fmt.Sprintf("%s/CopyTruncateSequential", tc.name), tc.run(tc, true, true)) - } -} - func TestMoveFile(t *testing.T) { if runtime.GOOS == windowsOS { t.Skip("Moving files while open is unsupported on Windows") diff --git a/pkg/stanza/go.mod b/pkg/stanza/go.mod index b9fd1507bdf1b..97ff161652d4b 100644 --- a/pkg/stanza/go.mod +++ b/pkg/stanza/go.mod @@ -10,7 +10,6 @@ require ( github.com/haimrubinstein/go-syslog/v3 v3.0.0 github.com/jpillora/backoff v1.0.0 github.com/json-iterator/go v1.1.12 - github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage v0.95.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.95.0 github.com/stretchr/testify v1.8.4 diff --git a/pkg/stanza/go.sum b/pkg/stanza/go.sum index ab53ce122f200..b972728910699 100644 --- a/pkg/stanza/go.sum +++ b/pkg/stanza/go.sum @@ -66,8 +66,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc h1:49ewVBwLcy+eYqI4R0ICilCI4dPjddpFXWv3liXzUxM= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc/go.mod h1:WXIHwGy+c7/IK2PiJ4oxuTHkpnkSut7TNFpKnI5llPU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= @@ -86,7 +84,6 @@ github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= @@ -196,7 +193,6 @@ google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/receiver/filelogreceiver/filelog_test.go b/receiver/filelogreceiver/filelog_test.go index cf2673b4bdaa2..ccc6917e80e56 100644 --- a/receiver/filelogreceiver/filelog_test.go +++ b/receiver/filelogreceiver/filelog_test.go @@ -6,7 +6,7 @@ package filelogreceiver import ( "context" "fmt" - "log" + "io" "os" "path/filepath" "runtime" @@ -15,7 +15,6 @@ import ( "testing" "time" - "github.com/observiq/nanojack" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" @@ -127,28 +126,16 @@ func TestReadRotatingFiles(t *testing.T) { tests := []rotationTest{ { - name: "CopyTruncateTimestamped", + name: "CopyTruncate", copyTruncate: true, - sequential: false, - }, - { - name: "CopyTruncateSequential", - copyTruncate: true, - sequential: true, }, } if runtime.GOOS != "windows" { // Windows has very poor support for moving active files, so rotation is less commonly used tests = append(tests, []rotationTest{ { - name: "MoveCreateTimestamped", + name: "MoveCreate", copyTruncate: false, - sequential: false, - }, - { - name: "MoveCreateSequential", - copyTruncate: false, - sequential: true, }, }...) } @@ -161,24 +148,24 @@ func TestReadRotatingFiles(t *testing.T) { type rotationTest struct { name string copyTruncate bool - sequential bool } func (rt *rotationTest) Run(t *testing.T) { t.Parallel() - tempDir := t.TempDir() - f := NewFactory() sink := new(consumertest.LogsSink) + tempDir := t.TempDir() cfg := rotationTestConfig(tempDir) // With a max of 100 logs per file and 1 backup file, rotation will occur // when more than 100 logs are written, and deletion when more than 200 are written. // Write 300 and validate that we got the all despite rotation and deletion. - logger := newRotatingLogger(t, tempDir, 100, 1, rt.copyTruncate, rt.sequential) + maxLinesPerFile := 100 numLogs := 300 + fileName := filepath.Join(tempDir, "test.log") + backupFileName := filepath.Join(tempDir, "test-backup.log") // Build expected outputs expectedTimestamp, _ := time.ParseInLocation("2006-01-02", "2020-08-25", time.Local) @@ -193,7 +180,37 @@ func (rt *rotationTest) Run(t *testing.T) { require.NoError(t, err, "failed to create receiver") require.NoError(t, rcvr.Start(context.Background(), componenttest.NewNopHost())) + file, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0600) + require.NoError(t, err) + for i := 0; i < numLogs; i++ { + if (i+1)%maxLinesPerFile == 0 { + if rt.copyTruncate { + // Recreate the backup file + require.NoError(t, os.Remove(backupFileName)) + backupFile, openErr := os.OpenFile(backupFileName, os.O_CREATE|os.O_RDWR, 0600) + require.NoError(t, openErr) + + // Copy the current file to the backup file + require.NoError(t, file.Sync()) + _, err = file.Seek(0, 0) + require.NoError(t, err) + _, err = io.Copy(backupFile, file) + require.NoError(t, err) + require.NoError(t, backupFile.Close()) + + // Truncate the original file + require.NoError(t, file.Truncate(0)) + _, err = file.Seek(0, 0) + require.NoError(t, err) + } else { + require.NoError(t, file.Close()) + require.NoError(t, os.Rename(fileName, backupFileName)) + file, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0600) + require.NoError(t, err) + } + } + msg := fmt.Sprintf("This is a simple log line with the number %3d", i) // Build the expected set by converting entries to pdata Logs... @@ -203,7 +220,8 @@ func (rt *rotationTest) Run(t *testing.T) { require.NoError(t, converter.Batch([]*entry.Entry{e})) // ... and write the logs lines to the actual file consumed by receiver. - logger.Printf("2020-08-25 %s", msg) + _, err := file.WriteString(fmt.Sprintf("2020-08-25 %s\n", msg)) + require.NoError(t, err) time.Sleep(time.Millisecond) } @@ -231,21 +249,6 @@ func consumeNLogsFromConverter(ch <-chan plog.Logs, count int, wg *sync.WaitGrou } } -func newRotatingLogger(t *testing.T, tempDir string, maxLines, maxBackups int, copyTruncate, sequential bool) *log.Logger { - path := filepath.Join(tempDir, "test.log") - rotator := &nanojack.Logger{ - Filename: path, - MaxLines: maxLines, - MaxBackups: maxBackups, - CopyTruncate: copyTruncate, - Sequential: sequential, - } - - t.Cleanup(func() { _ = rotator.Close() }) - - return log.New(rotator, "", 0) -} - func expectNLogs(sink *consumertest.LogsSink, expected int) func() bool { return func() bool { return sink.LogRecordCount() == expected } } diff --git a/receiver/filelogreceiver/go.mod b/receiver/filelogreceiver/go.mod index c12272a8f53fc..b13d1a07557b9 100644 --- a/receiver/filelogreceiver/go.mod +++ b/receiver/filelogreceiver/go.mod @@ -3,7 +3,6 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelo go 1.21 require ( - github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage v0.95.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.95.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.95.0 diff --git a/receiver/filelogreceiver/go.sum b/receiver/filelogreceiver/go.sum index 6752fab4f7c02..8db6564e99eda 100644 --- a/receiver/filelogreceiver/go.sum +++ b/receiver/filelogreceiver/go.sum @@ -62,8 +62,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc h1:49ewVBwLcy+eYqI4R0ICilCI4dPjddpFXWv3liXzUxM= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc/go.mod h1:WXIHwGy+c7/IK2PiJ4oxuTHkpnkSut7TNFpKnI5llPU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= @@ -81,7 +79,6 @@ github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= @@ -185,9 +182,7 @@ google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/receiver/otlpjsonfilereceiver/go.sum b/receiver/otlpjsonfilereceiver/go.sum index 49703964c211a..8db6564e99eda 100644 --- a/receiver/otlpjsonfilereceiver/go.sum +++ b/receiver/otlpjsonfilereceiver/go.sum @@ -62,8 +62,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc h1:49ewVBwLcy+eYqI4R0ICilCI4dPjddpFXWv3liXzUxM= -github.com/observiq/nanojack v0.0.0-20201106172433-343928847ebc/go.mod h1:WXIHwGy+c7/IK2PiJ4oxuTHkpnkSut7TNFpKnI5llPU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=