Upgrade to JDK 14

.travis.yml

@@ -1,6 +1,6 @@
 language: java
 jdk:
- - openjdk13
+ - openjdk14
 
 git:
  depth: false # see https://docs.travis-ci.com/user/sonarcloud/#accessing-full-scm-history 
@@ -12,4 +12,4 @@ addons:
  secure: "r6YbLdXaJ25cB7K8Ene2//Bu46DEdUPt55Nit8YNXPn6eHNzx9eH1f2o5BDQgNIwbQmomfyzqvEcoFuSg85V7viLeL2U0gKFQcvZRm8PMdkE3hMncxfXC1PdB+VIOBgqvTV7BvpnvPEqBUIeLLhToyC0/ull9pjgdlyRqhMch1MMuTkfLrvuuiAfX+Yu8yF/iAjw5J0ka5UXVRexXbLZQbzt4P98w0L0ciUaHTxHaPBuXUtqBsdMpM21pPm+dKoVHnEUctLy5BoypkZCTJfhGhVTL/iCp1YvDXUV7uiBGO5IYD0QYdncWKh5N6rlPtBmnuop6GyHHkppGYR3Dd3abH777vG72ucfT3b6yAphZvVTJ5sQ4H7kxqAkreKcEWMLhcFi5cTgx5cNFxJZIQwLp7eYvk6mSIj8QDzG3cJML76F19f5kf6/qDTZrv9RpwZh7QnRx4T3fWGlh3PYxoGY9X3G3HE8ooAgBty7Nl9vZ6k9ljSGAlBM3fm1zF3e19pYZcg9IhvIpIzTTx2PFSL7HQR2morLAI5J0VwE1kqCJ1wfCCrth3tQepqSnSmZe8ULL2IFbS/FQ09++tUzCz9KuZXWw14AqquLSHv83qirOl89/owrKraJq3b49KaG+XM18JIsmInOfHhQr2IdEknAFZaer3FhCmrRELRFVkRU38w="
 
 script:
- - ./gradlew jacocoTestReport sonarqube
+ - ./gradlew jacocoTestReport sonarqube --no-daemon

README.md

@@ -45,7 +45,7 @@ In AnaLog terms the *administrator* is a person who installs and configures AnaL
 From the administrator's perspective AnaLog:
 
 * is standalone Java application with built-in web server (based on [Spring Boot](https://spring.io/projects/spring-boot) framework)
-* works on [Java 12](http:https://jdk.java.net/12/) and above
+* works on [Java 14](http:https://jdk.java.net/14/) and above
 * has flexible configuration in 2 YAML files: for system settings and log choices (see [examples](https://github.com/Toparvion/analog/wiki))
 * must be installed on every server where the file logs must be fetched from
 * relies on `tail`, `docker` and `kubectl` binaries to fetch logs from corresponding sources

build.gradle

@@ -25,8 +25,8 @@ apply plugin: 'application'
 apply plugin: 'com.bmuschko.docker-remote-api'
 apply plugin: 'org.sonarqube'
 
-sourceCompatibility = JavaVersion.VERSION_13
-targetCompatibility = JavaVersion.VERSION_13
+sourceCompatibility = JavaVersion.VERSION_14
+targetCompatibility = JavaVersion.VERSION_14
 //tasks.withType(JavaCompile).each {
 // it.options.compilerArgs.add('--enable-preview') // to support Java new language features;
 //} // disabled to keep the app compatible with future JDK versions

src/main/java/tech/toparvion/analog/util/timestamp/DateFormat2RegexConverter.java

@@ -71,19 +71,11 @@ interface Presentation {
  *
  * @see DateTimeFormatter
  */
- private static final Presentation TEXT = lettersCount -> {
- switch (lettersCount) {
- case 1:
- case 2:
- case 3:
- return format("[\\w\\x20]{%d}", lettersCount);
- case 4:
- return "[\\w\\x20]+";
- case 5:
- return "[\\w\\x20]{1}";
- default:
- return "[\\w\\x20]+";
- }
+ private static final Presentation TEXT = lettersCount ->
+ switch (lettersCount) {
+ case 1, 2, 3 -> format("[\\w\\x20]{%d}", lettersCount);
+ case 5 -> "[\\w\\x20]{1}";
+ default -> "[\\w\\x20]+"; // the same as for 4 letters
  };
 
  /**
@@ -122,8 +114,6 @@ interface Presentation {
  * The format syntax is a subset of rules specified in {@link DateTimeFormatter}. Namely supported the following
  * symbols only: {@code u,y,M,L,d,E,a,h,K,k,H,m,s,S,n}. Quoting (both single quote and arbitrary text) is also
  * supported.<p>
- * In order to process timestamp'less log lines as fast as possible, the returned pattern contains a prefix
- * denoting searching from the start of a line only ({@code ^} symbol).<p>
  * Because log timestamps are not always located at the very beginning of a line, {@code
  * logTimestampFormat} may contain some additional characters. For instance, if log timestamp is wrapped with
  * square braces, the format string may look like {@code [dd.MM.yy HH:mm:ss.SSS}. The opening square bracket will

src/main/java/tech/toparvion/analog/util/timestamp/TimestampExtractor.java

@@ -12,6 +12,7 @@
 import java.time.*;
 import java.time.format.DateTimeFormatter;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -30,7 +31,16 @@
 @Service
 public class TimestampExtractor {
  private static final Logger log = LoggerFactory.getLogger(TimestampExtractor.class);
-
+ /**
+ * By default, AnaLog parses log lines' timestamp with English locale because otherwise it may fail parsing. For
+ * instance, if current system locale is 'ru' and the application tries to parse "{@code 23/Jun/2020:00:29:42}"
+ * timestamp, then it will fail with the following exception:<pre><code>
+ * java.time.format.DateTimeParseException: Text '23/Jun/2020:00:29:42' could not be parsed at index 3
+ * </code></pre>
+ * This is generally not a good idea to hard-code the locale, but it seems acceptable so far as there is no
+ * localized logs in production environments (AFAIK).
+ */
+ public static final Locale DEFAULT_TIMESTAMP_PARSER_LOCALE = Locale.ENGLISH;
  private final DateFormat2RegexConverter converter;
  /**
  * Registry of compiled regex patterns and pre-built dateTime formatters for known logs. Registry records aren't
@@ -58,7 +68,8 @@ public void registerNewTimestampFormat(String format, String logPath) {
  return;
  }
  Pattern pattern = converter.convertToRegex(format);
- DateTimeFormatter formatter = DateTimeFormatter.ofPattern(format);
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern(format)
+ .withLocale(DEFAULT_TIMESTAMP_PARSER_LOCALE);
  log.info("For logPath='{}' and its format='{}' new registry record was created: pattern='{}', formatter='{}'",
  logPath, format, pattern, formatter);
  registry.put(logPath, new PatternAndFormatter(pattern, formatter));
@@ -76,7 +87,7 @@ public void registerNewTimestampFormat(String format, String logPath) {
  public LocalDateTime extractTimestamp(Message<String> lineMessage) {
  String line = lineMessage.getPayload();
  if (line.startsWith("\tat ")) {
- // a kind of short-hand way to avoid wasting time on analyzing lines of java stacktraces
+ // a kind of short-hand way to avoid wasting time on analyzing lines of java stack traces
  return null;
  }
 
@@ -91,8 +102,8 @@ public LocalDateTime extractTimestamp(Message<String> lineMessage) {
  if (!timestampMatcher.find()) {
  return null;
  // TODO есть проблема: если число таких записей без метки будет слишком велико, то выделяющий записи агрегатор
- // выпустит их без "головы", то есть без предшествующей записи с меткой. Из-за этого на принимающей стороне их,
- // возможно, будет трудно куда-либо определить. Нужно подумать, как это победить, и есть ли такая проблема.
+ //  выпустит их без "головы", то есть без предшествующей записи с меткой. Из-за этого на принимающей стороне их,
+ //  возможно, будет трудно куда-либо определить. Нужно подумать, как это победить, и есть ли такая проблема.
  }
 
  String tsString = timestampMatcher.group();
@@ -101,19 +112,20 @@ public LocalDateTime extractTimestamp(Message<String> lineMessage) {
  try {
  parsedTimestamp = formatter.parse(tsString, LocalDateTime::from);
  } catch (DateTimeException e) {
- // in case no date specified in timestamp format, AnaLog supposes the date to be equal today
+ log.debug("Unable to parse timestamp string '{}' with formatter '{}'.", tsString, formatter, e);
+ // in case no date specified in timestamp format, AnaLog supposes the date to be equal to the current one
  LocalTime parsedTime = formatter.parse(tsString, LocalTime::from);
  parsedTimestamp = LocalDateTime.of(LocalDate.now(Clock.systemDefaultZone()), parsedTime);
  }
 
  return parsedTimestamp;
  }
 
- private static class PatternAndFormatter {
+ static class PatternAndFormatter {
  private final Pattern pattern;
  private final DateTimeFormatter formatter;
 
- private PatternAndFormatter(Pattern pattern, DateTimeFormatter formatter) {
+ PatternAndFormatter(Pattern pattern, DateTimeFormatter formatter) {
  this.pattern = pattern;
  this.formatter = formatter;
  }
@@ -127,4 +139,7 @@ DateTimeFormatter getFormatter() {
  }
  }
 
+ /*public*/ Map<String, PatternAndFormatter> getRegistry() {
+ return registry;
+ }
 }

src/test/java/tech/toparvion/analog/util/timestamp/DateFormat2RegexConverterTest.java

@@ -1,5 +1,6 @@
 package tech.toparvion.analog.util.timestamp;
 
+import org.assertj.core.api.ThrowableAssert.ThrowingCallable;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
@@ -10,6 +11,8 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static java.util.Locale.ENGLISH;
+import static org.assertj.core.api.Assertions.assertThatCode;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -135,4 +138,24 @@ void letterSeparatedSearch() {
  Matcher matcher = convertedPattern.matcher(sampleLogTimestamp);
  assertTrue(matcher.find());
  }
+
+ @Test
+ void testAmPmFormatter() {
+ var format = "LLL dd, yyyy K:mm:ss";
+ var formatter = DateTimeFormatter.ofPattern(format)
+ .withLocale(ENGLISH);
+ ThrowingCallable sutCall = () -> formatter.parse("Dec 21, 2020 07:05:00");
+// log.info("Parsed dateTime: {}", parsedDateTime);
+ assertThatCode(sutCall).doesNotThrowAnyException();
+ }
+
+ @Test
+ void testNginxLogFormat() {
+ var format = "dd/LLL/yyyy:HH:mm:ss";
+ var formatter = DateTimeFormatter.ofPattern(format)
+ .withLocale(ENGLISH);
+ ThrowingCallable sutCall = () -> formatter.parse("23/Jun/2020:00:29:42");
+// log.info("Parsed dateTime: {}", parsedDateTime);
+ assertThatCode(sutCall).doesNotThrowAnyException();
+ }
 }

src/test/java/tech/toparvion/analog/util/timestamp/TimestampExtractorTest.java

@@ -0,0 +1,176 @@
+package tech.toparvion.analog.util.timestamp;
+
+import org.assertj.core.api.Condition;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.springframework.integration.support.MessageBuilder;
+import tech.toparvion.analog.util.timestamp.TimestampExtractor.PatternAndFormatter;
+
+import java.io.File;
+import java.time.Clock;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.format.DateTimeFormatter;
+import java.util.regex.Pattern;
+
+import static java.time.temporal.ChronoUnit.SECONDS;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.within;
+import static org.mockito.Mockito.*;
+import static org.springframework.integration.file.FileHeaders.ORIGINAL_FILE;
+import static tech.toparvion.analog.util.timestamp.TimestampExtractor.DEFAULT_TIMESTAMP_PARSER_LOCALE;
+
+/**
+ * @author Toparvion
+ */
+class TimestampExtractorTest {
+
+ DateFormat2RegexConverter converter;
+ TimestampExtractor sut;
+
+ @BeforeEach
+ void setUp() {
+ converter = mock(DateFormat2RegexConverter.class);
+ sut = new TimestampExtractor(converter);
+ }
+
+ @Test
+ @DisplayName("A format for given log path is already registered")
+ void registerNewTimestampFormat_existentFormat() {
+ // given
+ var format = "someFormat";
+ var logPath = "logPath";
+ sut.getRegistry().put(logPath, mock(PatternAndFormatter.class));
+
+ // when
+ sut.registerNewTimestampFormat(format, logPath);
+
+ // then
+ verifyNoInteractions(converter);
+ assertThat(sut.getRegistry()).size().isEqualTo(1);
+ }
+ @Test
+ @DisplayName("A format for given log path is stored into the map")
+ void registerNewTimestampFormat_newFormat() {
+ // given
+ var format = "dd/LLL/yyyy:HH:mm:ss";
+ var logPath = "logPath";
+ Pattern pattern = Pattern.compile("");
+ when(converter.convertToRegex(format)).thenReturn(pattern);
+
+ // when
+ sut.registerNewTimestampFormat(format, logPath);
+
+ // then
+ verify(converter).convertToRegex(format);
+ Condition<PatternAndFormatter> pafCondition = new Condition<>() {
+ public boolean matches(PatternAndFormatter paf) {
+ return paf.getFormatter().getLocale().equals(DEFAULT_TIMESTAMP_PARSER_LOCALE)
+ && paf.getPattern().equals(pattern);
+ }
+ };
+ assertThat(sut.getRegistry()).hasSize(1)
+ .hasEntrySatisfying(logPath, pafCondition);
+
+ }
+
+ @Test
+ @DisplayName("The extracting exits shortly with null if the line starts with '\\tat'")
+ void extractTimestamp_shortHand() {
+ // given
+ var message = MessageBuilder.withPayload("\tat SomeElse.java").build();
+
+ // when
+ LocalDateTime timestamp = sut.extractTimestamp(message);
+
+ // then
+ assertThat(timestamp).isNull();
+ }
+
+ @Test
+ @DisplayName("The extracting returns null if no matched group found")
+ void extractTimestamp_noPafFound() {
+ // given
+ var logFile = mock(File.class);
+ var logPath = "/home/me/myapp/app.log";
+ when(logFile.getAbsolutePath()).thenReturn(logPath);
+ var message = MessageBuilder
+ .withPayload("2020-05-10 09:23:05,419 INFO [main] - tech.toparvion.analog.AnaLog")
+ .setHeader(ORIGINAL_FILE, logFile)
+ .build();
+ PatternAndFormatter paf = mock(PatternAndFormatter.class);
+ Pattern pattern = Pattern.compile("neverMatchingPattern");
+ when(paf.getPattern()).thenReturn(pattern);
+ sut.getRegistry().put(logPath, paf);
+
+ // when
+ LocalDateTime timestamp = sut.extractTimestamp(message);
+
+ //then
+ assertThat(timestamp).isNull();
+ //noinspection ResultOfMethodCallIgnored
+ verify(logFile).getAbsolutePath();
+ //noinspection ResultOfMethodCallIgnored
+ verify(paf).getPattern();
+ verifyNoMoreInteractions(paf);
+ }
+
+ @Test
+ @DisplayName("The extracting ends up with successfully parsed timestamp")
+ void extractTimestamp_successfulParsing() {
+ // given
+ var logFile = mock(File.class);
+ var logPath = "/home/me/myapp/app.log";
+ when(logFile.getAbsolutePath()).thenReturn(logPath);
+ var message = MessageBuilder
+ .withPayload("02.10.14 09:21:58 INFO [main] - tech.toparvion.analog.AnaLog")
+ .setHeader(ORIGINAL_FILE, logFile)
+ .build();
+ var pattern = Pattern.compile("\\d{2}\\.\\d{2}\\.\\d{2} \\d{2}:\\d{2}:\\d{2}");
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd.MM.yy HH:mm:ss");
+ PatternAndFormatter paf = new PatternAndFormatter(pattern, formatter);
+ sut.getRegistry().put(logPath, paf);
+
+ // when
+ LocalDateTime timestamp = sut.extractTimestamp(message);
+
+ //then
+ assertThat(timestamp)
+ .isNotNull()
+ .isEqualTo("2014-10-02T09:21:58");
+ //noinspection ResultOfMethodCallIgnored
+ verify(logFile).getAbsolutePath();
+ }
+
+ @Test
+ @DisplayName("The extracting can use current date if no date is specified in the log message")
+ void extractTimestamp_noDateSpecified() {
+ // given
+ var logFile = mock(File.class);
+ var logPath = "/home/me/myapp/app.log";
+ when(logFile.getAbsolutePath()).thenReturn(logPath);
+ var message = MessageBuilder
+ .withPayload("19:21:58 INFO [main] - tech.toparvion.analog.AnaLog")
+ .setHeader(ORIGINAL_FILE, logFile)
+ .build();
+ var pattern = Pattern.compile("\\d{2}:\\d{2}:\\d{2}");
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("HH:mm:ss");
+ PatternAndFormatter paf = new PatternAndFormatter(pattern, formatter);
+ sut.getRegistry().put(logPath, paf);
+
+ // when
+ LocalDateTime timestamp = sut.extractTimestamp(message);
+
+ //then
+ LocalDateTime logTimeWithCurrentDate =
+ LocalDateTime.of(LocalDate.now(Clock.systemDefaultZone()), LocalTime.of(19, 21, 58));
+ assertThat(timestamp)
+ .isNotNull()
+ .isCloseTo(logTimeWithCurrentDate, within(1, SECONDS));
+
+ //noinspection ResultOfMethodCallIgnored
+ verify(logFile).getAbsolutePath();
+ }
+}