From 7417288f43acc0d6b85c5bc70ca2d5e4aab4f84d Mon Sep 17 00:00:00 2001 From: olsona Date: Fri, 20 Jul 2018 12:41:18 -0500 Subject: [PATCH] Start on logging (#483) --- .../resources/log4j.properties | 2 +- .../src/main/resources/log4j.properties | 8 ++--- .../calendar/GoogleCalendarExporter.java | 6 +--- gradle.properties | 1 + .../api/launcher/ExtensionContext.java | 3 -- .../api/ApiExtensionContext.java | 13 ++----- portability-transfer/build.gradle | 3 +- .../transfer/JobMetadata.java | 2 ++ .../transfer/JobProcessor.java | 11 +++--- .../transfer/WorkerExtensionContext.java | 13 +++---- .../transfer/WorkerModule.java | 5 ++- .../transfer/logging/EncryptingLayout.java | 36 +++++++++++++++++++ 12 files changed, 62 insertions(+), 41 deletions(-) create mode 100644 portability-transfer/src/main/java/org/datatransferproject/transfer/logging/EncryptingLayout.java diff --git a/distributions/demo-google-deployment/resources/log4j.properties b/distributions/demo-google-deployment/resources/log4j.properties index 45c144a9e..ba6fc834e 100644 --- a/distributions/demo-google-deployment/resources/log4j.properties +++ b/distributions/demo-google-deployment/resources/log4j.properties @@ -18,6 +18,6 @@ log4j.rootLogger=DEBUG, A1 log4j.appender.A1=org.apache.log4j.ConsoleAppender -log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout=org.datatransferproject.transfer.logging.EncryptingLayout # Print the date in ISO 8601 format log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n diff --git a/distributions/demo-server/src/main/resources/log4j.properties b/distributions/demo-server/src/main/resources/log4j.properties index 7d6b7bc0f..a4262919e 100644 --- a/distributions/demo-server/src/main/resources/log4j.properties +++ b/distributions/demo-server/src/main/resources/log4j.properties @@ -1,5 +1,3 @@ -log4j.rootLogger=DEBUG, A1 -log4j.appender.A1=org.apache.log4j.ConsoleAppender -log4j.appender.A1.layout=org.apache.log4j.PatternLayout -# Print the date in ISO 8601 format -log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n +log4j.rootLogger=DEBUG, consoleAppender, fileAppender +log4j.appender.consoleAppender=org.apache.log4j.ConsoleAppender +log4j.appender.consoleAppender.layout=org.datatransferproject.transfer.logging.EncryptingLayout diff --git a/extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/calendar/GoogleCalendarExporter.java b/extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/calendar/GoogleCalendarExporter.java index be4ed13d5..9ec9a6e99 100644 --- a/extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/calendar/GoogleCalendarExporter.java +++ b/extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/calendar/GoogleCalendarExporter.java @@ -107,11 +107,7 @@ private static RecurrenceRule getRecurrenceRule(List ruleStrings) { "Recurrence entry " + st + " is not recognizable as an RRULE, RDATE, or EXDATE"); } } - RecurrenceRule rule = ruleBuilder.build(); - logger.debug("RRule: " + rule.getRRule()); - logger.debug("RDate: " + rule.getRDate()); - logger.debug("ExDate: " + rule.getExDate()); - return rule; + return ruleBuilder.build(); } private static CalendarModel convertToCalendarModel(CalendarListEntry calendarData) { diff --git a/gradle.properties b/gradle.properties index e7ff88039..f1b4ceae1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -35,3 +35,4 @@ okHttpVersion=3.9.1 azureKeyVaultVersion=1.0.0 javaDockerContainer=openjdk:8-jdk jerseyVersion=2.26 +log4jVersion=2.11.0 diff --git a/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/ExtensionContext.java b/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/ExtensionContext.java index dd38e5ceb..5b916db73 100644 --- a/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/ExtensionContext.java +++ b/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/ExtensionContext.java @@ -20,9 +20,6 @@ /** Provides information required to bootstrap extensions. */ public interface ExtensionContext { - /** Returns the system logger extension service can use to record events. */ - Logger getLogger(); - /** * Returns the runtime type manager. * diff --git a/portability-api/src/main/java/org/datatransferproject/api/ApiExtensionContext.java b/portability-api/src/main/java/org/datatransferproject/api/ApiExtensionContext.java index bde216e63..7a2a62242 100644 --- a/portability-api/src/main/java/org/datatransferproject/api/ApiExtensionContext.java +++ b/portability-api/src/main/java/org/datatransferproject/api/ApiExtensionContext.java @@ -16,22 +16,21 @@ package org.datatransferproject.api; import com.google.common.base.Preconditions; +import java.util.HashMap; +import java.util.Map; import org.datatransferproject.api.launcher.Constants.Environment; import org.datatransferproject.api.launcher.ExtensionContext; import org.datatransferproject.api.launcher.Flag; -import org.datatransferproject.api.launcher.Logger; import org.datatransferproject.api.launcher.TypeManager; import org.datatransferproject.config.extension.SettingsExtension; -import java.util.HashMap; -import java.util.Map; - /** * Provides a context for initializing extensions. * *

*/ public class ApiExtensionContext implements ExtensionContext { + private final Map, Object> registered = new HashMap<>(); private final TypeManager typeManager; private final SettingsExtension settingsExtension; @@ -57,12 +56,6 @@ public ApiExtensionContext(TypeManager typeManager, SettingsExtension settingsEx Preconditions.checkNotNull(baseApiUrl, "Required setting 'baseApiUrl' is missing"); } - @Override - public Logger getLogger() { - // TODO implement - return null; - } - @Override public TypeManager getTypeManager() { return typeManager; diff --git a/portability-transfer/build.gradle b/portability-transfer/build.gradle index f5b0bfaad..0507a1241 100644 --- a/portability-transfer/build.gradle +++ b/portability-transfer/build.gradle @@ -75,7 +75,8 @@ dependencies { // Logging compile("org.slf4j:slf4j-api:${slf4jVersion}") compile("org.slf4j:slf4j-log4j12:${slf4jVersion}") - + compile("org.apache.logging.log4j:log4j-api:${log4jVersion}") + compile("org.apache.logging.log4j:log4j-core:${log4jVersion}") testCompile("com.google.truth:truth:${truthVersion}") testCompile("junit:junit:${junitVersion}") diff --git a/portability-transfer/src/main/java/org/datatransferproject/transfer/JobMetadata.java b/portability-transfer/src/main/java/org/datatransferproject/transfer/JobMetadata.java index 54b033171..ab6891427 100644 --- a/portability-transfer/src/main/java/org/datatransferproject/transfer/JobMetadata.java +++ b/portability-transfer/src/main/java/org/datatransferproject/transfer/JobMetadata.java @@ -18,6 +18,7 @@ import com.google.common.base.Preconditions; import java.security.KeyPair; import java.util.UUID; +import org.datatransferproject.transfer.logging.EncryptingLayout; /** * A class that contains metadata for a transfer worker's job. @@ -54,6 +55,7 @@ static void init( dataType = initDataType; exportService = initExportService; importService = initImportService; + EncryptingLayout.setJobId(jobId); } // TODO: remove this diff --git a/portability-transfer/src/main/java/org/datatransferproject/transfer/JobProcessor.java b/portability-transfer/src/main/java/org/datatransferproject/transfer/JobProcessor.java index 7a3377627..4cc528a87 100644 --- a/portability-transfer/src/main/java/org/datatransferproject/transfer/JobProcessor.java +++ b/portability-transfer/src/main/java/org/datatransferproject/transfer/JobProcessor.java @@ -33,12 +33,11 @@ import org.slf4j.LoggerFactory; /** - * Process a job in two steps:
- * (1) Decrypt the stored credentials, which have been encrypted with this transfer worker's public - * key
- * (2)Run the copy job + * Process a job in two steps:
(1) Decrypt the stored credentials, which have been encrypted + * with this transfer worker's public key
(2)Run the copy job */ final class JobProcessor { + private static final Logger logger = LoggerFactory.getLogger(JobProcessor.class); private final JobStore store; @@ -58,7 +57,9 @@ final class JobProcessor { this.symmetricKeyGenerator = symmetricKeyGenerator; } - /** Process our job, whose metadata is available via {@link JobMetadata}. */ + /** + * Process our job, whose metadata is available via {@link JobMetadata}. + */ void processJob() { UUID jobId = JobMetadata.getJobId(); logger.debug("Begin processing jobId: {}", jobId); diff --git a/portability-transfer/src/main/java/org/datatransferproject/transfer/WorkerExtensionContext.java b/portability-transfer/src/main/java/org/datatransferproject/transfer/WorkerExtensionContext.java index b139ae56b..7f364d51c 100644 --- a/portability-transfer/src/main/java/org/datatransferproject/transfer/WorkerExtensionContext.java +++ b/portability-transfer/src/main/java/org/datatransferproject/transfer/WorkerExtensionContext.java @@ -21,17 +21,19 @@ import java.util.Map; import org.datatransferproject.api.launcher.Constants.Environment; import org.datatransferproject.api.launcher.ExtensionContext; -import org.datatransferproject.api.launcher.Logger; -import org.datatransferproject.api.launcher.TypeManager; import org.datatransferproject.api.launcher.Flag; +import org.datatransferproject.api.launcher.TypeManager; import org.datatransferproject.config.extension.SettingsExtension; import org.datatransferproject.launcher.impl.TypeManagerImpl; import org.datatransferproject.types.transfer.auth.TokenAuthData; import org.datatransferproject.types.transfer.auth.TokenSecretAuthData; import org.datatransferproject.types.transfer.auth.TokensAndUrlAuthData; -/** {@link ExtensionContext} used by the transfer worker. */ +/** + * {@link ExtensionContext} used by the transfer worker. + */ public class WorkerExtensionContext implements ExtensionContext { + private final TypeManager typeManager; private final Map, Object> registered = new HashMap<>(); private final SettingsExtension settingsExtension; @@ -54,11 +56,6 @@ public class WorkerExtensionContext implements ExtensionContext { Preconditions.checkNotNull(environment, "Required setting 'environment' is missing"); } - @Override - public Logger getLogger() { - return null; - } - @Override public TypeManager getTypeManager() { return typeManager; diff --git a/portability-transfer/src/main/java/org/datatransferproject/transfer/WorkerModule.java b/portability-transfer/src/main/java/org/datatransferproject/transfer/WorkerModule.java index ea2b12205..b02d70717 100644 --- a/portability-transfer/src/main/java/org/datatransferproject/transfer/WorkerModule.java +++ b/portability-transfer/src/main/java/org/datatransferproject/transfer/WorkerModule.java @@ -18,12 +18,10 @@ import static com.google.common.collect.MoreCollectors.onlyElement; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.google.common.collect.ImmutableList; import com.google.inject.Provides; import com.google.inject.Singleton; import java.io.IOException; -import java.util.LinkedHashMap; import java.util.List; import java.util.NoSuchElementException; import org.datatransferproject.api.launcher.ExtensionContext; @@ -36,7 +34,6 @@ import org.datatransferproject.spi.transfer.extension.TransferExtension; import org.datatransferproject.spi.transfer.provider.Exporter; import org.datatransferproject.spi.transfer.provider.Importer; -import org.datatransferproject.types.transfer.retry.RetryStrategy; import org.datatransferproject.types.transfer.retry.RetryStrategyLibrary; final class WorkerModule extends FlagBindingModule { @@ -128,4 +125,6 @@ ImmutableList getTransferExtensions() { RetryStrategyLibrary getRetryStrategyLibrary() throws IOException { return context.getSetting("retryLibrary", null); } + + } diff --git a/portability-transfer/src/main/java/org/datatransferproject/transfer/logging/EncryptingLayout.java b/portability-transfer/src/main/java/org/datatransferproject/transfer/logging/EncryptingLayout.java new file mode 100644 index 000000000..3ea05e5f2 --- /dev/null +++ b/portability-transfer/src/main/java/org/datatransferproject/transfer/logging/EncryptingLayout.java @@ -0,0 +1,36 @@ +package org.datatransferproject.transfer.logging; + +import com.google.common.base.Preconditions; +import java.util.UUID; +import org.apache.log4j.Layout; +import org.apache.log4j.helpers.ISO8601DateFormat; +import org.apache.log4j.spi.LoggingEvent; + +public class EncryptingLayout extends Layout { + + static UUID jobId; + + public static void setJobId(UUID inputJobId) { + Preconditions.checkState(jobId == null, "jobId has already been set"); + jobId = inputJobId; + } + + @Override + public String format(LoggingEvent event) { + // TODO: read formatting from a config file + return String.format("[%s] [%s]: %s - %s%s", + new ISO8601DateFormat().format(event.timeStamp), + jobId != null ? Long.toHexString(jobId.getMostSignificantBits()) : "undefined", + event.getLevel().toString(), + event.getRenderedMessage(), LINE_SEP); + } + + @Override + public boolean ignoresThrowable() { + return true; + } + + @Override + public void activateOptions() { + } +}