From 42fed78e830d91fec02878872bf3a1a11e989ea1 Mon Sep 17 00:00:00 2001 From: Matt Conroy Date: Mon, 27 Nov 2017 14:54:53 -0700 Subject: [PATCH 01/21] - Switch to a more global build file setup --- .gitignore | 2 + .travis.yml | 8 +++ bin/checkPublish | 9 ++++ build.sbt | 58 ++++++++++++++++++++ connectors/activemq/build.sbt | 61 --------------------- connectors/activemq/project/plugins.sbt | 1 - connectors/kafka/build.sbt | 68 ------------------------ connectors/kafka/project/plugins.sbt | 2 - core/build.sbt | 70 ------------------------- core/project/plugins.sbt | 1 - plugins/replay/build.sbt | 65 ----------------------- plugins/replay/project/plugins.sbt | 1 - plugins/retry/build.sbt | 59 --------------------- plugins/retry/project/plugins.sbt | 1 - plugins/stats/build.sbt | 62 ---------------------- project/Build.scala | 40 -------------- project/plugins.sbt | 6 +-- server/build.sbt | 65 ----------------------- 18 files changed, 79 insertions(+), 500 deletions(-) create mode 100755 bin/checkPublish create mode 100644 build.sbt delete mode 100644 connectors/activemq/project/plugins.sbt delete mode 100644 core/project/plugins.sbt delete mode 100644 plugins/replay/project/plugins.sbt delete mode 100644 plugins/retry/project/plugins.sbt delete mode 100644 project/Build.scala diff --git a/.gitignore b/.gitignore index 9bdc891..9ef42c0 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ activemq-data .vscode classpath.txt javaconfig.json + +build.properties diff --git a/.travis.yml b/.travis.yml index 94d5896..1344f79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,3 +4,11 @@ scala: jdk: - oraclejdk8 sudo: false + +env: + global: + - secure: uV1Qhx4Bgfiyc+X0GIS1LqjCfMJSm9fIdgoe63CPEPwhf6qTKfzpYOGOos6UTotvSSmIeLmUEUjJurYM1UOnRtltXClpz0lL/WrKDhoyGHdWwk+HrHAOkXhUncTloOA15gNfDS/8JeatT5J3FWXoLVXJBRBN41/uvZV0FTFKdiE= + - secure: oslYlw8hd54oQZUygcCYbYZiXRpUMqgDyeTxF7CdAtVpFnxK7lTnmJUXXp5S22CXEcK/n/8+V6l+DmvnWYOJ73QbDc11rrYOgrHWE3KH7Cxs0W5JonZUB5WZ2kMM3XcZlfXmggiJqtHNxHBLVoS3khM0+ta/RVcvgNCQM5ZlMao= + - secure: yOJT1AMsLq6x6SGr65dsRhtqvn5i3N9JEqtB3XPRMnNaHzQIg4RH0PoJMTzM0bLKG/gIEsMdB257nT7XkZjOVusrZLaf9N9c55zHNSAmoy1VlaPezWSePkrkKIscZIHUZD0psliGcE6mjO1MuqMXXipd00Zs8hgWrNxu6wlB6Is= +after_success: + - ./bin/checkPublish diff --git a/bin/checkPublish b/bin/checkPublish new file mode 100755 index 0000000..c5920c3 --- /dev/null +++ b/bin/checkPublish @@ -0,0 +1,9 @@ +#!/bin/bash +BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH; else echo $TRAVIS_PULL_REQUEST_BRANCH; fi) +echo Found branch $BRANCH + +if [ $BRANCH == 'master' ] + then + echo Publishing to Sonatype + sbt publishSigned sonatypeReleaseAll +fi diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..68be3f6 --- /dev/null +++ b/build.sbt @@ -0,0 +1,58 @@ +lazy val baseSettings = binksSettings ++ Seq( + organization := "com.github.dcshock", + version := "3.0", + scalaVersion := "2.11.7", + javacOptions ++= Seq("-source", "1.8"), + javacOptions in compile ++= Seq("-g:lines,vars,source", "-deprecation"), + javacOptions in doc += "-Xdoclint:none", + crossPaths := false, + autoScalaLibrary := false, + resolvers ++= Seq( + "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", + "Maven Central" at "http://repo1.maven.org/maven2", + "Fuse Snapshots" at "http://repo.fusesource.com/nexus/content/repositories/snapshots", + "Fuse" at "http://repo.fusesource.com/nexus/content/groups/public", + "Confluent Maven Repo" at "http://packages.confluent.io/maven/" + ), + publishMavenStyle := true, + credentials += Credentials( + "Sonatype Nexus Repository Manager", + "oss.sonatype.org", + sys.env.getOrElse("SONATYPE_USER", ""), + sys.env.getOrElse("SONATYPE_PASS", "") + ), + useGpg := false, + usePgpKeyHex("E46770E4F1ED27F3"), + pgpPublicRing := baseDirectory.value / "project" / ".gnupg" / "pubring.gpg", + pgpSecretRing := baseDirectory.value / "project" / ".gnupg" / "secring.gpg", + pgpPassphrase := sys.env.get("GPG_PASS").map(_.toArray), + pomIncludeRepository := { _ => false }, + pomExtra := ( + https://github.com/dcshock/forklift + + + BSD-style + http://www.opensource.org/licenses/bsd-license.php + repo + + + + git@github.com:dcshock/forklift.git + scm:git:git@github.com:dcshock/forklift.git + + + + dcshock + Matt Conroy + http://www.mattconroy.com + + ) +) + +lazy val core = project in file("core") settings baseSettings +lazy val replay = project.dependsOn(core) in file("plugins/replay") settings baseSettings +lazy val retry = project.dependsOn(core) in file("plugins/retry") settings baseSettings +lazy val stats = project.dependsOn(core) in file("plugins/stats") settings baseSettings +lazy val activemq = project.dependsOn(core) in file("connectors/activemq") settings baseSettings +lazy val kafka = project.dependsOn(core) in file("connectors/kafka") settings baseSettings +lazy val server = project.dependsOn(core, replay, retry, stats, activemq, kafka) in file("server") settings baseSettings diff --git a/connectors/activemq/build.sbt b/connectors/activemq/build.sbt index 2b01798..b088875 100644 --- a/connectors/activemq/build.sbt +++ b/connectors/activemq/build.sbt @@ -1,28 +1,6 @@ -organization := "com.github.dcshock" - name := "forklift-activemq" -version := "2.0" - -javacOptions ++= Seq("-source", "1.8") - -javacOptions in compile ++= Seq("-g:lines,vars,source") - -initialize := { - val _ = initialize.value - if (sys.props("java.specification.version") != "1.8") - sys.error("Java 8 is required for this project.") -} - -resolvers ++= Seq( - "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", - "Maven Central" at "http://repo1.maven.org/maven2", - "Fuse Snapshots" at "http://repo.fusesource.com/nexus/content/repositories/snapshots", - "Fuse" at "http://repo.fusesource.com/nexus/content/groups/public" -) - libraryDependencies ++= Seq( - "com.github.dcshock" % "forklift" % "2.0", "org.apache.activemq" % "activemq-client" % "5.14.0", "org.apache.activemq" % "activemq-broker" % "5.14.0", "com.fasterxml.jackson.core" % "jackson-databind" % "2.7.3", @@ -42,42 +20,3 @@ libraryDependencies ++= testDependencies.map(_ % "test") // so disable parallel test execution parallelExecution in Test := false testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a") - -// Remove scala dependency for pure Java libraries -autoScalaLibrary := false - -// Remove the scala version from the generated/published artifact -crossPaths := false - -publishMavenStyle := true - -publishTo := { - val nexus = "https://oss.sonatype.org/" - if (isSnapshot.value) - Some("snapshots" at nexus + "content/repositories/snapshots") - else - Some("releases" at nexus + "service/local/staging/deploy/maven2") -} - -pomIncludeRepository := { _ => false } - -pomExtra := ( - https://github.com/dcshock/forklift - - - BSD-style - http://www.opensource.org/licenses/bsd-license.php - repo - - - - git@github.com:dcshock/forklift.git - scm:git:git@github.com:dcshock/forklift.git - - - - dcshock - Matt Conroy - http://www.mattconroy.com - - ) diff --git a/connectors/activemq/project/plugins.sbt b/connectors/activemq/project/plugins.sbt deleted file mode 100644 index 4ce4d9e..0000000 --- a/connectors/activemq/project/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") diff --git a/connectors/kafka/build.sbt b/connectors/kafka/build.sbt index 12017ca..4308670 100644 --- a/connectors/kafka/build.sbt +++ b/connectors/kafka/build.sbt @@ -1,32 +1,6 @@ -organization := "com.github.dcshock" - name := "forklift-kafka" -version := "2.0" - -//required for some test dependencies -scalaVersion := "2.11.7" - -javacOptions ++= Seq("-source", "1.8") - -javacOptions in compile ++= Seq("-g:lines,vars,source") - -initialize := { - val _ = initialize.value - if (sys.props("java.specification.version") != "1.8") - sys.error("Java 8 is required for this project.") -} - -resolvers ++= Seq( - "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", - "Maven Central" at "http://repo1.maven.org/maven2", - "Fuse Snapshots" at "http://repo.fusesource.com/nexus/content/repositories/snapshots", - "Fuse" at "http://repo.fusesource.com/nexus/content/groups/public", - "Confluent Maven Repo" at "http://packages.confluent.io/maven/" -) - libraryDependencies ++= Seq( - "com.github.dcshock" % "forklift" % "2.0" , "com.fasterxml.jackson.core" % "jackson-databind" % "2.7.3", "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % "2.7.3", "org.apache.kafka" % "kafka-clients" % "0.10.1.1-cp1" exclude("org.slf4j","slf4j-log4j12"), @@ -55,45 +29,3 @@ testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a") // avro settings (javaSource in avroConfig) := baseDirectory(_/"target/generated-sources").value (sourceDirectory in avroConfig) := baseDirectory(_/"src/test/resources/schemas").value - -// Remove scala dependency for pure Java libraries -autoScalaLibrary := false - -// Remove the scala version from the generated/published artifact -crossPaths := false - -publishMavenStyle := true - -publishTo := { - val nexus = "https://oss.sonatype.org/" - if (isSnapshot.value) - Some("snapshots" at nexus + "content/repositories/snapshots") - else - Some("releases" at nexus + "service/local/staging/deploy/maven2") -} - -pomIncludeRepository := { _ => false } - -pomExtra := ( - https://github.com/dcshock/forklift-kafka - - - BSD-style - http://www.opensource.org/licenses/bsd-license.php - repo - - - - git@github.com:dcshock/forklift-kafka.git - scm:git:git@github.com:dcshock/forklift-kafka.git - - - - afrieze - Andrew Frieze - - - kuroshii - Bridger Howell - - ) diff --git a/connectors/kafka/project/plugins.sbt b/connectors/kafka/project/plugins.sbt index 3a29115..16ad420 100644 --- a/connectors/kafka/project/plugins.sbt +++ b/connectors/kafka/project/plugins.sbt @@ -1,3 +1 @@ addSbtPlugin("com.cavorite" % "sbt-avro-1-7" % "1.1.2") - -addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") diff --git a/core/build.sbt b/core/build.sbt index 376dd10..7b4c07c 100644 --- a/core/build.sbt +++ b/core/build.sbt @@ -1,28 +1,5 @@ -organization := "com.github.dcshock" - name := "forklift" -version := "2.1" - -javacOptions ++= Seq("-source", "1.8") - -javacOptions in compile ++= Seq("-g:lines,vars,source", "-deprecation") - -javacOptions in doc += "-Xdoclint:none" - -initialize := { - val _ = initialize.value - if (sys.props("java.specification.version") != "1.8") - sys.error("Java 8 is required for this project.") -} - -resolvers ++= Seq( - "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", - "Maven Central" at "http://repo1.maven.org/maven2", - "Fuse Snapshots" at "http://repo.fusesource.com/nexus/content/repositories/snapshots", - "Fuse" at "http://repo.fusesource.com/nexus/content/groups/public" -) - libraryDependencies ++= Seq( "com.google.guava" % "guava" % "18.0", "com.fasterxml.jackson.core" % "jackson-databind" % "2.7.3", @@ -38,50 +15,3 @@ lazy val testDependencies = Seq( ) libraryDependencies ++= testDependencies.map(_ % "test") - -// Remove scala dependency for pure Java libraries -autoScalaLibrary := false - -// Remove the scala version from the generated/published artifact -crossPaths := false - -publishMavenStyle := true - -publishTo := { - val nexus = "https://oss.sonatype.org/" - if (isSnapshot.value) - Some("snapshots" at nexus + "content/repositories/snapshots") - else - Some("releases" at nexus + "service/local/staging/deploy/maven2") -} - -pomIncludeRepository := { _ => false } - -pomExtra := ( - https://github.com/dcshock/forklift - - - BSD-style - http://www.opensource.org/licenses/bsd-license.php - repo - - - - git@github.com:dcshock/forklift.git - scm:git:git@github.com:dcshock/forklift.git - - - - dcshock - Matt Conroy - http://www.mattconroy.com - - - afrieze - Andrew Frieze - - - kuroshii - Bridger Howell - - ) diff --git a/core/project/plugins.sbt b/core/project/plugins.sbt deleted file mode 100644 index 4ce4d9e..0000000 --- a/core/project/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") diff --git a/plugins/replay/build.sbt b/plugins/replay/build.sbt index e7527bc..3cfeed6 100644 --- a/plugins/replay/build.sbt +++ b/plugins/replay/build.sbt @@ -1,26 +1,6 @@ -organization := "com.github.dcshock" - name := "forklift-replay" -version := "2.2" - -javacOptions ++= Seq("-source", "1.8") - -initialize := { - val _ = initialize.value - if (sys.props("java.specification.version") != "1.8") - sys.error("Java 8 is required for this project.") -} - -resolvers ++= Seq( - "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", - "Maven Central" at "http://repo1.maven.org/maven2", - "Fuse Snapshots" at "http://repo.fusesource.com/nexus/content/repositories/snapshots", - "Fuse" at "http://repo.fusesource.com/nexus/content/groups/public" -) - libraryDependencies ++= Seq( - "com.github.dcshock" % "forklift" % "2.0", "org.elasticsearch" % "elasticsearch" % "2.4.1", "com.fasterxml.jackson.core" % "jackson-databind" % "2.7.3", "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % "2.7.3" @@ -32,48 +12,3 @@ lazy val testDependencies = Seq( ) libraryDependencies ++= testDependencies.map(_ % "test") - -// Remove scala dependency for pure Java libraries -autoScalaLibrary := false - -// Remove the scala version from the generated/published artifact -crossPaths := false - -publishMavenStyle := true - -publishTo := { - val nexus = "https://oss.sonatype.org/" - if (isSnapshot.value) - Some("snapshots" at nexus + "content/repositories/snapshots") - else - Some("releases" at nexus + "service/local/staging/deploy/maven2") -} - -// Remove scala dependency for pure Java libraries -autoScalaLibrary := false - -// Remove the scala version from the generated/published artifact -crossPaths := false - -pomIncludeRepository := { _ => false } - -pomExtra := ( - https://github.com/dcshock/forklift - - - BSD-style - http://www.opensource.org/licenses/bsd-license.php - repo - - - - git@github.com:dcshock/forklift.git - scm:git:git@github.com:dcshock/forklift.git - - - - dcshock - Matt Conroy - http://www.mattconroy.com - - ) diff --git a/plugins/replay/project/plugins.sbt b/plugins/replay/project/plugins.sbt deleted file mode 100644 index 4ce4d9e..0000000 --- a/plugins/replay/project/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") diff --git a/plugins/retry/build.sbt b/plugins/retry/build.sbt index 481e67b..4be170f 100644 --- a/plugins/retry/build.sbt +++ b/plugins/retry/build.sbt @@ -1,26 +1,6 @@ -organization := "com.github.dcshock" - name := "forklift-retry" -version := "2.2" - -javacOptions ++= Seq("-source", "1.8") - -initialize := { - val _ = initialize.value - if (sys.props("java.specification.version") != "1.8") - sys.error("Java 8 is required for this project.") -} - -resolvers ++= Seq( - "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", - "Maven Central" at "http://repo1.maven.org/maven2", - "Fuse Snapshots" at "http://repo.fusesource.com/nexus/content/repositories/snapshots", - "Fuse" at "http://repo.fusesource.com/nexus/content/groups/public" -) - libraryDependencies ++= Seq( - "com.github.dcshock" % "forklift" % "2.0", "com.fasterxml.jackson.core" % "jackson-databind" % "2.7.3", "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % "2.7.3", "io.searchbox" % "jest" % "2.4.0" @@ -32,42 +12,3 @@ lazy val testDependencies = Seq( ) libraryDependencies ++= testDependencies.map(_ % "test") - -// Remove scala dependency for pure Java libraries -autoScalaLibrary := false - -// Remove the scala version from the generated/published artifact -crossPaths := false - -publishMavenStyle := true - -publishTo := { - val nexus = "https://oss.sonatype.org/" - if (isSnapshot.value) - Some("snapshots" at nexus + "content/repositories/snapshots") - else - Some("releases" at nexus + "service/local/staging/deploy/maven2") -} - -pomIncludeRepository := { _ => false } - -pomExtra := ( - https://github.com/dcshock/forklift - - - BSD-style - http://www.opensource.org/licenses/bsd-license.php - repo - - - - git@github.com:dcshock/forklift.git - scm:git:git@github.com:dcshock/forklift.git - - - - dcshock - Matt Conroy - http://www.mattconroy.com - - ) diff --git a/plugins/retry/project/plugins.sbt b/plugins/retry/project/plugins.sbt deleted file mode 100644 index 4ce4d9e..0000000 --- a/plugins/retry/project/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") diff --git a/plugins/stats/build.sbt b/plugins/stats/build.sbt index 24914a1..23f30bd 100644 --- a/plugins/stats/build.sbt +++ b/plugins/stats/build.sbt @@ -1,63 +1 @@ -organization := "com.github.dcshock" - name := "forklift-stats" - -version := "1.0" - -javacOptions ++= Seq("-source", "1.8") - -initialize := { - val _ = initialize.value - if (sys.props("java.specification.version") != "1.8") - sys.error("Java 8 is required for this project.") -} - -resolvers ++= Seq( - "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", - "Maven Central" at "http://repo1.maven.org/maven2", - "Fuse Snapshots" at "http://repo.fusesource.com/nexus/content/repositories/snapshots", - "Fuse" at "http://repo.fusesource.com/nexus/content/groups/public" -) - -libraryDependencies ++= Seq( - "com.github.dcshock" % "forklift" % "1.0" -) - -// Remove scala dependency for pure Java libraries -autoScalaLibrary := false - -// Remove the scala version from the generated/published artifact -crossPaths := false - -publishMavenStyle := true - -publishTo := { - val nexus = "https://oss.sonatype.org/" - if (isSnapshot.value) - Some("snapshots" at nexus + "content/repositories/snapshots") - else - Some("releases" at nexus + "service/local/staging/deploy/maven2") -} - -pomIncludeRepository := { _ => false } - -pomExtra := ( - https://github.com/dcshock/forklift - - - BSD-style - http://www.opensource.org/licenses/bsd-license.php - repo - - - - git@github.com:dcshock/forklift.git - scm:git:git@github.com:dcshock/forklift.git - - - - dcshock - Matt Conroy - http://www.mattconroy.com - - ) \ No newline at end of file diff --git a/project/Build.scala b/project/Build.scala deleted file mode 100644 index 44284a8..0000000 --- a/project/Build.scala +++ /dev/null @@ -1,40 +0,0 @@ -import sbt._ -import Keys._ - -object ForkliftBuild extends Build { - lazy val core = Project( - id = "core", - base = file("core") - ) - - lazy val replay = Project( - id = "replay", - base = file("plugins/replay") - ).dependsOn(core) - - lazy val retry = Project( - id = "retry", - base = file("plugins/retry") - ).dependsOn(core) - - lazy val stats = Project( - id = "stats", - base = file("plugins/stats") - ).dependsOn(core) - - lazy val activemq = Project( - id = "activemq", - base = file("connectors/activemq") - ).dependsOn(core) - - lazy val kafka = Project( - id = "kafka", - base = file("connectors/kafka") - ).dependsOn(core) - - lazy val server = Project( - id = "server", - base = file("server") - ).dependsOn(core, activemq, replay, retry, stats) - -} diff --git a/project/plugins.sbt b/project/plugins.sbt index a571e0b..1a0c34b 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,7 +1,5 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.0.0-M4") - addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.13.0") - -addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") - +addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.0") +addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.0") addSbtPlugin("com.cavorite" % "sbt-avro-1-7" % "1.1.2") diff --git a/server/build.sbt b/server/build.sbt index 5d22512..0d41c21 100644 --- a/server/build.sbt +++ b/server/build.sbt @@ -1,32 +1,8 @@ -organization := "com.github.dcshock" - name := "forklift-server" -version := "2.2" - enablePlugins(JavaAppPackaging) -javacOptions ++= Seq("-source", "1.8") - -initialize := { - val _ = initialize.value - if (sys.props("java.specification.version") != "1.8") - sys.error("Java 8 is required for this project.") -} - -resolvers ++= Seq( - "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", - "Maven Central" at "http://repo1.maven.org/maven2", - "Fuse Snapshots" at "http://repo.fusesource.com/nexus/content/repositories/snapshots", - "Fuse" at "http://repo.fusesource.com/nexus/content/groups/public" -) - libraryDependencies ++= Seq( - "com.github.dcshock" % "forklift" % "2.1", - "com.github.dcshock" % "forklift-activemq" % "2.0", - "com.github.dcshock" % "forklift-replay" % "2.2", - "com.github.dcshock" % "forklift-retry" % "2.2", - "com.github.dcshock" % "forklift-stats" % "1.0", "com.github.dcshock" % "consul-rest-client" % "0.10", "org.apache.activemq" % "activemq-broker" % "5.14.0", "io.searchbox" % "jest" % "2.0.0", @@ -41,44 +17,3 @@ libraryDependencies ++= Seq( "com.novocode" % "junit-interface" % "0.11" % "test", "commons-io" % "commons-io" % "2.4" % "test" ) - -// Remove scala dependency for pure Java libraries -autoScalaLibrary := false - -// Remove the scala version from the generated/published artifact -crossPaths := false - -publishMavenStyle := true - -publishTo := { - val nexus = "https://oss.sonatype.org/" - if (isSnapshot.value) - Some("snapshots" at nexus + "content/repositories/snapshots") - else - Some("releases" at nexus + "service/local/staging/deploy/maven2") -} - -pomIncludeRepository := { _ => false } - -pomExtra := ( - https://github.com/dcshock/forklift - - - BSD-style - http://www.opensource.org/licenses/bsd-license.php - repo - - - - git@github.com:dcshock/forklift.git - scm:git:git@github.com:dcshock/forklift.git - - - - dcshock - Matt Conroy - http://www.mattconroy.com - - ) - -useGpg := true From a8839f559019416cd8798da99819271ee2c37dbd Mon Sep 17 00:00:00 2001 From: Matt Conroy Date: Mon, 27 Nov 2017 15:20:35 -0700 Subject: [PATCH 02/21] - Add signing keys.. Note they are secured by a long passphrase. --- build.sbt | 4 +- project/.gnupg/gpg.conf | 196 ++++++++++++++++++++++++++++++++++++ project/.gnupg/pubring.gpg | Bin 0 -> 2240 bytes project/.gnupg/pubring.gpg~ | Bin 0 -> 2240 bytes project/.gnupg/secring.gpg | Bin 0 -> 4897 bytes 5 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 project/.gnupg/gpg.conf create mode 100644 project/.gnupg/pubring.gpg create mode 100644 project/.gnupg/pubring.gpg~ create mode 100644 project/.gnupg/secring.gpg diff --git a/build.sbt b/build.sbt index 68be3f6..e9e5ee2 100644 --- a/build.sbt +++ b/build.sbt @@ -23,8 +23,8 @@ lazy val baseSettings = binksSettings ++ Seq( ), useGpg := false, usePgpKeyHex("E46770E4F1ED27F3"), - pgpPublicRing := baseDirectory.value / "project" / ".gnupg" / "pubring.gpg", - pgpSecretRing := baseDirectory.value / "project" / ".gnupg" / "secring.gpg", + pgpPublicRing := file(sys.props("user.dir")) / "project" / ".gnupg" / "pubring.gpg", + pgpSecretRing := file(sys.props("user.dir")) / "project" / ".gnupg" / "secring.gpg", pgpPassphrase := sys.env.get("GPG_PASS").map(_.toArray), pomIncludeRepository := { _ => false }, pomExtra := ( diff --git a/project/.gnupg/gpg.conf b/project/.gnupg/gpg.conf new file mode 100644 index 0000000..942678f --- /dev/null +++ b/project/.gnupg/gpg.conf @@ -0,0 +1,196 @@ +# Options for GnuPG +# Copyright 1998, 1999, 2000, 2001, 2002, 2003, +# 2010 Free Software Foundation, Inc. +# +# This file is free software; as a special exception the author gives +# unlimited permission to copy and/or distribute it, with or without +# modifications, as long as this notice is preserved. +# +# This file is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY, to the extent permitted by law; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Unless you specify which option file to use (with the command line +# option "--options filename"), GnuPG uses the file ~/.gnupg/gpg.conf +# by default. +# +# An options file can contain any long options which are available in +# GnuPG. If the first non white space character of a line is a '#', +# this line is ignored. Empty lines are also ignored. +# +# See the man page for a list of options. + +# Uncomment the following option to get rid of the copyright notice + +#no-greeting + +# If you have more than 1 secret key in your keyring, you may want to +# uncomment the following option and set your preferred keyid. + +#default-key 621CC013 + +# If you do not pass a recipient to gpg, it will ask for one. Using +# this option you can encrypt to a default key. Key validation will +# not be done in this case. The second form uses the default key as +# default recipient. + +#default-recipient some-user-id +#default-recipient-self + +# By default GnuPG creates version 4 signatures for data files as +# specified by OpenPGP. Some earlier (PGP 6, PGP 7) versions of PGP +# require the older version 3 signatures. Setting this option forces +# GnuPG to create version 3 signatures. + +#force-v3-sigs + +# Because some mailers change lines starting with "From " to ">From " +# it is good to handle such lines in a special way when creating +# cleartext signatures; all other PGP versions do it this way too. +# To enable full OpenPGP compliance you may want to use this option. + +#no-escape-from-lines + +# When verifying a signature made from a subkey, ensure that the cross +# certification "back signature" on the subkey is present and valid. +# This protects against a subtle attack against subkeys that can sign. +# Defaults to --no-require-cross-certification. However for new +# installations it should be enabled. + +require-cross-certification + + +# If you do not use the Latin-1 (ISO-8859-1) charset, you should tell +# GnuPG which is the native character set. Please check the man page +# for supported character sets. This character set is only used for +# metadata and not for the actual message which does not undergo any +# translation. Note that future version of GnuPG will change to UTF-8 +# as default character set. + +#charset utf-8 + +# Group names may be defined like this: +# group mynames = paige 0x12345678 joe patti +# +# Any time "mynames" is a recipient (-r or --recipient), it will be +# expanded to the names "paige", "joe", and "patti", and the key ID +# "0x12345678". Note there is only one level of expansion - you +# cannot make an group that points to another group. Note also that +# if there are spaces in the recipient name, this will appear as two +# recipients. In these cases it is better to use the key ID. + +#group mynames = paige 0x12345678 joe patti + +# Some old Windows platforms require 8.3 filenames. If your system +# can handle long filenames, uncomment this. + +#no-mangle-dos-filenames + +# Lock the file only once for the lifetime of a process. If you do +# not define this, the lock will be obtained and released every time +# it is needed - normally this is not needed. + +#lock-once + +# GnuPG can send and receive keys to and from a keyserver. These +# servers can be HKP, email, or LDAP (if GnuPG is built with LDAP +# support). +# +# Example HKP keyservers: +# hkp://keys.gnupg.net +# +# Example LDAP keyservers: +# ldap://pgp.surfnet.nl:11370 +# +# Regular URL syntax applies, and you can set an alternate port +# through the usual method: +# hkp://keyserver.example.net:22742 +# +# If you have problems connecting to a HKP server through a buggy http +# proxy, you can use keyserver option broken-http-proxy (see below), +# but first you should make sure that you have read the man page +# regarding proxies (keyserver option honor-http-proxy) +# +# Most users just set the name and type of their preferred keyserver. +# Note that most servers (with the notable exception of +# ldap://keyserver.pgp.com) synchronize changes with each other. Note +# also that a single server name may actually point to multiple +# servers via DNS round-robin. hkp://keys.gnupg.net is an example of +# such a "server", which spreads the load over a number of physical +# servers. To see the IP address of the server actually used, you may use +# the "--keyserver-options debug". + +keyserver hkp://keys.gnupg.net +#keyserver http://http-keys.gnupg.net +#keyserver mailto:pgp-public-keys@keys.nl.pgp.net + +# Common options for keyserver functions: +# +# include-disabled = when searching, include keys marked as "disabled" +# on the keyserver (not all keyservers support this). +# +# no-include-revoked = when searching, do not include keys marked as +# "revoked" on the keyserver. +# +# verbose = show more information as the keys are fetched. +# Can be used more than once to increase the amount +# of information shown. +# +# use-temp-files = use temporary files instead of a pipe to talk to the +# keyserver. Some platforms (Win32 for one) always +# have this on. +# +# keep-temp-files = do not delete temporary files after using them +# (really only useful for debugging) +# +# honor-http-proxy = if the keyserver uses HTTP, honor the http_proxy +# environment variable +# +# broken-http-proxy = try to work around a buggy HTTP proxy +# +# auto-key-retrieve = automatically fetch keys as needed from the keyserver +# when verifying signatures or when importing keys that +# have been revoked by a revocation key that is not +# present on the keyring. +# +# no-include-attributes = do not include attribute IDs (aka "photo IDs") +# when sending keys to the keyserver. + +#keyserver-options auto-key-retrieve + +# Uncomment this line to display photo user IDs in key listings and +# when a signature from a key with a photo is verified. + +#show-photos + +# Use this program to display photo user IDs +# +# %i is expanded to a temporary file that contains the photo. +# %I is the same as %i, but the file isn't deleted afterwards by GnuPG. +# %k is expanded to the key ID of the key. +# %K is expanded to the long OpenPGP key ID of the key. +# %t is expanded to the extension of the image (e.g. "jpg"). +# %T is expanded to the MIME type of the image (e.g. "image/jpeg"). +# %f is expanded to the fingerprint of the key. +# %% is %, of course. +# +# If %i or %I are not present, then the photo is supplied to the +# viewer on standard input. If your platform supports it, standard +# input is the best way to do this as it avoids the time and effort in +# generating and then cleaning up a secure temp file. +# +# The default program is "xloadimage -fork -quiet -title 'KeyID 0x%k' stdin" +# On Mac OS X and Windows, the default is to use your regular JPEG image +# viewer. +# +# Some other viewers: +# photo-viewer "qiv %i" +# photo-viewer "ee %i" +# photo-viewer "display -title 'KeyID 0x%k'" +# +# This one saves a copy of the photo ID in your home directory: +# photo-viewer "cat > ~/photoid-for-key-%k.%t" +# +# Use your MIME handler to view photos: +# photo-viewer "metamail -q -d -b -c %T -s 'KeyID 0x%k' -f GnuPG" + diff --git a/project/.gnupg/pubring.gpg b/project/.gnupg/pubring.gpg new file mode 100644 index 0000000000000000000000000000000000000000..a98f74e246c5006099e4eaae497e384fddd96ca1 GIT binary patch literal 2240 zcmV;x2tW6k0u2OO6r%S55CFIOc=1d13c!;>JtfttF7pkRL_d$JV zv`keP-Qs72SiJ^4E;Uy*7RIsb`}EyBk|QzvGjm9QGV&-?x_$1KNgB}qBqSL^n)X}E zuKlqneLv3OhnO3RMI%PUG8qL3pU=ZEYln#NqkHHR`nHQ8sz`TC4T}RR8XS=a+$nOs zf!d`u#8VAaNhGivO$CV(Xm5d^tTT3>2(%gOAkzo(sHh$00QB36KE)u9Y@T@X)=g!W zvJ~Kw8DNlT!bZnD)a8_!gQFUiegSpP%}Uq?0Z_zO6NmN@0RK;=Un`;fi;4QaTPd=! z!=Yy2ZdqL&aUHBdQ?xMon|PonK+D286sR;zS=85Iwuh}ap$|=#9jUYtjQm)kp7~VA z3ROIcAQWazLcHL8n(^Fo9r=W`7y{Y>$SkcOl9=E2r&G3M+J%aFc!|6kq|7RAuwX)u zcGwe$u1aT&q|&&42l4V%BJU)hoKqNn6B+`x-&!W>uzFjp*-m@aPNx%0_?<_yp&-ut z$b!#DNs0N7G_f$d!pN5=C4Xg3CdI7#Z+~|A$|q7;CC@AYU(F1&l;FHA{yXtrK8Hy5 z-wcQ1Q_uY`xa|ND0RRECIAd>ZE@x?UXmw&PWMgw^Z)0m9C}VGJE@x?UXmw&PWMgw^ zZ)0mIAUthhbaX&tZ*FpLc`jpPK8XT31QP)W03!tgS`?!90viJd3ke7Z0|EgC6$kx0EqQq-1wSe%Mb!TWhg*(f(d|Cz{k5T zN9?`PC&J|$?1l{OSxpTQw2y-wA;Sx;ZB_R20YK7d9|}(pr+>`n=10X5LF*_!$*2sk z8xUBVv=*X3bjT5JKe?|ED9z=SrFVBDrHEZu%(l6bVeof|AQbGlmqsg9Q#jRyx@z9Z zX3*G5hhHYuLptPxg~;L4fhe?C4?r2_mQ^r}7VWP5Mf^5oVhLUKv-?lJM6~a=;{;1{ zqd{4CFct&iRP;5X83|*(^*7g z$rK8d;(OotKE8MOTRql;d7EOCBqY%439aApSnYU70FE|$J$0+Un)NVdWrpPj9LJLz?tbMZI|^DES;7C}dO{1obFL**^#d!3tq$r;v=w1#Gw zk{2pC&_Q=7#DqyyZ~&TQ=wl&mMRx0wU|9YpB6fh4!;U-*!ISM5KLsd61U>d!KjEC! zh&YOlt@SBE?R5cbzi6TqyONWa*zYqXR1Mf73De<7B$3dl^9tLO6ZnUSwYaqgm9)htB4#&uAAa%nhDx9(T1VWqHO4~3tcH4B1Z(i9s~CKZ`5zA zSizjOhpSFk7iBnanWMI+*~Sdc+pq!v1GxeX1X>iL_W=+9?Qxf{YBH^3uGtK+=^_TD zJ*0KR5G-5HQZi1zW#De|0mR#HsY&}aAvE3w3l9SiyoxaWCJsVo0 zZ{E7*FDUYi14tG2CLNIIt1f8gb;IBa17mTfzlOZrA)LiFPqz`c>dHsAx&ejR?H4wQhAB4XC^H5n*JI76!d5PikN?D;HRNA07(>dXZRic5(kIYfZhk`t*T|42dOUfjKS2m68Fse1!OZ zFUs~{rV03hyLq;)-FSnA++5Z9KaxZ0iNc)YaaPH9R560eVi z$#YDhjvOWYk+VMs%x!sFnx&%8P7cI8rHSnG#vV_xzO)R$oUnmL9qXI$5_Y}*q@5a- z79`$@@WVNAp}6W%&qjiW8{BmJl1$?Md8@VhroaV*wF}Amn`cz zudo!0JyX@uCu^~y@LWhS_$)OyHI&^jk4mRK01*KI0f_=11Q-Db00{*GS`?!90vikf z3JDP8XK>{4?I-iO#1H>XH3^Sc5<&9hP^mR$QNcA%JiCdE$udc{52Ug~`AO(N>PLWS z@Sd?#FL&f^pBBKwfWy+T4FiBwm(#E$Tmnpcwnak$#t*VApiY2lo+h(IP>5P6K1t?8 zYTkzh^*dC{*6OdQ@+*T+q^^sJYF&7fIq08 zOLnrB8ED%{TfMaS!;<;!QpCtizPeOwMI?w737&B9p z{);%6{%xcBe=`H-JAyE>iqn8kK)~8hOp$D*{5%Lhf8SdoEVHXt_S|>0cA|l%%}-OZrgZW}Q<+iF2WjC4o9{FiMpB6pxch-%jp~ OZz_~q7`EcD0ssR}ttl}8 literal 0 HcmV?d00001 diff --git a/project/.gnupg/pubring.gpg~ b/project/.gnupg/pubring.gpg~ new file mode 100644 index 0000000000000000000000000000000000000000..a98f74e246c5006099e4eaae497e384fddd96ca1 GIT binary patch literal 2240 zcmV;x2tW6k0u2OO6r%S55CFIOc=1d13c!;>JtfttF7pkRL_d$JV zv`keP-Qs72SiJ^4E;Uy*7RIsb`}EyBk|QzvGjm9QGV&-?x_$1KNgB}qBqSL^n)X}E zuKlqneLv3OhnO3RMI%PUG8qL3pU=ZEYln#NqkHHR`nHQ8sz`TC4T}RR8XS=a+$nOs zf!d`u#8VAaNhGivO$CV(Xm5d^tTT3>2(%gOAkzo(sHh$00QB36KE)u9Y@T@X)=g!W zvJ~Kw8DNlT!bZnD)a8_!gQFUiegSpP%}Uq?0Z_zO6NmN@0RK;=Un`;fi;4QaTPd=! z!=Yy2ZdqL&aUHBdQ?xMon|PonK+D286sR;zS=85Iwuh}ap$|=#9jUYtjQm)kp7~VA z3ROIcAQWazLcHL8n(^Fo9r=W`7y{Y>$SkcOl9=E2r&G3M+J%aFc!|6kq|7RAuwX)u zcGwe$u1aT&q|&&42l4V%BJU)hoKqNn6B+`x-&!W>uzFjp*-m@aPNx%0_?<_yp&-ut z$b!#DNs0N7G_f$d!pN5=C4Xg3CdI7#Z+~|A$|q7;CC@AYU(F1&l;FHA{yXtrK8Hy5 z-wcQ1Q_uY`xa|ND0RRECIAd>ZE@x?UXmw&PWMgw^Z)0m9C}VGJE@x?UXmw&PWMgw^ zZ)0mIAUthhbaX&tZ*FpLc`jpPK8XT31QP)W03!tgS`?!90viJd3ke7Z0|EgC6$kx0EqQq-1wSe%Mb!TWhg*(f(d|Cz{k5T zN9?`PC&J|$?1l{OSxpTQw2y-wA;Sx;ZB_R20YK7d9|}(pr+>`n=10X5LF*_!$*2sk z8xUBVv=*X3bjT5JKe?|ED9z=SrFVBDrHEZu%(l6bVeof|AQbGlmqsg9Q#jRyx@z9Z zX3*G5hhHYuLptPxg~;L4fhe?C4?r2_mQ^r}7VWP5Mf^5oVhLUKv-?lJM6~a=;{;1{ zqd{4CFct&iRP;5X83|*(^*7g z$rK8d;(OotKE8MOTRql;d7EOCBqY%439aApSnYU70FE|$J$0+Un)NVdWrpPj9LJLz?tbMZI|^DES;7C}dO{1obFL**^#d!3tq$r;v=w1#Gw zk{2pC&_Q=7#DqyyZ~&TQ=wl&mMRx0wU|9YpB6fh4!;U-*!ISM5KLsd61U>d!KjEC! zh&YOlt@SBE?R5cbzi6TqyONWa*zYqXR1Mf73De<7B$3dl^9tLO6ZnUSwYaqgm9)htB4#&uAAa%nhDx9(T1VWqHO4~3tcH4B1Z(i9s~CKZ`5zA zSizjOhpSFk7iBnanWMI+*~Sdc+pq!v1GxeX1X>iL_W=+9?Qxf{YBH^3uGtK+=^_TD zJ*0KR5G-5HQZi1zW#De|0mR#HsY&}aAvE3w3l9SiyoxaWCJsVo0 zZ{E7*FDUYi14tG2CLNIIt1f8gb;IBa17mTfzlOZrA)LiFPqz`c>dHsAx&ejR?H4wQhAB4XC^H5n*JI76!d5PikN?D;HRNA07(>dXZRic5(kIYfZhk`t*T|42dOUfjKS2m68Fse1!OZ zFUs~{rV03hyLq;)-FSnA++5Z9KaxZ0iNc)YaaPH9R560eVi z$#YDhjvOWYk+VMs%x!sFnx&%8P7cI8rHSnG#vV_xzO)R$oUnmL9qXI$5_Y}*q@5a- z79`$@@WVNAp}6W%&qjiW8{BmJl1$?Md8@VhroaV*wF}Amn`cz zudo!0JyX@uCu^~y@LWhS_$)OyHI&^jk4mRK01*KI0f_=11Q-Db00{*GS`?!90vikf z3JDP8XK>{4?I-iO#1H>XH3^Sc5<&9hP^mR$QNcA%JiCdE$udc{52Ug~`AO(N>PLWS z@Sd?#FL&f^pBBKwfWy+T4FiBwm(#E$Tmnpcwnak$#t*VApiY2lo+h(IP>5P6K1t?8 zYTkzh^*dC{*6OdQ@+*T+q^^sJYF&7fIq08 zOLnrB8ED%{TfMaS!;<;!QpCtizPeOwMI?w737&B9p z{);%6{%xcBe=`H-JAyE>iqn8kK)~8hOp$D*{5%Lhf8SdoEVHXt_S|>0cA|l%%}-OZrgZW}Q<+iF2WjC4o9{FiMpB6pxch-%jp~ OZz_~q7`EcD0ssR}ttl}8 literal 0 HcmV?d00001 diff --git a/project/.gnupg/secring.gpg b/project/.gnupg/secring.gpg new file mode 100644 index 0000000000000000000000000000000000000000..7367b678048c016a767aecac118dd745866ba0b8 GIT binary patch literal 4897 zcmZYB^*|U!U-3N(xGM+qpR?(71pLiVq5koYiHS-+ z``U24_Njp_KxT3?GLt4o>D8M-ekv^7f~f&shsdbY62EdLzc^g2cge&RG8{%G_azHZCyeNss%g?aGR-)7-n(KD=O%AF)mkPq=y(19P_=; z=QXU-8%Z~ksmu~>``?U6{`{Sq6;?u>76?F0PN*(o1Kthlkwjh-Vf@o7GvzFLjE%lW z8L~Gv3=~;!IO`kJxzSb2L7RE+a=Z&!Wykx)NGk-@^?xpyw?x*677=Kq(;=IQ;vQi| z`FA=ac)F4?45Zc?@?9GN*=N7p>Fz!?Qv#O(-?(ZRk~7z@%b_in%i%FT-qBsurQbQ6 z>&@g7JQqo#YE^CGN@v;van8yXGT=+x8ewqpz;00saJ0s+lESz4EDjJ;-u?^KOoMQ>cOoZo9GI9E8wu1G1_ zt8!VGGjUw3|NVQJ zFRC-s3n7_aC~Zw{WUuwA^L1)mm+R6C3egw1+hJb1LbdZE20LA42?LHM%+vZYr%FGZ zxdX5TSqh&sf00Ezq7(`uet^pb$!7L5&HB|Ew%^^{5g7%gnJSzwD+FQEXyRn% z#7S%hBZ-%KiyUfvhe5E)VJBa#D@P+cMY2fTQO=}PrkN97ioUSuH7=Ig2xPF~Qe)%$ z%Pf_bIHwL(Y)s-!mZ53vTVn=AN05$7rE3FiemF&TR-^|sC+j)Rw8xTQPxJO}-ep8L z8)fji&gZv1NGUP5s`FpY#8i1)Wkt1W4~!oEuf>p%T_uE6hMxZTv?fOyx!d(Yz zrw3qwOwG}+E`NKg`c1`zwG9am*{sIQD}y2=8*C6#iX%9MvTPYbk)1C;a?i|Q;Wcs4 z>jgqPrR7va!4~c$nM=7txBLF1r3$`<5@GS6dmK9YV`tOqB~^QG=~d730np{6iw%F| z&Yr91tGq4^AvQi#b+_i)Eu$glpt2!RtB=ZUtQ`x%XYzs>;{$wgH5G(})@D%&pNvLP zqyntIRCeJnf%l#BJn3 z>UXg)bU9pczc7?P69J$g454hN*@& zRxjktX69OXl&?IvBKajwXD$@RQ^D9Bi>)^m$L_2Th8~E=w}se{?Cr48qm$Khgh>YKZ_AhY5Q^H- zv|8=JWyU^pL3S!T12fi1rLTplc) z>`{6=?G>V+z-PmwfgDs9G(NZ?XZTAkELwhNh`K1{cA_x?*s?mWl|Ej*6mQ0B-T7J? zJM?6rhBr8czT{BGFypdsep=iSKaNs=*nPB8@{>(|-i;uU)5a9>Ua{S!JHJml1M|~C zJZk_B{OR+f_^wR#>)5Ch)_%?Esl&FewXz*=t@)esVtz9IwnAvIMz@DgfDx;SK#?JK zQfuWav40HoS7y0Em+!JBX!Y`ym7XIVM|WW1)qg(z6557T4-q5UNKdJnMd2<^JT`V7 zwg?LzOSrqO3*3Q$?f*A+21zG#4-Z+mi}ObpA09YdIvOa3Ndmydc!dT0&wzn6ARK%! zE)EC?z$U{5g7NS%$$=mM@C5)!5sZNcCfu=c-8nmEy^ty;#E7~w`*oXbF-{1Sv0{@& zgo1;0dq+BX-X3*Nv-ba{If{TB>uWqCYEB5FdpUq#NyXm^TapcFfk4lfimKI>+tELAn zlw$Kq%QmK(dx@OBH!nG8OXpH(Ox+^U4SrJ5amC2C+=st(Qt~`ZHiMyA7(Ij22I3_leJRHjGFWVi zm{QkUJw_CAJX;VS4Gjjt6V|!Wi5tWn}`;|U3{$3o3Hv@Cis^r<7==ogWrlp0yXDw9@n%EEpgYGPB0 z#-P|`tz-8!+j$&Fne3W{E^)t#$94}fuz?4HyM5`2==$|CcSu;Pm8k7t>}_iYAbe@( zy6@!7HI!R@oacIfzZ`;PaKY-dw{~~akI8q$bP})a6J2Y=!gZC6rb3fm+C0T%27NIe z;IYiWvK^(*r`l*{&6^HbhqALhQvamnc*Q~+=FS!fL8xc zw}G$6uFbxHZ$mqMT549kV?*Y8136FT8^dV7wjd8#C9eq=(#zVG^L~4GHScKJse*{> z62eE_)r;b3nIDsyWN^Ma`50!G6;EkCACxSMKDrowq1AwDhV_(U7 zQKykJtwo;>h`AND9lZbXjQlkvxB?lbCP1n{iFi6ANpu_pv_fmwAG~}Vz}UTWXPL%U z2IHvnnq}X?y-rk|=o-|epIycSg&`^A8q|A5AB_tx0y2_iNS}yhf<(8Ek8I>n1!wGt?YvKIpyTm{u z)w8?@mV&s#xBO4I8>@Lg1d4mq5#%V+&dy0bzGPx9KUUl8J&40%G#XgM+q(Zd)xL%5 zLDTIrxyzJ%`rhRl2uvU|gp`ToF(zL=3};GQQe?k3^6%d~o}JKx+nDIpy_d^ZThHqgf+O`{!6Q>?_kZA>lLK{q2kzps>$ZrHirUQK^k&}}W z(h`V}1}NO!uiR1+#bi_a7dH}-DZL0g&ah%yI-rykVT+5neY20?B;wy25Rl`iT%T?f zyKVCb4E*^m;nbr2%f1|GvzjCgM7BseEr;!V5sn^X0H>m9u9RBMOSkz0SsSiNlePST&k_@TiNe28 zf=S+ho&@y?*2}b3`$~in%>tv6*JCrWI7C=E^f%OXR%Du0W}tX zBi#b9Ak_c{5UlsBCCSC)D-f*pKvAl7c^I4B8S-Mr7K2Hi{8wyg6bZhSD^(Ge`?40x z^C9-&;7B+pu!M8}CGtHYj^+RX3YJ-<BP~jn))<;VOQ|ExHCw zXJANE>DElGu`25uQxWGrUZk`{k|QP8+EScDKl6hjBi@&+%<%Htb0>Qi_7)@m@yhBj zbyIYe!nw<+e(8dQL7VmyCf>G7Z2$B1xvaWM%RBUfJheSVYghtui0<_R(!~H{$oZE#3okl1P|EE}t)mvgSJ3Jrq71uFWtDuczl#^?|IA#G{S6CS z%vbX}IQ79Cj-qqFsRl~!4Jip5%xH+C;q^BN_8?_Fv1irbRrS+avIzM5zLaAi-I(B^ zqn)n=JGJgz7R5Pg>K?ZTE-(1Ijz!c;O9imO@D`sE`)u=ZGJG8w?gi$UrYl@8CvvB5WIiilT zfLQKE?W;US!MI;8i3`=8iDuK0^Tf2+;c*w`3nj)ODZ;XCkwc~(VULe@RpM~{Xdpc% z<^KW<{9k}U{x7rI2MPXZ2!Rt|#B%35?~p>)@B4%_B|D?zM)_4*2ud3j?o{?=58ejb zo#r<{UwiF16_EE11P{#AKLZ8pWX#qx83EP(T9gz3!vu|7g_^ZekQ$v@bDQVrt`sVqN>+4(4Q6bps2a^l`ch0NfW-`!7>#E zYMzbhRJJQBhTYA#1Ic&C+JoQJQ5`ytN=%%djptm%Z({zkHc{1=k783_8ZpJv@0Y}x zK(s$tU&Oi&`%RyG^2K8l9XF?G5cjaJaI#p3Xg8*YIcF>?>xdoUMJWJH zMaPO}qMb_ag9Jdk5}~|}F|)y1vc1b%>Pe1ekCM1DLFBup(KdxZGJ<*7U%0+(g6GtBZ@>2Rw@&+0UBE)JDd=*;D{~oV={s#pL@A6z!%`QBg%y`W3Zo zg$_jAESL-SVe)!UxJHM*iEBh Date: Mon, 27 Nov 2017 15:54:59 -0700 Subject: [PATCH 03/21] - Add devs back in --- build.sbt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build.sbt b/build.sbt index e9e5ee2..164bfed 100644 --- a/build.sbt +++ b/build.sbt @@ -46,6 +46,14 @@ lazy val baseSettings = binksSettings ++ Seq( Matt Conroy http://www.mattconroy.com + + afrieze + Andrew Frieze + + + kuroshii + Bridger Howell + ) ) From 2718d1e960fd9114779966eb6570ab2cc7bec5cc Mon Sep 17 00:00:00 2001 From: Matt Conroy Date: Mon, 27 Nov 2017 15:55:17 -0700 Subject: [PATCH 04/21] - Fix an OOM error with the kafka tests. --- .../src/test/java/forklift/integration/server/KafkaService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/connectors/kafka/src/test/java/forklift/integration/server/KafkaService.java b/connectors/kafka/src/test/java/forklift/integration/server/KafkaService.java index 9d88f01..e94e4f7 100644 --- a/connectors/kafka/src/test/java/forklift/integration/server/KafkaService.java +++ b/connectors/kafka/src/test/java/forklift/integration/server/KafkaService.java @@ -27,6 +27,7 @@ public void run() { Properties properties = new Properties(); properties.setProperty("broker.id", "1"); properties.setProperty("listeners", "PLAINTEXT://:" + listenPort); + properties.put("log.cleaner.dedupe.buffer.size", 2 * 1024 * 1024L); properties.setProperty("num.network.threads", "3"); properties.setProperty("num.io.threads", "8"); properties.setProperty("socket.send.buffer.bytes", "102400"); From 922afa7738538f74b9ec8310da8559d5b44f3e2d Mon Sep 17 00:00:00 2001 From: Matt Conroy Date: Mon, 27 Nov 2017 16:03:04 -0700 Subject: [PATCH 05/21] - Switch to travis build numbers for versions. --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 164bfed..cba10d6 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ lazy val baseSettings = binksSettings ++ Seq( organization := "com.github.dcshock", - version := "3.0", + version := sys.env.getOrElse("TRAVIS_BUILD_NUMBER", "SNAPSHOT"), scalaVersion := "2.11.7", javacOptions ++= Seq("-source", "1.8"), javacOptions in compile ++= Seq("-g:lines,vars,source", "-deprecation"), From da8a85d27184377fbf5c3553939a6f21874a30a7 Mon Sep 17 00:00:00 2001 From: Matt Conroy Date: Mon, 27 Nov 2017 16:15:11 -0700 Subject: [PATCH 06/21] - Remove binks --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index cba10d6..4f7497b 100644 --- a/build.sbt +++ b/build.sbt @@ -1,4 +1,4 @@ -lazy val baseSettings = binksSettings ++ Seq( +lazy val baseSettings = Seq( organization := "com.github.dcshock", version := sys.env.getOrElse("TRAVIS_BUILD_NUMBER", "SNAPSHOT"), scalaVersion := "2.11.7", From 8a90d7ab0ff92a991840b08f000b2db5d4a8af6d Mon Sep 17 00:00:00 2001 From: Matt Conroy Date: Tue, 28 Nov 2017 09:12:55 -0700 Subject: [PATCH 07/21] - Go back to having a user specify the release version. It's a lot easier to manage versions from a backward compatibility perspective with a user completing the analysis. --- README.adoc | 4 ++++ build.sbt | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index 7311188..f4fb480 100644 --- a/README.adoc +++ b/README.adoc @@ -10,6 +10,10 @@ link:doc/forklift.adoc[Documentation] == Releases link:doc/prev_releases.adoc[Previous Releases] +* *Novemeber 28th 2017* - v2.3 +** Update build to automate releases to sonatype +** Align release numbers of subcomponents to avoid having a version 1 of a plugin while a version 2 of the core is being used. + * *September 28th 2017* - v2.2 ** Eliminate unneeded queries to elasticsearch, improving preformance the performance replay processing ** Fix bug in re-runing failed retries in the retry plugin diff --git a/build.sbt b/build.sbt index 4f7497b..3bc7c11 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ lazy val baseSettings = Seq( organization := "com.github.dcshock", - version := sys.env.getOrElse("TRAVIS_BUILD_NUMBER", "SNAPSHOT"), + version := "2.3", scalaVersion := "2.11.7", javacOptions ++= Seq("-source", "1.8"), javacOptions in compile ++= Seq("-g:lines,vars,source", "-deprecation"), From dad5f8867212ae0a4f8348b82bbffa775c3973c4 Mon Sep 17 00:00:00 2001 From: Matt Conroy Date: Tue, 28 Nov 2017 15:56:50 -0700 Subject: [PATCH 08/21] - Update the sonatype/gpg info. --- .travis.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1344f79..1c0f80b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,13 @@ language: scala scala: - - "2.11.4" +- 2.11.4 jdk: - - oraclejdk8 +- oraclejdk8 sudo: false - env: global: - - secure: uV1Qhx4Bgfiyc+X0GIS1LqjCfMJSm9fIdgoe63CPEPwhf6qTKfzpYOGOos6UTotvSSmIeLmUEUjJurYM1UOnRtltXClpz0lL/WrKDhoyGHdWwk+HrHAOkXhUncTloOA15gNfDS/8JeatT5J3FWXoLVXJBRBN41/uvZV0FTFKdiE= - - secure: oslYlw8hd54oQZUygcCYbYZiXRpUMqgDyeTxF7CdAtVpFnxK7lTnmJUXXp5S22CXEcK/n/8+V6l+DmvnWYOJ73QbDc11rrYOgrHWE3KH7Cxs0W5JonZUB5WZ2kMM3XcZlfXmggiJqtHNxHBLVoS3khM0+ta/RVcvgNCQM5ZlMao= - - secure: yOJT1AMsLq6x6SGr65dsRhtqvn5i3N9JEqtB3XPRMnNaHzQIg4RH0PoJMTzM0bLKG/gIEsMdB257nT7XkZjOVusrZLaf9N9c55zHNSAmoy1VlaPezWSePkrkKIscZIHUZD0psliGcE6mjO1MuqMXXipd00Zs8hgWrNxu6wlB6Is= + - secure: f9pej7Im4e7dwwv9s/ctP1Mh+RgLF3tlUCPsewLZbHhv1plgbGQnKW10j0B/GZIy+Bc8M1+H/EnNhsHp9QE7mUOMJXM8kW/OlcBMIwxYkHFGBhHND2j7GXwUCXbjctwryjFxOg3hHMwQy+z3K4r9ShdrOnJ9cqLKo3OkTZ5Vi+M= + - secure: olvXVQJ6c/uCWXrm7dw/LsWCKEguylLybPoiM40trNIrHMpEcDWQ/2u+fvcQ9KWm8ztQeBSmXwlkHMCMicbh2iTX9W1hmxfzrFE8N33iVet74N72tsRfrFxcUI3HXjPfB3hCF3NbPAeplfxqYXRY2OzNOueigHb4oW6jMf1a97I= + - secure: JhqZSvXjlGDMHvVli9Cuv6hwOeCWLywagZsocigtIOI9b6lO4um9qokcNJTg+rbQE+bd7E65zMviJPnwMWzbuTdI4LG5Ldqxr4IQbFrkythWSDMWLHsziFPAYHPZMZPf0tALzf1F7bMj0yo4YulqeeAr/Iy7bpAQPVWPf+Lqox0= after_success: - - ./bin/checkPublish +- ./bin/checkPublish From 6667a0226bdf3454c5d404d54dff280bd86fca71 Mon Sep 17 00:00:00 2001 From: Matt Conroy Date: Tue, 28 Nov 2017 16:26:02 -0700 Subject: [PATCH 09/21] - Add the publish settings. --- build.sbt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build.sbt b/build.sbt index 3bc7c11..c5c47bf 100644 --- a/build.sbt +++ b/build.sbt @@ -14,6 +14,13 @@ lazy val baseSettings = Seq( "Fuse" at "http://repo.fusesource.com/nexus/content/groups/public", "Confluent Maven Repo" at "http://packages.confluent.io/maven/" ), + publishTo := { + val nexus = "https://oss.sonatype.org/" + if (isSnapshot.value) + Some("snapshots" at nexus + "content/repositories/snapshots") + else + Some("releases" at nexus + "service/local/staging/deploy/maven2") + }, publishMavenStyle := true, credentials += Credentials( "Sonatype Nexus Repository Manager", From 89abeb1412997bb54e5d1d8c81f77dfab84b747a Mon Sep 17 00:00:00 2001 From: Matt Conroy Date: Thu, 30 Nov 2017 12:13:08 -0700 Subject: [PATCH 10/21] - Skip publishing the root artifact since it is blank. --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index c5c47bf..4192e3e 100644 --- a/build.sbt +++ b/build.sbt @@ -71,3 +71,4 @@ lazy val stats = project.dependsOn(core) in file("plugins/stats") settings baseS lazy val activemq = project.dependsOn(core) in file("connectors/activemq") settings baseSettings lazy val kafka = project.dependsOn(core) in file("connectors/kafka") settings baseSettings lazy val server = project.dependsOn(core, replay, retry, stats, activemq, kafka) in file("server") settings baseSettings +lazy val root = project in file(".") aggregate(core, replay, retry, stats, activemq, kafka, server) settings(skip in publish := true) From d7d19541c9b107e42d60828ba9a189fafa165023 Mon Sep 17 00:00:00 2001 From: Matt Conroy Date: Thu, 30 Nov 2017 12:39:23 -0700 Subject: [PATCH 11/21] - Need a base publishing directory because "skip in publish" happens further into the lifecycle. --- build.sbt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 4192e3e..62f4a84 100644 --- a/build.sbt +++ b/build.sbt @@ -71,4 +71,8 @@ lazy val stats = project.dependsOn(core) in file("plugins/stats") settings baseS lazy val activemq = project.dependsOn(core) in file("connectors/activemq") settings baseSettings lazy val kafka = project.dependsOn(core) in file("connectors/kafka") settings baseSettings lazy val server = project.dependsOn(core, replay, retry, stats, activemq, kafka) in file("server") settings baseSettings -lazy val root = project in file(".") aggregate(core, replay, retry, stats, activemq, kafka, server) settings(skip in publish := true) + +lazy val rootSettings = baseSettings ++ Seq( + skip in publish := true +) +lazy val root = project in file(".") aggregate(core, replay, retry, stats, activemq, kafka, server) settings rootSettings From 55aadbcd61524b11e0b5d4073b41b1d8e6a39471 Mon Sep 17 00:00:00 2001 From: Bridger Howell Date: Fri, 1 Dec 2017 13:24:45 -0700 Subject: [PATCH 12/21] Switch to low-level ES rest client --- plugins/replay/build.sbt | 3 +- .../java/forklift/replay/ReplayESWriter.java | 193 ++++++++++++------ 2 files changed, 135 insertions(+), 61 deletions(-) diff --git a/plugins/replay/build.sbt b/plugins/replay/build.sbt index 3cfeed6..0c33f36 100644 --- a/plugins/replay/build.sbt +++ b/plugins/replay/build.sbt @@ -1,7 +1,8 @@ name := "forklift-replay" libraryDependencies ++= Seq( - "org.elasticsearch" % "elasticsearch" % "2.4.1", + "org.elasticsearch" % "elasticsearch" % "2.4.1", // retained for now, to support embedded ES clusters + "org.elasticsearch.client" % "rest" % "5.0.2", "com.fasterxml.jackson.core" % "jackson-databind" % "2.7.3", "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % "2.7.3" ) diff --git a/plugins/replay/src/main/java/forklift/replay/ReplayESWriter.java b/plugins/replay/src/main/java/forklift/replay/ReplayESWriter.java index e6746ef..f594a38 100644 --- a/plugins/replay/src/main/java/forklift/replay/ReplayESWriter.java +++ b/plugins/replay/src/main/java/forklift/replay/ReplayESWriter.java @@ -1,94 +1,167 @@ package forklift.replay; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.elasticsearch.index.VersionType; -import org.elasticsearch.index.engine.VersionConflictEngineException; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.HttpStatus; +import org.apache.http.entity.ContentType; +import org.apache.http.message.BasicHeader; +import org.apache.http.nio.entity.NStringEntity; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.InetSocketAddress; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.HashMap; public class ReplayESWriter extends ReplayStoreThread { private static final Logger log = LoggerFactory.getLogger(ReplayES.class); + private static final ObjectMapper mapper = new ObjectMapper(); - private final TransportClient client; + private final RestClient restClient; public ReplayESWriter(String hostname) { - this(hostname, 9300, "elasticsearch"); + this(hostname, 9200, "elasticsearch"); } + // clusterName is retained in constructor for compatibility public ReplayESWriter(String hostname, int port, String clusterName) { - final Settings settings = Settings.settingsBuilder() - .put("cluster.name", clusterName).build(); - - this.client = TransportClient.builder() - .settings(settings) - .build() - .addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(hostname, port))); + this.restClient = RestClient.builder(new HttpHost(hostname, port, "http")) + .setRequestConfigCallback(requestConfig -> requestConfig + .setConnectTimeout(3_000) + .setSocketTimeout(20_000)) + .setDefaultHeaders(new Header[] { + new BasicHeader("Accept", "application/json; charset=utf-8"), + new BasicHeader("Content-Type", "application/json; charset=utf-8") + }) + .setMaxRetryTimeoutMillis(20_000) + .build(); } @Override - protected void poll(ReplayESWriterMsg t) { - final String replayVersion = t.getFields().get("forklift-replay-version"); + protected void poll(ReplayESWriterMsg replayMessage) { + final String replayVersion = replayMessage.getFields().get("forklift-replay-version"); if ("3".equals(replayVersion)) { // latest version - String indexDate = t.getFields().get("first-processed-date"); - if (indexDate == null) - indexDate = LocalDate.now().format(DateTimeFormatter.BASIC_ISO_DATE); - final String index = "forklift-replay-" + indexDate; + processVersion3Replay(replayMessage); + } else if ("2".equals(replayVersion) || replayVersion == null) { // older versions didn't have the replay version or didn't persist it with the message + processVersion2OrEarlierReplay(replayMessage); + } else { + log.error("Unrecognized replay version: '{}' for message ID '{}' with fields '{}'", + replayVersion, replayMessage.getId(), replayMessage.getFields()); + } + } - try { - // Index the new information. - client.prepareIndex(index, "log") - .setVersion(t.getVersion()).setVersionType(VersionType.EXTERNAL_GTE) - .setId(t.getId()).setSource(t.getFields()).execute().actionGet(); - } catch (VersionConflictEngineException expected) { - log.debug("Newer replay message already exists", expected); + private void processVersion3Replay(final ReplayESWriterMsg replayMessage) { + String indexDate = replayMessage.getFields().get("first-processed-date"); + if (indexDate == null) { + indexDate = LocalDate.now().format(DateTimeFormatter.BASIC_ISO_DATE); + } + final String index = "forklift-replay-" + indexDate; + + indexReplayMessage(replayMessage, index); + } + + private void processVersion2OrEarlierReplay(ReplayESWriterMsg replayMessage) { + final String index = "forklift-replay-" + LocalDate.now().format(DateTimeFormatter.BASIC_ISO_DATE); + + for (String existingMessageIndex : searchForIndexesWithId(replayMessage.getId())) { + if (!existingMessageIndex.equals(index)) { + deleteMessageInIndex(replayMessage.getId(), existingMessageIndex); } - } else if ("2".equals(replayVersion) || replayVersion == null) { // older versions didn't have the replay version or didn't persist it with the message - final String index = "forklift-replay-" + LocalDate.now().format(DateTimeFormatter.BASIC_ISO_DATE); + } - // In order to ensure there is only one replay msg for a given id we have to clean the msg id from - // any previously created indexes. + indexReplayMessage(replayMessage, index); + } + + private void indexReplayMessage(final ReplayESWriterMsg replayMessage, final String index) { + final String id = replayMessage.getId(); + final String endpoint = "/" + index + "/log/" + id; + + final Map queryParams = new HashMap<>(); + queryParams.put("version_type", "external_gte"); + queryParams.put("version", "" + replayMessage.getVersion()); + + final HttpEntity entity; + try { + final String entityContents = mapper.writeValueAsString(replayMessage.getFields()); + entity = new NStringEntity(entityContents, ContentType.APPLICATION_JSON); + } catch (JsonProcessingException e) { + log.error("Could not write replay fields to JSON: (id {}, fields {})", replayMessage.getId(), replayMessage.getFields(), e); + return; + } + + try { + restClient.performRequest("PUT", endpoint, queryParams, entity); + } catch (ResponseException e) { + // conflicts are normal when versioned index requests are submitted in the wrong order + if (e.getResponse().getStatusLine().getStatusCode() == HttpStatus.SC_CONFLICT) { + return; + } + + log.error("Error indexing replay message (id {}, fields {})", id, replayMessage.getFields(), e); + } catch (IOException e) { + log.error("Error indexing replay message (id {}, fields {})", id, replayMessage.getFields(), e); + } + } + + private Iterable searchForIndexesWithId(final String id) { + final String endpoint = "/forklift-replay*/_search"; + final Map queryParams = new HashMap<>(); + queryParams.put("q", "_id:" + id); + queryParams.put("size", "50"); + + try { + final Response response = restClient.performRequest("GET", endpoint, queryParams); try { - final SearchResponse resp = client.prepareSearch("forklift-replay*").setTypes("log") - .setQuery(QueryBuilders.termQuery("_id", t.getId())) - .setFrom(0).setSize(50).setExplain(false) - .execute() - .actionGet(); - - if (resp != null && resp.getHits() != null && resp.getHits().getHits() != null) { - for (SearchHit hit : resp.getHits().getHits()) { - if (!hit.getIndex().equals(index)) - client.prepareDelete(hit.getIndex(), "log", t.getId()).execute().actionGet(); - } + final JsonNode jsonResponse = mapper.readTree(response.getEntity().getContent()); + final JsonNode hits = jsonResponse.get("hits").get("hits"); + final List indexes = new ArrayList<>(); + + for (final JsonNode hit : hits) { + final String hitIndex = hit.get("_index").asText(); + indexes.add(hitIndex); } + + return indexes; } catch (Exception e) { - log.error("", e); - log.error("Unable to search for old replay logs {}", t.getId()); + log.error("Error parsing elasticsearch response to search for id {}", id, e); } + } catch (IOException e) { + log.error("Error searching for indexes for id {}", id, e); + } - try { - // Index the new information. - client.prepareIndex(index, "log") - .setVersion(t.getVersion()).setVersionType(VersionType.EXTERNAL_GTE) - .setId(t.getId()).setSource(t.getFields()).execute().actionGet(); - } catch (VersionConflictEngineException expected) { - log.debug("Newer replay message already exists", expected); - } - } else { - log.error("Unrecognized replay version: '{}' for message ID '{}' with fields '{}' ", replayVersion, t.getId(), t.getFields()); + return Collections.emptyList(); + } + + private void deleteMessageInIndex(final String id, final String index) { + final String endpoint = "/" + index + "/log/" + id; + try { + restClient.performRequest("DELETE", endpoint); + } catch (IOException e) { + log.error("Error deleting message with id {} for index {}", id, index, e); } } public void shutdown() { - if (client != null) - client.close(); + if (restClient != null) { + try { + restClient.close(); + } catch (IOException e) { + log.error("Couldn't shutdown Elasticsearch REST client", e); + } + } } } From 8372c5fc9a3e5b710710359dbd9fe4d6275bc522 Mon Sep 17 00:00:00 2001 From: Bridger Howell Date: Fri, 1 Dec 2017 13:25:01 -0700 Subject: [PATCH 13/21] Remove ThreadSafe annoation no longer present in apache http --- server/src/main/java/forklift/ForkliftServer.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/forklift/ForkliftServer.java b/server/src/main/java/forklift/ForkliftServer.java index 900f7b3..b85aaa0 100644 --- a/server/src/main/java/forklift/ForkliftServer.java +++ b/server/src/main/java/forklift/ForkliftServer.java @@ -16,7 +16,6 @@ import forklift.retry.RetryHandler; import forklift.stats.StatsCollector; import org.apache.activemq.broker.BrokerService; -import org.apache.http.annotation.ThreadSafe; import java.io.File; import java.io.FileNotFoundException; @@ -41,7 +40,6 @@ * * @author zdavep */ -@ThreadSafe public final class ForkliftServer { private ExecutorService executor = Executors.newSingleThreadExecutor(); From 801e917bace82c072d7e6e2b3b09ee65af99c2e1 Mon Sep 17 00:00:00 2001 From: Bridger Howell Date: Fri, 1 Dec 2017 15:30:34 -0700 Subject: [PATCH 14/21] Remove false-positive error when deleting an already deleted log --- .../src/main/java/forklift/replay/ReplayESWriter.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/replay/src/main/java/forklift/replay/ReplayESWriter.java b/plugins/replay/src/main/java/forklift/replay/ReplayESWriter.java index f594a38..df1ad2f 100644 --- a/plugins/replay/src/main/java/forklift/replay/ReplayESWriter.java +++ b/plugins/replay/src/main/java/forklift/replay/ReplayESWriter.java @@ -150,6 +150,14 @@ private void deleteMessageInIndex(final String id, final String index) { final String endpoint = "/" + index + "/log/" + id; try { restClient.performRequest("DELETE", endpoint); + } catch (ResponseException e) { + // sometimes we might try to delete something that was already deleted by something else; + // in which case there is no error + if (e.getResponse().getStatusLine().getStatusCode() == HttpStatus.SC_NOT_FOUND) { + return; + } + + log.error("Error deleting message with id {} for index {}", id, index, e); } catch (IOException e) { log.error("Error deleting message with id {} for index {}", id, index, e); } From 0254de626590e07afb98f3c97782359f20aba5c0 Mon Sep 17 00:00:00 2001 From: Bridger Howell Date: Fri, 1 Dec 2017 16:27:35 -0700 Subject: [PATCH 15/21] Remove clusterName from ReplayES and ReplayESWriter interfaces --- .../main/java/forklift/replay/ReplayES.java | 38 +++++++++++++++++-- .../java/forklift/replay/ReplayESWriter.java | 31 +++++++++++---- 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/plugins/replay/src/main/java/forklift/replay/ReplayES.java b/plugins/replay/src/main/java/forklift/replay/ReplayES.java index c4cc227..7e25a9b 100644 --- a/plugins/replay/src/main/java/forklift/replay/ReplayES.java +++ b/plugins/replay/src/main/java/forklift/replay/ReplayES.java @@ -49,7 +49,7 @@ public class ReplayES { private final Consumer consumer; /** - * Constructs a new ReplayES instance using the default transport port (9200) for + * Constructs a new ReplayES instance using the default REST port (9200) for * sending messages to elasticsearch. * * @param clientOnly whether the plugin is only an elasticsearch client, or an @@ -57,6 +57,8 @@ public class ReplayES { * @param hostname the address of the ES host to send messages to * @param clusterName the name of the elasticsearch cluster to send logs to * @param connector the connector to use for queuing messages + * + * @deprecated use {@link #ReplayES(boolean, String, ForkliftConnectorI)} instead */ public ReplayES(boolean clientOnly, String hostname, String clusterName, ForkliftConnectorI connector) { this(clientOnly, hostname, 9200, clusterName, connector); @@ -72,8 +74,38 @@ public ReplayES(boolean clientOnly, String hostname, String clusterName, Forklif * @param port the port number of the ES transport port for the given host * @param clusterName the name of the elasticsearch cluster to send logs to * @param connector the connector to use for queuing messages + * + * @deprecated use {@link #ReplayES(boolean, String, int, ForkliftConnectorI)} instead */ public ReplayES(boolean clientOnly, String hostname, int port, String clusterName, ForkliftConnectorI connector) { + this(clientOnly, hostname, port, connector); + } + + /** + * Constructs a new ReplayES instance using the default REST port (9200) for + * sending messages to elasticsearch. + * + * @param clientOnly whether the plugin is only an elasticsearch client, or an + * embedded elasticsearch node should be started + * @param hostname the address of the ES host to send messages to + * @param connector the connector to use for queuing messages + */ + public ReplayES(boolean clientOnly, String hostname, ForkliftConnectorI connector) { + this(clientOnly, hostname, 9200, connector); + } + + + /** + * Constructs a new ReplayES instance sending messages to elasticsearch based + * on the given parameters. + * + * @param clientOnly whether the plugin is only an elasticsearch client, or an + * embedded elasticsearch node should be started + * @param hostname the address of the ES host to send messages to + * @param port the port number of the ES transport port for the given host + * @param connector the connector to use for queuing messages + */ + public ReplayES(boolean clientOnly, String hostname, int port, ForkliftConnectorI connector) { /* * Setup the connection to the server. If we are only a client we'll not setup a node locally to run. * This will help developers and smaller setups avoid the pain of setting up elastic search. @@ -96,7 +128,7 @@ public ReplayES(boolean clientOnly, String hostname, int port, String clusterNam } } - this.writer = new ReplayESWriter(hostname, port, clusterName); + this.writer = new ReplayESWriter(hostname, port); this.writer.start(); Runtime.getRuntime().addShutdownHook(new Thread() { @@ -144,7 +176,7 @@ public void shutdown() { } catch (InterruptedException ignored) { } - this.writer.shutdown(); + this.writer.close(); } @LifeCycle(value=ProcessStep.Pending, annotation=Replay.class) diff --git a/plugins/replay/src/main/java/forklift/replay/ReplayESWriter.java b/plugins/replay/src/main/java/forklift/replay/ReplayESWriter.java index df1ad2f..775c944 100644 --- a/plugins/replay/src/main/java/forklift/replay/ReplayESWriter.java +++ b/plugins/replay/src/main/java/forklift/replay/ReplayESWriter.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.Closeable; import java.io.IOException; import java.time.LocalDate; import java.time.format.DateTimeFormatter; @@ -27,22 +28,35 @@ import java.util.Map; import java.util.HashMap; -public class ReplayESWriter extends ReplayStoreThread { +public class ReplayESWriter extends ReplayStoreThread implements Closeable { private static final Logger log = LoggerFactory.getLogger(ReplayES.class); private static final ObjectMapper mapper = new ObjectMapper(); private final RestClient restClient; + /** + * Creates a log writer that sends replay information to elasticsearch over REST, + * using the given hostname and port {@code 9200}. + * + * @param hostname the name or address of the host to connect to + */ public ReplayESWriter(String hostname) { - this(hostname, 9200, "elasticsearch"); + this(hostname, 9200); } - // clusterName is retained in constructor for compatibility - public ReplayESWriter(String hostname, int port, String clusterName) { + /** + * Creates a log writer that sends replay information to elasticsearch over REST, + * using the given hostname and port. + * + * @param hostname the name or address of the host to connect to + * @param port the connection port + */ + public ReplayESWriter(String hostname, int port) { this.restClient = RestClient.builder(new HttpHost(hostname, port, "http")) - .setRequestConfigCallback(requestConfig -> requestConfig - .setConnectTimeout(3_000) - .setSocketTimeout(20_000)) + .setRequestConfigCallback(requestConfig -> + requestConfig + .setConnectTimeout(3_000) + .setSocketTimeout(20_000)) .setDefaultHeaders(new Header[] { new BasicHeader("Accept", "application/json; charset=utf-8"), new BasicHeader("Content-Type", "application/json; charset=utf-8") @@ -163,7 +177,8 @@ private void deleteMessageInIndex(final String id, final String index) { } } - public void shutdown() { + @Override + public void close() { if (restClient != null) { try { restClient.close(); From 6d20715a172f8e662d872a8cdf714685363808c8 Mon Sep 17 00:00:00 2001 From: Matt Conroy Date: Sat, 2 Dec 2017 19:59:07 -0700 Subject: [PATCH 16/21] - 3.0 due to breaking backward compatibility. --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 3bc7c11..9e45658 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ lazy val baseSettings = Seq( organization := "com.github.dcshock", - version := "2.3", + version := "3.0", scalaVersion := "2.11.7", javacOptions ++= Seq("-source", "1.8"), javacOptions in compile ++= Seq("-g:lines,vars,source", "-deprecation"), From cca355d9c02a7803a56d3f038e8b7b2dde4ff618 Mon Sep 17 00:00:00 2001 From: Matt Conroy Date: Sat, 2 Dec 2017 20:01:33 -0700 Subject: [PATCH 17/21] - Release notes. --- README.adoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.adoc b/README.adoc index f4fb480..c5f3b70 100644 --- a/README.adoc +++ b/README.adoc @@ -10,6 +10,9 @@ link:doc/forklift.adoc[Documentation] == Releases link:doc/prev_releases.adoc[Previous Releases] +* *December 2nd 2017* - v3.0 +** Upgrade replay logging to utilize Elastic Search 5 + * *Novemeber 28th 2017* - v2.3 ** Update build to automate releases to sonatype ** Align release numbers of subcomponents to avoid having a version 1 of a plugin while a version 2 of the core is being used. From 4b7067076a1251ba79b0ed675271e62dfd65f4d0 Mon Sep 17 00:00:00 2001 From: Bridger Howell Date: Wed, 7 Jun 2017 14:45:08 -0600 Subject: [PATCH 18/21] Make GroupedTopic hashable --- .../java/forklift/source/sources/GroupedTopicSource.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/forklift/source/sources/GroupedTopicSource.java b/core/src/main/java/forklift/source/sources/GroupedTopicSource.java index 9377e92..0d961dc 100644 --- a/core/src/main/java/forklift/source/sources/GroupedTopicSource.java +++ b/core/src/main/java/forklift/source/sources/GroupedTopicSource.java @@ -50,6 +50,11 @@ public void overrideGroup(String group) { this.group = group; } + @Override + public int hashCode() { + return Objects.hash(name, group); + } + @Override public boolean equals(Object o) { if (this == o) From 65c3816a8c68c4d5dd1f45079afbad1ae7ca62aa Mon Sep 17 00:00:00 2001 From: Bridger Howell Date: Fri, 9 Jun 2017 15:13:34 -0600 Subject: [PATCH 19/21] Retrieve controllers according to topic and group name --- .../forklift/connectors/KafkaConnector.java | 66 +++++++++++-------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/connectors/kafka/src/main/java/forklift/connectors/KafkaConnector.java b/connectors/kafka/src/main/java/forklift/connectors/KafkaConnector.java index 8044d32..ba292bd 100644 --- a/connectors/kafka/src/main/java/forklift/connectors/KafkaConnector.java +++ b/connectors/kafka/src/main/java/forklift/connectors/KafkaConnector.java @@ -39,24 +39,24 @@ public class KafkaConnector implements ForkliftConnectorI { private final String kafkaHosts; private final String schemaRegistries; - private final String groupId; + private final String defaultGroupId; private KafkaProducer kafkaProducer; private KafkaController controller; private ForkliftSerializer serializer; - private Map controllers = new HashMap<>(); + private Map controllers = new HashMap<>(); /** * Constructs a new instance of the KafkaConnector * * @param kafkaHosts list of kafka servers in host:port,... format * @param schemaRegistries list of schema registry servers in http://host:port,... format - * @param groupId the groupId to use when subscribing to topics + * @param defaultGroupId the default groupId to use when subscribing to topics */ - public KafkaConnector(String kafkaHosts, String schemaRegistries, String groupId) { + public KafkaConnector(String kafkaHosts, String schemaRegistries, String defaultGroupId) { this.kafkaHosts = kafkaHosts; this.schemaRegistries = schemaRegistries; - this.groupId = groupId; + this.defaultGroupId = defaultGroupId; this.serializer = new KafkaSerializer(this, newSerializer(), newDeserializer()); } @@ -101,7 +101,7 @@ private KafkaProducer createKafkaProducer() { return new KafkaProducer(producerProperties); } - private KafkaController createController(String topicName) { + private KafkaController createController(String topicName, String groupId) { Properties props = new Properties(); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaHosts); props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); @@ -137,48 +137,41 @@ public synchronized void stop() throws ConnectorException { @Override public ForkliftConsumerI getConsumerForSource(SourceI source) throws ConnectorException { - return source - .apply(QueueSource.class, queue -> getQueue(queue.getName())) - .apply(TopicSource.class, topic -> getTopic(topic.getName())) - .apply(GroupedTopicSource.class, topic -> getGroupedTopic(topic)) - .apply(RoleInputSource.class, roleSource -> { - final ForkliftConsumerI rawConsumer = getConsumerForSource(roleSource.getActionSource(this)); - return new RoleInputConsumerWrapper(rawConsumer); - }) - .elseUnsupportedError(); + if (source instanceof RoleInputSource) { + final RoleInputSource roleSource = (RoleInputSource) source; + final ForkliftConsumerI rawConsumer = getConsumerForSource(roleSource.getActionSource(this)); + return new RoleInputConsumerWrapper(rawConsumer); + } + return getGroupedTopic(mapToGroupedTopic(source)); } public synchronized ForkliftConsumerI getGroupedTopic(GroupedTopicSource source) throws ConnectorException { if (!source.groupSpecified()) { - source.overrideGroup(groupId); + source.overrideGroup(defaultGroupId); } - if (!source.getGroup().equals(groupId)) { //TODO actually support GroupedTopics - throw new ConnectorException("Unexpected group '" + source.getGroup() + "'; only the connector group '" + groupId + "' is allowed"); - } - - KafkaController controller = controllers.get(source.getName()); + KafkaController controller = controllers.get(source); if (controller != null && controller.isRunning()) { - log.warn("Consumer for topic already exists under this controller's groupname. Messages will be divided amongst consumers."); + log.warn("Consumer for topic and group already exists. Messages will be divided amongst consumers."); } else { - controller = createController(source.getName()); - this.controllers.put(source.getName(), controller); + controller = createController(source.getName(), source.getGroup()); + this.controllers.put(source, controller); controller.start(); } + return new KafkaTopicConsumer(source.getName(), controller); } @Override public ForkliftConsumerI getQueue(String name) throws ConnectorException { - return getGroupedTopic(new GroupedTopicSource(name, groupId)); + return getGroupedTopic(mapToGroupedTopic(new QueueSource(name))); } @Override public ForkliftConsumerI getTopic(String name) throws ConnectorException { - return getGroupedTopic(new GroupedTopicSource(name, groupId)); + return getGroupedTopic(mapToGroupedTopic(new TopicSource(name))); } - @Override public ForkliftProducerI getQueueProducer(String name) { return getTopicProducer(name); @@ -200,11 +193,28 @@ public ActionSource mapSource(LogicalSource source) { } protected GroupedTopicSource mapRoleInputSource(RoleInputSource roleSource) { - return new GroupedTopicSource("forklift-role-" + roleSource.getRole(), groupId); + return new GroupedTopicSource("forklift-role-" + roleSource.getRole(), defaultGroupId); } @Override public boolean supportsResponse() { return true; } + + /* visible for testing */ + protected GroupedTopicSource mapToGroupedTopic(SourceI source) { + return source + .apply(QueueSource.class, queueSource -> new GroupedTopicSource(queueSource.getName(), defaultGroupId)) + .apply(TopicSource.class, topicSource -> topicToGroupedTopic(topicSource)) + .apply(GroupedTopicSource.class, groupedTopicSource -> groupedTopicSource) + .elseUnsupportedError(); + } + + private GroupedTopicSource topicToGroupedTopic(TopicSource topicSource) { + if (topicSource.getContextClass() == null) { + return new GroupedTopicSource(topicSource.getName(), defaultGroupId); + } + final String groupName = defaultGroupId + "-" + topicSource.getContextClass().getSimpleName(); + return new GroupedTopicSource(topicSource.getName(), groupName); + } } From daaf595d9afbda0f52ddc2c9f007fdf0386d4b5b Mon Sep 17 00:00:00 2001 From: Bridger Howell Date: Mon, 16 Oct 2017 10:19:27 -0600 Subject: [PATCH 20/21] Add tests for that sources are mapped to GroupedTopics correctly --- .../connectors/KafkaConnectorTests.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 connectors/kafka/src/test/java/forklift/connectors/KafkaConnectorTests.java diff --git a/connectors/kafka/src/test/java/forklift/connectors/KafkaConnectorTests.java b/connectors/kafka/src/test/java/forklift/connectors/KafkaConnectorTests.java new file mode 100644 index 0000000..6abcdac --- /dev/null +++ b/connectors/kafka/src/test/java/forklift/connectors/KafkaConnectorTests.java @@ -0,0 +1,58 @@ +package forklift.connectors; + +import forklift.source.sources.GroupedTopicSource; +import forklift.source.sources.QueueSource; +import forklift.source.sources.TopicSource; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class KafkaConnectorTests { + private static final String GROUP_ID = "test-default"; + private KafkaConnector connector; + + @Before + public void setup() throws Exception { + connector = new KafkaConnector("blah", "blah", GROUP_ID); + } + + @Test + public void testQueueMapping() { + final String topicName = "test-topic"; + + final GroupedTopicSource mappedSource = connector.mapToGroupedTopic(new QueueSource(topicName)); + Assert.assertEquals(new GroupedTopicSource(topicName, GROUP_ID), mappedSource); + } + + @Test + public void testTopicMapping() { + final String topicName = "test-topic"; + final TopicSource consumerSource = new TopicSource(topicName); + consumerSource.setContextClass(FakeConsumer.class); + + final GroupedTopicSource mappedConsumerSource = connector.mapToGroupedTopic(consumerSource); + Assert.assertEquals(new GroupedTopicSource(topicName, "test-default-FakeConsumer"), mappedConsumerSource); + } + + @Test + public void testDefaultTopicMapping() { + final String topicName = "test-topic"; + final TopicSource anonymousSource = new TopicSource(topicName); + + final GroupedTopicSource mappedConsumerSource = connector.mapToGroupedTopic(anonymousSource); + Assert.assertEquals(new GroupedTopicSource(topicName, GROUP_ID), mappedConsumerSource); + } + + @Test + public void testGroupedTopicMapping() { + final String topicName = "test-topic"; + final String groupId = "test-group"; + final GroupedTopicSource unmappedSource = new GroupedTopicSource(topicName, groupId); + + final GroupedTopicSource mappedSource = connector.mapToGroupedTopic(unmappedSource); + Assert.assertEquals(unmappedSource, mappedSource); + } + + private final class FakeConsumer {} +} From 34dd22ba2e65874fe2ac1d1ecaf69db4162121ea Mon Sep 17 00:00:00 2001 From: Bridger Howell Date: Mon, 4 Dec 2017 09:35:36 -0700 Subject: [PATCH 21/21] Add info for 3.1 release --- README.adoc | 3 +++ build.sbt | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index c5f3b70..f94fa4d 100644 --- a/README.adoc +++ b/README.adoc @@ -10,6 +10,9 @@ link:doc/forklift.adoc[Documentation] == Releases link:doc/prev_releases.adoc[Previous Releases] +* *December 4th 2017* - v3.1 +** Kafka: Add internal support for separate dedicated consumers on the same topic + * *December 2nd 2017* - v3.0 ** Upgrade replay logging to utilize Elastic Search 5 diff --git a/build.sbt b/build.sbt index 4c4d145..e576c05 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ lazy val baseSettings = Seq( organization := "com.github.dcshock", - version := "3.0", + version := "3.1", scalaVersion := "2.11.7", javacOptions ++= Seq("-source", "1.8"), javacOptions in compile ++= Seq("-g:lines,vars,source", "-deprecation"),