Fix resolution of package configuration provider secrets

workers/advisor/src/main/kotlin/advisor/AdvisorRunner.kt

@@ -47,7 +47,7 @@ internal class AdvisorRunner {
  logger.error("The following advisors could not be resolved: {}.", invalidAdvisors)
  }
 
- val pluginConfigs = context.resolveConfigSecrets(config.config)
+ val pluginConfigs = context.resolvePluginConfigSecrets(config.config)
  val advisorConfig = AdvisorConfiguration(
  config.skipExcluded,
  pluginConfigs.mapValues { it.value.mapToOrt() }

workers/advisor/src/test/kotlin/AdvisorRunnerTest.kt

@@ -139,5 +139,5 @@ private fun mockContext(
  resolvedPluginConfig: Map<String, PluginConfiguration> = emptyMap()
 ): WorkerContext =
  mockk {
- coEvery { resolveConfigSecrets(jobConfig.config) } returns resolvedPluginConfig
+ coEvery { resolvePluginConfigSecrets(jobConfig.config) } returns resolvedPluginConfig
  }

workers/advisor/src/test/kotlin/AdvisorWorkerTest.kt

@@ -109,7 +109,7 @@ class AdvisorWorkerTest : StringSpec({
 
  val context = mockk<WorkerContext> {
  every { [email protected] } returns ortRun
- coEvery { resolveConfigSecrets(any()) } returns emptyMap()
+ coEvery { resolvePluginConfigSecrets(any()) } returns emptyMap()
  }
  val contextFactory = mockk<WorkerContextFactory> {
  every { createContext(ORT_RUN_ID) } returns context

workers/common/src/main/kotlin/common/context/WorkerContext.kt

@@ -26,6 +26,7 @@ import org.eclipse.apoapsis.ortserver.config.Path
 import org.eclipse.apoapsis.ortserver.model.Hierarchy
 import org.eclipse.apoapsis.ortserver.model.OrtRun
 import org.eclipse.apoapsis.ortserver.model.PluginConfiguration
+import org.eclipse.apoapsis.ortserver.model.ProviderPluginConfiguration
 import org.eclipse.apoapsis.ortserver.model.Secret
 
 /**
@@ -72,7 +73,17 @@ interface WorkerContext : AutoCloseable {
  * in contrast to [resolveSecrets], this function deals with secrets from the configuration instead of secrets
  * managed on behalf of customers.
  */
- suspend fun resolveConfigSecrets(config: Map<String, PluginConfiguration>?): Map<String, PluginConfiguration>
+ suspend fun resolvePluginConfigSecrets(config: Map<String, PluginConfiguration>?): Map<String, PluginConfiguration>
+
+ /**
+ * Resolve all the secrets referenced by the given [config]. If [config] is not *null*, obtain the referenced
+ * secrets from the [ProviderPluginConfiguration]s and resolve all values using the [configManager] instance. Note
+ * that in contrast to [resolveSecrets], this function deals with secrets from the configuration instead of secrets
+ * managed on behalf of customers.
+ */
+ suspend fun resolveProviderPluginConfigSecrets(
+ config: List<ProviderPluginConfiguration>?
+ ): List<ProviderPluginConfiguration>
 
  /**
  * Download the configuration file at the specified [path] from the resolved configuration context to the given

workers/common/src/main/kotlin/common/context/WorkerContextImpl.kt

@@ -36,6 +36,7 @@ import org.eclipse.apoapsis.ortserver.config.Path as ConfigPath
 import org.eclipse.apoapsis.ortserver.model.Hierarchy
 import org.eclipse.apoapsis.ortserver.model.OrtRun
 import org.eclipse.apoapsis.ortserver.model.PluginConfiguration
+import org.eclipse.apoapsis.ortserver.model.ProviderPluginConfiguration
 import org.eclipse.apoapsis.ortserver.model.Secret
 import org.eclipse.apoapsis.ortserver.model.repositories.OrtRunRepository
 import org.eclipse.apoapsis.ortserver.model.repositories.RepositoryRepository
@@ -104,14 +105,24 @@ internal class WorkerContextImpl(
  return parallelTransform(secrets.toList(), secretsCache, this::resolveSecret, ::extractSecretKey)
  }
 
- override suspend fun resolveConfigSecrets(
+ override suspend fun resolvePluginConfigSecrets(
  config: Map<String, PluginConfiguration>?
  ): Map<String, PluginConfiguration> =
  config?.let { c ->
  val secrets = c.values.flatMap { it.secrets.values }
  val resolvedSecrets = parallelTransform(secrets, configSecretsCache, this::resolveConfigSecret) { it }
 
- c.mapValues { entry -> entry.value.resolveSecrets(resolvedSecrets) }
+ c.mapValues { (_, pluginConfig) -> pluginConfig.resolveSecrets(resolvedSecrets) }
+ }.orEmpty()
+
+ override suspend fun resolveProviderPluginConfigSecrets(
+ config: List<ProviderPluginConfiguration>?
+ ): List<ProviderPluginConfiguration> =
+ config?.let { c ->
+ val secrets = c.flatMap { it.secrets.values }
+ val resolvedSecrets = parallelTransform(secrets, configSecretsCache, this::resolveConfigSecret) { it }
+
+ c.map { providerPluginConfig -> providerPluginConfig.resolveSecrets(resolvedSecrets) }
  }.orEmpty()
 
  override suspend fun downloadConfigurationFile(
@@ -250,3 +261,11 @@ private fun PluginConfiguration.resolveSecrets(secretValues: Map<String, String>
  val resolvedSecrets = secrets.mapValues { e -> secretValues.getValue(e.value) }
  return copy(secrets = resolvedSecrets)
 }
+
+/**
+ * Return a [ProviderPluginConfiguration] whose secrets are resolved according to the given map with [secretValues].
+ */
+private fun ProviderPluginConfiguration.resolveSecrets(secretValues: Map<String, String>): ProviderPluginConfiguration {
+ val resolvedSecrets = secrets.mapValues { e -> secretValues.getValue(e.value) }
+ return copy(secrets = resolvedSecrets)
+}

workers/common/src/test/kotlin/common/context/WorkerContextFactoryTest.kt

@@ -25,7 +25,9 @@ import com.typesafe.config.ConfigFactory
 import io.kotest.assertions.throwables.shouldThrow
 import io.kotest.core.spec.style.WordSpec
 import io.kotest.engine.spec.tempdir
-import io.kotest.matchers.maps.beEmpty
+import io.kotest.matchers.collections.beEmpty
+import io.kotest.matchers.collections.shouldContainExactly
+import io.kotest.matchers.maps.beEmpty as beEmptyMap
 import io.kotest.matchers.maps.shouldContainExactly
 import io.kotest.matchers.maps.shouldHaveSize
 import io.kotest.matchers.should
@@ -47,6 +49,7 @@ import org.eclipse.apoapsis.ortserver.config.Path
 import org.eclipse.apoapsis.ortserver.model.Hierarchy
 import org.eclipse.apoapsis.ortserver.model.OrtRun
 import org.eclipse.apoapsis.ortserver.model.PluginConfiguration
+import org.eclipse.apoapsis.ortserver.model.ProviderPluginConfiguration
 import org.eclipse.apoapsis.ortserver.model.Secret
 import org.eclipse.apoapsis.ortserver.model.repositories.OrtRunRepository
 import org.eclipse.apoapsis.ortserver.model.repositories.RepositoryRepository
@@ -305,13 +308,13 @@ class WorkerContextFactoryTest : WordSpec({
  }
  }
 
- "resolveConfigFiles" should {
+ "resolvePluginConfigSecrets" should {
  "return an empty Map for null input" {
  val helper = ContextFactoryTestHelper()
 
- val resolvedConfig = helper.context().resolveConfigSecrets(null)
+ val resolvedConfig = helper.context().resolvePluginConfigSecrets(null)
 
- resolvedConfig should beEmpty()
+ resolvedConfig should beEmptyMap()
  }
 
  "return plugin configurations with resolved secrets" {
@@ -343,12 +346,58 @@ class WorkerContextFactoryTest : WordSpec({
 
  val helper = ContextFactoryTestHelper()
 
- val resolvedConfig = helper.context().resolveConfigSecrets(config)
+ val resolvedConfig = helper.context().resolvePluginConfigSecrets(config)
 
  resolvedConfig shouldBe expectedConfig
  }
  }
 
+ "resolveProviderPluginConfigSecrets" should {
+ "return an empty Map for null input" {
+ val helper = ContextFactoryTestHelper()
+
+ val resolvedConfig = helper.context().resolveProviderPluginConfigSecrets(null)
+
+ resolvedConfig should beEmpty()
+ }
+
+ "return provider plugin configurations with resolved secrets" {
+ val pluginConfig1 = ProviderPluginConfiguration(
+ type = "type1",
+ options = mapOf("plugin1Option1" to "v1", "plugin1Option2" to "v2"),
+ secrets = mapOf("plugin1User" to "dbUser", "plugin1Password" to "dbPassword")
+ )
+ val pluginConfig2 = ProviderPluginConfiguration(
+ type = "type2",
+ options = mapOf("plugin2Option" to "v3"),
+ secrets = mapOf(
+ "plugin2ServiceUser" to "serviceUser",
+ "plugin2ServicePassword" to "servicePassword",
+ "plugin2DBAccess" to "dbPassword"
+ )
+ )
+ val config = listOf(pluginConfig1, pluginConfig2)
+
+ val resolvedConfig1 = pluginConfig1.copy(
+ secrets = mapOf("plugin1User" to "scott", "plugin1Password" to "tiger")
+ )
+ val resolvedConfig2 = pluginConfig2.copy(
+ secrets = mapOf(
+ "plugin2ServiceUser" to "svcUser",
+ "plugin2ServicePassword" to "svcPass",
+ "plugin2DBAccess" to "tiger"
+ )
+ )
+ val expectedConfig = listOf(resolvedConfig1, resolvedConfig2)
+
+ val helper = ContextFactoryTestHelper()
+
+ val resolvedConfig = helper.context().resolveProviderPluginConfigSecrets(config)
+
+ resolvedConfig shouldContainExactly expectedConfig
+ }
+ }
+
  "the ORT configuration" should {
  "should be set up" {
  mockkObject(WorkerOrtConfig)

workers/evaluator/src/main/kotlin/evaluator/EvaluatorRunner.kt

@@ -19,6 +19,8 @@
 
 package org.eclipse.apoapsis.ortserver.workers.evaluator
 
+import kotlinx.coroutines.runBlocking
+
 import org.eclipse.apoapsis.ortserver.config.Path
 import org.eclipse.apoapsis.ortserver.model.EvaluatorJobConfiguration
 import org.eclipse.apoapsis.ortserver.workers.common.context.WorkerContext
@@ -96,7 +98,10 @@ class EvaluatorRunner(
  val repositoryPackageConfigurations = ortResult.repository.config.packageConfigurations
  add(SimplePackageConfigurationProvider(repositoryPackageConfigurations))
 
- val packageConfigurationProviderConfigs = config.packageConfigurationProviders.map { it.mapToOrt() }
+ val packageConfigurationProviderConfigs = runBlocking {
+ workerContext.resolveProviderPluginConfigSecrets(config.packageConfigurationProviders)
+ .map { it.mapToOrt() }
+ }
  addAll(PackageConfigurationProviderFactory.create(packageConfigurationProviderConfigs).map { it.second })
  }.let { CompositePackageConfigurationProvider(it) }
 

workers/evaluator/src/test/kotlin/EvaluatorRunnerTest.kt

@@ -29,8 +29,12 @@ import io.kotest.matchers.should
 import io.kotest.matchers.shouldBe
 import io.kotest.matchers.string.shouldContain
 
+import io.mockk.coEvery
 import io.mockk.every
 import io.mockk.mockk
+import io.mockk.mockkObject
+import io.mockk.unmockkAll
+import io.mockk.verify
 
 import java.io.File
 
@@ -42,6 +46,7 @@ import org.eclipse.apoapsis.ortserver.model.EvaluatorJobConfiguration
 import org.eclipse.apoapsis.ortserver.model.ProviderPluginConfiguration
 import org.eclipse.apoapsis.ortserver.workers.common.OrtTestData
 import org.eclipse.apoapsis.ortserver.workers.common.context.WorkerContext
+import org.eclipse.apoapsis.ortserver.workers.common.mapToOrt
 
 import org.ossreviewtoolkit.model.OrtResult
 import org.ossreviewtoolkit.model.RuleViolation
@@ -50,6 +55,7 @@ import org.ossreviewtoolkit.model.config.LicenseFindingCuration
 import org.ossreviewtoolkit.model.config.LicenseFindingCurationReason
 import org.ossreviewtoolkit.model.config.PackageConfiguration
 import org.ossreviewtoolkit.model.yamlMapper
+import org.ossreviewtoolkit.plugins.packageconfigurationproviders.api.PackageConfigurationProviderFactory
 import org.ossreviewtoolkit.utils.ort.ORT_COPYRIGHT_GARBAGE_FILENAME
 import org.ossreviewtoolkit.utils.ort.ORT_EVALUATOR_RULES_FILENAME
 import org.ossreviewtoolkit.utils.ort.ORT_LICENSE_CLASSIFICATIONS_FILENAME
@@ -65,6 +71,8 @@ private const val UNKNOWN_RULES_KTS = "unknown.rules.kts"
 private val resolvedConfigContext = Context("resolvedContext")
 
 class EvaluatorRunnerTest : WordSpec({
+ afterEach { unmockkAll() }
+
  val runner = EvaluatorRunner(mockk())
 
  "run" should {
@@ -140,21 +148,23 @@ class EvaluatorRunnerTest : WordSpec({
  )
  )
 
+ val packageConfigurationProviderConfigs = listOf(
+ ProviderPluginConfiguration(
+ type = "Dir",
+ options = mapOf(
+ "path" to "src/test/resources/package-configurations",
+ "mustExist" to "true"
+ )
+ )
+ )
+
  val result = runner.run(
  ortResult,
  EvaluatorJobConfiguration(
  ruleSet = PACKAGE_CONFIGURATION_RULES,
- packageConfigurationProviders = listOf(
- ProviderPluginConfiguration(
- type = "Dir",
- options = mapOf(
- "path" to "src/test/resources/package-configurations",
- "mustExist" to "true"
- )
- )
- )
+ packageConfigurationProviders = packageConfigurationProviderConfigs
  ),
- createWorkerContext()
+ createWorkerContext(packageConfigurationProviderConfigs)
  )
 
  // The test data contains a package with a LicenseRef-detected1 and a LicenseRef-detected2 license finding
@@ -185,6 +195,52 @@ class EvaluatorRunnerTest : WordSpec({
  )
  }
 
+ "resolve secrets in the package configuration provider configurations" {
+ mockkObject(PackageConfigurationProviderFactory)
+ every { PackageConfigurationProviderFactory.create(any()) } returns mockk(relaxed = true)
+
+ val packageConfigurationProviderConfigs = listOf(
+ ProviderPluginConfiguration(
+ type = "Dir",
+ options = mapOf("path" to "path1"),
+ secrets = mapOf("secret1" to "ref1")
+ ),
+ ProviderPluginConfiguration(
+ type = "Dir",
+ options = mapOf("path" to "path2"),
+ secrets = mapOf("secret2" to "ref2")
+ )
+ )
+
+ val resolvedPackageConfigurationProviderConfigs = listOf(
+ ProviderPluginConfiguration(
+ type = "Dir",
+ options = mapOf("path" to "path1"),
+ secrets = mapOf("secret1" to "value1")
+ ),
+ ProviderPluginConfiguration(
+ type = "Dir",
+ options = mapOf("path" to "path2"),
+ secrets = mapOf("secret2" to "value2")
+ )
+ )
+
+ runner.run(
+ OrtTestData.result,
+ EvaluatorJobConfiguration(
+ ruleSet = SCRIPT_FILE,
+ packageConfigurationProviders = packageConfigurationProviderConfigs
+ ),
+ createWorkerContext(packageConfigurationProviderConfigs, resolvedPackageConfigurationProviderConfigs)
+ )
+
+ verify(exactly = 1) {
+ PackageConfigurationProviderFactory.create(
+ resolvedPackageConfigurationProviderConfigs.map { it.mapToOrt() }
+ )
+ }
+ }
+
  "use the resolutions from the repository configuration and resolutions file" {
  val result = runner.run(
  OrtTestData.result,
@@ -233,11 +289,15 @@ private fun createConfigManager(): ConfigManager {
  return configManager
 }
 
-private fun createWorkerContext(): WorkerContext {
+private fun createWorkerContext(
+ providerPluginConfigs: List<ProviderPluginConfiguration> = emptyList(),
+ resolvedProviderPluginConfigs: List<ProviderPluginConfiguration> = providerPluginConfigs
+): WorkerContext {
  val configManagerMock = createConfigManager()
 
  return mockk {
  every { configManager } returns configManagerMock
  every { ortRun.resolvedJobConfigContext } returns resolvedConfigContext.name
+ coEvery { resolveProviderPluginConfigSecrets(providerPluginConfigs) } returns resolvedProviderPluginConfigs
  }
 }

workers/evaluator/src/test/kotlin/EvaluatorWorkerTest.kt

@@ -23,6 +23,7 @@ import io.kotest.assertions.fail
 import io.kotest.core.spec.style.StringSpec
 import io.kotest.matchers.shouldBe
 
+import io.mockk.coEvery
 import io.mockk.coVerify
 import io.mockk.every
 import io.mockk.just
@@ -125,6 +126,7 @@ class EvaluatorWorkerTest : StringSpec({
  val context = mockk<WorkerContext> {
  every { [email protected] } returns ortRun
  every { [email protected] } returns configManager
+ coEvery { resolveProviderPluginConfigSecrets(any()) } returns mockk(relaxed = true)
  }
  val contextFactory = mockk<WorkerContextFactory> {
  every { createContext(ORT_RUN_ID) } returns context

workers/reporter/src/main/kotlin/reporter/ReporterRunner.kt

@@ -131,7 +131,10 @@ class ReporterRunner(
  val repositoryPackageConfigurations = resolvedOrtResult.repository.config.packageConfigurations
  add(SimplePackageConfigurationProvider(repositoryPackageConfigurations))
 
- val packageConfigurationProviderConfigs = config.packageConfigurationProviders.map { it.mapToOrt() }
+ val packageConfigurationProviderConfigs = runBlocking {
+ context.resolveProviderPluginConfigSecrets(config.packageConfigurationProviders)
+ .map { it.mapToOrt() }
+ }
  addAll(
  PackageConfigurationProviderFactory.create(packageConfigurationProviderConfigs)
  .map { it.second }
@@ -278,7 +281,7 @@ class ReporterRunner(
  launch { context.downloadAssetDirectories(config.assetDirectories, templateDir) }
 
  val transformedPluginOptions = config.config?.recombine(transformedOptions.await())
- context.resolveConfigSecrets(transformedPluginOptions)
+ context.resolvePluginConfigSecrets(transformedPluginOptions)
  }
 }