feat: refresh secret on delete when polling is disabled

bin/daemon.js

@@ -25,6 +25,7 @@ const {
  rolePermittedAnnotation,
  namingPermittedAnnotation,
  enforceNamespaceAnnotation,
+ pollInternalSecrets,
  watchTimeout,
  watchedNamespaces,
  instanceId
@@ -58,6 +59,7 @@ async function main () {
  enforceNamespaceAnnotation,
  customResourceManifest,
  pollingDisabled,
+ pollInternalSecrets,
  logger
  })
 

charts/kubernetes-external-secrets/templates/rbac.yaml

@@ -15,7 +15,7 @@ metadata:
 rules:
  - apiGroups: [""]
  resources: ["secrets"]
- verbs: ["create", "update"]
+ verbs: ["create", "update"{{ if (index .Values.env "POLL_INTERNAL_SECRETS") }}, "list"{{ end}}]
  - apiGroups: [""]
  resources: ["namespaces"]
  verbs: ["get", "watch", "list"]

charts/kubernetes-external-secrets/values.yaml

@@ -15,6 +15,7 @@ env:
  # USE_HUMAN_READABLE_LOG_LEVELS: true
  METRICS_PORT: 3001
  VAULT_ADDR: http:https://127.0.0.1:8200
+ # POLL_INTERNAL_SECRETS: "true"
  # Set a role to be used when assuming roles specified in external secret (AWS only)
  # AWS_INTERMEDIATE_ROLE_ARN:
  # GOOGLE_APPLICATION_CREDENTIALS: /app/gcp-creds/gcp-creds.json

config/environment.js

@@ -55,6 +55,8 @@ watchedNamespaces = watchedNamespaces
  // Remove empty values (in case there is a tailing comma).
  .filter(namespace => namespace)
 
+const pollInternalSecrets = 'POLL_INTERNAL_SECRETS' in process.env
+
 module.exports = {
  instanceId,
  vaultEndpoint,
@@ -72,6 +74,7 @@ module.exports = {
  logLevel,
  useHumanReadableLogLevels,
  logMessageKey,
+ pollInternalSecrets,
  watchTimeout,
  watchedNamespaces
 }

lib/daemon.js

@@ -5,10 +5,9 @@ class Daemon {
  /**
  * Create daemon.
  * @param {Object} backends - Backends for fetching secret properties.
- * @param {Object} kubeClient - Client for interacting with kubernetes cluster.
  * @param {Object} externalSecretEvents - Stream of external secret events.
  * @param {Object} logger - Logger for logging stuff.
- * @param {number} pollerIntervalMilliseconds - Interval time in milliseconds for polling secret properties.
+ * @param {PollerFactory} pollerFactory - A poller factory instance
  */
  constructor ({
  instanceId,

lib/kubernetes-secrets.js

@@ -0,0 +1,107 @@
+'use strict'
+
+/**
+ * Kubernetes secret observer.
+ * @param {string} timeoutId - An ID of setTimeout which schedules next poll of internal secrets.
+ * @param {Object} secrets - Object of secret names present in given namespace.
+ * Watch for Kubernetes secrets, and provide status promises.
+ */
+
+/** Kubernetes secret observer class. */
+class KubernetesSecrets {
+ /**
+ * Create secrets observer.
+ * @param {string} namespace - A namespace to poll for internal secrets.
+ * @param {number} intervalMilliseconds - Interval time in milliseconds for polling secret properties.
+ * @param {Object} logger - Logger for logging stuff.
+ * @param {Object} metrics - Metrics client.
+ */
+ constructor ({ namespace, kubeClient, intervalMilliseconds, logger, metrics }) {
+ this._intervalMilliseconds = intervalMilliseconds
+ this._logger = logger
+ this._metrics = metrics
+ this._kubeClient = kubeClient
+ this._timeoutId = null
+ this._secrets = {}
+ this._namespace = namespace
+ }
+
+ /**
+ * Return current set of present secret names
+ * @returns {Array} - secret names listing
+ */
+ get secretNames () {
+ return Object.keys(this._secrets)
+ }
+
+ /**
+ * Refresh Kubernetes secrets.
+ * Set timeout for next refresh.
+ */
+ async _listAndRefreshSecrets () {
+ const kubeNamespace = this._kubeClient.api.v1.namespaces(this._namespace)
+
+ const newSecrets = {}
+ const kubeSecrets = await kubeNamespace.secrets.get()
+
+ for (const kubeSecret of kubeSecrets.body.items) {
+ newSecrets[kubeSecret.metadata.name] = kubeSecret
+ }
+
+ this._secrets = newSecrets
+ this._metrics.observeSync({
+ name: 'all-internal-secrets-list',
+ namespace: this._namespace,
+ status: 'success'
+ })
+ this._timeoutId = setTimeout(this._listAndRefreshSecrets.bind(this), this._intervalMilliseconds)
+ }
+
+ /**
+ * Find out if given secret exists in a namespace.
+ * @param {string} secretName Name of secret
+ *
+ * @returns {boolean} A boolean status of k8s secret presence.
+ */
+ secretPresent (secretName) {
+ return (secretName in this._secrets)
+ }
+
+ /**
+ * Start this secrets observer.
+ */
+ start () {
+ this._logger.info(`starting kubernetes secrets observer for namespace ${this._namespace}.`)
+ this._timeoutId = setTimeout(this._listAndRefreshSecrets.bind(this), this._intervalMilliseconds)
+ return this
+ }
+
+ /**
+ * Stop this secrets observer.
+ */
+ stop () {
+ if (this._timeoutId != null) {
+ clearTimeout(this._timeoutId)
+ }
+ }
+
+ /**
+ * Get or create secret observer in given namespace.
+ */
+ static getOrCreateSecretObserver (props) {
+ const nsName = props.namespace
+
+ if (!(nsName in KubernetesSecrets.secretObservers)) {
+ KubernetesSecrets.secretObservers[nsName] =
+ new KubernetesSecrets(props)
+ }
+
+ return KubernetesSecrets.secretObservers[nsName]
+ }
+}
+
+// A static object containing namespace name as a key, and secret
+// observer instance as a value.
+KubernetesSecrets.secretObservers = {}
+
+module.exports = KubernetesSecrets

lib/kubernetes-secrets.test.js

@@ -0,0 +1,117 @@
+/* eslint-env mocha */
+'use strict'
+
+const { expect } = require('chai')
+const sinon = require('sinon')
+
+const KubernetesSecrets = require('./kubernetes-secrets')
+
+describe('KubernetesSecrets', () => {
+ let loggerMock
+ let metricsMock
+ const kubeNamespaceMock = sinon.mock()
+ const kubeClientMock = sinon.mock()
+
+ beforeEach(async () => {
+ loggerMock = sinon.mock()
+ loggerMock.info = sinon.stub()
+
+ metricsMock = sinon.mock()
+ metricsMock.observeSync = sinon.stub()
+
+ const fakeSecret1 = {
+ apiVersion: 'v1',
+ kind: 'Secret',
+ metadata: {
+ name: 'stub1'
+ }
+ }
+
+ const fakeSecret2 = {
+ apiVersion: 'v1',
+ kind: 'Secret',
+ metadata: {
+ name: 'stub2'
+ }
+ }
+
+ kubeClientMock.api = sinon.mock()
+ kubeClientMock.api.v1 = sinon.mock()
+ kubeClientMock.api.v1.namespaces = sinon.stub().returns(kubeNamespaceMock)
+
+ kubeNamespaceMock.get = sinon.stub().resolves(kubeNamespaceMock)
+ kubeNamespaceMock.secrets = sinon.mock()
+ kubeNamespaceMock.secrets.get = sinon.stub().resolves({ body: { items: [fakeSecret1, fakeSecret2] } })
+ })
+
+ afterEach(async () => {
+ sinon.restore()
+ })
+
+ it('caches secret after creating', async () => {
+ const os1 = KubernetesSecrets.getOrCreateSecretObserver(
+ {
+ namespace: 'ns1',
+ logger: loggerMock,
+ metrics: metricsMock,
+ intervalMilliseconds: 1000,
+ kubeClient: kubeClientMock
+ }
+ )
+
+ const os2 = KubernetesSecrets.getOrCreateSecretObserver(
+ {
+ namespace: 'ns1',
+ logger: loggerMock,
+ metrics: metricsMock,
+ intervalMilliseconds: 1000,
+ kubeClient: kubeClientMock
+ }
+ )
+
+ const os3 = KubernetesSecrets.getOrCreateSecretObserver(
+ {
+ namespace: 'ns2',
+ logger: loggerMock,
+ metrics: metricsMock,
+ intervalMilliseconds: 1000,
+ kubeClient: kubeClientMock
+ }
+ )
+
+ expect(os1).is.deep.equal(os2)
+ expect(os3).is.not.equal(os2)
+ expect(Object.keys(KubernetesSecrets.secretObservers)).is.deep.equal(['ns1', 'ns2'])
+ })
+
+ it('periodically refreshes internal secret state', async () => {
+ const os = KubernetesSecrets.getOrCreateSecretObserver(
+ {
+ namespace: kubeNamespaceMock,
+ logger: loggerMock,
+ metrics: metricsMock,
+ intervalMilliseconds: 2000,
+ kubeClient: kubeClientMock
+ }
+ )
+ await os._listAndRefreshSecrets()
+ clearTimeout(os._timeoutId)
+ expect(os.secretNames).is.deep.equal(['stub1', 'stub2'])
+ })
+
+ it('allows to testing of secret presence within state', async () => {
+ const os = KubernetesSecrets.getOrCreateSecretObserver(
+ {
+ namespace: kubeNamespaceMock,
+ logger: loggerMock,
+ metrics: metricsMock,
+ intervalMilliseconds: 2000,
+ kubeClient: kubeClientMock
+ }
+ )
+ await os._listAndRefreshSecrets()
+ clearTimeout(os._timeoutId)
+ expect(os.secretPresent('stub1')).equal(true)
+ expect(os.secretPresent('stub3')).equal(false)
+ })
+})

lib/poller-factory.js

@@ -11,6 +11,8 @@ class PollerFactory {
  * @param {Object} customResourceManifest - CRD manifest
  * @param {Object} logger - Logger for logging stuff.
  * @param {number} pollerIntervalMilliseconds - Interval time in milliseconds for polling secret properties.
+ * @param {boolean} pollingDisabled - If set up, disables polling of all secrets.
+ * @param {boolean} pollInternalSecrets - If set up, starts polling of internal secrets.
  * @param {String} rolePermittedAnnotation - namespace annotation that defines which roles can be assumed within this namespace
  */
  constructor ({
@@ -23,6 +25,7 @@ class PollerFactory {
  customResourceManifest,
  enforceNamespaceAnnotation,
  pollingDisabled,
+ pollInternalSecrets,
  logger
  }) {
  this._logger = logger
@@ -35,6 +38,7 @@ class PollerFactory {
  this._namingPermittedAnnotation = namingPermittedAnnotation
  this._enforceNamespaceAnnotation = enforceNamespaceAnnotation
  this._pollingDisabled = pollingDisabled
+ this._pollInternalSecrets = pollInternalSecrets
  }
 
  /**
@@ -53,6 +57,7 @@ class PollerFactory {
  namingPermittedAnnotation: this._namingPermittedAnnotation,
  enforceNamespaceAnnotation: this._enforceNamespaceAnnotation,
  pollingDisabled: this._pollingDisabled,
+ pollInternalSecrets: this._pollInternalSecrets,
  externalSecret
  })
 

lib/poller.js

@@ -3,6 +3,7 @@
 const clonedeep = require('lodash/cloneDeep')
 const merge = require('lodash/merge')
 const mapValues = require('lodash/mapValues')
+const KubernetesSecrets = require('./kubernetes-secrets')
 const { compileObjectTemplateKeys } = require('./utils')
 
 /**
@@ -33,6 +34,7 @@ class Poller {
  * @param {string} rolePermittedAnnotation - namespace annotation that defines which roles can be assumed within this namespace
  * @param {string} namingPermittedAnnotation - namespace annotation that defines which keys can be assumed within this namespace
  * @param {string} enforceNamespaceAnnotation - should enforce namespace annotations
+ * @param {Boolean} pollInternalSecrets - Whether supports internal secrets listing
  * @param {Object} metrics - Metrics client.
  */
  constructor ({
@@ -46,6 +48,7 @@ class Poller {
  namingPermittedAnnotation,
  enforceNamespaceAnnotation,
  pollingDisabled,
+ pollInternalSecrets,
  externalSecret
  }) {
  this._backends = backends
@@ -79,6 +82,15 @@ class Poller {
  this._status = this._kubeClient
  .apis[this._customResourceManifest.spec.group]
  .v1.namespaces(this._namespace)[this._customResourceManifest.spec.names.plural](this._name).status
+
+ this._internalSecrets = pollInternalSecrets
+ ? KubernetesSecrets.getOrCreateSecretObserver({
+ namespace: this._namespace,
+ intervalMilliseconds: this._intervalMilliseconds,
+ kubeClient: this._kubeClient,
+ logger: this._logger,
+ metrics: this._metrics
+ }) : null
  }
 
  /**
@@ -317,7 +329,12 @@ class Poller {
 
  // If polling is disabled we only react to changes in the ExternalSecret
  if (this._pollingDisabled) {
- return
+ // Check for internal secret being missing
+ if ((this._internalSecrets === null) ||
+ (this._internalSecrets.secretNames.length === 0) ||
+ (this._internalSecrets.secretPresent(this._name))) {
+ return
+ }
  }
 
  const now = Date.now()
@@ -359,7 +376,7 @@ class Poller {
  */
  start () {
  if (this._timeoutId) return this
-
+ if (this._internalSecrets) this._internalSecrets.start()
  this._logger.info(`starting poller for ${this._namespace}/${this._name}`)
  this._scheduleNextPoll()
 
@@ -372,6 +389,7 @@ class Poller {
  */
  stop () {
  if (!this._timeoutId) return this
+ if (this._internalSecrets) this._internalSecrets.stop()
 
  this._logger.info(`stopping poller for ${this._namespace}/${this._name}`)