[Resolve cloudreach#26] skip_index (cloudreach#27)

* [cloudreach#26] new params `skip_index`

* update terraform code

README.md

@@ -125,7 +125,8 @@ Using AWS environment variable you can easily modify the behaviour of the Lambda
 | Variable Name | Example Value | Description | Default Value | Required |
 | --- | --- | --- | --- | --- |
 | es_endpoint | search-es-demo-zveqnhnhjqm5flntemgmx5iuya.eu-west-1.es.amazonaws.com | AWS ES fqdn | `None` | True |
-| index | `logstash,cwl` | Index/indices to process comma separated, with `all` every index will be processed except `.kibana` | `all` | False |
+| index | `logstash,cwl` | Index/indices to process comma separated, with `all` every index will be processed except the one listed in `skip_index` | `all` | False |
+| skip_index | `.kibana,.kibana_5` | Index/indices to skip | `.kibana` | False |
 | index_format | `%Y.%m.%d` | Combined with `index` varible is used to evaluate the index age | `%Y.%m.%d` | False |
 | delete_after | `7` | Numbers of days to preserve | `15` | False |
 

es-cleanup.py

@@ -52,6 +52,7 @@ def __init__(self, event, context):
  self.cfg = {}
  self.cfg["es_endpoint"] = self.get_parameter("es_endpoint")
  self.cfg["index"] = self.get_parameter("index", "all").split(",")
+ self.cfg["skip_index"] = self.get_parameter("skip_index", ".kibana").split(",")
 
  self.cfg["delete_after"] = int(self.get_parameter("delete_after", 15))
  self.cfg["es_max_retry"] = int(self.get_parameter("es_max_retry", 3))
@@ -167,7 +168,7 @@ def lambda_handler(event, context):
  days=int(es.cfg["delete_after"]))
  for index in es.get_indices():
  print("Found index: {}".format(index["index"]))
- if index["index"].startswith(".kibana"):
+ if index["index"] in es.cfg["skip_index"]:
  # ignore .kibana index
  continue
 

terraform/README.md

@@ -17,6 +17,7 @@ Particularly it creates:
 | --- | --- | --- | --- | --- |
 | es_endpoint | search-es-demo-zveqnhnhjqm5flntemgmx5iuya.eu-west-1.es.amazonaws.com | AWS ES fqdn | `None` | True |
 | index | `logstash,cwl` | Index/indices to process comma separated, with `all` every index will be processed except `.kibana` | `all` | False |
+| skip_index | `.kibana,.kibana_5` | Index/indices to skip | `.kibana` | False |
 | index_format | `%Y.%m.%d` | Combined with `index` varible is used to evaluate the index age | `%Y.%m.%d` | False |
 | delete_after | `7` | Numbers of days to preserve | `15` | False |
 | python_version | `2.7` | Python version to be used | `2.7` | False |

terraform/lambda.tf

@@ -22,6 +22,7 @@ resource "aws_lambda_function" "es_cleanup" {
  variables = {
  es_endpoint = var.es_endpoint
  index = var.index
+ skip_index = var.skip_index
  delete_after = var.delete_after
  index_format = var.index_format
  sns_alert = var.sns_alert

terraform/variables.tf

@@ -23,6 +23,11 @@ variable "index" {
  default = "all"
 }
 
+variable "skip_index" {
+ description = "Index/indices to skip"
+ default = ".kibana"
+}
+
 variable "delete_after" {
  description = "Numbers of days to preserve"
  default = 15