From 2ac08b59b4b2658175e7ad83708e254a4dc4ffbd Mon Sep 17 00:00:00 2001 From: Smruti Soumitra Khuntia Date: Fri, 20 Dec 2019 17:40:56 +0530 Subject: [PATCH] Support for local storage This change adds a means of introducing new storage classes and local persistent volumes. Change-Id: I340c75f3d0a1678f3149f3cf62e4ab104823cc49 Co-Authored-By: Steven Fitzpatrick --- .../values_overrides/local-storage.yaml | 20 +++++++++ local-storage/Chart.yaml | 20 +++++++++ local-storage/requirements.yaml | 18 ++++++++ .../templates/persistent-volumes.yaml | 44 +++++++++++++++++++ local-storage/templates/storage-class.yaml | 28 ++++++++++++ local-storage/values.yaml | 41 +++++++++++++++++ .../values_overrides/local-storage.yaml | 35 +++++++++++++++ mariadb/values_overrides/local-storage.yaml | 9 ++++ .../values_overrides/local-storage.yaml | 7 +++ .../000-install-packages.sh | 1 + .../osh-infra-local-storage/005-deploy-k8s.sh | 1 + .../osh-infra-local-storage/010-ingress.sh | 1 + .../020-local-storage.sh | 43 ++++++++++++++++++ .../osh-infra-local-storage/030-mariadb.sh | 1 + .../osh-infra-local-storage/040-prometheus.sh | 37 ++++++++++++++++ .../050-elasticsearch.sh | 1 + .../060-volume-info.sh | 20 +++++++++ zuul.d/jobs.yaml | 25 +++++++++++ zuul.d/project.yaml | 2 + 19 files changed, 354 insertions(+) create mode 100644 elasticsearch/values_overrides/local-storage.yaml create mode 100644 local-storage/Chart.yaml create mode 100644 local-storage/requirements.yaml create mode 100644 local-storage/templates/persistent-volumes.yaml create mode 100644 local-storage/templates/storage-class.yaml create mode 100644 local-storage/values.yaml create mode 100644 local-storage/values_overrides/local-storage.yaml create mode 100644 mariadb/values_overrides/local-storage.yaml create mode 100644 prometheus/values_overrides/local-storage.yaml create mode 120000 tools/deployment/osh-infra-local-storage/000-install-packages.sh create mode 120000 tools/deployment/osh-infra-local-storage/005-deploy-k8s.sh create mode 120000 tools/deployment/osh-infra-local-storage/010-ingress.sh create mode 100755 tools/deployment/osh-infra-local-storage/020-local-storage.sh create mode 120000 tools/deployment/osh-infra-local-storage/030-mariadb.sh create mode 100755 tools/deployment/osh-infra-local-storage/040-prometheus.sh create mode 120000 tools/deployment/osh-infra-local-storage/050-elasticsearch.sh create mode 100755 tools/deployment/osh-infra-local-storage/060-volume-info.sh diff --git a/elasticsearch/values_overrides/local-storage.yaml b/elasticsearch/values_overrides/local-storage.yaml new file mode 100644 index 000000000..0d8b0d6f7 --- /dev/null +++ b/elasticsearch/values_overrides/local-storage.yaml @@ -0,0 +1,20 @@ +pod: + replicas: + data: 1 +storage: + data: + requests: + storage: 1Gi + storage_class: local-storage + master: + requests: + storage: 1Gi + storage_class: local-storage +manifests: + cron_curator: false + cron_verify_repositories: false + job_snapshot_repository: false + job_elasticsearch_templates: false + job_s3_user: false + job_s3_bucket: false + helm_tests: false diff --git a/local-storage/Chart.yaml b/local-storage/Chart.yaml new file mode 100644 index 000000000..f2671d737 --- /dev/null +++ b/local-storage/Chart.yaml @@ -0,0 +1,20 @@ +# Copyright 2020 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Local Storage +name: local-storage +version: 0.1.0 +maintainers: + - name: OpenStack-Helm Authors diff --git a/local-storage/requirements.yaml b/local-storage/requirements.yaml new file mode 100644 index 000000000..28ec01f4f --- /dev/null +++ b/local-storage/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2020 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts + version: 0.1.0 diff --git a/local-storage/templates/persistent-volumes.yaml b/local-storage/templates/persistent-volumes.yaml new file mode 100644 index 000000000..fe0ba7036 --- /dev/null +++ b/local-storage/templates/persistent-volumes.yaml @@ -0,0 +1,44 @@ +{{/* +Copyright 2020 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.persistent_volumes }} +{{- $envAll := . }} +{{- range .Values.conf.persistent_volumes }} +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ .name }} + labels: +{{ tuple $envAll "local-storage" $envAll.Release.Name | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + capacity: + storage: {{ .storage_capacity }} + accessModes: {{ .access_modes }} + persistentVolumeReclaimPolicy: {{ .reclaim_policy }} + storageClassName: {{ $envAll.Release.Name }} + local: + path: {{ .local_path }} + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: {{ $envAll.Values.labels.node_affinity.node_selector_key }} + operator: In + values: + - {{ $envAll.Values.labels.node_affinity.node_selector_value }} +{{- end }} +{{- end }} diff --git a/local-storage/templates/storage-class.yaml b/local-storage/templates/storage-class.yaml new file mode 100644 index 000000000..a92a00518 --- /dev/null +++ b/local-storage/templates/storage-class.yaml @@ -0,0 +1,28 @@ +{{/* +Copyright 2020 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.storage_class }} +{{- $envAll := . }} +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Release.Name }} + labels: +{{ tuple $envAll "local-storage" $envAll.Release.Name | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +{{- end }} diff --git a/local-storage/values.yaml b/local-storage/values.yaml new file mode 100644 index 000000000..02c56b5f4 --- /dev/null +++ b/local-storage/values.yaml @@ -0,0 +1,41 @@ +# Copyright 2020 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +labels: + node_affinity: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +conf: + persistent_volumes: + # For each mount path, one PV should be created. + # If there are two mount paths for local storage are available on two nodes, + # then two PVs details should be defined. Example: + # - name: local-pv-1 (name of the Persistent Volume 1) + # reclaimpolicy: Retain (Reclaim Policy for the PV local-pv-1) + # storage_capacity: "100Gi" (Storage capacity of the PV local-pv-1) + # access_modes: [ "ReadWriteOnce" ] (Access mode for the PV local-pv-1) + # local_path: /mnt/disk/vol1 (Mount path of the local disk, local-pv-1 will be created on) + # - name: local-pv-2 (name of the Persistent Volume 2) + # reclaimpolicy: Retain (Reclaim Policy for the PV local-pv-2) + # storage_capacity: "100Gi" (Storage capacity of the PV local-pv-2) + # access_modes: [ "ReadWriteOnce" ] (Access mode for the PV local-pv-2) + # local_path: /mnt/disk/vol2 (Mount path of the local disk, local-pv-2 will be created on) + # Similarly if three nodes each have disk mount path /var/lib/kubernetes + # which will be acting as local storage for each node, then Persistentvolumes + # should be updated with three entries. + +manifests: + storage_class: true + persistent_volumes: true diff --git a/local-storage/values_overrides/local-storage.yaml b/local-storage/values_overrides/local-storage.yaml new file mode 100644 index 000000000..6b8f341f6 --- /dev/null +++ b/local-storage/values_overrides/local-storage.yaml @@ -0,0 +1,35 @@ +conf: + persistent_volumes: + - name: local-persistent-volume-0 + reclaim_policy: Delete + storage_capacity: "1Gi" + access_modes: [ "ReadWriteOnce" ] + local_path: /srv/local-volume-0 + - name: local-persistent-volume-1 + reclaim_policy: Delete + storage_capacity: "1Gi" + access_modes: [ "ReadWriteOnce" ] + local_path: /srv/local-volume-1 + - name: local-persistent-volume-2 + reclaim_policy: Delete + storage_capacity: "1Gi" + access_modes: [ "ReadWriteOnce" ] + local_path: /srv/local-volume-2 + - name: local-persistent-volume-3 + reclaim_policy: Delete + storage_capacity: "1Gi" + access_modes: [ "ReadWriteOnce" ] + local_path: /srv/local-volume-3 + - name: local-persistent-volume-4 + reclaim_policy: Delete + storage_capacity: "1Gi" + access_modes: [ "ReadWriteOnce" ] + local_path: /srv/local-volume-4 + - name: local-persistent-volume-5 + reclaim_policy: Delete + storage_capacity: "1Gi" + access_modes: [ "ReadWriteOnce" ] + local_path: /srv/local-volume-5 +manifests: + storage_class: true + persistent_volumes: true diff --git a/mariadb/values_overrides/local-storage.yaml b/mariadb/values_overrides/local-storage.yaml new file mode 100644 index 000000000..11a4e9f23 --- /dev/null +++ b/mariadb/values_overrides/local-storage.yaml @@ -0,0 +1,9 @@ +pod: + replicas: + server: 1 +volume: + size: 1Gi + class_name: local-storage +monitoring: + prometheus: + enabled: false diff --git a/prometheus/values_overrides/local-storage.yaml b/prometheus/values_overrides/local-storage.yaml new file mode 100644 index 000000000..384260f42 --- /dev/null +++ b/prometheus/values_overrides/local-storage.yaml @@ -0,0 +1,7 @@ +pod: + replicas: + prometheus: 1 +storage: + requests: + storage: 1Gi + storage_class: local-storage diff --git a/tools/deployment/osh-infra-local-storage/000-install-packages.sh b/tools/deployment/osh-infra-local-storage/000-install-packages.sh new file mode 120000 index 000000000..d702c4899 --- /dev/null +++ b/tools/deployment/osh-infra-local-storage/000-install-packages.sh @@ -0,0 +1 @@ +../common/000-install-packages.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-local-storage/005-deploy-k8s.sh b/tools/deployment/osh-infra-local-storage/005-deploy-k8s.sh new file mode 120000 index 000000000..257a39f7a --- /dev/null +++ b/tools/deployment/osh-infra-local-storage/005-deploy-k8s.sh @@ -0,0 +1 @@ +../common/005-deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-local-storage/010-ingress.sh b/tools/deployment/osh-infra-local-storage/010-ingress.sh new file mode 120000 index 000000000..c3f099351 --- /dev/null +++ b/tools/deployment/osh-infra-local-storage/010-ingress.sh @@ -0,0 +1 @@ +../osh-infra-monitoring/020-ingress.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-local-storage/020-local-storage.sh b/tools/deployment/osh-infra-local-storage/020-local-storage.sh new file mode 100755 index 000000000..183f854db --- /dev/null +++ b/tools/deployment/osh-infra-local-storage/020-local-storage.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Copyright 2020 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +for i in {0..5}; do + sudo mkdir /srv/local-volume-$i; +done + +#NOTE: Lint and package chart +make local-storage + +#NOTE: Deploy command +: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} +: ${OSH_INFRA_EXTRA_HELM_ARGS_LOCAL_STORAGE:="$(./tools/deployment/common/get-values-overrides.sh local-storage)"} + +helm upgrade --install local-storage ./local-storage \ + --namespace=osh-infra \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_LOCAL_STORAGE} + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh osh-infra + +#NOTE: Validate Deployment info +helm status local-storage + +# Simple object validation +kubectl describe sc local-storage +kubectl get pv diff --git a/tools/deployment/osh-infra-local-storage/030-mariadb.sh b/tools/deployment/osh-infra-local-storage/030-mariadb.sh new file mode 120000 index 000000000..880f9f76c --- /dev/null +++ b/tools/deployment/osh-infra-local-storage/030-mariadb.sh @@ -0,0 +1 @@ +../osh-infra-monitoring/045-mariadb.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-local-storage/040-prometheus.sh b/tools/deployment/osh-infra-local-storage/040-prometheus.sh new file mode 100755 index 000000000..54abee017 --- /dev/null +++ b/tools/deployment/osh-infra-local-storage/040-prometheus.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Copyright 2020 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make prometheus + +#NOTE: Deploy command +: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} +: ${OSH_INFRA_EXTRA_HELM_ARGS_PROMETHEUS:="$(./tools/deployment/common/get-values-overrides.sh prometheus)"} + +helm upgrade --install prometheus ./prometheus \ + --namespace=osh-infra \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_PROMETHEUS} + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh osh-infra + +#NOTE: Validate Deployment info +helm status prometheus + +helm test prometheus diff --git a/tools/deployment/osh-infra-local-storage/050-elasticsearch.sh b/tools/deployment/osh-infra-local-storage/050-elasticsearch.sh new file mode 120000 index 000000000..adbbc3119 --- /dev/null +++ b/tools/deployment/osh-infra-local-storage/050-elasticsearch.sh @@ -0,0 +1 @@ +../osh-infra-logging/050-elasticsearch.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-local-storage/060-volume-info.sh b/tools/deployment/osh-infra-local-storage/060-volume-info.sh new file mode 100755 index 000000000..8289d5b47 --- /dev/null +++ b/tools/deployment/osh-infra-local-storage/060-volume-info.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2020 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +# Verifying persistent volumes +kubectl get pv diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 8385a59e8..49f597e8a 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -469,3 +469,28 @@ - ./tools/deployment/podsecuritypolicy/005-deploy-k8s.sh - ./tools/deployment/podsecuritypolicy/006-config-k8s-psp.sh - ./tools/deployment/podsecuritypolicy/007-podsecuritypolicy.sh + +- job: + name: openstack-helm-infra-local-storage + parent: openstack-helm-infra-functional + timeout: 7200 + pre-run: + - playbooks/osh-infra-upgrade-host.yaml + run: playbooks/osh-infra-gate-runner.yaml + post-run: playbooks/osh-infra-collect-logs.yaml + nodeset: openstack-helm-single-node + vars: + osh_params: + openstack_release: stein + container_distro_name: ubuntu + container_distro_version: bionic + feature_gates: local-storage + gate_scripts: + - ./tools/deployment/osh-infra-local-storage/000-install-packages.sh + - ./tools/deployment/osh-infra-local-storage/005-deploy-k8s.sh + - ./tools/deployment/osh-infra-local-storage/010-ingress.sh + - ./tools/deployment/osh-infra-local-storage/020-local-storage.sh + - ./tools/deployment/osh-infra-local-storage/030-mariadb.sh + - ./tools/deployment/osh-infra-local-storage/040-prometheus.sh + - ./tools/deployment/osh-infra-local-storage/050-elasticsearch.sh + - ./tools/deployment/osh-infra-local-storage/060-volume-info.sh diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index ec3e29410..b6bacf4ab 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -42,6 +42,8 @@ voting: false - openstack-helm-infra-apparmor: voting: false + - openstack-helm-infra-local-storage: + voting: false gate: jobs: - openstack-helm-lint