diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..5f5e311 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,56 @@ +--- +name: 🐛 Bug Report +about: Report a reproducible bug + +--- + + + + +### Environment + +* docker-redis-cluster version: +* redis-server version: +* git hash (If applicable): +* CLI command used to build the container: + + + +### Steps to Reproduce + +1. +2. +3. + + + + +### Expected Behavior + + + +### Observed Behavior diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000..24cf73e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,22 @@ +--- +name: Question +about: Ask a question + +--- + + diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml new file mode 100644 index 0000000..6003798 --- /dev/null +++ b/.github/workflows/dockerimage.yml @@ -0,0 +1,14 @@ +name: Docker Image CI + +on: + push: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Build the latest Docker image + run: make build diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..168dc6c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,18 @@ +# This travis run should only test and build the latest version of all major redis-server versions +language: python +python: + - "3.10" +services: + - docker +env: + # - REDIS_VERSION=3.0.7 + # - REDIS_VERSION=3.2.13 + # - REDIS_VERSION=4.0.14 + # - REDIS_VERSION=5.0.12 + # - REDIS_VERSION=6.0.20 + - REDIS_VERSION=6.2.14 + - REDIS_VERSION=7.0.15 + - REDIS_VERSION=7.2.5 + - REDIS_VERSION=7.4-rc1 +script: + docker build --build-arg redis_version=$REDIS_VERSION -t grokzen/redis-cluster . diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b4ee6ee --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,43 @@ +## 2024-06-25 + +* Added 7.2.x releases and published docker images +* added 7.4-rc1 release and published +* Updated all older generations of images +* New base image that contains more updated patches etc + +## 2022-12-18 + +* Added redis 7.0.x releases and published docker images +* Dropped redis 5.0.x releases and unpublished docker images +* Added `invoke list` command to show all possible targets you can build against + +## 2021-04-13 + +* Dropped the availability of redis-server major versions 3.0, 3.2, 4.0 from docker.hub + +## 2021-03-28 + +* Added versions 5.0.12, 6.0.12, 6.2.1 +* Updated latest to 6.2.1 + +## 2021-02-28 + +* Rebuilt most of the build commands from Makefile into py-invoke script + +## 2021-02-27 + +* Added versions 5.0.11, 6.0.11 & 6.2.0 +* Updated latest to 6.2.0 + +## 2021-01-17 + +* Updated README with documentation regarding new github discussions feature and how to use it. +* Add new github issues template to make it easier to request features/help/support +* Add release 5.0.10 +* Add support for building major version 6.2 and their RC releases +* Add release 6.0.10 +* Updated latest to 6.0.10 + +## 2020-11-24 + +* Added support for IPv6 for cluster and stand-alone instances diff --git a/Dockerfile b/Dockerfile index 01d6028..cafda1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,63 +1,52 @@ -# This tag use ubuntu 14.04 -FROM phusion/baseimage:0.9.16 +# Build based on redis:7.2.5 from "2024-05-22T23:17:59Z" +FROM redis@sha256:e422889e156ebea83856b6ff973bfe0c86bce867d80def228044eeecf925592b -MAINTAINER Johan Andersson +LABEL maintainer="Johan Andersson " # Some Environment Variables ENV HOME /root ENV DEBIAN_FRONTEND noninteractive +# Install system dependencies +RUN apt-get update -qq && \ + apt-get install --no-install-recommends -yqq \ + net-tools supervisor ruby rubygems locales gettext-base wget gcc make g++ build-essential libc6-dev tcl && \ + apt-get clean -yqq + # # Ensure UTF-8 lang and locale RUN locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LC_ALL en_US.UTF-8 -# Initial update and install of dependency that can add apt-repos -RUN apt-get -y update && apt-get install -y software-properties-common python-software-properties - -# Add global apt repos -RUN add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu precise universe" && \ - add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu precise main restricted universe multiverse" && \ - add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu precise-updates main restricted universe multiverse" && \ - add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" -RUN apt-get update && apt-get -y upgrade - -# Install system dependencies -RUN apt-get install -y gcc make g++ build-essential libc6-dev tcl git supervisor ruby +# Necessary for gem installs due to SHA1 being weak and old cert being revoked +ENV SSL_CERT_FILE=/usr/local/etc/openssl/cert.pem -# Must be installed seperate from ruby or it will complain about broken package -RUN apt-get install -y rubygems +RUN gem install redis -v 4.1.3 -# Install ruby dependencies so we can bootstrap the cluster via redis-trib.rb -RUN gem install redis +# This will always build the latest release/commit in the 7.2 branch +ARG redis_version=7.2 -# checkout the 3.0.6 tag (Will change to 3.2 tag when it is released as stable) -RUN git clone -b 3.0.6 https://github.com/antirez/redis.git +RUN wget -qO redis.tar.gz https://github.com/redis/redis/tarball/${redis_version} \ + && tar xfz redis.tar.gz -C / \ + && mv /redis-* /redis -# Build redis from source RUN (cd /redis && make) -# Because Git cannot track empty folders we have to create them manually... -RUN mkdir /redis-data && \ - mkdir /redis-data/7000 && \ - mkdir /redis-data/7001 && \ - mkdir /redis-data/7002 && \ - mkdir /redis-data/7003 && \ - mkdir /redis-data/7004 && \ - mkdir /redis-data/7005 && \ - mkdir /redis-data/7006 && \ - mkdir /redis-data/7007 - -# Add all config files for all clusters -ADD ./docker-data/redis-conf /redis-conf +RUN mkdir /redis-conf && mkdir /redis-data -# Add supervisord configuration -ADD ./docker-data/supervisord.conf /etc/supervisor/conf.d/supervisord.conf +COPY redis-cluster.tmpl /redis-conf/redis-cluster.tmpl +COPY redis.tmpl /redis-conf/redis.tmpl +COPY sentinel.tmpl /redis-conf/sentinel.tmpl # Add startup script -ADD ./docker-data/start.sh /start.sh -RUN chmod 755 /start.sh +COPY docker-entrypoint.sh /docker-entrypoint.sh + +# Add script that generates supervisor conf file based on environment variables +COPY generate-supervisor-conf.sh /generate-supervisor-conf.sh + +RUN chmod 755 /docker-entrypoint.sh -EXPOSE 7000 7001 7002 7003 7004 7005 7006 7007 +EXPOSE 7000 7001 7002 7003 7004 7005 7006 7007 5000 5001 5002 -CMD ["/bin/bash", "/start.sh"] +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["redis-cluster"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..df03aab --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +MIT LICENSE + +Copyright 2014-2024 Johan Andersson + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile index dd1c455..582283f 100644 --- a/Makefile +++ b/Makefile @@ -1,39 +1,26 @@ -CID_FILE = /tmp/grokzen-redis-cluster.cid -CID =`cat $(CID_FILE)` -IMAGE_NAME = grokzen/redis-cluster -PORTS = -p 7000:7000 -p 7001:7001 -p 7002:7002 -p 7003:7003 -p 7004:7004 -p 7005:7005 -p 7006:7006 -p 7007:7007 - help: @echo "Please use 'make ' where is one of" - @echo " build build the docker image containing a redis cluster" - @echo " rebuild rebuilds the image from scratch without using any cached layers" - @echo " run run the built docker image" - @echo " bash starts bash inside a running container." - @echo " clean removes the tmp cid file on disk" - @echo " cli run redis-cli inside the container on the server with port 7000" + @echo " build builds docker-compose containers" + @echo " up starts docker-compose containers" + @echo " down stops the running docker-compose containers" + @echo " rebuild rebuilds the image from scratch without using any cached layers" + @echo " bash starts bash inside a running container." + @echo " cli run redis-cli inside the container on the server with port 7000" build: - @echo "Building docker image..." - docker build -t ${IMAGE_NAME} . - -rebuild: - @echo "Rebuilding docker image..." - docker build --no-cache=true -t ${IMAGE_NAME} . + docker-compose build -run: - @echo "Running docker image..." - docker run -d $(PORTS) --cidfile $(CID_FILE) -i -t ${IMAGE_NAME} +up: + docker-compose up -bash: - docker exec -it $(CID) /bin/bash +down: + docker-compose stop -stop: - docker stop $(CID) - -make clean +rebuild: + docker-compose build --no-cache -clean: - # Cleanup cidfile on disk - -rm $(CID_FILE) +bash: + docker-compose exec redis-cluster /bin/bash cli: - docker exec -it $(CID) /redis/src/redis-cli -p 7000 + docker-compose exec redis-cluster /redis/src/redis-cli -p 7000 diff --git a/README.md b/README.md index 4c0f4a8..3dcea6a 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,323 @@ # docker-redis-cluster -[![Docker Stars](https://img.shields.io/docker/stars/grokzen/redis-cluster.svg)](hub]) -[![Docker Pulls](https://img.shields.io/docker/pulls/grokzen/redis-cluster.svg)](hub]) -[![Image Size](https://img.shields.io/imagelayers/image-size/grokzen/redis-cluster/latest.svg)](https://imagelayers.io/?images=grokzen/redis-cluster:latest) -[![Image Layers](https://img.shields.io/imagelayers/layers/grokzen/redis-cluster/latest.svg)](https://imagelayers.io/?images=grokzen/redis-cluster:latest) +[![Docker Stars](https://img.shields.io/docker/stars/grokzen/redis-cluster.svg)](https://hub.docker.com/r/grokzen/redis-cluster/) +[![Docker Pulls](https://img.shields.io/docker/pulls/grokzen/redis-cluster.svg)](https://hub.docker.com/r/grokzen/redis-cluster/) +[![Build Status](https://travis-ci.org/Grokzen/docker-redis-cluster.svg?branch=master)](https://travis-ci.org/Grokzen/docker-redis-cluster) -Docker image with redis built and installed from source. +Docker image with redis built and installed from source and a cluster is built. -The main usage for this container is to test redis cluster code. For example in https://github.com/Grokzen/redis-py-cluster repo. +To find all redis-server releases see them here https://github.com/antirez/redis/releases + + +## Discussions, help, guides + +Github have recently released their `Discussions` feature into beta for more repositories across the github space. This feature is enabled on this repo since a while back. + +Becuase we now have this feature, the issues feature will NOT be a place where you can now ask general questions or need simple help with this repo and what it provides. + +What can you expect to find in there? + + - A place where you can freely ask any question regarding this repo. + - Ask questions like `how do i do X?` + - General help with problems with this repo + - Guides written by me or any other contributer with useful examples and ansers to commonly asked questions and how to resolve thos problems. + - Approved answers to questions marked and promoted by me if help is provided by the community regarding some questions + + +## What this repo and container IS + +This repo exists as a resource to make it quick and simple to get a redis cluster up and running with no fuzz or issues with mininal effort. The primary use for this container is to get a cluster up and running in no time that you can use for demo/presentation/development. It is not intended or built for anything else. + +I also aim to have every single release of redis that supports a cluster available for use so you can run the exact version you want. + +I personally use this to develop redis cluster client code https://github.com/Grokzen/redis-py-cluster + + +## What this repo and container IS NOT + +This container that i have built is not supposed to be some kind of production container or one that is used within any environment other than running locally on your machine. It is not ment to be run on kubernetes or in any other prod/stage/test/dev environment as a fully working commponent in that environment. If that works for you and your use-case then awesome. But this container will not change to fit any other primary solution than to be used locally on your machine. + +If you are looking for something else or some production quality or kubernetes compatible solution then you are looking in the wrong repo. There is other projects or forks of this repo that is compatible for that situation/solution. + +For all other purposes other than what has been stated you are free to fork and/or rebuild this container using it as a template for what you need. + + +## Redis major version support and docker.hub availability + +Starting from `2020-04-01` this repo will only support and make available on docker.hub all minor versions in the latest 3 major versions of redis-server software. At this date the tags on docker.hub for major versions 3.0, 3.2 & 4.0, 5.0 will be removed and only 6.0, 6.2, 7.0 will be available to download. This do not mean that you will not be able to build your desired version from this repo but there is no guarantees or support or hacks that will support this out of the box. + +Moving forward when a new major release is shipped out, at the first minor release X.Y.1 version of the next major release, all tags from the last supported major version will be removed from docker.hub. This will give some time for the community to adapt and move forward in the versions before the older major version is removed from docker.hub. + +This major version schema support follows the same major version support that redis itself use. + + +## Redis instances inside the container The cluster is 6 redis instances running with 3 master & 3 slaves, one slave for each master. They run on ports 7000 to 7005. -It also contains 2 standalone instances that is not part of the cluster. They are running on port 7006 & 7007 +If the flag `-e "SENTINEL=true"` is passed there are 3 Sentinel nodes running on ports 5000 to 5002 matching cluster's master instances. + -The image will build the tag `3.0.6` from the redis git repo. +This image requires at least `Docker` version 1.10 but the latest version is recommended. -This image requires `Docker` above version 1.0 + + +# Important for Mac users + +If you are using this container to run a redis cluster on your mac computer, then you need to configure the container to use another IP address for cluster discovery as it can't use the default discovery IP that is hardcoded into the container. + +If you are using the docker-compose file to build the container, then you must export a environment variable on your machine before building the container. + +``` +# This will make redis do cluster discovery and bind all nodes to ip 127.0.0.1 internally + +export REDIS_CLUSTER_IP=0.0.0.0 +``` + +If you are downloading the container from dockerhub, you must add the internal IP environment variable to your `docker run` command. + +``` +docker run -e "IP=0.0.0.0" -p 7000-7005:7000-7005 grokzen/redis-cluster:latest +``` # Usage -If you want to use `docker-compose (fig)` please read next section. +This git repo is using `invoke` to pull, build, push docker images. You can use it to build your own images if you like. + +The invoke scripts in this repo is written only for python 3.7 and above + +Install `invoke` with `pip install invoke`. + +This script will run `N num of cpu - 1` parralell tasks based on your version input. + +To see available commands run `invoke -l` in the root folder of this repo. Example + +``` +(tmp-615229a94c330b9) ➜ docker-redis-cluster git:(invoke) ✗ invoke -l +"Configured multiprocess pool size: 3 +Available tasks: + + build + list + pull + push +``` + +Each command is only taking one required positional argument `version`. Example: + +``` +(tmp-615229a94c330b9) ➜ docker-redis-cluster git:(invoke) ✗ invoke build 7.0 +... +``` + +and it will run the build step on all versions that starts with 6.0. + +The only other optional usefull argument is `--cpu=N` and it will set how many paralell processes will be used. By default you will use n - 1 number of cpu cores that is available on your system. Commands like pull and push aare not very cpu intensive so using a higher number here might speed things up if you have good network bandwidth. + + +## Makefile (legacy) + +Makefile still has a few docker-compose commands that can be used + +To build your own image run: + + make build + +To run the container run: + + make up + +To stop the container run: + + make down + +To connect to your cluster you can use the redis-cli tool: + + redis-cli -c -p 7000 + +Or the built redis-cli tool inside the container that will connect to the cluster inside the container + + make cli + + +## Include sentinel instances + +Sentinel instances is not enabled by default. + +If running with plain docker send in `-e SENTINEL=true`. + +When running with docker-compose set the environment variable on your system `REDIS_USE_SENTINEL=true` and start your container. + + version: '2' + services: + redis-cluster: + ... + environment: + SENTINEL: 'true' + + +## Change number of nodes + +Be default, it is going to launch 3 masters with 1 slave per master. This is configurable through a number of environment variables: + +| Environment variable | Default | +| -------------------- |--------:| +| `INITIAL_PORT` | 7000 | +| `MASTERS` | 3 | +| `SLAVES_PER_MASTER` | 1 | + +Therefore, the total number of nodes (`NODES`) is going to be `$MASTERS * ( $SLAVES_PER_MASTER + 1 )` and ports are going to range from `$INITIAL_PORT` to `$INITIAL_PORT + NODES - 1`. + +At the docker-compose provided by this repository, ports 7000-7050 are already mapped to the hosts'. Either if you need more than 50 nodes in total or if you need to change the initial port number, you should override those values. + +Also note that the number of sentinels (if enabled) is the same as the number of masters. The docker-compose file already maps ports 5000-5010 by default. You should also override those values if you have more than 10 masters. + + version: '2' + services: + redis-cluster: + ... + environment: + INITIAL_PORT: 9000, + MASTERS: 2, + SLAVES_PER_MASTER: 2 + + +## IPv6 support + +By default, redis instances will bind and accept requests from any IPv4 network. +This is configurable by an environment variable that specifies which address a redis instance will bind to. +By using the IPv6 variant `::` as counterpart to IPv4s `0.0.0.0` an IPv6 cluster can be created. + +| Environment variable | Default | +| -------------------- | ------: | +| `BIND_ADDRESS` | 0.0.0.0 | + +Note that Docker also needs to be [configured](https://docs.docker.com/config/daemon/ipv6/) for IPv6 support. +Unfortunately Docker does not handle IPv6 NAT so, when acceptable, `--network host` can be used. + + # Example using plain docker + docker run -e "IP=::1" -e "BIND_ADDRESS=::" --network host grokzen/redis-cluster:latest + + +## Build alternative redis versions + +For a release to be buildable it needs to be present at this url: http://download.redis.io/releases/ + + +### docker build + +To build a different redis version use the argument `--build-arg` argument. + + # Example plain docker + docker build --build-arg redis_version=6.0.11 -t grokzen/redis-cluster . + + +### docker-compose + +To build a different redis version use the `--build-arg` argument. + + # Example docker-compose + docker-compose build --build-arg "redis_version=6.0.11" redis-cluster + + + +# Available tags + +The following tags with pre-built images is available on `docker-hub`. + +Latest release in the most recent stable branch will be used as `latest` version. + +- latest == 7.2.5 + +Redis 7.4-rc1 version: + +- 7.4-rc1 + +Redis 7.2.x version: + +- 7.2.5 +- 7.2.4 +- 7.2.3 +- 7.2.2 +- 7.2.1 +- 7.2.0 -Either download the latest build from docker hub with `docker pull grokzen/redis-cluster` +Redis 7.0.x version: -Or to build the image, use either `make build` or `make rebuild`. It will be built to the image name `grokzen/redis-cluster`. +- 7.0.15 +- 7.0.14 +- 7.0.13 +- 7.0.12 +- 7.0.11 +- 7.0.10 +- 7.0.9 +- 7.0.8 +- 7.0.7 +- 7.0.6 +- 7.0.5 +- 7.0.4 +- 7.0.3 +- 7.0.2 +- 7.0.1 +- 7.0.0 -To start the image use `make run`. It will be started in the background. To gain access to the running image you can get a bash session by running `make bash`. +Redis 6.2.x versions: -Redis cli can be used with `make cli` to gain access to one of the cluster servers. +- 6.2.14 +- 6.2.13 +- 6.2.12 +- 6.2.11 +- 6.2.10 +- 6.2.9 +- 6.2.8 +- 6.2.7 +- 6.2.6 +- 6.2.5 +- 6.2.4 +- 6.2.3 +- 6.2.2 +- 6.2.1 +- 6.2.0 +Redis 6.0.x versions: +- 6.0.20 +- 6.0.19 +- 6.0.18 +- 6.0.17 +- 6.0.16 +- 6.0.15 +- 6.0.14 +- 6.0.13 +- 6.0.12 +- 6.0.11 +- 6.0.10 +- 6.0.9 +- 6.0.8 +- 6.0.7 +- 6.0.6 +- 6.0.5 +- 6.0.4 +- 6.0.3 +- 6.0.2 +- 6.0.1 +- 6.0.0 -# Docker compose (fig) -This image contains a `compose.yml` file that can be used with `docker-compose (fig)` to run the image. Docker compose is simpler to use then the old `Makefile`. +## Unavailable major versions -Build the image with `docker-compose -f compose.yml build`. +The following major versions is no longer available to be downloaded from docker.hub. You can still build and run them directly from this repo. -Start the image after building with `docker-compose -f compose.yml up`. Add `-d` to run the server in background/detatched mode. +- 5.0 +- 4.0 +- 3.2 +- 3.0 +# License -# Known Issues +This repo is using the MIT LICENSE. -If you get a error when rebuilding the image that docker can't do dns lookup on `archive.ubuntu.com` then you need to modify docker to use google IPv4 DNS lookups. Read the following link http://dannytsang.co.uk/docker-on-digitalocean-cannot-resolve-hostnames/ and uncomment the line in `/etc/default/docker` and restart your docker daemon and it should be fixed. +You can find it in the file [LICENSE](LICENSE) diff --git a/compose.yml b/compose.yml deleted file mode 100644 index 0f14ec8..0000000 --- a/compose.yml +++ /dev/null @@ -1,12 +0,0 @@ -server: - build: . - hostname: server - expose: - - "7000:7000" - - "7001:7001" - - "7002:7002" - - "7003:7003" - - "7004:7004" - - "7005:7005" - - "7006:7006" - - "7007:7007" diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..ee761aa --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,2 @@ +invoke>=2.0.0 +requests>=2.28.2 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4856826 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +version: '2' +services: + redis-cluster: + environment: + IP: ${REDIS_CLUSTER_IP} + SENTINEL: ${REDIS_USE_SENTINEL} + STANDALONE: ${REDIS_USE_STANDALONE} + build: + context: . + args: + redis_version: '7.2.5' + hostname: server + ports: + - '7000-7050:7000-7050' + - '5000-5010:5000-5010' diff --git a/docker-data/redis-conf/7000/redis.conf b/docker-data/redis-conf/7000/redis.conf deleted file mode 100644 index ab660cb..0000000 --- a/docker-data/redis-conf/7000/redis.conf +++ /dev/null @@ -1,6 +0,0 @@ -port 7000 -cluster-enabled yes -cluster-config-file nodes.conf -cluster-node-timeout 5000 -appendonly yes -dir /redis-data/7000 diff --git a/docker-data/redis-conf/7001/redis.conf b/docker-data/redis-conf/7001/redis.conf deleted file mode 100644 index 9eb113a..0000000 --- a/docker-data/redis-conf/7001/redis.conf +++ /dev/null @@ -1,6 +0,0 @@ -port 7001 -cluster-enabled yes -cluster-config-file nodes.conf -cluster-node-timeout 5000 -appendonly yes -dir /redis-data/7001 diff --git a/docker-data/redis-conf/7002/redis.conf b/docker-data/redis-conf/7002/redis.conf deleted file mode 100644 index e4c796a..0000000 --- a/docker-data/redis-conf/7002/redis.conf +++ /dev/null @@ -1,6 +0,0 @@ -port 7002 -cluster-enabled yes -cluster-config-file nodes.conf -cluster-node-timeout 5000 -appendonly yes -dir /redis-data/7002 diff --git a/docker-data/redis-conf/7004/redis.conf b/docker-data/redis-conf/7004/redis.conf deleted file mode 100644 index dd17c7b..0000000 --- a/docker-data/redis-conf/7004/redis.conf +++ /dev/null @@ -1,6 +0,0 @@ -port 7004 -cluster-enabled yes -cluster-config-file nodes.conf -cluster-node-timeout 5000 -appendonly yes -dir /redis-data/7004 diff --git a/docker-data/redis-conf/7005/redis.conf b/docker-data/redis-conf/7005/redis.conf deleted file mode 100644 index 8749c77..0000000 --- a/docker-data/redis-conf/7005/redis.conf +++ /dev/null @@ -1,6 +0,0 @@ -port 7005 -cluster-enabled yes -cluster-config-file nodes.conf -cluster-node-timeout 5000 -appendonly yes -dir /redis-data/7005 diff --git a/docker-data/redis-conf/7006/redis.conf b/docker-data/redis-conf/7006/redis.conf deleted file mode 100644 index f8a8cfc..0000000 --- a/docker-data/redis-conf/7006/redis.conf +++ /dev/null @@ -1,3 +0,0 @@ -port 7006 -appendonly yes -dir /redis-data/7006 diff --git a/docker-data/redis-conf/7007/redis.conf b/docker-data/redis-conf/7007/redis.conf deleted file mode 100644 index c88adb2..0000000 --- a/docker-data/redis-conf/7007/redis.conf +++ /dev/null @@ -1,3 +0,0 @@ -port 7007 -appendonly yes -dir /redis-data/7007 diff --git a/docker-data/start.sh b/docker-data/start.sh deleted file mode 100644 index adf1776..0000000 --- a/docker-data/start.sh +++ /dev/null @@ -1,6 +0,0 @@ -supervisord -sleep 3 - -IP=`ifconfig | grep "inet addr:17" | cut -f2 -d ":" | cut -f1 -d " "` -echo "yes" | ruby /redis/src/redis-trib.rb create --replicas 1 ${IP}:7000 ${IP}:7001 ${IP}:7002 ${IP}:7003 ${IP}:7004 ${IP}:7005 -tail -f /var/log/supervisor/redis-1.log diff --git a/docker-data/supervisord.conf b/docker-data/supervisord.conf deleted file mode 100644 index c8a8182..0000000 --- a/docker-data/supervisord.conf +++ /dev/null @@ -1,50 +0,0 @@ -[supervisord] -nodaemon=false - -[program:redis-1] -command=/redis/src/redis-server /redis-conf/7000/redis.conf -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -autorestart=true - -[program:redis-2] -command=/redis/src/redis-server /redis-conf/7001/redis.conf -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -autorestart=true - -[program:redis-3] -command=/redis/src/redis-server /redis-conf/7002/redis.conf -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -autorestart=true - -[program:redis-4] -command=/redis/src/redis-server /redis-conf/7003/redis.conf -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -autorestart=true - -[program:redis-5] -command=/redis/src/redis-server /redis-conf/7004/redis.conf -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -autorestart=true - -[program:redis-6] -command=/redis/src/redis-server /redis-conf/7005/redis.conf -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -autorestart=true - -[program:redis-7] -command=/redis/src/redis-server /redis-conf/7006/redis.conf -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -autorestart=true - -[program:redis-8] -command=/redis/src/redis-server /redis-conf/7007/redis.conf -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -autorestart=true diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 0000000..0b2e67f --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,103 @@ +#!/bin/sh + +if [ "$1" = 'redis-cluster' ]; then + # Allow passing in cluster IP by argument or environmental variable + IP="${2:-$IP}" + + if [ -z "$IP" ]; then # If IP is unset then discover it + IP=$(hostname -I) + fi + + echo " -- IP Before trim: '$IP'" + IP=$(echo ${IP}) # trim whitespaces + echo " -- IP Before split: '$IP'" + IP=${IP%% *} # use the first ip + echo " -- IP After trim: '$IP'" + + if [ -z "$INITIAL_PORT" ]; then # Default to port 7000 + INITIAL_PORT=7000 + fi + + if [ -z "$MASTERS" ]; then # Default to 3 masters + MASTERS=3 + fi + + if [ -z "$SLAVES_PER_MASTER" ]; then # Default to 1 slave for each master + SLAVES_PER_MASTER=1 + fi + + if [ -z "$BIND_ADDRESS" ]; then # Default to any IPv4 address + BIND_ADDRESS=0.0.0.0 + fi + + max_port=$(($INITIAL_PORT + $MASTERS * ( $SLAVES_PER_MASTER + 1 ) - 1)) + first_standalone=$(($max_port + 1)) + if [ "$STANDALONE" = "true" ]; then + STANDALONE=2 + fi + if [ ! -z "$STANDALONE" ]; then + max_port=$(($max_port + $STANDALONE)) + fi + + for port in $(seq $INITIAL_PORT $max_port); do + mkdir -p /redis-conf/${port} + mkdir -p /redis-data/${port} + + if [ -e /redis-data/${port}/nodes.conf ]; then + rm /redis-data/${port}/nodes.conf + fi + + if [ -e /redis-data/${port}/dump.rdb ]; then + rm /redis-data/${port}/dump.rdb + fi + + if [ -e /redis-data/${port}/appendonly.aof ]; then + rm /redis-data/${port}/appendonly.aof + fi + + if [ "$port" -lt "$first_standalone" ]; then + PORT=${port} BIND_ADDRESS=${BIND_ADDRESS} envsubst < /redis-conf/redis-cluster.tmpl > /redis-conf/${port}/redis.conf + nodes="$nodes $IP:$port" + else + PORT=${port} BIND_ADDRESS=${BIND_ADDRESS} envsubst < /redis-conf/redis.tmpl > /redis-conf/${port}/redis.conf + fi + + if [ "$port" -lt $(($INITIAL_PORT + $MASTERS)) ]; then + if [ "$SENTINEL" = "true" ]; then + PORT=${port} SENTINEL_PORT=$((port - 2000)) envsubst < /redis-conf/sentinel.tmpl > /redis-conf/sentinel-${port}.conf + cat /redis-conf/sentinel-${port}.conf + fi + fi + + done + + bash /generate-supervisor-conf.sh $INITIAL_PORT $max_port > /etc/supervisor/supervisord.conf + + supervisord -c /etc/supervisor/supervisord.conf + sleep 3 + + # + ## Check the version of redis-cli and if we run on a redis server below 5.0 + ## If it is below 5.0 then we use the redis-trib.rb to build the cluster + # + /redis/src/redis-cli --version | grep -E "redis-cli 3.0|redis-cli 3.2|redis-cli 4.0" + + if [ $? -eq 0 ] + then + echo "Using old redis-trib.rb to create the cluster" + echo "yes" | eval ruby /redis/src/redis-trib.rb create --replicas "$SLAVES_PER_MASTER" "$nodes" + else + echo "Using redis-cli to create the cluster" + echo "yes" | eval /redis/src/redis-cli --cluster create --cluster-replicas "$SLAVES_PER_MASTER" "$nodes" + fi + + if [ "$SENTINEL" = "true" ]; then + for port in $(seq $INITIAL_PORT $(($INITIAL_PORT + $MASTERS))); do + redis-sentinel /redis-conf/sentinel-${port}.conf & + done + fi + + tail -f /var/log/supervisor/redis*.log +else + exec "$@" +fi diff --git a/generate-supervisor-conf.sh b/generate-supervisor-conf.sh new file mode 100644 index 0000000..1f96ace --- /dev/null +++ b/generate-supervisor-conf.sh @@ -0,0 +1,47 @@ +initial_port="$1" +max_port="$2" + +program_entry_template () +{ + local count=$1 + local port=$2 + echo " + +[program:redis-$count] +command=/redis/src/redis-server /redis-conf/$port/redis.conf +stdout_logfile=/var/log/supervisor/%(program_name)s.log +stderr_logfile=/var/log/supervisor/%(program_name)s.log +autorestart=true +" +} + +result_str=" +[unix_http_server] +file=/tmp/supervisor.sock ; path to your socket file + +[supervisord] +logfile=/supervisord.log ; supervisord log file +logfile_maxbytes=50MB ; maximum size of logfile before rotation +logfile_backups=10 ; number of backed up logfiles +loglevel=error ; info, debug, warn, trace +pidfile=/var/run/supervisord.pid ; pidfile location +nodaemon=false ; run supervisord as a daemon +minfds=1024 ; number of startup file descriptors +minprocs=200 ; number of process descriptors +user=root ; default user +childlogdir=/ ; where child log files will live + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket +" + +count=1 +for port in `seq $initial_port $max_port`; do + result_str="$result_str$(program_entry_template $count $port)" + count=$((count + 1)) +done + +echo "$result_str" diff --git a/docker-data/redis-conf/7003/redis.conf b/redis-cluster.tmpl similarity index 54% rename from docker-data/redis-conf/7003/redis.conf rename to redis-cluster.tmpl index e131d28..ebc9ce7 100644 --- a/docker-data/redis-conf/7003/redis.conf +++ b/redis-cluster.tmpl @@ -1,6 +1,8 @@ -port 7003 +bind ${BIND_ADDRESS} +port ${PORT} cluster-enabled yes cluster-config-file nodes.conf cluster-node-timeout 5000 appendonly yes -dir /redis-data/7003 +dir /redis-data/${PORT} +protected-mode no diff --git a/redis.tmpl b/redis.tmpl new file mode 100644 index 0000000..2081848 --- /dev/null +++ b/redis.tmpl @@ -0,0 +1,5 @@ +bind ${BIND_ADDRESS} +port ${PORT} +appendonly yes +dir /redis-data/${PORT} +protected-mode no diff --git a/sentinel.tmpl b/sentinel.tmpl new file mode 100644 index 0000000..7279a83 --- /dev/null +++ b/sentinel.tmpl @@ -0,0 +1,5 @@ +port ${SENTINEL_PORT} +sentinel monitor sentinel${PORT} 127.0.0.1 ${PORT} 2 +sentinel down-after-milliseconds sentinel${PORT} 5000 +sentinel failover-timeout sentinel${PORT} 60000 +sentinel parallel-syncs sentinel${PORT} 1 diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..a318792 --- /dev/null +++ b/tasks.py @@ -0,0 +1,161 @@ +import multiprocessing +import requests + +from multiprocessing import Pool +from invoke import task + + +latest_version_string = "7.2.5" + +# Unpublished versions +version_config_mapping = [] +version_config_mapping += [f"3.0.{i}" for i in range(0, 8)] +version_config_mapping += [f"3.2.{i}" for i in range(0, 14)] +version_config_mapping += [f"4.0.{i}" for i in range(0, 15)] +version_config_mapping += [f"5.0.{i}" for i in range(0, 13)] +version_config_mapping += [f"6.0.{i}" for i in range(0, 21)] + +# Published versions +version_config_mapping += [f"6.2.{i}" for i in range(0, 15)] +version_config_mapping += [f"7.0.{i}" for i in range(0, 16)] +version_config_mapping += [f"7.2.{i}" for i in range(0, 6)] +version_config_mapping += ["7.4-rc1"] + + +def version_name_to_version(version): + """ + Helper method that returns correct versions if you specify either + - all + - latest + + or it will filter your chosen version based on what you inputed as version argument + """ + if version == "all": + return version_config_mapping + elif version == "latest": + return [latest_version_string] + else: + return filter_versions(version) + + +def get_pool_size(cpu_from_cli): + if cpu_from_cli: + pool_size = int(cpu_from_cli) + else: + pool_size = multiprocessing.cpu_count() - 1 + + print(f"Configured multiprocess pool size: {pool_size}") + return pool_size + + +def filter_versions(desired_version): + result = [] + + for version in version_config_mapping: + if version.startswith(desired_version): + result.append(version) + + return result + + +def _docker_pull(config): + """ + Internal multiprocess method to run docker pull command + """ + c, version = config + print(f" -- Starting docker pull for version : {version}") + pull_command = f"docker pull grokzen/redis-cluster:{version}" + c.run(pull_command) + + +def _docker_build(config): + """ + Internal multiprocess method to run docker build command + """ + c, version = config + print(f" -- Starting docker build for version : {version}") + build_command = f"docker build --build-arg redis_version={version} -t grokzen/redis-cluster:{version} ." + c.run(build_command) + + +def _docker_push(config): + """ + Internal multiprocess method to run docker push command + """ + c, version = config + print(f" -- Starting docker push for version : {version}") + build_command = f"docker push grokzen/redis-cluster:{version}" + c.run(build_command) + + +@task +def pull(c, version, cpu=None): + print(f" -- Docker pull version docker-hub : {version}") + + pool = Pool(get_pool_size(cpu)) + pool.map( + _docker_pull, + [ + [c, version] + for version in version_name_to_version(version) + ], + ) + + +@task +def build(c, version, cpu=None): + print(f" -- Docker building version : {version}") + + pool = Pool(get_pool_size(cpu)) + pool.map( + _docker_build, + [ + [c, version] + for version in version_name_to_version(version) + ], + ) + + +@task +def push(c, version, cpu=None): + print(f" -- Docker push version to docker-hub : {version}") + + pool = Pool(get_pool_size(cpu)) + pool.map( + _docker_push, + [ + [c, version] + for version in version_name_to_version(version) + ], + ) + + +@task +def list(c): + from pprint import pprint + pprint(version_config_mapping, indent=2) + + +@task +def list_releases(c): + releases = [] + + for page in range(1, 5): + data = requests.get("https://api.github.com/repos/redis/redis/releases", params={"page": int(page)}) + + if data.status_code == 200: + for release in data.json(): + r = release["name"] + + if "rc" in r or r.startswith("5"): + pass + else: + releases.append(r) + else: + print("Error, stopping") + + for released_version in releases: + if released_version in version_config_mapping: + print(f"Release found - {released_version}") + else: + print(f"NOT found - {released_version}")