From d418c4443779b99dbd4a86cb4941385077bfc0f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Radek=20Gol=C3=A1=C5=88=20jr=2E?= Date: Tue, 24 Mar 2026 06:57:33 +0100 Subject: [PATCH] Initial Commit --- .gitignore | 219 +++++++++++++++++++++++++++++++++++++++ build/Dockerfile | 6 ++ build/entrypoint.sh | 9 ++ compose.yml | 40 +++++++ conf/redis.conf | 14 +++ conf/users.acl | 5 + notebooks/Untitled.ipynb | 74 +++++++++++++ scripts/cluster-init.sh | 7 ++ 8 files changed, 374 insertions(+) create mode 100644 .gitignore create mode 100644 build/Dockerfile create mode 100644 build/entrypoint.sh create mode 100644 compose.yml create mode 100644 conf/redis.conf create mode 100644 conf/users.acl create mode 100644 notebooks/Untitled.ipynb create mode 100644 scripts/cluster-init.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ec582a4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,219 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +# Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# poetry.lock +# poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +# pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml + + +notebooks/* +!notebooks/*.ipynb \ No newline at end of file diff --git a/build/Dockerfile b/build/Dockerfile new file mode 100644 index 0000000..a6f0deb --- /dev/null +++ b/build/Dockerfile @@ -0,0 +1,6 @@ +FROM redis:8.6.1 + +RUN apt update && apt install -y --no-install-recommends gettext && apt clean && rm -rf /var/lib/apt/lists/* +ADD ./entrypoint.sh /entrypoint.sh + +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/build/entrypoint.sh b/build/entrypoint.sh new file mode 100644 index 0000000..4321ba7 --- /dev/null +++ b/build/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -e + +if [ "${1%.tmpl}" != "$1" ]; then + cat "$1" | envsubst > /etc/redis.conf + set -- /etc/redis.conf +fi + +docker-entrypoint.sh "$@" \ No newline at end of file diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..8ddf0cf --- /dev/null +++ b/compose.yml @@ -0,0 +1,40 @@ +services: + redisinsight: + image: redis/redisinsight + ports: + - 5540:5540 + redis: + image: redis:8.6.1 + build: ./build + command: ["/conf/redis.conf"] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 1m30s + timeout: 30s + retries: 5 + start_period: 30s + volumes: + - ./conf:/conf:ro + - /data + expose: + - 6379 + - 16379 + deploy: + mode: replicated + endpoint_mode: dnsrr + replicas: 9 + clustersetup: + image: redis:8.6.1 + volumes: + - ./scripts/cluster-init.sh:/cluster-init.sh + entrypoint: /cluster-init.sh + depends_on: + redis: + condition: service_healthy + notebooks: + image: quay.io/jupyter/scipy-notebook:python-3.13 + ports: + - 8888:8888 + userns_mode: keep-id + volumes: + - ./notebooks:/home/jovyan diff --git a/conf/redis.conf b/conf/redis.conf new file mode 100644 index 0000000..5e45b34 --- /dev/null +++ b/conf/redis.conf @@ -0,0 +1,14 @@ +dir /data/ + +aclfile /conf/users.acl + +appendonly yes + +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 + +maxmemory 512mb + +masterauth replication +masteruser replication \ No newline at end of file diff --git a/conf/users.acl b/conf/users.acl new file mode 100644 index 0000000..b270023 --- /dev/null +++ b/conf/users.acl @@ -0,0 +1,5 @@ +user default off +user admin on >admin allkeys allchannels allcommands +user replication on >replication +psync +replconf +ping +user readonly on >readonly %R~* allchannels +@read +user readwrite on >readwrite %RW~* allchannels +@read +@write diff --git a/notebooks/Untitled.ipynb b/notebooks/Untitled.ipynb new file mode 100644 index 0000000..029d133 --- /dev/null +++ b/notebooks/Untitled.ipynb @@ -0,0 +1,74 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "05c0de49-0eb8-4cd2-89b2-c312111dda85", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install redis hiredis fastid faker tqdm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "90326242-f141-4035-9053-d3aab6cc9224", + "metadata": {}, + "outputs": [], + "source": [ + "from redis.cluster import RedisCluster\n", + "\n", + "client = RedisCluster(host=\"redis\", port=6379)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50b9f473-8924-4d03-acf8-71ecf25e54a8", + "metadata": {}, + "outputs": [], + "source": [ + "from fastid import ulid\n", + "from faker import Faker\n", + "from tqdm.notebook import trange\n", + "faker = Faker()\n", + "\n", + "with client.pipeline() as p:\n", + " for _ in trange(15_000_000):\n", + " p.hset(f\"profile:{ulid()}\", mapping={\"name\": \"John Doe\", \"sex\": \"M\"})\n", + " p.execute()\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23933aa3-ff5d-45f9-b402-59e58c02a2b3", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/cluster-init.sh b/scripts/cluster-init.sh new file mode 100644 index 0000000..2be3cd2 --- /dev/null +++ b/scripts/cluster-init.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +redis-cli --user admin --pass admin --cluster create --cluster-yes --cluster-replicas 2 \ + bsqbd-redis-1:6379 bsqbd-redis-2:6379 bsqbd-redis-3:6379 bsqbd-redis-4:6379 \ + bsqbd-redis-5:6379 bsqbd-redis-6:6379 bsqbd-redis-7:6379 bsqbd-redis-8:6379 \ + bsqbd-redis-9:6379 +