UPCE-FEI/BSQBD
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: data persistence in shared volume

Browse Source 
implemented a template to enable data persistence even with a shared volume across containers
This commit is contained in:
Radek Goláň jr
2026-04-20 11:39:35 +02:00
parent 11330e6b89
commit a9f9c834be
8 changed files with 113 additions and 18633 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/Dockerfile
+1 -1
View File
@@ -1,6 +1,6 @@
FROM redis:8.6.1 FROM redis:8.6.1
RUN apt update && apt install -y --no-install-recommends gettext && apt clean && rm -rf /var/lib/apt/lists/* RUN apt update && apt install -y --no-install-recommends gettext dnsutils && apt clean && rm -rf /var/lib/apt/lists/*
ADD ./entrypoint.sh /entrypoint.sh ADD ./entrypoint.sh /entrypoint.sh
ENTRYPOINT [ "/entrypoint.sh" ] ENTRYPOINT [ "/entrypoint.sh" ]
build/entrypoint.sh
Regular → Executable
+4
View File
@@ -1,6 +1,10 @@
#!/bin/sh #!/bin/sh
set -e set -e
export CONTAINER_NAME=$(dig -x "$(hostname -i)" +short | cut -d . -f 1)
mkdir -p /data/$CONTAINER_NAME
cd /data/$CONTAINER_NAME
if [ "${1%.tmpl}" != "$1" ]; then if [ "${1%.tmpl}" != "$1" ]; then
cat "$1" | envsubst > /etc/redis.conf cat "$1" | envsubst > /etc/redis.conf
set -- /etc/redis.conf set -- /etc/redis.conf
compose.yml
+6 -3
View File
@@ -4,9 +4,9 @@ services:
ports: ports:
- 5540:5540 - 5540:5540
redis: redis:
image: redis:8.6.1 image: golyalpha/redis:8.6.1
build: ./build build: ./build
command: ["/conf/redis.conf"] command: ["/conf/redis.conf.tmpl"]
healthcheck: healthcheck:
test: ["CMD", "redis-cli", "ping"] test: ["CMD", "redis-cli", "ping"]
interval: 1m30s interval: 1m30s
@@ -15,7 +15,7 @@ services:
start_period: 30s start_period: 30s
volumes: volumes:
- ./conf:/conf:ro - ./conf:/conf:ro
- /data - redis_data:/data
expose: expose:
- 6379 - 6379
- 16379 - 16379
@@ -38,3 +38,6 @@ services:
userns_mode: keep-id userns_mode: keep-id
volumes: volumes:
- ./notebooks:/home/jovyan - ./notebooks:/home/jovyan
volumes:
redis_data:
conf/redis.conf → conf/redis.conf.tmpl
+1 -1
View File
@@ -1,4 +1,4 @@
dir /data/ dir /data/${CONTAINER_NAME}/
aclfile /conf/users.acl aclfile /conf/users.acl
notebooks/fill_test.ipynb
+7 -54
View File
@@ -2,39 +2,17 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": null,
"id": "05c0de49-0eb8-4cd2-89b2-c312111dda85", "id": "05c0de49-0eb8-4cd2-89b2-c312111dda85",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: redis in /opt/conda/lib/python3.13/site-packages (7.4.0)\n",
"Requirement already satisfied: hiredis in /opt/conda/lib/python3.13/site-packages (3.3.1)\n",
"Requirement already satisfied: fastid in /opt/conda/lib/python3.13/site-packages (0.0.5)\n",
"Requirement already satisfied: tqdm in /opt/conda/lib/python3.13/site-packages (4.67.3)\n",
"Requirement already satisfied: kagglehub in /opt/conda/lib/python3.13/site-packages (1.0.0)\n",
"Requirement already satisfied: kagglesdk<1.0,>=0.1.14 in /opt/conda/lib/python3.13/site-packages (from kagglehub) (0.1.16)\n",
"Requirement already satisfied: packaging in /opt/conda/lib/python3.13/site-packages (from kagglehub) (26.0)\n",
"Requirement already satisfied: pyyaml in /opt/conda/lib/python3.13/site-packages (from kagglehub) (6.0.3)\n",
"Requirement already satisfied: requests in /opt/conda/lib/python3.13/site-packages (from kagglehub) (2.32.5)\n",
"Requirement already satisfied: protobuf in /opt/conda/lib/python3.13/site-packages (from kagglesdk<1.0,>=0.1.14->kagglehub) (6.33.5)\n",
"Requirement already satisfied: charset_normalizer<4,>=2 in /opt/conda/lib/python3.13/site-packages (from requests->kagglehub) (3.4.6)\n",
"Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.13/site-packages (from requests->kagglehub) (3.11)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.13/site-packages (from requests->kagglehub) (2.6.3)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.13/site-packages (from requests->kagglehub) (2026.2.25)\n",
"Note: you may need to restart the kernel to use updated packages.\n"
]
}
],
"source": [ "source": [
"%pip install redis hiredis fastid tqdm kagglehub" "%pip install redis hiredis fastid tqdm kagglehub"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": null,
"id": "90326242-f141-4035-9053-d3aab6cc9224", "id": "90326242-f141-4035-9053-d3aab6cc9224",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -49,25 +27,10 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": null,
"id": "50b9f473-8924-4d03-acf8-71ecf25e54a8", "id": "50b9f473-8924-4d03-acf8-71ecf25e54a8",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "887890574a2048a587b6ed4dd8eef7c6",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/1000000 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [ "source": [
"from fastid import ulid\n", "from fastid import ulid\n",
"from tqdm.notebook import trange\n", "from tqdm.notebook import trange\n",
@@ -80,20 +43,10 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": null,
"id": "23933aa3-ff5d-45f9-b402-59e58c02a2b3", "id": "23933aa3-ff5d-45f9-b402-59e58c02a2b3",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"data": {
"text/plain": [
"b'OK'"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [ "source": [
"schema = (\n", "schema = (\n",
" TextField(\"name\"),\n", " TextField(\"name\"),\n",
notebooks/r6_analysis.ipynb
+72 -1295
View File
File diff suppressed because it is too large Load Diff
notebooks/r6_load_resampled.ipynb
+11 -84
View File
@@ -2,39 +2,17 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": null,
"id": "071f6969-c01a-4369-a763-871f5b9e65b3", "id": "071f6969-c01a-4369-a763-871f5b9e65b3",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: redis in /opt/conda/lib/python3.13/site-packages (7.4.0)\n",
"Requirement already satisfied: hiredis in /opt/conda/lib/python3.13/site-packages (3.3.1)\n",
"Requirement already satisfied: fastid in /opt/conda/lib/python3.13/site-packages (0.0.5)\n",
"Requirement already satisfied: tqdm in /opt/conda/lib/python3.13/site-packages (4.67.3)\n",
"Requirement already satisfied: kagglehub in /opt/conda/lib/python3.13/site-packages (1.0.0)\n",
"Requirement already satisfied: kagglesdk<1.0,>=0.1.14 in /opt/conda/lib/python3.13/site-packages (from kagglehub) (0.1.16)\n",
"Requirement already satisfied: packaging in /opt/conda/lib/python3.13/site-packages (from kagglehub) (26.0)\n",
"Requirement already satisfied: pyyaml in /opt/conda/lib/python3.13/site-packages (from kagglehub) (6.0.3)\n",
"Requirement already satisfied: requests in /opt/conda/lib/python3.13/site-packages (from kagglehub) (2.32.5)\n",
"Requirement already satisfied: protobuf in /opt/conda/lib/python3.13/site-packages (from kagglesdk<1.0,>=0.1.14->kagglehub) (6.33.5)\n",
"Requirement already satisfied: charset_normalizer<4,>=2 in /opt/conda/lib/python3.13/site-packages (from requests->kagglehub) (3.4.6)\n",
"Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.13/site-packages (from requests->kagglehub) (3.11)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.13/site-packages (from requests->kagglehub) (2.6.3)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.13/site-packages (from requests->kagglehub) (2026.2.25)\n",
"Note: you may need to restart the kernel to use updated packages.\n"
]
}
],
"source": [ "source": [
"%pip install redis hiredis fastid tqdm kagglehub" "%pip install redis hiredis fastid tqdm kagglehub"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": null,
"id": "526cb932-59aa-489e-bbba-954ac645b633", "id": "526cb932-59aa-489e-bbba-954ac645b633",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -66,20 +44,10 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": null,
"id": "be4c5461-1db2-4226-b71e-cdc7f06615bd", "id": "be4c5461-1db2-4226-b71e-cdc7f06615bd",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"data": {
"text/plain": [
"'Downsample of full dump present: True'"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [ "source": [
"dataset = Path(kagglehub.dataset_download(\"awesomizer/rainbox-six-siege-dataset\"))\n", "dataset = Path(kagglehub.dataset_download(\"awesomizer/rainbox-six-siege-dataset\"))\n",
"downsampled = dataset/\"downsampled_S5.csv\"\n", "downsampled = dataset/\"downsampled_S5.csv\"\n",
@@ -88,25 +56,10 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": null,
"id": "2efb5699-cced-4df0-9d76-278e23874436", "id": "2efb5699-cced-4df0-9d76-278e23874436",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "2fa9c64b42394db7a5764896d7d936d0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"0it [00:00, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [ "source": [
"with downsampled.open(\"rb\") as f:\n", "with downsampled.open(\"rb\") as f:\n",
" rows = sum(1 for _ in tqdm(f)) - 1" " rows = sum(1 for _ in tqdm(f)) - 1"
@@ -114,25 +67,10 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": null,
"id": "9177760b-eec4-4ab4-8242-0dd8df5db9dd", "id": "9177760b-eec4-4ab4-8242-0dd8df5db9dd",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "f235da3945864d409e81037d42269756",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Importing data into Redis...: 0%| | 0/102698 [00:00<?, ?row/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [ "source": [
"with downsampled.open(errors=\"ignore\") as csvfile:\n", "with downsampled.open(errors=\"ignore\") as csvfile:\n",
" reader = DictReader(csvfile)\n", " reader = DictReader(csvfile)\n",
@@ -144,21 +82,10 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 6, "execution_count": null,
"id": "d8b75e35-8bef-4b2b-ae25-1982076c73fd", "id": "d8b75e35-8bef-4b2b-ae25-1982076c73fd",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"data": {
"text/plain": [
"'OK'"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [ "source": [
"from redis.commands.search.field import TagField, NumericField\n", "from redis.commands.search.field import TagField, NumericField\n",
"from redis.commands.search.index_definition import IndexDefinition, IndexType\n", "from redis.commands.search.index_definition import IndexDefinition, IndexType\n",
notebooks/r6_resample.ipynb
+9 -17193
View File
File diff suppressed because it is too large Load Diff