From f50f17d593231adf22792d48279dea2eca73c86f Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Wed, 9 Mar 2022 14:02:25 +0100 Subject: [PATCH 1/4] Docker worker enhancement * drops docker worker process in favor of background celery worker on main process * changes uploaded files folder into host-based folder --- docker-compose.yml | 33 ++++----------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a67fb2ff..bf32e64a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,10 +15,11 @@ services: - arnova-nginx-server:arnova build: . container_name: "konova-docker" - command: gunicorn konova.wsgi:application --bind 0.0.0.0:8000 + command: sh -c 'celery -A konova worker -l INFO --detach && gunicorn konova.wsgi:application --bind 0.0.0.0:8000' + restart: always volumes: - .:/konova - - konova_uploaded_files:/konova_uploaded_files + - /path/to/host/folder:/konova_uploaded_files - static_file_volume:/konova/static # Point to the volume for static files. Shared with nginx service expose: - 8000 @@ -36,31 +37,6 @@ services: - SMTP_PORT=25 - SMTP_REAL_REPLY_MAIL=ksp-servicestelle@sgdnord.rlp.de -# To provide a celery worker instance, we need to add the celery worker as an own service - celery_worker: - external_links: - - postgis:db - - arnova-nginx-server:arnova - build: . - container_name: "konova-worker-docker" - command: celery -A konova worker -l INFO - volumes: - - .:/konova - - konova_uploaded_files:/konova_uploaded_files - depends_on: - - konova - environment: - - POSTGRES_NAME=konova - - POSTGRES_PORT=5432 - - POSTGRES_PASSWORD=CHANGE_ME - - POSTGRES_USER=konova - - POSTGRES_HOST=db - - REDIS_HOST=redis - - SSO_HOST=CHANGE_ME_TO_SSO_HOST_URL - - SMTP_HOST=172.17.0.1 - - SMTP_PORT=25 - - SMTP_REAL_REPLY_MAIL=ksp-servicestelle@sgdnord.rlp.de - nginx: build: ./nginx container_name: "konova-nginx-server" @@ -79,5 +55,4 @@ networks: name: postgis_nat_it_backend volumes: - static_file_volume: - konova_uploaded_files: \ No newline at end of file + static_file_volume: \ No newline at end of file -- 2.45.2 From 7d29dddd226397fe7b3dfcb50c9bfe49d1d911dd Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Mon, 25 Apr 2022 16:07:38 +0200 Subject: [PATCH 2/4] WIP: Docker enhancement * reduce containers into a single one, holding nginx + celery + redis all at once --- Dockerfile | 6 ++++-- docker-compose.yml | 44 ++++++++++++++++++++++---------------------- nginx/nginx.conf | 2 +- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index aad14222..290c3d08 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM python:3.7-slim +FROM python:3.7 ENV PYTHONUNBUFFERED 1 WORKDIR /konova # Install some dependencies RUN apt update -RUN apt install -y gdal-bin +RUN apt install -y gdal-bin redis-server nginx # Copy requirements file into workspace COPY ./requirements.txt /konova/ @@ -13,6 +13,8 @@ RUN pip install --upgrade pip RUN pip install -r requirements.txt # Copy rest of project into workspace +RUN rm /etc/nginx/sites-enabled/default +COPY ./nginx/nginx.conf /etc/nginx/conf.d COPY . /konova/ # Move static files in designated folder diff --git a/docker-compose.yml b/docker-compose.yml index bf32e64a..74f8c0b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,13 +1,13 @@ version: '3.3' services: - redis: - image: redis - container_name: "konova-redis-cache" - volumes: - - /redis/data:/bitnami/redis/data - environment: - - REDIS_PASSWORD=CHANGE_ME + #redis: + # image: redis + # container_name: "konova-redis-cache" + # volumes: + # - /redis/data:/bitnami/redis/data + # environment: + # - REDIS_PASSWORD=CHANGE_ME konova: external_links: @@ -15,37 +15,37 @@ services: - arnova-nginx-server:arnova build: . container_name: "konova-docker" - command: sh -c 'celery -A konova worker -l INFO --detach && gunicorn konova.wsgi:application --bind 0.0.0.0:8000' + command: sh -c 'service nginx start && service redis-server start && celery -A konova worker --detach && gunicorn konova.wsgi:application --bind 0.0.0.0:8000' restart: always volumes: - .:/konova - /path/to/host/folder:/konova_uploaded_files - static_file_volume:/konova/static # Point to the volume for static files. Shared with nginx service - expose: - - 8000 - depends_on: - - redis + ports: + - "1337:80" + #depends_on: + # - redis environment: - POSTGRES_NAME=konova - POSTGRES_PORT=5432 - POSTGRES_PASSWORD=CHANGE_ME - POSTGRES_USER=konova - POSTGRES_HOST=db - - REDIS_HOST=redis + - REDIS_HOST=localhost - SSO_HOST=CHANGE_ME_TO_SSO_HOST_URL - SMTP_HOST=172.17.0.1 - SMTP_PORT=25 - SMTP_REAL_REPLY_MAIL=ksp-servicestelle@sgdnord.rlp.de - nginx: - build: ./nginx - container_name: "konova-nginx-server" - ports: - - "1337:80" - depends_on: - - konova - volumes: - - static_file_volume:/konova/static # Point to the volume for static files. Shared with konova service + #nginx: + # build: ./nginx + # container_name: "konova-nginx-server" + # ports: + # - "1337:80" + # depends_on: + # - konova + # volumes: + # - static_file_volume:/konova/static # Point to the volume for static files. Shared with konova service # Instead of an own, new network, we need to connect to the existing one, which is provided by the postgis container # NOTE: THIS NETWORK MUST EXIST diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 26b326ca..373393a2 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -1,5 +1,5 @@ upstream konova { - server konova:8000; + server localhost:8000; } server { -- 2.45.2 From 80de57a0879697b1bfbbd1e7adab40720022c956 Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Tue, 26 Apr 2022 08:55:07 +0200 Subject: [PATCH 3/4] WIP: Docker enhancement * reduces all needed containers into a single one * simplifies initial startup command by adding docker-entrypoint.sh --- Dockerfile | 8 +++++--- docker-compose.yml | 28 ++-------------------------- docker-entrypoint.sh | 7 +++++++ nginx/nginx.conf => nginx.conf | 0 nginx/Dockerfile | 4 ---- 5 files changed, 14 insertions(+), 33 deletions(-) create mode 100755 docker-entrypoint.sh rename nginx/nginx.conf => nginx.conf (100%) delete mode 100644 nginx/Dockerfile diff --git a/Dockerfile b/Dockerfile index 290c3d08..ad61d714 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,14 +7,16 @@ WORKDIR /konova RUN apt update RUN apt install -y gdal-bin redis-server nginx -# Copy requirements file into workspace +# Copy requirements file into workspace and install all dependencies COPY ./requirements.txt /konova/ RUN pip install --upgrade pip RUN pip install -r requirements.txt -# Copy rest of project into workspace +# Remove nginx default configuration and replace with own configuration RUN rm /etc/nginx/sites-enabled/default -COPY ./nginx/nginx.conf /etc/nginx/conf.d +COPY ./nginx.conf /etc/nginx/conf.d + +# Copy rest of project into workspace COPY . /konova/ # Move static files in designated folder diff --git a/docker-compose.yml b/docker-compose.yml index 74f8c0b4..c7bd3f4b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,30 +1,19 @@ version: '3.3' services: - #redis: - # image: redis - # container_name: "konova-redis-cache" - # volumes: - # - /redis/data:/bitnami/redis/data - # environment: - # - REDIS_PASSWORD=CHANGE_ME - konova: external_links: - postgis:db - arnova-nginx-server:arnova build: . container_name: "konova-docker" - command: sh -c 'service nginx start && service redis-server start && celery -A konova worker --detach && gunicorn konova.wsgi:application --bind 0.0.0.0:8000' + command: ./docker-entrypoint.sh restart: always volumes: - .:/konova - - /path/to/host/folder:/konova_uploaded_files - - static_file_volume:/konova/static # Point to the volume for static files. Shared with nginx service + - /data/apps/konova/uploaded_files:/konova_uploaded_files ports: - "1337:80" - #depends_on: - # - redis environment: - POSTGRES_NAME=konova - POSTGRES_PORT=5432 @@ -37,22 +26,9 @@ services: - SMTP_PORT=25 - SMTP_REAL_REPLY_MAIL=ksp-servicestelle@sgdnord.rlp.de - #nginx: - # build: ./nginx - # container_name: "konova-nginx-server" - # ports: - # - "1337:80" - # depends_on: - # - konova - # volumes: - # - static_file_volume:/konova/static # Point to the volume for static files. Shared with konova service - # Instead of an own, new network, we need to connect to the existing one, which is provided by the postgis container # NOTE: THIS NETWORK MUST EXIST networks: default: external: name: postgis_nat_it_backend - -volumes: - static_file_volume: \ No newline at end of file diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 00000000..3e405b72 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Start all needed services inside the container! + +service nginx start +service redis-server start +celery -A konova worker --detach +gunicorn konova.wsgi:application --bind 0.0.0.0:8000 \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx.conf similarity index 100% rename from nginx/nginx.conf rename to nginx.conf diff --git a/nginx/Dockerfile b/nginx/Dockerfile deleted file mode 100644 index 4c49d2ee..00000000 --- a/nginx/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM nginx:alpine - -RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d \ No newline at end of file -- 2.45.2 From 7027e0c02bd58ba8e813d14337713976048ee02d Mon Sep 17 00:00:00 2001 From: mpeltriaux Date: Tue, 26 Apr 2022 10:09:58 +0200 Subject: [PATCH 4/4] Docker enhancement * optimizes image build dependency * increases gunicorn default number of workers --- Dockerfile | 2 +- docker-entrypoint.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ad61d714..6e35526c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7 +FROM python:3.9-slim ENV PYTHONUNBUFFERED 1 WORKDIR /konova diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 3e405b72..e9f2cea2 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Start all needed services inside the container! - +# Start all needed services once the container is fired up! service nginx start service redis-server start celery -A konova worker --detach -gunicorn konova.wsgi:application --bind 0.0.0.0:8000 \ No newline at end of file +# Rule of thumb: (2*CPU)+1 as worker_num -> Use 5 as default (matches a dual core) +gunicorn --workers=5 konova.wsgi:application --bind=0.0.0.0:8000 \ No newline at end of file -- 2.45.2