diff --git a/.dockerignore b/.dockerignore index 536ff3c85..e9036795e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,90 @@ +# Version control (main + all nested repos like froide/*/.git) .git/ +.gitignore +.gitattributes +**/.git/ +**/.gitignore + +# IDE/Editor +.idea/ +.vscode/ +*.swp +*.swo +*~ +.DS_Store + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.egg-info/ +.eggs/ +dist/ +build/ +*.so +.Python +.env +.venv +env/ +venv/ +ENV/ + +# Node node_modules/ -*.egg-info +npm-debug.log* +pnpm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-store/ +.pnpm-cache/ +.pnpm-store/ +.pnpm-cache/ + +# Frontend build artifacts (exclude from build context) +**/dist/ +**/build/ +**/.vite/ +**/.nuxt/ +**/.output/ + +# Testing +.coverage +.pytest_cache/ +htmlcov/ +.tox/ +.nox/ +coverage.xml +*.cover + +# Frontend build artifacts +**/dist/ +**/build/ +**/.vite/ + +# Documentation +*.md +docs/ +*.rst + +# Docker +Dockerfile +docker-compose*.yml +.docker/ + +# Database +*.sqlite3 +*.db +*.sql + +# Logs +*.log +logs/ + +# Temporary +tmp/ +temp/ +*.tmp +*.temp + +# OS files +Thumbs.db diff --git a/.gitignore b/.gitignore index d03389873..70f369471 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,6 @@ htmlcov/ .coverage django.po .env.* -!fragdenstaat_de/locale/**/django.po \ No newline at end of file +!fragdenstaat_de/locale/**/django.po +.env +!.env.example diff --git a/compose-dev.yaml b/compose-dev.yaml index cd9e8657e..37226bf81 100644 --- a/compose-dev.yaml +++ b/compose-dev.yaml @@ -11,7 +11,7 @@ services: POSTGRES_PASSWORD: fragdenstaat_de elasticsearch: - build: ./deps/elasticsearch/ + build: ./docker/elasticsearch/ volumes: - es-data:/usr/share/elasticsearch/data - es-logs:/var/log diff --git a/devsetup.sh b/devsetup.sh index f6368b239..11c9e7a2a 100755 --- a/devsetup.sh +++ b/devsetup.sh @@ -103,18 +103,36 @@ pull() { } dependencies() { - source fds-env/bin/activate - echo "Installing $MAIN..." + # Check if running in Docker environment + if $DOCKER; then + uv pip install \ + --no-cache \ + --system \ + -r ./requirements-dev.txt + + for name in "${REPOS[@]}"; do + uv pip install --system -e "./$name" --config-setting editable_mode=compat + done + else + source fds-env/bin/activate + echo "Installing $MAIN..." - uv pip install -r $MAIN/requirements-dev.txt - install_precommit "$MAIN" + uv pip install -r $MAIN/requirements-dev.txt + install_precommit "$MAIN" - echo "Cloning / installing all editable dependencies..." + echo "Cloning / installing all editable dependencies..." + for name in "${REPOS[@]}"; do + uv pip install -e "./$name" --config-setting editable_mode=compat + install_precommit "$name" + done + fi +} - for name in "${REPOS[@]}"; do - uv pip install -e "./$name" --config-setting editable_mode=compat - install_precommit "$name" - done + +dockerized() { + pull + docker compose build + docker compose run --rm django python manage.py migrate --skip-checks } frontend() { @@ -144,6 +162,11 @@ frontend() { popd done + # Add a Docker-specific override + if $DOCKER; then + MAIN="." + fi + # Setup main project and link dependencies pushd "$MAIN" pnpm install @@ -187,7 +210,7 @@ if [ -z "$1" ]; then dependencies frontend messages - + echo "Done!" else if [[ $(type -t "$1") == function ]]; then diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..b400ebad3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,60 @@ +services: + django: + build: + context: . + dockerfile: ./docker/django/Dockerfile + command: > + bash -c "python manage.py runserver" + network_mode: "host" + volumes: + - .:/app + - /app/node_modules + depends_on: + - db + - elasticsearch + + frontend: + build: + context: . + dockerfile: ./docker/frontend/Dockerfile + command: "pnpm run dev" + network_mode: "host" + volumes: + - .:/app + - /app/node_modules + + db: + image: postgis/postgis:16-3.4 + volumes: + - pg-data:/var/lib/postgresql/ + environment: + POSTGRES_USER: fragdenstaat_de + POSTGRES_DB: fragdenstaat_de + POSTGRES_PASSWORD: fragdenstaat_de + network_mode: "host" + + elasticsearch: + build: ./docker/elasticsearch/ + volumes: + - es-data:/usr/share/elasticsearch/data + - es-logs:/var/log + environment: + - 'discovery.type=single-node' + - 'cluster.routing.allocation.disk.threshold_enabled=false' + - 'cluster.routing.allocation.disk.watermark.low=3gb' + - 'cluster.routing.allocation.disk.watermark.high=2gb' + - 'cluster.routing.allocation.disk.watermark.flood_stage=1gb' + - 'xpack.security.enabled=false' + network_mode: "host" + + redis: + image: redis:7-alpine + volumes: + - redis_data:/data + network_mode: "host" + +volumes: + es-data: + es-logs: + pg-data: + redis_data: diff --git a/docker/django/Dockerfile b/docker/django/Dockerfile new file mode 100644 index 000000000..d50fd24d5 --- /dev/null +++ b/docker/django/Dockerfile @@ -0,0 +1,47 @@ +FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + DJANGO_SETTINGS_MODULE=fragdenstaat_de.settings.development \ + DJANGO_CONFIGURATION=Dev \ + SHELL=/bin/bash \ + DOCKER=true +# Install system dependencies +RUN apt-get update && apt-get install -y \ + git \ + postgresql-client \ + postgis \ + gettext \ + gdal-bin \ + libgdal-dev \ + libpq-dev \ + build-essential \ + inotify-tools \ + curl \ + # Specific dependencies from README + imagemagick \ + libmagickwand-dev \ + libfreetype6-dev \ + poppler-utils \ + qpdf \ + ocrmypdf \ + libpango1.0-dev \ + libgeoip-dev \ + libmagic-dev \ + && rm -rf /var/lib/apt/lists/* + +# Set work directory +WORKDIR /app + +# Copy entire project +COPY . . + +# Install repo dependencies +RUN chmod +x ./devsetup.sh && \ + ./devsetup.sh dependencies + +# Compile messages +RUN python manage.py compilemessages -l de -i node_modules + +EXPOSE 8000 diff --git a/deps/elasticsearch/Dockerfile b/docker/elasticsearch/Dockerfile similarity index 100% rename from deps/elasticsearch/Dockerfile rename to docker/elasticsearch/Dockerfile diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile new file mode 100644 index 000000000..eead3102c --- /dev/null +++ b/docker/frontend/Dockerfile @@ -0,0 +1,28 @@ +FROM node:22-bookworm-slim + +# Set up pnpm home directory +ENV PNPM_HOME="/root/.local/share/pnpm" +ENV PATH="${PNPM_HOME}:${PATH}" +ENV DOCKER=true + +# Enable corepack +RUN corepack enable + +# Set work directory +WORKDIR /app + +# Copy content +COPY . ./ + +# Dynamically prepare pnpm version and activate +RUN pnpm_version=$(node -p "require('./package.json').packageManager?.replace('pnpm@', '')") && \ + echo "Using pnpm version: $pnpm_version" && \ + corepack prepare "pnpm@$pnpm_version" --activate + +# Install dependencies and run frontend setup +RUN chmod +x ./devsetup.sh && \ + ./devsetup.sh frontend + +EXPOSE 5173 + +CMD ["pnpm", "run", "dev"] diff --git a/dockersetup.md b/dockersetup.md new file mode 100644 index 000000000..92b8d88d3 --- /dev/null +++ b/dockersetup.md @@ -0,0 +1,60 @@ +First, clone the repository and enter: + +```bash +git clone https://github.com/okfde/fragdenstaat_de +cd fragdenstaat_de +``` + +There are currently two options for a setup with Docker: + +1. Only Database + Elasticsearch (See README.md) + +Use this when developing locally with Python/pnpm installed on your machine and setting up a virtual environment, but want to avoid setting up PostgreSQL and Elasticsearch manually. + +2. Full Docker setup + +This full setup builds the Docker image installing the necessary dependencies, compiling messages and making an initial migration. +Vite and Django serve from different containers. + +Make sure to have a local_settings.py file in /fragdenstaat_de/settings/. You can copy the example file from /fragdenstaat_de/settings/local_settings.py.example. + +Run the setup script + +```bash +./devsetuph.sh dockerized +``` +This clones the froide repositories and builds the docker image with all the dependencies and links, mimicking the setup for a virtual environment, just inside the container. + +Please note that the image can take over ten minutes to build and because of Vite incompatibilities, the containers run in "host" network mode, which may cause conflicts if those ports are already in use. It could be necessary to adjust the settings in Docker Desktop to allow the use of network host mode. + +Then start the containers with +```bash +docker compose up -d +``` + +Django will be listening on http://localhost:8000, Vite on http://localhost:5173 + +### Initial setup + +devsetup.sh automatically runs `python manage.py migrate --skip-checks` after build. However, you may still need to: + +1. Create a superuser +``` bash +docker compose exec -it django python manage.py createsuperuser +``` + +2. Load initial data +``` bash +docker compose exec django python manage.py loaddata tests/fixtures/cms.json +``` + +3. Create search index +``` bash +docker compose exec django python manage.py search_index --create +docker compose exec django python manage.py search_index --populate +``` + +For more convenient access you can use the container's shell: +``` bash +docker compose exec -it django bash +``` diff --git a/fragdenstaat_de/settings/local_settings.py.example b/fragdenstaat_de/settings/local_settings.py.example index 7caefc531..9085f91a7 100644 --- a/fragdenstaat_de/settings/local_settings.py.example +++ b/fragdenstaat_de/settings/local_settings.py.example @@ -18,13 +18,13 @@ class Dev(FragDenStaatBase): @property def INSTALLED_APPS(self): installed = super(Dev, self).INSTALLED_APPS - # installed += ['debug_toolbar'] + installed += ['debug_toolbar'] return installed @property def MIDDLEWARE(self): return super(Dev, self).MIDDLEWARE + [ - # 'debug_toolbar.middleware.DebugToolbarMiddleware', + 'debug_toolbar.middleware.DebugToolbarMiddleware', ] DATABASES = { @@ -34,13 +34,13 @@ class Dev(FragDenStaatBase): 'USER': 'fragdenstaat_de', # Not used with sqlite3. 'PASSWORD': 'fragdenstaat_de', # Not used with sqlite3. 'HOST': 'localhost', # Set to empty string for localhost. Not used with sqlite3. - 'PORT': '5436', # Set to empty string for default. Not used with sqlite3. + 'PORT': '5432', # Set to empty string for default. Not used with sqlite3. } } ELASTICSEARCH_DSL = { 'default': { - 'hosts': 'localhost:9206', + 'hosts': 'localhost:9200', 'timeout': 30, }, }