Docker-compose

docker-compose.yml

version: "3.8"

services:
  traefik:
    image: traefik:v2.10
    container_name: traefik
    restart: always
    command:
      - "--api.dashboard=true"
      - "--providers.file.directory=/traefik/dynamic"
      - "--providers.file.watch=true"
      - "--entrypoints.web.address=:80"
      - "--entrypoints.websecure.address=:443"
      - "--certificatesresolvers.letsencrypt.acme.email=${EMAIL_LETSENCRYPT}"
      - "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json"
      - "--certificatesresolvers.letsencrypt.acme.tlschallenge=true"
      - "--serverstransport.insecureSkipVerify=false"  # Verhindert SSL-Warnungen
      - "--global.sendAnonymousUsage=false"  # Verhindert, dass Traefik Telemetrie sendet
      - "--accesslog=true"
      - "--accesslog.filepath=/logs/access.log"
    ports:
      - "80:80"
      - "443:443"
    networks:
      - traefik_default
    env_file:
      - .env
    volumes:
      - "./traefik/dynamic:/traefik/dynamic:ro"
      - "./traefik/letsencrypt:/letsencrypt"
      - "./logs:/logs"

  # 🔹 PostgreSQL als Hauptdatenbank
  postgres:
    image: postgres:15
    container_name: postgres
    restart: always
    environment:
      POSTGRES_DB: ${POSTGRES_DB}
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
    networks:
      - traefik_default
    volumes:
      - postgres_data:/var/lib/postgresql/data

  # 🔹 ClickHouse für OLAP-Abfragen
  clickhouse:
    image: clickhouse/clickhouse-server
    container_name: clickhouse
    restart: always
    environment:
      CLICKHOUSE_DB: "ssd_analytics"
    networks:
      - traefik_default
    volumes:
      - clickhouse_data:/var/lib/clickhouse

  # 🔹 Airflow Webserver & Scheduler
  airflow:
    image: apache/airflow:2.6.3
    container_name: airflow
    restart: always
    depends_on:
      - postgres
      - clickhouse
    environment:
      - LOAD_EXAMPLES=False
      - AIRFLOW__CORE__EXECUTOR=SequentialExecutor
    volumes:
      - ./airflow/dags:/opt/airflow/dags
    networks:
      - traefik_default
    command: >
      bash -c "airflow db init &&
      airflow users create --username ${AIRFLOW_USER} --password ${AIRFLOW_PASSWORD} --firstname Admin --lastname User --role Admin --email ${EMAIL_LETSENCRYPT} &&
      airflow standalone"

  # 🔹 Metabase für OLAP-Analysen & Dashboards
  metabase:
    image: metabase/metabase:latest
    container_name: metabase
    restart: always
    depends_on:
      - postgres
      - clickhouse
    networks:
      - traefik_default
    environment:
      MB_DB_TYPE: postgres
      MB_DB_DBNAME: ${POSTGRES_DB}
      MB_DB_HOST: postgres
    MB_DB_USER: ${POSTGRES_USER}
      MB_DB_PASS: ${POSTGRES_PASSWORD}
    volumes:
      - ./metabase-data:/metabase-data


  # 🔹 pgAdmin für PostgreSQL-Administration
  pgadmin:
    image: dpage/pgadmin4:latest
    container_name: pgadmin
    restart: always
    environment:
      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_EMAIL}
      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_PASSWORD}
    networks:
      - traefik_default
    depends_on:
      - postgres

  # 🔹 Jupyter Notebook für Machine Learning
  jupyter:
    image: jupyter/tensorflow-notebook
    container_name: jupyter
    restart: always
    environment:
      JUPYTER_TOKEN: ${JUPYTER_TOKEN}
      JUPYTER_PASSWORD: ${JUPYTER_PASSWORD}
    volumes:
      - ./jupyter:/home/jovyan/work
    networks:
      - traefik_default
    command: >
      start-notebook.sh --NotebookApp.token='' --NotebookApp.password="${JUPYTER_HASH_PASSWORD}"

  mediawiki:
    image: mediawiki
    container_name: mediawiki
    restart: always
    networks:
      - traefik_default  # Traefik-Netzwerk
    environment:
      MEDIAWIKI_DB_TYPE: "sqlite"
    dns:
      - 8.8.8.8
      - 1.1.1.1
    volumes:
      - mediawiki_data:/var/www/html
      - mediawiki_db:/var/www/data

networks:
  traefik_default:
    external: true

volumes:
  postgres_data:
  clickhouse_data:
  metabase_data:
  mediawiki_data:
  mediawiki_db:

traefik/dynamic/traefik_dynamic.yml

http:
  routers:
    traefik:
      rule: "Host(`traefik.roth-it-solutions.de`)"
      service: "api@internal"
      entryPoints:
        - "websecure"
      middlewares:
        - "traefik-auth-secure"
      tls:
        certResolver: "letsencrypt"

    airflow:
      rule: "Host(`airflow.roth-it-solutions.de`)"
      service: "airflow"
      entryPoints:
        - "websecure"
      middlewares:
        - "traefik-auth-secure"
        - "rate-limit"
      tls:
        certResolver: "letsencrypt"

    metabase:
      rule: "Host(`metabase.roth-it-solutions.de`)"
      service: "metabase"
      middlewares:
        - "rate-limit"
      entryPoints:
        - "websecure"
      tls:
        certResolver: "letsencrypt"

    pgadmin:
      rule: "Host(`pgadmin.roth-it-solutions.de`)"
      service: "pgadmin"
      middlewares:
        - "rate-limit"
      entryPoints:
        - "websecure"
      tls:
        certResolver: "letsencrypt"

    jupyter:
      rule: "Host(`jupyter.roth-it-solutions.de`)"
      service: "jupyter"
      middlewares:
        - "traefik-auth-secure"
        - "rate-limit"
      entryPoints:
        - "websecure"
      tls:
        certResolver: "letsencrypt"

    mediawiki:
      rule: "Host(`wiki.roth-it-solutions.de`)"
      service: "mediawiki"
      entryPoints:
        - "websecure"
      tls:
        certResolver: "letsencrypt"
  middlewares:
    # kifeart
    traefik-auth:
      basicAuth:
        usersFile: "/traefik/dynamic/traefik_auth_users"

    traefik-auth-secure:
      chain:
        middlewares:
          - traefik-auth
          - remove-server-header

    rate-limit:
      rateLimit:
        average: 50
        burst: 100
        period: 1s

    remove-server-header:
      headers:
        frameDeny: true
        sslRedirect: true
        browserXssFilter: true
        contentTypeNosniff: true
        forceSTSHeader: true
        stsIncludeSubdomains: true
        stsPreload: true
        customResponseHeaders:
          Server: ""
          X-Powered-By: ""
          X-Traefik-Server: ""

  services:
    airflow:
      loadBalancer:
        servers:
          - url: "http://airflow:8080"

    metabase:
      loadBalancer:
        servers:
          - url: "http://metabase:3000"

    pgadmin:
      loadBalancer:
        servers:
          - url: "http://pgadmin:80"

    jupyter:
      loadBalancer:
        servers:
          - url: "http://jupyter:8888"

    mediawiki:
      loadBalancer:
        servers:
          - url: "http://mediawiki:80"

fix-iptables-conntrack.sh

#!/bin/bash

echo "🔧 Setze iptables-Regeln und optimiere conntrack..."

# **Aktive Netzwerkschnittstelle automatisch ermitteln**
IFACE=$(ip route | grep default | awk '{print $5}')
if [[ -z "$IFACE" ]]; then
    echo "⚠️ Keine aktive Netzwerkschnittstelle gefunden. Bitte manuell prüfen!"
    exit 1
fi
echo "🌍 Gefundene Netzwerkschnittstelle: $IFACE"

# Standardrichtlinien setzen (sollte nur gemacht werden, wenn gewünscht)
iptables -P INPUT DROP
iptables -P FORWARD DROP
iptables -P OUTPUT ACCEPT

# Bestehende Regeln leeren
iptables -F
iptables -X

# Loopback-Schnittstelle zulassen
iptables -A INPUT -i lo -j ACCEPT
iptables -A OUTPUT -o lo -j ACCEPT

# Verbindungen erlauben, die zu einer bestehenden Verbindung gehören
iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
iptables -A OUTPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
iptables -A FORWARD -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT

# **DNS erlauben** (damit `apt-get update` funktioniert)
iptables -A OUTPUT -p udp --dport 53 -j ACCEPT
iptables -A OUTPUT -p tcp --dport 53 -j ACCEPT
iptables -A INPUT -p udp --sport 53 -j ACCEPT
iptables -A INPUT -p tcp --sport 53 -j ACCEPT

# **HTTP und HTTPS erlauben** (für `apt-get update` und Web-Anfragen)
iptables -A OUTPUT -p tcp --dport 80 -j ACCEPT
iptables -A OUTPUT -p tcp --dport 443 -j ACCEPT

# **SSH-Zugang erlauben** (wichtig, falls du über SSH auf den Server zugreifst)
iptables -A INPUT -p tcp --dport 22 -j ACCEPT

# **Traefik-Ports erlauben (nur falls nötig)**
iptables -A INPUT -p tcp --dport 80 -j ACCEPT
iptables -A INPUT -p tcp --dport 443 -j ACCEPT

# **Docker-Netzwerkverkehr zulassen**
iptables -A INPUT -i docker0 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
iptables -A OUTPUT -o docker0 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
iptables -A FORWARD -i docker0 -o $IFACE -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
iptables -A FORWARD -i $IFACE -o docker0 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT

echo "🚀 Bereinige conntrack-Tabelle..."
conntrack -F
conntrack -D --state INVALID

echo "📌 Setze `conntrack`-Maximalwert auf 1.048.576..."
echo 1048576 > /proc/sys/net/netfilter/nf_conntrack_max
sysctl -w net.netfilter.nf_conntrack_max=1048576

# **Regeln dauerhaft speichern (falls iptables-persistent installiert ist)**
if command -v iptables-save &> /dev/null; then
    iptables-save > /etc/iptables.rules
    echo "📌 iptables-Regeln gespeichert unter /etc/iptables.rules"
fi

echo "✅ iptables-Regeln erfolgreich gesetzt & conntrack optimiert!"

iptables

sudo apt install iptables-persistent -y
sudo netfilter-persistent save
sudo netfilter-persistent reload