Deployment#

                       Internet
                           │
        ┌──────────────────┼───────────────────┬──────────────┐
        │                  │                   │              │
      :25                :587/:465           :143/:993      :443
    inbound MX         submission           IMAP           HTTPS (API,
        │                  │                   │            Webmail, Admin)
        └──────────────────┴───────────────────┘              │
                           │                                  │
                  ┌────────▼──────────────────────────────────▼────────┐
                  │                  ferroma container                 │
                  │  one process: SMTP, IMAP, HTTP API, queue workers, │
                  │  sync service, event bus                          │
                  └────────┬───────────────────────────────┬──────────┘
                           │                               │
                  ┌────────▼────────┐            ┌─────────▼──────────┐
                  │ postgres:16     │            │ volume ferroma-data│
                  │ (internal only) │            │  mail/ attachments/│
                  └─────────────────┘            │  tls/ backups/     │
                                                 └────────────────────┘

$TTL 3600
$ORIGIN example.com.

; --- address ---
@               IN  A       203.0.113.10
mail            IN  A       203.0.113.10
; Only publish AAAA if IPv6 genuinely works end to end. A host that
; advertises AAAA and cannot answer on it loses mail from dual-stack senders.
; mail          IN  AAAA    2001:db8::10

; --- mail routing ---
@               IN  MX  10  mail.example.com.

; A "null MX" says this domain accepts no mail. Do not publish it on a
; domain that has users:
; @             IN  MX  0   .

; --- sender authentication ---
@               IN  TXT     "v=spf1 mx -all"

; DKIM: paste the p= value from `ferroma dkim generate` / the Admin panel.
default._domainkey IN TXT  "v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA..."

; DMARC. Start at p=none to collect reports, then tighten.
_dmarc          IN  TXT     "v=DMARC1; p=quarantine; rua=mailto:[email protected]; ruf=mailto:[email protected]; adkim=r; aspf=r; pct=100"

; --- transport security ---
_mta-sts        IN  TXT     "v=STSv1; id=20260916000000"
mta-sts         IN  CNAME   mail.example.com.
_smtp._tls      IN  TXT     "v=TLSRPTv1; rua=mailto:[email protected]"

; --- issuance control ---
@               IN  CAA     0 issue "letsencrypt.org"
@               IN  CAA     0 iodef "mailto:[email protected]"

; --- optional service hostnames (specification §42) ---
; webmail       IN  CNAME   mail.example.com.
; admin         IN  CNAME   mail.example.com.

# What the world sees for your address. It must equal FERROMA_HOSTNAME.
dig +short -x 203.0.113.10
# expected: mail.example.com.

dig +short mail.example.com        # -> 203.0.113.10
dig +short -x 203.0.113.10         # -> mail.example.com.

@   IN  MX  10  mail.example.com.
@   IN  MX  20  mail2.example.com.

# Check what receivers will see.
dig +short TXT example.com | grep spf1

_mta-sts    IN  TXT     "v=STSv1; id=20260916000000"
mta-sts     IN  CNAME   mail.example.com.

# Served at https://mta-sts.example.com/.well-known/mta-sts.txt
version: STSv1
mode: enforce
mx: mail.example.com
max_age: 604800

# Everything at once.
dig +short MX  example.com
dig +short A   mail.example.com
dig +short -x  203.0.113.10
dig +short TXT example.com              | grep spf1
dig +short TXT default._domainkey.example.com
dig +short TXT _dmarc.example.com
dig +short TXT _mta-sts.example.com
curl -s https://mta-sts.example.com/.well-known/mta-sts.txt

# Ask a DNSBL whether your IP is already listed (use a real one, this is illustrative).
# 203.0.113.10 is documentation-reserved and will never be listed.
dig +short 10.113.0.203.zen.spamhaus.org

git clone … && cd ferroma
./scripts/deploy.sh

./scripts/deploy.sh status              # containers / health / database
./scripts/deploy.sh logs
./scripts/deploy.sh backup              # back up once, now (the nightly run is automatic)
./scripts/deploy.sh upgrade             # back up → rebuild the image → restart → wait for health
./scripts/deploy.sh restore /backups/20260916T030000Z
./scripts/deploy.sh dkim --enable       # turn signing on after the TXT record is published
./scripts/deploy.sh certs               # re-install a renewed certificate and restart (used by the certbot hook)
./scripts/deploy.sh doctor
./scripts/deploy.sh down [--volumes]

./scripts/deploy.sh --yes --domain example.com --admin [email protected] \
  --db-password "$DB_PW" \
  --tls-cert /etc/letsencrypt/live/mail.example.com/fullchain.pem \
  --tls-key  /etc/letsencrypt/live/mail.example.com/privkey.pem

cp .env.example .env
# Edit .env: at minimum POSTGRES_PASSWORD and FERROMA_JWT_SECRET.
docker compose up -d
docker compose logs -f ferroma

cp .env.example .env
# Edit .env and set every REQUIRED variable: see §4.
docker compose -f docker-compose.prod.yml pull
docker compose -f docker-compose.prod.yml up -d
docker compose -f docker-compose.prod.yml ps
docker compose -f docker-compose.prod.yml logs -f ferroma

FERROMA_TLS_ENABLED: 'true'
FERROMA_TLS_CERT: /etc/ferroma/tls/fullchain.pem
FERROMA_TLS_KEY: /etc/ferroma/tls/privkey.pem
FERROMA__API__SECURE_COOKIES: 'true'
FERROMA__SMTP__REQUIRE_TLS_FOR_AUTH: 'true'
FERROMA__IMAP__REQUIRE_TLS_FOR_LOGIN: 'true'
# The implicit-TLS listeners are off by default (0). Without these two lines the
# 465/993 published by compose map to a port nobody listens on — the connection
# is refused rather than reporting any error.
FERROMA__SMTP__SMTPS_PORT: '465'
FERROMA__IMAP__IMAPS_PORT: '993'
FERROMA_DKIM_ENABLED: ${FERROMA_DKIM_ENABLED:-false}
FERROMA_DKIM_KEY: /etc/ferroma/dkim/${FERROMA_DKIM_SELECTOR:-default}.private

backup:
  image: postgres:16-alpine
  entrypoint: ['/bin/sh', '/usr/local/bin/backup.sh']
  # A loop, not a one-shot: a one-shot script under restart: always writes another
  # full backup on every restart backoff, until the disk fills.
  command: ['--loop']
  restart: unless-stopped
  volumes:
    - ./scripts/backup.sh:/usr/local/bin/backup.sh:ro
    - ferroma-data:/mail:ro
    - backups:/backups

# Follow the log of one service.
docker compose -f docker-compose.prod.yml logs -f --tail=200 ferroma

# Restart just Ferroma (PostgreSQL keeps running).
docker compose -f docker-compose.prod.yml restart ferroma

# A shell inside the container, as the ferroma user.
docker compose -f docker-compose.prod.yml exec ferroma sh

# A psql session against the database.
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma

# Disk usage of the two volumes.
docker system df -v | grep -E 'ferroma-data|ferroma-postgres-data'

# Stop everything, keeping the volumes.
docker compose -f docker-compose.prod.yml down

# Stop everything and DESTROY the volumes. This deletes all mail and users.
# docker compose -f docker-compose.prod.yml down -v

FERROMA__SMTP__PORT=2525
FERROMA__TLS__ENABLED=true
FERROMA__API__SECURE_COOKIES=true
FERROMA__SMTP__REQUIRE_TLS_FOR_AUTH=true
FERROMA__IMAP__REQUIRE_TLS_FOR_LOGIN=true
FERROMA__API__TRUST_PROXY_HEADERS=true
FERROMA__LIMITS__MAX_MESSAGE_SIZE=52428800

DATABASE_URL                  FERROMA_HOSTNAME        FERROMA_DATA_DIR
FERROMA_LOG_LEVEL             FERROMA_LOG_FORMAT      FERROMA_SMTP_HOST
FERROMA_SMTP_PORT             FERROMA_SMTP_SUBMISSION_PORT
FERROMA_IMAP_PORT             FERROMA_API_HOST        FERROMA_API_PORT
FERROMA_API_PUBLIC_URL        FERROMA_JWT_SECRET      FERROMA_TLS_ENABLED
FERROMA_TLS_CERT              FERROMA_TLS_KEY         FERROMA_DKIM_ENABLED
FERROMA_DKIM_KEY              FERROMA_DKIM_SELECTOR

# Generate both, then put them in .env. Never commit .env.
openssl rand -base64 32      # POSTGRES_PASSWORD
openssl rand -base64 48      # FERROMA_JWT_SECRET

FERROMA__SMTP__SMTPS_PORT=465
FERROMA__IMAP__IMAPS_PORT=993

mkdir -p tls dkim
# Certificate and key, however you obtained them.
ls -l tls/fullchain.pem tls/privkey.pem

# In .env
FERROMA_TLS_ENABLED=true
FERROMA_TLS_CERT=/etc/ferroma/tls/fullchain.pem
FERROMA_TLS_KEY=/etc/ferroma/tls/privkey.pem

tls.cert_path and tls.key_path must be set together (or enable self_signed_fallback)

# Add to docker-compose.prod.yml's ferroma service.
    ports:
      - '25:25'
      - '587:587'
      - '465:465'
      - '143:143'
      - '993:993'
      - '127.0.0.1:8080:8080'      # HTTP, bound to loopback: the proxy reaches it
    environment:
      FERROMA_TLS_ENABLED: 'true'  # still needed for 465 and 993
      FERROMA__SMTP__SMTPS_PORT: '465'
      FERROMA__IMAP__IMAPS_PORT: '993'
      FERROMA__API__TRUST_PROXY_HEADERS: 'true'
      FERROMA__API__SECURE_COOKIES: 'true'

server {
    listen 443 ssl http2;
    server_name mail.example.com mta-sts.example.com;

    ssl_certificate     /etc/letsencrypt/live/mail.example.com/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/mail.example.com/privkey.pem;
    ssl_protocols       TLSv1.2 TLSv1.3;
    ssl_prefer_server_ciphers off;

    add_header Strict-Transport-Security "max-age=31536000" always;
    add_header X-Content-Type-Options nosniff always;
    add_header X-Frame-Options DENY always;
    add_header Referrer-Policy no-referrer always;

    client_max_body_size 30m;      # >= api.max_request_size and limits.max_message_size

    # The MTA-STS policy is served by ferroma-api, so forward it (point the CNAME
    # for mta-sts.<domain> at this host, and cover it in the certificate too).
    location /.well-known/mta-sts.txt {
        proxy_pass http://127.0.0.1:18080;
        proxy_set_header Host $host;
    }

    location / {
        proxy_pass http://127.0.0.1:18080;   # the prod stack uses 8080; §3.1's stack defaults to 18080
        proxy_http_version 1.1;
        proxy_set_header Host              $host;
        proxy_set_header X-Real-IP         $remote_addr;
        proxy_set_header X-Forwarded-For   $remote_addr;   # overwrite, never append
        proxy_set_header X-Forwarded-Proto $scheme;

        # The realtime socket needs the upgrade dance and a long read timeout.
        proxy_set_header Upgrade    $http_upgrade;
        proxy_set_header Connection "upgrade";
        proxy_read_timeout 3600s;
    }
}

# certbot, HTTP-01. Port 80 must be reachable.
sudo certbot certonly --standalone \
  -d mail.example.com -d mta-sts.example.com \
  --agree-tos -m [email protected] --no-eff-email

# Files land here; symlink or copy them into ./tls/.
sudo ls -l /etc/letsencrypt/live/mail.example.com/
#   fullchain.pem  -> tls/fullchain.pem
#   privkey.pem    -> tls/privkey.pem

# Copy into the mounted directory with the ownership the container expects
# (the image runs as uid 10001).
sudo install -o 10001 -g 10001 -m 0640 \
  /etc/letsencrypt/live/mail.example.com/fullchain.pem tls/fullchain.pem
sudo install -o 10001 -g 10001 -m 0600 \
  /etc/letsencrypt/live/mail.example.com/privkey.pem   tls/privkey.pem
docker compose -f docker-compose.prod.yml restart ferroma

# A renewal rehearsal: certbot really does issue (against staging) and fires the
# deploy hook above.
sudo certbot renew --dry-run

docker compose -f docker-compose.prod.yml up -d
docker compose -f docker-compose.prod.yml ps
docker compose -f docker-compose.prod.yml logs ferroma | tail -50

# Or drive it from the shell.
curl -s https://mail.example.com/api/v1/setup
curl -s -X POST https://mail.example.com/api/v1/setup \
  -H 'Content-Type: application/json' \
  -d '{"email":"[email protected]","password":"…","hostname":"mail.example.com","domain":"example.com"}'

curl -s -X POST https://mail.example.com/api/v1/domains \
  -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \
  -d '{"name":"example.com","description":"primary"}'

-- Domains are lower-cased; the schema enforces it.
INSERT INTO domains (name, description) VALUES ('example.com', 'primary')
RETURNING id, name, enabled;

# The password is hashed with Argon2id by the server; never insert a hash by hand.
curl -s -X POST https://mail.example.com/api/v1/users \
  -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \
  -d '{"email":"[email protected]","password":"…","display_name":"Alice","quota_bytes":1073741824}'

# Attach an address to that user. This creates the Maildir and the standard folders.
curl -s -X POST https://mail.example.com/api/v1/users/7/mailboxes \
  -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \
  -d '{"domain":"example.com","local_part":"alice","is_primary":true}'

# Confirm on disk, inside the container.
docker compose -f docker-compose.prod.yml exec ferroma \
  ls -la /var/lib/ferroma/mail/example.com/alice/Maildir

# Local delivery to a local address, by hand. Use a real From for a real test.
printf 'EHLO test\r\nMAIL FROM:<[email protected]>\r\nRCPT TO:<[email protected]>\r\nDATA\r\nSubject: hello\r\n\r\nfirst\r\n.\r\nQUIT\r\n' \
  | nc 127.0.0.1 25

# Did it land?
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -c \
  "SELECT id, uid, subject, sender, size_bytes, storage_path FROM messages ORDER BY id DESC LIMIT 5;"

# Preferred: the CLI generates a 2048-bit key and writes it into the domain record
# (or --out to a file).
docker compose exec ferroma ferroma dkim generate --domain example.com

# Print the TXT record to publish:
docker compose exec ferroma ferroma dkim show --domain example.com

# Or with OpenSSL. 2048-bit RSA is the size receivers
# expect; 1024 is too weak and 4096 is slow to verify.
openssl genrsa -out dkim/default.private 2048
openssl rsa -in dkim/default.private -pubout -out dkim/default.public

# The TXT record value, on one line:
printf 'v=DKIM1; k=rsa; p=%s\n' \
  "$(openssl rsa -in dkim/default.private -pubout 2>/dev/null \
     | grep -v '^-----' | tr -d '\n')"

# The private key must be readable only by the service user (uid 10001).
sudo chown 10001:10001 dkim/default.private
chmod 0600 dkim/default.private

default._domainkey IN TXT (
    "v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA"
    "…the rest of the base64…"
)

dig +short TXT default._domainkey.example.com
# expected (illustrative): "v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A..."

# config/ferroma.toml
[dkim]
enabled = true
selector = "default"
private_key_path = "/etc/ferroma/dkim/default.private"
canonicalization = "relaxed"
verify_inbound = true
add_auth_results = true

# Or through the environment, which is how docker-compose.prod.yml does it.
FERROMA_DKIM_ENABLED=true
FERROMA_DKIM_SELECTOR=default
FERROMA_DKIM_KEY=/etc/ferroma/dkim/default.private
docker compose -f docker-compose.prod.yml restart ferroma

# Send a message to a checker that reports the DKIM result.
swaks --server mail.example.com --port 587 --tls \
      --auth PLAIN --auth-user [email protected] --auth-password '…' \
      --from [email protected] --to [email protected] \
      --body "dkim test"

# Or read the Authentication-Results header in a message you sent to yourself.

docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -Atc \
  "SELECT storage_path FROM messages ORDER BY id DESC LIMIT 1"

/backups/20260916T030000Z/
├── ferroma.dump      pg_dump --format=custom --compress=6
├── schema.sql        pg_dump --schema-only: an empty but correct structure
├── maildir.tar.gz    tar -czf of the mail root
├── config.tar.gz     ferroma.toml, DKIM keys, TLS material
├── MANIFEST          version, created_at, database, postgres_version,
│                     ferroma_version, hostname
└── SHA256SUMS        sha256sum ./*, so a restore can prove the archive survived

backup:
  image: postgres:16-alpine
  environment:
    PGHOST: postgres
    PGUSER: ${POSTGRES_USER:-ferroma}
    PGPASSWORD: ${POSTGRES_PASSWORD}
    PGDATABASE: ${POSTGRES_DB:-ferroma}
    BACKUP_DIR: /backups
    RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-14}
    BACKUP_INTERVAL_SECONDS: ${BACKUP_INTERVAL_SECONDS:-86400}
  entrypoint: ['/bin/sh', '/usr/local/bin/backup.sh']
  command: ['--loop']
  restart: unless-stopped
  volumes:
    - ./scripts/backup.sh:/usr/local/bin/backup.sh:ro
    - ferroma-data:/mail:ro
    - backups:/backups

# One pass, then exit (--once is the explicit "run once").
docker compose -f docker-compose.prod.yml run --rm backup --once

# A loop in the foreground: every 24 h, retention 14 days.
docker compose -f docker-compose.prod.yml run --rm \
  -e BACKUP_INTERVAL_SECONDS=86400 -e RETENTION_DAYS=14 \
  backup --loop

# Off-host, which is what makes a backup a backup. Run it from the host:
docker run --rm -v ferroma-backups:/backups -v "$PWD:/out" alpine \
  tar -czf /out/ferroma-backups-$(date -u +%Y%m%d).tar.gz -C /backups .

docker compose -f docker-compose.prod.yml stop ferroma
docker compose -f docker-compose.prod.yml run --rm backup --once
docker compose -f docker-compose.prod.yml start ferroma

# Verify only: check the checksums and print the manifest, restore nothing. The
# service can keep running.
./scripts/deploy.sh restore /backups/20260916T030000Z --verify-only

# A full restore: the script stops Ferroma first, restores, then brings it back up
# and waits for the health check.
./scripts/deploy.sh restore /backups/20260916T030000Z

# The raw equivalent (external-db stack).
docker compose -f docker-compose.external-db.yml stop ferroma
docker compose -f docker-compose.external-db.yml --profile tools run --rm \
  restore /backups/20260916T030000Z
docker compose -f docker-compose.external-db.yml up -d ferroma

database ferroma is not empty (19 tables). Set FORCE_RESTORE=1 to overwrite,
or restore into a fresh database.

# external-db stack: adding -e FORCE_RESTORE=1 is enough.
docker compose -f docker-compose.external-db.yml stop ferroma
docker compose -f docker-compose.external-db.yml --profile tools run --rm \
  -e FORCE_RESTORE=1 restore /backups/20260916T030000Z
docker compose -f docker-compose.external-db.yml up -d ferroma

./scripts/deploy.sh restore /backups/20260916T030000Z --db-only
./scripts/deploy.sh restore /backups/20260916T030000Z --mail-only

# 1. The checks first: rows without bodies, files without rows, counters, uid_next.
#    See docs/storage.md §9 for the queries and the shell loops.
docker compose -f docker-compose.prod.yml exec ferroma sh -c '
  psql "$DATABASE_URL" -Atc "SELECT COUNT(*) FROM messages WHERE expunged_at IS NULL"'

# 2. Reconcile the counters that are allowed to drift.
#    FoldersRepository::recount and MailboxesRepository::recompute_usage, triggered
#    through POST /api/v1/storage/gc (an admin token is required) and the Admin
#    storage screen.
#    curl -s -X POST https://mail.example.com/api/v1/storage/gc \
#      -H "Authorization: Bearer $TOKEN"

# 3. Start Ferroma and watch the log for the first minute.
docker compose -f docker-compose.prod.yml up -d ferroma
docker compose -f docker-compose.prod.yml logs -f --tail=100 ferroma

# Restore into a scratch database on the same host, without touching production.
docker compose -f docker-compose.prod.yml exec postgres createdb -U ferroma ferroma_drill
docker compose -f docker-compose.prod.yml exec postgres \
  pg_restore --no-owner --no-privileges --dbname=ferroma_drill /backups/…/ferroma.dump
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma_drill -c 'SELECT COUNT(*) FROM messages;'
docker compose -f docker-compose.prod.yml exec postgres dropdb -U ferroma ferroma_drill

# 1. Back up first. Always. An upgrade is the second-most-likely time to need it.
docker compose -f docker-compose.prod.yml run --rm backup --once
# On the external-db stack one step does all of it: ./scripts/deploy.sh upgrade
# (back up → rebuild → restart → wait for health)

# 2. Read the release notes for migration and configuration changes.
#    A new required key, or a removed one, stops the new version at boot
#    (unknown keys are rejected).

# 3. Update the image tag. Prod never builds from source.
sed -i 's/^FERROMA_VERSION=.*/FERROMA_VERSION=0.2.0/' .env

# 4. Pull and recreate only the ferroma service.
docker compose -f docker-compose.prod.yml pull ferroma
docker compose -f docker-compose.prod.yml up -d ferroma

# 5. Watch it come up.
docker compose -f docker-compose.prod.yml logs -f --tail=100 ferroma

# Roll the image back.
sed -i 's/^FERROMA_VERSION=.*/FERROMA_VERSION=0.1.0/' .env
docker compose -f docker-compose.prod.yml pull ferroma
docker compose -f docker-compose.prod.yml up -d ferroma

docker compose -f docker-compose.external-db.yml stop ferroma
docker compose -f docker-compose.external-db.yml --profile tools run --rm \
  -e FORCE_RESTORE=1 restore /backups/<pre-upgrade-stamp>
docker compose -f docker-compose.external-db.yml up -d ferroma

{
  "status": "ok",
  "version": "0.1.0",
  "protocol_version": 1,
  "uptime_secs": 84213,
  "database": { "ok": true, "server_version": "PostgreSQL 16.15",
                "pool": { "size": 4, "idle": 3, "max": 20 } },
  "smtp": { "enabled": true, "connections": 3 },
  "imap": { "enabled": true, "connections": 1 },
  "queue": { "pending": 0, "delivering": 0, "retry": 2, "failed": 1 }
}

docker compose -f docker-compose.prod.yml exec ferroma \
  ferroma healthcheck --url http://127.0.0.1:8080/api/v1/health

# Or without the CLI.
docker compose -f docker-compose.prod.yml exec ferroma \
  sh -c 'wget -qO- http://127.0.0.1:8080/api/v1/health || echo unreachable'

# The queue, grouped.
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -c \
  "SELECT status, COUNT(*) FROM mail_queue GROUP BY status ORDER BY 2 DESC;"

# The oldest thing waiting to be retried.
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -c \
  "SELECT id, recipient, attempts, next_attempt_at, last_status_code, left(last_error,60)
     FROM mail_queue WHERE status IN ('pending','retry')
    ORDER BY next_attempt_at LIMIT 20;"

# Storage.
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -c \
  "SELECT pg_size_pretty(pg_database_size('ferroma')) AS db,
          (SELECT COUNT(*) FROM messages WHERE expunged_at IS NULL) AS live_messages,
          (SELECT COUNT(*) FROM users) AS users,
          (SELECT COUNT(*) FROM domains) AS domains;"
du -sh "$(docker volume inspect -f '{{.Mountpoint}}' ferroma-data)"

# Failed logins in the last day.
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -c \
  "SELECT email, ip, COUNT(*) FROM login_attempts
    WHERE NOT success AND created_at > NOW() - INTERVAL '1 day'
    GROUP BY email, ip ORDER BY 3 DESC LIMIT 20;"

# Just the errors.
docker compose -f docker-compose.prod.yml logs ferroma | grep -i '"level":"ERROR"'

# Everything about one message id.
docker compose -f docker-compose.prod.yml logs ferroma | grep '4821'

# A delivery that failed.
docker compose -f docker-compose.prod.yml logs ferroma | grep -i 'delivery'

# Fresh log lines as they happen, filtered.
docker compose -f docker-compose.prod.yml logs -f ferroma | grep -E 'WARN|ERROR'

# 1. Does the PTR match FERROMA_HOSTNAME, and does it point back?
dig +short -x 203.0.113.10            # must equal FERROMA_HOSTNAME
dig +short mail.example.com           # must be the same address
grep FERROMA_HOSTNAME .env

# 2. Is SPF present, single, and terminating in -all?
dig +short TXT example.com | grep spf1

# 3. Is the DKIM record published and does it match the key you sign with?
dig +short TXT default._domainkey.example.com
openssl rsa -in dkim/default.private -pubout 2>/dev/null | grep -v '^-----' | tr -d '\n'

# 4. Is DMARC present?
dig +short TXT _dmarc.example.com

# 5. Is the IP on a blocklist?
dig +short 10.113.0.203.zen.spamhaus.org

# 6. Is the queue reporting failures with a remote status code?
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -c \
  "SELECT recipient, last_status_code, last_status_text FROM mail_queue
    WHERE status = 'failed' ORDER BY updated_at DESC LIMIT 10;"

# 1. Is a connection from the internet reaching port 25 at all?
#    From a machine outside your network:
nc -vz mail.example.com 25

# 2. Does the MX record resolve, and is it the host you think?
dig +short MX example.com
dig +short A mail.example.com

# 3. Is Ferroma listening on 25 inside the container?
docker compose -f docker-compose.prod.yml exec ferroma \
  sh -c 'netstat -tlnp 2>/dev/null || ss -tlnp'

# 4. Did the connection even arrive? If there is no log line, it never got here.
docker compose -f docker-compose.prod.yml logs ferroma | grep -i 'connection\|reject\|550\|554'

# 5. Is the certificate valid from outside?
openssl s_client -starttls smtp -connect mail.example.com:25 -crlf < /dev/null 2>&1 | head -30

# 6. Does the greeting name match the PTR?
printf 'EHLO test\r\nQUIT\r\n' | nc mail.example.com 25

# 1. Is the queue actually growing, and is the first attempt recent?
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -c \
  "SELECT status, COUNT(*), MIN(next_attempt_at), MAX(created_at)
     FROM mail_queue GROUP BY status;"

# 2. What are the top failures saying?
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -c \
  "SELECT recipient, attempts, last_status_code, last_error
     FROM mail_queue WHERE status IN ('retry','failed')
    ORDER BY attempts DESC LIMIT 20;"

# 3. The attempt history of one stuck delivery.
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -c \
  "SELECT a.attempt, a.remote_mx, a.status_code, a.status_text, a.duration_ms, a.created_at
     FROM delivery_attempts a JOIN mail_queue q ON q.id = a.queue_id
    WHERE q.id = 1234 ORDER BY a.attempt;"

# 4. Can this host reach the remote MX on port 25 at all?
docker compose -f docker-compose.prod.yml exec ferroma \
  sh -c 'nc -vz gmail-smtp-in.l.google.com 25'

# 5. Is outbound 25 blocked by the provider? Test from the host.
nc -vz alt1.gmail-smtp-in.l.google.com 25

# 6. Is the dispatcher running at all?
docker compose -f docker-compose.prod.yml logs ferroma | grep -i 'queue\|dispatch'
grep -A6 '^\[queue\]' config/ferroma.toml

-- Who is sending the most?
SELECT user_id, COUNT(*) FROM mail_queue
 WHERE created_at > NOW() - INTERVAL '1 hour'
 GROUP BY user_id ORDER BY 2 DESC LIMIT 10;

# 1. Is IMAP listening?
docker compose -f docker-compose.prod.yml exec ferroma \
  sh -c 'netstat -tlnp 2>/dev/null | grep -E "143|993"'

# 2. What does the greeting and capability list say?
openssl s_client -crlf -connect mail.example.com:143 < /dev/null 2>&1 | head -20
# for implicit TLS:
openssl s_client -connect mail.example.com:993 < /dev/null 2>&1 | head -20

# 3. Try a real login.
printf 'a LOGIN [email protected] "…"\r\nb LOGOUT\r\n' | \
  openssl s_client -quiet -crlf -connect mail.example.com:143

# 4. Is TLS required, and is your client doing it?
grep -E 'require_tls_for_login|imaps_port' config/ferroma.toml
grep FERROMA__IMAP__REQUIRE_TLS_FOR_LOGIN .env

# 5. Is the account locked by the login throttle?
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -c \
  "SELECT id, email, enabled, failed_logins, locked_until FROM users WHERE email='[email protected]';"

# 6. What do the recent attempts say?
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -c \
  "SELECT email, ip, success, created_at FROM login_attempts
    ORDER BY created_at DESC LIMIT 20;"

# 7. The server's view of the failure.
docker compose -f docker-compose.prod.yml logs ferroma | grep -i 'imap\|login'

# 1. Health, from inside the container.
docker compose -f docker-compose.prod.yml exec ferroma \
  sh -c 'wget -qO- http://127.0.0.1:8080/api/v1/health || echo unreachable'

# 2. Is the container healthy?
docker compose -f docker-compose.prod.yml ps
docker inspect -f '{{.State.Health.Status}} restarts={{.RestartCount}}' ferroma

# 3. Why did it exit?
docker compose -f docker-compose.prod.yml logs --tail=200 ferroma | grep -iE 'error|panic|config'

# 4. A configuration typo stops the server at boot. That is by design.
docker compose -f docker-compose.prod.yml run --rm ferroma \
  ferroma serve --config /etc/ferroma/ferroma.toml

# What the database believes.
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -c \
  "SELECT m.id, d.name||'@'||m.local_part AS addr, u.used_bytes, u.quota_bytes
     FROM mailboxes m JOIN domains d ON d.id=m.domain_id JOIN users u ON u.id=m.user_id
    ORDER BY u.used_bytes DESC LIMIT 10;"

# What is actually on disk.
docker compose -f docker-compose.prod.yml exec ferroma \
  du -sb /var/lib/ferroma/mail/example.com/alice

# The files that make up the total.
docker compose -f docker-compose.prod.yml exec ferroma \
  sh -c 'find /var/lib/ferroma/mail/example.com/alice -type f -printf "%s %p\n" | sort -rn | head -20'

# Where.
du -sh /var/lib/docker/volumes/ferroma-data/_data/*
du -sh /var/lib/docker/volumes/ferroma-data/_data/mail/*

# Garbage from interrupted writes.
docker compose -f docker-compose.prod.yml exec ferroma \
  sh -c 'find /var/lib/ferroma/mail -type d -name tmp -exec du -sh {} +'

# Unreferenced attachment blobs.
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -Atc 'SELECT DISTINCT storage_path FROM attachments' | wc -l
docker compose -f docker-compose.prod.yml exec ferroma \
  sh -c 'find /var/lib/ferroma/attachments -type f ! -name "*.tmp" | wc -l'

# Expunged messages still holding their files.
docker compose -f docker-compose.prod.yml exec postgres \
  psql -U ferroma -d ferroma -c \
  "SELECT COUNT(*), pg_size_pretty(SUM(size_bytes)) FROM messages WHERE expunged_at IS NOT NULL;"