👁1views

Migrating a Raspberry Pi 5 WordPress Stack from SD Card to NVMe

cat > /home/pi/benchmark-nvme.sh <<'EOF'
#!/usr/bin/env bash
# benchmark-nvme.sh -- fio I/O benchmark for SD vs NVMe comparison
#
# Usage:
#   bash benchmark-nvme.sh                         # tests current Docker root
#   TEST_DIR=/mnt/nvme bash benchmark-nvme.sh      # tests NVMe directly
set -euo pipefail

TEST_DIR="${TEST_DIR:-/var/lib/docker}"
RUNTIME=30
SIZE="512M"

echo ""
echo "Pi 5 NVMe benchmark -- $(date '+%Y-%m-%d %H:%M')"
echo ""
echo "  Test directory : ${TEST_DIR}"
echo "  Device         : $(df -h "${TEST_DIR}" | awk 'NR==2{print $1}')"
echo "  Free space     : $(df -h "${TEST_DIR}" | awk 'NR==2{print $4}')"
echo ""

run_fio() {
    local name="$1" rw="$2" bs="$3" iodepth="$4"
    echo -n "  ${name}... "
    fio --name="${name}" \
        --directory="${TEST_DIR}" \
        --size="${SIZE}" \
        --rw="${rw}" \
        --bs="${bs}" \
        --iodepth="${iodepth}" \
        --numjobs=1 \
        --runtime="${RUNTIME}" \
        --time_based \
        --ioengine=libaio \
        --direct=1 \
        --group_reporting \
        --output-format=terse \
        --terse-version=3 2>/dev/null \
    | awk -F';' '
        /^3;/ {
            bw_read  = $6  / 1024;
            bw_write = $47 / 1024;
            iops_read  = $7;
            iops_write = $48;
            if (bw_read  > 0) printf "read  %6.0f MB/s  IOPS %6.0f\n", bw_read,  iops_read;
            if (bw_write > 0) printf "write %6.0f MB/s  IOPS %6.0f\n", bw_write, iops_write;
        }'
    rm -f "${TEST_DIR}/${name}.*"
}

echo "  Sequential"
run_fio "seq-read"  "read"  "128k" 8
run_fio "seq-write" "write" "128k" 8
echo ""
echo "  Random 4K (QD1 -- MariaDB single-threaded pattern)"
run_fio "rand4k-read-qd1"  "randread"  "4k" 1
run_fio "rand4k-write-qd1" "randwrite" "4k" 1
echo ""
echo "  Random 4K (QD32 -- parallel I/O)"
run_fio "rand4k-read-qd32"  "randread"  "4k" 32
run_fio "rand4k-write-qd32" "randwrite" "4k" 32
echo ""
EOF
chmod +x /home/pi/benchmark-nvme.sh

lsblk
# Expect: nvme0n1 listed with your drive size

cat > /home/pi/setup-nvme.sh <<'SCRIPT'
#!/usr/bin/env bash
# setup-nvme.sh -- Mount NVMe SSD and migrate Docker data-root to it
#
# Steps:
#   1. Detect the NVMe device
#   2. Partition and format (ext4, GPT, single partition)
#   3. Mount at /mnt/nvme with NVMe-optimised fstab entry
#   4. I/O scheduler udev rule (set to 'none')
#   5. Sysctl tuning (dirty page ratios for NVMe)
#   6. Migrate Docker data-root (stop, rsync, verify, clean up)
#   7. Summary
#
# Override device:
#   NVME_DEVICE=/dev/nvme1n1 bash setup-nvme.sh
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "${REPO_ROOT}"

NVME_DEVICE="${NVME_DEVICE:-/dev/nvme0n1}"
NVME_PARTITION="${NVME_DEVICE}p1"
NVME_MOUNT="/mnt/nvme"
DOCKER_NEW_ROOT="${NVME_MOUNT}/docker"
DOCKER_OLD_ROOT="/var/lib/docker"
DOCKER_BACKUP="${DOCKER_OLD_ROOT}.bak-sd"

log()  { echo ""; echo "[$(date '+%H:%M:%S')] $*"; }
ok()   { echo "  ok  $*"; }
skip() { echo "  ->  $* (already done, skipping)"; }
fail() { echo "  FAIL: $*" >&2; exit 1; }
info() { echo "  ..  $*"; }

echo ""
echo "andrewbakerninja-pi -- NVMe setup and Docker migration"
echo ""

if [[ "$(uname -m)" != "aarch64" ]]; then
    echo "WARNING: Not running on ARM64. Detected: $(uname -m)"
    read -r -p "Continue anyway? [y/N] " _c
    [[ "${_c,,}" == "y" ]] || exit 0
fi

# ----------------------------------------------------------------
# STEP 1 -- Detect NVMe device
# ----------------------------------------------------------------
log "Step 1: Detect NVMe device..."

if [[ ! -b "${NVME_DEVICE}" ]]; then
    for candidate in /dev/nvme0n1 /dev/nvme1n1 /dev/nvme0n2; do
        if [[ -b "${candidate}" ]]; then
            NVME_DEVICE="${candidate}"
            NVME_PARTITION="${NVME_DEVICE}p1"
            break
        fi
    done
fi

[[ -b "${NVME_DEVICE}" ]] || \
    fail "No NVMe device found. Run 'lsblk'. Override: NVME_DEVICE=/dev/nvmeXnY bash $0"

NVME_BLK="$(basename "${NVME_DEVICE}")"
NVME_MODEL="$(cat /sys/block/${NVME_BLK}/device/model 2>/dev/null | xargs || echo 'unknown')"
NVME_SIZE="$(lsblk -dn -o SIZE "${NVME_DEVICE}" 2>/dev/null || echo 'unknown')"

info "Device : ${NVME_DEVICE}"
info "Model  : ${NVME_MODEL}"
info "Size   : ${NVME_SIZE}"

if mount | grep -q "${NVME_DEVICE}.*on / "; then
    echo ""
    echo "  WARNING: ${NVME_DEVICE} appears to be the root filesystem."
    echo "  Run 'docker info | grep Root' -- if it shows /mnt/nvme/docker, nothing to do."
    read -r -p "  Continue anyway? [y/N] " _c
    [[ "${_c,,}" == "y" ]] || exit 0
fi

# ----------------------------------------------------------------
# STEP 2 -- Partition and format (idempotent)
# ----------------------------------------------------------------
log "Step 2: Partition and format NVMe..."

if [[ -b "${NVME_PARTITION}" ]]; then
    FS_TYPE="$(blkid -o value -s TYPE "${NVME_PARTITION}" 2>/dev/null || echo "")"
    if [[ "${FS_TYPE}" == "ext4" ]]; then
        skip "${NVME_PARTITION} is already ext4"
    else
        fail "${NVME_PARTITION} exists but is '${FS_TYPE:-unknown}'. Inspect with 'lsblk -f'."
    fi
else
    echo ""
    echo "  WARNING: ${NVME_DEVICE} has no partitions. ALL DATA WILL BE ERASED."
    read -r -p "  Confirm erase ${NVME_DEVICE}? Type 'yes' to proceed: " _confirm
    [[ "${_confirm}" == "yes" ]] || { echo "Aborted."; exit 0; }

    info "Creating GPT partition table..."
    sudo parted -s "${NVME_DEVICE}" mklabel gpt
    sudo parted -s "${NVME_DEVICE}" mkpart primary ext4 0% 100%
    sudo partprobe "${NVME_DEVICE}"
    sleep 2

    info "Formatting as ext4 (label: nvme-data)..."
    sudo mkfs.ext4 -L nvme-data -q "${NVME_PARTITION}"
    ok "${NVME_PARTITION} formatted as ext4"
fi

# ----------------------------------------------------------------
# STEP 3 -- Mount at /mnt/nvme
# ----------------------------------------------------------------
log "Step 3: Mount NVMe at ${NVME_MOUNT}..."

sudo mkdir -p "${NVME_MOUNT}"

NVME_UUID="$(blkid -o value -s UUID "${NVME_PARTITION}")"
[[ -n "${NVME_UUID}" ]] || fail "Could not read UUID from ${NVME_PARTITION}"
info "UUID: ${NVME_UUID}"

if mountpoint -q "${NVME_MOUNT}"; then
    skip "${NVME_MOUNT} already mounted"
else
    sudo mount -o noatime "${NVME_PARTITION}" "${NVME_MOUNT}"
    ok "Mounted ${NVME_PARTITION} -> ${NVME_MOUNT}"
fi

if grep -q "${NVME_UUID}" /etc/fstab; then
    skip "fstab entry already present"
else
    {
        echo ""
        echo "# NVMe data drive -- added by setup-nvme.sh"
        echo "UUID=${NVME_UUID}  ${NVME_MOUNT}  ext4  defaults,noatime,commit=60  0  2"
    } | sudo tee -a /etc/fstab > /dev/null
    ok "fstab: UUID=${NVME_UUID} -> ${NVME_MOUNT} (noatime, commit=60)"
fi

info "NVMe free space: $(df -h "${NVME_MOUNT}" | awk 'NR==2{print $4}')"

# ----------------------------------------------------------------
# STEP 4 -- I/O scheduler: 'none' for NVMe
# ----------------------------------------------------------------
log "Step 4: NVMe I/O scheduler (udev rule)..."

UDEV_RULE_FILE="/etc/udev/rules.d/60-nvme-scheduler.rules"
UDEV_RULE='ACTION=="add|change", SUBSYSTEM=="block", KERNEL=="nvme[0-9]*n[0-9]*", ATTR{queue/scheduler}="none"'

if [[ -f "${UDEV_RULE_FILE}" ]] && grep -q 'scheduler.*none' "${UDEV_RULE_FILE}" 2>/dev/null; then
    skip "NVMe udev scheduler rule already in place"
else
    echo "${UDEV_RULE}" | sudo tee "${UDEV_RULE_FILE}" > /dev/null
    sudo udevadm control --reload-rules
    sudo udevadm trigger --subsystem-match=block 2>/dev/null || true
    ok "udev rule written: ${UDEV_RULE_FILE}"
fi

SCHEDULER_PATH="/sys/block/${NVME_BLK}/queue/scheduler"
if [[ -f "${SCHEDULER_PATH}" ]]; then
    echo "none" | sudo tee "${SCHEDULER_PATH}" > /dev/null 2>&1 || true
    CURRENT="$(grep -o '\[.*\]' "${SCHEDULER_PATH}" 2>/dev/null | tr -d '[]' || echo '?')"
    info "Current scheduler: ${CURRENT}"
fi

# ----------------------------------------------------------------
# STEP 5 -- Sysctl tuning for NVMe
# ----------------------------------------------------------------
log "Step 5: NVMe sysctl tuning..."

NVME_SYSCTL="/etc/sysctl.d/99-pi-nvme.conf"
if [[ -f "${NVME_SYSCTL}" ]]; then
    skip "${NVME_SYSCTL} already present"
else
    sudo tee "${NVME_SYSCTL}" > /dev/null <<'EOF'
# andrewbakerninja-pi -- NVMe I/O sysctl tuning

# Allow more dirty pages before blocking writes (NVMe flushes them quickly)
vm.dirty_ratio = 20
vm.dirty_background_ratio = 5

# Flush dirty pages every 15s (default 5s; NVMe handles bursts efficiently)
vm.dirty_writeback_centisecs = 1500
vm.dirty_expire_centisecs    = 3000
EOF
    sudo sysctl -p "${NVME_SYSCTL}" > /dev/null
    ok "${NVME_SYSCTL} written and applied"
fi

# ----------------------------------------------------------------
# STEP 6 -- Migrate Docker data-root to NVMe
# ----------------------------------------------------------------
log "Step 6: Migrate Docker data-root to NVMe..."

CURRENT_ROOT="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo "")"

if [[ "${CURRENT_ROOT}" == "${DOCKER_NEW_ROOT}" ]]; then
    skip "Docker already using ${DOCKER_NEW_ROOT}"
else
    echo ""
    echo "  Docker data will be migrated:"
    printf "    From: %s  (SD card)\n" "${DOCKER_OLD_ROOT}"
    printf "    To:   %s  (NVMe)\n"   "${DOCKER_NEW_ROOT}"
    echo ""
    read -r -p "  Proceed? [y/N] " _m
    [[ "${_m,,}" == "y" ]] || { echo "Migration skipped."; exit 0; }

    # 6a. Stop the WordPress stack cleanly
    log "  6a. Stopping WordPress stack..."
    docker compose down --timeout 30 2>&1 | sed 's/^/    /' || true
    ok "Stack stopped"

    # 6b. Stop the Docker daemon so no files are modified during copy
    log "  6b. Stopping Docker daemon..."
    sudo systemctl stop docker
    sudo systemctl stop docker.socket 2>/dev/null || true
    ok "Docker daemon stopped"

    # 6c. Create destination directory
    sudo mkdir -p "${DOCKER_NEW_ROOT}"

    # 6d. Copy all Docker data to NVMe
    # -a preserves permissions, links, and timestamps
    # -H preserves hard links (critical for Docker overlay2 layer sharing)
    # -A preserves ACLs, -X preserves extended attributes
    log "  6d. Copying Docker data to NVMe..."
    info "Source : ${DOCKER_OLD_ROOT}"
    info "Dest   : ${DOCKER_NEW_ROOT}"
    sudo rsync -aHAX --info=progress2 \
        "${DOCKER_OLD_ROOT}/" "${DOCKER_NEW_ROOT}/"

    SRC_SIZE="$(sudo du -sh "${DOCKER_OLD_ROOT}"  2>/dev/null | cut -f1)"
    DST_SIZE="$(sudo du -sh "${DOCKER_NEW_ROOT}"  2>/dev/null | cut -f1)"
    ok "Copy complete -- source: ${SRC_SIZE}  destination: ${DST_SIZE}"

    # 6e. Update daemon.json with the new data-root path
    # Merges into existing JSON to preserve log rotation settings
    log "  6e. Updating /etc/docker/daemon.json..."
    if [[ -f /etc/docker/daemon.json ]]; then
        sudo python3 - <<PYEOF
import json
path = '/etc/docker/daemon.json'
with open(path) as f:
    cfg = json.load(f)
cfg['data-root'] = '${DOCKER_NEW_ROOT}'
with open(path, 'w') as f:
    json.dump(cfg, f, indent=2)
    f.write('\n')
PYEOF
    else
        sudo tee /etc/docker/daemon.json > /dev/null <<EOF
{
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
  },
  "data-root": "${DOCKER_NEW_ROOT}"
}
EOF
    fi
    ok "daemon.json updated -- data-root: ${DOCKER_NEW_ROOT}"

    # 6f. Rename the old SD directory as a safety net (not yet deleted)
    log "  6f. Renaming SD card Docker dir to .bak-sd..."
    sudo mv "${DOCKER_OLD_ROOT}" "${DOCKER_BACKUP}"
    ok "${DOCKER_BACKUP} created"

    # 6g. Restart Docker from the new NVMe path
    log "  6g. Starting Docker daemon (NVMe path)..."
    sudo systemctl start docker
    sleep 5

    # Verify Docker reports the correct root before proceeding
    NEW_ROOT="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo "unknown")"
    if [[ "${NEW_ROOT}" != "${DOCKER_NEW_ROOT}" ]]; then
        echo "  FAIL: Docker root mismatch, rolling back..."
        sudo systemctl stop docker
        sudo mv "${DOCKER_BACKUP}" "${DOCKER_OLD_ROOT}"
        sudo python3 - <<PYEOF
import json
path = '/etc/docker/daemon.json'
with open(path) as f:
    cfg = json.load(f)
cfg.pop('data-root', None)
with open(path, 'w') as f:
    json.dump(cfg, f, indent=2)
    f.write('\n')
PYEOF
        sudo systemctl start docker
        fail "Migration failed and was rolled back. Check: sudo journalctl -u docker --since today"
    fi
    ok "Docker root confirmed: ${NEW_ROOT}"

    # 6h. Bring up the stack and wait for a healthy response
    log "  6h. Starting WordPress stack..."
    docker compose up -d

    info "Waiting for Nginx health check (up to 120s)..."
    MAX_WAIT=120; ELAPSED=0; HEALTHY=false
    until docker compose exec -T nginx wget -qO- http://localhost/nginx-health &>/dev/null 2>&1; do
        if [[ ${ELAPSED} -ge ${MAX_WAIT} ]]; then
            echo "  WARNING: stack not healthy within ${MAX_WAIT}s. Check: docker compose logs"
            break
        fi
        printf "  Waiting... %ds\r" "${ELAPSED}"
        sleep 5; ELAPSED=$((ELAPSED + 5))
    done

    if docker compose exec -T nginx wget -qO- http://localhost/nginx-health &>/dev/null 2>&1; then
        HEALTHY=true
        ok "Stack is healthy"
    fi

    # 6i. Remove SD card backup only after the user confirms the stack is healthy
    echo ""
    if [[ "${HEALTHY}" == "true" ]]; then
        echo "  Stack verified healthy on NVMe."
        echo "  SD backup: ${DOCKER_BACKUP}  ($(sudo du -sh "${DOCKER_BACKUP}" 2>/dev/null | cut -f1))"
        echo ""
        read -r -p "  Delete the SD card backup now? [y/N] " _del
        if [[ "${_del,,}" == "y" ]]; then
            sudo rm -rf "${DOCKER_BACKUP}"
            ok "SD card backup removed"
        else
            echo "  Kept. Remove manually when satisfied:"
            echo "    sudo rm -rf ${DOCKER_BACKUP}"
        fi
    else
        echo "  SD backup retained at: ${DOCKER_BACKUP}"
        echo "  Remove manually once confirmed healthy:"
        echo "    sudo rm -rf ${DOCKER_BACKUP}"
    fi
fi

# ----------------------------------------------------------------
# STEP 7 -- Summary
# ----------------------------------------------------------------
log "NVMe setup complete!"

DOCKER_ROOT="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo 'unknown')"
SCHED="$(grep -o '\[.*\]' "/sys/block/${NVME_BLK}/queue/scheduler" 2>/dev/null | tr -d '[]' || echo '?')"

echo ""
echo "NVMe status"
echo ""
df -h "${NVME_MOUNT}" | awk 'NR==2 {
    printf "  Mount     : %s\n  Size      : %s total  |  %s used  |  %s free\n", $6, $2, $3, $4
}'
echo "  Docker    : ${DOCKER_ROOT}"
echo "  Scheduler : ${SCHED}  (optimal for NVMe)"
echo ""
echo "  Container status:"
docker compose ps --format "table {{.Name}}\t{{.Status}}" 2>/dev/null | sed 's/^/    /'
echo ""
echo "  Verify site:"
echo "    curl -sk -o /dev/null -w '%{http_code}' https://andrewbaker.ninja/"
echo ""
if [[ -d "${DOCKER_BACKUP}" ]]; then
    echo "  WARNING: SD backup still present: ${DOCKER_BACKUP}"
    echo "  Remove when confident:  sudo rm -rf ${DOCKER_BACKUP}"
    echo ""
fi
SCRIPT
chmod +x /home/pi/setup-nvme.sh

# Before (conservative -- written for "SSD/NVMe" generically)
innodb_io_capacity     = 2000
innodb_io_capacity_max = 4000

# After (confirmed NVMe on Pi 5 HAT+)
innodb_io_capacity     = 4000
innodb_io_capacity_max = 8000

cd ~/andrewbakerninja-pi
bash scripts/setup-nvme.sh

bash bootstrap.sh
# Step 9e/9 -- NVMe (optional)
# NVMe drive detected at /dev/nvme0n1.
# Running setup-nvme.sh to mount and migrate Docker data...

cat > /home/pi/verify-nvme.sh <<'EOF'
#!/usr/bin/env bash
# verify-nvme.sh -- confirm NVMe migration is healthy
set -euo pipefail

echo ""
echo "Docker root:"
docker info | grep "Docker Root Dir"
# Expected: Docker Root Dir: /mnt/nvme/docker

echo ""
echo "Container status:"
docker compose ps

echo ""
echo "NVMe mount:"
df -h /mnt/nvme

echo ""
echo "I/O scheduler:"
cat /sys/block/nvme0n1/queue/scheduler
# Expected: [none]

echo ""
echo "Site health:"
HTTP_CODE="$(curl -sk -o /dev/null -w '%{http_code}' https://andrewbaker.ninja/)"
echo "HTTP ${HTTP_CODE}"
[[ "${HTTP_CODE}" == "200" ]] && echo "Site responding" || echo "WARNING: unexpected status"
echo ""
EOF
chmod +x /home/pi/verify-nvme.sh