bug fixing and deploy and build diagnostics

2026-03-22 03:30:21 +00:00
parent 01942cea95
commit 13e4a738be
198 changed files with 21703 additions and 19587 deletions
--- a/scripts/deploy-tailscale.sh
+++ b/scripts/deploy-tailscale.sh
@@ -38,7 +38,7 @@ BUILD_DIR="/home/archipelago/archy"
 # Node registry
 TAILSCALE_NODES=(
    "archipelago@${TAILSCALE_ARCH1:-100.82.97.63}"
-    "archipelago@archipelago-2.tail2b6225.ts.net"
+    "archipelago@${TAILSCALE_ARCH2:-100.122.84.60}"
    "archipelago@${TAILSCALE_ARCH3:-100.124.105.113}"
 )
 TAILSCALE_NAMES=("Arch 1" "Arch 2" "Arch 3")
@@ -107,18 +107,27 @@ deploy_node() {
        echo "   Copy-only (cargo=$HAS_CARGO, npm=$HAS_NPM) — will copy from $BUILD_SOURCE"
    fi

-    # ── Step 4: Rootful→rootless migration ───────────────────────────
+    # ── Step 4: Rootful→rootless migration (one-time) ────────────────
    step "Checking for rootful containers (migration)"
    ssh $SSH_OPTS "$TARGET" '
-        # Count rootful containers (run as sudo = rootful podman)
-        ROOTFUL=$(podman ps -a --format "{{.Names}}" 2>/dev/null | grep -v "^$" | wc -l)
-        ROOTLESS=$(podman ps -a --format "{{.Names}}" 2>/dev/null | grep -v "^$" | wc -l)
-        echo "   Rootful: $ROOTFUL containers, Rootless: $ROOTLESS containers"
-        if [ "$ROOTFUL" -gt 0 ]; then
-            echo "   MIGRATING: Stopping $ROOTFUL rootful containers..."
-            podman stop --all --timeout 30 2>/dev/null || true
-            podman rm --all --force 2>/dev/null || true
-            echo "   Rootful containers removed (data preserved in /var/lib/archipelago/)"
+        MIGRATION_FLAG="/var/lib/archipelago/.rootless-migrated"
+        if [ -f "$MIGRATION_FLAG" ]; then
+            ROOTLESS=$(podman ps -a --format "{{.Names}}" 2>/dev/null | grep -v "^$" | wc -l)
+            echo "   Already migrated ($ROOTLESS rootless containers)"
+        else
+            # Check if rootful podman has any containers (sudo = rootful context)
+            ROOTFUL=$(sudo podman ps -a --format "{{.Names}}" 2>/dev/null | grep -v "^$" | wc -l)
+            ROOTLESS=$(podman ps -a --format "{{.Names}}" 2>/dev/null | grep -v "^$" | wc -l)
+            echo "   Rootful: $ROOTFUL, Rootless: $ROOTLESS"
+            if [ "$ROOTFUL" -gt 0 ] && [ "$ROOTFUL" != "$ROOTLESS" ]; then
+                echo "   MIGRATING: Stopping $ROOTFUL rootful containers..."
+                sudo podman stop --all --timeout 30 2>/dev/null || true
+                sudo podman rm --all --force 2>/dev/null || true
+                echo "   Rootful containers removed (data preserved in /var/lib/archipelago/)"
+            else
+                echo "   No rootful containers to migrate"
+            fi
+            sudo touch "$MIGRATION_FLAG"
        fi
    ' 2>&1

@@ -166,12 +175,17 @@ deploy_node() {
        # Transfer custom UI images (individual tarballs — never combined)
        echo "   Transferring custom UI images..."
        for ui_img in bitcoin-ui lnd-ui electrs-ui; do
-            HAS_IMG=$(ssh $SSH_OPTS "$BUILD_SOURCE" "podman images --format '{{.Repository}}' 2>/dev/null | grep -q '$ui_img' && echo yes || echo no" 2>/dev/null)
+            HAS_IMG=$(ssh $SSH_OPTS "$BUILD_SOURCE" "podman images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -q '${ui_img}:' && echo yes || echo no" 2>/dev/null)
            if [ "$HAS_IMG" = "yes" ]; then
                echo "     $ui_img..."
-                ssh $SSH_OPTS "$BUILD_SOURCE" "podman save 'localhost/${ui_img}:local'" > "/tmp/${ui_img}.tar"
-                ssh $SSH_OPTS "$TARGET" "podman load" < "/tmp/${ui_img}.tar" 2>&1 | tail -1
+                if ssh $SSH_OPTS "$BUILD_SOURCE" "podman save 'localhost/${ui_img}:local' 2>/dev/null" > "/tmp/${ui_img}.tar" 2>/dev/null && [ -s "/tmp/${ui_img}.tar" ]; then
+                    ssh $SSH_OPTS "$TARGET" "podman load" < "/tmp/${ui_img}.tar" 2>&1 | tail -1
+                else
+                    echo "     $ui_img: not available on build server, skipping"
+                fi
                rm -f "/tmp/${ui_img}.tar"
+            else
+                echo "     $ui_img: not found on build server, skipping"
            fi
        done

@@ -221,7 +235,7 @@ deploy_node() {
    AIUI_DIST="$PROJECT_DIR/../AIUI/packages/app/dist"
    if [ -d "$AIUI_DIST" ] && [ -f "$AIUI_DIST/index.html" ]; then
        ssh $SSH_OPTS "$TARGET" "sudo mkdir -p /opt/archipelago/web-ui/aiui && sudo rm -rf /opt/archipelago/web-ui/aiui/*"
-        (cd "$AIUI_DIST" && tar cf - .) | ssh $SSH_OPTS "$TARGET" "sudo tar xf - -C /opt/archipelago/web-ui/aiui/"
+        (cd "$AIUI_DIST" && tar --no-xattrs -cf - .) | ssh $SSH_OPTS "$TARGET" "sudo tar xf - -C /opt/archipelago/web-ui/aiui/ 2>/dev/null"
        ssh $SSH_OPTS "$TARGET" "sudo chown -R 1000:1000 /opt/archipelago/web-ui/aiui"
        echo "   AIUI deployed."
    else
@@ -304,20 +318,23 @@ deploy_node() {
        sudo mkdir -p /var/lib/archipelago/dwn/messages /var/lib/archipelago/dwn/protocols
        sudo mkdir -p /var/lib/archipelago/content/files /var/lib/archipelago/federation
        sudo mkdir -p /var/lib/archipelago/identities /var/lib/archipelago/tor-config
+        sudo mkdir -p /var/lib/archipelago/searxng /var/lib/archipelago/vaultwarden
+        sudo mkdir -p /var/lib/archipelago/photoprism /var/lib/archipelago/filebrowser
+        sudo mkdir -p /var/lib/archipelago/nextcloud
        sudo chown -R archipelago:archipelago /var/lib/archipelago/dwn /var/lib/archipelago/content \
            /var/lib/archipelago/federation /var/lib/archipelago/identities /var/lib/archipelago/tor-config 2>/dev/null || true

        echo "   Fixing rootless podman UID mapping..."
        # Containers running as root (UID 0 → host UID 100000)
-        for dir in lnd electrumx btcpay nbxplorer immich jellyfin vaultwarden \
+        for dir in lnd electrumx btcpay nbxplorer jellyfin vaultwarden \
                   home-assistant fedimint fedimint-gateway photoprism ollama filebrowser \
-                   nextcloud uptime-kuma onlyoffice nginx-proxy-manager portainer nostr-rs-relay; do
+                   nextcloud uptime-kuma onlyoffice nginx-proxy-manager portainer nostr-rs-relay searxng; do
            [ -d "/var/lib/archipelago/$dir" ] && sudo chown -R 100000:100000 "/var/lib/archipelago/$dir" 2>/dev/null
        done
        # Bitcoin Knots: container UID 101 → host UID 100101
        [ -d /var/lib/archipelago/bitcoin ] && sudo chown -R 100101:100101 /var/lib/archipelago/bitcoin 2>/dev/null
        # Postgres: container UID 70 → host UID 100070
-        for dir in postgres-btcpay immich-db penpot-postgres; do
+        for dir in postgres-btcpay; do
            [ -d "/var/lib/archipelago/$dir" ] && sudo chown -R 100070:100070 "/var/lib/archipelago/$dir" 2>/dev/null
        done
        # MariaDB: container UID 999 → host UID 100999
@@ -413,7 +430,7 @@ deploy_node() {

    DB_PASSWORDS=$(ssh $SSH_OPTS "$TARGET" '
        SECRETS_DIR="/var/lib/archipelago/secrets"
-        for svc in mempool btcpay immich penpot mysql-root; do
+        for svc in mempool btcpay mysql-root; do
            if [ ! -f "$SECRETS_DIR/${svc}-db-password" ]; then
                openssl rand -base64 24 | sudo tee "$SECRETS_DIR/${svc}-db-password" > /dev/null
                sudo chmod 600 "$SECRETS_DIR/${svc}-db-password"
@@ -421,8 +438,6 @@ deploy_node() {
        done
        echo "MEMPOOL_DB_PASS=$(sudo cat "$SECRETS_DIR/mempool-db-password")"
        echo "BTCPAY_DB_PASS=$(sudo cat "$SECRETS_DIR/btcpay-db-password")"
-        echo "IMMICH_DB_PASS=$(sudo cat "$SECRETS_DIR/immich-db-password")"
-        echo "PENPOT_DB_PASS=$(sudo cat "$SECRETS_DIR/penpot-db-password")"
        echo "MYSQL_ROOT_PASS=$(sudo cat "$SECRETS_DIR/mysql-root-db-password")"
        # Fedimint gateway
        if [ ! -f "$SECRETS_DIR/fedimint-gateway-password" ]; then
@@ -485,7 +500,7 @@ deploy_node() {
                    --security-opt no-new-privileges:true \
                    -p 8332:8332 -p 8333:8333 \
                    -v /var/lib/archipelago/bitcoin:/home/bitcoin/.bitcoin \
-                    ${BITCOIN_KNOTS_IMAGE:-docker.io/bitcoinknots/bitcoin:v28.1} \
+                    $BITCOIN_KNOTS_IMAGE \
                    -server=1 \$BTC_EXTRA_ARGS \
                    -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0:8332 \
                    -rpcuser=$BITCOIN_RPC_USER -rpcpassword=$BITCOIN_RPC_PASS \
@@ -503,7 +518,7 @@ deploy_node() {
                    -v /var/lib/archipelago/mysql-mempool:/var/lib/mysql \
                    -e MYSQL_DATABASE=mempool -e MYSQL_USER=mempool \
                    -e MYSQL_PASSWORD=$MEMPOOL_DB_PASS -e MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASS \
-                    docker.io/mariadb:10.11
+                    $MARIADB_IMAGE
                sleep 3
            fi
            MYSQL_CNT=\$(\$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -E 'mysql-mempool|archy-mempool-db' | head -1)
@@ -521,7 +536,7 @@ deploy_node() {
                        -e DAEMON_URL=http://$BITCOIN_RPC_USER:$BITCOIN_RPC_PASS@bitcoin-knots:8332/ \
                        -e COIN=Bitcoin -e DB_DIRECTORY=/data \
                        -e SERVICES=tcp://:50001,rpc://0.0.0.0:8000 \
-                        docker.io/lukechilds/electrumx:v1.18.0
+                        $ELECTRUMX_IMAGE
                fi
            fi

@@ -535,24 +550,32 @@ deploy_node() {
                    -e CORE_RPC_USERNAME=archipelago -e CORE_RPC_PASSWORD=$BITCOIN_RPC_PASS \
                    -e DATABASE_ENABLED=true -e DATABASE_HOST=\$MYSQL_CNT -e DATABASE_DATABASE=mempool \
                    -e DATABASE_USERNAME=mempool -e DATABASE_PASSWORD=$MEMPOOL_DB_PASS \
-                    docker.io/mempool/backend:v2.5.0
+                    $MEMPOOL_BACKEND_IMAGE
            fi

            if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q archy-mempool-web; then
                echo '   Creating mempool frontend...'
                \$DOCKER run -d --name archy-mempool-web --restart unless-stopped \$NET_OPT \
                    -p 4080:8080 -e FRONTEND_HTTP_PORT=8080 -e BACKEND_MAINNET_HTTP_HOST=mempool-api \
-                    docker.io/mempool/frontend:v2.5.0
+                    $MEMPOOL_WEB_IMAGE
            fi

            echo '   === BTCPay Stack ==='
+            # Recreate btcpay-db if postgres version mismatch (15→16 incompatible)
+            if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then
+                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then
+                    echo '   Recreating archy-btcpay-db (was stopped/broken)...'
+                    \$DOCKER rm -f archy-btcpay-db 2>/dev/null
+                    \$DOCKER rm -f postgres-btcpay 2>/dev/null
+                fi
+            fi
            if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qE 'archy-btcpay-db|postgres-btcpay'; then
                echo '   Creating archy-btcpay-db...'
                sudo mkdir -p /var/lib/archipelago/postgres-btcpay
                \$DOCKER run -d --name archy-btcpay-db --restart unless-stopped \$NET_OPT \
                    -v /var/lib/archipelago/postgres-btcpay:/var/lib/postgresql/data \
                    -e POSTGRES_DB=btcpay -e POSTGRES_USER=btcpay -e POSTGRES_PASSWORD=$BTCPAY_DB_PASS \
-                    docker.io/postgres:15-alpine
+                    $BTCPAY_POSTGRES_IMAGE
                sleep 3
            fi
            \$DOCKER exec archy-btcpay-db psql -U postgres -tc \"SELECT 1 FROM pg_database WHERE datname='nbxplorer'\" 2>/dev/null | grep -q 1 || \
@@ -570,7 +593,7 @@ deploy_node() {
                        -e NBXPLORER_BIND=0.0.0.0:32838 -e NBXPLORER_BTCRPCURL=http://bitcoin-knots:8332 \
                        -e NBXPLORER_BTCRPCUSER=archipelago -e NBXPLORER_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \
                        -e NBXPLORER_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=nbxplorer;Include Error Detail=true' \
-                        docker.io/nicolasdorier/nbxplorer:2.6.0
+                        $NBXPLORER_IMAGE
                    sleep 5
                fi
            fi
@@ -588,18 +611,20 @@ deploy_node() {
                    -e BTCPAY_BTCRPCURL=http://bitcoin-knots:8332 \
                    -e BTCPAY_BTCRPCUSER=archipelago -e BTCPAY_BTCRPCPASSWORD=$BITCOIN_RPC_PASS \
                    -e BTCPAY_POSTGRES='User ID=btcpay;Password=$BTCPAY_DB_PASS;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true' \
-                    docker.io/btcpayserver/btcpayserver:1.13.5
+                    $BTCPAY_IMAGE
            fi

            echo '   === LND ==='
+            # Always update LND config with current RPC credentials
+            sudo mkdir -p /var/lib/archipelago/lnd
            if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx lnd; then
                if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx lnd; then
                    \$DOCKER start lnd 2>/dev/null || true
                else
                    echo '   Creating LND...'
-                    sudo mkdir -p /var/lib/archipelago/lnd
-                    if [ ! -f /var/lib/archipelago/lnd/lnd.conf ]; then
-                        cat > /tmp/lnd.conf <<'LNDCONF'
+                    # Always write/update lnd.conf with current RPC credentials
+                    RPC_PASS=\$(sudo cat /var/lib/archipelago/secrets/bitcoin-rpc-password 2>/dev/null)
+                    cat > /tmp/lnd.conf <<LNDCONF
 [Application Options]
 listen=0.0.0.0:9735
 rpclisten=0.0.0.0:10009
@@ -615,27 +640,36 @@ bitcoin.node=bitcoind
 [Bitcoind]
 bitcoind.rpchost=bitcoin-knots:8332
 bitcoind.rpcuser=archipelago
-bitcoind.rpcpass=$BITCOIN_RPC_PASS
+bitcoind.rpcpass=\$RPC_PASS
 bitcoind.rpcpolling=true
 bitcoind.estimatemode=ECONOMICAL

 [autopilot]
 autopilot.active=false
 LNDCONF
-                        sudo cp /tmp/lnd.conf /var/lib/archipelago/lnd/lnd.conf
-                        rm -f /tmp/lnd.conf
-                    fi
+                    sudo cp /tmp/lnd.conf /var/lib/archipelago/lnd/lnd.conf
+                    sudo chown 100000:100000 /var/lib/archipelago/lnd/lnd.conf 2>/dev/null
+                    rm -f /tmp/lnd.conf
                    \$DOCKER run -d --name lnd --restart unless-stopped --network archy-net \
                        --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
                        --security-opt no-new-privileges:true \
                        -p 9735:9735 -p 10009:10009 -p 8080:8080 \
                        -v /var/lib/archipelago/lnd:/root/.lnd \
-                        docker.io/lightninglabs/lnd:v0.18.4-beta
+                        $LND_IMAGE
                fi
            fi

            echo '   === Fedimint ==='
-            if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx fedimint; then
+            # Recreate fedimint if it exists but is broken (wrong env vars)
+            if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx fedimint; then
+                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx fedimint; then
+                    echo '   Recreating fedimint (was stopped/broken)...'
+                    \$DOCKER rm -f fedimint 2>/dev/null
+                else
+                    echo '   Fedimint already running'
+                fi
+            fi
+            if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx fedimint; then
                echo '   Creating Fedimint...'
                sudo mkdir -p /var/lib/archipelago/fedimint
                \$DOCKER run -d --name fedimint --restart unless-stopped \$NET_OPT \
@@ -648,12 +682,21 @@ LNDCONF
                    -e FM_BIND_API=0.0.0.0:8174 -e FM_BIND_UI=0.0.0.0:8175 \
                    -e FM_P2P_URL=fedimint://\$TARGET_IP:8173 -e FM_API_URL=ws://\$TARGET_IP:8174 \
                    -e FM_BITCOIND_URL=http://\$TARGET_IP:8332 \
-                    docker.io/fedimint/fedimintd:v0.10.0
+                    -e FM_REQ_RELEASE_NOTES_ACK_V0_4=true \
+                    $FEDIMINT_IMAGE
            fi

-            if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q fedimint-gateway; then
+            # Recreate fedimint-gateway if broken
+            if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q fedimint-gateway; then
+                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q fedimint-gateway; then
+                    echo '   Recreating fedimint-gateway (was stopped/broken)...'
+                    \$DOCKER rm -f fedimint-gateway 2>/dev/null
+                fi
+            fi
+            if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q fedimint-gateway; then
                echo '   Creating fedimint-gateway...'
                sudo mkdir -p /var/lib/archipelago/fedimint-gateway
+                FEDI_PASS=\$(sudo cat /var/lib/archipelago/secrets/fedimint-gateway-password 2>/dev/null || echo 'archipelago')
                LND_CERT=/var/lib/archipelago/lnd/tls.cert
                LND_MACAROON=/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon
                if \$DOCKER ps --format '{{.Names}}' | grep -q '^lnd\$' && sudo test -f \$LND_CERT && sudo test -f \$LND_MACAROON; then
@@ -663,9 +706,9 @@ LNDCONF
                        -p 8176:8176 -v /var/lib/archipelago/fedimint-gateway:/data \
                        -v /var/lib/archipelago/lnd/tls.cert:/lnd/tls.cert:ro \
                        -v /var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon:/lnd/admin.macaroon:ro \
-                        docker.io/fedimint/gatewayd:v0.10.0 \
+                        $FEDIMINT_GATEWAY_IMAGE \
                        gatewayd --data-dir /data --listen 0.0.0.0:8176 \
-                        --bcrypt-password-hash '$FEDI_HASH' \
+                        --password \"\$FEDI_PASS\" \
                        --network bitcoin --bitcoind-url http://\$TARGET_IP:8332 \
                        --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \
                        lnd --lnd-rpc-host \$TARGET_IP:10009 --lnd-tls-cert /lnd/tls.cert --lnd-macaroon /lnd/admin.macaroon
@@ -674,42 +717,15 @@ LNDCONF
                        --cap-drop ALL --cap-add CHOWN --cap-add FOWNER --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
                        --security-opt no-new-privileges:true \
                        -p 8176:8176 -p 9737:9737 -v /var/lib/archipelago/fedimint-gateway:/data \
-                        docker.io/fedimint/gatewayd:v0.10.0 \
+                        $FEDIMINT_GATEWAY_IMAGE \
                        gatewayd --data-dir /data --listen 0.0.0.0:8176 \
-                        --bcrypt-password-hash '$FEDI_HASH' \
+                        --password \"\$FEDI_PASS\" \
                        --network bitcoin --bitcoind-url http://\$TARGET_IP:8332 \
                        --bitcoind-username $BITCOIN_RPC_USER --bitcoind-password $BITCOIN_RPC_PASS \
                        ldk --ldk-lightning-port 9737 --ldk-alias archipelago-gateway
                fi
            fi

-            echo '   === Immich ==='
-            if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
-                echo '   Creating Immich stack...'
-                sudo mkdir -p /var/lib/archipelago/immich /var/lib/archipelago/immich-db
-                \$DOCKER network create immich-net 2>/dev/null || true
-                if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q immich_postgres; then
-                    \$DOCKER run -d --name immich_postgres --restart unless-stopped --network immich-net \
-                        -v /var/lib/archipelago/immich-db:/var/lib/postgresql/data \
-                        -e POSTGRES_PASSWORD=$IMMICH_DB_PASS -e POSTGRES_USER=postgres -e POSTGRES_DB=immich \
-                        ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0 2>/dev/null || true
-                    sleep 5
-                fi
-                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_redis; then
-                    \$DOCKER run -d --name immich_redis --restart unless-stopped --network immich-net \
-                        docker.io/valkey/valkey:7-alpine 2>/dev/null || true
-                    sleep 2
-                fi
-                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
-                    \$DOCKER run -d --name immich_server --restart unless-stopped --network immich-net \
-                        -p 2283:2283 -v /var/lib/archipelago/immich:/usr/src/app/upload \
-                        -e DB_HOSTNAME=immich_postgres -e DB_USERNAME=postgres -e DB_PASSWORD=$IMMICH_DB_PASS \
-                        -e DB_DATABASE_NAME=immich -e REDIS_HOSTNAME=immich_redis \
-                        -e UPLOAD_LOCATION=/usr/src/app/upload \
-                        ghcr.io/immich-app/immich-server:release 2>/dev/null || true
-                fi
-            fi
-
            echo '   === Simple apps ==='
            # Home Assistant
            if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx homeassistant; then
@@ -721,7 +737,7 @@ LNDCONF
                        --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
                        --security-opt no-new-privileges:true \
                        -p 8123:8123 -v /var/lib/archipelago/home-assistant:/config -e TZ=UTC \
-                        docker.io/homeassistant/home-assistant:2024.1
+                        $HOMEASSISTANT_IMAGE
                fi
            fi
            # Grafana
@@ -741,7 +757,7 @@ LNDCONF
                        --user 0:0 \
                        -p 3000:3000 -v /var/lib/archipelago/grafana:/var/lib/grafana \
                        -e GF_PATHS_DATA=/var/lib/grafana -e GF_USERS_ALLOW_SIGN_UP=false \
-                        docker.io/grafana/grafana:10.2.0
+                        $GRAFANA_IMAGE
                fi
            fi
            # Jellyfin
@@ -755,68 +771,84 @@ LNDCONF
                        -p 8096:8096 \
                        -v /var/lib/archipelago/jellyfin/config:/config \
                        -v /var/lib/archipelago/jellyfin/cache:/cache \
-                        docker.io/jellyfin/jellyfin:10.8.13
+                        $JELLYFIN_IMAGE
                fi
            fi
-            # Vaultwarden
-            if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx vaultwarden; then
-                if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx vaultwarden; then
-                    \$DOCKER start vaultwarden 2>/dev/null || true
-                else
-                    sudo mkdir -p /var/lib/archipelago/vaultwarden
-                    \$DOCKER run -d --name vaultwarden --restart unless-stopped \
-                        --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add NET_BIND_SERVICE \
-                        --security-opt no-new-privileges:true \
-                        -p 8082:80 -v /var/lib/archipelago/vaultwarden:/data \
-                        docker.io/vaultwarden/server:1.30.0-alpine
+            # Vaultwarden — recreate if broken (permissions/DB)
+            if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx vaultwarden; then
+                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx vaultwarden; then
+                    \$DOCKER rm -f vaultwarden 2>/dev/null
                fi
            fi
-            # SearXNG
-            if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx searxng; then
-                if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx searxng; then
-                    \$DOCKER start searxng 2>/dev/null || true
-                else
-                    \$DOCKER run -d --name searxng --restart unless-stopped \
-                        --cap-drop ALL --security-opt no-new-privileges:true \
-                        -p 8888:8080 ${SEARXNG_IMAGE:-docker.io/searxng/searxng:2024.11.17}
+            if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx vaultwarden; then
+                sudo mkdir -p /var/lib/archipelago/vaultwarden
+                sudo chown -R 100000:100000 /var/lib/archipelago/vaultwarden 2>/dev/null
+                \$DOCKER run -d --name vaultwarden --restart unless-stopped \
+                    --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add NET_BIND_SERVICE \
+                    --security-opt no-new-privileges:true \
+                    -p 8082:80 -v /var/lib/archipelago/vaultwarden:/data \
+                    $VAULTWARDEN_IMAGE
+            fi
+            # SearXNG — recreate if broken (permission denied on settings.yml)
+            if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx searxng; then
+                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx searxng; then
+                    \$DOCKER rm -f searxng 2>/dev/null
                fi
            fi
-            # FileBrowser
-            if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx filebrowser; then
+            if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx searxng; then
+                sudo mkdir -p /var/lib/archipelago/searxng
+                \$DOCKER run -d --name searxng --restart unless-stopped \
+                    --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
+                    --security-opt no-new-privileges:true \
+                    -v /var/lib/archipelago/searxng:/etc/searxng \
+                    -p 8888:8080 $SEARXNG_IMAGE
+            fi
+            # FileBrowser — recreate if broken (permission denied on :80)
+            if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx filebrowser; then
+                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx filebrowser; then
+                    \$DOCKER rm -f filebrowser 2>/dev/null
+                fi
+            fi
+            if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx filebrowser; then
                sudo mkdir -p /var/lib/archipelago/filebrowser
                \$DOCKER run -d --name filebrowser --restart=always \
+                    --cap-add NET_BIND_SERVICE \
                    -p 8083:80 -v /var/lib/archipelago/filebrowser:/srv \
-                    docker.io/filebrowser/filebrowser:v2.27.0
+                    $FILEBROWSER_IMAGE
            fi

            echo '   === Additional apps ==='
-            # Nextcloud
-            if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx nextcloud; then
-                if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx nextcloud; then
-                    \$DOCKER start nextcloud 2>/dev/null || true
-                else
-                    sudo mkdir -p /var/lib/archipelago/nextcloud
-                    \$DOCKER run -d --name nextcloud --restart unless-stopped \
-                        --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
-                        --security-opt no-new-privileges:true \
-                        -p 8085:80 -v /var/lib/archipelago/nextcloud:/var/www/html \
-                        docker.io/library/nextcloud:28
+            # Nextcloud — recreate if wrong image version (28→30 not supported, need 29)
+            if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx nextcloud; then
+                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx nextcloud; then
+                    echo '   Recreating nextcloud (was stopped/broken)...'
+                    \$DOCKER rm -f nextcloud 2>/dev/null
                fi
            fi
-            # PhotoPrism
-            if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx photoprism; then
-                if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx photoprism; then
-                    \$DOCKER start photoprism 2>/dev/null || true
-                else
-                    sudo mkdir -p /var/lib/archipelago/photoprism
-                    \$DOCKER run -d --name photoprism --restart unless-stopped \
-                        --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
-                        --security-opt no-new-privileges:true \
-                        -p 2342:2342 -v /var/lib/archipelago/photoprism:/photoprism/storage \
-                        -e PHOTOPRISM_ADMIN_PASSWORD=archipelago -e PHOTOPRISM_DEFAULT_LOCALE=en \
-                        ${PHOTOPRISM_IMAGE:-docker.io/photoprism/photoprism:240915}
+            if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx nextcloud; then
+                sudo mkdir -p /var/lib/archipelago/nextcloud
+                \$DOCKER run -d --name nextcloud --restart unless-stopped \
+                    --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
+                    --security-opt no-new-privileges:true \
+                    -p 8085:80 -v /var/lib/archipelago/nextcloud:/var/www/html \
+                    $NEXTCLOUD_IMAGE
+            fi
+            # PhotoPrism — recreate if broken (permissions)
+            if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx photoprism; then
+                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx photoprism; then
+                    \$DOCKER rm -f photoprism 2>/dev/null
                fi
            fi
+            if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx photoprism; then
+                sudo mkdir -p /var/lib/archipelago/photoprism
+                sudo chown -R 100000:100000 /var/lib/archipelago/photoprism 2>/dev/null
+                \$DOCKER run -d --name photoprism --restart unless-stopped \
+                    --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \
+                    --security-opt no-new-privileges:true \
+                    -p 2342:2342 -v /var/lib/archipelago/photoprism:/photoprism/storage \
+                    -e PHOTOPRISM_ADMIN_PASSWORD=archipelago -e PHOTOPRISM_DEFAULT_LOCALE=en \
+                    $PHOTOPRISM_IMAGE
+            fi
            # OnlyOffice
            if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx onlyoffice; then
                if \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx onlyoffice; then
@@ -825,7 +857,7 @@ LNDCONF
                    \$DOCKER run -d --name onlyoffice --restart unless-stopped \
                        --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add DAC_OVERRIDE \
                        --security-opt no-new-privileges:true \
-                        -p 9980:80 docker.io/onlyoffice/documentserver:7.5.1
+                        -p 9980:80 $ONLYOFFICE_IMAGE
                fi
            fi
            # Nginx Proxy Manager
@@ -840,7 +872,7 @@ LNDCONF
                        -p 81:81 -p 8084:80 -p 8443:443 \
                        -v /var/lib/archipelago/nginx-proxy-manager/data:/data \
                        -v /var/lib/archipelago/nginx-proxy-manager/letsencrypt:/etc/letsencrypt \
-                        ${NPM_IMAGE:-docker.io/jc21/nginx-proxy-manager:2}
+                        $NPM_IMAGE
                fi
            fi
            # Portainer
@@ -854,58 +886,9 @@ LNDCONF
                        --security-opt no-new-privileges:true \
                        -p 9000:9000 -v /var/lib/archipelago/portainer:/data \
                        -v /run/user/1000/podman/podman.sock:/var/run/docker.sock \
-                        docker.io/portainer/portainer-ce:2.19.4
+                        $PORTAINER_IMAGE
                fi
            fi
-            # Penpot stack (postgres + valkey + backend + exporter + frontend)
-            if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-frontend; then
-                echo '   Creating Penpot stack...'
-                sudo mkdir -p /var/lib/archipelago/penpot-assets /var/lib/archipelago/penpot-postgres
-                \$DOCKER network create penpot-net 2>/dev/null || true
-                if ! \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q penpot-postgres; then
-                    \$DOCKER run -d --name penpot-postgres --restart unless-stopped --network penpot-net \
-                        -v /var/lib/archipelago/penpot-postgres:/var/lib/postgresql/data \
-                        -e POSTGRES_DB=penpot -e POSTGRES_USER=penpot -e POSTGRES_PASSWORD=$PENPOT_DB_PASS \
-                        docker.io/postgres:15
-                    sleep 5
-                fi
-                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-valkey; then
-                    \$DOCKER run -d --name penpot-valkey --restart unless-stopped --network penpot-net \
-                        -e VALKEY_EXTRA_FLAGS='--maxmemory 128mb --maxmemory-policy volatile-lfu' \
-                        docker.io/valkey/valkey:8.1
-                    sleep 3
-                fi
-                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-backend; then
-                    \$DOCKER run -d --name penpot-backend --restart unless-stopped --network penpot-net \
-                        -v /var/lib/archipelago/penpot-assets:/opt/data/assets \
-                        -e PENPOT_PUBLIC_URI=http://\${TARGET_IP}:9001 \
-                        -e PENPOT_SECRET_KEY=archipelago-penpot-secret-key-change-in-production \
-                        -e PENPOT_DATABASE_URI=postgresql://penpot-postgres/penpot \
-                        -e PENPOT_DATABASE_USERNAME=penpot -e PENPOT_DATABASE_PASSWORD=$PENPOT_DB_PASS \
-                        -e PENPOT_REDIS_URI=redis://penpot-valkey/0 \
-                        -e PENPOT_OBJECTS_STORAGE_BACKEND=fs \
-                        -e PENPOT_OBJECTS_STORAGE_FS_DIRECTORY=/opt/data/assets \
-                        -e PENPOT_FLAGS='disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies' \
-                        ${PENPOT_BACKEND_IMAGE:-docker.io/penpotapp/backend:2.4.2}
-                    sleep 5
-                fi
-                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-exporter; then
-                    \$DOCKER run -d --name penpot-exporter --restart unless-stopped --network penpot-net \
-                        -e PENPOT_SECRET_KEY=archipelago-penpot-secret-key-change-in-production \
-                        -e PENPOT_PUBLIC_URI=http://penpot-frontend:8080 \
-                        -e PENPOT_REDIS_URI=redis://penpot-valkey/0 \
-                        ${PENPOT_EXPORTER_IMAGE:-docker.io/penpotapp/exporter:2.4.2}
-                    sleep 2
-                fi
-                if ! \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-frontend; then
-                    \$DOCKER run -d --name penpot-frontend --restart unless-stopped --network penpot-net \
-                        -p 9001:8080 -v /var/lib/archipelago/penpot-assets:/opt/data/assets \
-                        -e PENPOT_PUBLIC_URI=http://\${TARGET_IP}:9001 \
-                        -e PENPOT_FLAGS='disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies' \
-                        ${PENPOT_FRONTEND_IMAGE:-docker.io/penpotapp/frontend:2.4.2}
-                fi
-            fi
-
            echo '   === Custom UI containers ==='
            # Build custom UI containers if source exists
            for ui in bitcoin-ui lnd-ui electrs-ui; do
@@ -955,13 +938,26 @@ LNDCONF
            echo \"   Total containers running: \$TOTAL\"
        " 2>&1 | sed 's/^/   /'

-        # ── Step 23: Tor ─────────────────────────────────────────────
+        # ── Step 23: Tor (robust setup) ──────────────────────────────
        step "Setting up Tor"
-        ssh $SSH_OPTS "$TARGET" "
+        ssh $SSH_OPTS "$TARGET" '
            sudo mkdir -p /var/lib/archipelago/tor
+
+            # Install Tor if missing
+            if ! command -v tor >/dev/null 2>&1; then
+                echo "   Installing Tor..."
+                sudo apt-get update -qq && sudo apt-get install -y -qq tor 2>/dev/null
+            fi
+
+            if ! command -v tor >/dev/null 2>&1; then
+                echo "   ERROR: Tor installation failed"
+                exit 0
+            fi
+
+            # Write services.json
            SERVICES_JSON=/var/lib/archipelago/tor/services.json
-            if [ ! -f \"\$SERVICES_JSON\" ]; then
-                sudo python3 -c '
+            if [ ! -f "$SERVICES_JSON" ]; then
+                sudo python3 -c "
 import json
 services = [
    {\"name\": \"archipelago\", \"local_port\": 80, \"enabled\": True},
@@ -974,21 +970,29 @@ services = [
 ]
 with open(\"/var/lib/archipelago/tor/services.json\", \"w\") as f:
    json.dump({\"services\": services}, f, indent=2)
-'
+"
            fi
-            if command -v tor >/dev/null 2>&1; then
-                sudo systemctl enable tor 2>/dev/null
-                sudo systemctl restart tor@default 2>/dev/null
-                echo '   System Tor running'
+
+            # Enable + start Tor service (try both unit names)
+            sudo systemctl enable tor 2>/dev/null || true
+            sudo systemctl enable tor@default 2>/dev/null || true
+
+            # Restart Tor — try tor@default first (Debian pattern), fallback to tor
+            if sudo systemctl restart tor@default 2>/dev/null; then
+                echo "   Tor running (tor@default)"
+            elif sudo systemctl restart tor 2>/dev/null; then
+                echo "   Tor running (tor)"
            else
-                sudo apt-get update -qq && sudo apt-get install -y -qq tor 2>/dev/null || true
-                if command -v tor >/dev/null 2>&1; then
-                    sudo systemctl enable tor 2>/dev/null
-                    sudo systemctl restart tor@default 2>/dev/null
-                    echo '   Tor installed and started'
-                fi
+                echo "   WARNING: Tor failed to start — check journalctl -u tor"
            fi
-        " 2>&1 | tail -5 | sed 's/^/   /' || true
+
+            # Verify Tor is actually running
+            if systemctl is-active tor@default >/dev/null 2>&1 || systemctl is-active tor >/dev/null 2>&1; then
+                echo "   Tor verified active"
+            else
+                echo "   WARNING: Tor not active after restart attempt"
+            fi
+        ' 2>&1 | sed 's/^/   /'
    fi

    # ── Step 24: UFW forward policy ──────────────────────────────────
@@ -1041,6 +1045,40 @@ with open(\"/var/lib/archipelago/tor/services.json\", \"w\") as f:
    step "Running container doctor"
    "$SCRIPT_DIR/container-doctor.sh" "$TARGET" 2>&1 | tail -10 | sed 's/^/   /' || true

+    # ── Step 26b: Restart stopped containers + verify health ──────
+    step "Verifying all containers running"
+    ssh $SSH_OPTS "$TARGET" '
+        DOCKER=podman; command -v podman >/dev/null 2>&1 || DOCKER=docker
+
+        # Fix permissions before restart attempts (rootless UID mapping)
+        for dir in vaultwarden photoprism nextcloud filebrowser searxng; do
+            [ -d "/var/lib/archipelago/$dir" ] && sudo chown -R 100000:100000 "/var/lib/archipelago/$dir" 2>/dev/null
+        done
+
+        # Restart any exited containers (unless user-stopped)
+        USER_STOPPED="/var/lib/archipelago/user-stopped.json"
+        for ctr in $($DOCKER ps -a --filter "status=exited" --format "{{.Names}}" 2>/dev/null); do
+            if [ -f "$USER_STOPPED" ] && grep -q "\"$ctr\"" "$USER_STOPPED" 2>/dev/null; then
+                continue
+            fi
+            echo "   Restarting exited container: $ctr"
+            $DOCKER start "$ctr" 2>/dev/null || echo "   WARNING: Failed to start $ctr"
+        done
+
+        # Summary
+        RUNNING=$($DOCKER ps --format "{{.Names}}" 2>/dev/null | wc -l)
+        EXITED=$($DOCKER ps -a --filter "status=exited" --format "{{.Names}}" 2>/dev/null | wc -l)
+        echo "   Containers: $RUNNING running, $EXITED exited"
+
+        # Verify Tor is still active
+        if systemctl is-active tor@default >/dev/null 2>&1 || systemctl is-active tor >/dev/null 2>&1; then
+            echo "   Tor: active"
+        else
+            echo "   Tor: NOT RUNNING — attempting restart..."
+            sudo systemctl restart tor@default 2>/dev/null || sudo systemctl restart tor 2>/dev/null || echo "   Tor restart failed"
+        fi
+    ' 2>&1 | sed 's/^/   /'
+
    # ── Step 27: Deploy manifest ─────────────────────────────────────
    step "Writing deploy manifest"
    DEPLOY_TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
--- a/scripts/deploy-to-target.sh
+++ b/scripts/deploy-to-target.sh
@@ -54,6 +54,7 @@ DRY_RUN=false
 CANARY=false
 TAILSCALE=false
 TAILSCALE_NODE=""
+FLEET=false
 for arg in "$@"; do
    case $arg in
        --quick) QUICK=true ;;
@@ -65,9 +66,30 @@ for arg in "$@"; do
        --canary) CANARY=true ;;
        --tailscale) TAILSCALE=true ;;
        --tailscale-node=*) TAILSCALE_NODE="${arg#*=}" ;;
+        --fleet) FLEET=true ;;
    esac
 done

+# Fleet deploy: .228 → .198 → all 3 Tailscale nodes (all 5 servers)
+if [ "$FLEET" = true ]; then
+    echo "╔════════════════════════════════════════════════════════════════╗"
+    echo "║  FLEET DEPLOY — All 5 nodes (.228, .198, Arch 1/2/3)         ║"
+    echo "╚════════════════════════════════════════════════════════════════╝"
+    echo ""
+    echo "Phase 1: Build + deploy to .228 (primary build server)"
+    "$0" --live || { echo "FAILED: .228 deploy"; exit 1; }
+    echo ""
+    echo "Phase 2: Copy to .198 (LAN secondary)"
+    "$0" --both || { echo "WARNING: .198 deploy failed (continuing)"; }
+    echo ""
+    echo "Phase 3: Deploy to all Tailscale nodes (Arch 1/2/3)"
+    "$SCRIPT_DIR/deploy-tailscale.sh" --all || { echo "WARNING: Some Tailscale nodes failed"; }
+    echo ""
+    echo "════════════════════════════════════════════════════════════════"
+    echo "Fleet deploy complete."
+    exit 0
+fi
+
 # Tailscale deploy: delegate to deploy-tailscale.sh
 if [ "$TAILSCALE" = true ]; then
    echo "Deploying to all Tailscale nodes..."
@@ -300,6 +322,8 @@ fi
 # When --both: deploy to 228 first, then copy to 198
 if [ "$BOTH" = true ]; then
    echo "Deploying to both servers (228, then 198)..."
+    # Release lock so the recursive --live call can acquire it
+    rm -rf "$LOCK_DIR" 2>/dev/null; trap - EXIT
    "$0" --live
    echo ""
    echo "📤 Copying to 192.168.1.198 (no rsync/cargo on that node)..."
--- a/scripts/first-boot-containers.sh
+++ b/scripts/first-boot-containers.sh
@@ -125,7 +125,7 @@ BTCCONF
 fi

 # Generate per-installation database passwords if not already saved
-for svc in mempool btcpay immich penpot mysql-root; do
+for svc in mempool btcpay mysql-root; do
    if [ ! -f "$SECRETS_DIR/${svc}-db-password" ]; then
        openssl rand -base64 24 > "$SECRETS_DIR/${svc}-db-password"
        chmod 600 "$SECRETS_DIR/${svc}-db-password"
@@ -133,8 +133,6 @@ for svc in mempool btcpay immich penpot mysql-root; do
 done
 MEMPOOL_DB_PASS=$(cat "$SECRETS_DIR/mempool-db-password")
 BTCPAY_DB_PASS=$(cat "$SECRETS_DIR/btcpay-db-password")
-IMMICH_DB_PASS=$(cat "$SECRETS_DIR/immich-db-password")
-PENPOT_DB_PASS=$(cat "$SECRETS_DIR/penpot-db-password")
 MYSQL_ROOT_PASS=$(cat "$SECRETS_DIR/mysql-root-db-password")

 # Generate Fedimint gateway password and bcrypt hash
@@ -236,7 +234,7 @@ $DOCKER network create archy-net 2>/dev/null || true
 # → host UID 100000+N). Must run BEFORE container creation.
 log "Fixing rootless podman UID mapping..."
 # Containers running as root (UID 0 → host UID 100000)
-for dir in lnd electrumx btcpay nbxplorer immich jellyfin vaultwarden \
+for dir in lnd electrumx btcpay nbxplorer jellyfin vaultwarden \
           home-assistant fedimint fedimint-gateway photoprism ollama filebrowser \
           nextcloud uptime-kuma onlyoffice nginx-proxy-manager portainer nostr-rs-relay; do
    [ -d "/var/lib/archipelago/$dir" ] && chown -R 100000:100000 "/var/lib/archipelago/$dir" 2>/dev/null
@@ -244,7 +242,7 @@ done
 # Bitcoin Knots: container UID 101 → host UID 100101
 [ -d /var/lib/archipelago/bitcoin ] && chown -R 100101:100101 /var/lib/archipelago/bitcoin 2>/dev/null
 # Postgres: container UID 70 → host UID 100070
-for dir in postgres-btcpay immich-db penpot-postgres; do
+for dir in postgres-btcpay; do
    [ -d "/var/lib/archipelago/$dir" ] && chown -R 100070:100070 "/var/lib/archipelago/$dir" 2>/dev/null
 done
 # MariaDB: container UID 999 → host UID 100999
@@ -270,7 +268,6 @@ mem_limit() {
    lnd)                 echo "512m";;
    electrumx)           echo "1g";;
    nextcloud)           echo "1g";;
-    immich_server)       echo "1g";;
    btcpay-server)       echo "1g";;
    homeassistant)       echo "512m";;
    fedimint)            echo "512m";;
@@ -289,8 +286,6 @@ mem_limit() {
    filebrowser)         echo "256m";;
    portainer)           echo "256m";;
    nginx-proxy-manager) echo "256m";;
-    immich_postgres)     echo "256m";;
-    immich_redis)        echo "128m";;
    tailscale)           echo "256m";;
    indeedhub|archy-bitcoin-ui|archy-lnd-ui|archy-electrs-ui) echo "128m";;
    *)                   echo "512m";;
@@ -322,7 +317,7 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qE 'bitcoin-knots|arch
    --security-opt no-new-privileges:true \
    -p 8332:8332 -p 8333:8333 \
    -v /var/lib/archipelago/bitcoin:/home/bitcoin/.bitcoin \
-    "${BITCOIN_KNOTS_IMAGE:-docker.io/bitcoinknots/bitcoin:v28.1}" \
+    "${BITCOIN_KNOTS_IMAGE:-docker.io/bitcoinknots/bitcoin:28.1}" \
    -server=1 $BTC_EXTRA_ARGS \
    -rpcallowip=0.0.0.0/0 -rpcbind=0.0.0.0:8332 \
    -proxy=host.containers.internal:9050 -listen=1 -bind=0.0.0.0:8333 \
@@ -800,110 +795,6 @@ fi
 track_container "tailscale"

 # Immich stack (postgres + redis + server - ML optional)
-# Remove old single-container 'immich' if present (wrong port 2283:3001, conflicts with immich_server)
-if $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx immich; then
-  log "Removing old immich container (use immich_server stack)..."
-  $DOCKER stop immich 2>/dev/null || true
-  $DOCKER rm -f immich 2>/dev/null || true
-  $DOCKER start immich_server 2>/dev/null || true
-fi
-if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
-  log "Creating Immich stack..."
-  mkdir -p /var/lib/archipelago/immich /var/lib/archipelago/immich-db
-  $DOCKER network create immich-net 2>/dev/null || true
-  if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q immich_postgres; then
-    $DOCKER run -d --name immich_postgres --restart unless-stopped \
-      --health-cmd="pg_isready -U postgres || exit 1" --health-interval=30s --health-timeout=5s --health-retries=3 \
-      --memory=$(mem_limit immich_postgres) --network immich-net \
-      -v /var/lib/archipelago/immich-db:/var/lib/postgresql/data \
-      -e "POSTGRES_PASSWORD=$IMMICH_DB_PASS" -e POSTGRES_USER=postgres -e POSTGRES_DB=immich \
-      ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0 2>>"$LOG" || true
-    sleep 3
-    for i in 1 2 3 4 5 6 7 8 9 10; do
-      $DOCKER exec immich_postgres pg_isready -U postgres 2>/dev/null && break
-      sleep 2
-    done
-  fi
-  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_redis; then
-    $DOCKER run -d --name immich_redis --restart unless-stopped \
-      --health-cmd="redis-cli ping || exit 1" --health-interval=30s --health-timeout=5s --health-retries=3 \
-      --memory=$(mem_limit immich_redis) --network immich-net \
-      docker.io/valkey/valkey:7-alpine 2>>"$LOG" || true
-    sleep 2
-  fi
-  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
-    $DOCKER run -d --name immich_server --restart unless-stopped \
-      --health-cmd="curl -sf http://localhost:2283/ || exit 1" --health-interval=30s --health-timeout=5s --health-retries=3 \
-      --memory=$(mem_limit immich_server) --network immich-net \
-      -p 2283:2283 -v /var/lib/archipelago/immich:/usr/src/app/upload \
-      -e DB_HOSTNAME=immich_postgres -e DB_USERNAME=postgres -e "DB_PASSWORD=$IMMICH_DB_PASS" \
-      -e DB_DATABASE_NAME=immich -e REDIS_HOSTNAME=immich_redis \
-      -e UPLOAD_LOCATION=/usr/src/app/upload \
-      ghcr.io/immich-app/immich-server:release 2>>"$LOG" || true
-  fi
-fi
-track_container "immich_server"
-
-# Penpot stack (postgres + valkey + backend + exporter + frontend)
-if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-frontend; then
-  log "Creating Penpot stack..."
-  mkdir -p /var/lib/archipelago/penpot-assets /var/lib/archipelago/penpot-postgres
-  $DOCKER network create penpot-net 2>/dev/null || true
-  if ! $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -q penpot-postgres; then
-    $DOCKER run -d --name penpot-postgres --restart unless-stopped \
-      --health-cmd="pg_isready -U penpot || exit 1" --health-interval=30s --health-timeout=5s --health-retries=3 \
-      --memory=$(mem_limit penpot-postgres) --network penpot-net \
-      -v /var/lib/archipelago/penpot-postgres:/var/lib/postgresql/data \
-      -e POSTGRES_DB=penpot -e POSTGRES_USER=penpot -e "POSTGRES_PASSWORD=$PENPOT_DB_PASS" \
-      docker.io/postgres:15 2>>"$LOG" || true
-    sleep 5
-  fi
-  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-valkey; then
-    $DOCKER run -d --name penpot-valkey --restart unless-stopped \
-      --health-cmd="redis-cli ping || exit 1" --health-interval=30s --health-timeout=5s --health-retries=3 \
-      --memory=$(mem_limit penpot-valkey) --network penpot-net \
-      -e VALKEY_EXTRA_FLAGS="--maxmemory 128mb --maxmemory-policy volatile-lfu" \
-      docker.io/valkey/valkey:8.1 2>>"$LOG" || true
-    sleep 3
-  fi
-  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-backend; then
-    $DOCKER run -d --name penpot-backend --restart unless-stopped \
-      --health-cmd="curl -sf http://localhost:6060/ || exit 1" --health-interval=30s --health-timeout=5s --health-retries=3 \
-      --memory=$(mem_limit penpot-backend) --network penpot-net \
-      -v /var/lib/archipelago/penpot-assets:/opt/data/assets \
-      -e PENPOT_PUBLIC_URI="http://${TARGET_IP}:9001" \
-      -e PENPOT_SECRET_KEY=archipelago-penpot-secret-key-change-in-production \
-      -e PENPOT_DATABASE_URI=postgresql://penpot-postgres/penpot \
-      -e PENPOT_DATABASE_USERNAME=penpot -e "PENPOT_DATABASE_PASSWORD=$PENPOT_DB_PASS" \
-      -e PENPOT_REDIS_URI=redis://penpot-valkey/0 \
-      -e PENPOT_OBJECTS_STORAGE_BACKEND=fs \
-      -e PENPOT_OBJECTS_STORAGE_FS_DIRECTORY=/opt/data/assets \
-      -e PENPOT_FLAGS=disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies \
-      "${PENPOT_BACKEND_IMAGE:-docker.io/penpotapp/backend:2.4.2}" 2>>"$LOG" || true
-    sleep 5
-  fi
-  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-exporter; then
-    $DOCKER run -d --name penpot-exporter --restart unless-stopped \
-      --health-cmd="curl -sf http://localhost:6061/ || exit 1" --health-interval=30s --health-timeout=5s --health-retries=3 \
-      --memory=$(mem_limit penpot-exporter) --network penpot-net \
-      -e PENPOT_SECRET_KEY=archipelago-penpot-secret-key-change-in-production \
-      -e PENPOT_PUBLIC_URI=http://penpot-frontend:8080 \
-      -e PENPOT_REDIS_URI=redis://penpot-valkey/0 \
-      "${PENPOT_EXPORTER_IMAGE:-docker.io/penpotapp/exporter:2.4.2}" 2>>"$LOG" || true
-    sleep 2
-  fi
-  if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q penpot-frontend; then
-    $DOCKER run -d --name penpot-frontend --restart unless-stopped \
-      --health-cmd="curl -sf http://localhost:80/ || exit 1" --health-interval=30s --health-timeout=5s --health-retries=3 \
-      --memory=$(mem_limit penpot-frontend) --network penpot-net \
-      -p 9001:8080 -v /var/lib/archipelago/penpot-assets:/opt/data/assets \
-      -e PENPOT_PUBLIC_URI="http://${TARGET_IP}:9001" \
-      -e PENPOT_FLAGS=disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies \
-      "${PENPOT_FRONTEND_IMAGE:-docker.io/penpotapp/frontend:2.4.2}" 2>>"$LOG" || true
-  fi
-fi
-track_container "penpot-frontend"
-
 # 8. Nostr relays (optional - only if images were loaded; deploy does not create these on first boot)
 # nostr-rs-relay and strfry are in ISO image bundle; create if image exists
 if $DOCKER images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -q 'nostr-rs-relay'; then
@@ -925,7 +816,7 @@ if $DOCKER images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -q 'str
      --health-cmd="curl -sf http://localhost:7777/ || exit 1" --health-interval=30s --health-timeout=5s --health-retries=3 \
      --memory=$(mem_limit strfry) \
      -p 7777:7777 -v /var/lib/archipelago/strfry:/data \
-      "${STRFRY_IMAGE:-docker.io/pluja/strfry:latest}" 2>>"$LOG" || true
+      "${STRFRY_IMAGE:-docker.io/pluja/strfry:1.0.4}" 2>>"$LOG" || true
  fi
 fi

--- a/scripts/image-versions.sh
+++ b/scripts/image-versions.sh
@@ -6,19 +6,20 @@
 #        source "$(dirname "$0")/image-versions.sh" 2>/dev/null || true

 # Bitcoin stack
-BITCOIN_KNOTS_IMAGE="docker.io/bitcoinknots/bitcoin:v28.1"
+BITCOIN_KNOTS_IMAGE="docker.io/bitcoinknots/bitcoin:28.1"
 LND_IMAGE="docker.io/lightninglabs/lnd:v0.18.5-beta"
 ELECTRUMX_IMAGE="docker.io/lukechilds/electrumx:v1.16.0"

 # Mempool stack
-MEMPOOL_API_IMAGE="docker.io/mempool/frontend:v3.0.0"
+MEMPOOL_BACKEND_IMAGE="docker.io/mempool/backend:v3.0.0"
 MEMPOOL_WEB_IMAGE="docker.io/mempool/frontend:v3.0.0"
 MARIADB_IMAGE="docker.io/library/mariadb:11.4"

 # BTCPay
-BTCPAY_IMAGE="docker.io/btcpayserver/btcpayserver:1.14.5"
+BTCPAY_IMAGE="docker.io/btcpayserver/btcpayserver:1.13.7"
 NBXPLORER_IMAGE="docker.io/nicolasdorier/nbxplorer:2.5.13"
-POSTGRES_IMAGE="docker.io/library/postgres:16"
+POSTGRES_IMAGE="docker.io/library/postgres:15"
+BTCPAY_POSTGRES_IMAGE="docker.io/library/postgres:15"

 # Apps
 HOMEASSISTANT_IMAGE="ghcr.io/home-assistant/home-assistant:2024.12"
@@ -28,8 +29,8 @@ JELLYFIN_IMAGE="docker.io/jellyfin/jellyfin:10.10.3"
 PHOTOPRISM_IMAGE="docker.io/photoprism/photoprism:240915"
 OLLAMA_IMAGE="docker.io/ollama/ollama:0.5.4"
 VAULTWARDEN_IMAGE="docker.io/vaultwarden/server:1.32.5"
-NEXTCLOUD_IMAGE="docker.io/library/nextcloud:30"
-SEARXNG_IMAGE="docker.io/searxng/searxng:2024.11.17"
+NEXTCLOUD_IMAGE="docker.io/library/nextcloud:29"
+SEARXNG_IMAGE="docker.io/searxng/searxng:2026.3.20-6c7e9c197"
 ONLYOFFICE_IMAGE="docker.io/onlyoffice/documentserver:8.2"
 FILEBROWSER_IMAGE="docker.io/filebrowser/filebrowser:v2"
 NPM_IMAGE="docker.io/jc21/nginx-proxy-manager:2"
@@ -45,20 +46,14 @@ FEDIMINT_IMAGE="docker.io/fedimint/fedimintd:v0.5.1"
 FEDIMINT_GATEWAY_IMAGE="docker.io/fedimint/gatewayd:v0.5.1"

 # Media
-IMMICH_IMAGE="ghcr.io/immich-app/immich-server:v1.123.0"
-IMMICH_POSTGRES_IMAGE="ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0"
-IMMICH_VALKEY_IMAGE="docker.io/valkey/valkey:7-alpine"
 REDIS_IMAGE="docker.io/library/redis:7"

-# Penpot
-PENPOT_BACKEND_IMAGE="docker.io/penpotapp/backend:2.4.2"
-PENPOT_FRONTEND_IMAGE="docker.io/penpotapp/frontend:2.4.2"
-PENPOT_EXPORTER_IMAGE="docker.io/penpotapp/exporter:2.4.2"
+# Valkey (general purpose)
 VALKEY_IMAGE="docker.io/valkey/valkey:8"

 # Nostr
 NOSTR_RS_RELAY_IMAGE="docker.io/scsibug/nostr-rs-relay:0.9.0"
-STRFRY_IMAGE="docker.io/pluja/strfry:latest"  # No stable tag available yet
+STRFRY_IMAGE="docker.io/pluja/strfry:1.0.4"

 # IndeedHub stack (local builds use :local tag, not :latest)
 MINIO_IMAGE="docker.io/minio/minio:RELEASE.2024-11-07T00-52-20Z"
@@ -68,8 +63,6 @@ INDEEDHUB_REDIS_IMAGE="docker.io/library/redis:7-alpine"
 # DWN (Decentralized Web Node)
 DWN_SERVER_IMAGE="ghcr.io/tbd54566975/dwn-server:main"

-# Penpot postgres (separate from BTCPay postgres — different version)
-PENPOT_POSTGRES_IMAGE="docker.io/library/postgres:15"

 # Base images
 NGINX_ALPINE_IMAGE="docker.io/library/nginx:alpine"
--- a/scripts/lib/common.sh
+++ b/scripts/lib/common.sh
@@ -7,7 +7,7 @@
 # Provides: logging, SSH helpers, health checks, disk checks, memory limits

 # Guard against double-sourcing
-[ -n "$_ARCHY_COMMON_LOADED" ] && return 0
+[ -n "${_ARCHY_COMMON_LOADED:-}" ] && return 0
 _ARCHY_COMMON_LOADED=1

 # ── Colored logging ─────────────────────────────────────────────────────