114 lines
3.5 KiB
Bash
114 lines
3.5 KiB
Bash
#!/bin/bash
|
|
#
|
|
# Cassandra Cluster Sequential Deployment Script
|
|
# This script deploys Cassandra nodes sequentially to avoid race conditions
|
|
# during cluster formation.
|
|
#
|
|
|
|
set -e
|
|
|
|
STACK_NAME="cassandra"
|
|
STACK_FILE="cassandra-stack.yml"
|
|
|
|
echo "=== Cassandra Cluster Sequential Deployment ==="
|
|
echo ""
|
|
|
|
# Check if stack file exists
|
|
if [ ! -f "$STACK_FILE" ]; then
|
|
echo "ERROR: $STACK_FILE not found in current directory"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Step 1: Deploying cassandra-1 (seed node)..."
|
|
docker stack deploy -c "$STACK_FILE" "$STACK_NAME"
|
|
|
|
# Scale down cassandra-2 and cassandra-3 temporarily
|
|
docker service scale "${STACK_NAME}_cassandra-2=0" > /dev/null 2>&1
|
|
docker service scale "${STACK_NAME}_cassandra-3=0" > /dev/null 2>&1
|
|
|
|
echo "Waiting for cassandra-1 to become healthy (this takes ~5-8 minutes)..."
|
|
echo "Checking every 30 seconds..."
|
|
|
|
# Wait for cassandra-1 to be running
|
|
COUNTER=0
|
|
MAX_WAIT=20 # 20 * 30 seconds = 10 minutes max
|
|
while [ $COUNTER -lt $MAX_WAIT ]; do
|
|
REPLICAS=$(docker service ls --filter "name=${STACK_NAME}_cassandra-1" --format "{{.Replicas}}")
|
|
if [ "$REPLICAS" = "1/1" ]; then
|
|
echo "✓ cassandra-1 is running"
|
|
# Give it extra time to fully initialize
|
|
echo "Waiting additional 2 minutes for cassandra-1 to fully initialize..."
|
|
sleep 120
|
|
break
|
|
fi
|
|
echo " cassandra-1 status: $REPLICAS (waiting...)"
|
|
sleep 30
|
|
COUNTER=$((COUNTER + 1))
|
|
done
|
|
|
|
if [ $COUNTER -eq $MAX_WAIT ]; then
|
|
echo "ERROR: cassandra-1 failed to start within 10 minutes"
|
|
echo "Check logs with: docker service logs ${STACK_NAME}_cassandra-1"
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
echo "Step 2: Starting cassandra-2..."
|
|
docker service scale "${STACK_NAME}_cassandra-2=1"
|
|
|
|
echo "Waiting for cassandra-2 to become healthy (this takes ~5-8 minutes)..."
|
|
COUNTER=0
|
|
while [ $COUNTER -lt $MAX_WAIT ]; do
|
|
REPLICAS=$(docker service ls --filter "name=${STACK_NAME}_cassandra-2" --format "{{.Replicas}}")
|
|
if [ "$REPLICAS" = "1/1" ]; then
|
|
echo "✓ cassandra-2 is running"
|
|
echo "Waiting additional 2 minutes for cassandra-2 to join cluster..."
|
|
sleep 120
|
|
break
|
|
fi
|
|
echo " cassandra-2 status: $REPLICAS (waiting...)"
|
|
sleep 30
|
|
COUNTER=$((COUNTER + 1))
|
|
done
|
|
|
|
if [ $COUNTER -eq $MAX_WAIT ]; then
|
|
echo "ERROR: cassandra-2 failed to start within 10 minutes"
|
|
echo "Check logs with: docker service logs ${STACK_NAME}_cassandra-2"
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
echo "Step 3: Starting cassandra-3..."
|
|
docker service scale "${STACK_NAME}_cassandra-3=1"
|
|
|
|
echo "Waiting for cassandra-3 to become healthy (this takes ~5-8 minutes)..."
|
|
COUNTER=0
|
|
while [ $COUNTER -lt $MAX_WAIT ]; do
|
|
REPLICAS=$(docker service ls --filter "name=${STACK_NAME}_cassandra-3" --format "{{.Replicas}}")
|
|
if [ "$REPLICAS" = "1/1" ]; then
|
|
echo "✓ cassandra-3 is running"
|
|
echo "Waiting additional 2 minutes for cassandra-3 to join cluster..."
|
|
sleep 120
|
|
break
|
|
fi
|
|
echo " cassandra-3 status: $REPLICAS (waiting...)"
|
|
sleep 30
|
|
COUNTER=$((COUNTER + 1))
|
|
done
|
|
|
|
if [ $COUNTER -eq $MAX_WAIT ]; then
|
|
echo "ERROR: cassandra-3 failed to start within 10 minutes"
|
|
echo "Check logs with: docker service logs ${STACK_NAME}_cassandra-3"
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== Deployment Complete ==="
|
|
echo ""
|
|
echo "All 3 Cassandra nodes should now be running and forming a cluster."
|
|
echo ""
|
|
echo "Verify cluster status by SSH'ing to any worker node and running:"
|
|
echo " docker exec -it \$(docker ps -q --filter \"name=cassandra\") nodetool status"
|
|
echo ""
|
|
echo "You should see 3 nodes with status 'UN' (Up Normal)."
|
|
echo ""
|