feat: add orchestrator scripts and Docker setup for droplet management; include SSH key handling and environment configuration
This commit is contained in:
267
orchestrator/orchestrating.sh
Normal file
267
orchestrator/orchestrating.sh
Normal file
@@ -0,0 +1,267 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Record start time
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
# Function to check list of required environment variables
|
||||
check_env_vars() {
|
||||
local missing_vars=()
|
||||
for var in "$@"; do
|
||||
if [ -z "${!var}" ]; then
|
||||
missing_vars+=("$var")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing_vars[@]} -ne 0 ]; then
|
||||
echo "Error: The following environment variables are not set: ${missing_vars[*]}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to check if doctl is available
|
||||
check_doctl() {
|
||||
if ! command -v doctl &> /dev/null; then
|
||||
echo "Error: doctl is not installed or not in PATH"
|
||||
echo "Please install doctl: https://docs.digitalocean.com/reference/doctl/how-to/install/"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to setup SSH private key
|
||||
setup_ssh_key() {
|
||||
if [ -z "$SSH_PRIVATE_KEY_DECODED" ]; then
|
||||
echo "Error: SSH_PRIVATE_KEY_DECODED environment variable is not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create SSH directory if it doesn't exist
|
||||
mkdir -p ~/.ssh
|
||||
chmod 700 ~/.ssh
|
||||
|
||||
# Decode base64 encoded SSH private key
|
||||
echo "$SSH_PRIVATE_KEY_DECODED" | base64 -d > ~/.ssh/id_rsa
|
||||
|
||||
# Ensure proper line endings for OpenSSH key format
|
||||
sed -i 's/\r$//' ~/.ssh/id_rsa
|
||||
|
||||
# Add final newline if missing
|
||||
if [ -s ~/.ssh/id_rsa ] && [ "$(tail -c1 ~/.ssh/id_rsa | wc -l)" -eq 0 ]; then
|
||||
echo >> ~/.ssh/id_rsa
|
||||
fi
|
||||
|
||||
chmod 600 ~/.ssh/id_rsa
|
||||
|
||||
# Validate SSH key formats
|
||||
if ! ssh-keygen -y -f ~/.ssh/id_rsa > /dev/null 2>&1; then
|
||||
echo "Error: Invalid SSH private key format"
|
||||
echo "Key content preview (first 100 chars):"
|
||||
head -c 100 ~/.ssh/id_rsa
|
||||
echo
|
||||
echo "Key file size: $(wc -c < ~/.ssh/id_rsa) bytes"
|
||||
echo "Key file lines: $(wc -l < ~/.ssh/id_rsa) lines"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "SSH private key has been set up and validated successfully"
|
||||
}
|
||||
|
||||
# Function to destroy droplet
|
||||
destroy_droplet() {
|
||||
local droplet_id="$1"
|
||||
|
||||
if [ -z "$droplet_id" ]; then
|
||||
echo "Error: No droplet ID provided for destruction"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Destroying droplet with ID: $droplet_id"
|
||||
|
||||
# Authenticate doctl
|
||||
doctl auth init --access-token "$DO_TOKEN"
|
||||
|
||||
# Destroy droplet
|
||||
if doctl compute droplet delete "$droplet_id" --force; then
|
||||
echo "Droplet $droplet_id destroyed successfully"
|
||||
return 0
|
||||
else
|
||||
echo "Error: Failed to destroy droplet $droplet_id"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to create droplet
|
||||
create_droplet() {
|
||||
local name="${1:-runner-${RUNNER_ID}}"
|
||||
local size="${2:-s-1vcpu-1gb}"
|
||||
local image="${3:-200811356}"
|
||||
local region="${4:-sgp1}"
|
||||
|
||||
echo "Creating droplet: $name"
|
||||
echo "Size: $size, Image: $image, Region: $region"
|
||||
|
||||
# Authenticate doctl
|
||||
doctl auth init --access-token "$DO_TOKEN"
|
||||
|
||||
# Create droplet
|
||||
local droplet_id=$(doctl compute droplet create "$name" \
|
||||
--size "$size" \
|
||||
--image "$image" \
|
||||
--region "$region" \
|
||||
--ssh-keys $(doctl compute ssh-key list --format ID --no-header | tr '\n' ',' | sed 's/,$//' || echo "") \
|
||||
--format ID \
|
||||
--no-header \
|
||||
--wait)
|
||||
|
||||
if [ -n "$droplet_id" ]; then
|
||||
echo "Droplet created successfully with ID: $droplet_id"
|
||||
echo "Droplet name: $name"
|
||||
|
||||
echo "droplet_id=$droplet_id" >> $GITHUB_OUTPUT
|
||||
|
||||
# Get droplet IP
|
||||
local droplet_ip=$(doctl compute droplet get "$droplet_id" --format PublicIPv4 --no-header)
|
||||
echo "Droplet IP: $droplet_ip"
|
||||
echo "droplet_ip=$droplet_ip" >> $GITHUB_OUTPUT
|
||||
|
||||
# Test SSH connection
|
||||
if test_ssh_connection "$droplet_ip"; then
|
||||
echo "Droplet is ready and accessible via SSH"
|
||||
|
||||
# Setup VM with required services
|
||||
if setup_vm "$droplet_ip"; then
|
||||
echo "VM setup completed successfully"
|
||||
else
|
||||
echo "Error: VM setup failed"
|
||||
echo "Destroying failed droplet..."
|
||||
destroy_droplet "$droplet_id"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
echo "Warning: Droplet created but SSH connection failed"
|
||||
echo "Destroying failed droplet..."
|
||||
destroy_droplet "$droplet_id"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
else
|
||||
echo "Error: Failed to create droplet"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to test SSH connection
|
||||
test_ssh_connection() {
|
||||
local droplet_ip="$1"
|
||||
local max_attempts=10
|
||||
local attempt=1
|
||||
local wait_time=30
|
||||
|
||||
echo "Testing SSH connection to $droplet_ip..."
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
echo "Attempt $attempt/$max_attempts: Testing SSH connection..."
|
||||
|
||||
# Test SSH connection with verbose error handling
|
||||
ssh_output=$(ssh -i ~/.ssh/id_rsa \
|
||||
-o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=10 \
|
||||
-o BatchMode=yes \
|
||||
-o PasswordAuthentication=no \
|
||||
root@"$droplet_ip" "echo 'SSH connection successful'" 2>&1)
|
||||
|
||||
ssh_exit_code=$?
|
||||
|
||||
if [ $ssh_exit_code -eq 0 ]; then
|
||||
echo "SSH connection to $droplet_ip established successfully"
|
||||
echo "Response: $ssh_output"
|
||||
return 0
|
||||
else
|
||||
echo "SSH connection failed (exit code: $ssh_exit_code)"
|
||||
echo "Error output: $ssh_output"
|
||||
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Waiting $wait_time seconds before retry..."
|
||||
sleep $wait_time
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Error: Failed to establish SSH connection to $droplet_ip after $max_attempts attempts"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Function to setup VM after successful SSH connection
|
||||
setup_vm() {
|
||||
local droplet_ip="$1"
|
||||
|
||||
if [ -z "$droplet_ip" ]; then
|
||||
echo "Error: No droplet IP provided for VM setup"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Setting up VM at $droplet_ip..."
|
||||
|
||||
# Execute setup commands via SSH
|
||||
echo "Step 1: Running prepare.sh config.yml..."
|
||||
if ! ssh -i ~/.ssh/id_rsa \
|
||||
-o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=30 \
|
||||
root@"$droplet_ip" \
|
||||
"./prepare.sh config.yml RUNNER_ID $RUNNER_ID"; then
|
||||
echo "Error: Failed to run prepare.sh config.yml"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Step 2: Running prepare.sh docker-compose.yml..."
|
||||
if ! ssh -i ~/.ssh/id_rsa \
|
||||
-o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=30 \
|
||||
root@"$droplet_ip" \
|
||||
"./prepare.sh docker-compose.yml RUNNER_ID $RUNNER_ID"; then
|
||||
echo "Error: Failed to run prepare.sh docker-compose.yml"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Step 3: Starting docker compose services..."
|
||||
if ! ssh -i ~/.ssh/id_rsa \
|
||||
-o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=30 \
|
||||
root@"$droplet_ip" \
|
||||
"docker compose up -d"; then
|
||||
echo "Error: Failed to start docker compose services"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Step 4: Checking docker compose status..."
|
||||
ssh -i ~/.ssh/id_rsa \
|
||||
-o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=30 \
|
||||
root@"$droplet_ip" \
|
||||
"docker compose ps"
|
||||
|
||||
echo "VM setup completed successfully"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Check required environment variables
|
||||
check_env_vars "SPEC" "RUNNER_ID" "DO_TOKEN" "SSH_PRIVATE_KEY_DECODED"
|
||||
|
||||
# Check if doctl is available
|
||||
check_doctl
|
||||
|
||||
# Setup SSH private key
|
||||
setup_ssh_key
|
||||
|
||||
create_droplet
|
||||
|
||||
# Calculate and display execution time
|
||||
END_TIME=$(date +%s)
|
||||
EXECUTION_TIME=$((END_TIME - START_TIME))
|
||||
echo "Script execution time: ${EXECUTION_TIME} seconds"
|
||||
echo "ready=true" >> $GITHUB_OUTPUT
|
||||
Reference in New Issue
Block a user