sync branch

Dockerfile

@@ -1,4 +1,4 @@
-# Use a specific Node.js version for better reproducibility
+# Use Node.js 23.3.0 as specified in package.json
 FROM node:23.3.0-slim AS builder
 
 # Install pnpm globally and install necessary build tools
@@ -11,36 +11,50 @@ RUN npm install -g pnpm@9.4.0 && \
 # Set Python 3 as the default python
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
-# Set the working directory
 WORKDIR /app
 
-# Copy package.json and other configuration files
+# Copy all workspace files first
 COPY package.json pnpm-lock.yaml pnpm-workspace.yaml .npmrc turbo.json ./
-
-# Copy the rest of the application code
-COPY agent ./agent
 COPY packages ./packages
+COPY agent ./agent
 COPY scripts ./scripts
-COPY characters ./characters
 
 # Install dependencies and build the project
-RUN pnpm install \
-    && pnpm build-docker \
-    && pnpm prune --prod
+RUN pnpm install && \
+    pnpm build-docker && \
+    pnpm prune --prod
 
-# Create a new stage for the final image
+# Final stage
 FROM node:23.3.0-slim
 
-# Install runtime dependencies if needed
+# Install runtime dependencies and certificates first
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        git \
+        python3 \
+        curl \
+        gnupg \
+        ca-certificates && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install pnpm
+RUN npm install -g pnpm@9.4.0
+
+# Install runtime dependencies and Google Cloud SDK
 RUN npm install -g pnpm@9.4.0 && \
     apt-get update && \
-    apt-get install -y git python3 && \
+    apt-get install -y git python3 curl gnupg && \
+    echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \
+    apt-get update && \
+    apt-get install -y google-cloud-sdk && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 
-# Copy built artifacts and production dependencies from the builder stage
+# Copy only necessary files from builder
 COPY --from=builder /app/package.json ./
 COPY --from=builder /app/pnpm-workspace.yaml ./
 COPY --from=builder /app/.npmrc ./
@@ -49,7 +63,64 @@ COPY --from=builder /app/node_modules ./node_modules
 COPY --from=builder /app/agent ./agent
 COPY --from=builder /app/packages ./packages
 COPY --from=builder /app/scripts ./scripts
-COPY --from=builder /app/characters ./characters
 
-# Set the command to run the application
-CMD ["pnpm", "start"]
+# Create characters directory
+RUN mkdir -p characters
+
+# Add debugging to startup command
+CMD sh -c '\
+    echo "Debug: Starting container initialization" && \
+    last_update="" && \
+    while true; do \
+        # Check metadata for updates
+        current_update=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/character-update-trigger") && \
+        if [ "$current_update" != "$last_update" ]; then \
+            echo "Update triggered" && \
+            # Get list of active characters
+            active_characters=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/active-characters") && \
+            echo "Active characters: $active_characters" && \
+            # Copy all character files (even inactive ones)
+            echo "Copying all character files..." && \
+            gsutil -m cp "gs://${AGENTS_BUCKET_NAME}/${DEPLOYMENT_ID}/*.character.json" /app/characters/ && \
+            echo "All character files:" && \
+            ls -la /app/characters/ && \
+            # Filter for only active characters
+            character_files=$(echo "$active_characters" | jq -r ".[]" | while read char; do \
+                find /app/characters -name "${char}.character.json"; \
+            done | paste -sd "," -) && \
+            if [ -n "$character_files" ]; then \
+                echo "Restarting with active character files: $character_files" && \
+                pkill -f "pnpm start" || true && \
+                pnpm start --non-interactive --characters="$character_files" & \
+            fi && \
+            last_update=$current_update; \
+        fi && \
+        sleep 30; \
+    done & \
+    \
+    # Initial startup using all available characters
+    echo "Debug: Environment variables:" && \
+    env | grep -E "AGENTS_BUCKET_NAME|DEPLOYMENT_ID" && \
+    echo "Debug: Copying character files..." && \
+    gsutil -m cp "gs://${AGENTS_BUCKET_NAME}/${DEPLOYMENT_ID}/*.character.json" /app/characters/ && \
+    echo "Debug: Character files:" && \
+    ls -la /app/characters/ && \
+    # Get initial active characters from metadata
+    active_characters=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/active-characters") && \
+    if [ -n "$active_characters" ]; then \
+        character_files=$(echo "$active_characters" | jq -r ".[]" | while read char; do \
+            find /app/characters -name "${char}.character.json"; \
+        done | paste -sd "," -) && \
+        echo "Starting with active character files: $character_files" && \
+        pnpm start --non-interactive --characters="$character_files"; \
+    else \
+        # If no active characters specified, use all available ones
+        character_files=$(find /app/characters -name "*.character.json" | paste -sd "," -) && \
+        if [ -n "$character_files" ]; then \
+            echo "Starting with all character files: $character_files" && \
+            pnpm start --non-interactive --characters="$character_files"; \
+        else \
+            echo "ERROR: No character files found in /app/characters/" && \
+            exit 1; \
+        fi; \
+    fi'

cloudbuild-base.yaml

@@ -0,0 +1,38 @@
+steps:
+  # Build the base image
+  - name: 'gcr.io/cloud-builders/docker'
+    env:
+      - 'DOCKER_BUILDKIT=1'
+    args:
+      - 'build'
+      - '-t'
+      - 'us-east1-docker.pkg.dev/${PROJECT_ID}/qi-agents/base:latest'
+      - '.'
+
+  # Push the base image
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      - 'push'
+      - 'us-east1-docker.pkg.dev/${PROJECT_ID}/qi-agents/base:latest'
+
+  # Tag with commit hash
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      - 'tag'
+      - 'us-east1-docker.pkg.dev/${PROJECT_ID}/qi-agents/base:latest'
+      - 'us-east1-docker.pkg.dev/${PROJECT_ID}/qi-agents/base:${SHORT_SHA}'
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      - 'push'
+      - 'us-east1-docker.pkg.dev/${PROJECT_ID}/qi-agents/base:${SHORT_SHA}'
+
+options:
+  env:
+    - 'DOCKER_BUILDKIT=1'
+  logging: CLOUD_LOGGING_ONLY
+  machineType: 'E2_HIGHCPU_8'
+
+images:
+  - 'us-east1-docker.pkg.dev/${PROJECT_ID}/qi-agents/base:latest'
+  - 'us-east1-docker.pkg.dev/${PROJECT_ID}/qi-agents/base:${SHORT_SHA}'

cloudbuild.yaml

@@ -0,0 +1,55 @@
+steps:
+# Setup character files
+- name: 'gcr.io/cloud-builders/gcloud'
+  entrypoint: 'bash'
+  args:
+  - '-c'
+  - |
+    echo "Setting up deployment for ${_DEPLOYMENT_ID}"
+    mkdir -p /app/characters
+    gsutil cp gs://qi-agents-service/${_DEPLOYMENT_ID}/*.character.json /app/characters/
+
+# Create or update instance with random zone selection
+- name: 'gcr.io/cloud-builders/gcloud'
+  entrypoint: 'bash'
+  args:
+  - '-c'
+  - |
+    ZONES=("a" "b" "c" "d" "f")
+    RANDOM_ZONE="${ZONES[$((RANDOM % ${#ZONES[@]}))]}"
+    ZONE="us-east1-$RANDOM_ZONE"
+
+    if [[ $(gcloud compute instances list --filter="name=${_DEPLOYMENT_ID}" --format="get(name)") ]]; then
+      echo "Updating existing instance ${_DEPLOYMENT_ID}"
+      gcloud compute instances update-container ${_DEPLOYMENT_ID} \
+        --container-image us-east1-docker.pkg.dev/${PROJECT_ID}/qi-agents/base:latest \
+        --container-mount-host-path mount-path=/app/characters,host-path=/app/characters \
+        --container-env BUCKET_PATH=${_DEPLOYMENT_ID} \
+        --zone $ZONE
+    else
+      echo "Creating new instance ${_DEPLOYMENT_ID}"
+      gcloud compute instances create-with-container ${_DEPLOYMENT_ID} \
+        --container-image us-east1-docker.pkg.dev/${PROJECT_ID}/qi-agents/base:latest \
+        --machine-type ${_MACHINE_TYPE} \
+        --zone $ZONE \
+        --network agents-vpc \
+        --subnet agents-subnet-primary \
+        --no-address \
+        --boot-disk-size 30GB \
+        --container-mount-host-path mount-path=/app/characters,host-path=/app/characters \
+        --container-env BUCKET_PATH=${_DEPLOYMENT_ID} \
+        --scopes=cloud-platform \
+        --service-account=${_SERVICE_ACCOUNT} \
+        --metadata-from-file startup-script=startup.sh \
+        --tags=agent-vm
+    fi
+
+substitutions:
+  _MACHINE_TYPE: e2-small
+  _SERVICE_ACCOUNT: qi-agents-service@qi-agents-as-a-service.iam.gserviceaccount.com
+  _DEPLOYMENT_ID: ""
+  _VERSION: ""
+
+options:
+  logging: CLOUD_LOGGING_ONLY
+  machineType: 'E2_HIGHCPU_8'

startup.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+set -x
+
+echo "Starting initialization script..."
+
+# Create data directory
+AGENT_DATA_DIR="/var/lib/qi-agents"
+echo "Creating data directory in $AGENT_DATA_DIR..."
+sudo mkdir -p "$AGENT_DATA_DIR/data" || {
+    echo "Failed to create data directory"
+    exit 1
+}
+
+# Set permissions
+echo "Setting directory permissions..."
+sudo chmod -R 777 "$AGENT_DATA_DIR" || {
+    echo "Failed to set permissions"
+    exit 1
+}
+
+# Get deployment ID from metadata
+DEPLOYMENT_ID=$(curl -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/name")
+echo "Deployment ID: ${DEPLOYMENT_ID}"
+
+# Configure gcloud auth
+echo "Configuring gcloud auth..."
+gcloud auth configure-docker us-east1-docker.pkg.dev --quiet
+
+echo "Startup completed successfully"