diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 17ede21..1810b77 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -15,14 +15,13 @@ jobs: strategy: matrix: couchbase-version: - - enterprise-6.6.0 - enterprise-6.6.6 - community-7.0.2 - enterprise-7.0.5 - community-7.1.1 - - enterprise-7.1.5 + - enterprise-7.1.6 - community-7.2.2 - - enterprise-7.2.2 + - enterprise-7.2.3 steps: - uses: actions/checkout@v4 diff --git a/Dockerfile b/Dockerfile index f6ba3ae..785f98b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,44 +1,40 @@ -ARG COUCHBASE_TAG=community-6.5.1 -FROM couchbase:${COUCHBASE_TAG} - -# Configure apt-get for NodeJS -# Install NPM and NodeJS and jq, with apt-get cleanup -RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - && \ - apt-get install -yq build-essential nodejs jq && \ - apt-get autoremove && apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Upgrade to jq 1.6 -RUN wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && \ - chmod +x jq-linux64 && \ - mv jq-linux64 $(which jq) - -# Install couchbase-index-manager -RUN mkdir -p /npm-packages && \ - npm config set prefix /npm-packages && \ - npm install -g --unsafe-perm couchbase-index-manager-cli@2.0.0 && \ - rm -rf /tmp/* /var/tmp/* -ENV PATH="/npm-packages/bin:$PATH" +ARG COUCHBASE_TAG=enterprise-7.2.3 +FROM btburnett3/couchbase-quickinit:${COUCHBASE_TAG} as base + +# Install Node 12 because fakeit isn't compatible with newer versions +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash && \ + rm -f $HOME/.npmrc +ENV NVM_DIR=/root/.nvm +RUN \. $NVM_DIR/nvm.sh && \ + nvm install 12.22.12 + + + +FROM base as restore + +# Install build essentials in a separate stage because we only need it during restore +RUN apt-get update && \ + apt-get install build-essential -y # Copy package.json -WORKDIR /scripts +WORKDIR /fakeit COPY ./scripts/package*.json ./ # Install fakeit -RUN npm ci && \ - rm -rf /tmp/* /var/tmp/* +RUN \. $NVM_DIR/nvm.sh && \ + nvm use 12.22.12 && \ + npm ci -# Copy startup scripts -COPY ./scripts/ /scripts/ -COPY ./startup/ /startup/ -# Configure default environment -ENV CB_DATARAM=512 CB_INDEXRAM=256 CB_SEARCHRAM=256 CB_ANALYTICSRAM=1024 CB_EVENTINGRAM=256 \ - CB_SERVICES=kv,n1ql,index,fts CB_INDEXSTORAGE=forestdb \ - CB_USERNAME=Administrator CB_PASSWORD=password \ - FAKEIT_BUCKETTIMEOUT=5000 -RUN mkdir /nodestatus -VOLUME /nodestatus +FROM base as final + +# Copy node_modules +COPY --from=restore /fakeit/node_modules/ /fakeit/node_modules/ -ENTRYPOINT ["./configure-node.sh"] +# Copy startup scripts +COPY ./scripts/ /fakeit/ +RUN echo "/fakeit/run-fakeit.sh" >> /scripts/additional-init.sh + +# Configure default environment +ENV FAKEIT_BUCKETTIMEOUT=5000 diff --git a/README.md b/README.md index 1e5ec9b..7dd7181 100644 --- a/README.md +++ b/README.md @@ -6,17 +6,17 @@ couchbasefakeit is a Docker image designed for testing and local development. I This can be very useful in reducing developer friction, providing a way to quickly and easily spin up a Couchbase server preinitialized for your application. By including an Dockerfile and associated configuration files within your source control repo, you can version your development data definitions along with your application. +> **NOTE:** couchbasefakeit is built on top of [couchbase-quickinit](https://github.com/brantburnett/couchbase-quickinit), adding support for data generation using FakeIt. If you do not require data generation, simply use `couchbase-quickinit` directly. + ## Pulling The latest version can be pulled using: ```sh -docker pull btburnett3/couchbasefakeit:latest +docker pull btburnett3/couchbasefakeit:enterprise-7.2.3 ``` -The `latest` tag will be the latest Enterprise edition of Couchbase, with the latest release of FakeIt. - -Specific versions may also be available, such as `enterprise-4.6.3`. This would be the Enterprise edition of Couchbase, version 4.6.3. +Other may also be available, such as `community-7.0.2`. This would be the Community edition of Couchbase, version 7.0.2. ## Using couchbasefakeit diff --git a/example/docker-compose.yml b/example/docker-compose.yml index ff6cd82..daef1a7 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: couchbase: - image: btburnett3/couchbasefakeit:enterprise-7.0.3 + image: btburnett3/couchbasefakeit:enterprise-7.2.3 environment: - CB_SERVICES=kv,n1ql,index,fts,eventing,cbas volumes: diff --git a/scripts/configure-analytics.sh b/scripts/configure-analytics.sh deleted file mode 100755 index 384af3a..0000000 --- a/scripts/configure-analytics.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -bucketName=$1 - -# Run Analytics Dataset scripts - -if [ -e "/startup/$bucketName/analytics/dataset.json" ]; then - echo "Creating datasets on $bucketName..." - - while read statement - do - echo "Dataset Statement: $statement" - curl -Ss -u $CB_USERNAME:$CB_PASSWORD http://localhost:8095/analytics/service --data-urlencode "statement=$statement" - done < <(cat /startup/$bucketName/analytics/dataset.json | jq -r '.statements | .[]') -fi - -# Run Analytics index scripts - -if [ -e "/startup/$bucketName/analytics/indexes.json" ]; then - echo "Building analytics indexes on $bucketName..." - - while read statement - do - echo "Index Statement: $statement" - curl -Ss -u $CB_USERNAME:$CB_PASSWORD http://localhost:8095/analytics/service --data-urlencode "statement=$statement" - done < <(cat /startup/$bucketName/analytics/indexes.json | jq -r '.statements | .[]') -fi diff --git a/scripts/configure-node.sh b/scripts/configure-node.sh deleted file mode 100755 index 0b9ecd0..0000000 --- a/scripts/configure-node.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -set -m - -/entrypoint.sh couchbase-server & - -if [ ! -e "/nodestatus/initialized" ] ; then - export CB_VERSION=$(cat /opt/couchbase/VERSION.txt | grep -o "^[0-9]*\.[0-9]*\.[0-9]*") - echo "CB_VERSION=$CB_VERSION" - - scriptPath=$(dirname $(realpath $0)) - - $scriptPath/init-node.sh - $scriptPath/wait-for-services.sh - $scriptPath/create-buckets.sh - $scriptPath/run-fakeit.sh - - while read bucketName - do - $scriptPath/create-views.sh $bucketName - $scriptPath/create-n1ql-indexes.sh $bucketName - $scriptPath/create-fts-indexes.sh $bucketName - $scriptPath/configure-analytics.sh $bucketName - done < <(cat /startup/buckets.json | jq -r '.[].name') - - $scriptPath/create-events.sh - $scriptPath/create-rbac-users.sh - - # Done - echo "Couchbase Server initialized." - echo "Initialized `date +"%D %T"`" > /nodestatus/initialized -else - echo "Couchbase Server already initialized." -fi - -# Wait for Couchbase Server shutdown -fg 1 diff --git a/scripts/create-buckets.sh b/scripts/create-buckets.sh deleted file mode 100755 index e4ff10f..0000000 --- a/scripts/create-buckets.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Create buckets defined in /startup/buckets.json -echo "Creating buckets..." - -while read bucketSettings -do - curl -Ss http://127.0.0.1:8091/pools/default/buckets -u "$CB_USERNAME:$CB_PASSWORD" $bucketSettings -done < <(cat /startup/buckets.json | jq -r '.[] | to_entries | map(.key + "=" + (.value | tostring)) | @sh "-d " + join(" -d ")') - -# Wait for the buckets to be healthy -bucketCount=$(cat /startup/buckets.json | jq -r '.[].name' | wc -l) -until [ `curl -Ss http://127.0.0.1:8091/pools/default/buckets -u $CB_USERNAME:$CB_PASSWORD | \ - jq -r .[].nodes[].status | grep '^healthy$' | wc -l` -eq $bucketCount ]; -do - echo "Waiting for bucket initialization..." - sleep 1 -done - -echo "Couchbase Version: $CB_VERSION" -if [[ $CB_VERSION > "6." ]]; then - echo "About to create scopes and collections..." - - scriptPath=$(dirname $(realpath $0)) - # Scopes and collections need created after a bucket is up and running - while read bucketName - do - $scriptPath/create-collections.sh $bucketName - done < <(cat /startup/buckets.json | jq -r '.[].name') -fi diff --git a/scripts/create-collections.sh b/scripts/create-collections.sh deleted file mode 100755 index 030fbca..0000000 --- a/scripts/create-collections.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -bucketName=$1 - -if [ -e "/startup/$bucketName/collections.json" ]; then - echo "Creating scopes and collections in $bucketName..." - - # Create the scopes and collections - while read scopeName - do - echo "Scope: $scopeName" - # Check to see if the scope name provided is the default scope. If not, create the scope - if [ $scopeName != "_default" ]; then - echo "Creating scope: $scopeName..." - curl -Ss -X POST -u "$CB_USERNAME:$CB_PASSWORD" http://localhost:8091/pools/default/buckets/$bucketName/scopes -d name=$scopeName && echo - fi - - # Create the collections within the appropriate scope - while read collectionName - do - echo "Creating collection: $collectionName..." - curl -Ss -X POST -u "$CB_USERNAME:$CB_PASSWORD" http://localhost:8091/pools/default/buckets/$bucketName/scopes/$scopeName/collections -d name=$collectionName && echo - done < <(cat /startup/$bucketName/collections.json | jq -r ".scopes.$scopeName | .collections | .[]") - done < <(cat /startup/$bucketName/collections.json | jq -r '.scopes | keys[]') -fi diff --git a/scripts/create-events.sh b/scripts/create-events.sh deleted file mode 100755 index 71f12f1..0000000 --- a/scripts/create-events.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Create and deploy events - -if [ -e "/startup/events/" ]; then - echo "Creating events..." - - for filename in /startup/$bucketName/events/*.json; do - eventName=$(basename $filename .json) - - cat event_template.json | - jq -c ".appname = \"$eventName\"" | - jq --argfile source $filename -c ".depcfg = \$source.depcfg" | - jq --argfile source $filename -c ".settings = .settings * \$source.settings" | - jq --rawfile source /startup/events/$eventName.js -c ".appcode = \$source" | - curl -Ss -X POST -u "$CB_USERNAME:$CB_PASSWORD" -H "Content-Type: application/json" \ - -d @- http://127.0.0.1:8096/api/v1/functions/$eventName \ - && echo - done -fi diff --git a/scripts/create-fts-indexes.sh b/scripts/create-fts-indexes.sh deleted file mode 100755 index 5e954ea..0000000 --- a/scripts/create-fts-indexes.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -bucketName=$1 - -# Create FTS indexes - -if [ -e "/startup/$bucketName/fts" ]; then - echo "Creating FTS indexes on $bucketName..." - - for filename in /startup/$bucketName/fts/*.json; do - indexName=$(basename $filename .json) - - if [ "$indexName" != "aliases" ]; then - echo "Creating FTS index $indexName..." - - # Replace the bucket name and index name - # and strip the UUIDs from the file before creating the index - - cat $filename | - jq -c ".name = \"$indexName\" | .sourceName = \"$bucketName\" | del(.uuid) | del(.sourceUUID)" | - curl -Ss -X PUT -u "$CB_USERNAME:$CB_PASSWORD" -H "Content-Type: application/json" \ - -d @- http://127.0.0.1:8094/api/index/$indexName - fi - done - - # Create FTS aliases - - if [ -e "/startup/$bucketName/fts/aliases.json" ]; then - while read aliasPair - do - # aliasPair will have form key=["value1","value2"], where key is alias name and values are target indexes - # So make aliasPair an array split by = - aliasPair=(${aliasPair//=/ }) - - jsonBody=$(jq -c ".name = \"${aliasPair[0]}\"" /scripts/fts_alias_template.json) - - # Add a target for each one in the JSON array on the right side of the pair - while read -r target; do - jsonBody=$(echo $jsonBody | jq -c ".params.targets.$target = {}") - done < <(echo ${aliasPair[1]} | jq -r ".[]") - - echo "Creating FTS alias ${aliasPair[0]}..." - - echo $jsonBody | - curl -Ss -X PUT -u "$CB_USERNAME:$CB_PASSWORD" -H "Content-Type: application/json" \ - -d @- http://127.0.0.1:8094/api/index/${aliasPair[0]} - done < <(cat /startup/$bucketName/fts/aliases.json | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]") - fi -fi diff --git a/scripts/create-n1ql-indexes.sh b/scripts/create-n1ql-indexes.sh deleted file mode 100755 index 0324910..0000000 --- a/scripts/create-n1ql-indexes.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -bucketName=$1 - -# Run index scripts - -if [ -e "/startup/$bucketName/indexes.n1ql" ]; then - echo "Building indexes on $bucketName..." - - /opt/couchbase/bin/cbq -e http://127.0.0.1:8093/ -u $CB_USERNAME -p $CB_PASSWORD -q=true -f="/startup/$bucketName/indexes.n1ql" - - # Wait for index build completion - until [ `/opt/couchbase/bin/cbq -e http://127.0.0.1:8093/ -u $CB_USERNAME -p $CB_PASSWORD -q=true \ - -s="SELECT COUNT(*) as unbuilt FROM system:indexes WHERE keyspace_id = '$bucket' AND state <> 'online'" | \ - sed -n -e '/{/,$p' | \ - jq -r '.results[].unbuilt'` -eq 0 ]; - do - echo "Waiting for index build on $bucketName..." - sleep 2 - done -fi - -# Also check for couchbase-index-manager YAML index definitions - -if [ -e "/startup/$bucketName/indexes/" ]; then - echo "Building indexes on $bucketName..." - - couchbase-index-manager \ - -u $CB_USERNAME -p $CB_PASSWORD \ - sync -f $bucketName /startup/$bucketName/indexes/ -fi diff --git a/scripts/create-rbac-users.sh b/scripts/create-rbac-users.sh deleted file mode 100755 index 34fcf43..0000000 --- a/scripts/create-rbac-users.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Create rbac users - -if [ -e "/startup/rbac-users.json" ]; then - while read rbacSettings - do - rbacName=$(echo $rbacSettings | jq -r '.["rbacName"]') - rbacUsername=$(echo $rbacSettings | jq -r '.["rbacUsername"]') - rbacPassword=$(echo $rbacSettings | jq -r '.["rbacPassword"]') - rbacRoles=$(echo $rbacSettings | jq -r '.roles | join(",")') - - curl -Ss -X PUT http://127.0.0.1:8091/settings/rbac/users/local/$rbacUsername \ - -u $CB_USERNAME:$CB_PASSWORD \ - -d password=$rbacPassword \ - -d name="$rbacName" \ - -d roles=$rbacRoles && echo - done < <(cat /startup/rbac-users.json | jq -c '.[]') -fi diff --git a/scripts/create-views.sh b/scripts/create-views.sh deleted file mode 100755 index 4b992af..0000000 --- a/scripts/create-views.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Create view design documents - -bucketName=$1 - -if [ -e "/startup/$bucketName/views.json" ]; then - echo "Building views on $bucketName..." - - while read designDocName - do - cat /startup/$bucketName/views.json | - jq -r ".$designDocName" | - curl -Ss -X PUT -u "$CB_USERNAME:$CB_PASSWORD" -H "Content-Type: application/json" \ - -d @- http://127.0.0.1:8092/$bucketName/_design/$designDocName - done < <(cat /startup/$bucketName/views.json | jq -r 'keys[]') -fi diff --git a/scripts/event_template.json b/scripts/event_template.json deleted file mode 100644 index 5bd17fb..0000000 --- a/scripts/event_template.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "appcode": "", - "depcfg": { - }, - "appname": "", - "settings": { - "dcp_stream_boundary": "everything", - "deadline_timeout": 62, - "deployment_status": true, - "description": "", - "execution_timeout": 60, - "log_level": "DEBUG", - "n1ql_consistency": "none", - "processing_status": true, - "user_prefix": "eventing", - "worker_count": 1 - } -} diff --git a/scripts/fts_alias_template.json b/scripts/fts_alias_template.json deleted file mode 100644 index 8a67fda..0000000 --- a/scripts/fts_alias_template.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "", - "type": "fulltext-alias", - "sourceType": "nil", - "sourceName": "", - "params": { - "targets": { - } - } -} diff --git a/scripts/init-node.sh b/scripts/init-node.sh deleted file mode 100755 index 3338ae1..0000000 --- a/scripts/init-node.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -echo "Initializing Couchbase Server $CB_VERSION..." - -if [[ $CB_VERSION > "5." && $CB_INDEXSTORAGE == "forestdb" ]]; then - # Couchbase 5.0 and later doesn't support forestdb, switch to plasma instead - export CB_INDEXSTORAGE=plasma - - echo "Switching from forestdb to plasma for index storage." -fi - -sleep 5 - -# Configure cluster, first request may need more time so retry -for attempt in $(seq 60) -do - echo "Cluster name: $CB_CLUSTER_NAME" - - # Change the name of the cluster if the user provided a cluster name - if [[ ! -z $CB_CLUSTER_NAME ]]; then - echo "Changing cluster name to $CB_CLUSTER_NAME..." - curl -Ss -X POST http://127.0.0.1:8091/pools/default -d clusterName=$CB_CLUSTER_NAME && echo - fi - - curl -Ss -X POST http://127.0.0.1:8091/pools/default \ - -d memoryQuota=$CB_DATARAM \ - -d indexMemoryQuota=$CB_INDEXRAM \ - -d ftsMemoryQuota=$CB_SEARCHRAM \ - -d cbasMemoryQuota=$CB_ANALYTICSRAM \ - -d eventingMemoryQuota=$CB_EVENTINGRAM \ - && echo \ - && break - - echo "Waiting for Couchbase startup..." - sleep 1 -done - -curl -Ss http://127.0.0.1:8091/node/controller/setupServices -d services=${CB_SERVICES//,/%2C} && echo -curl -Ss http://127.0.0.1:8091/settings/indexes -d storageMode=$CB_INDEXSTORAGE && echo -curl -Ss http://127.0.0.1:8091/settings/web -d port=8091 -d "username=$CB_USERNAME" -d "password=$CB_PASSWORD" && echo diff --git a/scripts/run-fakeit.sh b/scripts/run-fakeit.sh index bbcb083..b347f45 100755 --- a/scripts/run-fakeit.sh +++ b/scripts/run-fakeit.sh @@ -1,5 +1,8 @@ #!/bin/bash +\. $NVM_DIR/nvm.sh +nvm use 12.22.12 + # Run fakeit while read bucketName do @@ -12,13 +15,13 @@ do do # Try and create a document to see if the bucket is initialized, try again if it errored # If the bucket isn't initialized fakeit will error - if node /scripts/is-the-bucket-ready.js $bucketName $CB_USERNAME $CB_PASSWORD $CB_VERSION; then + if node /fakeit/is-the-bucket-ready.js $bucketName $CB_USERNAME $CB_PASSWORD $CB_VERSION; then if [[ $CB_VERSION < "5." ]]; then - /scripts/node_modules/.bin/fakeit couchbase \ + /fakeit/node_modules/.bin/fakeit couchbase \ --bucket "$bucketName" --timeout $FAKEIT_BUCKETTIMEOUT \ "/startup/$bucketName/models" else - /scripts/node_modules/.bin/fakeit couchbase \ + /fakeit/node_modules/.bin/fakeit couchbase \ --bucket "$bucketName" -u "$CB_USERNAME" -p "$CB_PASSWORD" --timeout $FAKEIT_BUCKETTIMEOUT \ "/startup/$bucketName/models" fi @@ -39,3 +42,5 @@ do done fi done < <(cat /startup/buckets.json | jq -r '.[].name') + +nvm use system diff --git a/scripts/wait-for-services.sh b/scripts/wait-for-services.sh deleted file mode 100755 index 2d773b5..0000000 --- a/scripts/wait-for-services.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -if [[ $CB_SERVICES == *"n1ql"* ]]; then - # Wait for the query service to be up and running - for attempt in $(seq 10) - do - curl -s http://127.0.0.1:8093/admin/ping > /dev/null \ - && break - - echo "Waiting for query service..." - sleep 1 - done - - # We're seeing sporadic issues with "Operation not supported" creating indexes - # As painful as it is, an extra sleep is called for to make sure N1QL is fully up and running - sleep 5 -fi - -if [[ $CB_SERVICES == *"fts"* ]]; then - # Wait for the FTS service to be up and running - for attempt in $(seq 10) - do - curl -s -u $CB_USERNAME:$CB_PASSWORD http://127.0.0.1:8094/api/index > /dev/null \ - && break - - echo "Waiting for FTS service..." - sleep 1 - done -fi - -if [[ $CB_SERVICES == *"cbas"* ]]; then - # Wait for the analytics service to be up and running - for attempt in $(seq 10) - do - curl -s -u $CB_USERNAME:$CB_PASSWORD http://localhost:8095/analytics/config/service > /dev/null \ - && break - - echo "Waiting for analytics service..." - sleep 1 - done -fi - -if [[ $CB_SERVICES == *"eventing"* ]]; then - # Wait for the eventing service to be up and running - for attempt in $(seq 10) - do - curl -s -u $CB_USERNAME:$CB_PASSWORD http://127.0.0.1:8096/api/v1/functions > /dev/null \ - && break - - echo "Waiting for eventing service..." - sleep 1 - done -fi diff --git a/startup/buckets.json b/startup/buckets.json deleted file mode 100644 index 62f9661..0000000 --- a/startup/buckets.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "name": "default", - "ramQuotaMB": 100, - "bucketType": "couchbase", - "authType": "sasl", - "saslPassword": "", - "evictionPolicy": "valueOnly", - "replicaNumber": 0, - "flushEnabled": 1 - } -]