wip

RareCompute · Nov 25, 2024 · 83561fd · 83561fd
1 parent 8c6e565
commit 83561fd
Show file tree

Hide file tree

Showing 4 changed files with 112 additions and 89 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1,2 @@
+# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#about-code-owners
+* @Liana64
diff --git a/.github/workflows/qlora.yaml b/.github/workflows/qlora.yaml
@@ -22,8 +22,14 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Set repository owner
+        id: set_repo_owner_lower
+        shell: bash
+        run: |
+          echo "REPOSITORY_OWNER=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
+
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
         with:
           platforms: arm64,amd64
 
@@ -39,7 +45,7 @@ jobs:
         uses: docker/metadata-action@v4
         with:
           images: |
-            ${{ env.REGISTRY }}/${{ github.actor }}/${{ env.IMAGE_NAME }}
+            ${{ env.REGISTRY }}/${{ env.REPOSITORY_OWNER }}/${{ env.IMAGE_NAME }}
           flavor: latest=auto
           tags: type=raw,value=latest,enable={{is_default_branch}}
 
@@ -52,7 +58,7 @@ jobs:
           push: true
           platforms: arm64,amd64
           context: ./${{ env.IMAGE_NAME }}/python3.11-slim
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.image_tag }}
+          tags: ${{ env.REGISTRY }}/${{ env.REPOSITORY_OWNER }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.image_tag }}
           build-args: BASE_IMAGE=python:3.11-slim
   build-qlora-cuda118:
     runs-on: ubuntu-latest
@@ -63,8 +69,14 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Set repository owner
+        id: set_repo_owner_lower
+        shell: bash
+        run: |
+          echo "REPOSITORY_OWNER=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
+
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
         with:
           platforms: arm64,amd64
 
@@ -80,7 +92,7 @@ jobs:
         uses: docker/metadata-action@v4
         with:
           images: |
-            ${{ env.REGISTRY }}/${{ github.actor }}/${{ env.IMAGE_NAME }}
+            ${{ env.REGISTRY }}/${{ env.REPOSITORY_OWNER }}/${{ env.IMAGE_NAME }}
           flavor: latest=auto
           tags: type=raw,value=latest,enable={{is_default_branch}}
 
@@ -93,87 +105,93 @@ jobs:
           push: true
           platforms: arm64,amd64
           context: ./${{ env.IMAGE_NAME }}/11.8.0-cudnn8-runtime-ubuntu22.04
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.image_tag }}-cu118
+          tags: ${{ env.REGISTRY }}/${{ env.REPOSITORY_OWNER }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.image_tag }}
           build-args: BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04
-  build-qlora-cuda121:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: arm64,amd64
-
-      - name: Login to the container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract image metadata
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: |
-            ${{ env.REGISTRY }}/${{ github.actor }}/${{ env.IMAGE_NAME }}
-          flavor: latest=auto
-          tags: type=raw,value=latest,enable={{is_default_branch}}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v6
-        with:
-          push: true
-          platforms: arm64,amd64
-          context: ./${{ env.IMAGE_NAME }}/12.1.0-cudnn8-runtime-ubuntu22.04
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.image_tag }}-cu121
-          build-args: BASE_IMAGE=nvidia/cuda:12.1.0-cudnn8-runtime-ubuntu22.04
-  build-qlora-cuda124:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: arm64,amd64
-
-      - name: Login to the container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract image metadata
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: |
-            ${{ env.REGISTRY }}/${{ github.actor }}/${{ env.IMAGE_NAME }}
-          flavor: latest=auto
-          tags: type=raw,value=latest,enable={{is_default_branch}}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v6
-        with:
-          push: true
-          platforms: arm64,amd64
-          context: ./${{ env.IMAGE_NAME }}/12.4.1-cudnn-runtime-ubuntu22.04
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.image_tag }}-cu124
-          build-args: BASE_IMAGE=nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
+  #  build-qlora-cuda121:
+  #    runs-on: ubuntu-latest
+  #    permissions:
+  #      contents: read
+  #      packages: write
+  #    steps:
+  #      - name: Checkout repository
+  #        uses: actions/checkout@v4
+  #
+  #      #      - name: Convert repository name to lowercase
+  #      #        run: echo "IMAGE_NAME=$(echo ${GITHUB_REPOSITORY} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
+  #
+  #      - name: Set up QEMU
+  #        uses: docker/setup-qemu-action@v3
+  #        with:
+  #          platforms: arm64,amd64
+  #
+  #      - name: Login to the container registry
+  #        uses: docker/login-action@v3
+  #        with:
+  #          registry: ${{ env.REGISTRY }}
+  #          username: ${{ github.actor }}
+  #          password: ${{ secrets.GITHUB_TOKEN }}
+  #
+  #      - name: Extract image metadata
+  #        id: meta
+  #        uses: docker/metadata-action@v4
+  #        with:
+  #          images: |
+  #            ${{ env.REGISTRY }}/${{ github.actor }}/${{ env.IMAGE_NAME }}
+  #          flavor: latest=auto
+  #          tags: type=raw,value=latest,enable={{is_default_branch}}
+  #
+  #      - name: Set up Docker Buildx
+  #        uses: docker/setup-buildx-action@v3
+  #
+  #      - name: Build and push Docker image
+  #        uses: docker/build-push-action@v6
+  #        with:
+  #          push: true
+  #          platforms: arm64,amd64
+  #          context: ./${{ env.IMAGE_NAME }}/12.1.0-cudnn8-runtime-ubuntu22.04
+  #          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.image_tag }}-cu121
+  #          build-args: BASE_IMAGE=nvidia/cuda:12.1.0-cudnn8-runtime-ubuntu22.04
+  #  build-qlora-cuda124:
+  #    runs-on: ubuntu-latest
+  #    permissions:
+  #      contents: read
+  #      packages: write
+  #    steps:
+  #      - name: Checkout repository
+  #        uses: actions/checkout@v4
+  #
+  #      #      - name: Convert repository name to lowercase
+  #      #        run: echo "IMAGE_NAME=$(echo ${GITHUB_REPOSITORY} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
+  #
+  #      - name: Set up QEMU
+  #        uses: docker/setup-qemu-action@v3
+  #        with:
+  #          platforms: arm64,amd64
+  #
+  #      - name: Login to the container registry
+  #        uses: docker/login-action@v3
+  #        with:
+  #          registry: ${{ env.REGISTRY }}
+  #          username: ${{ github.actor }}
+  #          password: ${{ secrets.GITHUB_TOKEN }}
+  #
+  #      - name: Extract image metadata
+  #        id: meta
+  #        uses: docker/metadata-action@v4
+  #        with:
+  #          images: |
+  #            ${{ env.REGISTRY }}/${{ github.actor }}/${{ env.IMAGE_NAME }}
+  #          flavor: latest=auto
+  #          tags: type=raw,value=latest,enable={{is_default_branch}}
+  #
+  #      - name: Set up Docker Buildx
+  #        uses: docker/setup-buildx-action@v3
+  #
+  #      - name: Build and push Docker image
+  #        uses: docker/build-push-action@v6
+  #        with:
+  #          push: true
+  #          platforms: arm64,amd64
+  #          context: ./${{ env.IMAGE_NAME }}/12.4.1-cudnn-runtime-ubuntu22.04
+  #          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.image_tag }}-cu124
+  #          build-args: BASE_IMAGE=nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
diff --git a/qlora/11.8.0-cudnn8-runtime-ubuntu22.04/Dockerfile b/qlora/11.8.0-cudnn8-runtime-ubuntu22.04/Dockerfile
@@ -3,7 +3,7 @@
 ARG PROJECT_NAME=qlora
 ARG PROJECT_URL=https://github.com/artidoro/qlora
 ARG BASE_IMAGE=nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04
-FROM ${BASE_IMAGE} AS builder
+FROM python:3.11-slim AS builder
 
 # Build
 # ----------------------

diff --git a/qlora/README.md b/qlora/README.md
@@ -0,0 +1,3 @@
+From [artidoro/qlora](https://github.com/artidoro/qlora):
+
+> We present QLoRA, an efficient finetuning approach that reduces memory usage enough to finetune a 65B parameter model on a single 48GB GPU while preserving full 16-bit finetuning task performance. QLoRA backpropagates gradients through a frozen, 4-bit quantized pretrained language model into Low Rank Adapters (LoRA). Our best model family, which we name Guanaco, outperforms all previous openly released models on the Vicuna benchmark, reaching 99.3% of the performance level of ChatGPT while only requiring 24 hours of finetuning on a single GPU. QLoRA introduces a number of innovations to save memory without sacrificing performance: (a) 4-bit NormalFloat (NF4), a new data type that is information theoretically optimal for normally distributed weights (b) Double Quantization to reduce the average memory footprint by quantizing the quantization constants, and (c) Paged Optimizers to manage memory spikes. We use QLoRA to finetune more than 1,000 models, providing a detailed analysis of instruction following and chatbot performance across 8 instruction datasets, multiple model types (LLaMA, T5), and model scales that would be infeasible to run with regular finetuning (e.g. 33B and 65B parameter models). Our results show that QLoRA finetuning on a small high-quality dataset leads to state-of-the-art results, even when using smaller models than the previous SoTA. We provide a detailed analysis of chatbot performance based on both human and GPT-4 evaluations showing that GPT-4 evaluations are a cheap and reasonable alternative to human evaluation. Furthermore, we find that current chatbot benchmarks are not trustworthy to accurately evaluate the performance levels of chatbots. We release all of our models and code, including CUDA kernels for 4-bit training.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#about-code-owners
		* @Liana64
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		From [artidoro/qlora](https://github.com/artidoro/qlora):

		> We present QLoRA, an efficient finetuning approach that reduces memory usage enough to finetune a 65B parameter model on a single 48GB GPU while preserving full 16-bit finetuning task performance. QLoRA backpropagates gradients through a frozen, 4-bit quantized pretrained language model into Low Rank Adapters (LoRA). Our best model family, which we name Guanaco, outperforms all previous openly released models on the Vicuna benchmark, reaching 99.3% of the performance level of ChatGPT while only requiring 24 hours of finetuning on a single GPU. QLoRA introduces a number of innovations to save memory without sacrificing performance: (a) 4-bit NormalFloat (NF4), a new data type that is information theoretically optimal for normally distributed weights (b) Double Quantization to reduce the average memory footprint by quantizing the quantization constants, and (c) Paged Optimizers to manage memory spikes. We use QLoRA to finetune more than 1,000 models, providing a detailed analysis of instruction following and chatbot performance across 8 instruction datasets, multiple model types (LLaMA, T5), and model scales that would be infeasible to run with regular finetuning (e.g. 33B and 65B parameter models). Our results show that QLoRA finetuning on a small high-quality dataset leads to state-of-the-art results, even when using smaller models than the previous SoTA. We provide a detailed analysis of chatbot performance based on both human and GPT-4 evaluations showing that GPT-4 evaluations are a cheap and reasonable alternative to human evaluation. Furthermore, we find that current chatbot benchmarks are not trustworthy to accurately evaluate the performance levels of chatbots. We release all of our models and code, including CUDA kernels for 4-bit training.