From 517731f022faa94d8894ec4d8a3282d791611bba Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Fri, 19 Jul 2024 13:00:35 +0300 Subject: [PATCH 01/31] after docker image is built by github pr, set env vars & run tests --- .github/workflows/docker-bulid.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/docker-bulid.yml b/.github/workflows/docker-bulid.yml index 58afe8cf7..ef94a81d8 100644 --- a/.github/workflows/docker-bulid.yml +++ b/.github/workflows/docker-bulid.yml @@ -21,4 +21,15 @@ jobs: push: false tags: assafelovic/gpt-researcher:latest file: Dockerfile + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest pytest-asyncio gpt-researcher + - name: Set environment variables + run: | + echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> $GITHUB_ENV + echo "TAVILY_API_KEY=${{ secrets.TAVILY_API_KEY }}" >> $GITHUB_ENV + - name: Run tests + run: | + pytest tests/all-6-report-types.py From c5245d74b3093cb8037686e03f3fafc64c8cd737 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Tue, 23 Jul 2024 19:39:04 +0300 Subject: [PATCH 02/31] tests --- .../{docker-bulid.yml => docker-build.yml} | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) rename .github/workflows/{docker-bulid.yml => docker-build.yml} (51%) diff --git a/.github/workflows/docker-bulid.yml b/.github/workflows/docker-build.yml similarity index 51% rename from .github/workflows/docker-bulid.yml rename to .github/workflows/docker-build.yml index ef94a81d8..995d41f63 100644 --- a/.github/workflows/docker-bulid.yml +++ b/.github/workflows/docker-build.yml @@ -2,34 +2,33 @@ name: GitHub Actions Workflow run-name: ${{ github.actor }} has started docker build workflow. on: pull_request: - types: [opened, edited, ready_for_review] + types: [opened, edited, ready_for_review, synchronize] jobs: docker: runs-on: ubuntu-latest steps: - name: Git checkout uses: actions/checkout@master + - name: Set up QEMU uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 with: driver: docker + - name: Build Dockerfile uses: docker/build-push-action@v4 with: push: false - tags: assafelovic/gpt-researcher:latest + tags: gptresearcher/gpt-researcher:latest file: Dockerfile - - name: Install dependencies + + - name: Run Docker container run: | - python -m pip install --upgrade pip - pip install pytest pytest-asyncio gpt-researcher - - name: Set environment variables - run: | - echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> $GITHUB_ENV - echo "TAVILY_API_KEY=${{ secrets.TAVILY_API_KEY }}" >> $GITHUB_ENV - - name: Run tests - run: | - pytest tests/all-6-report-types.py - + docker run --rm \ + -e OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} \ + -e TAVILY_API_KEY=${{ secrets.TAVILY_API_KEY }} \ + gptresearcher/gpt-researcher:latest \ + /bin/sh -c "pip install pytest pytest-asyncio && pytest tests/all-6-report-types.py" \ No newline at end of file From 81e2bfaec6e050b7e80ea0ff15cc9fe7150963a4 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Tue, 23 Jul 2024 19:47:30 +0300 Subject: [PATCH 03/31] fix --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 995d41f63..223fb6471 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -31,4 +31,4 @@ jobs: -e OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} \ -e TAVILY_API_KEY=${{ secrets.TAVILY_API_KEY }} \ gptresearcher/gpt-researcher:latest \ - /bin/sh -c "pip install pytest pytest-asyncio && pytest tests/all-6-report-types.py" \ No newline at end of file + /bin/sh -c "pip install --user pytest pytest-asyncio && PATH=$PATH:/root/.local/bin && pytest tests/all-6-report-types.py" \ No newline at end of file From 498b83f8d3998467ca6ef77a6b6d65314686cf9e Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Wed, 24 Jul 2024 08:40:36 +0300 Subject: [PATCH 04/31] requirements.txt install test libs --- .github/workflows/docker-build.yml | 2 +- requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 223fb6471..4ea2de01d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -31,4 +31,4 @@ jobs: -e OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} \ -e TAVILY_API_KEY=${{ secrets.TAVILY_API_KEY }} \ gptresearcher/gpt-researcher:latest \ - /bin/sh -c "pip install --user pytest pytest-asyncio && PATH=$PATH:/root/.local/bin && pytest tests/all-6-report-types.py" \ No newline at end of file + /bin/sh -c "pytest tests/all-6-report-types.py" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ad2feb2eb..334902eb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,5 +28,5 @@ unstructured json_repair # uncomment for testing -# pytest -# pytest-asyncio \ No newline at end of file +pytest +pytest-asyncio \ No newline at end of file From dfd7b8ef089f45c71f6236d46367f14305a314fb Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Wed, 24 Jul 2024 08:55:59 +0300 Subject: [PATCH 05/31] pass secrets --- .github/workflows/docker-build.yml | 5 +++-- Dockerfile | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 4ea2de01d..14e111db3 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -24,11 +24,12 @@ jobs: push: false tags: gptresearcher/gpt-researcher:latest file: Dockerfile + build-args: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} - name: Run Docker container run: | docker run --rm \ - -e OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} \ - -e TAVILY_API_KEY=${{ secrets.TAVILY_API_KEY }} \ gptresearcher/gpt-researcher:latest \ /bin/sh -c "pytest tests/all-6-report-types.py" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 43a6e4143..9240c96cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,12 @@ RUN pip install -r multi_agents/requirements.txt FROM gpt-researcher-install AS gpt-researcher +ARG OPENAI_API_KEY +ARG TAVILY_API_KEY + +ENV OPENAI_API_KEY=${OPENAI_API_KEY} +ENV TAVILY_API_KEY=${TAVILY_API_KEY} + RUN useradd -ms /bin/bash gpt-researcher \ && chown -R gpt-researcher:gpt-researcher /usr/src/app From 53a6f3abeed33a9dce356c16bcb2469f7e507f46 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Wed, 24 Jul 2024 09:01:12 +0300 Subject: [PATCH 06/31] run tests with docker-compose --- .github/workflows/docker-build.yml | 17 ++++++++++++----- docker-compose.yml | 9 +++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 14e111db3..587fa4f4a 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -18,7 +18,7 @@ jobs: with: driver: docker - - name: Build Dockerfile + - name: Build Docker images uses: docker/build-push-action@v4 with: push: false @@ -28,8 +28,15 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} - - name: Run Docker container + - name: Set up Docker Compose run: | - docker run --rm \ - gptresearcher/gpt-researcher:latest \ - /bin/sh -c "pytest tests/all-6-report-types.py" \ No newline at end of file + sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + + - name: Run tests with Docker Compose + run: | + docker-compose run --rm gpt-researcher-tests + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} + LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1bdc8b6f9..9c7ebd179 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,3 +26,12 @@ services: restart: always ports: - 3000:3000 + + gpt-researcher-tests: + image: gptresearcher/gpt-researcher + build: ./ + environment: + OPENAI_API_KEY: ${OPENAI_API_KEY} + TAVILY_API_KEY: ${TAVILY_API_KEY} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + command: /bin/sh -c "pytest tests/all-6-report-types.py" \ No newline at end of file From 4b67c8270ddf8d42f65f1ce7f237be84da55fe4b Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Wed, 24 Jul 2024 09:07:25 +0300 Subject: [PATCH 07/31] compose --- docker-compose.yml | 79 ++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9c7ebd179..587fa4f4a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,37 +1,42 @@ -version: '3' -services: - gpt-researcher: - pull_policy: build - image: gptresearcher/gpt-researcher - build: ./ - environment: - OPENAI_API_KEY: ${OPENAI_API_KEY} - TAVILY_API_KEY: ${TAVILY_API_KEY} - LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} - restart: always - ports: - - 8000:8000 - gptr-nextjs: - pull_policy: build - image: gptresearcher/gptr-nextjs - stdin_open: true - environment: - - CHOKIDAR_USEPOLLING=true - build: - dockerfile: Dockerfile.dev - context: multi_agents/frontend - volumes: - - /app/node_modules - - ./multi_agents/frontend:/app - restart: always - ports: - - 3000:3000 - - gpt-researcher-tests: - image: gptresearcher/gpt-researcher - build: ./ - environment: - OPENAI_API_KEY: ${OPENAI_API_KEY} - TAVILY_API_KEY: ${TAVILY_API_KEY} - LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} - command: /bin/sh -c "pytest tests/all-6-report-types.py" \ No newline at end of file +name: GitHub Actions Workflow +run-name: ${{ github.actor }} has started docker build workflow. +on: + pull_request: + types: [opened, edited, ready_for_review, synchronize] +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Git checkout + uses: actions/checkout@master + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + driver: docker + + - name: Build Docker images + uses: docker/build-push-action@v4 + with: + push: false + tags: gptresearcher/gpt-researcher:latest + file: Dockerfile + build-args: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} + + - name: Set up Docker Compose + run: | + sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + + - name: Run tests with Docker Compose + run: | + docker-compose run --rm gpt-researcher-tests + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} + LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }} \ No newline at end of file From 7c038ee6cb6778cb65bde6b0a13a2769c934329d Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Wed, 24 Jul 2024 09:10:59 +0300 Subject: [PATCH 08/31] cr --- .github/workflows/docker-build.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 587fa4f4a..4e9646adc 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -24,10 +24,7 @@ jobs: push: false tags: gptresearcher/gpt-researcher:latest file: Dockerfile - build-args: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} - + - name: Set up Docker Compose run: | sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose From 173bdbcd048561a239d3bd920dea25f37bcd95e6 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Wed, 24 Jul 2024 09:12:13 +0300 Subject: [PATCH 09/31] compose --- .github/workflows/docker-build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 4e9646adc..86ba583e6 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -18,12 +18,12 @@ jobs: with: driver: docker - - name: Build Docker images - uses: docker/build-push-action@v4 - with: - push: false - tags: gptresearcher/gpt-researcher:latest - file: Dockerfile + # - name: Build Docker images + # uses: docker/build-push-action@v4 + # with: + # push: false + # tags: gptresearcher/gpt-researcher:latest + # file: Dockerfile - name: Set up Docker Compose run: | From 7b07d8c243fa390f868ee1579ec1431552e88707 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Fri, 16 Aug 2024 10:59:05 +0000 Subject: [PATCH 10/31] type definitions for nextjs --- application/next-env.d.ts | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 application/next-env.d.ts diff --git a/application/next-env.d.ts b/application/next-env.d.ts new file mode 100644 index 000000000..4f11a03dc --- /dev/null +++ b/application/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/basic-features/typescript for more information. From 1eb7d3e20e62038d38d215313e72a91e4d538330 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Fri, 16 Aug 2024 11:14:42 +0000 Subject: [PATCH 11/31] docker-compose fix --- docker-compose.yml | 79 ++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 587fa4f4a..518e4921c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,42 +1,37 @@ -name: GitHub Actions Workflow -run-name: ${{ github.actor }} has started docker build workflow. -on: - pull_request: - types: [opened, edited, ready_for_review, synchronize] -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Git checkout - uses: actions/checkout@master - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - driver: docker - - - name: Build Docker images - uses: docker/build-push-action@v4 - with: - push: false - tags: gptresearcher/gpt-researcher:latest - file: Dockerfile - build-args: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} - - - name: Set up Docker Compose - run: | - sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - - - name: Run tests with Docker Compose - run: | - docker-compose run --rm gpt-researcher-tests - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} - LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }} \ No newline at end of file +version: '3' +services: + gpt-researcher: + pull_policy: build + image: gptresearcher/gpt-researcher + build: ./ + environment: + OPENAI_API_KEY: ${OPENAI_API_KEY} + TAVILY_API_KEY: ${TAVILY_API_KEY} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + restart: always + ports: + - 8000:8000 + gptr-nextjs: + pull_policy: build + image: gptresearcher/gptr-nextjs + stdin_open: true + environment: + - CHOKIDAR_USEPOLLING=true + build: + dockerfile: Dockerfile.dev + context: application + volumes: + - /app/node_modules + - ./application:/app + restart: always + ports: + - 3000:3000 + + gpt-researcher-tests: + image: gptresearcher/gpt-researcher + build: ./ + environment: + OPENAI_API_KEY: ${OPENAI_API_KEY} + TAVILY_API_KEY: ${TAVILY_API_KEY} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + command: /bin/sh -c "pytest tests/all-6-report-types.py" \ No newline at end of file From 3a8b066a232de1d8428f77d7bf8ead8b5fede480 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Fri, 16 Aug 2024 11:18:40 +0000 Subject: [PATCH 12/31] automated tests should only run: (a) When the PR is created (b) When a commit is made to an open PR --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 86ba583e6..4b3ca3f9b 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -2,7 +2,7 @@ name: GitHub Actions Workflow run-name: ${{ github.actor }} has started docker build workflow. on: pull_request: - types: [opened, edited, ready_for_review, synchronize] + types: [opened, synchronize] jobs: docker: runs-on: ubuntu-latest From 61cf2a3049a414f9604eedf2e2f3a85891d76871 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Fri, 16 Aug 2024 11:33:07 +0000 Subject: [PATCH 13/31] are keys on main repo? --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 4b3ca3f9b..96837ff18 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -23,7 +23,7 @@ jobs: # with: # push: false # tags: gptresearcher/gpt-researcher:latest - # file: Dockerfile + # file: Dockerfile - name: Set up Docker Compose run: | From 5ea880f0727d0194bd375358f9d240639b0a9864 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Fri, 16 Aug 2024 11:47:24 +0000 Subject: [PATCH 14/31] pytest & pytest-asyncio moved out of requirements.txt into the docker-compose file - to only be installed for automated tests --- docker-compose.yml | 2 +- requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 518e4921c..d49abdc38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,4 +34,4 @@ services: OPENAI_API_KEY: ${OPENAI_API_KEY} TAVILY_API_KEY: ${TAVILY_API_KEY} LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} - command: /bin/sh -c "pytest tests/all-6-report-types.py" \ No newline at end of file + command: /bin/sh -c "pip install pytest pytest-asyncio && pytest tests/all-6-report-types.py" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 334902eb1..ad2feb2eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,5 +28,5 @@ unstructured json_repair # uncomment for testing -pytest -pytest-asyncio \ No newline at end of file +# pytest +# pytest-asyncio \ No newline at end of file From c5155e1c1104c1eed6cf33891ef7acc3e8cae31d Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Fri, 16 Aug 2024 11:57:33 +0000 Subject: [PATCH 15/31] ensure we're using the Python interpreter in automated tests to run pytest --- docker-compose.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d49abdc38..e059f44df 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,4 +34,8 @@ services: OPENAI_API_KEY: ${OPENAI_API_KEY} TAVILY_API_KEY: ${TAVILY_API_KEY} LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} - command: /bin/sh -c "pip install pytest pytest-asyncio && pytest tests/all-6-report-types.py" \ No newline at end of file + command: > + /bin/sh -c " + pip install pytest pytest-asyncio && + python -m pytest tests/all-6-report-types.py + " \ No newline at end of file From 471e3c2a803502910068e67bb2108567619f2dbf Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Fri, 16 Aug 2024 12:05:02 +0000 Subject: [PATCH 16/31] 3-report-types.py --- docker-compose.yml | 2 +- tests/{all-6-report-types.py => 3-report-types.py} | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) rename tests/{all-6-report-types.py => 3-report-types.py} (94%) diff --git a/docker-compose.yml b/docker-compose.yml index e059f44df..f16edade6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,5 +37,5 @@ services: command: > /bin/sh -c " pip install pytest pytest-asyncio && - python -m pytest tests/all-6-report-types.py + python -m pytest tests/3-report-types.py " \ No newline at end of file diff --git a/tests/all-6-report-types.py b/tests/3-report-types.py similarity index 94% rename from tests/all-6-report-types.py rename to tests/3-report-types.py index b938589f5..13b4d6df8 100644 --- a/tests/all-6-report-types.py +++ b/tests/3-report-types.py @@ -7,10 +7,7 @@ report_types = [ "research_report", "custom_report", - "subtopic_report", - "summary_report", - "detailed_report", - "quick_report" + "subtopic_report" ] # Define a common query and sources for testing From e1957fd129f54c2e04cb7ac2b0066b0c7717b20a Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Sun, 18 Aug 2024 07:30:08 +0000 Subject: [PATCH 17/31] tests env --- .github/workflows/docker-build.yml | 8 ++++++++ ollama/ollama/id_ed25519 | 7 +++++++ ollama/ollama/id_ed25519.pub | 1 + 3 files changed, 16 insertions(+) create mode 100644 ollama/ollama/id_ed25519 create mode 100644 ollama/ollama/id_ed25519.pub diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 96837ff18..8d60db289 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -4,6 +4,14 @@ on: pull_request: types: [opened, synchronize] jobs: + docker: + runs-on: ubuntu-latest + environment: tests # Specify the environment to use for this job + env: + # Ensure these environment variables are set for the entire job + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} + LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }} docker: runs-on: ubuntu-latest steps: diff --git a/ollama/ollama/id_ed25519 b/ollama/ollama/id_ed25519 new file mode 100644 index 000000000..a03c2503a --- /dev/null +++ b/ollama/ollama/id_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtz +c2gtZWQyNTUxOQAAACB8VSZQZJCtS6k1JqLharHSOiFk/DClAaHQTDoozBDbZgAA +AIiHWaTwh1mk8AAAAAtzc2gtZWQyNTUxOQAAACB8VSZQZJCtS6k1JqLharHSOiFk +/DClAaHQTDoozBDbZgAAAEDX0WO+D+FFjJ4INEye5RxMy/6YpzdyaoBDMIyc1W2H +9HxVJlBkkK1LqTUmouFqsdI6IWT8MKUBodBMOijMENtmAAAAAAECAwQF +-----END OPENSSH PRIVATE KEY----- diff --git a/ollama/ollama/id_ed25519.pub b/ollama/ollama/id_ed25519.pub new file mode 100644 index 000000000..c7d60c847 --- /dev/null +++ b/ollama/ollama/id_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHxVJlBkkK1LqTUmouFqsdI6IWT8MKUBodBMOijMENtm From 5c51a8d107360fb892998f81a72d39e2f2427c66 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Sun, 18 Aug 2024 07:41:46 +0000 Subject: [PATCH 18/31] env tests --- .github/workflows/docker-build.yml | 2 -- README.md | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 8d60db289..4d587237f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -12,8 +12,6 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }} - docker: - runs-on: ubuntu-latest steps: - name: Git checkout uses: actions/checkout@master diff --git a/README.md b/README.md index 635a00f62..97f5aa54e 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ The agent can produce detailed, factual and unbiased research reports, with customization options for focusing on relevant resources and outlines. Inspired by the recent [Plan-and-Solve](https://arxiv.org/abs/2305.04091) and [RAG](https://arxiv.org/abs/2005.11401) papers, GPT Researcher addresses issues of misinformation, speed, determinism and reliability, offering a more stable performance and increased speed through parallelized agent work, as opposed to synchronous operations. -**Our mission is to empower individuals and organizations with accurate, unbiased, and factual information by leveraging the power of AI.** +**Our mission is to empower individuals and organizations with accurate, unbiased, and factual information by leveraging the power of AI.** ## Why GPT Researcher? From 46bc71a24edc7e6d20b32387821770b248c4cbec Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Sun, 18 Aug 2024 08:03:20 +0000 Subject: [PATCH 19/31] ci fixes --- .github/workflows/docker-build.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 4d587237f..560ac634f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -3,6 +3,7 @@ run-name: ${{ github.actor }} has started docker build workflow. on: pull_request: types: [opened, synchronize] + jobs: docker: runs-on: ubuntu-latest @@ -14,32 +15,27 @@ jobs: LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }} steps: - name: Git checkout - uses: actions/checkout@master - + uses: actions/checkout@v3 + - name: Set up QEMU uses: docker/setup-qemu-action@v2 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 with: driver: docker - + # - name: Build Docker images # uses: docker/build-push-action@v4 # with: # push: false # tags: gptresearcher/gpt-researcher:latest - # file: Dockerfile - + # file: Dockerfile + - name: Set up Docker Compose run: | sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose - - name: Run tests with Docker Compose run: | - docker-compose run --rm gpt-researcher-tests - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} - LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }} \ No newline at end of file + docker-compose run --rm gpt-researcher-tests \ No newline at end of file From 822b68a9ce667781188f42d25ccbfec61a9b81af Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Sun, 18 Aug 2024 08:15:44 +0000 Subject: [PATCH 20/31] remove custom-report test for now --- tests/3-report-types.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/3-report-types.py b/tests/3-report-types.py index 13b4d6df8..6b676083d 100644 --- a/tests/3-report-types.py +++ b/tests/3-report-types.py @@ -6,7 +6,6 @@ # Define the report types to test report_types = [ "research_report", - "custom_report", "subtopic_report" ] From f538e0469a24034bba8d99b88cd1487543d0cabb Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Sun, 18 Aug 2024 12:57:21 +0000 Subject: [PATCH 21/31] 1 test --- tests/3-report-types.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/3-report-types.py b/tests/3-report-types.py index 6b676083d..0bd82576d 100644 --- a/tests/3-report-types.py +++ b/tests/3-report-types.py @@ -5,8 +5,7 @@ # Define the report types to test report_types = [ - "research_report", - "subtopic_report" + "research_report" ] # Define a common query and sources for testing From 6591184581dccbb8d4fd8ca6a8f46e3ff451af74 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Sun, 18 Aug 2024 14:23:24 +0000 Subject: [PATCH 22/31] tests --- tests/3-report-types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/3-report-types.py b/tests/3-report-types.py index 0bd82576d..e9086aaa5 100644 --- a/tests/3-report-types.py +++ b/tests/3-report-types.py @@ -10,6 +10,7 @@ # Define a common query and sources for testing query = "What are the latest advancements in AI?" +tone = sources = ["https://en.wikipedia.org/wiki/Artificial_intelligence", "https://www.ibm.com/watson/ai"] # Define the output directory From 2735dda7998cf082ec2e1a805e36cd46620b84ce Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Mon, 19 Aug 2024 01:16:12 +0000 Subject: [PATCH 23/31] verify test runs --- tests/3-report-types.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/3-report-types.py b/tests/3-report-types.py index e9086aaa5..0bd82576d 100644 --- a/tests/3-report-types.py +++ b/tests/3-report-types.py @@ -10,7 +10,6 @@ # Define a common query and sources for testing query = "What are the latest advancements in AI?" -tone = sources = ["https://en.wikipedia.org/wiki/Artificial_intelligence", "https://www.ibm.com/watson/ai"] # Define the output directory From 0f561a1d3eb2a82ce544134ca6e9e26d0634a8e1 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Mon, 19 Aug 2024 01:21:13 +0000 Subject: [PATCH 24/31] replicate working tests --- docker-compose.yml | 6 +----- requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f16edade6..f050b5e28 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,8 +34,4 @@ services: OPENAI_API_KEY: ${OPENAI_API_KEY} TAVILY_API_KEY: ${TAVILY_API_KEY} LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} - command: > - /bin/sh -c " - pip install pytest pytest-asyncio && - python -m pytest tests/3-report-types.py - " \ No newline at end of file + command: pytest -v tests/3-report-types.py \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ad2feb2eb..334902eb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,5 +28,5 @@ unstructured json_repair # uncomment for testing -# pytest -# pytest-asyncio \ No newline at end of file +pytest +pytest-asyncio \ No newline at end of file From d4cca6e0b8a5ee593c7c010eb55d57d81f09e4c1 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Mon, 19 Aug 2024 01:31:05 +0000 Subject: [PATCH 25/31] edit tests --- tests/3-report-types.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/3-report-types.py b/tests/3-report-types.py index 0bd82576d..2a42aa3ff 100644 --- a/tests/3-report-types.py +++ b/tests/3-report-types.py @@ -5,12 +5,13 @@ # Define the report types to test report_types = [ - "research_report" + "research_report", + "subtopic_report" ] # Define a common query and sources for testing query = "What are the latest advancements in AI?" -sources = ["https://en.wikipedia.org/wiki/Artificial_intelligence", "https://www.ibm.com/watson/ai"] +# sources = ["https://en.wikipedia.org/wiki/Artificial_intelligence", "https://www.ibm.com/watson/ai"] # Define the output directory output_dir = "./outputs" @@ -23,7 +24,7 @@ async def test_gpt_researcher(report_type): os.makedirs(output_dir) # Create an instance of GPTResearcher - researcher = GPTResearcher(query=query, report_type=report_type, source_urls=sources) + researcher = GPTResearcher(query=query, report_type=report_type) # Conduct research and write the report await researcher.conduct_research() From 492f9f0847bb8dfa3a2d4dbbc451fb6ceca4b327 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Mon, 19 Aug 2024 01:47:40 +0000 Subject: [PATCH 26/31] pytest install via docker - not requirements.txt --- docker-compose.yml | 6 +++++- requirements.txt | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f050b5e28..f16edade6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,4 +34,8 @@ services: OPENAI_API_KEY: ${OPENAI_API_KEY} TAVILY_API_KEY: ${TAVILY_API_KEY} LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} - command: pytest -v tests/3-report-types.py \ No newline at end of file + command: > + /bin/sh -c " + pip install pytest pytest-asyncio && + python -m pytest tests/3-report-types.py + " \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 334902eb1..ad2feb2eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,5 +28,5 @@ unstructured json_repair # uncomment for testing -pytest -pytest-asyncio \ No newline at end of file +# pytest +# pytest-asyncio \ No newline at end of file From f0745c6ad9b7f133fd0cd0a383b257bd7f27b043 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Mon, 19 Aug 2024 01:51:45 +0000 Subject: [PATCH 27/31] verify permissions --- .github/workflows/docker-build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 560ac634f..d230b8ef3 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -1,5 +1,8 @@ name: GitHub Actions Workflow run-name: ${{ github.actor }} has started docker build workflow. +permissions: + contents: read + pull-requests: write on: pull_request: types: [opened, synchronize] From 5b6cce6bb3392d048bf2b8460245035b8e27fd1f Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Mon, 19 Aug 2024 02:01:12 +0000 Subject: [PATCH 28/31] conditionally run tests only if env vars are set in github repo --- .github/workflows/docker-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index d230b8ef3..45ebd2a84 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -11,6 +11,7 @@ jobs: docker: runs-on: ubuntu-latest environment: tests # Specify the environment to use for this job + if: ${{ secrets.OPENAI_API_KEY && secrets.TAVILY_API_KEY }} env: # Ensure these environment variables are set for the entire job OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} From e6da3572d30e40202bf0cc7ea403f734bfd8cd65 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Mon, 19 Aug 2024 02:04:12 +0000 Subject: [PATCH 29/31] cleanup --- .github/workflows/docker-build.yml | 1 - ollama/ollama/id_ed25519 | 7 ------- ollama/ollama/id_ed25519.pub | 1 - 3 files changed, 9 deletions(-) delete mode 100644 ollama/ollama/id_ed25519 delete mode 100644 ollama/ollama/id_ed25519.pub diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 45ebd2a84..d230b8ef3 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -11,7 +11,6 @@ jobs: docker: runs-on: ubuntu-latest environment: tests # Specify the environment to use for this job - if: ${{ secrets.OPENAI_API_KEY && secrets.TAVILY_API_KEY }} env: # Ensure these environment variables are set for the entire job OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/ollama/ollama/id_ed25519 b/ollama/ollama/id_ed25519 deleted file mode 100644 index a03c2503a..000000000 --- a/ollama/ollama/id_ed25519 +++ /dev/null @@ -1,7 +0,0 @@ ------BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtz -c2gtZWQyNTUxOQAAACB8VSZQZJCtS6k1JqLharHSOiFk/DClAaHQTDoozBDbZgAA -AIiHWaTwh1mk8AAAAAtzc2gtZWQyNTUxOQAAACB8VSZQZJCtS6k1JqLharHSOiFk -/DClAaHQTDoozBDbZgAAAEDX0WO+D+FFjJ4INEye5RxMy/6YpzdyaoBDMIyc1W2H -9HxVJlBkkK1LqTUmouFqsdI6IWT8MKUBodBMOijMENtmAAAAAAECAwQF ------END OPENSSH PRIVATE KEY----- diff --git a/ollama/ollama/id_ed25519.pub b/ollama/ollama/id_ed25519.pub deleted file mode 100644 index c7d60c847..000000000 --- a/ollama/ollama/id_ed25519.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHxVJlBkkK1LqTUmouFqsdI6IWT8MKUBodBMOijMENtm From 9d5b08fbc77f7b5d415eccc287d4cfe0c3c13c07 Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Mon, 19 Aug 2024 02:54:00 +0000 Subject: [PATCH 30/31] enable manual triggering of GPTR tests flow --- .github/workflows/docker-build.yml | 9 +++++---- docker-compose.yml | 2 +- tests/{3-report-types.py => report-types.py} | 0 3 files changed, 6 insertions(+), 5 deletions(-) rename tests/{3-report-types.py => report-types.py} (100%) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index d230b8ef3..eb391799f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -1,11 +1,12 @@ -name: GitHub Actions Workflow -run-name: ${{ github.actor }} has started docker build workflow. +name: GPTR tests +run-name: ${{ github.actor }} ran the GPTR tests flow permissions: contents: read pull-requests: write on: - pull_request: - types: [opened, synchronize] + workflow_dispatch: # Add this line to enable manual triggering + # pull_request: + # types: [opened, synchronize] jobs: docker: diff --git a/docker-compose.yml b/docker-compose.yml index f16edade6..6a282d7b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,5 +37,5 @@ services: command: > /bin/sh -c " pip install pytest pytest-asyncio && - python -m pytest tests/3-report-types.py + python -m pytest tests/report-types.py " \ No newline at end of file diff --git a/tests/3-report-types.py b/tests/report-types.py similarity index 100% rename from tests/3-report-types.py rename to tests/report-types.py From ff9380651b3fefc7e735dd3cea223035ce7e9f9c Mon Sep 17 00:00:00 2001 From: ElishaKay Date: Mon, 19 Aug 2024 03:38:11 +0000 Subject: [PATCH 31/31] removed nextjs folder change in docker (handled in separate PR) --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6a282d7b5..227317c78 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,10 +19,10 @@ services: - CHOKIDAR_USEPOLLING=true build: dockerfile: Dockerfile.dev - context: application + context: multi_agents/frontend volumes: - /app/node_modules - - ./application:/app + - ./multi_agents/frontend:/app restart: always ports: - 3000:3000