Merge branch 'geekan:main' into send18

.coveragerc

@@ -0,0 +1,7 @@
+[run]
+source = 
+    ./metagpt/
+omit = 
+    */metagpt/environment/android/*
+    */metagpt/ext/android_assistant/*
+    */metagpt/ext/werewolf/*

.devcontainer/devcontainer.json

@@ -3,7 +3,7 @@
 {
 	"name": "Python 3",
 	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-	"image": "mcr.microsoft.com/devcontainers/python:0-3.11",
+	"image": "metagpt/metagpt:latest",
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
 	// "features": {},
@@ -18,7 +18,7 @@
 			]
 		}
 	},
-	
+
 	// Use 'postCreateCommand' to run commands after the container is created.
 	"postCreateCommand": "./.devcontainer/postCreateCommand.sh"
 

.github/ISSUE_TEMPLATE/show_me_the_bug.md

@@ -19,6 +19,7 @@ assignees: ''
 - LLM type and model name:
 - System version:
 - Python version:
+- MetaGPT version or branch:
 
 <!-- Dependent packagess：the packages version cause the bug(like `pydantic 1.10.8`), installation method（like `pip install metagpt` or `pip install from source` or `run in docker`） -->
 

.github/workflows/build-package.yaml

@@ -1,8 +1,9 @@
 name: Build and upload python package
 
 on:
+  workflow_dispatch:
   release:
-    types: [created]
+    types: [created, published]
 
 jobs:
   deploy:

.github/workflows/fulltest.yaml

@@ -30,7 +30,10 @@ jobs:
         cache: 'pip'
     - name: Install dependencies
       run: |
-        sh tests/scripts/run_install_deps.sh
+        python -m pip install --upgrade pip
+        pip install -e .[test]
+        npm install -g @mermaid-js/mermaid-cli
+        playwright install --with-deps
     - name: Run reverse proxy script for ssh service
       if: contains(github.ref, '-debugger')
       continue-on-error: true

.github/workflows/pre-commit.yaml

@@ -11,6 +11,7 @@ on:
 jobs:
   pre-commit-check:
     runs-on: ubuntu-latest
+    environment: pre-commit
     steps:
     - name: Checkout Source Code
       uses: actions/checkout@v2

.github/workflows/unittest.yaml

@@ -27,20 +27,57 @@ jobs:
         cache: 'pip'
     - name: Install dependencies
       run: |
-        sh tests/scripts/run_install_deps.sh
+        python -m pip install --upgrade pip
+        pip install -e .[test]
+        npm install -g @mermaid-js/mermaid-cli
+        playwright install --with-deps
     - name: Test with pytest
       run: |
         export ALLOW_OPENAI_API_CALL=0
         mkdir -p ~/.metagpt && cp tests/config2.yaml ~/.metagpt/config2.yaml
-        pytest tests/ --doctest-modules --cov=./metagpt/ --cov-report=xml:cov.xml --cov-report=html:htmlcov --durations=20 | tee unittest.txt
+        pytest --continue-on-collection-errors tests/ \
+          --ignore=tests/metagpt/environment/android_env \
+          --ignore=tests/metagpt/ext/android_assistant \
+          --ignore=tests/metagpt/ext/stanford_town \
+          --ignore=tests/metagpt/provider/test_bedrock_api.py \
+          --ignore=tests/metagpt/rag/factories/test_embedding.py \
+          --ignore=tests/metagpt/ext/werewolf/actions/test_experience_operation.py \
+          --ignore=tests/metagpt/provider/test_openai.py \
+          --ignore=tests/metagpt/planner/test_action_planner.py \
+          --ignore=tests/metagpt/planner/test_basic_planner.py \
+          --ignore=tests/metagpt/actions/test_project_management.py \
+          --ignore=tests/metagpt/actions/test_write_code.py \
+          --ignore=tests/metagpt/actions/test_write_code_review.py \
+          --ignore=tests/metagpt/actions/test_write_prd.py \
+          --ignore=tests/metagpt/environment/werewolf_env/test_werewolf_ext_env.py \
+          --ignore=tests/metagpt/memory/test_brain_memory.py \
+          --ignore=tests/metagpt/roles/test_assistant.py \
+          --ignore=tests/metagpt/roles/test_engineer.py \
+          --ignore=tests/metagpt/serialize_deserialize/test_write_code_review.py \
+          --ignore=tests/metagpt/test_environment.py \
+          --ignore=tests/metagpt/test_llm.py \
+          --ignore=tests/metagpt/tools/test_metagpt_oas3_api_svc.py \
+          --ignore=tests/metagpt/tools/test_moderation.py \
+          --ignore=tests/metagpt/tools/test_search_engine.py \
+          --ignore=tests/metagpt/tools/test_tool_convert.py \
+          --ignore=tests/metagpt/tools/test_web_browser_engine_playwright.py \
+          --ignore=tests/metagpt/utils/test_mermaid.py \
+          --ignore=tests/metagpt/utils/test_redis.py \
+          --ignore=tests/metagpt/utils/test_tree.py \
+          --ignore=tests/metagpt/serialize_deserialize/test_sk_agent.py \
+          --ignore=tests/metagpt/utils/test_text.py \
+          --ignore=tests/metagpt/actions/di/test_write_analysis_code.py \
+          --ignore=tests/metagpt/provider/test_ark.py \
+          --doctest-modules --cov=./metagpt/ --cov-report=xml:cov.xml --cov-report=html:htmlcov \
+          --durations=20 | tee unittest.txt
     - name: Show coverage report
       run: |
         coverage report -m
     - name: Show failed tests and overall summary
       run: |
         grep -E "FAILED tests|ERROR tests|[0-9]+ passed," unittest.txt
-        failed_count=$(grep -E "FAILED|ERROR" unittest.txt | wc -l)
-        if [[ "$failed_count" -gt 0 ]]; then
+        failed_count=$(grep -E "FAILED tests|ERROR tests" unittest.txt | wc -l | tr -d '[:space:]')
+        if [[ $failed_count -gt 0 ]]; then
           echo "$failed_count failed lines found! Task failed."
           exit 1
         fi

.gitignore

@@ -1,7 +1,7 @@
 ### Python template
 
 # Byte-compiled / optimized / DLL files
-__pycache__/
+__pycache__
 *.py[cod]
 *$py.class
 

MANIFEST.in

@@ -0,0 +1,3 @@
+recursive-include metagpt/ext/stanford_town/prompts *.txt
+recursive-include metagpt/ext/stanford_town/static_dirs *.csv
+recursive-include metagpt/ext/stanford_town/static_dirs *.json

README.md

@@ -26,16 +26,16 @@
 </p>
 
 ## News
-🚀 Mar. 14, 2024: Our Data Interpreter paper is on [arxiv](https://arxiv.org/abs/2402.18679). Check the [example](https://docs.deepwisdom.ai/main/en/DataInterpreter/) and [code](https://github.com/geekan/MetaGPT/tree/main/examples/di)!
+🚀 Mar. 29, 2024: [v0.8.0](https://github.com/geekan/MetaGPT/releases/tag/v0.8.0) released. Now you can use Data Interpreter ([arxiv](https://arxiv.org/abs/2402.18679), [example](https://docs.deepwisdom.ai/main/en/DataInterpreter/), [code](https://github.com/geekan/MetaGPT/tree/main/examples/di)) via pypi package import. Meanwhile, we integrated RAG module and supported multiple new LLMs.
 
 🚀 Feb. 08, 2024: [v0.7.0](https://github.com/geekan/MetaGPT/releases/tag/v0.7.0) released, supporting assigning different LLMs to different Roles. We also introduced [Data Interpreter](https://github.com/geekan/MetaGPT/blob/main/examples/di/README.md), a powerful agent capable of solving a wide range of real-world problems.
 
 🚀 Jan. 16, 2024: Our paper [MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework
-](https://arxiv.org/abs/2308.00352) accepted for oral presentation **(top 1.2%)** at ICLR 2024, **ranking #1** in the LLM-based Agent category.
+](https://openreview.net/forum?id=VtmBAGCN7o) accepted for **oral presentation (top 1.2%)** at ICLR 2024, **ranking #1** in the LLM-based Agent category.
 
 🚀 Jan. 03, 2024: [v0.6.0](https://github.com/geekan/MetaGPT/releases/tag/v0.6.0) released, new features include serialization, upgraded OpenAI package and supported multiple LLM, provided [minimal example for debate](https://github.com/geekan/MetaGPT/blob/main/examples/debate_simple.py) etc.
 
-🚀 Dec. 15, 2023: [v0.5.0](https://github.com/geekan/MetaGPT/releases/tag/v0.5.0) released, introducing some experimental features such as **incremental development**, **multilingual**, **multiple programming languages**, etc.
+🚀 Dec. 15, 2023: [v0.5.0](https://github.com/geekan/MetaGPT/releases/tag/v0.5.0) released, introducing some experimental features such as incremental development, multilingual, multiple programming languages, etc.
 
 🔥 Nov. 08, 2023: MetaGPT is selected into [Open100: Top 100 Open Source achievements](https://www.benchcouncil.org/evaluation/opencs/annual.html).
 
@@ -55,88 +55,69 @@
 
 <p align="center">Software Company Multi-Agent Schematic (Gradually Implementing)</p>
 
-## Install
+## Get Started
 
-### Pip installation
+### Installation
 
 > Ensure that Python 3.9+ is installed on your system. You can check this by using: `python --version`.  
 > You can use conda like this: `conda create -n metagpt python=3.9 && conda activate metagpt`
 
 ```bash
-pip install metagpt
-# https://docs.deepwisdom.ai/main/en/guide/get_started/configuration.html
-metagpt --init-config  # it will create ~/.metagpt/config2.yaml, just modify it to your needs
+pip install --upgrade metagpt
+# or `pip install --upgrade git+https://github.com/geekan/MetaGPT.git`
+# or `git clone https://github.com/geekan/MetaGPT && cd MetaGPT && pip install --upgrade -e .`
 ```
 
+For detailed installation guidance, please refer to [cli_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-stable-version)
+ or [docker_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-with-docker)
+
 ### Configuration
 
+You can init the config of MetaGPT by running the following command, or manually create `~/.metagpt/config2.yaml` file:
+```bash
+# Check https://docs.deepwisdom.ai/main/en/guide/get_started/configuration.html for more details
+metagpt --init-config  # it will create ~/.metagpt/config2.yaml, just modify it to your needs
+```
+
 You can configure `~/.metagpt/config2.yaml` according to the [example](https://github.com/geekan/MetaGPT/blob/main/config/config2.example.yaml) and [doc](https://docs.deepwisdom.ai/main/en/guide/get_started/configuration.html):
 
 ```yaml
 llm:
-  api_type: "openai"  # or azure / ollama / open_llm etc. Check LLMType for more options
-  model: "gpt-4-turbo-preview"  # or gpt-3.5-turbo-1106 / gpt-4-1106-preview
+  api_type: "openai"  # or azure / ollama / groq etc. Check LLMType for more options
+  model: "gpt-4-turbo"  # or gpt-3.5-turbo
   base_url: "https://api.openai.com/v1"  # or forward url / other llm url
   api_key: "YOUR_API_KEY"
 ```
 
 ### Usage
 
-After installation, you can use it as CLI
+After installation, you can use MetaGPT at CLI
 
 ```bash
 metagpt "Create a 2048 game"  # this will create a repo in ./workspace
 ```
 
-or you can use it as library
+or use it as library
 
 ```python
 from metagpt.software_company import generate_repo, ProjectRepo
 repo: ProjectRepo = generate_repo("Create a 2048 game")  # or ProjectRepo("<path>")
 print(repo)  # it will print the repo structure with files
 ```
 
-detail installation please refer to [cli_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-stable-version)
- or [docker_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-with-docker)
-
-### Docker installation
-<details><summary><strong>⏬ Step 1: Download metagpt image and prepare config2.yaml </strong><i>:: click to expand ::</i></summary>
-<div>
-
-```bash
-docker pull metagpt/metagpt:latest
-mkdir -p /opt/metagpt/{config,workspace}
-docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config2.yaml > /opt/metagpt/config/config2.yaml
-vim /opt/metagpt/config/config2.yaml # Change the config
-```
-
-</div>
-</details>
+You can also use [Data Interpreter](https://github.com/geekan/MetaGPT/tree/main/examples/di) to write code:
 
-<details><summary><strong>⏬ Step 2: Run metagpt container </strong><i>:: click to expand ::</i></summary>
-<div>
-
-```bash
-docker run --name metagpt -d \
-    --privileged \
-    -v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \
-    -v /opt/metagpt/workspace:/app/metagpt/workspace \
-    metagpt/metagpt:latest
-```
-
-</div>
-</details>
+```python
+import asyncio
+from metagpt.roles.di.data_interpreter import DataInterpreter
 
-<details><summary><strong>⏬ Step 3: Use metagpt </strong><i>:: click to expand ::</i></summary>
-<div>
+async def main():
+    di = DataInterpreter()
+    await di.run("Run data analysis on sklearn Iris dataset, include a plot")
 
-```bash
-docker exec -it metagpt /bin/bash
-$ metagpt "Create a 2048 game"  # this will create a repo in ./workspace
+asyncio.run(main())  # or await main() in a jupyter notebook setting
 ```
 
-</div>
-</details>
 
 ### QuickStart & Demo Video
 - Try it on [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT)
@@ -156,17 +137,21 @@ https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace
 - 🧑‍💻 Contribution
   - [Develop Roadmap](docs/ROADMAP.md)
 - 🔖 Use Cases
+  - [Data Interpreter](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/interpreter/intro.html)
   - [Debate](https://docs.deepwisdom.ai/main/en/guide/use_cases/multi_agent/debate.html)
   - [Researcher](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/researcher.html)
   - [Recepit Assistant](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/receipt_assistant.html)
 - ❓ [FAQs](https://docs.deepwisdom.ai/main/en/guide/faq.html)
 
 ## Support
 
-### Discard Join US
-📢 Join Our [Discord Channel](https://discord.gg/ZRHeExS6xv)!
+### Discord Join US
+
+📢 Join Our [Discord Channel](https://discord.gg/ZRHeExS6xv)! Looking forward to seeing you there! 🎉
 
-Looking forward to seeing you there! 🎉
+### Contributor form
+
+📝 [Fill out the form](https://airtable.com/appInfdG0eJ9J4NNL/pagK3Fh1sGclBvVkV/form) to become a contributor. We are looking forward to your participation!
 
 ### Contact Information
 
@@ -179,16 +164,17 @@ We will respond to all questions within 2-3 business days.
 
 ## Citation
 
-If you use MetaGPT or Data Interpreter in a research paper, please cite our work as follows:
+To stay updated with the latest research and development, follow [@MetaGPT_](https://twitter.com/MetaGPT_) on Twitter. 
+
+To cite [MetaGPT](https://openreview.net/forum?id=VtmBAGCN7o) or [Data Interpreter](https://arxiv.org/abs/2402.18679) in publications, please use the following BibTeX entries.
 
 ```bibtex
-@misc{hong2023metagpt,
-      title={MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework}, 
-      author={Sirui Hong and Mingchen Zhuge and Jonathan Chen and Xiawu Zheng and Yuheng Cheng and Ceyao Zhang and Jinlin Wang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu and Jürgen Schmidhuber},
-      year={2023},
-      eprint={2308.00352},
-      archivePrefix={arXiv},
-      primaryClass={cs.AI}
+@inproceedings{hong2024metagpt,
+      title={Meta{GPT}: Meta Programming for A Multi-Agent Collaborative Framework},
+      author={Sirui Hong and Mingchen Zhuge and Jonathan Chen and Xiawu Zheng and Yuheng Cheng and Jinlin Wang and Ceyao Zhang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu and J{\"u}rgen Schmidhuber},
+      booktitle={The Twelfth International Conference on Learning Representations},
+      year={2024},
+      url={https://openreview.net/forum?id=VtmBAGCN7o}
 }
 @misc{hong2024data,
       title={Data Interpreter: An LLM Agent For Data Science}, 
@@ -198,6 +184,5 @@ If you use MetaGPT or Data Interpreter in a research paper, please cite our work
       archivePrefix={arXiv},
       primaryClass={cs.AI}
 }
-
 ```
 

SECURITY.md

@@ -4,9 +4,9 @@
 
 | Version | Supported          |
 |---------|--------------------|
- | 7.x     | :x:                |
- | 6.x     | :x:                |
-| < 6.x   | :x:                |
+ | 0.7.x   | :x:                |
+ | 0.6.x   | :x:                |
+| < 0.6.x | :x:                |
 
 
 ## Reporting a Vulnerability