[distro] refactor: cleanup dependencies in setup script (Jiayi-Pan#37)

* [distro] refactor: cleanup dependencies in setup script

* Update requirements.txt

Author: eric-haibin-lin

requirements.txt

@@ -1,10 +1,12 @@
-transformers 
-hydra-core 
-tensordict==0.5.0
-numpy 
-pytest 
-pybind11 
+# vllm==0.6.3 # vllm is installed in image building to avoid ray conflicts
+# TODO: add version info to requirements
+accelerate
 codetiming
-yapf
-wandb
-git+https://github.com/NVIDIA/TransformerEngine.git@stable
+datasets
+dill
+hydra-core
+numpy
+pybind11
+ray==2.10
+tensordict
+transformers

setup.py

@@ -20,29 +20,19 @@
 with open(os.path.join(version_folder, 'verl/version/version')) as f:
     __version__ = f.read().strip()
 
-# TODO: add version info to requirements
-install_requires = [
-    'torch==2.4.0',
-    'tensordict',
-    'transformers',
-    'codetiming',
-    'pybind11',
-    'hydra-core',
-    'numpy',
-    'yapf',
-    "dill",
-    "accelerate"
-]
+
+with open('requirements.txt') as f:
+    required = f.read().splitlines()
+    install_requires = [item.strip() for item in required if item.strip()[0] != '#']
 
 install_optional = [
     'vllm==0.6.3',
+    'torch==2.4.0', # required by vllm
 ]
 
 extras_require = {
-    'demo': ['hydra-core', 'transformers', ''],
     'single-controller': ['ray', 'kubernetes'],
-    'single-controller-ray': ['ray'],
-    'test': ['fsspec', 'pytest', 'datasets']
+    'test': ['pytest']
 }
 
 from pathlib import Path