Workflow: Add scheduler workflow for stability

.github/dependabot.yml

@@ -1,3 +1,17 @@
+# ByteDance Volcengine EMR, Copyright 2024.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 version: 2
 updates:
   - package-ecosystem: "github-actions"

.github/workflows/ci.yml

@@ -1,3 +1,17 @@
+# ByteDance Volcengine EMR, Copyright 2024.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 name: CI
 
 on:
@@ -14,6 +28,7 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     strategy:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]

.github/workflows/codestyle.yml

@@ -1,3 +1,17 @@
+# ByteDance Volcengine EMR, Copyright 2024.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 name: CodeStyle
 
 on: [push]

.github/workflows/scheduled_workflow.yml

@@ -0,0 +1,54 @@
+# ByteDance Volcengine EMR, Copyright 2024.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Scheduled Workflow
+
+on:
+  schedule:
+    # Runs every day at midnight
+    - cron: '0 0 * * *'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 180 # 3hours
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        fsspec-version: ["2023.5.0", "2024.9.0"]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        set -x
+        python -m pip install --upgrade pip
+        pip install --user poetry
+        poetry lock
+        poetry install --with dev
+        poetry add fsspec==${{ matrix.fsspec-version }}
+        poetry show fsspec
+    - name: Prepare Env
+      run: |
+        echo "Preparing environment variables"
+        echo "TOS_ACCESS_KEY=${{ secrets.TOS_ACCESS_KEY }}" >> $GITHUB_ENV
+        echo "TOS_SECRET_KEY=${{ secrets.TOS_SECRET_KEY }}" >> $GITHUB_ENV
+        echo "TOS_REGION=${{ vars.TOS_REGION }}" >> $GITHUB_ENV
+        echo "TOS_ENDPOINT=${{ vars.TOS_ENDPOINT }}" >> $GITHUB_ENV
+        echo "TOSFS_LOGGING_LEVEL=${{ vars.TOSFS_LOGGING_LEVEL }}" >> $GITHUB_ENV
+    - name: Run tests
+      run: make test

tosfs/core.py

@@ -136,7 +136,7 @@ def __init__(
             The maximum number of HTTP connections that can be opened in the
             connection pool (default is 1024).
         connection_time : int, optional
-            The time to keep a connection open (default is 10).
+            The time to keep a connection open in seconds (default is 10).
         socket_timeout : int, optional
             The socket read and write timeout time for a single request after
             a connection is successfully established, in seconds.

tosfs/tests/conftest.py

@@ -38,6 +38,9 @@ def tosfs(_tosfs_env_prepare: None) -> TosFileSystem:
         endpoint_url=os.environ.get("TOS_ENDPOINT"),
         region=os.environ.get("TOS_REGION"),
         credentials_provider=EnvCredentialsProvider(),
+        max_retry_num=1000,
+        connection_time=30,
+        socket_timeout=180,
         multipart_size=4 << 20,
         multipart_threshold=4 << 20,
     )