deploy: 57f3a0e

telekom · May 24, 2024 · 2a74356 · 2a74356
1 parent c5315f9
commit 2a74356
Showing 1 changed file with 14 additions and 2 deletions.
diff --git a/_modules/mltb2/files.html b/_modules/mltb2/files.html
@@ -230,10 +230,22 @@ <h1>Source code for mltb2.files</h1><div class="highlight"><pre>
             <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_result_dir_path</span> <span class="o">/</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">uuid</span><span class="si">}</span><span class="s2">.lock&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">touch</span><span class="p">()</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">_own_lock_uuids</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">uuid</span><span class="p">)</span>
 
-<div class="viewcode-block" id="FileBasedRestartableBatchDataProcessor.read_batch"><a class="viewcode-back" href="../../api-reference/files.html#mltb2.files.FileBasedRestartableBatchDataProcessor.read_batch">[docs]</a>    <span class="k">def</span> <span class="nf">read_batch</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]:</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;Read the next batch of data.&quot;&quot;&quot;</span>
+    <span class="k">def</span> <span class="nf">_get_remaining_data</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]:</span>
         <span class="n">locked_or_done_uuids</span><span class="p">:</span> <span class="n">Set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_locked_or_done_uuids</span><span class="p">()</span>
         <span class="n">remaining_data</span> <span class="o">=</span> <span class="p">[</span><span class="n">d</span> <span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="k">if</span> <span class="n">d</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">uuid_name</span><span class="p">]</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">locked_or_done_uuids</span><span class="p">]</span>
+        <span class="k">return</span> <span class="n">remaining_data</span>
+
+<div class="viewcode-block" id="FileBasedRestartableBatchDataProcessor.read_batch"><a class="viewcode-back" href="../../api-reference/files.html#mltb2.files.FileBasedRestartableBatchDataProcessor.read_batch">[docs]</a>    <span class="k">def</span> <span class="nf">read_batch</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Read the next batch of data.&quot;&quot;&quot;</span>
+        <span class="n">remaining_data</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_remaining_data</span><span class="p">()</span>
+
+        <span class="c1"># if we think we are done, delete all lock files and check again</span>
+        <span class="c1"># this is because lock files might be orphaned</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">remaining_data</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">lock_file_path</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_result_dir_path</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span><span class="s2">&quot;*.lock&quot;</span><span class="p">):</span>
+                <span class="n">lock_file_path</span><span class="o">.</span><span class="n">unlink</span><span class="p">(</span><span class="n">missing_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+            <span class="n">remaining_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_remaining_data</span><span class="p">()</span>
+
         <span class="n">random</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span><span class="n">remaining_data</span><span class="p">)</span>
         <span class="n">next_batch_size</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">remaining_data</span><span class="p">))</span>
         <span class="n">next_batch</span> <span class="o">=</span> <span class="n">remaining_data</span><span class="p">[:</span><span class="n">next_batch_size</span><span class="p">]</span>