pylhc · JoschD · Nov 13, 2023 · Nov 1, 2023 · Nov 2, 2023 · Nov 2, 2023
diff --git a/.gitignore b/.gitignore
@@ -243,9 +243,14 @@ Temporary Items
 # Neovim
 .nvimlog
 
+# Intellij
 /.idea/codeStyles/codeStyleConfig.xml
 /.idea/misc.xml
 /.idea/modules.xml
 /.idea/inspectionProfiles/profiles_settings.xml
 /.idea/vcs.xml
 /.idea/PyhDToolkit.iml
+
+# Other
+tst_*
+
diff --git a/doc/Makefile b/doc/Makefile
@@ -48,9 +48,9 @@ html:
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
 josch:
-	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) /home/jdilly/Software/Documentation/submitter-doc
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) ../../Documentation/submitter-doc
 	@echo
-	@echo "Build finished. The HTML pages are in /home/jdilly/Software/Documentation/submitter-doc."
+	@echo "Build finished. The HTML pages are in ../../Documentation/submitter-doc."
 
 dirhtml:
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml

diff --git a/doc/_static/css/custom.css b/doc/_static/css/custom.css
@@ -1,7 +1,41 @@
+:root {
+  --nav-side-width: 300px;  /* default is 300px */
+  /* for 100% width */
+  /*--nav-content-width: 100%;*/
+  /*--local-toc-width: 300px;*/
+  /*--nav-content-width-wide: calc(100% - var(--local-toc-width)); /* 100% here is fullscreen */
+  /*--local-toc-left: calc(100% - var(--local-toc-width)); /* 100% here is w/o sidebar */
+
+  /* for fixed widths */
+  --nav-content-width: 800px;  /* default is 800px */
+  --nav-content-width-wide: var(--nav-content-width);
+  --local-toc-width: calc(100% - var(--nav-content-width-wide));
+  --local-toc-left: calc(var(--nav-content-width-wide) + var(--nav-side-width));
+}
+
+/* main content width */
+.wy-nav-content {
+  max-width: var(--nav-content-width);
+}
+
+/* Sidebar width */
+.wy-nav-side {
+  width: var(--nav-side-width);
+}
+
 .wy-side-nav-search {
   background: rgb(243,244,247);
 }
 
+.wy-side-nav-search > a {
+  color: black;
+}
+
+.wy-side-nav-search> a img.logo {
+  width: 50%;
+}
+
+
 .wy-side-nav-search > div.version {
   color: black;
 }
@@ -182,3 +216,60 @@ em.sig-param span.default_value {
 .rst-content table.field-list th {
   padding: 16px;
 }
+
+
+/* Create local table of contents
+   ------------------------------
+  inspired by https://github.com/readthedocs/sphinx_rtd_theme/pull/919
+  and https://github.com/readthedocs/sphinx_rtd_theme/issues/764
+  see also _templates/layout.html
+ */
+
+#local-table-of-contents {
+  padding-bottom: 20px;
+  /* display: none; */
+}
+
+/* Mask entry of main header (chapter) */
+#local-table-of-contents a[href="#"]{
+  /*display: none;*/
+}
+
+/* indent subsections */
+#local-table-of-contents ul > ul {
+  padding-left: 0px;
+  margin-left: 20px;
+  padding-right: 0;
+  padding-bottom: 5px;
+}
+
+
+#local-table-of-contents-title {
+  margin-bottom: 10px;
+}
+
+/* Show in Sidebar if window width is larger than nav-side + nav-content + toc-width */
+@media screen and (min-width: 1200px) {
+  .wy-nav-content {
+    max-width: var(--nav-content-width-wide);
+  }
+
+  #local-table-of-contents {
+    display: block;
+    position: fixed;
+    margin-left: 15px;
+    overflow-y: auto;
+    height: 95%;
+    top: 45px;
+    left: var(--local-toc-left);
+    width: var(--local-toc-width);
+  }
+
+  #local-table-of-contents-title {
+    display: block;
+    font-size: 16px;
+    width: 100%;
+    padding-top: 10px;
+    padding-bottom: 5px;
+  }
+}
diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html
@@ -0,0 +1,12 @@
+{% extends "!layout.html" %}
+{% block document %} 
+  {%- if toc|length > title|length + 75 %}
+  <nav id="local-table-of-contents" role="navigation" aria-labelledby="local-table-of-contents-title">
+    <h4 id="local-table-of-contents-title">On This Page</h4>
+    {{ toc }}
+  </nav>
+  {%- endif %}
+
+  {{ super() }}
+{% endblock %}
+
diff --git a/doc/conf.py b/doc/conf.py
@@ -31,20 +31,10 @@
     sys.path.insert(0, str(TOPLEVEL_DIR))
 
 
-def about_package(init_posixpath: pathlib.Path) -> dict:
-    """
-    Return package information defined with dunders in __init__.py as a dictionary, when
-    provided with a PosixPath to the __init__.py file.
-    """
-    about_text: str = init_posixpath.read_text()
-    return {
-        entry.split(" = ")[0]: entry.split(" = ")[1].strip('"')
-        for entry in about_text.strip().split("\n")
-        if entry.startswith("__")
-    }
 
-
-ABOUT_PYLHC_SUBMITTER = about_package(ABOUT_FILE)
+ABOUT_PYLHC_SUBMITTER: dict = {}
+with ABOUT_FILE.open("r") as f:
+    exec(f.read(), ABOUT_PYLHC_SUBMITTER)
 
 
 # -- General configuration ------------------------------------------------
@@ -66,9 +56,11 @@ def about_package(init_posixpath: pathlib.Path) -> dict:
     "sphinx.ext.githubpages",
     "sphinx.ext.napoleon",
 ]
+autosectionlabel_prefix_document = True
+autosectionlabel_maxdepth = 2
 
 # Add any paths that contain templates here, relative to this directory.
-# templates_path = ['_templates']
+templates_path = ['_templates']
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
@@ -84,6 +76,11 @@ def about_package(init_posixpath: pathlib.Path) -> dict:
 copyright_ = "2019, pyLHC/OMC-TEAM"
 author = ABOUT_PYLHC_SUBMITTER["__author__"]
 
+# Override link in 'Edit on Github'
+rst_prolog = f"""
+:github_url: {ABOUT_PYLHC_SUBMITTER['__url__']}
+"""
+
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
@@ -98,7 +95,7 @@ def about_package(init_posixpath: pathlib.Path) -> dict:
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.

diff --git a/doc/entrypoints/autosix.rst b/doc/entrypoints/autosix.rst
@@ -1,2 +1,3 @@
 .. automodule:: pylhc_submitter.autosix
     :members:
+    :noindex:
diff --git a/doc/entrypoints/job_submitter.rst b/doc/entrypoints/job_submitter.rst
@@ -1,2 +1,3 @@
 .. automodule:: pylhc_submitter.job_submitter
     :members:
+    :noindex:
diff --git a/doc/modules/constants.rst b/doc/modules/constants.rst
@@ -1,15 +1,21 @@
 Constants Definitions
-**************************
+*********************
 
 
 .. automodule:: pylhc_submitter.constants.general
     :members:
+    :noindex:
 
 
 .. automodule:: pylhc_submitter.constants.external_paths
     :members:
+    :noindex:
 
+.. automodule:: pylhc_submitter.constants.job_submitter
+    :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.constants.autosix
     :members:
+    :noindex:
 
diff --git a/doc/modules/htc.rst b/doc/modules/htc.rst
diff --git a/doc/modules/sixdesk_tools.rst b/doc/modules/sixdesk_tools.rst
@@ -3,21 +3,28 @@ Sixdesk Tools
 
 .. automodule:: pylhc_submitter.sixdesk_tools.stages
     :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.sixdesk_tools.create_workspace
     :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.sixdesk_tools.submit
     :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.sixdesk_tools.post_process_da
     :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.sixdesk_tools.extract_data_from_db
     :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.sixdesk_tools.utils
     :members:
+    :noindex:
 
 .. automodule:: pylhc_submitter.sixdesk_tools.troubleshooting
     :members:
+    :noindex:
diff --git a/doc/modules/submitter.rst b/doc/modules/submitter.rst
@@ -0,0 +1,19 @@
+Submitter
+*********
+
+.. automodule:: pylhc_submitter.submitter.htc_utils
+    :members:
+    :noindex:
+
+
+.. automodule:: pylhc_submitter.submitter.iotools
+    :members:
+    :noindex:
+
+.. automodule:: pylhc_submitter.submitter.mask
+    :members:
+    :noindex:
+
+.. automodule:: pylhc_submitter.submitter.runners
+    :members:
+    :noindex:
diff --git a/doc/modules/utils.rst b/doc/modules/utils.rst
@@ -3,7 +3,9 @@ Utilities
 
 .. automodule:: pylhc_submitter.utils.iotools
     :members:
+    :noindex:
 
 
 .. automodule:: pylhc_submitter.utils.logging_tools
     :members:
+    :noindex:
diff --git a/pylhc_submitter/__init__.py b/pylhc_submitter/__init__.py
@@ -10,7 +10,7 @@
 __title__ = "pylhc_submitter"
 __description__ = "pylhc-submitter contains scripts to simplify the creation and submission of jobs to HTCondor at CERN"
 __url__ = "https://github.com/pylhc/submitter"
-__version__ = "1.1.1"
+__version__ = "2.0.0"
 __author__ = "pylhc"
 __author_email__ = "[email protected]"
 __license__ = "MIT"

diff --git a/pylhc_submitter/autosix.py b/pylhc_submitter/autosix.py
@@ -193,34 +193,17 @@
 
 import numpy as np
 import tfs
-from generic_parser import EntryPointParameters, entrypoint, DotDict
+from generic_parser import EntryPointParameters, entrypoint
 from generic_parser.entry_datatypes import DictAsString
 
-from pylhc_submitter.constants.autosix import (
-    HEADER_BASEDIR,
-    SIXENV_REQUIRED,
-    SIXENV_OPTIONAL,
-    AutoSixEnvironment,
-)
-from pylhc_submitter.htc.mask import generate_jobdf_index
-from pylhc_submitter.job_submitter import (
-    JOBSUMMARY_FILE,
-    COLUMN_JOBID,
-)
-from pylhc_submitter.sixdesk_tools.create_workspace import (
-    set_max_materialize
-)
-from pylhc_submitter.sixdesk_tools.stages import Stage, STAGE_ORDER
-from pylhc_submitter.sixdesk_tools.utils import (
-    is_locked,
-    check_mask,
-)
-from pylhc_submitter.utils.iotools import (
-    PathOrStr,
-    save_config,
-    make_replace_entries_iterable,
-    keys_to_path
-)
+from pylhc_submitter.constants.autosix import (HEADER_BASEDIR, SIXENV_OPTIONAL, SIXENV_REQUIRED,
+                                               AutoSixEnvironment)
+from pylhc_submitter.constants.job_submitter import COLUMN_JOBID, JOBSUMMARY_FILE
+from pylhc_submitter.submitter.mask import generate_jobdf_index
+from pylhc_submitter.sixdesk_tools.stages import STAGE_ORDER, Stage
+from pylhc_submitter.sixdesk_tools.utils import check_mask, is_locked
+from pylhc_submitter.utils.iotools import (PathOrStr, keys_to_path, make_replace_entries_iterable,
+                                           save_config)
 from pylhc_submitter.utils.logging_tools import log_setup
 
 LOG = logging.getLogger(__name__)

diff --git a/pylhc_submitter/constants/job_submitter.py b/pylhc_submitter/constants/job_submitter.py
@@ -0,0 +1,33 @@
+
+"""
+Constants: Job Submitter
+----------------------------------
+
+Collections of constants and paths used in the job-submitter.
+"""
+from pylhc_submitter.constants.external_paths import MADX_BIN, PYTHON2_BIN, PYTHON3_BIN
+
+JOBSUMMARY_FILE = "Jobs.tfs"
+JOBDIRECTORY_PREFIX = "Job"
+CONFIG_FILE = "config.ini"
+
+SCRIPT_EXTENSIONS = {
+    "madx": ".madx",
+    "python3": ".py",
+    "python2": ".py",
+}
+
+EXECUTEABLEPATH = {
+    "madx": MADX_BIN,
+    "python3": PYTHON3_BIN,
+    "python2": PYTHON2_BIN,
+}
+
+
+COLUMN_JOBID = "JobId"
+COLUMN_SHELL_SCRIPT = "ShellScript"
+COLUMN_JOB_DIRECTORY = "JobDirectory"
+COLUMN_DEST_DIRECTORY = "DestDirectory"
+COLUMN_JOB_FILE = "JobFile"
+
+NON_PARAMETER_COLUMNS = (COLUMN_SHELL_SCRIPT, COLUMN_JOB_DIRECTORY, COLUMN_JOB_FILE, COLUMN_DEST_DIRECTORY)