Redo logger and fix examples

.github/workflows/ci.yml

@@ -13,6 +13,13 @@ on:
       - '!*post*'
   pull_request:
   workflow_dispatch:
+  schedule:
+    #        ┌───────── minute (0 - 59)
+    #        │ ┌───────── hour (0 - 23)
+    #        │ │ ┌───────── day of the month (1 - 31)
+    #        │ │ │ ┌───────── month (1 - 12 or JAN-DEC)
+    #        │ │ │ │ ┌───────── day of the week (0 - 6 or SUN-SAT)
+    - cron: '0 7 * * *'  # Every day at 07:00 UTC
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.gitignore

@@ -213,6 +213,7 @@ examples/**/*.asdf
 examples/**/*.csv
 examples/jsoc.stanford.edu/
 jsoc.stanford.edu/
+docs/sg_execution_times.rst
 
 ### Pycharm(?)
 .idea

.pre-commit-config.yaml

@@ -11,17 +11,17 @@ repos:
         args: ['--in-place', '--remove-all-unused-imports', '--remove-unused-variable']
         exclude: ".*(.fits|.fts|.fit|.txt|tca.*|extern.*|.rst|.md|docs/conf.py)$"
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.1.6'
+    rev: 'v0.1.9'
     hooks:
       - id: ruff
         args: ['--fix', '--unsafe-fixes']
   -   repo: https://github.com/psf/black
-      rev: 23.11.0
+      rev: 23.12.1
       hooks:
       - id: black
         exclude: ".*(.fits|.fts|.fit|.txt|.csv)$"
   -   repo: https://github.com/PyCQA/isort
-      rev: 5.12.0
+      rev: 5.13.2
       hooks:
       - id: isort
         exclude: ".*(.fits|.fts|.fit|.txt|.csv)$"

docs/conf.py

@@ -14,7 +14,6 @@
 project = "drms"
 author = "The SunPy Project"
 copyright = f"{datetime.datetime.now().year}, {author}"  # NOQA: A001
-
 # The full version, including alpha/beta/rc tags
 release = __version__
 is_development = ".dev" in __version__
@@ -37,25 +36,14 @@
     "sphinx.ext.todo",
     "sphinx.ext.viewcode",
 ]
-
-# Set automodapi to generate files inside the generated directory
 automodapi_toctreedirnm = "generated/api"
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
 source_suffix = ".rst"
-# The master toctree document.
 master_doc = "index"
-# The reST default role (used for this markup: `text`) to use for all
-# documents. Set to the "smart" one.
 default_role = "obj"
 
 # -- Options for hoverxref -----------------------------------------------------
 if os.environ.get("READTHEDOCS"):
-    # Building on Read the Docs
     hoverxref_api_host = "https://readthedocs.org"
     if os.environ.get("PROXIED_API_ENDPOINT"):
         # Use the proxied API endpoint
@@ -65,7 +53,6 @@
 hoverxref_tooltip_maxwidth = 600  # RTD main window is 696px
 hoverxref_auto_ref = True
 hoverxref_mathjax = True
-# hoverxref has to be applied to these
 hoverxref_domains = ["py"]
 hoverxref_role_types = {
     # roles with py domain
@@ -111,7 +98,9 @@
 }
 
 # -- Options for HTML output -------------------------------------------------
-# Render inheritance diagrams in SVG
+# JSOC email os env
+# see https://github.com/sunpy/sunpy/wiki/Home:-JSOC
+os.environ["JSOC_EMAIL"] = "[email protected]"
 graphviz_output_format = "svg"
 sphinx_gallery_conf = {
     "backreferences_dir": Path("generated") / "modules",

drms/__init__.py

@@ -9,14 +9,18 @@
 * Homepage: https://github.com/sunpy/drms
 * Documentation: https://docs.sunpy.org/projects/drms/en/stable/
 """
+import logging
 from pathlib import Path
 
-from .client import Client, ExportRequest, SeriesInfo
-from .config import ServerConfig, register_server
-from .exceptions import DrmsError, DrmsExportError, DrmsOperationNotSupported, DrmsQueryError
-from .json import HttpJsonClient, HttpJsonRequest, JsocInfoConstants
-from .utils import to_datetime
-from .version import version as __version__
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s: %(message)s")
+
+from .client import Client, ExportRequest, SeriesInfo  # NOQA: E402
+from .config import ServerConfig, register_server  # NOQA: E402
+from .exceptions import DrmsError, DrmsExportError, DrmsOperationNotSupported, DrmsQueryError  # NOQA: E402
+from .json import HttpJsonClient, HttpJsonRequest, JsocInfoConstants  # NOQA: E402
+from .utils import to_datetime  # NOQA: E402
+from .version import version as __version__  # NOQA: E402
 
 
 def _get_bibtex():
@@ -53,4 +57,5 @@ def _get_bibtex():
     "SeriesInfo",
     "ServerConfig",
     "to_datetime",
+    "logger",
 ]

drms/client.py

@@ -1,7 +1,6 @@
 import os
 import re
 import time
-import logging
 from pathlib import Path
 from collections import OrderedDict
 from urllib.error import URLError, HTTPError
@@ -11,6 +10,8 @@
 import numpy as np
 import pandas as pd
 
+from drms import logger
+
 from .exceptions import DrmsExportError, DrmsOperationNotSupported, DrmsQueryError
 from .json import HttpJsonClient
 from .utils import _extract_series_name, _pd_to_numeric_coerce, _split_arg
@@ -275,7 +276,7 @@ def dir(self):  # NOQA: A003
         """
         (string) Common directory of the requested files on the server.
         """
-        if self.has_finished(skip_update=True):
+        if self.has_finished(skip_update=False):
             self._raise_on_error()
         else:
             self.wait()
@@ -290,7 +291,7 @@ def data(self):
         Returns a pandas.DataFrame containing the records and filenames
         of the export request (DataFrame columns: 'record', 'filename').
         """
-        if self.has_finished(skip_update=True):
+        if self.has_finished(skip_update=False):
             self._raise_on_error()
         else:
             self.wait()
@@ -301,7 +302,7 @@ def tarfile(self):
         """
         (string) Filename, if a TAR file was requested.
         """
-        if self.has_finished(skip_update=True):
+        if self.has_finished(skip_update=False):
             self._raise_on_error()
         else:
             self.wait()
@@ -313,7 +314,7 @@ def keywords(self):
         """
         (string) Filename of textfile containing record keywords.
         """
-        if self.has_finished(skip_update=True):
+        if self.has_finished(skip_update=False):
             self._raise_on_error()
         else:
             self.wait()
@@ -449,13 +450,13 @@ def wait(self, *, timeout=None, sleep=5, retries_notfound=5):
         retries_notfound = int(retries_notfound)
 
         # We are done, if the request has already finished.
-        if self.has_finished(skip_update=True):
+        if self.has_finished(skip_update=False):
             self._raise_on_error()
             return True
 
         while True:
             idstr = str(None) if self._requestid is None else (f"{self._requestid}")
-            logging.info(f"Export request pending. [id={idstr}, status={self._status}]")
+            logger.info(f"Export request pending. [id={idstr}, status={self._status}]")
 
             # Use the user-provided sleep value or the server's wait value.
             # In case neither is available, wait for 5 seconds.
@@ -471,7 +472,7 @@ def wait(self, *, timeout=None, sleep=5, retries_notfound=5):
                 if t_start + timeout + wait_secs - time.time() < 0:
                     return False
 
-            logging.info(f"Waiting for {int(round(wait_secs))} seconds...")
+            logger.info(f"Waiting for {int(round(wait_secs))} seconds...")
             time.sleep(wait_secs)
 
             if self.has_finished():
@@ -481,7 +482,7 @@ def wait(self, *, timeout=None, sleep=5, retries_notfound=5):
                 # Raise exception, if no retries are left.
                 if retries_notfound <= 0:
                     self._raise_on_error(notfound_ok=False)
-                logging.info(f"Request not found on server, {retries_notfound} retries left.")
+                logger.info(f"Request not found on server, {retries_notfound} retries left.")
                 retries_notfound -= 1
 
     def download(self, directory, *, index=None, fname_from_rec=None):
@@ -565,18 +566,18 @@ def download(self, directory, *, index=None, fname_from_rec=None):
             fpath = Path(out_dir) / filename
             fpath_new = self._next_available_filename(fpath)
             fpath_tmp = self._next_available_filename(f"{fpath_new}.part")
-            logging.info(f"Downloading file {int(i + 1)} of {int(ndata)}...")
-            logging.info(f"    record: {di.record}")
-            logging.info(f"  filename: {di.filename}")
+            logger.info(f"Downloading file {int(i + 1)} of {int(ndata)}...")
+            logger.info(f"    record: {di.record}")
+            logger.info(f"    filename: {di.filename}")
             try:
                 urlretrieve(di.url, fpath_tmp)
             except (HTTPError, URLError):
                 fpath_new = None
-                logging.info("  -> Error: Could not download file")
+                logger.info("    -> Error: Could not download file")
             else:
                 fpath_new = self._next_available_filename(fpath)
                 Path(fpath_tmp).rename(fpath_new)
-                logging.info(f"  -> {os.path.relpath(fpath_new)}")
+                logger.info(f"    -> {os.path.relpath(fpath_new)}")
             downloads.append(fpath_new)
 
         res = data[["record", "url"]].copy()
@@ -650,7 +651,7 @@ def _generate_filenamefmt(self, sname):
             si = self.info(sname)
         except Exception as e:  # NOQA: BLE001
             # Cannot generate filename format for unknown series.
-            logging.warning(f"Cannot generate filename format for unknown series '{sname}' with {e}")
+            logger.warning(f"Cannot generate filename format for unknown series '{sname}' with {e}")
             return None
 
         pkfmt_list = []
@@ -700,7 +701,7 @@ def _filename_from_export_record(self, rs, *, old_fname=None):
             si = self.info(sname)
         except Exception as e:  # NOQA: BLE001
             # Cannot generate filename for unknown series.
-            logging.warning(f"Cannot generate filename format for unknown series '{sname}' with {e}")
+            logger.warning(f"Cannot generate filename format for unknown series '{sname}' with {e}")
             return None
 
         if pkeys is not None:

drms/json.py

@@ -1,9 +1,10 @@
 import json as _json
-import logging
 from enum import Enum
 from urllib.parse import urlencode, quote_plus
 from urllib.request import HTTPError, urlopen
 
+from drms import logger
+
 from .config import ServerConfig, _server_configs
 from .utils import _split_arg
 
@@ -86,7 +87,7 @@ def __repr__(self):
         return f"<HttpJsonClient: {self._server.name}>"
 
     def _json_request(self, url):
-        logging.info(url)
+        logger.debug(f"URL for request: {url}")
         return HttpJsonRequest(url, self._server.encoding)
 
     @property

drms/main.py

@@ -1,21 +1,18 @@
 import sys
-import logging
 import argparse
 
 
 def main():
     import drms
 
     args = parse_args(sys.argv[1:])
-    # Create a Client instance
     client = drms.Client(server=args.server, email=args.email)
-    logging.info(f"client: {client}")
+    drms.logger.info(f"client: {client}")
 
 
 def parse_args(args):
     import drms
 
-    # Handle command line options
     parser = argparse.ArgumentParser(description="drms, access HMI, AIA and MDI data with python")
     parser.add_argument(
         "--version",

setup.cfg

@@ -67,6 +67,7 @@ console_scripts =
 [tool:pytest]
 testpaths = "drms" "docs"
 norecursedirs = ".tox" "build" "docs[\/]_build" "docs[\/]generated" "*.egg-info" "examples" ".history" "paper" "drms[\/]_dev"
+log_cli = True
 doctest_plus = enabled
 doctest_optionflags = NORMALIZE_WHITESPACE FLOAT_CMP ELLIPSIS
 addopts = --doctest-rst -p no:unraisableexception -p no:threadexception

tox.ini

@@ -5,8 +5,6 @@ envlist =
     build_docs
     codestyle
 requires =
-    setuptools
-    pip
     tox-pypi-filter
 
 [testenv]
@@ -44,6 +42,7 @@ deps =
 extras =
     dev
 commands =
+    pip freeze --all --no-input
     sunpy: pytest -vvv -s -ra --pyargs sunpy.net.jsoc --timeout=120 --remote-data=any {posargs}
     !online: {env:PYTEST_COMMAND} {posargs}
     online: {env:PYTEST_COMMAND} --timeout=120 --remote-data=any {posargs}
@@ -53,7 +52,8 @@ changedir = docs
 description = Invoke sphinx-build to build the HTML docs
 extras = dev
 commands =
-    sphinx-build -j auto --color -W --keep-going -b html -d _build/.doctrees . _build/html {posargs}
+    pip freeze --all --no-input
+    sphinx-build -j 1 --color -W --keep-going -b html -d _build/.doctrees . _build/html {posargs}
     python -c 'import pathlib; print("Documentation available under file://\{0\}".format(pathlib.Path(r"{toxinidir}") / "docs" / "_build" / "html"/ "index.html"))'
 
 [testenv:codestyle]