2.1rc1

.env

@@ -2,6 +2,7 @@
 MAIL_TO_ADDRS_FILE=mail_to.txt
 
 DKR_USER=nappl_fswms
+DKR_GROUP=nappl
 
 DOCKER=/usr/bin/docker
 

cli.py

@@ -3,7 +3,7 @@
 import os.path
 
 from volumes import vols
-from util import load_env, clean_all
+from util import load_env, clean_all, chown_all
 from dkr import run_gdal
 
 load_env(ns=globals())

dkr.py

@@ -1,22 +1,28 @@
 
+import os, os.path, pwd, grp
 import docker
-import os.path
+import logging as log
 
 from util import load_env
 
-import logging as log
 
 load_env(ns=globals())
 
 def build_gdal():
   client = docker.from_env()
+  user = pwd.getpwnam(DKR_USER)
+  group = grp.getgrnam(DKR_GROUP)
+  user_id = user[2]
+  group_id = group[2]
   gdal_image = client.images.build(
     path=DKR_BUILD_DIR_HOST,
     tag=DKR_IMAGE_TAG,
     buildargs={
+      'DKR_BUILD_DIR': DKR_BUILD_DIR,
       'DKR_USER': DKR_USER,
       'DKR_GROUP': DKR_GROUP,
-      'DKR_BUILD_DIR': DKR_BUILD_DIR
+      'DKR_USER_ID': str(user_id),
+      'DKR_GROUP_ID': str(group_id)
     }
   )
 
@@ -35,7 +41,7 @@ def run_gdal(cmd, volumes=None):
         auto_remove=True,
         volumes=volumes,
         tty=True,
-        detach=True
+        detach=True,
     )
     for chunk in container.logs(stream=True):
       print(chunk.decode('UTF-8'), end='')

dkr_update

@@ -6,16 +6,18 @@ import logging as log
 sys.path.append('/build')
 from bulk_maxes import YearMaxesArchive
 from fw2_archive import ForWarn2Archive
-from util import init_log, mail_results
+from util import init_log, mail_results, chown_all
 
 init_log()
 
-#bulk_archive = YearMaxesArchive()
-#bulk_archive.update(update_precursors=True)
+bulk_archive = YearMaxesArchive()
+bulk_archive.update(update_precursors=True)
 
 archive = ForWarn2Archive()
 dates = archive.update()
 
+chown_all()
+
 if len(dates):
   log.info('Emailing results...')
   mail_results(dates)

gdal_docker/Dockerfile

@@ -3,19 +3,20 @@ FROM ubuntu:18.04
 ARG DKR_BUILD_DIR
 ARG DKR_USER
 ARG DKR_GROUP
+ARG DKR_USER_ID
+ARG DKR_GROUP_ID
 
 MAINTAINER NEMAC Dev Team <[email protected]>
 
-#RUN useradd $DKR_USER
-#RUN mkdir $DKR_BUILD_DIR
-#RUN chown $DKR_USER:$DKR_GROUP $DKR_BUILD_DIR
-
 # SMTP
 EXPOSE 587
 EXPOSE 25
 # HTTP
 EXPOSE 80
 
+RUN groupadd -g ${DKR_GROUP_ID} ${DKR_GROUP} && \
+    useradd -l -u ${DKR_USER_ID} -g ${DKR_GROUP_ID} ${DKR_USER}
+
 WORKDIR $DKR_BUILD_DIR
 
 ADD ./install $DKR_BUILD_DIR/install

precursor_archive.py

@@ -103,13 +103,12 @@ def _update_24day_max(self, y, jd):
     return out_path, ptype, True
 
 
-  def _update_8day_max(self, y, jd, verbose=False):
+  def _update_8day_max(self, y, jd):
     _dir = self._get_dir(jd)
     try:
       std_path = self._get_file_path(y, jd, nrt=False)
       if os.path.exists(std_path):
-        if verbose:
-          log.info(f'Found std file at {std_path}...')
+        log.debug(f'Found std file at {std_path}...')
         return std_path, 'std', False
       std_path = self.api.get(y, jd, out_dir=_dir, check=True)
       return std_path, 'std', True

test.py

@@ -5,8 +5,7 @@
 from gimms import Gimms, DataNotFoundError
 from precursor_archive import PrecursorArchive
 
-precursor_dir = './precursors'
-test_root = './test/precursors'
+from test_util import *
 
 tmp_dir = './tmp'
 out_dir='./tmp'
@@ -18,18 +17,22 @@
 class TestPrecursorArchive(unittest.TestCase):
 
   def setUp(self):
-    self.archive = PrecursorArchive(root_dir=test_root)
+    test_root = get_test_dir(PRECURSORS_DIR)
+    if not os.path.exists(test_root):
+        os.mkdir(test_root)
+        link_archive(PRECURSORS_DIR, test_root)
+    self.archive = PrecursorArchive(root_dir=get_test_dir(PRECURSORS_DIR))
 
   def test__update_all(self):
     pass
 
-  def test___update_date(self):
+  def test__update_date(self):
     pass
 
-  def test___update_24day_max(self):
+  def test__update_24day_max(self):
     pass
 
-  def test___update_8day_max(self):
+  def test__update_8day_max(self):
     pass
 
   def test__get_24day_max_input_paths(self):
@@ -38,7 +41,7 @@ def test__get_24day_max_input_paths(self):
   def test__get_dir(self):
     pass
 
-  def test___get_best_8day_max_path(self):
+  def test__get_best_8day_max_path(self):
     pass
 
   def test__get_previous_date(self):
@@ -133,12 +136,5 @@ def test__filename_template(self):
     self.assertTrue(f_sat.endswith(ext))
 
 
-def link_archive(src=precursor_dir, dst=test_root):
-  def copy3(src, dst):
-    src = os.path.realpath(src)
-    dst = os.path.realpath(dst)
-    os.symlink(src, dst)
-  shutil.copytree(src, dst, copy_function=copy3)
-
 if __name__ == '__main__':
   unittest.main()

test_util.py

@@ -0,0 +1,15 @@
+
+from util import load_env
+
+load_env(ns=globals())
+
+def get_test_dir(folder):
+    return os.path.join('./test', folder)
+
+def link_archive(src=PRECURSORS_DIR, dst=get_test_dir(PRECURSORS_DIR)):
+  def copy3(src, dst):
+    src = os.path.realpath(src)
+    dst = os.path.realpath(dst)
+    os.symlink(src, dst)
+  shutil.copytree(src, dst, copy_function=copy3)
+

util.py

@@ -1,9 +1,10 @@
 import smtplib
 from email.message import EmailMessage
 
-from subprocess import Popen, PIPE, STDOUT
 import os, os.path, sys, re, traceback, shutil, datetime
+from shutil import chown
 from pytz import timezone
+from subprocess import Popen, PIPE, STDOUT
 
 from tempfile import NamedTemporaryFile
 import base64
@@ -55,6 +56,24 @@ class OverwriteError(Exception):
 ## Helpers
 #
 
+def chown_all():
+  # The call is coming from... inside the container!
+  log.debug(f'Changing owner:group of all files to {DKR_USER}:{DKR_GROUP}')
+  folders = [
+    os.path.realpath(DKR_BUILD_DIR),
+    os.path.join(DKR_BUILD_DIR, FW2_ARCHIVE_DIR_NORMAL),
+    os.path.join(DKR_BUILD_DIR, FW2_ARCHIVE_DIR_MUTED),
+    os.path.join(DKR_BUILD_DIR, PRECURSORS_DIR),
+    os.path.join(DKR_BUILD_DIR, ALL_YEAR_MAXES_DIR),
+  ]
+  for folder in folders:
+    shutil.chown(os.path.realpath(folder), user=DKR_USER, group=DKR_GROUP)
+    for root, dirs, files in os.walk(folder):
+      for f in files:
+        shutil.chown(os.path.join(root, f), user=DKR_USER, group=DKR_GROUP)
+      for _dir in dirs:
+        shutil.chown(os.path.join(root, _dir), user=DKR_USER, group=DKR_GROUP) 
+
 
 def clean_all(base_dir='.', dryrun=False):
   exts = [ 'img', 'gz', 'tif', 'vrt' ]
@@ -63,6 +82,7 @@ def clean_all(base_dir='.', dryrun=False):
   for f in files:
     for ext in exts:
       if f.endswith(ext):
+        log.info(f'Removing {f}')
         os.remove(f)