Merge branch 'master' of https://github.com/astroumd/lmtoy

bin/SLpipeline.sh

@@ -10,7 +10,7 @@
 #  @todo   optional PI parameters
 #          option to have a data+time ID in the name, by default it will be blank?
 
-_version="SLpipeline: 2-feb-2024"
+_version="SLpipeline: 24-mar-2024"
 
 echo ""
 echo "LMTOY>> VERSION $(cat $LMTOY/VERSION)"
@@ -40,6 +40,8 @@ rsync=""        # rsync address for the TAP file (used at LMT/malt)
 oid=""          # experimental parallel processing using __$oid  == currently not in use ==
 goal=Science    # Science, or override with: Pointing,Focus
 webrun=""       # optional directive for webrun to do parameter checking (SEQ/map, SEQ/Bs, RSR, ....)
+qagrade=""      # if given, the final grade recorded for the archive (QAFAIL enforces -1)
+public=""       # if given, the public data for archiving. Default is 1 year after today.
 
 #  Optional instrument specific pipeline can be added as well but are not known here
 #  A few examples:
@@ -330,6 +332,16 @@ echo "obsnum_list=$obsnum_list" >> $pdir/lmtoy_$obsnum.rc
 # record the processing time
 echo "date=\"$(lmtoy_date)\"     # end " >> $pdir/lmtoy_$obsnum.rc
 
+# record the qagrade, if one was given
+if [ ! -z $qagrade ]; then
+    echo "qagrade=$qagrade" >> $pdir/lmtoy_$obsnum.rc
+fi
+
+# record the public date, if one was given
+if [ ! -z $public ]; then
+    echo "date_public=$public" >> $pdir/lmtoy_$obsnum.rc
+fi
+
 # make a metadata yaml file for later ingestion into DataVerse
 if [ $meta -gt 0 ]; then
     cd $pdir

bin/lmtoy_functions.sh

@@ -3,7 +3,7 @@
 #   some functions to share for lmtoy pipeline operations
 #   beware, in bash shell variables are common variables between this and the caller
 
-lmtoy_version="16-mar-2024"
+lmtoy_version="28-mar-2024"
 
 echo "LMTOY>> lmtoy_functions $lmtoy_version via $0"
 
@@ -463,10 +463,12 @@ function lmtoy_rsr1 {
 		tabplot spec2.tab 1 2,3,4 111-4 111 line=1,1 color=2,3,4 ycoord=0 yapp=spec2.$dev/$dev
 	    fi
 
-	    printf_red LineCheck1 $(txtpar linecheck.log %1*1000,%2*1000,%3,%4/%3*c/1000*2.355 p0=a=,1,2 p1=b=,1,2 p2=c=,1,2 p3=d=,1,2)
-	    printf_red LineCheck2 $(txtpar linecheck.log %1*1000,%2*1000,%3,%4/%3*c/1000*2.355 p0=a=,2,2 p1=b=,2,2 p2=c=,2,2 p3=d=,2,2)
-
-	    #PJT txtpar linecheck.log %1*1000,%2*1000 p0=b=,1,2  p1=b=,1,3
+	    linecheck1="$(txtpar linecheck.log %1*1000,%2*1000,%3,%4/%3*c/1000*2.355 p0=a=,1,2 p1=b=,1,2 p2=c=,1,2 p3=d=,1,2)"
+	    linecheck2="$(txtpar linecheck.log %1*1000,%2*1000,%3,%4/%3*c/1000*2.355 p0=a=,2,2 p1=b=,2,2 p2=c=,2,2 p3=d=,2,2)"
+	    echo linecheck1="\"$linecheck1\"" >> $rc
+	    echo linecheck2="\"$linecheck2\"" >> $rc
+	    printf_red LineCheck1 $linecheck1
+	    printf_red LineCheck2 $linecheck2
 
 	fi
 

bin/mk_metadata.py

@@ -35,8 +35,10 @@
 from dvpipe.pipelines.metadatagroup import LmtMetadataGroup, example
 from dvpipe.pipelines.metadatablock import MetadataBlock
 from lmtoy import data_prod_id
+from datetime import date
+from dateutil.relativedelta import relativedelta
 
-_version = "7-feb-2024"
+_version = "24-mar-2024"
 
 def header(rc, key, debug=False):
     """
@@ -132,11 +134,17 @@ def get_version():
     fp.close()
     return lines[0].strip()
 
-def get_publicDate(projectId):
+def get_publicDate(rc, debug=False):
     """
-    return the date the data will go public
+    return the date the data will go public. ISO format  yyyy-mm-dd
+    should be a year after "now", unless one was present in the RC file
+    (e.g. when passed as public=yyyy-mm-dd to SLpipeline.sh)
     """
-    return "2099-12-31"    # @todo
+    public = header(rc,"date_public",debug)
+    if public != None:
+        return public
+    public_date = date.today() + relativedelta(years=1)
+    return public_date.isoformat()
 
 # constants
 _c  = 299792.458
@@ -184,6 +192,15 @@ def guess_line(restfreq, debug=False):
             return (l[1],l[2])
     return ("Unknown","Unknown")
 
+def get_qagrade(rc, debug=False):
+    qagrade = header(rc,"qagrade",debug)
+    if qagrade == None:
+        qagrade = 0
+    else:
+        qagrade = int(qagrade)
+    return qagrade
+
+
 
 if __name__ == "__main__":
 
@@ -227,7 +244,7 @@ def guess_line(restfreq, debug=False):
     lmtdata.add_metadata("projectID",    header(rc,"ProjectId",debug))
     lmtdata.add_metadata("projectTitle", header(rc,"projectTitle",debug))
     lmtdata.add_metadata("PIName",       header(rc,"PIName",debug))
-    lmtdata.add_metadata("publicDate",   get_publicDate(header(rc,"ProjectId")))
+    lmtdata.add_metadata("publicDate",   get_publicDate(rc,debug))
     lmtdata.add_metadata("isPolarimetry",      False)    # or True if HWP mode not ABSENT
     lmtdata.add_metadata("halfWavePlateMode", "ABSENT")  # or FIXED or ROTATING
     # 0 = unprocessed, 1 = pipeline processed, 2 = DA improvement
@@ -346,7 +363,8 @@ def guess_line(restfreq, debug=False):
         band["bandwidth"] = bw*nchan/nchan0*u.Unit("GHz")
         band["beam"] = lmt_beam(skyfreq)
         band["winrms"] = rms*u.Unit("mK")
-        band["qaGrade"] = 0;                               # 0 .. 5   (0 means not graded) @todo
+        band["qaGrade"] = get_qagrade(rc, debug)   # -1 .. 5 (0 means not graded)
+
         band["nchan"] =  nchan
         band["bandName"] = "OTHER"    # we don't have special names for the spectral line bands
         lmtdata.add_metadata("band",band)
@@ -373,7 +391,7 @@ def guess_line(restfreq, debug=False):
             band["bandwidth"] = bw*nchan/nchan0*u.Unit("GHz")
             band["beam"] = lmt_beam(skyfreq)
             band["winrms"] = rms*u.Unit("mK")
-            band["qaGrade"] = 0;     # -1 .. 5 (0 means not graded)
+            band["qaGrade"] = get_qagrade(rc, debug)   # -1 .. 5 (0 means not graded)
             band["nchan"] = nchan
             band["bandwidth"] = bw*u.Unit("GHz")
 
@@ -406,7 +424,7 @@ def guess_line(restfreq, debug=False):
            band["winrms"] = -1.0
         else:
            band["winrms"] = float(header(rc,"rms",debug))*u.Unit("K")
-        band["qaGrade"] = 0;     # -1 .. 5 (0 means not graded)        
+        band["qaGrade"] = get_qagrade(rc, debug)   # -1 .. 5 (0 means not graded)        <
         band["nchan"] = int(header(rc,"nchan",debug))
         band["formula"] = ""        # not applicable for RSR
         band["transition"] = ""     # not applicable for RSR

docs/cheatsheet.md

@@ -248,6 +248,10 @@ Something needs to be written down how a remote PI webrun is impacted when new d
 
          mv 123456 123456__otfcal
 
+4. Update one or more NEMO programs
+
+         mknemo -u txtpar tabcols tabrows
+
 4. Removing all your jobs from the sbatch queue:
 
          scancel $(squeue --me | tail +2 | awk '{print $1}')