Merge branch 'master' of https://github.com/astroumd/lmtoy

astroumd · May 7, 2024 · f7abe44 · f7abe44
2 parents abb2e60 + 19ce11a
commit f7abe44
Show file tree

Hide file tree

Showing 7 changed files with 131 additions and 22 deletions.
diff --git a/bin/mk_last100.sh b/bin/mk_last100.sh
@@ -1,18 +1,22 @@
 #! /bin/bash
 #
 
-_version="8-mar-2024"
+_version="5-may-2024"
 
 log=$1
 url1="http://taps.lmtgtm.org/lmtslr/%s/TAP/"
 url2="http://taps.lmtgtm.org/lmtslr/%s/TAP/%s/README.html"
+url3="http://taps.lmtgtm.org/lmtslr/%s/"
+url4="http://taps.lmtgtm.org/lmtslr/%s/%s/README.html"
+
 
 echo "<html>"
 echo '<script src="https://www.kryogenix.org/code/browser/sorttable/sorttable.js"></script>'
 
-echo "Last 100 observations SLpipeline'd on malt and supposedly available as lightweight TAPS on Unity"
+echo "Last 100 observations SLpipeline'd on <B>malt</B> should be available as lightweight TAPS on Unity."
+echo "Plus links to future final full processing on Unity (if available)."
 echo "<br>"
-echo "(click on column name to sort by that column) Created: $(date)"
+echo "(click on column name to sort by that column) <br>Created: $(date)"
 
 echo '<table border=1 class="sortable">'
 echo '  <tr class="item">'
@@ -28,11 +32,17 @@ echo "    </th>"
 echo "  </tr>"
 
 while read -r line; do
+   comment=$(echo $line | cut -c1)
+   if [ "$comment" = '#' ]; then
+       continue
+   fi
    date=$(echo $line | tabcols - 1)
    obsnum=$(echo $line | tabcols - 2)
    pid=$(echo $line | tabcols - 3)
    u1=$(printf $url1 $pid)
    u2=$(printf $url2 $pid $obsnum)
+   u3=$(printf $url3 $pid)
+   u4=$(printf $url4 $pid $obsnum)
    echo '  <tr class="item">'
    echo '    <td>'
    echo "      $date"
@@ -43,6 +53,12 @@ while read -r line; do
    echo '    <td>'
    echo "      <A HREF=$u1>$pid</A>"
    echo '    </td>'
+   echo '    <td>'
+   echo "      <A HREF=$u4>[$obsnum]</A>"
+   echo '    </td>'
+   echo '    <td>'
+   echo "      <A HREF=$u3>[$pid]</A>"
+   echo '    </td>'
    echo '  </tr>'
 done < $log
 

diff --git a/bin/work_lmt_missing b/bin/work_lmt_missing
@@ -3,6 +3,12 @@
 # Usage:  work_lmt_missing
 #
 
+if [ -z "$1" ]; then
+    echo "Usage: $0 logfile"
+    echo "   A logfile from lmtinfo.py is needed here, and statistics on these projects will be presented"
+    echo "   e.g. lmtinfo.py grep 2024-S1 Science SEQ > 2024_seq.log"
+    exit 0
+fi
 
 #        should be some output of 'lmtinfo.py grep Science'
 log=$1
@@ -34,4 +40,6 @@ while IFS= read -r line; do
    else
       echo "MISSING $p/$o $i"
    fi
-done < $log 
+done < $log
+
+
diff --git a/docs/cheatsheet2.md b/docs/cheatsheet2.md
@@ -22,6 +22,11 @@ we will set as an environment variable, for example
 
    This should have been set up for you. If not, contact PeterT.
 
+   You should pretty much always make sure these files are in sync with github, so do this to get
+   the latest:
+
+         git pull
+
 
 2. Inspect the **README.md**, **comments.txt** and **mk_runs.py** files for human and programmatic
    information, and keep them up to date.
@@ -35,30 +40,30 @@ we will set as an environment variable, for example
 
          make runs
 
-   The most complex ones are two-IF SEQ, where it creates a run1a, run1b and run1c file.
+   The most complex observations ones are two-IF SEQ, where it creates a run1a, run1b and run1c file.
    These refer to a fresh start for both banks (run1a), then a specific run for bank=0 (run1b)
    and finally a specific run for bank=1 (run1c).
 
          sbatch_lmtoy.sh $PID.run1a
-	 squeue --me | nl -v 0
-	 
-	 <patiently wait until all your obsnums have exited from the queue>
-      
+         squeue --me | nl -v 0
+
+   *patiently wait until all your obsnums have exited from the queue*
+
          sbatch_lmtoy.sh $PID.run1b
 
-         <patiently wait for all the runs to finish>
+   *patiently wait for all the runs to finish*
 
          sbatch_lmtoy.sh $PID.run1c
 
-         <patiently wait for all the runs to finish>
+   *patiently wait for all the runs to finish*
 
    When only one bank is present, only run1a and run1b is needed. For RSR data only run1a is needed.
 
 4. When all these single obsnums are finished, make a summary
 
          make summary
 
-   The results can be viewed on
+   The results can then be viewed on
 
          http://taps.lmtgtm.org/lmtslr/$PID
 

diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -26,7 +26,7 @@
 project = 'lmtoy'
 # The full version, including alpha/beta/rc tags
 # release = pdrtpy.VERSION
-release = "0.7"
+release = "1.0"
 year = str(localtime().tm_year)
 #author = pdrtpy.AUTHORS
 author = "Peter Teuben"

diff --git a/etc/ProjectId.csv b/etc/ProjectId.csv
@@ -1,3 +1,76 @@
+#   Table of:   ProjectId, Title, PI
+#
+#   any projectId with the string "Commissioning" is recognized as such and not used here
+
+# 2024
+
+2024-S1-MX-14 , Imaging M84 and Sombrero at < 50 gravitational radii: jets and accretion inflow , David H. Hughes
+2024-S1-MX-19 , TolTEC imaging of the progenitors of the Coma cluster at z~5 towards the FRESCO/GOODS-N field , David H. Hughes
+2024-S1-MX-2 , Molecular Clouds in the Fermi Bubble , Laurent Loinard   
+2024-S1-MX-11 , Unveiling a hidden gem: an extraordinarily bright strongly lensed galaxy behind Milky Way dust clouds , Eric Faustino Jiménez-Andrade
+2024-S1-MX-29 , MACS J0018.5 - A Bullet Cluster Aimed in Our Direction , Alfredo Montaña
+2024-S1-MX-13 , A sample of SMBH shadows , David H. Hughes
+2024-S1-MX-37 , Large scale mapping of iCOMs in the prototypical shock laboratory L1157 , Arturo I. Gomez-Ruiz
+2024-S1-MX-24 , Feeding a YSO from the surrounding Molecular Cloud , Roberto Galvan-Madrid
+2024-S1-MX-22 , Identifying Dusty Star-Forming Galaxies in the Early Universe , Alfredo Montaña
+2024-S1-MX-31 , Two belts or one? Toward a complete map of tau Ceti's planetary system , Miguel Chavez Dagostino
+2024-S1-MX-33 , A millimeter key to the multi-wavelength characterization of Fast Evolving Luminous Transients , Alberto Carramiñana
+2024-S1-MX-23 , TolTEC imaging of filamentary structures in SSA22 , David H. Hughes
+2024-S1-MX-1 , A New “Frontier” for the LMT: Dust Obscured Activity in Galaxies that Dominate the Cosmic Star Formation History , Alfredo Montaña
+2024-S1-MX-34 , Mapping the emission of molecular gas in metal-poor nearby galaxies , Olga Vega
+2024-S1-MX-40 , Confirming and Characterizing Galaxy Proto-Clusters in the Early Universe , Alfredo Montaña
+2024-S1-MX-3 , LMT redshift identification of a dusty starburst candidate at z > 6 , Alfredo Montaña
+2024-S1-MX-5 , Mapping the physical conditions of star forming regions , Nadia Mariel Murillo Mejias
+2024-S1-MX-16 , LMT spectroscopy of strong outflows in star-forming galaxies at Cosmic Noon , David H. Hughes 
+2024-S1-MX-20 , Molecular Gas Reservoir in a Sample of Nearby Bulgeless Galaxies , Daniel Rosa Gonzalez
+2024-S1-MX-35 , The mass-loss history of Phoenix giants: toward restrictive constraints on merger models , Miguel Chavez Dagostino
+2024-S1-MX-25 , Pilot program to unveil overdensities around SPIRE Dropouts with TolTEC , Emmaly Aguilar Pérez
+2024-S1-MX-26 , Molecular Emission Detection Program for Seyfert Galaxies , Castalia Alenka Negrete Peñaloza
+2024-S1-MX-18 , Observations of the 57Fe+23 Hyperfine Transition Towards the Virgo Cluster , Daniel Rosa Gonzalez
+2024-S1-MX-38 , Untangling the shocks in the luminous SZE galaxy cluster RXJ1347.5-1145 , David Omar Sánchez-Argüelles 
+2024-S1-MX-6 , TolTEC Mapping of the Lockman-SpReSO Field , Irene Cruz-Gonzalez
+2024-S1-MX-28 , Searching for circumstellar dust in our closest cold neighbor in the northern hemisphere: The Barnard’s Star , Miguel Chavez Dagostino
+2024-S1-MX-36 , Dust in the Whirlpool , Jacopo Fritz
+2024-S1-MX-30 , Search for a new Class II methanol maser to Constrain Maser Models , Stan Kurtz
+2024-S1-MX-9 , Molecular Gas in Compton Thick AGN , Irene Cruz-Gonzalez
+2024-S1-MX-10 , Dust properties of an SMG to AGN transitional galaxy , Itziar Aretxaga
+2024-S1-MX-15 , A search for recently formed dust and molecules in the circumstellar nebula of WR140 , Divakara Mayya
+2024-S1-MX-17 , Luminous-lensed galaxies in Herschel-ATLAS survey: completing high-redshift measurements , Ivan Rodriguez Montoya
+2024-S1-MX-39 , Imaging the Circumstellar Envelope and the Magnetic Field Around the IRAS 17028-1004 Nebula , Enrique Omar Serrano Bernal 
+2024-S1-MX-4 , Density wave triggering of star formation across spiral arms. , Eric E. Martinez-Garcia
+2024-S1-MX-32 , The Strange Case of Masers in Empty Space , Stan Kurtz
+2024-S1-SP-2 , The nature of millimeter continuum emission in LBV stars , Ricardo Rizzo
+2024-S1-SP-6 , Dust in the bulgeless face-on galaxy NGC3938 , Javier Zaragoza Cardiel  
+2024-S1-SP-5 , Molecular Gas in the Antennae Tidal Dwarf Galaxy , Javier Zaragoza Cardiel  
+2024-S1-SP-7 , Molecular Gas of FIR Sources in the Lockman-SpReSO Field , Miguel Sánchez Portal 
+2024-S1-SP-1 , Thermodynamics of the ICM in the CATARSIS clusters , Patricia Sánchez-Blázquez
+2024-S1-UM-4 , TolTEC Observations of the Dwarf Galaxy NGC 4449 , Michael McCrackan
+2024-S1-UM-9 , "RSR Spectroscopy of Massive ,  Dust-Obscured
+2024-S1-UM-1 , Origin and evolution of molecular gas in the ram pressure stripped galactic tail of NGC4388 , Q. Daniel Wang
+2024-S1-UM-6 , Invisible Giants: Identifying the Power Source of Lyman-alpha Blobs , Roxana Popescu
+2024-S1-UM-7 , Probing the Interstellar Medium of Dusty Star-forming Galaxies in a Protocluster at Cosmic Noon , Q. Daniel Wang
+2024-S1-UM-12 , High Resolution SZE Map of a Possibly Disturbed Galaxy Cluster , Carlos Alberto Garcia Diaz
+2024-S1-UM-11 , Testing [C II] Emission as a CGM Tracer around z>6 Galaxies , Min Yun
+2024-S1-UM-5 , 2024 Event Horizon Telescope Observations on the LMT , Gopal Narayanan
+2024-S1-UM-8 , "Mapping dust ,  free-free
+2024-S1-UM-3 , Investigating Core Formation Environments in MonR2 with SEQUOIA/LMT , Amanda M Lee
+2024-S1-UM-10 , A novel observation of pressure fluctuations in the distant cluster RX J1226.9+3332 , Nat DeNigris
+2024-S1-UM-2 , TolTEC Mapping of the JWST NEP-TDF , Zhiyuan Ma
+2024-S1-US-18 , Filling the Spatial Gap: Dust Polarization in Rho Oph A , Erin Cox
+2024-S1-US-3 , CO in Molecular Cloud Boundary Regions with H2 Emission , Paul Goldsmith
+2024-S1-US-19 , Probing The Central Region of the Prestellar Core L183 , Yancy Shirley
+2024-S1-US-5 , Exploring the evolution of total gas contents with the observations of galaxies within z=0.11-0.17 , Hansung Gim
+2024-S1-US-4 , TolTEC Observations of the Extreme Star-forming Region Sgr B2 , Dylan Michelson Pare
+2024-S1-US-16 , Gas and Galaxy Transformation in Abell 262 , Marc Verheijen
+2024-S1-US-13 , Determining the Mass and Spectral Index of Peter Pan Disk J0501 , William Schap
+2024-S1-US-2 , Galaxy groups at mm-wavelengths: the key to AGN feedback , Gerrit Schellenberger
+2024-S1-US-17 , LMT Spectroscopy of Hidden Monsters at z > 4 , Mark Dickinson
+2024-S1-US-20 , When the Wind Blows: hi-res mapping of outflow feedback from massive protostars , Jonathan C. Tan
+2024-S1-US-14 , Investigating The Unusual Spectral Index Seen In IC 348 , William Schap
+2024-S1-US-10 , TolTEC Mapping of the Spitzer IRAC Dark Field , Haojing Yan
+2024-S1-US-7 , Exploration of a Lensed z=2.66 Protocluster Discovered by Planck+LMT+ALMA , Nicholas Foo
+
+
 # 2023
 
 2023-S1-MX-1,A Multicolor View of the Black Hole Environment in M87,Arturo I. Gomez-Ruiz

diff --git a/etc/ProjectId.tab b/etc/ProjectId.tab
@@ -1,3 +1,4 @@
+# this tab file is not used anymore, use the corresponding csv file
 #     missing lmtinfo variables that dataverse needs
 # projectID       PIName               projectTitle
 

diff --git a/examples/upload_project.sh b/examples/upload_project.sh
@@ -13,10 +13,11 @@
 # dvname    - Name of the dataverse to upload to. Default: lmtdata
 # envfile   - file with API credentials  [$HOME/.ssh/su_prod.env]
 # in        - input directory with project/obsnum/*.tar
-# out       - output dir
+# out       - output dir, default: 'ready_for_upload'
 # overwrite - overwrite the output dir.  
 #             If 1, existing directory will be removed. 
 #             If anything else , script will exit if $out exists.
+# publish   - Upload and publish the data to dataverse after preparation
 # verbose   - If non-zero, echo dvpipe commands before executing
 #
 #-------------------------------------------------------------
@@ -30,6 +31,7 @@ envfile=$HOME/.ssh/su_prod.env
 in=data_prod  
 out=ready_for_upload
 overwrite=0
+publish=1
 verbose=0
 #--HELP
 
@@ -54,6 +56,7 @@ echo "envfile:   $envfile"
 echo "in:        $in"
 echo "out:       $out"
 echo "overwrite: $overwrite"
+echo "publish:   $publish"
 echo "verbose:   $verbose"
 
 #-------------------------------------------------------------
@@ -76,6 +79,7 @@ if [ -d "$out" ]; then
     echo "#### Error: $out exists. Use 'overwrite=1' to automatically remove it"
     exit 255
 fi
+mkdir $out
 
 # Probably here, you have to run mk_metadata meta=2
 
@@ -93,11 +97,13 @@ for directory in ${in}/*; do
 done;
 
 # upload and publish (-b major)
-for index in ${out}/*.yaml; do
-    if [ "$verbose" -ne 0 ] || [ $dryrun -ne 0 ] ; then
-        echo "dvpipe -c config_prod.yaml -e ${envfile} -g dataset upload -a none -b major -i $index -p $dvname"
-    fi
-    if [ "$dryrun" -eq 0 ];then
-        dvpipe -c config_prod.yaml -e ${envfile} -g dataset upload -a none -b major -i $index -p $dvname
-    fi
-done
+if [ "$publish" -eq 1 ]; then
+    for index in ${out}/*.yaml; do
+        if [ "$verbose" -ne 0 ] || [ $dryrun -ne 0 ] ; then
+            echo "dvpipe -c config_prod.yaml -e ${envfile} -g dataset upload -a none -b major -i $index -p $dvname"
+        fi
+        if [ "$dryrun" -eq 0 ];then
+            dvpipe -c config_prod.yaml -e ${envfile} -g dataset upload -a none -b major -i $index -p $dvname
+        fi
+    done
+fi