Merge pull request #30 from fact-project/fix_checkavailability

Fix checkavailability

erna/scripts/check_availability.py

@@ -13,13 +13,12 @@
 from ..automatic_processing.custom_fields import night_int_to_date
 from ..utils import load_config
 
-log = logging.getLogger()
-log.setLevel(logging.INFO)
+log = logging.getLogger('erna')
 
 handler = logging.StreamHandler()
 formatter = logging.Formatter(fmt='%(asctime)s|%(levelname)s|%(name)s|%(message)s')
 handler.setFormatter(formatter)
-log.addHandler(handler)
+logging.getLogger().addHandler(handler)
 
 db_specification = 'mysql+pymysql://{user}:{password}@{host}/{database}'
 
@@ -38,37 +37,43 @@ def check_availability(run, basedir='/fact/raw'):
         basedir, str(run.night.year), '{:02d}'.format(run.night.month),
         '{:02d}'.format(run.night.day), '{:%Y%m%d}_{:03d}'.format(run.night, run.run_id)
     )
-    log.debug('Basename: {}'.format(basename))
 
-    if run.run_type == 1:
-        f = RawDataFile.select_night_runid(run.night, run.run_id)
-        available = isfile(basename + '.fits.fz') or isfile(basename + '.fits.gz')
-        log.debug('Available: {}'.format(available))
-
-        f.available = available
-        f.save()
-
-    elif run.run_type == 2 and run.drs_step == 2:
+    if run.drs_step == 2:
         log.debug('is a drs file')
-        f = DrsFile.select_night_runid(run.night, run.run_id)
-        available = isfile(basename + '.drs.fits.gz')
-        log.debug('Available: {}'.format(available))
-        f.available = available
-        f.save()
+
+        try:
+            f = DrsFile.get(night=run.night, run_id=run.run_id)
+            available = isfile(basename + '.drs.fits.gz')
+            log.debug('Available: {}'.format(available))
+            if available != f.available:
+                f.available = available
+                f.save()
+        except DrsFile.DoesNotExist:
+            log.info('DrsFile {:%Y%m%d}_{:03d} not in database'.format(
+                run.night, run.run_id
+            ))
 
     else:
-        log.debug('Neither drs nor data file')
+        try:
+            f = RawDataFile.get(night=run.night, run_id=run.run_id)
+            available = isfile(basename + '.fits.fz') or isfile(basename + '.fits.gz')
+            log.debug('Available: {}'.format(available))
+
+            if available != f.available:
+                f.available = available
+                f.save()
+        except RawDataFile.DoesNotExist:
+            log.info('Run {:%Y%m%d}_{:03d} not in database'.format(
+                run.night, run.run_id
+            ))
 
 
 @click.command()
-@click.option('--year', help='The year to update (default all)')
-@click.option('--month', help='The month to update (default all)')
-@click.option('--day', help='The day to update (default all)')
 @click.option('--config', '-c', help='Yaml file containing database credentials')
 @click.option('--verbose', '-v', help='Set logging level to DEBUG', is_flag=True)
 @click.option('--start', type=parse_date, default=str(datetime.date(2011, 10, 1)))
 @click.option('--end', type=parse_date, default=str(datetime.date.today()))
-def main(year, month, day, config, verbose, start, end):
+def main(config, verbose, start, end):
     '''
     Check if RawDataFiles and DrsFiles are available.
     Goes through the database entries and checks if the file is where it is expected

erna/scripts/fill_database.py

@@ -55,7 +55,6 @@ def main(start, end, config):
     )
     runs['fNight'] = pd.to_datetime(runs.fNight.astype(str), format='%Y%m%d')
 
-
     # fill all non drs runs into raw_data_files
     fill_data_runs(runs.query('fDrsStep != 2'), database=database)
     # fill all drs runs into drs_files