Skip to content

Commit

Permalink
Merge pull request #58 from fact-project/rename_cols
Browse files Browse the repository at this point in the history
Rename cols
  • Loading branch information
kbruegge authored Dec 15, 2017
2 parents 6e6b688 + 3661982 commit b48079f
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 4 deletions.
20 changes: 17 additions & 3 deletions erna/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,24 @@
from datetime import timedelta

from fact.io import to_h5py
from fact.instrument import camera_distance_mm_to_deg

from . import datacheck_conditions as dcc
from .datacheck import get_runs, get_drs_runs
from .hdf_utils import rename_columns

logger = logging.getLogger(__name__)


def add_theta_deg_columns(df):
for i in range(6):
incol = 'theta' if i == 0 else 'theta_off_{}'.format(i)
outcol = 'theta_deg' if i == 0 else 'theta_deg_off_{}'.format(i)
if incol in df.columns:
df[outcol] = camera_distance_mm_to_deg(df[incol])



def build_path(row, path_to_data, extension):
"""
builds a path to the fact data given the night, extension and filename
Expand All @@ -32,7 +43,7 @@ def test_drs_path(df, key):
"""
mask = df[key].apply(os.path.exists)
df['drs_file_exists'] = mask

return df


Expand Down Expand Up @@ -106,6 +117,9 @@ def collect_output(job_outputs, output_path, df_started_runs=None, **kwargs):
df_returned_data.total_on_time_in_seconds = total_on_time_in_seconds
df_returned_data.failed_jobs=difference

df_returned_data.columns = rename_columns(df_returned_data.columns)
add_theta_deg_columns(df_returned_data)

name, extension = os.path.splitext(output_path)
if extension not in ['.json', '.h5', '.hdf5', '.hdf' , '.csv']:
logger.warn("Did not recognize file extension {}. Writing to JSON".format(extension))
Expand Down Expand Up @@ -211,7 +225,7 @@ def load(
earlier_drs_entries = earlier_drs_entries.fillna(axis="index", method="ffill")
later_drs_entries = drs_data.reindex(data.index, method="backfill")
later_drs_entries = later_drs_entries.fillna(axis="index", method="ffill")

# when backfilling the drs obeservations the last rows might be invalid and contain nans.
# We cannot drop them becasue the tables have to have the same length.
# in that case simply fill them up.
Expand Down Expand Up @@ -241,7 +255,7 @@ def load(
"NIGHT",
"RUNID",
])

mapping = mapping.dropna(how='any')

logger.info("Fetched {} data runs and approx {} drs entries from database where time delta is less than {} minutes".format(len(mapping), mapping['drs_path'].nunique(), timedelta_in_minutes))
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

setup(
name='erna',
version='0.5.2',
version='0.6.0',
description='Easy RuN Access. Tools that help to do batch processing of FACT data',
url='https://github.com/fact-project/erna',
author='Kai Brügge, Jens Buss, Maximilian Nöthe',
Expand Down

0 comments on commit b48079f

Please sign in to comment.