From 0c7373537aa9d5180f019fb901279745d4380fb8 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 13 Jul 2022 13:52:27 -0600 Subject: [PATCH 001/157] Update requirements --- requirements-dev.in | 1 - requirements-dev.txt | 37 +++++++++++++++++-------------------- requirements.in | 5 +++-- requirements.txt | 7 +++++++ 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/requirements-dev.in b/requirements-dev.in index 8bcde3d5..baeada75 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,6 +1,5 @@ -rrequirements.txt -black>=22.0, <=23.0 pre-commit>=2.0, <3.0 pytest>=7.0, <8.0 tox>=3.0, <=4.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index ab9fe9b1..5ff3c8f8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,10 +8,10 @@ asgiref==3.5.0 # via # -r requirements.txt # django +atomicwrites==1.4.0 + # via pytest attrs==21.4.0 # via pytest -black==22.1.0 - # via -r requirements-dev.in certifi==2021.10.8 # via # -r requirements.txt @@ -22,8 +22,10 @@ charset-normalizer==2.0.12 # via # -r requirements.txt # requests -click==8.0.4 - # via black +colorama==0.4.5 + # via + # pytest + # tox distlib==0.3.4 # via virtualenv django==3.2.12 @@ -40,7 +42,6 @@ idna==3.3 # requests importlib-metadata==4.11.2 # via - # click # pluggy # pre-commit # pytest @@ -50,25 +51,24 @@ iniconfig==1.1.1 # via pytest its-preselector @ git+https://github.com/NTIA/Preselector@1.0.0 # via -r requirements.txt -mypy-extensions==0.4.3 - # via black nodeenv==1.6.0 # via pre-commit +numexpr==2.8.3 + # via -r requirements.txt numpy==1.21.5 # via # -r requirements.txt + # numexpr # scipy # sigmf packaging==21.3 # via + # -r requirements.txt + # numexpr # pytest # tox -pathspec==0.9.0 - # via black platformdirs==2.5.1 - # via - # black - # virtualenv + # via virtualenv pluggy==1.0.0 # via # pytest @@ -79,8 +79,10 @@ py==1.11.0 # via # pytest # tox -pyparsing==3.0.7 - # via packaging +pyparsing==3.0.9 + # via + # -r requirements.txt + # packaging pytest==7.0.1 # via -r requirements-dev.in python-dateutil==2.8.2 @@ -121,18 +123,13 @@ toml==0.10.2 # pre-commit # tox tomli==2.0.1 - # via - # black - # pytest + # via pytest tox==3.24.5 # via -r requirements-dev.in -typed-ast==1.5.2 - # via black typing-extensions==4.1.1 # via # -r requirements.txt # asgiref - # black # importlib-metadata urllib3==1.26.8 # via diff --git a/requirements.in b/requirements.in index e55d35ab..06accdbc 100644 --- a/requirements.in +++ b/requirements.in @@ -2,6 +2,7 @@ django>=3.0, <4.0 numpy>=1.0, <2.0 python-dateutil>=2.0, < 3.0 ruamel.yaml>=0.1, <1.0 -scipy>=1.0, <2.0 +scipy>=1.6.0, <2.0 +numexpr>=2.8.3, <3.0 SigMF @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive -its-preselector @ git+https://github.com/NTIA/Preselector@1.0.0#egg=its-preselector \ No newline at end of file +its-preselector @ git+https://github.com/NTIA/Preselector@1.0.0#egg=its-preselector diff --git a/requirements.txt b/requirements.txt index 9cb3ae9f..3d11c041 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,11 +16,18 @@ idna==3.3 # via requests its-preselector @ git+https://github.com/NTIA/Preselector@1.0.0 # via -r requirements.in +numexpr==2.8.3 + # via -r requirements.in numpy==1.21.5 # via # -r requirements.in + # numexpr # scipy # sigmf +packaging==21.3 + # via numexpr +pyparsing==3.0.9 + # via packaging python-dateutil==2.8.2 # via -r requirements.in pytz==2021.3 From f783737a17ca7bb7a278b7b420f241b7efb02459 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 13 Jul 2022 13:57:45 -0600 Subject: [PATCH 002/157] New DSP code --- scos_actions/actions/fft.py | 162 --------------- scos_actions/signal_processing/apd.py | 84 ++++++++ scos_actions/signal_processing/calibration.py | 131 ++++++++++-- scos_actions/signal_processing/fft.py | 187 ++++++++++++++++++ .../signal_processing/power_analysis.py | 134 +++++++++++++ .../signal_processing/unit_conversion.py | 145 ++++++++++++++ scos_actions/signal_processing/utils.py | 16 -- 7 files changed, 669 insertions(+), 190 deletions(-) delete mode 100644 scos_actions/actions/fft.py create mode 100644 scos_actions/signal_processing/apd.py create mode 100644 scos_actions/signal_processing/fft.py create mode 100644 scos_actions/signal_processing/power_analysis.py create mode 100644 scos_actions/signal_processing/unit_conversion.py delete mode 100644 scos_actions/signal_processing/utils.py diff --git a/scos_actions/actions/fft.py b/scos_actions/actions/fft.py deleted file mode 100644 index 93b222f7..00000000 --- a/scos_actions/actions/fft.py +++ /dev/null @@ -1,162 +0,0 @@ -import os -from enum import Enum -import logging -import numpy as np -from scipy.signal import windows - -logger = logging.getLogger(__name__) - - -class M4sDetector(Enum): - min = "fft_min_power" - max = "fft_max_power" - mean = "fft_mean_power" - median = "fft_median_power" - sample = "fft_sample_power" - - -def m4s_detector(array): - """Take min, max, mean, median, and random sample of n-dimensional array. - - Detector is applied along each column. - - :param array: an (m x n) array of real frequency-domain linear power values - :returns: a (5 x n) in the order min, max, mean, median, sample in the case - that `detector` is `m4s`, otherwise a (1 x n) array - - """ - amin = np.min(array, axis=0) - amax = np.max(array, axis=0) - mean = np.mean(array, axis=0) - median = np.median(array, axis=0) - random_sample = array[np.random.randint(0, array.shape[0], 1)][0] - m4s = np.array([amin, amax, mean, median, random_sample], dtype=np.float32) - - return m4s - - -def mean_detector(array): - mean = np.mean(array, axis=0) - return mean - - -def get_frequency_domain_data(time_data, sample_rate, fft_size, fft_window, fft_window_acf): - logger.debug( - 'Converting {} samples at {} to freq domain with fft_size {}'.format(len(time_data), sample_rate, fft_size)) - # Resize time data for FFTs - num_ffts = int(len(time_data) / fft_size) - time_data = np.resize(time_data, (num_ffts, fft_size)) - # Apply the FFT window - data = time_data * fft_window - # Take and shift the fft (center frequency) - complex_fft = np.fft.fft(data) - complex_fft = np.fft.fftshift(complex_fft) - complex_fft /= 2 - # Convert from V/Hz to V - complex_fft /= fft_size - # Apply the window's amplitude correction factor - complex_fft *= fft_window_acf - return complex_fft - - -def convert_volts_to_watts(complex_fft): - # Convert to power P=V^2/R - power_fft = np.abs(complex_fft) - power_fft = np.square(power_fft) - power_fft /= 50 - return power_fft - - -def apply_detector(power_fft): - # Run the M4S detector - power_fft_m4s = m4s_detector(power_fft) - return power_fft_m4s - - -def convert_watts_to_dbm(power_fft): - # If testing, don't flood output with divide-by-zero warnings from np.log10 - # if settings.RUNNING_TESTS: - if "PYTEST_CURRENT_TEST" in os.environ: - np_error_settings_savepoint = np.seterr(divide="ignore") - # Convert to dBm dBm = dB +30; dB = 10log(W) - power_fft_dbm = 10 * np.log10(power_fft) + 30 - return power_fft_dbm - - -def get_fft_window(window_type, window_length): - # Generate the window with the right number of points - window = None - if window_type == "Bartlett": - window = windows.bartlett(window_length) - if window_type == "Blackman": - window = windows.blackman(window_length) - if window_type == "Blackman Harris": - window = windows.blackmanharris(window_length) - if window_type == "Flat Top": - window = windows.flattop(window_length) - if window_type == "Hamming": - window = windows.hamming(window_length) - if window_type == "Hanning": - window = windows.hann(window_length) - - # If no window matched, use a rectangular window - if window is None: - window = np.ones(window_length) - - # Return the window - return window - - -def get_fft_window_correction(window, correction_type="amplitude"): - # Calculate the requested correction factor - window_correction = 1 # Assume no correction - if correction_type == "amplitude": - window_correction = 1 / np.mean(window) - if correction_type == "energy": - window_correction = np.sqrt(1 / np.mean(window ** 2)) - - # Return the window correction factor - return window_correction - - -def get_fft_frequencies(fft_size, sample_rate, center_frequency): - time_step = 1 / sample_rate - frequencies = np.fft.fftfreq(fft_size, time_step) - frequencies = np.fft.fftshift(frequencies) + center_frequency - return frequencies - - -def get_m4s_watts(fft_size, measurement_result, fft_window, fft_window_acf): - complex_fft = get_frequency_domain_data( - measurement_result["data"], measurement_result["sample_rate"], fft_size, fft_window, fft_window_acf - ) - power_fft = convert_volts_to_watts(complex_fft) - power_fft_m4s = apply_detector(power_fft) - return power_fft_m4s - - -def get_m4s_dbm(fft_size, measurement_result, fft_window, fft_window_acf): - m4_watts = get_m4s_watts(fft_size, measurement_result, fft_window, fft_window_acf) - power_fft_dbm = convert_watts_to_dbm(m4_watts) - return power_fft_dbm - - -def get_mean_detector_watts(fft_size, measurement_result, fft_window, fft_window_acf): - complex_fft = get_frequency_domain_data( - measurement_result["data"], measurement_result["sample_rate"], fft_size, fft_window, fft_window_acf - ) - power_fft = convert_volts_to_watts(complex_fft) - return mean_detector(power_fft) - - -def get_enbw(sample_rate, fft_size, fft_window_enbw): - enbw = sample_rate - enbw *= fft_window_enbw - enbw /= fft_size - return enbw - -def get_fft_window_correction_factors(fft_window): - fft_window_acf = get_fft_window_correction(fft_window, "amplitude") - fft_window_ecf = get_fft_window_correction(fft_window, "energy") - fft_window_enbw = (fft_window_acf / fft_window_ecf) ** 2 - return fft_window_acf, fft_window_ecf, fft_window_enbw diff --git a/scos_actions/signal_processing/apd.py b/scos_actions/signal_processing/apd.py new file mode 100644 index 00000000..a4b6f1d2 --- /dev/null +++ b/scos_actions/signal_processing/apd.py @@ -0,0 +1,84 @@ +import logging +from typing import Tuple + +import numexpr as ne +import numpy as np + +from scos_actions.signal_processing.unit_conversion import convert_linear_to_dB + +logger = logging.getLogger(__name__) + + +def get_apd( + time_data: np.ndarray, bin_size_dB: float = None +) -> Tuple[np.ndarray, np.ndarray]: + """Estimate the APD by sampling the CCDF. + + The size of the output depends on ``bin_size_dB``, which + determines the effective downsampling of IQ data into an APD dataset. + Higher bin sizes will lead to smaller data sizes but less resolution. + Inversely, smaller bin sizes will result in larger data size output + with higher resolution. + + Not setting ``bin_size_dB`` will result in no downsampling of the data + and will output the same data size as ``time_data``. + + No additional scaling is applied, so resulting amplitude units are + dBV. Typical applications will require converting this result to + power units. + + :param time_data: Input complex baseband IQ samples. + :param bin_size_dB: Amplitude granularity, in dB, for estimating the APD. + If not specified, the APD will not be downsampled (default behavior). + :return: A tuple (p, a) of NumPy arrays, where p contains the APD + probabilities, and a contains the APD amplitudes. + """ + # Convert IQ to amplitudes + all_amps = ne.evaluate("abs(time_data).real") + + # Replace any 0 value amplitudes with NaN + all_amps[all_amps == 0] = np.nan + + # Convert amplitudes from V to dBV + all_amps = convert_linear_to_dB(all_amps) + + if bin_size_dB is None: + # No downsampling + a = np.sort(all_amps) + p = 1 - ((np.arange(len(a)) + 1) / len(a)) + else: + # Generate bins based on bin_size_dB for downsampling + a = np.arange( + np.nanmin(all_amps), np.nanmax(all_amps) + bin_size_dB, bin_size_dB + ) + # Get counts of amplitudes exceeding each bin value + p = sample_ccdf(all_amps, a) + + # Replace peak amplitude 0 count with NaN + p[-1] = np.nan + logger.debug(f"APD result length: {len(a)} samples.") + + return p, a + + +def sample_ccdf(a: np.ndarray, edges: np.ndarray, density: bool = True) -> np.ndarray: + """ + Computes the fraction (or total number) of samples in `a` that + exceed each edge value. + + :param a: the vector of input samples + :param edges: sample threshold values at which to characterize the distribution + :param density: if True, the sample counts are normalized by `a.size` + :return: The empirical complementary cumulative distribution + """ + # 'left' makes the bin interval open-ended on the left side + # (the CCDF is "number of samples exceeding interval") + edge_inds = np.searchsorted(edges, a, side="left") + bin_counts = np.bincount(edge_inds, minlength=edges.size + 1) + ccdf = (a.size - bin_counts.cumsum())[:-1] + + if density: + ccdf = ccdf.astype("float64") + ccdf /= a.size + + return ccdf diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py index d9db101c..7748ac50 100644 --- a/scos_actions/signal_processing/calibration.py +++ b/scos_actions/signal_processing/calibration.py @@ -1,21 +1,128 @@ import numpy as np -from scipy.constants import Boltzmann as k_b +from scipy.constants import Boltzmann import logging - +from typing import Tuple +from scos_actions.hardware import preselector +from scos_actions.signal_processing.unit_conversion import ( + convert_celsius_to_kelvins, + convert_dB_to_linear, + convert_fahrenheit_to_celsius, + convert_linear_to_dB, +) logger = logging.getLogger(__name__) -def y_factor(pwr_noise_on_watts, pwr_noise_off_watts, ENR, ENBW, T_room=290.): - # Y-Factor calculations (element-wise from power arrays) - logger.debug('ENR:{}'.format(ENR)) - logger.debug('ENBW:{}'.format(ENBW)) - logger.debug('mean power on: {}'.format(np.mean(pwr_noise_on_watts))) - logger.debug('mean power off: {}'.format(np.mean(pwr_noise_off_watts))) + +class CalibrationException(Exception): + """Basic exception handling for calibration functions.""" + + def __init__(self, msg): + super().__init__(msg) + + +def y_factor( + pwr_noise_on_watts: np.ndarray, + pwr_noise_off_watts: np.ndarray, + enr_linear: float, + enbw_hz: float, + temp_kelvins: float = 300.0, +) -> Tuple[float, float]: + """ + Perform Y-Factor calculations of noise figure and gain. + + Noise factor and linear gain are computed element-wise from + the input arrays using the Y-Factor method. The linear values + are then averaged and converted to dB. + + :param pwr_noise_on_watts: Array of power values, in Watts, + recorded with the calibration noise source on. + :param pwr_noise_off_watts: Array of power values, in Watts, + recorded with the calibration noise source off. + :param enr_linear: Calibration noise source excess noise + ratio, in linear units. + :param enbw_hz: Equivalent noise bandwidth, in Hz. + :param temp_kelvins: Temperature, in Kelvins. If not given, + a default value of 300 K is used. + :return: A tuple (noise_figure, gain) containing the calculated + noise figure and gain, both in dB, from the Y-factor method. + """ + logger.debug(f"ENR: {convert_linear_to_dB(enr_linear)} dB") + logger.debug(f"ENBW: {enbw_hz} Hz") + logger.debug(f"Mean power on: {np.mean(pwr_noise_on_watts)} W") + logger.debug(f"Mean power off: {np.mean(pwr_noise_off_watts)} W") y = pwr_noise_on_watts / pwr_noise_off_watts - noise_factor = ENR / (y - 1) - gain_watts = pwr_noise_on_watts / (k_b * T_room * ENBW * (ENR + noise_factor)) + noise_factor = enr_linear / (y - 1.0) + gain_watts = pwr_noise_on_watts / ( + Boltzmann * temp_kelvins * enbw_hz * (enr_linear + noise_factor) + ) # Get mean values from arrays and convert to dB - noise_figure = 10. * np.log10(np.mean(noise_factor)) # dB - gain = 10. * np.log10(np.mean(gain_watts)) + noise_figure = convert_linear_to_dB(np.mean(noise_factor)) + gain = convert_linear_to_dB(np.mean(gain_watts)) return noise_figure, gain + +def get_linear_enr(cal_source_idx: int = None) -> float: + """ + Get the excess noise ratio of a calibration source. + + Specifying ``cal_source_idx`` is optional as long as there is + only one calibration source. It is required if multiple + calibration sources are present. + + The preselector is loaded from scos_actions.hardware. + + :param cal_source_idx: The index of the specified + calibration source in preselector.cal_sources. + :return: The excess noise ratio of the specified + calibration source, in linear units. + :raises CalibrationException: If multiple calibration sources are + available but `cal_source_idx` is not specified. + :raises IndexError: If the specified calibration source + index is out of range for the current preselector. + """ + if len(preselector.cal_sources) == 0: + raise CalibrationException("No calibration sources defined in preselector.") + elif len(preselector.cal_sources) == 1 and cal_source_idx is None: + # Default to the only cal source available + cal_source_idx = 0 + elif len(preselector.cal_sources) > 1 and cal_source_idx is None: + # Must specify index if multiple sources available + raise CalibrationException( + "Preselector contains multiple calibration sources, " + + "and the source index was not specified." + ) + try: + enr_dB = preselector.cal_sources[cal_source_idx].enr + except IndexError: + raise IndexError( + f"Calibration source index {cal_source_idx} out of range " + + "while trying to get ENR value." + ) + enr_linear = convert_dB_to_linear(enr_dB) + return enr_linear + + +def get_temperature(sensor_idx: int = None) -> Tuple[float, float, float]: + """ + Get the temperature from a preselector sensor. + + The preselector is loaded from scos_actions.hardware + + :param sensor_idx: The index of the desired temperature + sensor in the preselector. + :raises CalibrationException: If no sensor index is provided, or + if no value is returned after querying the sensor. + :return: A tuple of floats (temp_k, temp_c, temp_f) containing + the retrieved temperature in Kelvins, degrees Celsius, and + degrees Fahrenheit, respectively. + """ + if sensor_idx is None: + raise CalibrationException("Temperature sensor index not specified.") + temp = preselector.get_sensor_value(sensor_idx) + if temp is None: + raise CalibrationException("Failed to get temperature from sensor.") + logger.debug(f"Got temperature from sensor: {temp} deg. Fahrenheit") + temp_f = float(temp) + temp_c = convert_fahrenheit_to_celsius(temp_f) + temp_k = convert_celsius_to_kelvins(temp_c) + return temp_k, temp_c, temp_f diff --git a/scos_actions/signal_processing/fft.py b/scos_actions/signal_processing/fft.py new file mode 100644 index 00000000..901a3740 --- /dev/null +++ b/scos_actions/signal_processing/fft.py @@ -0,0 +1,187 @@ +import logging +import os +from enum import Enum, EnumMeta + +import numpy as np +from scipy.fft import fft as sp_fft +from scipy.signal import get_window + +logger = logging.getLogger(__name__) + + +def get_fft( + time_data: np.ndarray, + fft_size: int, + norm: str = "forward", + fft_window: np.ndarray = None, + num_ffts: int = 0, + workers: int = os.cpu_count() // 2, +) -> np.ndarray: + """ + Compute the 1-D DFT using the FFT algorithm. + + The input time domain samples are reshaped based on fft_size + and num_ffts. Then, the window is applied. The FFT is performed + and frequencies are shifted. The FFT is calculated using the + scipy.fft.fft method, which is leveraged for parallelization. + + This function only scales the FFT output by 1/fft_size. No + other power scaling, including RF/baseband conversion, is applied. + It is recommended to first apply statistical detectors, if any, + and apply power scaling as needed after converting values to dB, + if applicable - this approach generally results in faster + computation, and keeps power scaling details contained within + individual actions. + + By default, as many FFTs as possible are computed, based on the + length of the time_data input and the fft_size. If num_ffts is + specified, the time domain data will be truncated to length + (fft_size * num_ffts) before FFTs are computed. If num_ffts is not + specified, but the length of the input time domain data is not + evenly divisible by fft_size, the time domain data will still be + truncated. + + :param time_data: An array of time domain samples, which can be + complex. + :param fft_size: Length of FFT (N_Bins). + :param norm: Normalization mode. Valid options are 'backward', + 'ortho', and 'forward'. Backward applies no normalization, + while 'forward' applies 1/fft_size scaling and 'ortho' applies + 1/sqrt(fft_size) scaling. Defaults to 'forward'. + :param fft_window: An array of window samples (see get_fft_window). + If not given, no windowing is performed (equivalent to rectangular + windowing). + :param num_ffts: The number of FFTs of length fft_size to compute. + Setting this to zero or a negative number results in "as many + as possible" behavior, which is also the default behavior if + num_ffts is not specified. + :param workers: Maximum number of workers to use for parallel + computation. See scipy.fft.fft for more details. + :return: The transformed input, scaled based on the specified + normalization mode. + """ + # Get num_ffts for default case: as many as possible + if num_ffts <= 0: + num_ffts = int(len(time_data) // fft_size) + + # Determine if truncation will occur and raise a warning if so + if len(time_data) != fft_size * num_ffts: + thrown_away_samples = len(time_data) - (fft_size * num_ffts) + msg = "Time domain data length is not divisible by num_ffts.\nTime" + msg += "domain data will be truncated; Throwing away last " + msg += f"{thrown_away_samples} sample(s)." + logger.warning(msg) + + # Resize time data for FFTs + time_data = np.reshape(time_data[: num_ffts * fft_size], (num_ffts, fft_size)) + + # Apply the FFT window if provided + if fft_window is not None: + time_data *= fft_window + + # Take and shift the FFT + complex_fft = sp_fft(time_data, norm=norm, workers=workers) + complex_fft = np.fft.fftshift(complex_fft) + return complex_fft + + +def get_fft_window(window_type: str, window_length: int) -> np.ndarray: + """ + Generate a periodic window of the specified length. + + Supported values for window_type: boxcar, triang, blackman, + hamming, hann (also "Hanning" supported for backwards compatibility), + bartlett, flattop, parzen, bohman, blackmanharris, nuttall, barthann, + cosine, exponential, tukey, and taylor. + + If an invalid window type is specified, a boxcar (rectangular) + window will be used instead. + + :param window_type: A string supported by scipy.signal.get_window. + Only windows which do not require additional parameters are + supported. Whitespace and capitalization are ignored. + :param window_length: The number of samples in the window. + :return: An array of window samples, of length window_length and + type window_type. + """ + # String formatting for backwards-compatibility + window_type = window_type.lower().strip().replace(" ", "") + + # Catch Hanning window for backwards-compatibility + if window_type == "hanning": + window_type = "hann" + + # Get window samples + try: + window = get_window(window_type, window_length) + except ValueError: + logger.debug( + "Error generating FFT window. Attempting to" + + " use a rectangular window..." + ) + window = get_window("boxcar", window_length) + + # Return the window + return window + + +def get_fft_window_correction(window: np.ndarray, correction_type: str) -> float: + """ + Get the amplitude or energy correction factor for a window. + + :param window: The array of window samples. + :param correction_type: Which correction factor to return. + Must be one of 'amplitude' or 'energy'. + :return: The specified window correction factor. + :raises ValueError: If the correction type is neither 'energy' + nor 'amplitude'. + """ + if correction_type == "amplitude": + window_correction = 1.0 / np.mean(window) + elif correction_type == "energy": + window_correction = np.sqrt(1.0 / np.mean(window**2)) + else: + raise ValueError(f"Invalid window correction type: {correction_type}") + + return window_correction + + +def get_fft_frequencies( + fft_size: int, sample_rate: float, center_frequency: float +) -> list: + """ + Get the frequency axis for an FFT. + + The units of sample_rate and center_frequency should be the same: + if both are given in Hz, the returned frequency values will be in + Hz. If both are in MHz, the returned frequency values will be in + MHz. It is recommended to keep them both in Hz. + + :param fft_size: The length, in samples, of the FFT (N_Bins). + :param sample_rate: The sample rate for the transformed time domain + samples, in Hz. + :param center_frequency: The center frequency, in Hz. + :return: A list of values representing the frequency axis of the + FFT. + """ + time_step = 1.0 / sample_rate + frequencies = np.fft.fftfreq(fft_size, time_step) + frequencies = np.fft.fftshift(frequencies) + center_frequency + return frequencies.tolist() + + +def get_fft_enbw(fft_window: np.ndarray, sample_rate: float) -> float: + """ + Get the equivalent noise bandwidth of an FFT bin. + + The FFT size is inferred from the number of samples + in the input window. + + :param fft_window: An array of window samples. + :param sample_rate: The sampling rate, in Hz. + """ + # window_enbw is (amplitude_correction/energy_correction)^2 + # Here, get_fft_window_correction is not used in order to + # simplify the calculation. + window_enbw = np.mean(fft_window**2) / (np.mean(fft_window) ** 2) + return (sample_rate / len(fft_window)) * window_enbw diff --git a/scos_actions/signal_processing/power_analysis.py b/scos_actions/signal_processing/power_analysis.py new file mode 100644 index 00000000..8ea99bf4 --- /dev/null +++ b/scos_actions/signal_processing/power_analysis.py @@ -0,0 +1,134 @@ +import logging +from enum import Enum, EnumMeta + +import numexpr as ne +import numpy as np + +logger = logging.getLogger(__name__) + + +def calculate_power_watts(val_volts, impedance_ohms: float = 50.0): + """ + Calculate power in Watts from time domain samples in Volts. + + Calculation: (abs(val_volts)^2) / impedance_ohms + + NumPy is used for scalar inputs. + NumExpr is used to speed up the operation for arrays. + + The calculation assumes 50 Ohm impedance by default. + + :param val_volts: A value, or array of values, in Volts. + The input may be complex or real. + :param impedance_ohms: The impedance value to use when + converting from Volts to Watts. + :return: The input val_volts, converted to Watts. The + returned quantity is always real. + """ + if np.isscalar(val_volts): + power = (np.abs(val_volts) ** 2) / impedance_ohms + else: + power = ne.evaluate("(abs(val_volts).real**2)/impedance_ohms") + return power + + +def create_power_detector(name: str, detectors: list) -> EnumMeta: + """ + Construct a power detector based on a list of selected detectors. + + This allows for constructing new detectors while preserving the + order of the 5 possible detector types in all instances. The five + possible detector types to include are min, max, mean, median, and + sample. + + The returned enumeration can be passed to ``apply_power_detector()``. + + :param name: The name of the returned detector enumeration. + :param detectors: A list of strings specifying the detectors. Valid + contents are: 'min', 'max', 'mean', 'median', and 'sample'. + + :return: The detector enumeration created based on the input parameters. + """ + # Construct 2-tuples to create enumeration + _args = [] + if "min" in detectors: + _args.append(("min", "min_power")) + if "max" in detectors: + _args.append(("max", "max_power")) + if "mean" in detectors: + _args.append(("mean", "mean_power")) + if "median" in detectors: + _args.append(("median", "median_power")) + if "sample" in detectors: + _args.append(("sample", "sample_power")) + return Enum(name, tuple(_args)) + + +def apply_power_detector( + data: np.ndarray, detector: EnumMeta, dtype: type = None +) -> np.ndarray: + """ + Apply statistical detectors to a 2-D array of samples. + + Statistical detectors are applied along axis 0 (column-wise), + and the sample detector selects a single row from the 2-D + array at random. + + If the input samples are power FFT samples, they are expected + to be packed in the shape (N_FFTs, N_Bins). + + The shape of the output depends on the number of detectors + specified. The order of the results always follows min, max, mean, + median, sample - regardless of which detectors are used. This + ordering matches that of the detector enumerations. + + Create a detector using ``create_power_detector()`` + + :param data: A 2-D array of real, linear samples. + :param detector: A detector enumeration containing any combination + of 'min', 'max', 'mean', 'median', and 'sample'. Also see the + create_fft_detector and create_time_domain_detector documentation. + :param dtype: Data type of values within the returned array. If not + provided, the type is determined by NumPy as the minimum type + required to hold the values (see numpy.array). + :return: A 2-D array containing the selected detector results + as the specified dtype. The number of rows is equal to the + number of detectors applied, and the number of columns is equal + to the number of columns in the input array. + """ + # Currently this is identical to apply_fft_detector: make general? + # Get detector names from detector enumeration + detectors = [d.name for _, d in enumerate(detector)] + # Get functions based on specified detector + detector_functions = [] + if "min" in detectors: + detector_functions.append(np.min) + if "max" in detectors: + detector_functions.append(np.max) + if "mean" in detectors: + detector_functions.append(np.mean) + if "median" in detectors: + detector_functions.append(np.median) + # Apply statistical detectors + result = [d(data, axis=0) for d in detector_functions] + # Add sample detector result if configured + if "sample" in detectors: + rng = np.random.default_rng() + result.append(data[rng.integers(0, data.shape[0], 1)][0]) + del rng + return np.array(result, dtype=dtype) + + +def filter_quantiles(x: np.ndarray, q_lo: float, q_hi: float) -> np.ndarray: + """ + Replace values outside specified quantiles with NaN. + + :param x: Input N-dimensional data array. + :param q_lo: Lower quantile, 0 <= q_lo < q_hi. + :param q_hi: Upper quantile, q_lo < q_hi <= 1. + :return: The input data array, with values outside the + specified quantile replaced with NaN (numpy.nan). + """ + lo, hi = np.quantile(x, [q_lo, q_hi]) # Works on flattened array + nan = np.nan + return ne.evaluate("x + where((x<=lo)|(x>hi), nan, 0)") diff --git a/scos_actions/signal_processing/unit_conversion.py b/scos_actions/signal_processing/unit_conversion.py new file mode 100644 index 00000000..2a4726e4 --- /dev/null +++ b/scos_actions/signal_processing/unit_conversion.py @@ -0,0 +1,145 @@ +import os +import warnings +from typing import Union + +import numexpr as ne +import numpy as np + + +def suppress_divide_by_zero_when_testing(): + # If testing, don't output divide-by-zero warnings from log10 + # This handles NumPy and NumExpr warnings + if "PYTEST_CURRENT_TEST" in os.environ: + warnings.filterwarnings("ignore", message="divide by zero") + np_error_settings_savepoint = np.seterr(divide="ignore") + + +def convert_watts_to_dBm( + val_watts: Union[float, np.ndarray] +) -> Union[float, np.ndarray]: + """ + Convert from Watts to dBm. + + Calculation: ``10 * log10(val_watts) + 30`` + + NumPy is used for scalar inputs. + NumExpr is used to speed up the operation for arrays. + + :param val_watts: A value, or array of values, in Watts. + :return: The input val_watts, converted to dBm. + """ + suppress_divide_by_zero_when_testing() + if np.isscalar(val_watts): + val_dBm = 10.0 * np.log10(val_watts) + 30 + else: + val_dBm = ne.evaluate("10*log10(val_watts)+30") + return val_dBm + + +def convert_dBm_to_watts(val_dBm: Union[float, np.ndarray]) -> Union[float, np.ndarray]: + """ + Convert from dBm to Watts. + + Calculation: ``10^((val_dBm - 30) / 10)`` + + NumPy is used for scalar inputs. + NumExpr is used to speed up the operation for arrays. + + :param val_dBm: A value, or array of values, in dBm. + :return: The input val_dBm, converted to Watts. + """ + if np.isscalar(val_dBm): + val_watts = 10.0 ** ((val_dBm - 30) / 10) + else: + val_watts = ne.evaluate("10**((val_dBm-30)/10)") + return val_watts + + +def convert_linear_to_dB( + val_linear: Union[float, np.ndarray] +) -> Union[float, np.ndarray]: + """ + Convert from linear units to dB. + + Calculation: ``10 * log10(val_linear)`` + + NumPy is used for scalar inputs. + NumExpr is used to speed up the operation for arrays. + + :param val_linear: A value, or array of values, in linear + units. + :return: The input val_linear, converted to dB. + """ + suppress_divide_by_zero_when_testing() + if np.isscalar(val_linear): + val_dB = 10.0 * np.log10(val_linear) + else: + val_dB = ne.evaluate("10*log10(val_linear)") + return val_dB + + +def convert_dB_to_linear(val_dB: Union[float, np.ndarray]) -> Union[float, np.ndarray]: + """ + Convert from dB to linear units. + + Calculation: ``10^(val_dB / 10)`` + + NumPy is used for scalar inputs. + NumExpr is used to speed up the operation for arrays. + + :param val_dB: A value, or array of values, in dB. + :return: The input val_dB, converted to corresponding + linear units. + """ + if np.isscalar(val_dB): + val_linear = 10.0 ** (val_dB / 10.0) + else: + val_linear = ne.evaluate("10**(val_dB/10)") + return val_linear + + +def convert_kelvins_to_celsius( + val_kelvins: Union[float, np.ndarray] +) -> Union[float, np.ndarray]: + """ + Convert from Kelvins to degrees Celsius. + + Calculation: ``val_kelvins - 273.15`` + + :param val_kelvins: A value, or array of values, in + Kelvins. + :return: The input val_kelvins, converted to degrees + Celsius. + """ + return val_kelvins - 273.15 + + +def convert_celsius_to_kelvins( + val_celsius: Union[float, np.ndarray] +) -> Union[float, np.ndarray]: + """ + Convert from degrees Celsius to Kelvins. + + Calculation: ``val_celsius + 273.15`` + + :param val_celsius: A value, or array of values, in + degrees Celsius. + :return: The input val_celsius, converted to Kelvins. + """ + return val_celsius + 273.15 + + +def convert_fahrenheit_to_celsius( + val_fahrenheit: Union[float, np.ndarray] +) -> Union[float, np.ndarray]: + """ + Convert from degrees Fahrenheit to degrees Celsius. + + Calculation: ``(val_fahrenheit - 32) * (5 / 9)`` + + :param val_fahrenheit: A value, or array of values, in + degrees Fahrenheit. + :return: The input val_fahrenheit, converted to degrees + Celsius. + """ + return (val_fahrenheit - 32.0) * (5.0 / 9.0) diff --git a/scos_actions/signal_processing/utils.py b/scos_actions/signal_processing/utils.py deleted file mode 100644 index 3f3f2ef2..00000000 --- a/scos_actions/signal_processing/utils.py +++ /dev/null @@ -1,16 +0,0 @@ -import numpy as np - - -def dbm_to_watts(input): - # Convert an array of values from dBm to Watts - # return 10 ** ((np.array(input) - 30) / 10) - return (10. ** (np.array(input) / 10.)) / 1000. - - -def dBw_to_watts(val): - return 10 ** (val / 10) - - -def get_enbw(window, Fs): - # Return the equivalent noise bandwidth for a given window at a given sampling rate - return Fs * np.sum(window ** 2) / np.sum(window) ** 2 From 126bebcd150b904b3e3634d5ed90114133bd8e16 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 13 Jul 2022 13:57:55 -0600 Subject: [PATCH 003/157] Remove unused import --- scos_actions/hardware/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scos_actions/hardware/__init__.py b/scos_actions/hardware/__init__.py index 1d0bff21..bc4c7efe 100644 --- a/scos_actions/hardware/__init__.py +++ b/scos_actions/hardware/__init__.py @@ -22,7 +22,6 @@ def load_preselector(preselector_config_file): return preselector -from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer sigan = MockSignalAnalyzer(randomize_values=True) gps = MockGPS() preselector = load_preselector(PRESELECTOR_CONFIG_FILE) From 8effef7299f98a961b2e439b30f259e500cb8465 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 13 Jul 2022 13:58:43 -0600 Subject: [PATCH 004/157] Initialize freq_high and freq_low to None --- scos_actions/actions/metadata/measurement_global.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scos_actions/actions/metadata/measurement_global.py b/scos_actions/actions/metadata/measurement_global.py index 2d0c05ed..f45464b2 100644 --- a/scos_actions/actions/metadata/measurement_global.py +++ b/scos_actions/actions/metadata/measurement_global.py @@ -8,13 +8,14 @@ def __init__(self): super().__init__() def create_metadata(self, sigmf_builder: SigMFBuilder, measurement_result: dict): + freq_low = None + freq_high = None if 'frequency_low' in measurement_result: freq_low = measurement_result['frequency_low'] elif 'frequency' in measurement_result: freq_low = measurement_result['frequency'] freq_high = measurement_result['frequency'] - if 'frequency_high' in measurement_result: freq_high = measurement_result['frequency_high'] From d28a9a1952d655e400e4b0dd9b1623f083c4fdcd Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 13 Jul 2022 13:59:01 -0600 Subject: [PATCH 005/157] Add general get_param --- scos_actions/actions/action_utils.py | 32 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/scos_actions/actions/action_utils.py b/scos_actions/actions/action_utils.py index 1a519fde..f11cce5b 100644 --- a/scos_actions/actions/action_utils.py +++ b/scos_actions/actions/action_utils.py @@ -1,19 +1,19 @@ +class ParameterException(Exception): + """Basic exception handling for missing parameters.""" -def get_num_samples_and_fft_size( params): - if not "nffts" in params: - raise Exception("nffts missing from measurement parameters") - num_ffts = params["nffts"] - if not "fft_size" in params: - raise Exception("fft_size missing from measurement parameters") - fft_size = params["fft_size"] - num_samples = num_ffts * fft_size - return num_samples, fft_size + def __init__(self, param): + super().__init__(f"{param} missing from measurement parameters.") -def get_num_skip( params): - nskip = None - if "nskip" in params: - nskip = params["nskip"] - else: - raise Exception("nskip missing from measurement parameters") - return nskip \ No newline at end of file +def get_param(p: str, params: dict): + """ + Get a parameter by key from a parameter dictionary. + + :param p: The parameter name (key). + :param params: The parameter dictionary. + :return: The specified parameter (value). + :raises ParameterException: If p is not a key in params. + """ + if p not in params: + raise ParameterException(p) + return params[p] From db62535d40647e977a50472983681b6d36ab7eb5 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 13 Jul 2022 14:01:48 -0600 Subject: [PATCH 006/157] Update test for new FFT code --- scos_actions/actions/tests/test_acquire_single_freq_fft.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scos_actions/actions/tests/test_acquire_single_freq_fft.py b/scos_actions/actions/tests/test_acquire_single_freq_fft.py index f53aef86..de733369 100644 --- a/scos_actions/actions/tests/test_acquire_single_freq_fft.py +++ b/scos_actions/actions/tests/test_acquire_single_freq_fft.py @@ -1,4 +1,3 @@ -from scos_actions.actions.fft import M4sDetector from scos_actions.actions.interfaces.signals import measurement_action_completed from scos_actions.actions.tests.utils import SENSOR_DEFINITION, check_metadata_fields from scos_actions.discover import test_actions as actions From 3c0d771f835ea24a93567408b6c4069a3fb86cfd Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 13 Jul 2022 14:02:32 -0600 Subject: [PATCH 007/157] Update FFT annotation for generalized detectors --- scos_actions/actions/metadata/annotations/fft_annotation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/actions/metadata/annotations/fft_annotation.py b/scos_actions/actions/metadata/annotations/fft_annotation.py index e0f27730..a5bb6ca2 100644 --- a/scos_actions/actions/metadata/annotations/fft_annotation.py +++ b/scos_actions/actions/metadata/annotations/fft_annotation.py @@ -14,7 +14,7 @@ def create_metadata(self, sigmf_builder: SigMFBuilder, measurement_result): "ntia-algorithm:number_of_samples_in_fft": measurement_result['fft_size'], "ntia-algorithm:window": measurement_result['window'], "ntia-algorithm:equivalent_noise_bandwidth": measurement_result['enbw'], - "ntia-algorithm:detector": self.detector, + "ntia-algorithm:detector": 'fft_' + self.detector, "ntia-algorithm:number_of_ffts": measurement_result['nffts'], "ntia-algorithm:units": 'dBm', "ntia-algorithm:reference": '"preselector input"', From cee5dff76f0452e1f4c4b843928191f3c38ca81d Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 13 Jul 2022 14:03:44 -0600 Subject: [PATCH 008/157] FIx Django deprecation warning --- scos_actions/actions/interfaces/signals.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scos_actions/actions/interfaces/signals.py b/scos_actions/actions/interfaces/signals.py index ef60c4a0..c2c735aa 100644 --- a/scos_actions/actions/interfaces/signals.py +++ b/scos_actions/actions/interfaces/signals.py @@ -1,9 +1,10 @@ -import django.dispatch +from django.dispatch import Signal -measurement_action_completed = django.dispatch.Signal( - providing_args=["task_id", "data", "metadata"] -) -location_action_completed = django.dispatch.Signal( - providing_args=["latitude", "longitude"] -) -monitor_action_completed = django.dispatch.Signal(providing_args=["sigan_healthy"]) +# Provides arguments 'task_id', 'data', 'metadata' +measurement_action_completed = Signal() + +# Provides arguments: 'latitude', 'longitude' +location_action_completed = Signal() + +# Provides arguments: 'sigan_healthy' +monitor_action_completed = Signal() From e1e7d0103744244c8de7491937173cdce7b6b410 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 13 Jul 2022 14:04:28 -0600 Subject: [PATCH 009/157] Make action class use get_param --- scos_actions/actions/interfaces/action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scos_actions/actions/interfaces/action.py b/scos_actions/actions/interfaces/action.py index 98f8e745..aabd3513 100644 --- a/scos_actions/actions/interfaces/action.py +++ b/scos_actions/actions/interfaces/action.py @@ -6,6 +6,7 @@ from scos_actions.hardware import sigan as mock_sigan from scos_actions.capabilities import capabilities from scos_actions.hardware import preselector +from scos_actions.actions.action_utils import get_param logger = logging.getLogger(__name__) @@ -72,10 +73,9 @@ def summary(self): def description(self): return self.__doc__ - @property def name(self): - return self.parameter_map['name'] + return get_param("name", self.parameter_map) def get_parameter_map(self, params): if isinstance(params, list): From e14a325e3e7cc98a43bf334b5fa8b91019b2bf89 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 13 Jul 2022 14:04:53 -0600 Subject: [PATCH 010/157] Refactor actions for code changes --- .../actions/acquire_single_freq_fft.py | 124 ++++++---- .../actions/acquire_single_freq_tdomain_iq.py | 59 +++-- .../acquire_stepped_freq_tdomain_iq.py | 58 +++-- scos_actions/actions/calibrate_y_factor.py | 217 +++++++++++------- 4 files changed, 285 insertions(+), 173 deletions(-) diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py index 8c1401a0..cc57aae8 100644 --- a/scos_actions/actions/acquire_single_freq_fft.py +++ b/scos_actions/actions/acquire_single_freq_fft.py @@ -16,7 +16,7 @@ # - Markdown reference: https://commonmark.org/help/ # - SCOS Markdown Editor: https://ntia.github.io/scos-md-editor/ # -r"""Apply m4s detector over {nffts} {fft_size}-pt FFTs at {center_frequency:.2f} MHz. +r"""Apply M4S detector over {nffts} {fft_size}-pt FFTs at {center_frequency:.2f} MHz. # {name} @@ -89,69 +89,88 @@ import logging from scos_actions import utils from scos_actions.actions.interfaces.measurement_action import MeasurementAction -from scos_actions.actions.fft import ( - M4sDetector, +from scos_actions.signal_processing.fft import ( + get_fft, + get_fft_enbw, get_fft_frequencies, - get_m4s_dbm, get_fft_window, - get_fft_window_correction_factors, - get_enbw + get_fft_window_correction, ) from scos_actions.actions.sigmf_builder import Domain, MeasurementType, SigMFBuilder from scos_actions.actions.metadata.annotations.fft_annotation import FrequencyDomainDetectionAnnotation from scos_actions.hardware import gps as mock_gps -from scos_actions.actions.action_utils import get_num_samples_and_fft_size -from scos_actions.actions.action_utils import get_num_skip + +from scos_actions.actions.action_utils import get_param +from scos_actions.signal_processing.power_analysis import ( + apply_power_detector, + calculate_power_watts, + create_power_detector, +) +from scos_actions.signal_processing.unit_conversion import convert_watts_to_dBm +from numpy import float32, log10, ndarray logger = logging.getLogger(__name__) class SingleFrequencyFftAcquisition(MeasurementAction): - """Perform m4s detection over requested number of single-frequency FFTs. - - :param parameters: The dictionary of parameters needed for the action and the signal analyzer. + """Perform M4S detection over requested number of single-frequency FFTs. - The action will set any matching attributes found in the signal analyzer object. The following - parameters are required by the action: + The action will set any matching attributes found in the signal + analyzer object. The following parameters are required by the action: name: name of the action frequency: center frequency in Hz fft_size: number of points in FFT (some 2^n) nffts: number of consecutive FFTs to pass to detector - or the parameters required by the signal analyzer, see the documentation from the Python - package for the signal analyzer being used. + For the parameters required by the signal analyzer, see the + documentation from the Python package for the signal analyzer being + used. - :param sigan: instance of SignalAnalyzerInterface + :param parameters: The dictionary of parameters needed for the + action and the signal analyzer. + :param sigan: Instance of SignalAnalyzerInterface. """ def __init__(self, parameters, sigan, gps=mock_gps): super().__init__(parameters, sigan, gps) + # Pull parameters from action config + self.fft_size = get_param("fft_size", self.parameter_map) + self.nffts = get_param("nffts", self.parameter_map) + self.nskip = get_param("nskip", self.parameter_map) + self.frequency_Hz = get_param("frequency", self.parameter_map) + # FFT setup + self.fft_detector = create_power_detector( + "M4sDetector", ["min", "max", "mean", "median", "sample"] + ) + self.fft_window_type = "flattop" + self.num_samples = self.fft_size * self.nffts + self.fft_window = get_fft_window(self.fft_window_type, self.fft_size) + self.fft_window_acf = get_fft_window_correction(self.fft_window, "amplitude") - def execute(self, schedule_entry, task_id): + def execute(self, schedule_entry, task_id) -> dict: + # Acquire IQ data and generate M4S result start_time = utils.get_datetime_str_now() - nskip = get_num_skip(self.parameter_map) - num_samples, fft_size = get_num_samples_and_fft_size(self.parameter_map) - measurement_result = self.acquire_data(num_samples, nskip) - fft_window = get_fft_window("Flat Top", fft_size) - fft_window_acf, fft_window_ecf, fft_window_enbw = get_fft_window_correction_factors(fft_window) - enbw = get_enbw(measurement_result["sample_rate"], fft_size, fft_window_enbw) - measurement_result['data'] = get_m4s_dbm(fft_size, measurement_result, fft_window, fft_window_acf) + measurement_result = self.acquire_data(self.num_samples, self.nskip) + # Actual sample rate may differ from configured value + sample_rate_Hz = measurement_result["sample_rate"] + m4s_result = self.apply_m4s(measurement_result) + + # Save measurement results + measurement_result['data'] = m4s_result measurement_result['start_time'] = start_time measurement_result['end_time'] = utils.get_datetime_str_now() - measurement_result['enbw'] = enbw + measurement_result['enbw'] = get_fft_enbw(self.fft_window, sample_rate_Hz) frequencies = get_fft_frequencies( - self.parameter_map["fft_size"], - measurement_result["sample_rate"], - self.parameter_map["frequency"], - ).tolist() + self.fft_size, sample_rate_Hz, self.frequency_Hz + ) measurement_result.update(self.parameter_map) measurement_result['description'] = self.description measurement_result['domain'] = Domain.FREQUENCY.value measurement_result['frequency_start'] = frequencies[0] measurement_result['frequency_stop'] = frequencies[-1] measurement_result['frequency_step'] = frequencies[1] - frequencies[0] - measurement_result['window'] = 'flattop' + measurement_result['window'] = self.fft_window_type measurement_result['calibration_datetime'] = self.sigan.sensor_calibration_data['calibration_datetime'] measurement_result['task_id'] = task_id measurement_result['measurement_type'] = MeasurementType.SINGLE_FREQUENCY.value @@ -159,26 +178,44 @@ def execute(self, schedule_entry, task_id): measurement_result['sensor_cal'] = self.sigan.sensor_calibration_data return measurement_result + def apply_m4s(self, measurement_result: dict) -> ndarray: + # 'forward' normalization applies 1/fft_size normalization + complex_fft = get_fft( + measurement_result["data"], + self.fft_size, + "forward", + self.fft_window, + self.nffts, + ) + power_fft = calculate_power_watts(complex_fft) + m4s_result = apply_power_detector(power_fft, self.fft_detector, float32) + m4s_result = convert_watts_to_dBm(m4s_result) + m4s_result -= 3 # Baseband/RF power conversion + m4s_result += 10 * log10(self.fft_window_acf) # Window correction + return m4s_result + @property def description(self): - center_frequency = self.parameters["frequency"] / 1e6 - nffts = self.parameters["nffts"] - fft_size = self.parameters["fft_size"] + frequency_MHz = self.frequency_Hz / 1e6 used_keys = ["frequency", "nffts", "fft_size", "name"] - acq_plan = f"The signal analyzer is tuned to {center_frequency:.2f} MHz and the following parameters are set:\n" + acq_plan = ( + f"The signal analyzer is tuned to {frequency_MHz:.2f} MHz" + f" and the following parameters are set:\n" + ) for name, value in self.parameters.items(): if name not in used_keys: acq_plan += f"{name} = {value}\n" acq_plan += ( - f"\nThen, ${nffts} \times {fft_size}$ samples are acquired gap-free." + f"\nThen, ${self.nffts} \times {self.fft_size}$ samples " + "are acquired gap-free." ) definitions = { "name": self.name, - "center_frequency": center_frequency, + "center_frequency": frequency_MHz, "acquisition_plan": acq_plan, - "fft_size": fft_size, - "nffts": nffts, + "fft_size": self.fft_size, + "nffts": self.nffts, } # __doc__ refers to the module docstring at the top of the file @@ -186,12 +223,13 @@ def description(self): def get_sigmf_builder(self, measurement_result) -> SigMFBuilder: sigmf_builder = super().get_sigmf_builder(measurement_result) - for i, detector in enumerate(M4sDetector): - fft_annotation = FrequencyDomainDetectionAnnotation("fft_" + detector.name + "_power", - i * self.parameter_map["fft_size"], - self.parameter_map["fft_size"]) + for i, detector in enumerate(self.fft_detector): + fft_annotation = FrequencyDomainDetectionAnnotation( + detector.value, i * self.fft_size, self.fft_size + ) sigmf_builder.add_metadata_generator( - type(fft_annotation).__name__ + '_' + "fft_" + detector.name + "_power", fft_annotation) + type(fft_annotation).__name__ + "_" + detector.value, fft_annotation + ) return sigmf_builder def is_complex(self) -> bool: diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py index aab68b4b..cce42279 100644 --- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py +++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py @@ -34,7 +34,7 @@ import logging from scos_actions import utils -from scos_actions.actions.action_utils import get_num_skip +from scos_actions.actions.action_utils import get_param from scos_actions.actions.interfaces.measurement_action import MeasurementAction from scos_actions.actions.sigmf_builder import Domain, MeasurementType, SigMFBuilder from scos_actions.hardware import gps as mock_gps @@ -43,35 +43,46 @@ logger = logging.getLogger(__name__) +# Define parameter keys +FREQUENCY = "frequency" +SAMPLE_RATE = "sample_rate" +DURATION_MS = "duration_ms" +NUM_SKIP = "nskip" + class SingleFrequencyTimeDomainIqAcquisition(MeasurementAction): """Acquire IQ data at each of the requested frequencies. - :param parameters: The dictionary of parameters needed for the action and the signal analyzer. - - The action will set any matching attributes found in the signal analyzer object. The following - parameters are required by the action: + The action will set any matching attributes found in the + signal analyzer object. The following parameters are + required by the action: name: name of the action frequency: center frequency in Hz duration_ms: duration to acquire in ms - or the parameters required by the signal analyzer, see the documentation from the Python - package for the signal analyzer being used. + For the parameters required by the signal analyzer, see the + documentation from the Python package for the signal analyzer + being used. - :param sigan: instance of SignalAnalyzerInterface + :param parameters: The dictionary of parameters needed for + the action and the signal analyzer. + :param sigan: instance of SignalAnalyzerInterface. """ def __init__(self, parameters, sigan, gps=mock_gps): super().__init__(parameters=parameters, sigan=sigan, gps=gps) + # Pull parameters from action config + self.nskip = get_param(NUM_SKIP, self.parameter_map) + self.duration_ms = get_param(DURATION_MS, self.parameter_map) + self.frequency_Hz = get_param(FREQUENCY, self.parameter_map) - def execute(self, schedule_entry, task_id): + def execute(self, schedule_entry, task_id) -> dict: start_time = utils.get_datetime_str_now() - nskip = get_num_skip(self.parameter_map) - # Use the signal analyzer's actual reported sample rate instead of requested rate + # Use the sigan's actual reported instead of requested sample rate sample_rate = self.sigan.sample_rate - num_samples = int(sample_rate * self.parameter_map["duration_ms"] * 1e-3) - measurement_result = self.acquire_data(num_samples, nskip) + num_samples = int(sample_rate * self.duration_ms * 1e-3) + measurement_result = self.acquire_data(num_samples, self.nskip) measurement_result['start_time'] = start_time end_time = utils.get_datetime_str_now() measurement_result.update(self.parameter_map) @@ -85,27 +96,31 @@ def execute(self, schedule_entry, task_id): measurement_result['sensor_cal'] = self.sigan.sensor_calibration_data return measurement_result - def get_sigmf_builder(self, measurement_result) -> SigMFBuilder: + def get_sigmf_builder(self, measurement_result: dict) -> SigMFBuilder: sigmf_builder = super().get_sigmf_builder(measurement_result) time_domain_annotation = TimeDomainAnnotation(0, self.received_samples) - sigmf_builder.add_metadata_generator(type(time_domain_annotation).__name__, time_domain_annotation) + sigmf_builder.add_metadata_generator( + type(time_domain_annotation).__name__, time_domain_annotation + ) return sigmf_builder @property def description(self): """Parameterize and return the module-level docstring.""" - center_frequency = self.parameter_map["frequency"] / 1e6 - duration_ms = self.parameter_map["duration_ms"] - used_keys = ["frequency", "duration_ms", "name"] - acq_plan = f"The signal analyzer is tuned to {center_frequency:.2f} MHz and the following parameters are set:\n" + frequency_MHz = self.frequency_Hz / 1e6 + used_keys = [FREQUENCY, DURATION_MS, "name"] + acq_plan = ( + f"The signal analyzer is tuned to {frequency_MHz:.2f} " + + "MHz and the following parameters are set:\n" + ) for name, value in self.parameter_map.items(): if name not in used_keys: acq_plan += f"{name} = {value}\n" - acq_plan += f"\nThen, acquire samples for {duration_ms} ms\n." + acq_plan += f"\nThen, acquire samples for {self.duration_ms} ms\n." defs = { "name": self.name, - "center_frequency": center_frequency, + "center_frequency": frequency_MHz, "acquisition_plan": acq_plan, } @@ -113,7 +128,7 @@ def description(self): return __doc__.format(**defs) def transform_data(self, measurement_result): - return measurement_result['data'].astype(complex64) + return measurement_result["data"].astype(complex64) def is_complex(self) -> bool: return True diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py index dcb2046b..e7d13800 100644 --- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py +++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py @@ -40,39 +40,48 @@ import numpy as np from scos_actions import utils -from scos_actions.actions.action_utils import get_num_skip from scos_actions.actions.acquire_single_freq_tdomain_iq import ( SingleFrequencyTimeDomainIqAcquisition, ) +from scos_actions.actions.action_utils import get_param from scos_actions.actions.interfaces.signals import measurement_action_completed -from scos_actions.actions.sigmf_builder import Domain, MeasurementType, SigMFBuilder +from scos_actions.actions.sigmf_builder import Domain, MeasurementType from scos_actions.hardware import gps as mock_gps logger = logging.getLogger(__name__) +# Define parameter keys +FREQUENCY = "frequency" +SAMPLE_RATE = "sample_rate" +DURATION_MS = "duration_ms" +NUM_SKIP = "nskip" + class SteppedFrequencyTimeDomainIqAcquisition(SingleFrequencyTimeDomainIqAcquisition): """Acquire IQ data at each of the requested frequencies. - :param parameters: The dictionary of parameters needed for the action and the signal analyzer. - - The action will set any matching attributes found in the signal analyzer object. The following - parameters are required by the action: + The action will set any matching attributes found in the + signal analyzer object. The following parameters are required + by the action: - name: name of the action - frequency: an iterable of center frequencies in Hz - duration_ms: an iterable of measurement durations per center_frequency in ms + name: The name of the action. + frequency: An iterable of center frequencies, in Hz. + duration_ms: An iterable of measurement durations, per + center_frequency, in ms - For the parameters required by the signal analyzer, see the documentation from the Python - package for the signal analyzer being used. + For the parameters required by the signal analyzer, see the + documentation from the Python package for the signal analyzer + being used. + :param parameters: The dictionary of parameters needed for + the action and the signal analyzer. :param sigan: instance of SignalAnalyzerInterface """ def __init__(self, parameters, sigan, gps=mock_gps): super().__init__(parameters=parameters, sigan=sigan, gps=gps) self.sorted_measurement_parameters = [] - num_center_frequencies = len(parameters["frequency"]) + num_center_frequencies = len(parameters[FREQUENCY]) # convert dictionary of lists from yaml file to list of dictionaries longest_length = 0 @@ -88,7 +97,7 @@ def __init__(self, parameters, sigan, gps=mock_gps): continue sorted_params[key] = parameters[key][i] self.sorted_measurement_parameters.append(sorted_params) - self.sorted_measurement_parameters.sort(key=lambda params: params["frequency"]) + self.sorted_measurement_parameters.sort(key=lambda params: params[FREQUENCY]) self.sigan = sigan # make instance variable to allow mocking self.num_center_frequencies = num_center_frequencies @@ -102,9 +111,10 @@ def __call__(self, schedule_entry_json, task_id): ): start_time = utils.get_datetime_str_now() self.configure(measurement_params) + duration_ms = get_param(DURATION_MS, measurement_params) + nskip = get_param(NUM_SKIP, measurement_params) sample_rate = self.sigan.sample_rate - num_samples = int(sample_rate * measurement_params["duration_ms"] * 1e-3) - nskip = get_num_skip(measurement_params) + num_samples = int(sample_rate * duration_ms * 1e-3) measurement_result = super().acquire_data(num_samples, nskip) measurement_result.update(measurement_params) end_time = utils.get_datetime_str_now() @@ -114,11 +124,13 @@ def __call__(self, schedule_entry_json, task_id): measurement_result['measurement_type'] = MeasurementType.SINGLE_FREQUENCY.value measurement_result['task_id'] = task_id measurement_result['description'] = self.description - measurement_result['name'] = self.parameter_map['name'] + measurement_result['name'] = self.name measurement_result['sigan_cal'] = self.sigan.sigan_calibration_data measurement_result['sensor_cal'] = self.sigan.sensor_calibration_data sigmf_builder = self.get_sigmf_builder(measurement_result) - self.create_metadata(sigmf_builder, schedule_entry_json,measurement_result, recording_id) + self.create_metadata( + sigmf_builder, schedule_entry_json, measurement_result, recording_id + ) measurement_action_completed.send( sender=self.__class__, task_id=task_id, @@ -131,10 +143,10 @@ def description(self): """Parameterize and return the module-level docstring.""" acquisition_plan = "" - used_keys = ["frequency", "duration_ms", "name"] + used_keys = [FREQUENCY, DURATION_MS, "name"] acq_plan_template = "The signal analyzer is tuned to {center_frequency:.2f} MHz and the following parameters are set:\n" acq_plan_template += "{parameters}" - acq_plan_template += "Then, acquire samples for {duration_ms} ms\n." + acq_plan_template += "Then, acquire samples for {duration_ms} ms.\n" for measurement_params in self.sorted_measurement_parameters: parameters = "" @@ -143,13 +155,13 @@ def description(self): parameters += f"{name} = {value}\n" acquisition_plan += acq_plan_template.format( **{ - "center_frequency": measurement_params["frequency"] / 1e6, + "center_frequency": measurement_params[FREQUENCY] / 1e6, "parameters": parameters, - "duration_ms": measurement_params["duration_ms"], + "duration_ms": measurement_params[DURATION_MS], } ) - durations = [v["duration_ms"] for v in self.sorted_measurement_parameters] + durations = [v[DURATION_MS] for v in self.sorted_measurement_parameters] min_duration_ms = np.sum(durations) defs = { @@ -157,7 +169,7 @@ def description(self): "num_center_frequencies": self.num_center_frequencies, "center_frequencies": ", ".join( [ - "{:.2f} MHz".format(param["frequency"] / 1e6) + "{:.2f} MHz".format(param[FREQUENCY] / 1e6) for param in self.sorted_measurement_parameters ] ), diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 6c114b1f..717f65a4 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -18,13 +18,15 @@ # r"""Perform a Y-Factor Calibration. Supports calibration of gain and noise figure for one or more channels. -For each center frequency, sets the preselector to the noise diode path, turns noise diode on, performs and M4 measurmenet, -turns the noise diode off and performs another M4 measurements. Uses the mean power on and mean power off data to compute the noise figure and gain. -For each M4 measurement, it applies an m4s detector over {nffts} {fft_size}-pt FFTs at {frequencies} MHz. +For each center frequency, sets the preselector to the noise diode path, turns +noise diode on, performs a mean FFT measurement, turns the noise diode off and +performs another mean FFT measurement. The mean power on and mean power off +data are used to compute the noise figure and gain. For each measurement, the +mean detector is applied over {nffts} {fft_size}-pt FFTs at {frequencies} MHz. # {name} -## Radio setup and sample acquisition +## Signal analyzer setup and sample acquisition Each time this task runs, the following process is followed: {acquisition_plan} @@ -32,7 +34,7 @@ ## Time-domain processing First, the ${nffts} \times {fft_size}$ continuous samples are acquired from -the radio. If specified, a voltage scaling factor is applied to the complex +the signal analyzer. If specified, a voltage scaling factor is applied to the complex time-domain signals. Then, the data is reshaped into a ${nffts} \times {fft_size}$ matrix: @@ -55,50 +57,63 @@ where $M = {fft_size}$ is the number of points in the window, is applied to each row of the matrix. +## Frequency-domain processing +### To-do: add details of FFT processing + +## Y-Factor Method + +### To-do: add details of Y-Factor method """ import logging import time +from numpy import ndarray from scipy.constants import Boltzmann -from scipy.signal import windows -#from scos_actions.signal_processing.utils import get_enbw -from scos_actions.signal_processing.calibration import y_factor + from scos_actions import utils from scos_actions.hardware import gps as mock_gps from scos_actions.settings import sensor_calibration from scos_actions.settings import SENSOR_CALIBRATION_FILE from scos_actions.actions.interfaces.action import Action -from scos_actions.actions.action_utils import ( - get_num_samples_and_fft_size, - get_num_skip +from scos_actions.actions.action_utils import get_param +from scos_actions.signal_processing.fft import get_fft, get_fft_enbw, get_fft_window + +from scos_actions.signal_processing.calibration import ( + get_linear_enr, + get_temperature, + y_factor, ) -from scos_actions.actions.fft import ( - get_fft_window, - get_fft_window_correction_factors, - get_mean_detector_watts, - get_enbw +from scos_actions.signal_processing.power_analysis import ( + apply_power_detector, + calculate_power_watts, + create_power_detector, ) -from scos_actions.hardware import preselector import os +logger = logging.getLogger(__name__) + RF_PATH = 'rf_path' NOISE_DIODE_ON = {RF_PATH: 'noise_diode_on'} NOISE_DIODE_OFF = {RF_PATH: 'noise_diode_off'} -SAMPLE_RATE = 'sample_rate' -FFT_SIZE = 'fft_size' -logger = logging.getLogger(__name__) +# Define parameter keys +FREQUENCY = "frequency" +SAMPLE_RATE = "sample_rate" +FFT_SIZE = "fft_size" +NUM_FFTS = "nffts" +NUM_SKIP = "nskip" +# TODO: Should calibration source index and temperature sensor number +# be required parameters? class YFactorCalibration(Action): - """Perform a single or stepped y-factor calibration. - - :param parameters: The dictionary of parameters needed for the action and the radio. + """Perform a single- or stepped-frequency Y-factor calibration. - The action will set any matching attributes found in the radio object. The following - parameters are required by the action: + The action will set any matching attributes found in the signal + analyzer object. The following parameters are required by the + action: name: name of the action frequency: center frequency in Hz @@ -106,20 +121,29 @@ class YFactorCalibration(Action): nffts: number of consecutive FFTs to pass to detector - For the parameters required by the radio, see the documentation for the radio being used. + For the parameters required by the signal analyzer, see the + documentation from the Python package for the signal analyzer + being used. - :param radio: instance of RadioInterface + :param parameters: The dictionary of parameters needed for the + action and the signal analyzer. + :param sigan: instance of SignalAnalyzerInterface. """ def __init__(self, parameters, sigan, gps=mock_gps): logger.debug('Initializing calibration action') super().__init__(parameters, sigan, gps) + # Specify calibration source and temperature sensor indices + self.cal_source_idx = 0 + self.temp_sensor_idx = 1 + # FFT setup + self.fft_detector = create_power_detector("MeanDetector", ["mean"]) + self.fft_window_type = "flattop" def __call__(self, schedule_entry_json, task_id): """This is the entrypoint function called by the scheduler.""" self.test_required_components() - start_time = utils.get_datetime_str_now() - frequencies = self.parameter_map['frequency'] + frequencies = self.parameter_map[FREQUENCY] detail = '' if isinstance(frequencies, list): for i in range(len(frequencies)): @@ -131,14 +155,18 @@ def __call__(self, schedule_entry_json, task_id): elif isinstance(frequencies, float): detail = self.calibrate(self.parameters) - end_time = utils.get_datetime_str_now() return detail def calibrate(self, params): + # Set noise diode on logger.debug('Setting noise diode on') super().configure_preselector(NOISE_DIODE_ON) time.sleep(.25) + + # Debugging logger.debug('Before configure, Preamp = ' + str(self.sigan.preamp_enable)) + + # Configure signal analyzer self.sigan.preamp_enable = True super().configure_sigan(params) param_map = self.get_parameter_map(params) @@ -146,55 +174,90 @@ def calibrate(self, params): logger.debug('Preamp = ' + str(self.sigan.preamp_enable)) logger.debug('Ref_level: ' + str(self.sigan.reference_level)) logger.debug('Attenuation:' + str(self.sigan.attenuation)) - logger.debug('acquiring m4') - nskip = get_num_skip(param_map) - num_samples, fft_size = get_num_samples_and_fft_size(param_map) - noise_on_measurement_result = self.sigan.acquire_time_domain_samples(num_samples, num_samples_skip=nskip, gain_adjust=False) - fft_window = get_fft_window("Flat Top", fft_size) - fft_window_acf, fft_window_ecf, fft_window_enbw = get_fft_window_correction_factors(fft_window) - mean_on_watts = get_mean_detector_watts(fft_size, noise_on_measurement_result,fft_window,fft_window_acf) + + # Get parameters from action config + fft_size = get_param(FFT_SIZE, param_map) + nffts = get_param(NUM_FFTS, param_map) + nskip = get_param(NUM_SKIP, param_map) + fft_window = get_fft_window(self.fft_window_type, fft_size) + num_samples = fft_size * nffts + + logger.debug("Acquiring mean FFT") + + # Get noise diode on mean FFT result + noise_on_measurement_result = self.sigan.acquire_time_domain_samples( + num_samples, num_samples_skip=nskip, gain_adjust=False + ) + sample_rate = noise_on_measurement_result["sample_rate"] + mean_on_watts = self.apply_mean_fft( + noise_on_measurement_result, fft_size, fft_window, nffts + ) + + # Set noise diode off logger.debug('Setting noise diode off') self.configure_preselector(NOISE_DIODE_OFF) time.sleep(.25) - logger.debug('Acquiring noise off M4') - noise_off_measurement_result = self.sigan.acquire_time_domain_samples(num_samples, num_samples_skip=nskip, gain_adjust=False) - mean_off_watts = get_mean_detector_watts(fft_size, noise_off_measurement_result, fft_window, fft_window_acf) - enbw = get_enbw(param_map[SAMPLE_RATE], fft_size, fft_window_enbw) - enr = self.get_enr() - logger.debug('ENR: ' + str(enr)) - temp_k, temp_c, temp_f = self.get_temperature() - noise_floor = Boltzmann * temp_k * enbw - logger.debug('Noise floor: ' + str(noise_floor)) - noise_figure, gain = y_factor(mean_on_watts, mean_off_watts, enr, enbw, T_room=temp_k) - logger.debug('Noise Figure:' + str(noise_figure)) - logger.debug('Gain: ' + str(gain)) - sensor_calibration.update(param_map, utils.get_datetime_str_now(), gain, noise_figure, temp_c, - SENSOR_CALIBRATION_FILE) + + # Get noise diode off mean FFT result + logger.debug('Acquiring noise off mean FFT') + noise_off_measurement_result = self.sigan.acquire_time_domain_samples( + num_samples, num_samples_skip=nskip, gain_adjust=False + ) + mean_off_watts = self.apply_mean_fft( + noise_off_measurement_result, fft_size, fft_window, nffts + ) + + # Y-Factor + enbw_hz = get_fft_enbw(fft_window, sample_rate) + enr_linear = get_linear_enr(self.cal_source_idx) + temp_k, temp_c, _ = get_temperature(self.temp_sensor_idx) + noise_figure, gain = y_factor( + mean_on_watts, mean_off_watts, enr_linear, enbw_hz, temp_k + ) + sensor_calibration.update( + param_map, + utils.get_datetime_str_now(), + gain, + noise_figure, + temp_c, + SENSOR_CALIBRATION_FILE, + ) + + # Debugging + noise_floor = Boltzmann * temp_k * enbw_hz + logger.debug(f'Noise floor: {noise_floor} Watts') + logger.debug(f'Noise Figure: {noise_figure} dB') + logger.debug(f'Gain: {gain} dB') + return 'Noise Figure:{}, Gain:{}'.format(noise_figure, gain) - def get_enr(self): - # todo deal with multiple cal sources - if len(preselector.cal_sources) == 0: - raise Exception('No calibrations sources defined in preselector.') - elif len(preselector.cal_sources) > 1: - raise Exception('Preselector contains multiple calibration sources.') - else: - enr_dB = preselector.cal_sources[0].enr - linear_enr = 10 ** (enr_dB / 10.0) - return linear_enr + def apply_mean_fft( + self, measurement_result: dict, fft_size: int, fft_window: ndarray, nffts: int + ) -> ndarray: + complex_fft = get_fft( + measurement_result["data"], fft_size, "backward", fft_window, nffts + ) + power_fft = calculate_power_watts(complex_fft) + mean_result = apply_power_detector(power_fft, self.fft_detector) + return mean_result @property def description(self): - if (isinstance(self.parameter_map['frequency'], float)): - frequencies = self.parameter_map["frequency"] / 1e6 - nffts = self.parameter_map["nffts"] - fft_size = self.parameter_map[FFT_SIZE] + if isinstance(get_param(FREQUENCY, self.parameter_map), float): + frequencies = get_param(FREQUENCY, self.parameter_map) / 1e6 + nffts = get_param(NUM_FFTS, self.parameter_map) + fft_size = get_param(FFT_SIZE, self.parameter_map) else: - frequencies = utils.list_to_string(self.parameter_map['frequency']) - nffts = utils.list_to_string(self.parameter_map["nffts"]) - fft_size = utils.list_to_string(self.parameter_map[FFT_SIZE]) - acq_plan = f"Performs a y-factor calibration at frequencies: {frequencies}, nffts:{nffts}, fft_size: {fft_size}\n" + frequencies = utils.list_to_string( + [f / 1e6 for f in get_param(FREQUENCY, self.parameter_map)] + ) + nffts = utils.list_to_string(get_param(NUM_FFTS, self.parameter_map)) + fft_size = utils.list_to_string(get_param(FFT_SIZE, self.parameter_map)) + acq_plan = ( + f"Performs a y-factor calibration at frequencies: " + f"{frequencies}, nffts:{nffts}, fft_size: {fft_size}\n" + ) definitions = { "name": self.name, "frequencies": frequencies, @@ -205,22 +268,6 @@ def description(self): # __doc__ refers to the module docstring at the top of the file return __doc__.format(**definitions) - # todo support multiple temperature sensors - def get_temperature(self): - kelvin_temp = 290.0 - celsius_temp = kelvin_temp - 273.15 - fahrenheit = (celsius_temp * 9. / 5.) + 32 - temp = preselector.get_sensor_value(1) - logger.debug('Temp: ' + str(temp)) - if temp is None: - logger.warning('Temperature is None. Using 290 K instead.') - else: - fahrenheit = float(temp) - celsius_temp = ((5.0 * (fahrenheit - 32)) / 9.0) - kelvin_temp = celsius_temp + 273.15 - logger.debug('Temperature: ' + str(kelvin_temp)) - return kelvin_temp, celsius_temp, fahrenheit - def test_required_components(self): """Fail acquisition if a required component is not available.""" if not self.sigan.is_available: From d237901b23a143620b06732740880f0bd5c25001 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 13 Jul 2022 14:05:22 -0600 Subject: [PATCH 011/157] Update and improve pre-commit hooks --- .isort.cfg | 2 +- .pre-commit-config.yaml | 48 +++++++++++++++++++++++++---------------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 48d2be3d..fcb6f2a5 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -4,4 +4,4 @@ include_trailing_comma=True force_grid_wrap=0 use_parentheses=True line_length=88 -known_third_party = dateutil,django,numpy,pytest,ruamel,scipy,setuptools,sigmf +known_third_party = dateutil,django,numexpr,numpy,pytest,pytz,ruamel,scipy,setuptools,sigmf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f7390a5f..820fccc0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,31 +1,41 @@ +default_language_version: + python: python3.7 repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.1.0 hooks: - - id: trailing-whitespace - - id: end-of-file-fixer + - id: check-ast + types: [file, python] + - id: check-case-conflict - id: check-docstring-first + types: [file, python] + - id: check-merge-conflict - id: check-yaml + types: [file, yaml] - id: debug-statements - - repo: https://github.com/asottile/seed-isort-config - rev: v2.2.0 + types: [file, python] + - id: detect-private-key + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/asottile/pyupgrade + rev: v2.34.0 hooks: - - id: seed-isort-config - language_version: python3.7 - - repo: https://github.com/pre-commit/mirrors-isort - rev: v5.10.1 + - id: pyupgrade + - repo: https://github.com/pycqa/isort + rev: 5.10.1 hooks: - id: isort - language_version: python3.7 - - repo: https://github.com/ambv/black - rev: 22.1.0 + name: isort (python) + types: [file, python] + args: ["--profile", "black", "--filter-files"] + - repo: https://github.com/psf/black + rev: 22.6.0 hooks: - id: black - language_version: python3.7 - # TODO markdownlint broken - # - repo: https://github.com/markdownlint/markdownlint - # rev: v0.11.0 - # hooks: - # - id: markdownlint - # args: [-s, .ml_style.rb, README.md] - # exclude: GitHubRepoPublicReleaseApproval.md|LICENSE.md + types: [file, python] + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: v0.31.1 + hooks: + - id: markdownlint + types: [file, markdown] + exclude: GitHubRepoPublicReleaseApproval.md|LICENSE.md From 8783abca4895203cf566c3af8f5de386837d21c9 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 20 Jul 2022 13:11:30 -0600 Subject: [PATCH 012/157] Updated FFT result scaling --- scos_actions/actions/acquire_single_freq_fft.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py index cc57aae8..379b38de 100644 --- a/scos_actions/actions/acquire_single_freq_fft.py +++ b/scos_actions/actions/acquire_single_freq_fft.py @@ -106,7 +106,7 @@ calculate_power_watts, create_power_detector, ) -from scos_actions.signal_processing.unit_conversion import convert_watts_to_dBm +from scos_actions.signal_processing.unit_conversion import convert_watts_to_dBm, convert_linear_to_dB from numpy import float32, log10, ndarray logger = logging.getLogger(__name__) @@ -179,6 +179,7 @@ def execute(self, schedule_entry, task_id) -> dict: return measurement_result def apply_m4s(self, measurement_result: dict) -> ndarray: + # IQ samples already scaled based on calibration # 'forward' normalization applies 1/fft_size normalization complex_fft = get_fft( measurement_result["data"], @@ -190,8 +191,11 @@ def apply_m4s(self, measurement_result: dict) -> ndarray: power_fft = calculate_power_watts(complex_fft) m4s_result = apply_power_detector(power_fft, self.fft_detector, float32) m4s_result = convert_watts_to_dBm(m4s_result) - m4s_result -= 3 # Baseband/RF power conversion - m4s_result += 10 * log10(self.fft_window_acf) # Window correction + # Scaling applied: + # RF/Baseband power conversion (-3 dB) + # FFT window amplitude correction + m4s_result -= 3 + m4s_result += convert_linear_to_dB(self.fft_window_acf) return m4s_result @property From 0d830dc55e545303054c267545d70ecd0557a955 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 20 Jul 2022 13:31:53 -0600 Subject: [PATCH 013/157] Add YAML parameter key name definitions --- scos_actions/actions/acquire_single_freq_fft.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py index 379b38de..35147cbe 100644 --- a/scos_actions/actions/acquire_single_freq_fft.py +++ b/scos_actions/actions/acquire_single_freq_fft.py @@ -111,6 +111,13 @@ logger = logging.getLogger(__name__) +# Define paramter keys +FREQUENCY = "frequency" +SAMPLE_RATE = "sample_rate" +NUM_SKIP = "nskip" +NUM_FFTS = "nffts" +FFT_SIZE = "fft_size" + class SingleFrequencyFftAcquisition(MeasurementAction): """Perform M4S detection over requested number of single-frequency FFTs. @@ -135,10 +142,10 @@ class SingleFrequencyFftAcquisition(MeasurementAction): def __init__(self, parameters, sigan, gps=mock_gps): super().__init__(parameters, sigan, gps) # Pull parameters from action config - self.fft_size = get_param("fft_size", self.parameter_map) - self.nffts = get_param("nffts", self.parameter_map) - self.nskip = get_param("nskip", self.parameter_map) - self.frequency_Hz = get_param("frequency", self.parameter_map) + self.fft_size = get_param(FFT_SIZE, self.parameter_map) + self.nffts = get_param(NUM_FFTS, self.parameter_map) + self.nskip = get_param(NUM_SKIP, self.parameter_map) + self.frequency_Hz = get_param(FREQUENCY, self.parameter_map) # FFT setup self.fft_detector = create_power_detector( "M4sDetector", ["min", "max", "mean", "median", "sample"] @@ -201,7 +208,7 @@ def apply_m4s(self, measurement_result: dict) -> ndarray: @property def description(self): frequency_MHz = self.frequency_Hz / 1e6 - used_keys = ["frequency", "nffts", "fft_size", "name"] + used_keys = [FREQUENCY, NUM_FFTS, FFT_SIZE, "name"] acq_plan = ( f"The signal analyzer is tuned to {frequency_MHz:.2f} MHz" f" and the following parameters are set:\n" From 74aa4cb79320059c2a4d164ae3b56890ad07589b Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 20 Jul 2022 13:51:28 -0600 Subject: [PATCH 014/157] Update density scaling to use NumExpr --- scos_actions/signal_processing/apd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/signal_processing/apd.py b/scos_actions/signal_processing/apd.py index a4b6f1d2..f6c77392 100644 --- a/scos_actions/signal_processing/apd.py +++ b/scos_actions/signal_processing/apd.py @@ -79,6 +79,6 @@ def sample_ccdf(a: np.ndarray, edges: np.ndarray, density: bool = True) -> np.nd if density: ccdf = ccdf.astype("float64") - ccdf /= a.size + ne.evaluate("ccdf/a_size", local_dict={'ccdf': ccdf, 'a_size': a.size}, out=ccdf) return ccdf From 2fec2795199327b9bc693e2b40715cfb6f2aa7ed Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 20 Jul 2022 14:10:15 -0600 Subject: [PATCH 015/157] Create single frequency APD action --- scos_actions/actions/__init__.py | 4 +- .../actions/acquire_single_freq_apd.py | 168 ++++++++++++++++++ .../test_single_frequency_apd_action.yml | 8 + 3 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 scos_actions/actions/acquire_single_freq_apd.py create mode 100644 scos_actions/configs/actions/test_single_frequency_apd_action.yml diff --git a/scos_actions/actions/__init__.py b/scos_actions/actions/__init__.py index 9b55316e..a5ae0b09 100644 --- a/scos_actions/actions/__init__.py +++ b/scos_actions/actions/__init__.py @@ -2,6 +2,7 @@ from .acquire_single_freq_tdomain_iq import SingleFrequencyTimeDomainIqAcquisition from .acquire_stepped_freq_tdomain_iq import SteppedFrequencyTimeDomainIqAcquisition from .calibrate_y_factor import YFactorCalibration +from .acquire_single_freq_apd import SingleFrequencyApdAcquisition @@ -13,7 +14,8 @@ "single_frequency_fft": SingleFrequencyFftAcquisition, "stepped_frequency_time_domain_iq": SteppedFrequencyTimeDomainIqAcquisition, "single_frequency_time_domain_iq": SingleFrequencyTimeDomainIqAcquisition, - "y_factor_cal": YFactorCalibration + "y_factor_cal": YFactorCalibration, + "single_frequency_apd": SingleFrequencyApdAcquisition, } diff --git a/scos_actions/actions/acquire_single_freq_apd.py b/scos_actions/actions/acquire_single_freq_apd.py new file mode 100644 index 00000000..d8acba7b --- /dev/null +++ b/scos_actions/actions/acquire_single_freq_apd.py @@ -0,0 +1,168 @@ +# What follows is a parameterizable description of the algorithm used by this +# action. The first line is the summary and should be written in plain text. +# Everything following that is the extended description, which can be written +# in Markdown and MathJax. Each name in curly brackets '{}' will be replaced +# with the value specified in the `description` method which can be found at +# the very bottom of this file. Since this parameterization step affects +# everything in curly brackets, math notation such as {m \over n} must be +# escaped to {{m \over n}}. +# +# To print out this docstring after parameterization, see +# scos-sensor/scripts/print_action_docstring.py. You can then paste that into the +# SCOS Markdown Editor (link below) to see the final rendering. +# +# Resources: +# - MathJax reference: https://math.meta.stackexchange.com/q/5020 +# - Markdown reference: https://commonmark.org/help/ +# - SCOS Markdown Editor: https://ntia.github.io/scos-md-editor/ +# +r"""Estimate the amplitude probability distribution of {num_samps} continuous time domain power samples at {center_frequency:.2f} MHz. + +# To-do: write comprehensive module docstring + +# {name} + +## Signal Analyzer setup and sample acquisition + +Each time this task runs, the following process is followed: +{acquisition_plan} + +## Time-domain processing + +First, the {num_samps} continuous samples are acquired from the signal analyzer. If +specified, a voltage scaling factor is applied to the time-domain signals. The time +domain complex voltage samples are converted to real-valued amplitudes which are used to +generate the APD. After the APD is generated, the amplitude values are further converted +to power values in dBm, with calculations assuming a 50 Ohm system. + +If no downsampling is specified (by either not providing the apd_bin_size_dB parameter, or +setting its value to be less than or equal to zero), the APD is estimated using the routine +provided in [IEEE P802.15-04-0428-00](https://www.ieee802.org/15/pub/2004/15-04-0428-00-003a-estimating-and-graphing-amplitude-probability-distribution-function.pdf). + +If downsampling is specified, the APD is estimated by sampling the complementary +cumulative distribution function (CCDF) of the sample amplitudes. Bin edges are generated based on the specified value +of the apd_bin_size_dB parameter and the minimum and maximum recorded amplitude samples. The fraction of +amplitude samples exceeding each bin edge is computed, and used to construct an estimate of the APD. + +## Power Conversion + +After the APD is estimated, the amplitude values are converted to power values in dBm, assuming +a 50 Ohm system. A 3 dB power correction is applied to account for RF/baseband conversion. The resulting +power values are therefore referenced to the calibration point. + +## Results + +The current implementation stores a concatenated data array containing both the probability +and amplitude axes of the estimated APD. This should be altered as future updates to SigMF extensions +provide better ways to store such data. +""" +import logging +import numexpr as ne +import numpy as np +from scos_actions.actions.interfaces.measurement_action import (MeasurementAction) +from scos_actions.actions.action_utils import get_param, ParameterException +from scos_actions.actions.sigmf_builder import Domain, MeasurementType +from scos_actions.signal_processing.apd import get_apd +from scos_actions.signal_processing.unit_conversion import convert_linear_to_dB +from scos_actions.hardware import gps as mock_gps +from scos_actions import utils + + +logger = logging.getLogger(__name__) + +# Define parameter keys +FREQUENCY = "frequency" +SAMPLE_RATE = "sample_rate" +DURATION_MS = "duration_ms" +NUM_SKIP = "nskip" +APD_BIN_SIZE = "apd_bin_size_dB" + + +class SingleFrequencyApdAcquisition(MeasurementAction): + """Performs an APD analysis of the requested number of samples at the specified sample rate. + + The action will set any matching attributes found in the signal analyzer object. The following + parameters are required by the action: + + name: name of the action + frequency: center frequency in Hz + duration_ms: duration to acquire in ms + apd_bin_size_dB: amplitude resolution of IQ data, reduces data size. defaults to 0.5 + + For the parameters required by the signal analyzer, see the documentation from the Python + package for the signal analyzer being used. + + :param parameters: Dictionary of parameters needed for the action and signal analyzer. + :param sigan: instance of SignalAnalyzerInterface + """ + def __init__(self, parameters, sigan, gps=mock_gps): + super().__init__(parameters, sigan, gps) + self.is_complex = False + # Pull parameters from action config + self.nskip = get_param(NUM_SKIP, self.parameter_map) + self.duration_ms = get_param(DURATION_MS, self.parameter_map) + self.frequency_Hz = get_param(FREQUENCY, self.parameter_map) + self.sample_rate_Hz = get_param(SAMPLE_RATE, self.parameter_map) + self.num_samples = int(self.sample_rate * self.duration_ms * 1e-3) + try: + self.apd_bin_size = get_param(APD_BIN_SIZE, self.parameter_map) + except ParameterException: + logger.warning("APD bin size not configured. Using no downsampling.") + self.apd_bin_size = None + if self.apd_bin_size <= 0: + logger.warning("APD bin size set to zero or negative. Using no downsampling.") + self.apd_bin_size = None + + def execute(self, schedule_entry, task_id) -> dict: + # Acquire IQ data and generate APD result + start_time = utils.get_datetime_str_now() + measurement_result = self.acquire_data(self.num_samples, self.nskip) + apd_result = self.get_power_apd(measurement_result) + # apd_result is (p, a) concatenated into a single array + + # Save measurement results + measurement_result["data"] = apd_result + measurement_result.update(self.parameter_map) + measurement_result['start_time'] = start_time + measurement_result['end_time'] = utils.get_datetime_str_now() + measurement_result["domain"] = Domain.TIME.value + measurement_result['measurement_type'] = MeasurementType.SINGLE_FREQUENCY.value + measurement_result['description'] = self.description + measurement_result['calibration_datetime'] = self.sigan.sensor_calibration_data['calibration_datetime'] + measurement_result['task_id'] = task_id + measurement_result['sigan_cal'] = self.sigan.sigan_calibration_data + measurement_result['sensor_cal'] = self.sigan.sensor_calibration_data + return measurement_result + + def get_power_apd(self, measurement_result: dict) -> np.ndarray: + # Calibration gain scaling already applied to IQ samples + p, a = get_apd(measurement_result["data"], self.apd_bin_size) + # Scaling applied: + # a * 2 : dBV --> dB(V^2) + # a - impedance_dB : dB(V^2) --> dBW (hard-coded for 50 Ohm systems) + # a + 27 : dBW --> dBm (+30), RF/baseband conversion (-3) + scale_factor = 27 - convert_linear_to_dB(50.) + ne.evaluate("(2*a)+scale_factor", out=a) + # For now: concatenate axes to store as a single array + return np.concatenate((p, a)) + + @property + def description(self): + """Parameterize and return the module-level docstring.""" + frequency_MHz = self.frequency_Hz / 1e6 + used_keys = [FREQUENCY, APD_BIN_SIZE, "name"] + acq_plan = f"The signal analyzer is tuned to {frequency_MHz:.2f} MHz and the following parameters are set:\n" + for name, value in self.parameters.items(): + if name not in used_keys: + acq_plan += f"{name} = {value}\n" + + definitions = { + "name": self.name, + "num_samps": self.num_samples, + "center_frequency": frequency_MHz, + "acquisition_plan": acq_plan, + "apd_bin_size_dB": self.apd_bin_size_dB # Currently unused in docstring + } + + # __doc__ refers to the module docstring at the top of the file + return __doc__.format(**definitions) \ No newline at end of file diff --git a/scos_actions/configs/actions/test_single_frequency_apd_action.yml b/scos_actions/configs/actions/test_single_frequency_apd_action.yml new file mode 100644 index 00000000..00e31d85 --- /dev/null +++ b/scos_actions/configs/actions/test_single_frequency_apd_action.yml @@ -0,0 +1,8 @@ +single_frequency_apd: + name: test_single_frequency_apd_action + frequency: 739e6 + gain: 40 + sample_rate: 15.36e6 + nskip: 15.36e4 + duration_ms: 1000 + apd_bin_size_dB: 0.5 From c7c158a3b3ec200bb61df8c4fcd8a67945b24654 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 20 Jul 2022 14:14:05 -0600 Subject: [PATCH 016/157] Revert "Create single frequency APD action" This reverts commit 2fec2795199327b9bc693e2b40715cfb6f2aa7ed. --- scos_actions/actions/__init__.py | 4 +- .../actions/acquire_single_freq_apd.py | 168 ------------------ .../test_single_frequency_apd_action.yml | 8 - 3 files changed, 1 insertion(+), 179 deletions(-) delete mode 100644 scos_actions/actions/acquire_single_freq_apd.py delete mode 100644 scos_actions/configs/actions/test_single_frequency_apd_action.yml diff --git a/scos_actions/actions/__init__.py b/scos_actions/actions/__init__.py index a5ae0b09..9b55316e 100644 --- a/scos_actions/actions/__init__.py +++ b/scos_actions/actions/__init__.py @@ -2,7 +2,6 @@ from .acquire_single_freq_tdomain_iq import SingleFrequencyTimeDomainIqAcquisition from .acquire_stepped_freq_tdomain_iq import SteppedFrequencyTimeDomainIqAcquisition from .calibrate_y_factor import YFactorCalibration -from .acquire_single_freq_apd import SingleFrequencyApdAcquisition @@ -14,8 +13,7 @@ "single_frequency_fft": SingleFrequencyFftAcquisition, "stepped_frequency_time_domain_iq": SteppedFrequencyTimeDomainIqAcquisition, "single_frequency_time_domain_iq": SingleFrequencyTimeDomainIqAcquisition, - "y_factor_cal": YFactorCalibration, - "single_frequency_apd": SingleFrequencyApdAcquisition, + "y_factor_cal": YFactorCalibration } diff --git a/scos_actions/actions/acquire_single_freq_apd.py b/scos_actions/actions/acquire_single_freq_apd.py deleted file mode 100644 index d8acba7b..00000000 --- a/scos_actions/actions/acquire_single_freq_apd.py +++ /dev/null @@ -1,168 +0,0 @@ -# What follows is a parameterizable description of the algorithm used by this -# action. The first line is the summary and should be written in plain text. -# Everything following that is the extended description, which can be written -# in Markdown and MathJax. Each name in curly brackets '{}' will be replaced -# with the value specified in the `description` method which can be found at -# the very bottom of this file. Since this parameterization step affects -# everything in curly brackets, math notation such as {m \over n} must be -# escaped to {{m \over n}}. -# -# To print out this docstring after parameterization, see -# scos-sensor/scripts/print_action_docstring.py. You can then paste that into the -# SCOS Markdown Editor (link below) to see the final rendering. -# -# Resources: -# - MathJax reference: https://math.meta.stackexchange.com/q/5020 -# - Markdown reference: https://commonmark.org/help/ -# - SCOS Markdown Editor: https://ntia.github.io/scos-md-editor/ -# -r"""Estimate the amplitude probability distribution of {num_samps} continuous time domain power samples at {center_frequency:.2f} MHz. - -# To-do: write comprehensive module docstring - -# {name} - -## Signal Analyzer setup and sample acquisition - -Each time this task runs, the following process is followed: -{acquisition_plan} - -## Time-domain processing - -First, the {num_samps} continuous samples are acquired from the signal analyzer. If -specified, a voltage scaling factor is applied to the time-domain signals. The time -domain complex voltage samples are converted to real-valued amplitudes which are used to -generate the APD. After the APD is generated, the amplitude values are further converted -to power values in dBm, with calculations assuming a 50 Ohm system. - -If no downsampling is specified (by either not providing the apd_bin_size_dB parameter, or -setting its value to be less than or equal to zero), the APD is estimated using the routine -provided in [IEEE P802.15-04-0428-00](https://www.ieee802.org/15/pub/2004/15-04-0428-00-003a-estimating-and-graphing-amplitude-probability-distribution-function.pdf). - -If downsampling is specified, the APD is estimated by sampling the complementary -cumulative distribution function (CCDF) of the sample amplitudes. Bin edges are generated based on the specified value -of the apd_bin_size_dB parameter and the minimum and maximum recorded amplitude samples. The fraction of -amplitude samples exceeding each bin edge is computed, and used to construct an estimate of the APD. - -## Power Conversion - -After the APD is estimated, the amplitude values are converted to power values in dBm, assuming -a 50 Ohm system. A 3 dB power correction is applied to account for RF/baseband conversion. The resulting -power values are therefore referenced to the calibration point. - -## Results - -The current implementation stores a concatenated data array containing both the probability -and amplitude axes of the estimated APD. This should be altered as future updates to SigMF extensions -provide better ways to store such data. -""" -import logging -import numexpr as ne -import numpy as np -from scos_actions.actions.interfaces.measurement_action import (MeasurementAction) -from scos_actions.actions.action_utils import get_param, ParameterException -from scos_actions.actions.sigmf_builder import Domain, MeasurementType -from scos_actions.signal_processing.apd import get_apd -from scos_actions.signal_processing.unit_conversion import convert_linear_to_dB -from scos_actions.hardware import gps as mock_gps -from scos_actions import utils - - -logger = logging.getLogger(__name__) - -# Define parameter keys -FREQUENCY = "frequency" -SAMPLE_RATE = "sample_rate" -DURATION_MS = "duration_ms" -NUM_SKIP = "nskip" -APD_BIN_SIZE = "apd_bin_size_dB" - - -class SingleFrequencyApdAcquisition(MeasurementAction): - """Performs an APD analysis of the requested number of samples at the specified sample rate. - - The action will set any matching attributes found in the signal analyzer object. The following - parameters are required by the action: - - name: name of the action - frequency: center frequency in Hz - duration_ms: duration to acquire in ms - apd_bin_size_dB: amplitude resolution of IQ data, reduces data size. defaults to 0.5 - - For the parameters required by the signal analyzer, see the documentation from the Python - package for the signal analyzer being used. - - :param parameters: Dictionary of parameters needed for the action and signal analyzer. - :param sigan: instance of SignalAnalyzerInterface - """ - def __init__(self, parameters, sigan, gps=mock_gps): - super().__init__(parameters, sigan, gps) - self.is_complex = False - # Pull parameters from action config - self.nskip = get_param(NUM_SKIP, self.parameter_map) - self.duration_ms = get_param(DURATION_MS, self.parameter_map) - self.frequency_Hz = get_param(FREQUENCY, self.parameter_map) - self.sample_rate_Hz = get_param(SAMPLE_RATE, self.parameter_map) - self.num_samples = int(self.sample_rate * self.duration_ms * 1e-3) - try: - self.apd_bin_size = get_param(APD_BIN_SIZE, self.parameter_map) - except ParameterException: - logger.warning("APD bin size not configured. Using no downsampling.") - self.apd_bin_size = None - if self.apd_bin_size <= 0: - logger.warning("APD bin size set to zero or negative. Using no downsampling.") - self.apd_bin_size = None - - def execute(self, schedule_entry, task_id) -> dict: - # Acquire IQ data and generate APD result - start_time = utils.get_datetime_str_now() - measurement_result = self.acquire_data(self.num_samples, self.nskip) - apd_result = self.get_power_apd(measurement_result) - # apd_result is (p, a) concatenated into a single array - - # Save measurement results - measurement_result["data"] = apd_result - measurement_result.update(self.parameter_map) - measurement_result['start_time'] = start_time - measurement_result['end_time'] = utils.get_datetime_str_now() - measurement_result["domain"] = Domain.TIME.value - measurement_result['measurement_type'] = MeasurementType.SINGLE_FREQUENCY.value - measurement_result['description'] = self.description - measurement_result['calibration_datetime'] = self.sigan.sensor_calibration_data['calibration_datetime'] - measurement_result['task_id'] = task_id - measurement_result['sigan_cal'] = self.sigan.sigan_calibration_data - measurement_result['sensor_cal'] = self.sigan.sensor_calibration_data - return measurement_result - - def get_power_apd(self, measurement_result: dict) -> np.ndarray: - # Calibration gain scaling already applied to IQ samples - p, a = get_apd(measurement_result["data"], self.apd_bin_size) - # Scaling applied: - # a * 2 : dBV --> dB(V^2) - # a - impedance_dB : dB(V^2) --> dBW (hard-coded for 50 Ohm systems) - # a + 27 : dBW --> dBm (+30), RF/baseband conversion (-3) - scale_factor = 27 - convert_linear_to_dB(50.) - ne.evaluate("(2*a)+scale_factor", out=a) - # For now: concatenate axes to store as a single array - return np.concatenate((p, a)) - - @property - def description(self): - """Parameterize and return the module-level docstring.""" - frequency_MHz = self.frequency_Hz / 1e6 - used_keys = [FREQUENCY, APD_BIN_SIZE, "name"] - acq_plan = f"The signal analyzer is tuned to {frequency_MHz:.2f} MHz and the following parameters are set:\n" - for name, value in self.parameters.items(): - if name not in used_keys: - acq_plan += f"{name} = {value}\n" - - definitions = { - "name": self.name, - "num_samps": self.num_samples, - "center_frequency": frequency_MHz, - "acquisition_plan": acq_plan, - "apd_bin_size_dB": self.apd_bin_size_dB # Currently unused in docstring - } - - # __doc__ refers to the module docstring at the top of the file - return __doc__.format(**definitions) \ No newline at end of file diff --git a/scos_actions/configs/actions/test_single_frequency_apd_action.yml b/scos_actions/configs/actions/test_single_frequency_apd_action.yml deleted file mode 100644 index 00e31d85..00000000 --- a/scos_actions/configs/actions/test_single_frequency_apd_action.yml +++ /dev/null @@ -1,8 +0,0 @@ -single_frequency_apd: - name: test_single_frequency_apd_action - frequency: 739e6 - gain: 40 - sample_rate: 15.36e6 - nskip: 15.36e4 - duration_ms: 1000 - apd_bin_size_dB: 0.5 From f34718996c863787004b9be9ce0b64351d9f5106 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 20 Jul 2022 16:45:55 -0600 Subject: [PATCH 017/157] Added DSP file for filtering --- scos_actions/signal_processing/filtering.py | 27 +++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 scos_actions/signal_processing/filtering.py diff --git a/scos_actions/signal_processing/filtering.py b/scos_actions/signal_processing/filtering.py new file mode 100644 index 00000000..78978caf --- /dev/null +++ b/scos_actions/signal_processing/filtering.py @@ -0,0 +1,27 @@ +import logging +import numpy as np +from scipy.signal import ellip, ellipord + +logger = logging.getLogger(__name__) + +def generate_elliptic_iir_low_pass_filter( + rp_dB: float, + rs_dB: float, + cutoff_Hz: float, + width_Hz: float, + sample_rate_Hz: float +) -> np.ndarray: + """ + Generate an elliptic IIR low pass filter. + + :param rp_dB: Maximum passband ripple below unity gain, in dB. + :param rs_dB: Minimum stopband attenuation, in dB. + :param cutoff_Hz: Filter cutoff frequency, in Hz. + :param width_Hz: Passband-to-stopband transition width, in Hz. + :param sample_rate_Hz: Sampling rate, in Hz. + :return: Second-order sections representation of the IIR filter. + """ + ord, wn = ellipord(cutoff_Hz, cutoff_Hz + width_Hz, rp_dB, rs_dB, False, sample_rate_Hz) + sos = ellip(ord, rp_dB, rs_dB, wn, 'lowpass', False, 'sos', sample_rate_Hz) + logger.debug(f'Generated low-pass IIR filter with order {ord}.') + return sos \ No newline at end of file From 7c30b1c815946e917fc198e15a83f94e2cee02cd Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 20 Jul 2022 17:26:55 -0600 Subject: [PATCH 018/157] Make frequency shifting in FFT optional --- scos_actions/signal_processing/fft.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scos_actions/signal_processing/fft.py b/scos_actions/signal_processing/fft.py index 901a3740..bff93b5a 100644 --- a/scos_actions/signal_processing/fft.py +++ b/scos_actions/signal_processing/fft.py @@ -15,6 +15,7 @@ def get_fft( norm: str = "forward", fft_window: np.ndarray = None, num_ffts: int = 0, + shift: bool = True, workers: int = os.cpu_count() // 2, ) -> np.ndarray: """ @@ -55,6 +56,8 @@ def get_fft( Setting this to zero or a negative number results in "as many as possible" behavior, which is also the default behavior if num_ffts is not specified. + :param shift: If True, shift the zero-frequency component to the + center of the spectrum. :param workers: Maximum number of workers to use for parallel computation. See scipy.fft.fft for more details. :return: The transformed input, scaled based on the specified @@ -79,9 +82,12 @@ def get_fft( if fft_window is not None: time_data *= fft_window - # Take and shift the FFT + # Take the FFT complex_fft = sp_fft(time_data, norm=norm, workers=workers) - complex_fft = np.fft.fftshift(complex_fft) + + # Shift the frequencies if desired + if shift: + complex_fft = np.fft.fftshift(complex_fft) return complex_fft From 50a7c7c846ea629ae41919420e9f8a02bc8565a6 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 20 Jul 2022 17:35:08 -0600 Subject: [PATCH 019/157] Specify axis for shifting FFT --- scos_actions/signal_processing/fft.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scos_actions/signal_processing/fft.py b/scos_actions/signal_processing/fft.py index bff93b5a..aeac70e0 100644 --- a/scos_actions/signal_processing/fft.py +++ b/scos_actions/signal_processing/fft.py @@ -85,9 +85,9 @@ def get_fft( # Take the FFT complex_fft = sp_fft(time_data, norm=norm, workers=workers) - # Shift the frequencies if desired + # Shift the frequencies if desired (only along second axis) if shift: - complex_fft = np.fft.fftshift(complex_fft) + complex_fft = np.fft.fftshift(complex_fft, axes=(1,)) return complex_fft From 7e70c2533b7c3a7c6a5575de93857402bc0a004c Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 20 Jul 2022 17:43:02 -0600 Subject: [PATCH 020/157] Add pseudo-power calculation --- .../signal_processing/power_analysis.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/scos_actions/signal_processing/power_analysis.py b/scos_actions/signal_processing/power_analysis.py index 8ea99bf4..c7bf3bb2 100644 --- a/scos_actions/signal_processing/power_analysis.py +++ b/scos_actions/signal_processing/power_analysis.py @@ -26,12 +26,38 @@ def calculate_power_watts(val_volts, impedance_ohms: float = 50.0): returned quantity is always real. """ if np.isscalar(val_volts): - power = (np.abs(val_volts) ** 2) / impedance_ohms + power = (np.abs(val_volts) ** 2.) / impedance_ohms else: power = ne.evaluate("(abs(val_volts).real**2)/impedance_ohms") return power +def calculate_pseudo_power(val): + """ + Calculate the 'pseudo-power' (magnitude-squared) of input samples. + + Calculation: abs(val)^2 + + 'Pseudo-power' is useful in certain applications to avoid + computing the power of many samples before data reduction + by a detector. + + NumPy is used for scalar inputs. + NumExpr is used to speed up the operation for arrays. + + :param val: A value, or array of values, to be converted + to 'pseudo-power' (magnitude-squared). The input may be + complex or real. + :return: The input val, converted to 'pseudo-power' (magnitude- + squared). The returned quantity is always real. + """ + if np.isscalar(val): + ps_pwr = np.abs(val) ** 2. + else: + ps_pwr = ne.evaluate("abs(val).real**2") + return ps_pwr + + def create_power_detector(name: str, detectors: list) -> EnumMeta: """ Construct a power detector based on a list of selected detectors. From d3af2785211b967552be95e0c8137a30aea6ed08 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 20 Jul 2022 17:49:26 -0600 Subject: [PATCH 021/157] Remove unused import --- scos_actions/actions/acquire_single_freq_fft.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py index 35147cbe..67e3fc8b 100644 --- a/scos_actions/actions/acquire_single_freq_fft.py +++ b/scos_actions/actions/acquire_single_freq_fft.py @@ -107,7 +107,7 @@ create_power_detector, ) from scos_actions.signal_processing.unit_conversion import convert_watts_to_dBm, convert_linear_to_dB -from numpy import float32, log10, ndarray +from numpy import float32, ndarray logger = logging.getLogger(__name__) From 56dd67d2f540abccb89a27805e870fe2486c1e9a Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 21 Jul 2022 11:16:45 -0600 Subject: [PATCH 022/157] Added reference FIR low pass filter --- scos_actions/signal_processing/filtering.py | 35 +++++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/scos_actions/signal_processing/filtering.py b/scos_actions/signal_processing/filtering.py index 78978caf..6a581f62 100644 --- a/scos_actions/signal_processing/filtering.py +++ b/scos_actions/signal_processing/filtering.py @@ -1,6 +1,6 @@ import logging import numpy as np -from scipy.signal import ellip, ellipord +from scipy.signal import ellip, ellipord, kaiserord, firwin logger = logging.getLogger(__name__) @@ -9,11 +9,14 @@ def generate_elliptic_iir_low_pass_filter( rs_dB: float, cutoff_Hz: float, width_Hz: float, - sample_rate_Hz: float + sample_rate_Hz: float, ) -> np.ndarray: """ Generate an elliptic IIR low pass filter. + Apply this filter to data using scipy.signal.sosfilt or + scipy.signal.sosfiltfilt (for forwards-backwards filtering). + :param rp_dB: Maximum passband ripple below unity gain, in dB. :param rs_dB: Minimum stopband attenuation, in dB. :param cutoff_Hz: Filter cutoff frequency, in Hz. @@ -24,4 +27,30 @@ def generate_elliptic_iir_low_pass_filter( ord, wn = ellipord(cutoff_Hz, cutoff_Hz + width_Hz, rp_dB, rs_dB, False, sample_rate_Hz) sos = ellip(ord, rp_dB, rs_dB, wn, 'lowpass', False, 'sos', sample_rate_Hz) logger.debug(f'Generated low-pass IIR filter with order {ord}.') - return sos \ No newline at end of file + return sos + + +def generate_fir_low_pass_filter( + attenuation_dB: float, + width_Hz: float, + cutoff_Hz: float, + sample_rate_Hz: float +) -> np.ndarray: + """ + Generate a FIR low pass filter using the Kaiser window method. + + Apply this filter to data using scipy.signal.lfilter or + scipy.signal.filtfilt (for forwards-backwards filtering). + In either case, use the coefficients output by this method as + the numerator parameter, with a denominator of 1.0. + + :param atten_dB: Minimum stopband attenuation, in dB. + :param width_Hz: Width of the transition region, in Hz. + :param cutoff_Hz: Filter cutoff frequency, in Hz. + :param sample_rate_Hz: Sampling rate, in Hz. + :return: Coeffiecients of the FIR low pass filter. + """ + ord, beta = kaiserord(attenuation_dB, width_Hz / (0.5 * sample_rate_Hz)) + taps = firwin(ord + 1, cutoff_Hz, width_Hz, ('kaiser', beta), 'lowpass', True, fs=sample_rate_Hz) + logger.debug(f"Generated Type {'I' if ord % 2 == 0 else 'II'} low-pass FIR filter with order {ord} and length {ord + 1}.") + return taps \ No newline at end of file From dd13fb29f9b6eb04a9b66aec6505a3ca334f5a21 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 21 Jul 2022 12:32:16 -0600 Subject: [PATCH 023/157] remove for loops for dict to list conversion --- .../actions/acquire_stepped_freq_tdomain_iq.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py index e7d13800..c52f6634 100644 --- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py +++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py @@ -84,19 +84,8 @@ def __init__(self, parameters, sigan, gps=mock_gps): num_center_frequencies = len(parameters[FREQUENCY]) # convert dictionary of lists from yaml file to list of dictionaries - longest_length = 0 - for key, value in parameters.items(): - if key == "name": - continue - if len(value) > longest_length: - longest_length = len(value) - for i in range(longest_length): - sorted_params = {} - for key in parameters.keys(): - if key == "name": - continue - sorted_params[key] = parameters[key][i] - self.sorted_measurement_parameters.append(sorted_params) + parameters.pop('name') + self.sorted_measurement_parameters = [dict(zip(parameters, v)) for v in zip(*parameters.values())] self.sorted_measurement_parameters.sort(key=lambda params: params[FREQUENCY]) self.sigan = sigan # make instance variable to allow mocking From 357d66aedb26b95ce239f3983cef5c32b1273ec2 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 21 Jul 2022 15:34:42 -0600 Subject: [PATCH 024/157] Move sigmf builder to metadata --- scos_actions/actions/acquire_single_freq_fft.py | 2 +- scos_actions/actions/acquire_single_freq_tdomain_iq.py | 2 +- scos_actions/actions/acquire_stepped_freq_tdomain_iq.py | 2 +- scos_actions/actions/interfaces/measurement_action.py | 2 +- .../actions/metadata/annotations/calibration_annotation.py | 2 +- scos_actions/actions/metadata/annotations/fft_annotation.py | 2 +- scos_actions/actions/metadata/annotations/sensor_annotation.py | 2 +- .../actions/metadata/annotations/time_domain_annotation.py | 2 +- scos_actions/actions/metadata/measurement_global.py | 2 +- scos_actions/actions/metadata/metadata.py | 2 +- scos_actions/actions/{ => metadata}/sigmf_builder.py | 0 11 files changed, 10 insertions(+), 10 deletions(-) rename scos_actions/actions/{ => metadata}/sigmf_builder.py (100%) diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py index 67e3fc8b..e12c7800 100644 --- a/scos_actions/actions/acquire_single_freq_fft.py +++ b/scos_actions/actions/acquire_single_freq_fft.py @@ -96,7 +96,7 @@ get_fft_window, get_fft_window_correction, ) -from scos_actions.actions.sigmf_builder import Domain, MeasurementType, SigMFBuilder +from scos_actions.actions.metadata.sigmf_builder import Domain, MeasurementType, SigMFBuilder from scos_actions.actions.metadata.annotations.fft_annotation import FrequencyDomainDetectionAnnotation from scos_actions.hardware import gps as mock_gps diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py index cce42279..44330c69 100644 --- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py +++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py @@ -36,7 +36,7 @@ from scos_actions import utils from scos_actions.actions.action_utils import get_param from scos_actions.actions.interfaces.measurement_action import MeasurementAction -from scos_actions.actions.sigmf_builder import Domain, MeasurementType, SigMFBuilder +from scos_actions.actions.metadata.sigmf_builder import Domain, MeasurementType, SigMFBuilder from scos_actions.hardware import gps as mock_gps from scos_actions.actions.metadata.annotations.time_domain_annotation import TimeDomainAnnotation from numpy import complex64 diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py index c52f6634..5e2ea233 100644 --- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py +++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py @@ -45,7 +45,7 @@ ) from scos_actions.actions.action_utils import get_param from scos_actions.actions.interfaces.signals import measurement_action_completed -from scos_actions.actions.sigmf_builder import Domain, MeasurementType +from scos_actions.actions.metadata.sigmf_builder import Domain, MeasurementType from scos_actions.hardware import gps as mock_gps logger = logging.getLogger(__name__) diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py index 9f80a272..e23b3060 100644 --- a/scos_actions/actions/interfaces/measurement_action.py +++ b/scos_actions/actions/interfaces/measurement_action.py @@ -8,7 +8,7 @@ from scos_actions.actions.metadata.annotations.calibration_annotation import CalibrationAnnotation from scos_actions.actions.metadata.measurement_global import MeasurementMetadata from scos_actions.actions.metadata.annotations.sensor_annotation import SensorAnnotation -from scos_actions.actions.sigmf_builder import SigMFBuilder +from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder logger = logging.getLogger(__name__) diff --git a/scos_actions/actions/metadata/annotations/calibration_annotation.py b/scos_actions/actions/metadata/annotations/calibration_annotation.py index 3c8c7828..0eabf87e 100644 --- a/scos_actions/actions/metadata/annotations/calibration_annotation.py +++ b/scos_actions/actions/metadata/annotations/calibration_annotation.py @@ -1,5 +1,5 @@ from scos_actions.actions.metadata.metadata import Metadata -from scos_actions.actions.sigmf_builder import SigMFBuilder +from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder class CalibrationAnnotation(Metadata): diff --git a/scos_actions/actions/metadata/annotations/fft_annotation.py b/scos_actions/actions/metadata/annotations/fft_annotation.py index a5bb6ca2..26815178 100644 --- a/scos_actions/actions/metadata/annotations/fft_annotation.py +++ b/scos_actions/actions/metadata/annotations/fft_annotation.py @@ -1,5 +1,5 @@ from scos_actions.actions.metadata.metadata import Metadata -from scos_actions.actions.sigmf_builder import SigMFBuilder +from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder class FrequencyDomainDetectionAnnotation(Metadata): diff --git a/scos_actions/actions/metadata/annotations/sensor_annotation.py b/scos_actions/actions/metadata/annotations/sensor_annotation.py index 2e7f4434..3efb30ef 100644 --- a/scos_actions/actions/metadata/annotations/sensor_annotation.py +++ b/scos_actions/actions/metadata/annotations/sensor_annotation.py @@ -1,5 +1,5 @@ from scos_actions.actions.metadata.metadata import Metadata -from scos_actions.actions.sigmf_builder import SigMFBuilder +from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder class SensorAnnotation(Metadata): diff --git a/scos_actions/actions/metadata/annotations/time_domain_annotation.py b/scos_actions/actions/metadata/annotations/time_domain_annotation.py index b0c8c244..5e1de40a 100644 --- a/scos_actions/actions/metadata/annotations/time_domain_annotation.py +++ b/scos_actions/actions/metadata/annotations/time_domain_annotation.py @@ -1,5 +1,5 @@ from scos_actions.actions.metadata.metadata import Metadata -from scos_actions.actions.sigmf_builder import SigMFBuilder +from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder class TimeDomainAnnotation(Metadata): diff --git a/scos_actions/actions/metadata/measurement_global.py b/scos_actions/actions/metadata/measurement_global.py index f45464b2..24e4ec0c 100644 --- a/scos_actions/actions/metadata/measurement_global.py +++ b/scos_actions/actions/metadata/measurement_global.py @@ -1,5 +1,5 @@ from scos_actions.actions.metadata.metadata import Metadata -from scos_actions.actions.sigmf_builder import SigMFBuilder +from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder class MeasurementMetadata(Metadata): diff --git a/scos_actions/actions/metadata/metadata.py b/scos_actions/actions/metadata/metadata.py index c6ee4cbd..93b8416b 100644 --- a/scos_actions/actions/metadata/metadata.py +++ b/scos_actions/actions/metadata/metadata.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from scos_actions.actions.sigmf_builder import SigMFBuilder +from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder class Metadata(ABC): diff --git a/scos_actions/actions/sigmf_builder.py b/scos_actions/actions/metadata/sigmf_builder.py similarity index 100% rename from scos_actions/actions/sigmf_builder.py rename to scos_actions/actions/metadata/sigmf_builder.py From 2bacecb82dc96aadd8cb8bd56dd25fe1c32dc29b Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 21 Jul 2022 15:58:58 -0600 Subject: [PATCH 025/157] move get_parameter_map to utils --- scos_actions/actions/calibrate_y_factor.py | 2 +- scos_actions/actions/interfaces/action.py | 15 ++------------- scos_actions/utils.py | 13 ++++++++++++- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 717f65a4..f77aca9e 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -169,7 +169,7 @@ def calibrate(self, params): # Configure signal analyzer self.sigan.preamp_enable = True super().configure_sigan(params) - param_map = self.get_parameter_map(params) + param_map = utils.get_parameter_map(params) if logger.isEnabledFor(logging.DEBUG): logger.debug('Preamp = ' + str(self.sigan.preamp_enable)) logger.debug('Ref_level: ' + str(self.sigan.reference_level)) diff --git a/scos_actions/actions/interfaces/action.py b/scos_actions/actions/interfaces/action.py index aabd3513..713c3bfd 100644 --- a/scos_actions/actions/interfaces/action.py +++ b/scos_actions/actions/interfaces/action.py @@ -1,7 +1,7 @@ -import copy import logging from abc import ABC, abstractmethod +from scos_actions.utils import get_parameter_map from scos_actions.hardware import gps as mock_gps from scos_actions.hardware import sigan as mock_sigan from scos_actions.capabilities import capabilities @@ -36,7 +36,7 @@ def __init__(self, parameters, sigan=mock_sigan, gps=mock_gps): self.sigan = sigan self.gps = gps self.sensor_definition = capabilities['sensor'] - self.parameter_map = self.get_parameter_map(self.parameters) + self.parameter_map = get_parameter_map(self.parameters) def configure(self, measurement_params): self.configure_sigan(measurement_params) @@ -77,17 +77,6 @@ def description(self): def name(self): return get_param("name", self.parameter_map) - def get_parameter_map(self, params): - if isinstance(params, list): - key_map = {} - for param in params: - for key, value in param.items(): - key_map[key] = value - return key_map - elif isinstance(params, dict): - return copy.deepcopy(params) - - @abstractmethod def __call__(self, schedule_entry, task_id): pass diff --git a/scos_actions/utils.py b/scos_actions/utils.py index 57f29067..feb08d4f 100644 --- a/scos_actions/utils.py +++ b/scos_actions/utils.py @@ -2,7 +2,7 @@ from datetime import datetime from dateutil import parser import json - +import copy logger = logging.getLogger(__name__) @@ -45,3 +45,14 @@ def get_parameters(i, parameters): def list_to_string(a_list): string_list = [str(i) for i in a_list ] return ','.join(string_list) + + +def get_parameter_map(params): + if isinstance(params, list): + key_map = {} + for param in params: + for key, value in param.items(): + key_map[key] = value + return key_map + elif isinstance(params, dict): + return copy.deepcopy(params) From 83cc81317a94b5f6f94f40bfbc732f41060aebf1 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 21 Jul 2022 16:03:54 -0600 Subject: [PATCH 026/157] Rename get_parameters --- scos_actions/actions/calibrate_y_factor.py | 2 +- scos_actions/tests/test_utils.py | 2 +- scos_actions/utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index f77aca9e..ced45d4c 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -147,7 +147,7 @@ def __call__(self, schedule_entry_json, task_id): detail = '' if isinstance(frequencies, list): for i in range(len(frequencies)): - iteration_params = utils.get_parameters(i, self.parameter_map) + iteration_params = utils.get_iteration_parameters(i, self.parameter_map) if i == 0: detail += self.calibrate(iteration_params) else: diff --git a/scos_actions/tests/test_utils.py b/scos_actions/tests/test_utils.py index 26d157c8..40128034 100644 --- a/scos_actions/tests/test_utils.py +++ b/scos_actions/tests/test_utils.py @@ -4,7 +4,7 @@ class MyTestCase(unittest.TestCase): def test_get_parameters(self): parameters = {"name": 'test_params', 'frequency': [100,200,300], 'gain': [0,10,40], 'sample_rate': [1, 2,3]} - iteration_0_params = utils.get_parameters(0, parameters) + iteration_0_params = utils.get_iteration_parameters(0, parameters) self.assertEqual(3, len(iteration_0_params)) self.assertEqual(iteration_0_params['frequency'], 100) self.assertEqual(iteration_0_params['gain'], 0) diff --git a/scos_actions/utils.py b/scos_actions/utils.py index feb08d4f..cd6623ee 100644 --- a/scos_actions/utils.py +++ b/scos_actions/utils.py @@ -35,7 +35,7 @@ def load_from_json(fname): except Exception: logger.exception("Unable to load JSON file {}".format(fname)) -def get_parameters(i, parameters): +def get_iteration_parameters(i, parameters): iteration_params = {} for key in parameters: if key != 'name': From 4faa0897195d934e9fdee2f70ff37dfe6abe7ef6 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 21 Jul 2022 16:07:46 -0600 Subject: [PATCH 027/157] Refactor to consolidate utils --- .../actions/acquire_single_freq_fft.py | 10 ++++----- .../actions/acquire_single_freq_tdomain_iq.py | 8 +++---- .../acquire_stepped_freq_tdomain_iq.py | 6 ++--- scos_actions/actions/action_utils.py | 16 -------------- scos_actions/actions/calibrate_y_factor.py | 22 +++++++++---------- scos_actions/actions/interfaces/action.py | 5 ++--- scos_actions/utils.py | 22 +++++++++++++++++++ 7 files changed, 47 insertions(+), 42 deletions(-) diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py index e12c7800..986aa055 100644 --- a/scos_actions/actions/acquire_single_freq_fft.py +++ b/scos_actions/actions/acquire_single_freq_fft.py @@ -100,7 +100,7 @@ from scos_actions.actions.metadata.annotations.fft_annotation import FrequencyDomainDetectionAnnotation from scos_actions.hardware import gps as mock_gps -from scos_actions.actions.action_utils import get_param +from scos_actions.utils import get_parameter from scos_actions.signal_processing.power_analysis import ( apply_power_detector, calculate_power_watts, @@ -142,10 +142,10 @@ class SingleFrequencyFftAcquisition(MeasurementAction): def __init__(self, parameters, sigan, gps=mock_gps): super().__init__(parameters, sigan, gps) # Pull parameters from action config - self.fft_size = get_param(FFT_SIZE, self.parameter_map) - self.nffts = get_param(NUM_FFTS, self.parameter_map) - self.nskip = get_param(NUM_SKIP, self.parameter_map) - self.frequency_Hz = get_param(FREQUENCY, self.parameter_map) + self.fft_size = get_parameter(FFT_SIZE, self.parameter_map) + self.nffts = get_parameter(NUM_FFTS, self.parameter_map) + self.nskip = get_parameter(NUM_SKIP, self.parameter_map) + self.frequency_Hz = get_parameter(FREQUENCY, self.parameter_map) # FFT setup self.fft_detector = create_power_detector( "M4sDetector", ["min", "max", "mean", "median", "sample"] diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py index 44330c69..2779f138 100644 --- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py +++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py @@ -34,7 +34,7 @@ import logging from scos_actions import utils -from scos_actions.actions.action_utils import get_param +from scos_actions.utils import get_parameter from scos_actions.actions.interfaces.measurement_action import MeasurementAction from scos_actions.actions.metadata.sigmf_builder import Domain, MeasurementType, SigMFBuilder from scos_actions.hardware import gps as mock_gps @@ -73,9 +73,9 @@ class SingleFrequencyTimeDomainIqAcquisition(MeasurementAction): def __init__(self, parameters, sigan, gps=mock_gps): super().__init__(parameters=parameters, sigan=sigan, gps=gps) # Pull parameters from action config - self.nskip = get_param(NUM_SKIP, self.parameter_map) - self.duration_ms = get_param(DURATION_MS, self.parameter_map) - self.frequency_Hz = get_param(FREQUENCY, self.parameter_map) + self.nskip = get_parameter(NUM_SKIP, self.parameter_map) + self.duration_ms = get_parameter(DURATION_MS, self.parameter_map) + self.frequency_Hz = get_parameter(FREQUENCY, self.parameter_map) def execute(self, schedule_entry, task_id) -> dict: start_time = utils.get_datetime_str_now() diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py index 5e2ea233..56c74a07 100644 --- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py +++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py @@ -43,7 +43,7 @@ from scos_actions.actions.acquire_single_freq_tdomain_iq import ( SingleFrequencyTimeDomainIqAcquisition, ) -from scos_actions.actions.action_utils import get_param +from scos_actions.utils import get_parameter from scos_actions.actions.interfaces.signals import measurement_action_completed from scos_actions.actions.metadata.sigmf_builder import Domain, MeasurementType from scos_actions.hardware import gps as mock_gps @@ -100,8 +100,8 @@ def __call__(self, schedule_entry_json, task_id): ): start_time = utils.get_datetime_str_now() self.configure(measurement_params) - duration_ms = get_param(DURATION_MS, measurement_params) - nskip = get_param(NUM_SKIP, measurement_params) + duration_ms = get_parameter(DURATION_MS, measurement_params) + nskip = get_parameter(NUM_SKIP, measurement_params) sample_rate = self.sigan.sample_rate num_samples = int(sample_rate * duration_ms * 1e-3) measurement_result = super().acquire_data(num_samples, nskip) diff --git a/scos_actions/actions/action_utils.py b/scos_actions/actions/action_utils.py index f11cce5b..b28b04f6 100644 --- a/scos_actions/actions/action_utils.py +++ b/scos_actions/actions/action_utils.py @@ -1,19 +1,3 @@ -class ParameterException(Exception): - """Basic exception handling for missing parameters.""" - def __init__(self, param): - super().__init__(f"{param} missing from measurement parameters.") -def get_param(p: str, params: dict): - """ - Get a parameter by key from a parameter dictionary. - - :param p: The parameter name (key). - :param params: The parameter dictionary. - :return: The specified parameter (value). - :raises ParameterException: If p is not a key in params. - """ - if p not in params: - raise ParameterException(p) - return params[p] diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index ced45d4c..c43050f0 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -77,7 +77,7 @@ from scos_actions.settings import sensor_calibration from scos_actions.settings import SENSOR_CALIBRATION_FILE from scos_actions.actions.interfaces.action import Action -from scos_actions.actions.action_utils import get_param +from scos_actions.utils import get_parameter from scos_actions.signal_processing.fft import get_fft, get_fft_enbw, get_fft_window from scos_actions.signal_processing.calibration import ( @@ -176,9 +176,9 @@ def calibrate(self, params): logger.debug('Attenuation:' + str(self.sigan.attenuation)) # Get parameters from action config - fft_size = get_param(FFT_SIZE, param_map) - nffts = get_param(NUM_FFTS, param_map) - nskip = get_param(NUM_SKIP, param_map) + fft_size = get_parameter(FFT_SIZE, param_map) + nffts = get_parameter(NUM_FFTS, param_map) + nskip = get_parameter(NUM_SKIP, param_map) fft_window = get_fft_window(self.fft_window_type, fft_size) num_samples = fft_size * nffts @@ -244,16 +244,16 @@ def apply_mean_fft( @property def description(self): - if isinstance(get_param(FREQUENCY, self.parameter_map), float): - frequencies = get_param(FREQUENCY, self.parameter_map) / 1e6 - nffts = get_param(NUM_FFTS, self.parameter_map) - fft_size = get_param(FFT_SIZE, self.parameter_map) + if isinstance(get_parameter(FREQUENCY, self.parameter_map), float): + frequencies = get_parameter(FREQUENCY, self.parameter_map) / 1e6 + nffts = get_parameter(NUM_FFTS, self.parameter_map) + fft_size = get_parameter(FFT_SIZE, self.parameter_map) else: frequencies = utils.list_to_string( - [f / 1e6 for f in get_param(FREQUENCY, self.parameter_map)] + [f / 1e6 for f in get_parameter(FREQUENCY, self.parameter_map)] ) - nffts = utils.list_to_string(get_param(NUM_FFTS, self.parameter_map)) - fft_size = utils.list_to_string(get_param(FFT_SIZE, self.parameter_map)) + nffts = utils.list_to_string(get_parameter(NUM_FFTS, self.parameter_map)) + fft_size = utils.list_to_string(get_parameter(FFT_SIZE, self.parameter_map)) acq_plan = ( f"Performs a y-factor calibration at frequencies: " f"{frequencies}, nffts:{nffts}, fft_size: {fft_size}\n" diff --git a/scos_actions/actions/interfaces/action.py b/scos_actions/actions/interfaces/action.py index 713c3bfd..41491e28 100644 --- a/scos_actions/actions/interfaces/action.py +++ b/scos_actions/actions/interfaces/action.py @@ -1,12 +1,11 @@ import logging from abc import ABC, abstractmethod -from scos_actions.utils import get_parameter_map +from scos_actions.utils import get_parameter_map, get_parameter from scos_actions.hardware import gps as mock_gps from scos_actions.hardware import sigan as mock_sigan from scos_actions.capabilities import capabilities from scos_actions.hardware import preselector -from scos_actions.actions.action_utils import get_param logger = logging.getLogger(__name__) @@ -75,7 +74,7 @@ def description(self): @property def name(self): - return get_param("name", self.parameter_map) + return get_parameter("name", self.parameter_map) @abstractmethod def __call__(self, schedule_entry, task_id): diff --git a/scos_actions/utils.py b/scos_actions/utils.py index cd6623ee..619211a6 100644 --- a/scos_actions/utils.py +++ b/scos_actions/utils.py @@ -7,6 +7,12 @@ logger = logging.getLogger(__name__) +class ParameterException(Exception): + """Basic exception handling for missing parameters.""" + def __init__(self, param): + super().__init__(f"{param} missing from measurement parameters.") + + def get_datetime_str_now(): return datetime.utcnow().isoformat(timespec="milliseconds") + "Z" @@ -35,6 +41,7 @@ def load_from_json(fname): except Exception: logger.exception("Unable to load JSON file {}".format(fname)) + def get_iteration_parameters(i, parameters): iteration_params = {} for key in parameters: @@ -42,6 +49,7 @@ def get_iteration_parameters(i, parameters): iteration_params[key] = parameters[key][i] return iteration_params + def list_to_string(a_list): string_list = [str(i) for i in a_list ] return ','.join(string_list) @@ -56,3 +64,17 @@ def get_parameter_map(params): return key_map elif isinstance(params, dict): return copy.deepcopy(params) + + +def get_parameter(p: str, params: dict): + """ + Get a parameter by key from a parameter dictionary. + + :param p: The parameter name (key). + :param params: The parameter dictionary. + :return: The specified parameter (value). + :raises ParameterException: If p is not a key in params. + """ + if p not in params: + raise ParameterException(p) + return params[p] From 8105190d71faa254bfe6000b2c5a5387594a33f1 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 22 Jul 2022 14:51:57 -0600 Subject: [PATCH 028/157] Consolidate parameters and parameter_map --- .../actions/acquire_single_freq_fft.py | 10 +++---- .../actions/acquire_single_freq_tdomain_iq.py | 10 +++---- scos_actions/actions/calibrate_y_factor.py | 27 +++++++++---------- scos_actions/actions/interfaces/action.py | 8 +++--- scos_actions/utils.py | 11 -------- 5 files changed, 27 insertions(+), 39 deletions(-) diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py index 986aa055..dcd25d50 100644 --- a/scos_actions/actions/acquire_single_freq_fft.py +++ b/scos_actions/actions/acquire_single_freq_fft.py @@ -142,10 +142,10 @@ class SingleFrequencyFftAcquisition(MeasurementAction): def __init__(self, parameters, sigan, gps=mock_gps): super().__init__(parameters, sigan, gps) # Pull parameters from action config - self.fft_size = get_parameter(FFT_SIZE, self.parameter_map) - self.nffts = get_parameter(NUM_FFTS, self.parameter_map) - self.nskip = get_parameter(NUM_SKIP, self.parameter_map) - self.frequency_Hz = get_parameter(FREQUENCY, self.parameter_map) + self.fft_size = get_parameter(FFT_SIZE, self.parameters) + self.nffts = get_parameter(NUM_FFTS, self.parameters) + self.nskip = get_parameter(NUM_SKIP, self.parameters) + self.frequency_Hz = get_parameter(FREQUENCY, self.parameters) # FFT setup self.fft_detector = create_power_detector( "M4sDetector", ["min", "max", "mean", "median", "sample"] @@ -171,7 +171,7 @@ def execute(self, schedule_entry, task_id) -> dict: frequencies = get_fft_frequencies( self.fft_size, sample_rate_Hz, self.frequency_Hz ) - measurement_result.update(self.parameter_map) + measurement_result.update(self.parameters) measurement_result['description'] = self.description measurement_result['domain'] = Domain.FREQUENCY.value measurement_result['frequency_start'] = frequencies[0] diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py index 2779f138..f8efc527 100644 --- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py +++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py @@ -73,9 +73,9 @@ class SingleFrequencyTimeDomainIqAcquisition(MeasurementAction): def __init__(self, parameters, sigan, gps=mock_gps): super().__init__(parameters=parameters, sigan=sigan, gps=gps) # Pull parameters from action config - self.nskip = get_parameter(NUM_SKIP, self.parameter_map) - self.duration_ms = get_parameter(DURATION_MS, self.parameter_map) - self.frequency_Hz = get_parameter(FREQUENCY, self.parameter_map) + self.nskip = get_parameter(NUM_SKIP, self.parameters) + self.duration_ms = get_parameter(DURATION_MS, self.parameters) + self.frequency_Hz = get_parameter(FREQUENCY, self.parameters) def execute(self, schedule_entry, task_id) -> dict: start_time = utils.get_datetime_str_now() @@ -85,7 +85,7 @@ def execute(self, schedule_entry, task_id) -> dict: measurement_result = self.acquire_data(num_samples, self.nskip) measurement_result['start_time'] = start_time end_time = utils.get_datetime_str_now() - measurement_result.update(self.parameter_map) + measurement_result.update(self.parameters) measurement_result['end_time'] = end_time measurement_result['domain'] = Domain.TIME.value measurement_result['measurement_type'] = MeasurementType.SINGLE_FREQUENCY.value @@ -113,7 +113,7 @@ def description(self): f"The signal analyzer is tuned to {frequency_MHz:.2f} " + "MHz and the following parameters are set:\n" ) - for name, value in self.parameter_map.items(): + for name, value in self.parameters.items(): if name not in used_keys: acq_plan += f"{name} = {value}\n" acq_plan += f"\nThen, acquire samples for {self.duration_ms} ms\n." diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index c43050f0..1572eb8a 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -143,11 +143,11 @@ def __init__(self, parameters, sigan, gps=mock_gps): def __call__(self, schedule_entry_json, task_id): """This is the entrypoint function called by the scheduler.""" self.test_required_components() - frequencies = self.parameter_map[FREQUENCY] + frequencies = self.parameters[FREQUENCY] detail = '' if isinstance(frequencies, list): for i in range(len(frequencies)): - iteration_params = utils.get_iteration_parameters(i, self.parameter_map) + iteration_params = utils.get_iteration_parameters(i, self.parameters) if i == 0: detail += self.calibrate(iteration_params) else: @@ -169,16 +169,15 @@ def calibrate(self, params): # Configure signal analyzer self.sigan.preamp_enable = True super().configure_sigan(params) - param_map = utils.get_parameter_map(params) if logger.isEnabledFor(logging.DEBUG): logger.debug('Preamp = ' + str(self.sigan.preamp_enable)) logger.debug('Ref_level: ' + str(self.sigan.reference_level)) logger.debug('Attenuation:' + str(self.sigan.attenuation)) # Get parameters from action config - fft_size = get_parameter(FFT_SIZE, param_map) - nffts = get_parameter(NUM_FFTS, param_map) - nskip = get_parameter(NUM_SKIP, param_map) + fft_size = get_parameter(FFT_SIZE, params) + nffts = get_parameter(NUM_FFTS, params) + nskip = get_parameter(NUM_SKIP, params) fft_window = get_fft_window(self.fft_window_type, fft_size) num_samples = fft_size * nffts @@ -215,7 +214,7 @@ def calibrate(self, params): mean_on_watts, mean_off_watts, enr_linear, enbw_hz, temp_k ) sensor_calibration.update( - param_map, + params, utils.get_datetime_str_now(), gain, noise_figure, @@ -244,16 +243,16 @@ def apply_mean_fft( @property def description(self): - if isinstance(get_parameter(FREQUENCY, self.parameter_map), float): - frequencies = get_parameter(FREQUENCY, self.parameter_map) / 1e6 - nffts = get_parameter(NUM_FFTS, self.parameter_map) - fft_size = get_parameter(FFT_SIZE, self.parameter_map) + if isinstance(get_parameter(FREQUENCY, self.parameters), float): + frequencies = get_parameter(FREQUENCY, self.parameters) / 1e6 + nffts = get_parameter(NUM_FFTS, self.parameters) + fft_size = get_parameter(FFT_SIZE, self.parameters) else: frequencies = utils.list_to_string( - [f / 1e6 for f in get_parameter(FREQUENCY, self.parameter_map)] + [f / 1e6 for f in get_parameter(FREQUENCY, self.parameters)] ) - nffts = utils.list_to_string(get_parameter(NUM_FFTS, self.parameter_map)) - fft_size = utils.list_to_string(get_parameter(FFT_SIZE, self.parameter_map)) + nffts = utils.list_to_string(get_parameter(NUM_FFTS, self.parameters)) + fft_size = utils.list_to_string(get_parameter(FFT_SIZE, self.parameters)) acq_plan = ( f"Performs a y-factor calibration at frequencies: " f"{frequencies}, nffts:{nffts}, fft_size: {fft_size}\n" diff --git a/scos_actions/actions/interfaces/action.py b/scos_actions/actions/interfaces/action.py index 41491e28..7037bef7 100644 --- a/scos_actions/actions/interfaces/action.py +++ b/scos_actions/actions/interfaces/action.py @@ -1,7 +1,8 @@ import logging from abc import ABC, abstractmethod +from copy import deepcopy -from scos_actions.utils import get_parameter_map, get_parameter +from scos_actions.utils import get_parameter from scos_actions.hardware import gps as mock_gps from scos_actions.hardware import sigan as mock_sigan from scos_actions.capabilities import capabilities @@ -31,11 +32,10 @@ class Action(ABC): PRESELECTOR_PATH_KEY='rf_path' def __init__(self, parameters, sigan=mock_sigan, gps=mock_gps): - self.parameters = parameters + self.parameters = deepcopy(parameters) self.sigan = sigan self.gps = gps self.sensor_definition = capabilities['sensor'] - self.parameter_map = get_parameter_map(self.parameters) def configure(self, measurement_params): self.configure_sigan(measurement_params) @@ -74,7 +74,7 @@ def description(self): @property def name(self): - return get_parameter("name", self.parameter_map) + return get_parameter("name", self.parameters) @abstractmethod def __call__(self, schedule_entry, task_id): diff --git a/scos_actions/utils.py b/scos_actions/utils.py index 619211a6..b6461482 100644 --- a/scos_actions/utils.py +++ b/scos_actions/utils.py @@ -55,17 +55,6 @@ def list_to_string(a_list): return ','.join(string_list) -def get_parameter_map(params): - if isinstance(params, list): - key_map = {} - for param in params: - for key, value in param.items(): - key_map[key] = value - return key_map - elif isinstance(params, dict): - return copy.deepcopy(params) - - def get_parameter(p: str, params: dict): """ Get a parameter by key from a parameter dictionary. From eb93babcbfe93b573b38fc93c8eef0c5f92e6a51 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 22 Jul 2022 15:27:53 -0600 Subject: [PATCH 029/157] Improve iterable parameter mapping --- scos_actions/actions/calibrate_y_factor.py | 18 ++++++------- scos_actions/tests/test_utils.py | 10 +++---- scos_actions/utils.py | 31 +++++++++++++++++----- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 1572eb8a..8bfd3763 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -145,15 +145,15 @@ def __call__(self, schedule_entry_json, task_id): self.test_required_components() frequencies = self.parameters[FREQUENCY] detail = '' - if isinstance(frequencies, list): - for i in range(len(frequencies)): - iteration_params = utils.get_iteration_parameters(i, self.parameters) - if i == 0: - detail += self.calibrate(iteration_params) - else: - detail += os.linesep + self.calibrate(iteration_params) - elif isinstance(frequencies, float): - detail = self.calibrate(self.parameters) + # iteration_params is iterable even if it contains only one set of parameters + iteration_params = utils.get_iterable_parameters(self.parameters) + + # Calibrate + for i, p in enumerate(iteration_params): + if i == 0: + detail += self.calibrate(p) + else: + detail += os.linesep + self.calibrate(p) return detail diff --git a/scos_actions/tests/test_utils.py b/scos_actions/tests/test_utils.py index 40128034..90cc971a 100644 --- a/scos_actions/tests/test_utils.py +++ b/scos_actions/tests/test_utils.py @@ -4,11 +4,11 @@ class MyTestCase(unittest.TestCase): def test_get_parameters(self): parameters = {"name": 'test_params', 'frequency': [100,200,300], 'gain': [0,10,40], 'sample_rate': [1, 2,3]} - iteration_0_params = utils.get_iteration_parameters(0, parameters) - self.assertEqual(3, len(iteration_0_params)) - self.assertEqual(iteration_0_params['frequency'], 100) - self.assertEqual(iteration_0_params['gain'], 0) - self.assertEqual(iteration_0_params['sample_rate'], 1) + iteration_params = utils.get_iterable_parameters(parameters) + self.assertEqual(3, len(iteration_params)) + self.assertEqual(iteration_params[0]['frequency'], 100) + self.assertEqual(iteration_params[0]['gain'], 0) + self.assertEqual(iteration_params[0]['sample_rate'], 1) if __name__ == '__main__': diff --git a/scos_actions/utils.py b/scos_actions/utils.py index b6461482..5d0b6c80 100644 --- a/scos_actions/utils.py +++ b/scos_actions/utils.py @@ -42,12 +42,31 @@ def load_from_json(fname): logger.exception("Unable to load JSON file {}".format(fname)) -def get_iteration_parameters(i, parameters): - iteration_params = {} - for key in parameters: - if key != 'name': - iteration_params[key] = parameters[key][i] - return iteration_params +def get_iterable_parameters(parameters): + """Convert parameter dictionary into iterable list.""" + # Copy input and remove name key + params = dict(parameters) + del params["name"] + # Convert all elements to lists if they are not already + for p_key, p_val in params.items(): + if not isinstance(p_val, list): + params[p_key] = [p_val] + # Find longest set of parameters + max_param_length = max([len(p) for p in params.values()]) + for p_key, p_val in params.items(): + if len(p_val) < max_param_length: + if len(p_val) == 1: + # Repeat parameter to max length + # TODO: Add logger warning when this happens + params[p_key] = p_val * max_param_length + else: + # Don't make assumptions otherwise. Raise an error. + # TODO: Change this to ParameterException + raise Exception + # Construct iterable parameter mapping + result = [dict(zip(params, v)) for v in zip(*params.values())] + result.sort(key=lambda param: param["frequency"]) + return result def list_to_string(a_list): From a334867559caef4b51e026df045568b5e46453dd Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 22 Jul 2022 15:59:17 -0600 Subject: [PATCH 030/157] improve get_iterable_parameters documentation, logging, leverage dictionary comprehension --- scos_actions/utils.py | 62 ++++++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/scos_actions/utils.py b/scos_actions/utils.py index 5d0b6c80..3cc4c9dd 100644 --- a/scos_actions/utils.py +++ b/scos_actions/utils.py @@ -8,9 +8,9 @@ class ParameterException(Exception): - """Basic exception handling for missing parameters.""" - def __init__(self, param): - super().__init__(f"{param} missing from measurement parameters.") + """Basic exception handling for parameter-related problems.""" + def __init__(self, msg): + super().__init__(msg) def get_datetime_str_now(): @@ -42,27 +42,53 @@ def load_from_json(fname): logger.exception("Unable to load JSON file {}".format(fname)) -def get_iterable_parameters(parameters): - """Convert parameter dictionary into iterable list.""" - # Copy input and remove name key - params = dict(parameters) +def get_iterable_parameters(parameters: dict): + """ + Convert parameter dictionary into iterable list. + + The input parameters, as read from the YAML file, will be + converted into an iterable list, in which each element is + an individual set of corresponding parameters. This is useful + for multi-frequency measurements, for example. + + The 'name' key is ignored, and not included in the returned list. + + This method also allows for YAML config files to specify + single values for any parameters which should be held constant + in every iteration. For example, specify only a single gain value + to use the same gain value for all measurements in a stepped-frequency + acquisition. + + The output list is automatically sorted by frequency, but it can + be manually resorted by any key if desired. + + :param parameters: The parameter dictionary, as loaded by the action. + :return: An iterable list of parameter dictionaries based on the input. + If only single values are given for all parameters in the input, a + list will still be returned, containing a single dictionary. + :raises ParameterException: If a parameter in the input has a number of + values which is neither 1 nor the maximum number of values specified + for any parameter. + """ + # Create copy of parameters with all values as lists + params = {k:(v if isinstance(v, list) else [v]) for k, v in parameters.items()} del params["name"] - # Convert all elements to lists if they are not already - for p_key, p_val in params.items(): - if not isinstance(p_val, list): - params[p_key] = [p_val] # Find longest set of parameters max_param_length = max([len(p) for p in params.values()]) - for p_key, p_val in params.items(): - if len(p_val) < max_param_length: + if max_param_length > 1: + for p_key, p_val in params.items(): if len(p_val) == 1: # Repeat parameter to max length - # TODO: Add logger warning when this happens + msg = f'Parameter {p_key} has only one value specified.\n' + msg += 'It will be used for all iterations in the action.' + logger.warning(msg) params[p_key] = p_val * max_param_length - else: + elif len(p_val) < max_param_length: # Don't make assumptions otherwise. Raise an error. - # TODO: Change this to ParameterException - raise Exception + msg = f'Parameter {p_key} has {len(p_val)} specified values.\n' + msg += 'YAML parameters must have either 1 value or a number of values equal to ' + msg += f'that of the parameter with the most values provided ({max_param_length}).' + raise ParameterException(msg) # Construct iterable parameter mapping result = [dict(zip(params, v)) for v in zip(*params.values())] result.sort(key=lambda param: param["frequency"]) @@ -84,5 +110,5 @@ def get_parameter(p: str, params: dict): :raises ParameterException: If p is not a key in params. """ if p not in params: - raise ParameterException(p) + raise ParameterException(f"{p} missing from measurement parameters.") return params[p] From e40f0ba7650ea62082f766d35702d86ca03b6370 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 22 Jul 2022 16:05:31 -0600 Subject: [PATCH 031/157] Simplified multi-frequency YAML config --- .../test_multi_frequency_iq_action.yml | 48 ++----------------- 1 file changed, 4 insertions(+), 44 deletions(-) diff --git a/scos_actions/configs/actions/test_multi_frequency_iq_action.yml b/scos_actions/configs/actions/test_multi_frequency_iq_action.yml index ca7f70c2..92f2308e 100644 --- a/scos_actions/configs/actions/test_multi_frequency_iq_action.yml +++ b/scos_actions/configs/actions/test_multi_frequency_iq_action.yml @@ -11,47 +11,7 @@ stepped_frequency_time_domain_iq: - 782e6 - 793e6 - 802e6 - gain: - - 40 - - 40 - - 40 - - 40 - - 40 - - 40 - - 40 - - 40 - - 40 - - 40 - sample_rate: - - 15.36e6 - - 15.36e6 - - 15.36e6 - - 15.36e6 - - 15.36e6 - - 15.36e6 - - 15.36e6 - - 15.36e6 - - 15.36e6 - - 15.36e6 - duration_ms: - - 80 - - 80 - - 80 - - 80 - - 80 - - 80 - - 80 - - 80 - - 80 - - 80 - nskip: - - 15.36e4 - - 15.36e4 - - 15.36e4 - - 15.36e4 - - 15.36e4 - - 15.36e4 - - 15.36e4 - - 15.36e4 - - 15.36e4 - - 15.36e4 + gain: 40 + sample_rate: 15.36e6 + duration_ms: 80 + nskip: 15.36e4 From 3b285e7eb5b2f4431b9e95ecdc4fd0edf292d1c0 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 22 Jul 2022 16:48:13 -0600 Subject: [PATCH 032/157] Remove unused code --- scos_actions/actions/calibrate_y_factor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 8bfd3763..d5b54b34 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -143,9 +143,9 @@ def __init__(self, parameters, sigan, gps=mock_gps): def __call__(self, schedule_entry_json, task_id): """This is the entrypoint function called by the scheduler.""" self.test_required_components() - frequencies = self.parameters[FREQUENCY] detail = '' # iteration_params is iterable even if it contains only one set of parameters + # it is also sorted by frequency from low to high iteration_params = utils.get_iterable_parameters(self.parameters) # Calibrate From 9b7d14937371bbdde54d8af2a524f9b013181f34 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 22 Jul 2022 16:49:38 -0600 Subject: [PATCH 033/157] Update stepped IQ action to use new iterable params --- scos_actions/actions/acquire_stepped_freq_tdomain_iq.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py index 56c74a07..77850682 100644 --- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py +++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py @@ -83,11 +83,9 @@ def __init__(self, parameters, sigan, gps=mock_gps): self.sorted_measurement_parameters = [] num_center_frequencies = len(parameters[FREQUENCY]) - # convert dictionary of lists from yaml file to list of dictionaries - parameters.pop('name') - self.sorted_measurement_parameters = [dict(zip(parameters, v)) for v in zip(*parameters.values())] - self.sorted_measurement_parameters.sort(key=lambda params: params[FREQUENCY]) - + # Create iterable parameter set + self.sorted_measurement_parameters = utils.get_iterable_parameters(parameters) + self.sigan = sigan # make instance variable to allow mocking self.num_center_frequencies = num_center_frequencies From b128608bde8a7230a5ccfb80c1d8d0ffad2016ea Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 25 Jul 2022 15:35:05 -0600 Subject: [PATCH 034/157] Fix y-factor description multi-parameter handling --- scos_actions/actions/calibrate_y_factor.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index d5b54b34..143e5570 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -242,17 +242,21 @@ def apply_mean_fft( @property def description(self): + # Get parameters; they may be single values or lists + frequencies = get_parameter(FREQUENCY, self.parameters) + nffts = get_parameter(NUM_FFTS, self.parameters) + fft_size = get_parameter(FFT_SIZE, self.parameters) - if isinstance(get_parameter(FREQUENCY, self.parameters), float): - frequencies = get_parameter(FREQUENCY, self.parameters) / 1e6 - nffts = get_parameter(NUM_FFTS, self.parameters) - fft_size = get_parameter(FFT_SIZE, self.parameters) - else: + # Convert parameter lists to strings if needed + if isinstance(frequencies, list): frequencies = utils.list_to_string( [f / 1e6 for f in get_parameter(FREQUENCY, self.parameters)] ) + if isinstance(nffts, list): nffts = utils.list_to_string(get_parameter(NUM_FFTS, self.parameters)) + if isinstance(fft_size, list): fft_size = utils.list_to_string(get_parameter(FFT_SIZE, self.parameters)) + acq_plan = ( f"Performs a y-factor calibration at frequencies: " f"{frequencies}, nffts:{nffts}, fft_size: {fft_size}\n" From 6c14279b14538a28ca2c8f325730487f18d31150 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 10:20:38 -0600 Subject: [PATCH 035/157] Add ignore-nan handling to power detectors --- .../signal_processing/power_analysis.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scos_actions/signal_processing/power_analysis.py b/scos_actions/signal_processing/power_analysis.py index c7bf3bb2..8f47d940 100644 --- a/scos_actions/signal_processing/power_analysis.py +++ b/scos_actions/signal_processing/power_analysis.py @@ -91,7 +91,7 @@ def create_power_detector(name: str, detectors: list) -> EnumMeta: def apply_power_detector( - data: np.ndarray, detector: EnumMeta, dtype: type = None + data: np.ndarray, detector: EnumMeta, dtype: type = None, ignore_nan: bool = True, ) -> np.ndarray: """ Apply statistical detectors to a 2-D array of samples. @@ -117,6 +117,9 @@ def apply_power_detector( :param dtype: Data type of values within the returned array. If not provided, the type is determined by NumPy as the minimum type required to hold the values (see numpy.array). + :param ignore_nan: If true, statistical detectors (min/max/mean/median) + will ignore any NaN values. NaN values may still appear in the + random sample detector result. :return: A 2-D array containing the selected detector results as the specified dtype. The number of rows is equal to the number of detectors applied, and the number of columns is equal @@ -125,16 +128,22 @@ def apply_power_detector( # Currently this is identical to apply_fft_detector: make general? # Get detector names from detector enumeration detectors = [d.name for _, d in enumerate(detector)] + + if ignore_nan: + detector_functions = [np.nanmin, np.nanmax, np.nanmean, np.nanmedian] + else: + detector_functions = [np.min, np.max, np.mean, np.median] + # Get functions based on specified detector detector_functions = [] if "min" in detectors: - detector_functions.append(np.min) + detector_functions.append(detector_functions[0]) if "max" in detectors: - detector_functions.append(np.max) + detector_functions.append(detector_functions[1]) if "mean" in detectors: - detector_functions.append(np.mean) + detector_functions.append(detector_functions[2]) if "median" in detectors: - detector_functions.append(np.median) + detector_functions.append(detector_functions[3]) # Apply statistical detectors result = [d(data, axis=0) for d in detector_functions] # Add sample detector result if configured From b325664705b46ad33a4b5a95cd6cc64ec2748abf Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 10:22:18 -0600 Subject: [PATCH 036/157] Do not ignore NaN by default --- scos_actions/signal_processing/power_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/signal_processing/power_analysis.py b/scos_actions/signal_processing/power_analysis.py index 8f47d940..ee2c0528 100644 --- a/scos_actions/signal_processing/power_analysis.py +++ b/scos_actions/signal_processing/power_analysis.py @@ -91,7 +91,7 @@ def create_power_detector(name: str, detectors: list) -> EnumMeta: def apply_power_detector( - data: np.ndarray, detector: EnumMeta, dtype: type = None, ignore_nan: bool = True, + data: np.ndarray, detector: EnumMeta, dtype: type = None, ignore_nan: bool = False, ) -> np.ndarray: """ Apply statistical detectors to a 2-D array of samples. From ab9264fa3c9d9f5fc82749edc32f2f7b59deed02 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 11:29:49 -0600 Subject: [PATCH 037/157] Test remove dateutil requirement --- requirements.in | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.in b/requirements.in index 06accdbc..729f3738 100644 --- a/requirements.in +++ b/requirements.in @@ -1,6 +1,5 @@ django>=3.0, <4.0 numpy>=1.0, <2.0 -python-dateutil>=2.0, < 3.0 ruamel.yaml>=0.1, <1.0 scipy>=1.6.0, <2.0 numexpr>=2.8.3, <3.0 From e099cf9aa5ce487ec8527cc03acb961321882bd9 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 11:31:41 -0600 Subject: [PATCH 038/157] Updated requirements --- requirements-dev.txt | 9 --------- requirements.txt | 6 +----- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 5ff3c8f8..22206adf 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,8 +8,6 @@ asgiref==3.5.0 # via # -r requirements.txt # django -atomicwrites==1.4.0 - # via pytest attrs==21.4.0 # via pytest certifi==2021.10.8 @@ -22,10 +20,6 @@ charset-normalizer==2.0.12 # via # -r requirements.txt # requests -colorama==0.4.5 - # via - # pytest - # tox distlib==0.3.4 # via virtualenv django==3.2.12 @@ -85,8 +79,6 @@ pyparsing==3.0.9 # packaging pytest==7.0.1 # via -r requirements-dev.in -python-dateutil==2.8.2 - # via -r requirements.txt pytz==2021.3 # via # -r requirements.txt @@ -110,7 +102,6 @@ sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive six==1.16.0 # via # -r requirements.txt - # python-dateutil # sigmf # tox # virtualenv diff --git a/requirements.txt b/requirements.txt index 3d11c041..d93004b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,8 +28,6 @@ packaging==21.3 # via numexpr pyparsing==3.0.9 # via packaging -python-dateutil==2.8.2 - # via -r requirements.in pytz==2021.3 # via django requests==2.27.1 @@ -43,9 +41,7 @@ scipy==1.7.3 sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive # via -r requirements.in six==1.16.0 - # via - # python-dateutil - # sigmf + # via sigmf sqlparse==0.4.2 # via django typing-extensions==4.1.1 From 6b33d6b3d63d780b5113eef46127cf2dc4fe6ecd Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 12:01:05 -0600 Subject: [PATCH 039/157] Remove unused import --- scos_actions/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scos_actions/utils.py b/scos_actions/utils.py index 3cc4c9dd..0ba45dfe 100644 --- a/scos_actions/utils.py +++ b/scos_actions/utils.py @@ -2,7 +2,6 @@ from datetime import datetime from dateutil import parser import json -import copy logger = logging.getLogger(__name__) From 9c8570ecf538569d421707b47b7fa2d9cf50dbf8 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 12:02:08 -0600 Subject: [PATCH 040/157] Re-add dateutil requirement --- requirements.in | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.in b/requirements.in index 729f3738..06accdbc 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,6 @@ django>=3.0, <4.0 numpy>=1.0, <2.0 +python-dateutil>=2.0, < 3.0 ruamel.yaml>=0.1, <1.0 scipy>=1.6.0, <2.0 numexpr>=2.8.3, <3.0 From 254ecf13e71a637c290c3aa703235952d3c12607 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 12:05:22 -0600 Subject: [PATCH 041/157] Updated requirements --- requirements-dev.txt | 3 +++ requirements.txt | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 22206adf..af59dbed 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -79,6 +79,8 @@ pyparsing==3.0.9 # packaging pytest==7.0.1 # via -r requirements-dev.in +python-dateutil==2.8.2 + # via -r requirements.txt pytz==2021.3 # via # -r requirements.txt @@ -102,6 +104,7 @@ sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive six==1.16.0 # via # -r requirements.txt + # python-dateutil # sigmf # tox # virtualenv diff --git a/requirements.txt b/requirements.txt index d93004b8..3d11c041 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,6 +28,8 @@ packaging==21.3 # via numexpr pyparsing==3.0.9 # via packaging +python-dateutil==2.8.2 + # via -r requirements.in pytz==2021.3 # via django requests==2.27.1 @@ -41,7 +43,9 @@ scipy==1.7.3 sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive # via -r requirements.in six==1.16.0 - # via sigmf + # via + # python-dateutil + # sigmf sqlparse==0.4.2 # via django typing-extensions==4.1.1 From 1faf99d85b8309bf172200b1001350c4d77c6e6f Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 12:18:21 -0600 Subject: [PATCH 042/157] Add preselector control actions --- scos_actions/actions/enable_antenna.py | 19 ++++++++++++++++++ .../actions/enable_noise_diode_off.py | 19 ++++++++++++++++++ scos_actions/actions/enable_noise_diode_on.py | 20 +++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 scos_actions/actions/enable_antenna.py create mode 100644 scos_actions/actions/enable_noise_diode_off.py create mode 100644 scos_actions/actions/enable_noise_diode_on.py diff --git a/scos_actions/actions/enable_antenna.py b/scos_actions/actions/enable_antenna.py new file mode 100644 index 00000000..2bbd0c7f --- /dev/null +++ b/scos_actions/actions/enable_antenna.py @@ -0,0 +1,19 @@ +"""Set the preselector to the antenna""" + +import logging + +from scos_actions.actions.interfaces.action import Action +from scos_actions.hardware import preselector + +logger = logging.getLogger(__name__) + + +class EnableAntenna(Action): + """Set the preselector to the antenna""" + + def __init__(self, sigan): + super().__init__(parameters={'name':'enable_antenna'}, sigan=sigan) + + def __call__(self, schedule_entry_json, task_id): + logger.debug("Enabling antenna") + preselector.set_state('antenna') diff --git a/scos_actions/actions/enable_noise_diode_off.py b/scos_actions/actions/enable_noise_diode_off.py new file mode 100644 index 00000000..a90ec964 --- /dev/null +++ b/scos_actions/actions/enable_noise_diode_off.py @@ -0,0 +1,19 @@ +"""Set the preselector to noise diode off""" + +import logging + +from scos_actions.actions.interfaces.action import Action +from scos_actions.hardware import preselector + +logger = logging.getLogger(__name__) + + +class EnableNoiseDiodeOff(Action): + """Set the preselector to noise diode off""" + + def __init__(self, sigan): + super().__init__(parameters={'name': 'enable_noise_diode_off'}, sigan=sigan) + + def __call__(self, schedule_entry_json, task_id): + logger.debug("Setting noise diode OFF") + preselector.set_state('noise_diode_off') diff --git a/scos_actions/actions/enable_noise_diode_on.py b/scos_actions/actions/enable_noise_diode_on.py new file mode 100644 index 00000000..628175b0 --- /dev/null +++ b/scos_actions/actions/enable_noise_diode_on.py @@ -0,0 +1,20 @@ +"""Set the preselector to noise diode on""" + +import logging + +from scos_actions.actions.interfaces.action import Action +from scos_actions.hardware import preselector + +logger = logging.getLogger(__name__) + + +class EnableNoiseDiodeOn(Action): + """Set the preselector to noise diode on""" + + def __init__(self, sigan): + super().__init__(parameters={'name': 'enable_noise_diode_on'}, sigan=sigan) + + def __call__(self, schedule_entry_json, task_id): + logger.debug("Setting noise diode ON") + preselector.set_state('noise_diode_on') + From 045552fa5c849e3a71ba27018835006482e4a0c9 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 12:20:47 -0600 Subject: [PATCH 043/157] Move preselector control actions to new directory --- scos_actions/actions/preselector_control/__init__.py | 0 scos_actions/actions/{ => preselector_control}/enable_antenna.py | 0 .../actions/{ => preselector_control}/enable_noise_diode_off.py | 0 .../actions/{ => preselector_control}/enable_noise_diode_on.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 scos_actions/actions/preselector_control/__init__.py rename scos_actions/actions/{ => preselector_control}/enable_antenna.py (100%) rename scos_actions/actions/{ => preselector_control}/enable_noise_diode_off.py (100%) rename scos_actions/actions/{ => preselector_control}/enable_noise_diode_on.py (100%) diff --git a/scos_actions/actions/preselector_control/__init__.py b/scos_actions/actions/preselector_control/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scos_actions/actions/enable_antenna.py b/scos_actions/actions/preselector_control/enable_antenna.py similarity index 100% rename from scos_actions/actions/enable_antenna.py rename to scos_actions/actions/preselector_control/enable_antenna.py diff --git a/scos_actions/actions/enable_noise_diode_off.py b/scos_actions/actions/preselector_control/enable_noise_diode_off.py similarity index 100% rename from scos_actions/actions/enable_noise_diode_off.py rename to scos_actions/actions/preselector_control/enable_noise_diode_off.py diff --git a/scos_actions/actions/enable_noise_diode_on.py b/scos_actions/actions/preselector_control/enable_noise_diode_on.py similarity index 100% rename from scos_actions/actions/enable_noise_diode_on.py rename to scos_actions/actions/preselector_control/enable_noise_diode_on.py From 7b18f2bde5d0cc6d0c8377c24a39501032a6ada1 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 12:24:06 -0600 Subject: [PATCH 044/157] Fix power detector bug --- scos_actions/signal_processing/power_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scos_actions/signal_processing/power_analysis.py b/scos_actions/signal_processing/power_analysis.py index ee2c0528..0611c189 100644 --- a/scos_actions/signal_processing/power_analysis.py +++ b/scos_actions/signal_processing/power_analysis.py @@ -135,7 +135,6 @@ def apply_power_detector( detector_functions = [np.min, np.max, np.mean, np.median] # Get functions based on specified detector - detector_functions = [] if "min" in detectors: detector_functions.append(detector_functions[0]) if "max" in detectors: From f2bf424c2c435315e0c263883113927ba81f7d20 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 12:26:39 -0600 Subject: [PATCH 045/157] Fix unit test for parameter mapping updates --- scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py index aaf811a3..7b3d7fde 100644 --- a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py +++ b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py @@ -46,4 +46,7 @@ def test_num_samples_skip(): action = actions["test_multi_frequency_iq_action"] assert action.description action(SINGLE_TIMEDOMAIN_IQ_MULTI_RECORDING_ACQUISITION, 1) - assert action.sigan._num_samples_skip == action.parameters["nskip"][-1] + if isinstance(action.parameters["nskip"], list): + assert action.sigan._num_samples_skip == action.parameters["nskip"][-1] + else: + assert action.sigan._num_samples_skip == action.parameters["nskip"] From aa425c972e71ecb020a09381f46b6500eccdad51 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 16:49:25 -0600 Subject: [PATCH 046/157] Move metadata classes out of actions directory --- scos_actions/actions/acquire_single_freq_fft.py | 4 ++-- scos_actions/actions/acquire_single_freq_tdomain_iq.py | 4 ++-- scos_actions/actions/acquire_stepped_freq_tdomain_iq.py | 2 +- scos_actions/actions/interfaces/measurement_action.py | 8 ++++---- scos_actions/{actions => }/metadata/__init__.py | 0 .../{actions => }/metadata/annotations/__init__.py | 0 .../metadata/annotations/calibration_annotation.py | 4 ++-- .../{actions => }/metadata/annotations/fft_annotation.py | 4 ++-- .../metadata/annotations/sensor_annotation.py | 4 ++-- .../metadata/annotations/time_domain_annotation.py | 4 ++-- scos_actions/{actions => }/metadata/measurement_global.py | 4 ++-- scos_actions/{actions => }/metadata/metadata.py | 2 +- scos_actions/{actions => }/metadata/sigmf_builder.py | 0 13 files changed, 20 insertions(+), 20 deletions(-) rename scos_actions/{actions => }/metadata/__init__.py (100%) rename scos_actions/{actions => }/metadata/annotations/__init__.py (100%) rename scos_actions/{actions => }/metadata/annotations/calibration_annotation.py (92%) rename scos_actions/{actions => }/metadata/annotations/fft_annotation.py (90%) rename scos_actions/{actions => }/metadata/annotations/sensor_annotation.py (84%) rename scos_actions/{actions => }/metadata/annotations/time_domain_annotation.py (82%) rename scos_actions/{actions => }/metadata/measurement_global.py (91%) rename scos_actions/{actions => }/metadata/metadata.py (82%) rename scos_actions/{actions => }/metadata/sigmf_builder.py (100%) diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py index dcd25d50..81f945aa 100644 --- a/scos_actions/actions/acquire_single_freq_fft.py +++ b/scos_actions/actions/acquire_single_freq_fft.py @@ -96,8 +96,8 @@ get_fft_window, get_fft_window_correction, ) -from scos_actions.actions.metadata.sigmf_builder import Domain, MeasurementType, SigMFBuilder -from scos_actions.actions.metadata.annotations.fft_annotation import FrequencyDomainDetectionAnnotation +from scos_actions.metadata.sigmf_builder import Domain, MeasurementType, SigMFBuilder +from scos_actions.metadata.annotations.fft_annotation import FrequencyDomainDetectionAnnotation from scos_actions.hardware import gps as mock_gps from scos_actions.utils import get_parameter diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py index f8efc527..3e3b89ee 100644 --- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py +++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py @@ -36,9 +36,9 @@ from scos_actions import utils from scos_actions.utils import get_parameter from scos_actions.actions.interfaces.measurement_action import MeasurementAction -from scos_actions.actions.metadata.sigmf_builder import Domain, MeasurementType, SigMFBuilder +from scos_actions.metadata.sigmf_builder import Domain, MeasurementType, SigMFBuilder from scos_actions.hardware import gps as mock_gps -from scos_actions.actions.metadata.annotations.time_domain_annotation import TimeDomainAnnotation +from scos_actions.metadata.annotations.time_domain_annotation import TimeDomainAnnotation from numpy import complex64 logger = logging.getLogger(__name__) diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py index 77850682..f0ad0167 100644 --- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py +++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py @@ -45,7 +45,7 @@ ) from scos_actions.utils import get_parameter from scos_actions.actions.interfaces.signals import measurement_action_completed -from scos_actions.actions.metadata.sigmf_builder import Domain, MeasurementType +from scos_actions.metadata.sigmf_builder import Domain, MeasurementType from scos_actions.hardware import gps as mock_gps logger = logging.getLogger(__name__) diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py index e23b3060..359842de 100644 --- a/scos_actions/actions/interfaces/measurement_action.py +++ b/scos_actions/actions/interfaces/measurement_action.py @@ -5,10 +5,10 @@ from scos_actions.hardware import gps as mock_gps from scos_actions.hardware import sigan as mock_sigan from scos_actions.actions.interfaces.signals import measurement_action_completed -from scos_actions.actions.metadata.annotations.calibration_annotation import CalibrationAnnotation -from scos_actions.actions.metadata.measurement_global import MeasurementMetadata -from scos_actions.actions.metadata.annotations.sensor_annotation import SensorAnnotation -from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder +from scos_actions.metadata.annotations.calibration_annotation import CalibrationAnnotation +from scos_actions.metadata.measurement_global import MeasurementMetadata +from scos_actions.metadata.annotations.sensor_annotation import SensorAnnotation +from scos_actions.metadata.sigmf_builder import SigMFBuilder logger = logging.getLogger(__name__) diff --git a/scos_actions/actions/metadata/__init__.py b/scos_actions/metadata/__init__.py similarity index 100% rename from scos_actions/actions/metadata/__init__.py rename to scos_actions/metadata/__init__.py diff --git a/scos_actions/actions/metadata/annotations/__init__.py b/scos_actions/metadata/annotations/__init__.py similarity index 100% rename from scos_actions/actions/metadata/annotations/__init__.py rename to scos_actions/metadata/annotations/__init__.py diff --git a/scos_actions/actions/metadata/annotations/calibration_annotation.py b/scos_actions/metadata/annotations/calibration_annotation.py similarity index 92% rename from scos_actions/actions/metadata/annotations/calibration_annotation.py rename to scos_actions/metadata/annotations/calibration_annotation.py index 0eabf87e..147f8f69 100644 --- a/scos_actions/actions/metadata/annotations/calibration_annotation.py +++ b/scos_actions/metadata/annotations/calibration_annotation.py @@ -1,5 +1,5 @@ -from scos_actions.actions.metadata.metadata import Metadata -from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder +from scos_actions.metadata.metadata import Metadata +from scos_actions.metadata.sigmf_builder import SigMFBuilder class CalibrationAnnotation(Metadata): diff --git a/scos_actions/actions/metadata/annotations/fft_annotation.py b/scos_actions/metadata/annotations/fft_annotation.py similarity index 90% rename from scos_actions/actions/metadata/annotations/fft_annotation.py rename to scos_actions/metadata/annotations/fft_annotation.py index 26815178..85a3ba7a 100644 --- a/scos_actions/actions/metadata/annotations/fft_annotation.py +++ b/scos_actions/metadata/annotations/fft_annotation.py @@ -1,5 +1,5 @@ -from scos_actions.actions.metadata.metadata import Metadata -from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder +from scos_actions.metadata.metadata import Metadata +from scos_actions.metadata.sigmf_builder import SigMFBuilder class FrequencyDomainDetectionAnnotation(Metadata): diff --git a/scos_actions/actions/metadata/annotations/sensor_annotation.py b/scos_actions/metadata/annotations/sensor_annotation.py similarity index 84% rename from scos_actions/actions/metadata/annotations/sensor_annotation.py rename to scos_actions/metadata/annotations/sensor_annotation.py index 3efb30ef..b3a8aa8c 100644 --- a/scos_actions/actions/metadata/annotations/sensor_annotation.py +++ b/scos_actions/metadata/annotations/sensor_annotation.py @@ -1,5 +1,5 @@ -from scos_actions.actions.metadata.metadata import Metadata -from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder +from scos_actions.metadata.metadata import Metadata +from scos_actions.metadata.sigmf_builder import SigMFBuilder class SensorAnnotation(Metadata): diff --git a/scos_actions/actions/metadata/annotations/time_domain_annotation.py b/scos_actions/metadata/annotations/time_domain_annotation.py similarity index 82% rename from scos_actions/actions/metadata/annotations/time_domain_annotation.py rename to scos_actions/metadata/annotations/time_domain_annotation.py index 5e1de40a..3f2f57c1 100644 --- a/scos_actions/actions/metadata/annotations/time_domain_annotation.py +++ b/scos_actions/metadata/annotations/time_domain_annotation.py @@ -1,5 +1,5 @@ -from scos_actions.actions.metadata.metadata import Metadata -from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder +from scos_actions.metadata.metadata import Metadata +from scos_actions.metadata.sigmf_builder import SigMFBuilder class TimeDomainAnnotation(Metadata): diff --git a/scos_actions/actions/metadata/measurement_global.py b/scos_actions/metadata/measurement_global.py similarity index 91% rename from scos_actions/actions/metadata/measurement_global.py rename to scos_actions/metadata/measurement_global.py index 24e4ec0c..1c7eb198 100644 --- a/scos_actions/actions/metadata/measurement_global.py +++ b/scos_actions/metadata/measurement_global.py @@ -1,5 +1,5 @@ -from scos_actions.actions.metadata.metadata import Metadata -from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder +from scos_actions.metadata.metadata import Metadata +from scos_actions.metadata.sigmf_builder import SigMFBuilder class MeasurementMetadata(Metadata): diff --git a/scos_actions/actions/metadata/metadata.py b/scos_actions/metadata/metadata.py similarity index 82% rename from scos_actions/actions/metadata/metadata.py rename to scos_actions/metadata/metadata.py index 93b8416b..7e73c92e 100644 --- a/scos_actions/actions/metadata/metadata.py +++ b/scos_actions/metadata/metadata.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from scos_actions.actions.metadata.sigmf_builder import SigMFBuilder +from scos_actions.metadata.sigmf_builder import SigMFBuilder class Metadata(ABC): diff --git a/scos_actions/actions/metadata/sigmf_builder.py b/scos_actions/metadata/sigmf_builder.py similarity index 100% rename from scos_actions/actions/metadata/sigmf_builder.py rename to scos_actions/metadata/sigmf_builder.py From a4363012e9274b18c69d71fa9f038cff9a9dcdbb Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 17:05:53 -0600 Subject: [PATCH 047/157] Replace deprecated markdownlint styling --- .markdownlint.yaml | 3 +++ .ml_style.rb | 6 ------ 2 files changed, 3 insertions(+), 6 deletions(-) create mode 100644 .markdownlint.yaml delete mode 100644 .ml_style.rb diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 00000000..d7265749 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,3 @@ +default: true +MD013: + line_length: 88 \ No newline at end of file diff --git a/.ml_style.rb b/.ml_style.rb deleted file mode 100644 index 750d9e88..00000000 --- a/.ml_style.rb +++ /dev/null @@ -1,6 +0,0 @@ -# markdownlint style -# https://github.com/markdownlint/markdownlint/blob/master/docs/creating_styles.md -# Enable all rules by default -all - -rule 'MD013', :line_length => 88 From 0de931240e2985ecb89011c8a450f77f8782984c Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 17:24:50 -0600 Subject: [PATCH 048/157] Delete old/empty file --- scos_actions/actions/action_utils.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 scos_actions/actions/action_utils.py diff --git a/scos_actions/actions/action_utils.py b/scos_actions/actions/action_utils.py deleted file mode 100644 index b28b04f6..00000000 --- a/scos_actions/actions/action_utils.py +++ /dev/null @@ -1,3 +0,0 @@ - - - From e683b14bd9eadb5ad2711da527b9c3c2d109727b Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 26 Jul 2022 17:38:30 -0600 Subject: [PATCH 049/157] Consolidate isort config --- .isort.cfg | 7 ------- .pre-commit-config.yaml | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index fcb6f2a5..00000000 --- a/.isort.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[settings] -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -line_length=88 -known_third_party = dateutil,django,numexpr,numpy,pytest,pytz,ruamel,scipy,setuptools,sigmf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 820fccc0..67568c2e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - id: isort name: isort (python) types: [file, python] - args: ["--profile", "black", "--filter-files"] + args: ["--profile", "black", "--filter-files", "--gitignore"] - repo: https://github.com/psf/black rev: 22.6.0 hooks: From 45993aaa87d233baa5aa61442857053d9e4e0f45 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Jul 2022 15:00:45 -0600 Subject: [PATCH 050/157] Restore older power scaling --- scos_actions/actions/calibrate_y_factor.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 143e5570..19bd740b 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -78,7 +78,7 @@ from scos_actions.settings import SENSOR_CALIBRATION_FILE from scos_actions.actions.interfaces.action import Action from scos_actions.utils import get_parameter -from scos_actions.signal_processing.fft import get_fft, get_fft_enbw, get_fft_window +from scos_actions.signal_processing.fft import get_fft, get_fft_enbw, get_fft_window, get_fft_window_correction from scos_actions.signal_processing.calibration import ( get_linear_enr, @@ -90,6 +90,9 @@ calculate_power_watts, create_power_detector, ) + +from scos_actions.signal_processing.unit_conversion import convert_watts_to_dBm + import os logger = logging.getLogger(__name__) @@ -179,6 +182,7 @@ def calibrate(self, params): nffts = get_parameter(NUM_FFTS, params) nskip = get_parameter(NUM_SKIP, params) fft_window = get_fft_window(self.fft_window_type, fft_size) + fft_acf = get_fft_window_correction(fft_window, 'amplitude') num_samples = fft_size * nffts logger.debug("Acquiring mean FFT") @@ -189,7 +193,7 @@ def calibrate(self, params): ) sample_rate = noise_on_measurement_result["sample_rate"] mean_on_watts = self.apply_mean_fft( - noise_on_measurement_result, fft_size, fft_window, nffts + noise_on_measurement_result, fft_size, fft_window, nffts, fft_acf ) # Set noise diode off @@ -223,20 +227,22 @@ def calibrate(self, params): ) # Debugging - noise_floor = Boltzmann * temp_k * enbw_hz - logger.debug(f'Noise floor: {noise_floor} Watts') + noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) + logger.debug(f'Noise floor: {noise_floor_dBm} dBm') logger.debug(f'Noise Figure: {noise_figure} dB') logger.debug(f'Gain: {gain} dB') - return 'Noise Figure:{}, Gain:{}'.format(noise_figure, gain) + return 'Noise Figure: {}, Gain: {}'.format(noise_figure, gain) def apply_mean_fft( - self, measurement_result: dict, fft_size: int, fft_window: ndarray, nffts: int + self, measurement_result: dict, fft_size: int, fft_window: ndarray, nffts: int, fft_window_cf: float ) -> ndarray: complex_fft = get_fft( - measurement_result["data"], fft_size, "backward", fft_window, nffts + measurement_result["data"], fft_size, "forward", fft_window, nffts ) power_fft = calculate_power_watts(complex_fft) + power_fft /= 2 # RF/baseband conversion + power_fft *= fft_window_cf # Window correction mean_result = apply_power_detector(power_fft, self.fft_detector) return mean_result From 9f3fecc2e16ea73f41e814a68fedee646b205784 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Jul 2022 16:19:05 -0600 Subject: [PATCH 051/157] Fix missing parameter --- scos_actions/actions/calibrate_y_factor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 19bd740b..8acd5721 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -207,7 +207,7 @@ def calibrate(self, params): num_samples, num_samples_skip=nskip, gain_adjust=False ) mean_off_watts = self.apply_mean_fft( - noise_off_measurement_result, fft_size, fft_window, nffts + noise_off_measurement_result, fft_size, fft_window, nffts, fft_acf ) # Y-Factor From 8ff4299fd7d5365259b88e6924ec3cf2400d38a1 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Jul 2022 17:15:19 -0600 Subject: [PATCH 052/157] Adjust FFT scaling for comparison --- scos_actions/actions/calibrate_y_factor.py | 28 ++++++++++--------- scos_actions/signal_processing/calibration.py | 8 +++--- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 8acd5721..fd4e2b3f 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -166,16 +166,11 @@ def calibrate(self, params): super().configure_preselector(NOISE_DIODE_ON) time.sleep(.25) - # Debugging - logger.debug('Before configure, Preamp = ' + str(self.sigan.preamp_enable)) - # Configure signal analyzer - self.sigan.preamp_enable = True super().configure_sigan(params) - if logger.isEnabledFor(logging.DEBUG): - logger.debug('Preamp = ' + str(self.sigan.preamp_enable)) - logger.debug('Ref_level: ' + str(self.sigan.reference_level)) - logger.debug('Attenuation:' + str(self.sigan.attenuation)) + logger.debug('Preamp = ' + str(self.sigan.preamp_enable)) + logger.debug('Ref_level: ' + str(self.sigan.reference_level)) + logger.debug('Attenuation:' + str(self.sigan.attenuation)) # Get parameters from action config fft_size = get_parameter(FFT_SIZE, params) @@ -185,7 +180,7 @@ def calibrate(self, params): fft_acf = get_fft_window_correction(fft_window, 'amplitude') num_samples = fft_size * nffts - logger.debug("Acquiring mean FFT") + logger.debug("Acquiring mean FFT with noise diode ON") # Get noise diode on mean FFT result noise_on_measurement_result = self.sigan.acquire_time_domain_samples( @@ -202,7 +197,7 @@ def calibrate(self, params): time.sleep(.25) # Get noise diode off mean FFT result - logger.debug('Acquiring noise off mean FFT') + logger.debug('Acquiring mean FFT with noise diode OFF') noise_off_measurement_result = self.sigan.acquire_time_domain_samples( num_samples, num_samples_skip=nskip, gain_adjust=False ) @@ -238,11 +233,18 @@ def apply_mean_fft( self, measurement_result: dict, fft_size: int, fft_window: ndarray, nffts: int, fft_window_cf: float ) -> ndarray: complex_fft = get_fft( - measurement_result["data"], fft_size, "forward", fft_window, nffts + time_data=measurement_result["data"], + fft_size=fft_size, + norm="forward", + fft_window=fft_window, + num_ffts=nffts, + shift=True ) + complex_fft /= 2 # INCORRECT SCALING for testing + complex_fft *= fft_window_cf power_fft = calculate_power_watts(complex_fft) - power_fft /= 2 # RF/baseband conversion - power_fft *= fft_window_cf # Window correction + # power_fft /= 2 # RF/baseband conversion + # power_fft *= fft_window_cf # Window correction mean_result = apply_power_detector(power_fft, self.fft_detector) return mean_result diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py index 7748ac50..dec764f4 100644 --- a/scos_actions/signal_processing/calibration.py +++ b/scos_actions/signal_processing/calibration.py @@ -65,14 +65,14 @@ def get_linear_enr(cal_source_idx: int = None) -> float: """ Get the excess noise ratio of a calibration source. - Specifying ``cal_source_idx`` is optional as long as there is + Specifying `cal_source_idx` is optional as long as there is only one calibration source. It is required if multiple calibration sources are present. - The preselector is loaded from scos_actions.hardware. + The preselector is loaded from `scos_actions.hardware`. :param cal_source_idx: The index of the specified - calibration source in preselector.cal_sources. + calibration source in `preselector.cal_sources`. :return: The excess noise ratio of the specified calibration source, in linear units. :raises CalibrationException: If multiple calibration sources are @@ -106,7 +106,7 @@ def get_temperature(sensor_idx: int = None) -> Tuple[float, float, float]: """ Get the temperature from a preselector sensor. - The preselector is loaded from scos_actions.hardware + The preselector is loaded from `scos_actions.hardware`. :param sensor_idx: The index of the desired temperature sensor in the preselector. From b6d0ee6ef26fb689037e043e3df4e1ce096760f5 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Jul 2022 17:23:39 -0600 Subject: [PATCH 053/157] Testing scaling --- scos_actions/actions/calibrate_y_factor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index fd4e2b3f..8978b10d 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -240,10 +240,10 @@ def apply_mean_fft( num_ffts=nffts, shift=True ) - complex_fft /= 2 # INCORRECT SCALING for testing + # TESTING SCALING complex_fft *= fft_window_cf power_fft = calculate_power_watts(complex_fft) - # power_fft /= 2 # RF/baseband conversion + power_fft /= 2 # RF/baseband conversion # power_fft *= fft_window_cf # Window correction mean_result = apply_power_detector(power_fft, self.fft_detector) return mean_result From 82b31fcfd5320fee4384b000c475774ddcd0d784 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Jul 2022 17:32:44 -0600 Subject: [PATCH 054/157] Fix window correction --- scos_actions/actions/calibrate_y_factor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 8978b10d..62bda0f8 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -241,10 +241,9 @@ def apply_mean_fft( shift=True ) # TESTING SCALING - complex_fft *= fft_window_cf power_fft = calculate_power_watts(complex_fft) power_fft /= 2 # RF/baseband conversion - # power_fft *= fft_window_cf # Window correction + power_fft *= fft_window_cf ** 2. # Window correction mean_result = apply_power_detector(power_fft, self.fft_detector) return mean_result From a1124adf24a1e04a6e7dd6a2f982f772981d6d66 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Jul 2022 17:48:30 -0600 Subject: [PATCH 055/157] Update window corrections --- scos_actions/actions/acquire_single_freq_fft.py | 2 +- scos_actions/actions/calibrate_y_factor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py index 81f945aa..d6299c56 100644 --- a/scos_actions/actions/acquire_single_freq_fft.py +++ b/scos_actions/actions/acquire_single_freq_fft.py @@ -202,7 +202,7 @@ def apply_m4s(self, measurement_result: dict) -> ndarray: # RF/Baseband power conversion (-3 dB) # FFT window amplitude correction m4s_result -= 3 - m4s_result += convert_linear_to_dB(self.fft_window_acf) + m4s_result += 2. * convert_linear_to_dB(self.fft_window_acf) return m4s_result @property diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 62bda0f8..c7e6d5b9 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -241,9 +241,9 @@ def apply_mean_fft( shift=True ) # TESTING SCALING + complex_fft *= fft_window_cf # Window correction power_fft = calculate_power_watts(complex_fft) power_fft /= 2 # RF/baseband conversion - power_fft *= fft_window_cf ** 2. # Window correction mean_result = apply_power_detector(power_fft, self.fft_detector) return mean_result From ddcc1c7ebe12bb3dac5dd5aac69300b5f338882d Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 27 Jul 2022 17:48:49 -0600 Subject: [PATCH 056/157] Remove unused imports, add speedup for windowing --- scos_actions/signal_processing/fft.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scos_actions/signal_processing/fft.py b/scos_actions/signal_processing/fft.py index aeac70e0..53e2f92c 100644 --- a/scos_actions/signal_processing/fft.py +++ b/scos_actions/signal_processing/fft.py @@ -1,7 +1,7 @@ import logging import os -from enum import Enum, EnumMeta +import numexpr as ne import numpy as np from scipy.fft import fft as sp_fft from scipy.signal import get_window @@ -80,7 +80,7 @@ def get_fft( # Apply the FFT window if provided if fft_window is not None: - time_data *= fft_window + time_data = ne.evaluate("time_data*fft_window") # Take the FFT complex_fft = sp_fft(time_data, norm=norm, workers=workers) From 356c694569fc33d67d275c6faf5a26053eba25b8 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 13:18:13 -0600 Subject: [PATCH 057/157] Update power detector for 1-D or 2-D inputs --- .../signal_processing/power_analysis.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/scos_actions/signal_processing/power_analysis.py b/scos_actions/signal_processing/power_analysis.py index 0611c189..bc88fc87 100644 --- a/scos_actions/signal_processing/power_analysis.py +++ b/scos_actions/signal_processing/power_analysis.py @@ -94,14 +94,19 @@ def apply_power_detector( data: np.ndarray, detector: EnumMeta, dtype: type = None, ignore_nan: bool = False, ) -> np.ndarray: """ - Apply statistical detectors to a 2-D array of samples. + Apply statistical detectors to a 1- or 2-D array of samples. - Statistical detectors are applied along axis 0 (column-wise), - and the sample detector selects a single row from the 2-D - array at random. + For 2-D input data, statistical detectors are applied along + axis 0 (column-wise), and the sample detector selects a single + row at random. - If the input samples are power FFT samples, they are expected - to be packed in the shape (N_FFTs, N_Bins). + For 1-D input data, statistical detectors are applied along the + array, producing a single value output for each selected detector, + and the sample detector selects a single value from the input at + random. + + If the input samples are power FFT samples, stored in a 2-D array, + they are expected to be packed in the shape (N_FFTs, N_Bins). The shape of the output depends on the number of detectors specified. The order of the results always follows min, max, mean, @@ -110,7 +115,7 @@ def apply_power_detector( Create a detector using ``create_power_detector()`` - :param data: A 2-D array of real, linear samples. + :param data: A 1- or 2-D array of real-valued samples in linear units. :param detector: A detector enumeration containing any combination of 'min', 'max', 'mean', 'median', and 'sample'. Also see the create_fft_detector and create_time_domain_detector documentation. @@ -120,12 +125,12 @@ def apply_power_detector( :param ignore_nan: If true, statistical detectors (min/max/mean/median) will ignore any NaN values. NaN values may still appear in the random sample detector result. - :return: A 2-D array containing the selected detector results - as the specified dtype. The number of rows is equal to the - number of detectors applied, and the number of columns is equal - to the number of columns in the input array. + :return: A 1- or 2-D array containing the selected detector results + as the specified dtype. For 1-D inputs, the 1-D output length is + equal to the number of detectors applied. For 2-D inputs, the number + of rows is equal to the number of detectors applied, and the number + of columns is equal to the number of columns in the input array. """ - # Currently this is identical to apply_fft_detector: make general? # Get detector names from detector enumeration detectors = [d.name for _, d in enumerate(detector)] From 59a5cda3f769ee6dfbfeea6fad598a17c3530adc Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 13:25:27 -0600 Subject: [PATCH 058/157] Add dBm value to debug --- scos_actions/signal_processing/calibration.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py index dec764f4..77b7a97a 100644 --- a/scos_actions/signal_processing/calibration.py +++ b/scos_actions/signal_processing/calibration.py @@ -8,6 +8,7 @@ convert_dB_to_linear, convert_fahrenheit_to_celsius, convert_linear_to_dB, + convert_watts_to_dBm ) logger = logging.getLogger(__name__) @@ -46,10 +47,15 @@ def y_factor( :return: A tuple (noise_figure, gain) containing the calculated noise figure and gain, both in dB, from the Y-factor method. """ - logger.debug(f"ENR: {convert_linear_to_dB(enr_linear)} dB") - logger.debug(f"ENBW: {enbw_hz} Hz") - logger.debug(f"Mean power on: {np.mean(pwr_noise_on_watts)} W") - logger.debug(f"Mean power off: {np.mean(pwr_noise_off_watts)} W") + if logger.isEnabledFor(logging.DEBUG): + mean_on_watts = np.mean(pwr_noise_on_watts) + mean_off_watts = np.mean(pwr_noise_off_watts) + mean_on_dBm = convert_watts_to_dBm(mean_on_watts) + mean_off_dBm = convert_watts_to_dBm(mean_off_watts) + logger.debug(f"ENR: {convert_linear_to_dB(enr_linear)} dB") + logger.debug(f"ENBW: {enbw_hz} Hz") + logger.debug(f"Mean power on: {mean_on_watts:.2f} W = {mean_on_dBm:.2f} dBm") + logger.debug(f"Mean power off: {mean_off_watts:.2f} W = {mean_off_dBm:.2f} dBm") y = pwr_noise_on_watts / pwr_noise_off_watts noise_factor = enr_linear / (y - 1.0) gain_watts = pwr_noise_on_watts / ( From e9185f4e782c77f6684be033fcebb7219d0bf59b Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 13:29:05 -0600 Subject: [PATCH 059/157] Test time-domain y-factor --- scos_actions/actions/calibrate_y_factor.py | 78 ++++++++++++++++++++-- 1 file changed, 74 insertions(+), 4 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index c7e6d5b9..46872dee 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -137,6 +137,7 @@ def __init__(self, parameters, sigan, gps=mock_gps): logger.debug('Initializing calibration action') super().__init__(parameters, sigan, gps) # Specify calibration source and temperature sensor indices + # TODO: Should these be part of the action config? self.cal_source_idx = 0 self.temp_sensor_idx = 1 # FFT setup @@ -154,13 +155,84 @@ def __call__(self, schedule_entry_json, task_id): # Calibrate for i, p in enumerate(iteration_params): if i == 0: - detail += self.calibrate(p) + detail += 'OLD ' + self.calibrate_old(p) + detail += 'NEW ' + self.calibrate(p) else: - detail += os.linesep + self.calibrate(p) + detail += os.linesep + 'OLD ' + self.calibrate_old(p) + detail += os.linesep + 'NEW ' + self.calibrate(p) return detail def calibrate(self, params): + # New implementation, time domain calculation + logger.debug('Setting noise diode on') + super().configure_preselector(NOISE_DIODE_ON) + time.sleep(0.25) + + # Configure signal analyzer + super().configure_sigan(params) + logger.debug('Preamp = ' + str(self.sigan.preamp_enable)) + logger.debug('Ref_level: ' + str(self.sigan.reference_level)) + logger.debug('Attenuation:' + str(self.sigan.attenuation)) + + # Use num_samples as defined by FFT parameters + # TODO: Change action config to remove FFT parameters + fft_size = get_parameter(FFT_SIZE, params) + nffts = get_parameter(NUM_FFTS, params) + nskip = get_parameter(NUM_SKIP, params) + num_samples = fft_size * nffts + + # Acquire data with noise diode on + logger.debug('Acquiring IQ samples with noise diode ON') + noise_on_result = self.sigan.acquire_time_domain_samples( + num_samples, num_samples_skip=nskip, gain_adjust=False + ) + sample_rate = noise_on_result["sample_rate"] + + # Set noise diode off + logger.debug('Setting noise diode off') + self.configure_preselector(NOISE_DIODE_OFF) + time.sleep(0.25) + + # Acquire data with noise diode off + logger.debug('Acquiring IQ samples with noise diode OFF') + noise_off_result = self.sigan.acquire_time_domain_samples( + num_samples, num_samples_skip=nskip, gain_adjust=False + ) + assert noise_off_result["sample_rate"] == sample_rate, "Sample rate mismatch for noise diode on/off measurements." + + # Apply IIR filtering to both captures + # TODO + + # Get power values from each capture + power_on_watts = calculate_power_watts(noise_on_result["data"]) / 2. # Divide by 2 for RF/baseband conversion + power_off_watts = calculate_power_watts(noise_off_result["data"]) / 2. + + # Y-Factor + enbw_hz = sample_rate # TODO: Get actual ENBW value + enr_linear = get_linear_enr(self.cal_source_idx) + temp_k, temp_c, _ = get_temperature(self.temp_sensor_idx) + noise_figure, gain = y_factor( + power_on_watts, power_off_watts, enr_linear, enbw_hz, temp_k + ) + sensor_calibration.update( + params, + utils.get_datetime_str_now(), + gain, + noise_figure, + temp_c, + SENSOR_CALIBRATION_FILE, + ) + + # Debugging + noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) + logger.debug(f'Noise floor: {noise_floor_dBm:.3f} dBm') + logger.debug(f'Noise Figure: {noise_figure:.3f} dB') + logger.debug(f'Gain: {gain:.3f} dB') + + return 'Noise Figure: {}, Gain: {}'.format(noise_figure, gain) + + def calibrate_old(self, params): # Set noise diode on logger.debug('Setting noise diode on') super().configure_preselector(NOISE_DIODE_ON) @@ -285,5 +357,3 @@ def test_required_components(self): raise RuntimeError(msg) - - From 54c8fd801eb8f682a2804d821d050a8932b16c22 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 14:31:19 -0600 Subject: [PATCH 060/157] Revert "Test time-domain y-factor" This reverts commit e9185f4e782c77f6684be033fcebb7219d0bf59b. --- scos_actions/actions/calibrate_y_factor.py | 78 ++-------------------- 1 file changed, 4 insertions(+), 74 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 46872dee..c7e6d5b9 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -137,7 +137,6 @@ def __init__(self, parameters, sigan, gps=mock_gps): logger.debug('Initializing calibration action') super().__init__(parameters, sigan, gps) # Specify calibration source and temperature sensor indices - # TODO: Should these be part of the action config? self.cal_source_idx = 0 self.temp_sensor_idx = 1 # FFT setup @@ -155,84 +154,13 @@ def __call__(self, schedule_entry_json, task_id): # Calibrate for i, p in enumerate(iteration_params): if i == 0: - detail += 'OLD ' + self.calibrate_old(p) - detail += 'NEW ' + self.calibrate(p) + detail += self.calibrate(p) else: - detail += os.linesep + 'OLD ' + self.calibrate_old(p) - detail += os.linesep + 'NEW ' + self.calibrate(p) + detail += os.linesep + self.calibrate(p) return detail def calibrate(self, params): - # New implementation, time domain calculation - logger.debug('Setting noise diode on') - super().configure_preselector(NOISE_DIODE_ON) - time.sleep(0.25) - - # Configure signal analyzer - super().configure_sigan(params) - logger.debug('Preamp = ' + str(self.sigan.preamp_enable)) - logger.debug('Ref_level: ' + str(self.sigan.reference_level)) - logger.debug('Attenuation:' + str(self.sigan.attenuation)) - - # Use num_samples as defined by FFT parameters - # TODO: Change action config to remove FFT parameters - fft_size = get_parameter(FFT_SIZE, params) - nffts = get_parameter(NUM_FFTS, params) - nskip = get_parameter(NUM_SKIP, params) - num_samples = fft_size * nffts - - # Acquire data with noise diode on - logger.debug('Acquiring IQ samples with noise diode ON') - noise_on_result = self.sigan.acquire_time_domain_samples( - num_samples, num_samples_skip=nskip, gain_adjust=False - ) - sample_rate = noise_on_result["sample_rate"] - - # Set noise diode off - logger.debug('Setting noise diode off') - self.configure_preselector(NOISE_DIODE_OFF) - time.sleep(0.25) - - # Acquire data with noise diode off - logger.debug('Acquiring IQ samples with noise diode OFF') - noise_off_result = self.sigan.acquire_time_domain_samples( - num_samples, num_samples_skip=nskip, gain_adjust=False - ) - assert noise_off_result["sample_rate"] == sample_rate, "Sample rate mismatch for noise diode on/off measurements." - - # Apply IIR filtering to both captures - # TODO - - # Get power values from each capture - power_on_watts = calculate_power_watts(noise_on_result["data"]) / 2. # Divide by 2 for RF/baseband conversion - power_off_watts = calculate_power_watts(noise_off_result["data"]) / 2. - - # Y-Factor - enbw_hz = sample_rate # TODO: Get actual ENBW value - enr_linear = get_linear_enr(self.cal_source_idx) - temp_k, temp_c, _ = get_temperature(self.temp_sensor_idx) - noise_figure, gain = y_factor( - power_on_watts, power_off_watts, enr_linear, enbw_hz, temp_k - ) - sensor_calibration.update( - params, - utils.get_datetime_str_now(), - gain, - noise_figure, - temp_c, - SENSOR_CALIBRATION_FILE, - ) - - # Debugging - noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) - logger.debug(f'Noise floor: {noise_floor_dBm:.3f} dBm') - logger.debug(f'Noise Figure: {noise_figure:.3f} dB') - logger.debug(f'Gain: {gain:.3f} dB') - - return 'Noise Figure: {}, Gain: {}'.format(noise_figure, gain) - - def calibrate_old(self, params): # Set noise diode on logger.debug('Setting noise diode on') super().configure_preselector(NOISE_DIODE_ON) @@ -357,3 +285,5 @@ def test_required_components(self): raise RuntimeError(msg) + + From b900c03abda410a431ab5cd05714f42798dd8981 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 14:39:47 -0600 Subject: [PATCH 061/157] Revert "Revert "Test time-domain y-factor"" This reverts commit 54c8fd801eb8f682a2804d821d050a8932b16c22. --- scos_actions/actions/calibrate_y_factor.py | 78 ++++++++++++++++++++-- 1 file changed, 74 insertions(+), 4 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index c7e6d5b9..46872dee 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -137,6 +137,7 @@ def __init__(self, parameters, sigan, gps=mock_gps): logger.debug('Initializing calibration action') super().__init__(parameters, sigan, gps) # Specify calibration source and temperature sensor indices + # TODO: Should these be part of the action config? self.cal_source_idx = 0 self.temp_sensor_idx = 1 # FFT setup @@ -154,13 +155,84 @@ def __call__(self, schedule_entry_json, task_id): # Calibrate for i, p in enumerate(iteration_params): if i == 0: - detail += self.calibrate(p) + detail += 'OLD ' + self.calibrate_old(p) + detail += 'NEW ' + self.calibrate(p) else: - detail += os.linesep + self.calibrate(p) + detail += os.linesep + 'OLD ' + self.calibrate_old(p) + detail += os.linesep + 'NEW ' + self.calibrate(p) return detail def calibrate(self, params): + # New implementation, time domain calculation + logger.debug('Setting noise diode on') + super().configure_preselector(NOISE_DIODE_ON) + time.sleep(0.25) + + # Configure signal analyzer + super().configure_sigan(params) + logger.debug('Preamp = ' + str(self.sigan.preamp_enable)) + logger.debug('Ref_level: ' + str(self.sigan.reference_level)) + logger.debug('Attenuation:' + str(self.sigan.attenuation)) + + # Use num_samples as defined by FFT parameters + # TODO: Change action config to remove FFT parameters + fft_size = get_parameter(FFT_SIZE, params) + nffts = get_parameter(NUM_FFTS, params) + nskip = get_parameter(NUM_SKIP, params) + num_samples = fft_size * nffts + + # Acquire data with noise diode on + logger.debug('Acquiring IQ samples with noise diode ON') + noise_on_result = self.sigan.acquire_time_domain_samples( + num_samples, num_samples_skip=nskip, gain_adjust=False + ) + sample_rate = noise_on_result["sample_rate"] + + # Set noise diode off + logger.debug('Setting noise diode off') + self.configure_preselector(NOISE_DIODE_OFF) + time.sleep(0.25) + + # Acquire data with noise diode off + logger.debug('Acquiring IQ samples with noise diode OFF') + noise_off_result = self.sigan.acquire_time_domain_samples( + num_samples, num_samples_skip=nskip, gain_adjust=False + ) + assert noise_off_result["sample_rate"] == sample_rate, "Sample rate mismatch for noise diode on/off measurements." + + # Apply IIR filtering to both captures + # TODO + + # Get power values from each capture + power_on_watts = calculate_power_watts(noise_on_result["data"]) / 2. # Divide by 2 for RF/baseband conversion + power_off_watts = calculate_power_watts(noise_off_result["data"]) / 2. + + # Y-Factor + enbw_hz = sample_rate # TODO: Get actual ENBW value + enr_linear = get_linear_enr(self.cal_source_idx) + temp_k, temp_c, _ = get_temperature(self.temp_sensor_idx) + noise_figure, gain = y_factor( + power_on_watts, power_off_watts, enr_linear, enbw_hz, temp_k + ) + sensor_calibration.update( + params, + utils.get_datetime_str_now(), + gain, + noise_figure, + temp_c, + SENSOR_CALIBRATION_FILE, + ) + + # Debugging + noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) + logger.debug(f'Noise floor: {noise_floor_dBm:.3f} dBm') + logger.debug(f'Noise Figure: {noise_figure:.3f} dB') + logger.debug(f'Gain: {gain:.3f} dB') + + return 'Noise Figure: {}, Gain: {}'.format(noise_figure, gain) + + def calibrate_old(self, params): # Set noise diode on logger.debug('Setting noise diode on') super().configure_preselector(NOISE_DIODE_ON) @@ -285,5 +357,3 @@ def test_required_components(self): raise RuntimeError(msg) - - From 8550728b53c19ad268128805ab886c914b77bf5a Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 14:57:47 -0600 Subject: [PATCH 062/157] Revert "Revert "Revert "Test time-domain y-factor""" This reverts commit b900c03abda410a431ab5cd05714f42798dd8981. --- scos_actions/actions/calibrate_y_factor.py | 78 ++-------------------- 1 file changed, 4 insertions(+), 74 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 46872dee..c7e6d5b9 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -137,7 +137,6 @@ def __init__(self, parameters, sigan, gps=mock_gps): logger.debug('Initializing calibration action') super().__init__(parameters, sigan, gps) # Specify calibration source and temperature sensor indices - # TODO: Should these be part of the action config? self.cal_source_idx = 0 self.temp_sensor_idx = 1 # FFT setup @@ -155,84 +154,13 @@ def __call__(self, schedule_entry_json, task_id): # Calibrate for i, p in enumerate(iteration_params): if i == 0: - detail += 'OLD ' + self.calibrate_old(p) - detail += 'NEW ' + self.calibrate(p) + detail += self.calibrate(p) else: - detail += os.linesep + 'OLD ' + self.calibrate_old(p) - detail += os.linesep + 'NEW ' + self.calibrate(p) + detail += os.linesep + self.calibrate(p) return detail def calibrate(self, params): - # New implementation, time domain calculation - logger.debug('Setting noise diode on') - super().configure_preselector(NOISE_DIODE_ON) - time.sleep(0.25) - - # Configure signal analyzer - super().configure_sigan(params) - logger.debug('Preamp = ' + str(self.sigan.preamp_enable)) - logger.debug('Ref_level: ' + str(self.sigan.reference_level)) - logger.debug('Attenuation:' + str(self.sigan.attenuation)) - - # Use num_samples as defined by FFT parameters - # TODO: Change action config to remove FFT parameters - fft_size = get_parameter(FFT_SIZE, params) - nffts = get_parameter(NUM_FFTS, params) - nskip = get_parameter(NUM_SKIP, params) - num_samples = fft_size * nffts - - # Acquire data with noise diode on - logger.debug('Acquiring IQ samples with noise diode ON') - noise_on_result = self.sigan.acquire_time_domain_samples( - num_samples, num_samples_skip=nskip, gain_adjust=False - ) - sample_rate = noise_on_result["sample_rate"] - - # Set noise diode off - logger.debug('Setting noise diode off') - self.configure_preselector(NOISE_DIODE_OFF) - time.sleep(0.25) - - # Acquire data with noise diode off - logger.debug('Acquiring IQ samples with noise diode OFF') - noise_off_result = self.sigan.acquire_time_domain_samples( - num_samples, num_samples_skip=nskip, gain_adjust=False - ) - assert noise_off_result["sample_rate"] == sample_rate, "Sample rate mismatch for noise diode on/off measurements." - - # Apply IIR filtering to both captures - # TODO - - # Get power values from each capture - power_on_watts = calculate_power_watts(noise_on_result["data"]) / 2. # Divide by 2 for RF/baseband conversion - power_off_watts = calculate_power_watts(noise_off_result["data"]) / 2. - - # Y-Factor - enbw_hz = sample_rate # TODO: Get actual ENBW value - enr_linear = get_linear_enr(self.cal_source_idx) - temp_k, temp_c, _ = get_temperature(self.temp_sensor_idx) - noise_figure, gain = y_factor( - power_on_watts, power_off_watts, enr_linear, enbw_hz, temp_k - ) - sensor_calibration.update( - params, - utils.get_datetime_str_now(), - gain, - noise_figure, - temp_c, - SENSOR_CALIBRATION_FILE, - ) - - # Debugging - noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) - logger.debug(f'Noise floor: {noise_floor_dBm:.3f} dBm') - logger.debug(f'Noise Figure: {noise_figure:.3f} dB') - logger.debug(f'Gain: {gain:.3f} dB') - - return 'Noise Figure: {}, Gain: {}'.format(noise_figure, gain) - - def calibrate_old(self, params): # Set noise diode on logger.debug('Setting noise diode on') super().configure_preselector(NOISE_DIODE_ON) @@ -357,3 +285,5 @@ def test_required_components(self): raise RuntimeError(msg) + + From 3a6c88006cc8e95dce607790b2e479787b861d63 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 15:13:53 -0600 Subject: [PATCH 063/157] Revert "Revert "Revert "Revert "Test time-domain y-factor"""" This reverts commit 8550728b53c19ad268128805ab886c914b77bf5a. --- scos_actions/actions/calibrate_y_factor.py | 78 ++++++++++++++++++++-- 1 file changed, 74 insertions(+), 4 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index c7e6d5b9..46872dee 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -137,6 +137,7 @@ def __init__(self, parameters, sigan, gps=mock_gps): logger.debug('Initializing calibration action') super().__init__(parameters, sigan, gps) # Specify calibration source and temperature sensor indices + # TODO: Should these be part of the action config? self.cal_source_idx = 0 self.temp_sensor_idx = 1 # FFT setup @@ -154,13 +155,84 @@ def __call__(self, schedule_entry_json, task_id): # Calibrate for i, p in enumerate(iteration_params): if i == 0: - detail += self.calibrate(p) + detail += 'OLD ' + self.calibrate_old(p) + detail += 'NEW ' + self.calibrate(p) else: - detail += os.linesep + self.calibrate(p) + detail += os.linesep + 'OLD ' + self.calibrate_old(p) + detail += os.linesep + 'NEW ' + self.calibrate(p) return detail def calibrate(self, params): + # New implementation, time domain calculation + logger.debug('Setting noise diode on') + super().configure_preselector(NOISE_DIODE_ON) + time.sleep(0.25) + + # Configure signal analyzer + super().configure_sigan(params) + logger.debug('Preamp = ' + str(self.sigan.preamp_enable)) + logger.debug('Ref_level: ' + str(self.sigan.reference_level)) + logger.debug('Attenuation:' + str(self.sigan.attenuation)) + + # Use num_samples as defined by FFT parameters + # TODO: Change action config to remove FFT parameters + fft_size = get_parameter(FFT_SIZE, params) + nffts = get_parameter(NUM_FFTS, params) + nskip = get_parameter(NUM_SKIP, params) + num_samples = fft_size * nffts + + # Acquire data with noise diode on + logger.debug('Acquiring IQ samples with noise diode ON') + noise_on_result = self.sigan.acquire_time_domain_samples( + num_samples, num_samples_skip=nskip, gain_adjust=False + ) + sample_rate = noise_on_result["sample_rate"] + + # Set noise diode off + logger.debug('Setting noise diode off') + self.configure_preselector(NOISE_DIODE_OFF) + time.sleep(0.25) + + # Acquire data with noise diode off + logger.debug('Acquiring IQ samples with noise diode OFF') + noise_off_result = self.sigan.acquire_time_domain_samples( + num_samples, num_samples_skip=nskip, gain_adjust=False + ) + assert noise_off_result["sample_rate"] == sample_rate, "Sample rate mismatch for noise diode on/off measurements." + + # Apply IIR filtering to both captures + # TODO + + # Get power values from each capture + power_on_watts = calculate_power_watts(noise_on_result["data"]) / 2. # Divide by 2 for RF/baseband conversion + power_off_watts = calculate_power_watts(noise_off_result["data"]) / 2. + + # Y-Factor + enbw_hz = sample_rate # TODO: Get actual ENBW value + enr_linear = get_linear_enr(self.cal_source_idx) + temp_k, temp_c, _ = get_temperature(self.temp_sensor_idx) + noise_figure, gain = y_factor( + power_on_watts, power_off_watts, enr_linear, enbw_hz, temp_k + ) + sensor_calibration.update( + params, + utils.get_datetime_str_now(), + gain, + noise_figure, + temp_c, + SENSOR_CALIBRATION_FILE, + ) + + # Debugging + noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) + logger.debug(f'Noise floor: {noise_floor_dBm:.3f} dBm') + logger.debug(f'Noise Figure: {noise_figure:.3f} dB') + logger.debug(f'Gain: {gain:.3f} dB') + + return 'Noise Figure: {}, Gain: {}'.format(noise_figure, gain) + + def calibrate_old(self, params): # Set noise diode on logger.debug('Setting noise diode on') super().configure_preselector(NOISE_DIODE_ON) @@ -285,5 +357,3 @@ def test_required_components(self): raise RuntimeError(msg) - - From 677485297875bfc12596ebe878daff23ae4c6766 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 15:23:35 -0600 Subject: [PATCH 064/157] Add IIR filter --- scos_actions/actions/calibrate_y_factor.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 46872dee..09403f1e 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -71,6 +71,7 @@ from numpy import ndarray from scipy.constants import Boltzmann +from scipy.signal import sosfilt from scos_actions import utils from scos_actions.hardware import gps as mock_gps @@ -92,6 +93,7 @@ ) from scos_actions.signal_processing.unit_conversion import convert_watts_to_dBm +from scos_actions.signal_processing.filtering import generate_elliptic_iir_low_pass_filter import os @@ -143,6 +145,9 @@ def __init__(self, parameters, sigan, gps=mock_gps): # FFT setup self.fft_detector = create_power_detector("MeanDetector", ["mean"]) self.fft_window_type = "flattop" + # IIR Filter + # Hard-coded parameters for now + self.iir_sos = generate_elliptic_iir_low_pass_filter(0.1, 40, 5e6, 8e3, 14e6) def __call__(self, schedule_entry_json, task_id): """This is the entrypoint function called by the scheduler.""" @@ -159,7 +164,7 @@ def __call__(self, schedule_entry_json, task_id): detail += 'NEW ' + self.calibrate(p) else: detail += os.linesep + 'OLD ' + self.calibrate_old(p) - detail += os.linesep + 'NEW ' + self.calibrate(p) + detail += ' NEW ' + self.calibrate(p) return detail @@ -203,10 +208,13 @@ def calibrate(self, params): # Apply IIR filtering to both captures # TODO + logger.debug("Applying IIR filter to IQ captures") + noise_on_filtered = sosfilt(self.iir_sos, noise_on_result["data"]) + noise_off_filtered = sosfilt(self.iir_sos, noise_off_result["data"]) # Get power values from each capture - power_on_watts = calculate_power_watts(noise_on_result["data"]) / 2. # Divide by 2 for RF/baseband conversion - power_off_watts = calculate_power_watts(noise_off_result["data"]) / 2. + power_on_watts = calculate_power_watts(noise_on_filtered) / 2. # Divide by 2 for RF/baseband conversion + power_off_watts = calculate_power_watts(noise_off_filtered) / 2. # Y-Factor enbw_hz = sample_rate # TODO: Get actual ENBW value From eb19d9ca84240f57ef9262f8f2605e294a3b7f62 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 15:32:13 -0600 Subject: [PATCH 065/157] Consolidate y-factor methods for comparison --- scos_actions/actions/calibrate_y_factor.py | 157 +++++++-------------- 1 file changed, 52 insertions(+), 105 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 09403f1e..cf895631 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -160,160 +160,107 @@ def __call__(self, schedule_entry_json, task_id): # Calibrate for i, p in enumerate(iteration_params): if i == 0: - detail += 'OLD ' + self.calibrate_old(p) - detail += 'NEW ' + self.calibrate(p) + detail += self.calibrate(p) else: - detail += os.linesep + 'OLD ' + self.calibrate_old(p) - detail += ' NEW ' + self.calibrate(p) + detail += os.linesep + self.calibrate(p) return detail - def calibrate(self, params): - # New implementation, time domain calculation - logger.debug('Setting noise diode on') - super().configure_preselector(NOISE_DIODE_ON) - time.sleep(0.25) + def calibrate(self, params): # Configure signal analyzer super().configure_sigan(params) logger.debug('Preamp = ' + str(self.sigan.preamp_enable)) logger.debug('Ref_level: ' + str(self.sigan.reference_level)) logger.debug('Attenuation:' + str(self.sigan.attenuation)) - # Use num_samples as defined by FFT parameters - # TODO: Change action config to remove FFT parameters + # Get parameters from action config fft_size = get_parameter(FFT_SIZE, params) nffts = get_parameter(NUM_FFTS, params) nskip = get_parameter(NUM_SKIP, params) + fft_window = get_fft_window(self.fft_window_type, fft_size) + fft_acf = get_fft_window_correction(fft_window, 'amplitude') num_samples = fft_size * nffts - - # Acquire data with noise diode on - logger.debug('Acquiring IQ samples with noise diode ON') - noise_on_result = self.sigan.acquire_time_domain_samples( - num_samples, num_samples_skip=nskip, gain_adjust=False - ) - sample_rate = noise_on_result["sample_rate"] - # Set noise diode off - logger.debug('Setting noise diode off') - self.configure_preselector(NOISE_DIODE_OFF) - time.sleep(0.25) - - # Acquire data with noise diode off - logger.debug('Acquiring IQ samples with noise diode OFF') - noise_off_result = self.sigan.acquire_time_domain_samples( - num_samples, num_samples_skip=nskip, gain_adjust=False - ) - assert noise_off_result["sample_rate"] == sample_rate, "Sample rate mismatch for noise diode on/off measurements." - - # Apply IIR filtering to both captures - # TODO - logger.debug("Applying IIR filter to IQ captures") - noise_on_filtered = sosfilt(self.iir_sos, noise_on_result["data"]) - noise_off_filtered = sosfilt(self.iir_sos, noise_off_result["data"]) - - # Get power values from each capture - power_on_watts = calculate_power_watts(noise_on_filtered) / 2. # Divide by 2 for RF/baseband conversion - power_off_watts = calculate_power_watts(noise_off_filtered) / 2. - - # Y-Factor - enbw_hz = sample_rate # TODO: Get actual ENBW value - enr_linear = get_linear_enr(self.cal_source_idx) - temp_k, temp_c, _ = get_temperature(self.temp_sensor_idx) - noise_figure, gain = y_factor( - power_on_watts, power_off_watts, enr_linear, enbw_hz, temp_k - ) - sensor_calibration.update( - params, - utils.get_datetime_str_now(), - gain, - noise_figure, - temp_c, - SENSOR_CALIBRATION_FILE, - ) - - # Debugging - noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) - logger.debug(f'Noise floor: {noise_floor_dBm:.3f} dBm') - logger.debug(f'Noise Figure: {noise_figure:.3f} dB') - logger.debug(f'Gain: {gain:.3f} dB') - - return 'Noise Figure: {}, Gain: {}'.format(noise_figure, gain) - - def calibrate_old(self, params): # Set noise diode on logger.debug('Setting noise diode on') super().configure_preselector(NOISE_DIODE_ON) time.sleep(.25) - # Configure signal analyzer - super().configure_sigan(params) - logger.debug('Preamp = ' + str(self.sigan.preamp_enable)) - logger.debug('Ref_level: ' + str(self.sigan.reference_level)) - logger.debug('Attenuation:' + str(self.sigan.attenuation)) - - # Get parameters from action config - fft_size = get_parameter(FFT_SIZE, params) - nffts = get_parameter(NUM_FFTS, params) - nskip = get_parameter(NUM_SKIP, params) - fft_window = get_fft_window(self.fft_window_type, fft_size) - fft_acf = get_fft_window_correction(fft_window, 'amplitude') - num_samples = fft_size * nffts - - logger.debug("Acquiring mean FFT with noise diode ON") - - # Get noise diode on mean FFT result + # Get noise diode on IQ + logger.debug("Acquiring IQ samples with noise diode ON") noise_on_measurement_result = self.sigan.acquire_time_domain_samples( num_samples, num_samples_skip=nskip, gain_adjust=False ) sample_rate = noise_on_measurement_result["sample_rate"] - mean_on_watts = self.apply_mean_fft( - noise_on_measurement_result, fft_size, fft_window, nffts, fft_acf - ) # Set noise diode off logger.debug('Setting noise diode off') self.configure_preselector(NOISE_DIODE_OFF) time.sleep(.25) - # Get noise diode off mean FFT result - logger.debug('Acquiring mean FFT with noise diode OFF') + # Get noise diode off IQ + logger.debug('Acquiring IQ samples with noise diode OFF') noise_off_measurement_result = self.sigan.acquire_time_domain_samples( num_samples, num_samples_skip=nskip, gain_adjust=False ) - mean_off_watts = self.apply_mean_fft( - noise_off_measurement_result, fft_size, fft_window, nffts, fft_acf + + + # Apply IIR filtering to both captures + logger.debug("Applying IIR filter to IQ captures") + noise_on_filtered = sosfilt(self.iir_sos, noise_on_measurement_result["data"]) + noise_off_filtered = sosfilt(self.iir_sos, noise_off_measurement_result["data"]) + + # Get power values in time domain + td_on_watts = calculate_power_watts(noise_on_filtered) / 2. # Divide by 2 for RF/baseband conversion + td_off_watts = calculate_power_watts(noise_off_filtered) / 2. + + # Get mean power FFT results + fft_on_watts = self.apply_mean_fft( + noise_on_filtered, fft_size, fft_window, nffts, fft_acf + ) + fft_off_watts = self.apply_mean_fft( + noise_off_filtered, fft_size, fft_window, nffts, fft_acf ) # Y-Factor enbw_hz = get_fft_enbw(fft_window, sample_rate) + enbw_hz_td = sample_rate # TODO Get actual ENBW enr_linear = get_linear_enr(self.cal_source_idx) temp_k, temp_c, _ = get_temperature(self.temp_sensor_idx) - noise_figure, gain = y_factor( - mean_on_watts, mean_off_watts, enr_linear, enbw_hz, temp_k + td_noise_figure, td_gain = y_factor( + td_on_watts, td_off_watts, enr_linear, sample_rate, temp_k ) - sensor_calibration.update( - params, - utils.get_datetime_str_now(), - gain, - noise_figure, - temp_c, - SENSOR_CALIBRATION_FILE, + fft_noise_figure, fft_gain = y_factor( + fft_on_watts, fft_off_watts, enr_linear, enbw_hz, temp_k ) + # Don't update the sensor calibration while testing + # sensor_calibration.update( + # params, + # utils.get_datetime_str_now(), + # gain, + # noise_figure, + # temp_c, + # SENSOR_CALIBRATION_FILE, + # ) + # Debugging noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) - logger.debug(f'Noise floor: {noise_floor_dBm} dBm') - logger.debug(f'Noise Figure: {noise_figure} dB') - logger.debug(f'Gain: {gain} dB') - - return 'Noise Figure: {}, Gain: {}'.format(noise_figure, gain) + logger.debug(f'Noise floor: {noise_floor_dBm:.2f} dBm') + logger.debug(f'Noise Figure (FFT): {fft_noise_figure:.2f} dB') + logger.debug(f'Gain (FFT): {fft_gain:.2f} dB') + logger.debug(f'Noise figure (TD): {td_noise_figure:.2f} dB') + logger.debug(f"Gain (TD): {td_gain:.2f} dB") + + # Detail results contain only FFT version of result for now + return 'Noise Figure: {}, Gain: {}'.format(fft_noise_figure, fft_gain) def apply_mean_fft( - self, measurement_result: dict, fft_size: int, fft_window: ndarray, nffts: int, fft_window_cf: float + self, iqdata: ndarray, fft_size: int, fft_window: ndarray, nffts: int, fft_window_cf: float ) -> ndarray: complex_fft = get_fft( - time_data=measurement_result["data"], + time_data=iqdata, fft_size=fft_size, norm="forward", fft_window=fft_window, From f59168e0dc98d27df73d5249e9c5bf56292bab88 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 15:40:22 -0600 Subject: [PATCH 066/157] Update placeholder ENBW --- scos_actions/actions/calibrate_y_factor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index cf895631..639ebe85 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -225,7 +225,7 @@ def calibrate(self, params): # Y-Factor enbw_hz = get_fft_enbw(fft_window, sample_rate) - enbw_hz_td = sample_rate # TODO Get actual ENBW + enbw_hz_td = 10e6 # TODO Get actual ENBW enr_linear = get_linear_enr(self.cal_source_idx) temp_k, temp_c, _ = get_temperature(self.temp_sensor_idx) td_noise_figure, td_gain = y_factor( From d790872fd6c46bb0068e0b5ef62f83a031095f52 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 15:53:20 -0600 Subject: [PATCH 067/157] Added actual sensor ENBW (hard-coded temporarily) --- scos_actions/actions/calibrate_y_factor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 639ebe85..c51487dc 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -225,7 +225,7 @@ def calibrate(self, params): # Y-Factor enbw_hz = get_fft_enbw(fft_window, sample_rate) - enbw_hz_td = 10e6 # TODO Get actual ENBW + enbw_hz_td = 11.607e6 # TODO Parameterize this enr_linear = get_linear_enr(self.cal_source_idx) temp_k, temp_c, _ = get_temperature(self.temp_sensor_idx) td_noise_figure, td_gain = y_factor( @@ -268,9 +268,9 @@ def apply_mean_fft( shift=True ) # TESTING SCALING + power_fft /= 2 # RF/baseband conversion complex_fft *= fft_window_cf # Window correction power_fft = calculate_power_watts(complex_fft) - power_fft /= 2 # RF/baseband conversion mean_result = apply_power_detector(power_fft, self.fft_detector) return mean_result From 994be5682e436aff37a901d01b0e221b271e7d9f Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 15:53:27 -0600 Subject: [PATCH 068/157] Added debug messages --- scos_actions/signal_processing/fft.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scos_actions/signal_processing/fft.py b/scos_actions/signal_processing/fft.py index 53e2f92c..8b464885 100644 --- a/scos_actions/signal_processing/fft.py +++ b/scos_actions/signal_processing/fft.py @@ -65,7 +65,9 @@ def get_fft( """ # Get num_ffts for default case: as many as possible if num_ffts <= 0: + logger.info("Number of FFTs not specified. Using as many as possible.") num_ffts = int(len(time_data) // fft_size) + logger.info(f"Number of FFTs set to {num_ffts} based on specified FFT size {fft_size}") # Determine if truncation will occur and raise a warning if so if len(time_data) != fft_size * num_ffts: @@ -80,6 +82,7 @@ def get_fft( # Apply the FFT window if provided if fft_window is not None: + logger.debug("Applying window before FFT") time_data = ne.evaluate("time_data*fft_window") # Take the FFT From fe02862302439c3695b6d19953df6d652087136f Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 15:58:37 -0600 Subject: [PATCH 069/157] Fix incorrect variable name --- scos_actions/actions/calibrate_y_factor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index c51487dc..ed62a1ab 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -268,7 +268,7 @@ def apply_mean_fft( shift=True ) # TESTING SCALING - power_fft /= 2 # RF/baseband conversion + complex_fft /= 2 # RF/baseband conversion complex_fft *= fft_window_cf # Window correction power_fft = calculate_power_watts(complex_fft) mean_result = apply_power_detector(power_fft, self.fft_detector) From 1ae322de6bb222793250a5bdf4350b7279a9dc47 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 28 Jul 2022 15:59:57 -0600 Subject: [PATCH 070/157] Use copies of IQ arrays for testing --- scos_actions/actions/calibrate_y_factor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index ed62a1ab..f169225a 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -212,15 +212,15 @@ def calibrate(self, params): noise_off_filtered = sosfilt(self.iir_sos, noise_off_measurement_result["data"]) # Get power values in time domain - td_on_watts = calculate_power_watts(noise_on_filtered) / 2. # Divide by 2 for RF/baseband conversion - td_off_watts = calculate_power_watts(noise_off_filtered) / 2. + td_on_watts = calculate_power_watts(noise_on_filtered.copy()) / 2. # Divide by 2 for RF/baseband conversion + td_off_watts = calculate_power_watts(noise_off_filtered.copy()) / 2. # Get mean power FFT results fft_on_watts = self.apply_mean_fft( - noise_on_filtered, fft_size, fft_window, nffts, fft_acf + noise_on_filtered.copy(), fft_size, fft_window, nffts, fft_acf ) fft_off_watts = self.apply_mean_fft( - noise_off_filtered, fft_size, fft_window, nffts, fft_acf + noise_off_filtered.copy(), fft_size, fft_window, nffts, fft_acf ) # Y-Factor From f5f4e490657884690ce83b2b668142149d15ab54 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 29 Jul 2022 12:23:42 -0600 Subject: [PATCH 071/157] Initial IIR filter parameterization --- scos_actions/actions/calibrate_y_factor.py | 45 +++++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index f169225a..220a7cb0 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -78,7 +78,7 @@ from scos_actions.settings import sensor_calibration from scos_actions.settings import SENSOR_CALIBRATION_FILE from scos_actions.actions.interfaces.action import Action -from scos_actions.utils import get_parameter +from scos_actions.utils import ParameterException, get_parameter from scos_actions.signal_processing.fft import get_fft, get_fft_enbw, get_fft_window, get_fft_window_correction from scos_actions.signal_processing.calibration import ( @@ -109,6 +109,12 @@ FFT_SIZE = "fft_size" NUM_FFTS = "nffts" NUM_SKIP = "nskip" +IIR_APPLY = 'iir_apply' +IIR_RP = 'iir_rp_dB' +IIR_RS = 'iir_rs_dB' +IIR_CUTOFF = 'iir_cutoff_Hz' +IIR_WIDTH = 'iir_width_Hz' + # TODO: Should calibration source index and temperature sensor number # be required parameters? @@ -139,15 +145,31 @@ def __init__(self, parameters, sigan, gps=mock_gps): logger.debug('Initializing calibration action') super().__init__(parameters, sigan, gps) # Specify calibration source and temperature sensor indices - # TODO: Should these be part of the action config? + # TODO: Add cal source and sensor indices to YAML config self.cal_source_idx = 0 self.temp_sensor_idx = 1 # FFT setup + # TODO: Remove these self.fft_detector = create_power_detector("MeanDetector", ["mean"]) self.fft_window_type = "flattop" # IIR Filter - # Hard-coded parameters for now - self.iir_sos = generate_elliptic_iir_low_pass_filter(0.1, 40, 5e6, 8e3, 14e6) + try: + self.iir_apply = get_parameter(IIR_APPLY, parameters) + except ParameterException: + # Do not apply IIR filter if iir_apply is not in config + logger.info("Config parameter 'iir_apply' not provided. " + + "No IIR filtering will be used during calibration.") + self.iir_apply = False + if self.iir_apply: + # Check if any parameters are provided as lists, and if so, save + # filter generation for the calibration loop + self.iir_rp_dB = get_parameter(IIR_RP, parameters) + self.iir_rs_dB = get_parameter(IIR_RS, parameters) + self.iir_cutoff_Hz = get_parameter(IIR_CUTOFF, parameters) + self.iir_width_Hz = get_parameter(IIR_WIDTH, parameters) + # TODO: Get sample rate dynamically for filter design + self.iir_sos = generate_elliptic_iir_low_pass_filter( + self.iir_rp_dB, self.iir_rs_dB, self.iir_cutoff_Hz, self.iir_width_Hz, 14e6) def __call__(self, schedule_entry_json, task_id): """This is the entrypoint function called by the scheduler.""" @@ -170,9 +192,6 @@ def __call__(self, schedule_entry_json, task_id): def calibrate(self, params): # Configure signal analyzer super().configure_sigan(params) - logger.debug('Preamp = ' + str(self.sigan.preamp_enable)) - logger.debug('Ref_level: ' + str(self.sigan.reference_level)) - logger.debug('Attenuation:' + str(self.sigan.attenuation)) # Get parameters from action config fft_size = get_parameter(FFT_SIZE, params) @@ -204,7 +223,7 @@ def calibrate(self, params): noise_off_measurement_result = self.sigan.acquire_time_domain_samples( num_samples, num_samples_skip=nskip, gain_adjust=False ) - + assert sample_rate == noise_off_measurement_result["sample_rate"], "Sample rate mismatch" # Apply IIR filtering to both captures logger.debug("Applying IIR filter to IQ captures") @@ -225,12 +244,18 @@ def calibrate(self, params): # Y-Factor enbw_hz = get_fft_enbw(fft_window, sample_rate) - enbw_hz_td = 11.607e6 # TODO Parameterize this + # TODO Parameterize ENBW + # ENBW should differ based on whether or not IIR filtering is used + # enbw_hz_td = 11.607e6 # For RSA 507A + enbw_hz_td = 10.016e6 # Roughly based on IIR filter enr_linear = get_linear_enr(self.cal_source_idx) temp_k, temp_c, _ = get_temperature(self.temp_sensor_idx) + + # New method td_noise_figure, td_gain = y_factor( - td_on_watts, td_off_watts, enr_linear, sample_rate, temp_k + td_on_watts, td_off_watts, enr_linear, enbw_hz_td, temp_k ) + # Old method for comparison fft_noise_figure, fft_gain = y_factor( fft_on_watts, fft_off_watts, enr_linear, enbw_hz, temp_k ) From 8c1683fce4c2f4da8bd2d5ee1f3d1389d9b70f2a Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 29 Jul 2022 13:36:13 -0600 Subject: [PATCH 072/157] Implement configurable filter regeneration --- scos_actions/actions/calibrate_y_factor.py | 59 ++++++++++++++-------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 220a7cb0..0d40178e 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -152,34 +152,38 @@ def __init__(self, parameters, sigan, gps=mock_gps): # TODO: Remove these self.fft_detector = create_power_detector("MeanDetector", ["mean"]) self.fft_window_type = "flattop" - # IIR Filter + # IIR Filter Setup try: self.iir_apply = get_parameter(IIR_APPLY, parameters) except ParameterException: - # Do not apply IIR filter if iir_apply is not in config logger.info("Config parameter 'iir_apply' not provided. " + "No IIR filtering will be used during calibration.") self.iir_apply = False - if self.iir_apply: - # Check if any parameters are provided as lists, and if so, save - # filter generation for the calibration loop + + if self.iir_apply is True: + # If any parameters are multiply-specified, generate the filter + # in the calibration loop instead of here. self.iir_rp_dB = get_parameter(IIR_RP, parameters) self.iir_rs_dB = get_parameter(IIR_RS, parameters) self.iir_cutoff_Hz = get_parameter(IIR_CUTOFF, parameters) self.iir_width_Hz = get_parameter(IIR_WIDTH, parameters) - # TODO: Get sample rate dynamically for filter design - self.iir_sos = generate_elliptic_iir_low_pass_filter( - self.iir_rp_dB, self.iir_rs_dB, self.iir_cutoff_Hz, self.iir_width_Hz, 14e6) + self.sample_rate = get_parameter(SAMPLE_RATE, parameters) + if not any([isinstance(v, list) for v in [self.iir_rp_dB, self.iir_rs_dB, self.iir_cutoff_Hz, self.iir_width_Hz, self.sample_rate]]): + # Generate single filter ahead of calibration loop + self.iir_sos = generate_elliptic_iir_low_pass_filter( + self.iir_rp_dB, self.iir_rs_dB, self.iir_cutoff_Hz, self.iir_width_Hz, self.sample_rate + ) + self.regenerate_iir = False + else: + self.regenerate_iir = True def __call__(self, schedule_entry_json, task_id): """This is the entrypoint function called by the scheduler.""" self.test_required_components() - detail = '' - # iteration_params is iterable even if it contains only one set of parameters - # it is also sorted by frequency from low to high iteration_params = utils.get_iterable_parameters(self.parameters) + detail = '' - # Calibrate + # Run calibration routine for i, p in enumerate(iteration_params): if i == 0: detail += self.calibrate(p) @@ -194,6 +198,7 @@ def calibrate(self, params): super().configure_sigan(params) # Get parameters from action config + iir_apply = get_parameter(IIR_APPLY, params) fft_size = get_parameter(FFT_SIZE, params) nffts = get_parameter(NUM_FFTS, params) nskip = get_parameter(NUM_SKIP, params) @@ -225,21 +230,35 @@ def calibrate(self, params): ) assert sample_rate == noise_off_measurement_result["sample_rate"], "Sample rate mismatch" - # Apply IIR filtering to both captures - logger.debug("Applying IIR filter to IQ captures") - noise_on_filtered = sosfilt(self.iir_sos, noise_on_measurement_result["data"]) - noise_off_filtered = sosfilt(self.iir_sos, noise_off_measurement_result["data"]) + # Apply IIR filtering to both captures if configured + if iir_apply: + if self.regenerate_iir: + logger.debug("Generating IIR filter") + rp_dB = get_parameter(IIR_RP, params) + rs_dB = get_parameter(IIR_RS, params) + cutoff_Hz = get_parameter(IIR_CUTOFF, params) + width_Hz = get_parameter(IIR_WIDTH, params) + self.iir_sos = generate_elliptic_iir_low_pass_filter( + rp_dB, rs_dB, cutoff_Hz, width_Hz, sample_rate + ) + logger.debug("Applying IIR filter to IQ captures") + noise_on_data = sosfilt(self.iir_sos, noise_on_measurement_result["data"]) + noise_off_data = sosfilt(self.iir_sos, noise_off_measurement_result["data"]) + else: + logger.debug('Skipping IIR filtering') + noise_on_data = noise_on_measurement_result["data"] + noise_off_data = noise_off_measurement_result["data"] # Get power values in time domain - td_on_watts = calculate_power_watts(noise_on_filtered.copy()) / 2. # Divide by 2 for RF/baseband conversion - td_off_watts = calculate_power_watts(noise_off_filtered.copy()) / 2. + td_on_watts = calculate_power_watts(noise_on_data.copy()) / 2. # Divide by 2 for RF/baseband conversion + td_off_watts = calculate_power_watts(noise_off_data.copy()) / 2. # Get mean power FFT results fft_on_watts = self.apply_mean_fft( - noise_on_filtered.copy(), fft_size, fft_window, nffts, fft_acf + noise_on_data.copy(), fft_size, fft_window, nffts, fft_acf ) fft_off_watts = self.apply_mean_fft( - noise_off_filtered.copy(), fft_size, fft_window, nffts, fft_acf + noise_off_data.copy(), fft_size, fft_window, nffts, fft_acf ) # Y-Factor From 55d1f9e7df3f76d98979dd578f8c5e90d9287448 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 29 Jul 2022 13:38:05 -0600 Subject: [PATCH 073/157] More consistent use of instance variables --- scos_actions/actions/calibrate_y_factor.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 0d40178e..229c211e 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -238,12 +238,14 @@ def calibrate(self, params): rs_dB = get_parameter(IIR_RS, params) cutoff_Hz = get_parameter(IIR_CUTOFF, params) width_Hz = get_parameter(IIR_WIDTH, params) - self.iir_sos = generate_elliptic_iir_low_pass_filter( + iir_sos = generate_elliptic_iir_low_pass_filter( rp_dB, rs_dB, cutoff_Hz, width_Hz, sample_rate ) + else: + iir_sos = self.iir_sos logger.debug("Applying IIR filter to IQ captures") - noise_on_data = sosfilt(self.iir_sos, noise_on_measurement_result["data"]) - noise_off_data = sosfilt(self.iir_sos, noise_off_measurement_result["data"]) + noise_on_data = sosfilt(iir_sos, noise_on_measurement_result["data"]) + noise_off_data = sosfilt(iir_sos, noise_off_measurement_result["data"]) else: logger.debug('Skipping IIR filtering') noise_on_data = noise_on_measurement_result["data"] @@ -266,7 +268,7 @@ def calibrate(self, params): # TODO Parameterize ENBW # ENBW should differ based on whether or not IIR filtering is used # enbw_hz_td = 11.607e6 # For RSA 507A - enbw_hz_td = 10.016e6 # Roughly based on IIR filter + enbw_hz_td = (cutoff_Hz + width_Hz) * 2. # Roughly based on IIR filter enr_linear = get_linear_enr(self.cal_source_idx) temp_k, temp_c, _ = get_temperature(self.temp_sensor_idx) From dfa7e8806cf78a5093ea9a7d881a6cfa184892e2 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 29 Jul 2022 13:46:13 -0600 Subject: [PATCH 074/157] Parameterize cal source and temp sensor indices --- scos_actions/actions/calibrate_y_factor.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 229c211e..4586e689 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -114,6 +114,8 @@ IIR_RS = 'iir_rs_dB' IIR_CUTOFF = 'iir_cutoff_Hz' IIR_WIDTH = 'iir_width_Hz' +CAL_SOURCE_IDX = 'cal_source_idx' +TEMP_SENSOR_IDX = 'temp_sensor_idx' # TODO: Should calibration source index and temperature sensor number # be required parameters? @@ -144,10 +146,6 @@ class YFactorCalibration(Action): def __init__(self, parameters, sigan, gps=mock_gps): logger.debug('Initializing calibration action') super().__init__(parameters, sigan, gps) - # Specify calibration source and temperature sensor indices - # TODO: Add cal source and sensor indices to YAML config - self.cal_source_idx = 0 - self.temp_sensor_idx = 1 # FFT setup # TODO: Remove these self.fft_detector = create_power_detector("MeanDetector", ["mean"]) @@ -198,10 +196,13 @@ def calibrate(self, params): super().configure_sigan(params) # Get parameters from action config + cal_source_idx = get_parameter(CAL_SOURCE_IDX, params) + temp_sensor_idx = get_parameter(TEMP_SENSOR_IDX, params) iir_apply = get_parameter(IIR_APPLY, params) fft_size = get_parameter(FFT_SIZE, params) nffts = get_parameter(NUM_FFTS, params) nskip = get_parameter(NUM_SKIP, params) + fft_window = get_fft_window(self.fft_window_type, fft_size) fft_acf = get_fft_window_correction(fft_window, 'amplitude') num_samples = fft_size * nffts @@ -269,8 +270,8 @@ def calibrate(self, params): # ENBW should differ based on whether or not IIR filtering is used # enbw_hz_td = 11.607e6 # For RSA 507A enbw_hz_td = (cutoff_Hz + width_Hz) * 2. # Roughly based on IIR filter - enr_linear = get_linear_enr(self.cal_source_idx) - temp_k, temp_c, _ = get_temperature(self.temp_sensor_idx) + enr_linear = get_linear_enr(cal_source_idx) + temp_k, temp_c, _ = get_temperature(temp_sensor_idx) # New method td_noise_figure, td_gain = y_factor( From 359c82b3d3a7a4af705ac12e5d62dc48370053de Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 29 Jul 2022 14:24:43 -0600 Subject: [PATCH 075/157] Add test actions for y-factor --- .../test_multi_frequency_y_factor_action.yml | 27 +++++++++++++++++++ .../test_single_frequency_y_factor_action.yml | 17 ++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 scos_actions/configs/actions/test_multi_frequency_y_factor_action.yml create mode 100644 scos_actions/configs/actions/test_single_frequency_y_factor_action.yml diff --git a/scos_actions/configs/actions/test_multi_frequency_y_factor_action.yml b/scos_actions/configs/actions/test_multi_frequency_y_factor_action.yml new file mode 100644 index 00000000..d830f4c5 --- /dev/null +++ b/scos_actions/configs/actions/test_multi_frequency_y_factor_action.yml @@ -0,0 +1,27 @@ +y_factor_cal: + name: test_multi_frequency_y_factor_action + # Preselector configuration + cal_source_idx: 0 # Index of calibration source in preselector + temp_sensor_idx: 1 # Index of temperature sensor in preselector + # Sigan Settings + gain: 40 + sample_rate: 15.36e6 + duration_ms: 1000 + nskip: 15.36e4 + frequency: + - 700.5e6 + - 709e6 + - 731.5e6 + - 739e6 + - 751e6 + - 763e6 + - 772e6 + - 782e6 + - 793e6 + - 802e6 + # IIR Filter Settings + iir_apply: True + iir_rp_dB: 0.1 + iir_rs_dB: 40 + iir_cutoff_Hz: 7.68e6 + iir_width_Hz: 8e3 \ No newline at end of file diff --git a/scos_actions/configs/actions/test_single_frequency_y_factor_action.yml b/scos_actions/configs/actions/test_single_frequency_y_factor_action.yml new file mode 100644 index 00000000..d28cedc0 --- /dev/null +++ b/scos_actions/configs/actions/test_single_frequency_y_factor_action.yml @@ -0,0 +1,17 @@ +y_factor_cal: + name: test_single_frequency_y_factor_action + # Preselector configuration + cal_source_idx: 0 # Index of calibration source in preselector + temp_sensor_idx: 1 # Index of temperature sensor in preselector + # Sigan Settings + gain: 40 + sample_rate: 15.36e6 + duration_ms: 1000 + nskip: 15.36e4 + frequency: 739e6 + # IIR Filter Settings + iir_apply: True + iir_rp_dB: 0.1 + iir_rs_dB: 40 + iir_cutoff_Hz: 7.68e6 + iir_width_Hz: 8e3 \ No newline at end of file From 15900921385f6f9403e4dc76b8ef4d89ccb21953 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 1 Aug 2022 16:43:45 -0600 Subject: [PATCH 076/157] Update README.md --- README.md | 59 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 16762fb0..a40fef0a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # NTIA/ITS SCOS Actions Plugin +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) + This repository contains common actions and interfaces to be re-used by scos-sensor plugins. See the [scos-sensor README]( https://github.com/NTIA/scos-sensor/blob/master/README.md) @@ -20,30 +22,33 @@ architecture. ## Overview of Repo Structure -- scos_actions/actions: This includes the base Action class, signals, and the following +- `scos_actions/actions`: This includes the base Action class, signals, and the following common action classes: - - acquire_single_freq_fft: performs FFTs and calculates mean, median, min, max, and + - `acquire_single_freq_fft`: performs FFTs and calculates mean, median, min, max, and sample statistics at a single center frequency. - - acquire_single_freq_tdomain_iq: acquires IQ data at a single center frequency. - - acquire_stepped_freq_tdomain_iq: acquires IQ data at multiple center frequencies. - - sync_gps: gets GPS location and syncs the host to GPS time - - monitor_sigan: ensures a signal analyzer is available and is able to maintain a + - `acquire_single_freq_tdomain_iq`: acquires IQ data at a single center frequency. + - `acquire_stepped_freq_tdomain_iq`: acquires IQ data at multiple center frequencies. + - `calibrate_y_facvtor`: performs calibration using the Y-Factor method. + - `sync_gps`: gets GPS location and syncs the host to GPS time + - `monitor_sigan`: ensures a signal analyzer is available and is able to maintain a connection to the computer. -- scos_actions/configs/actions: This folder contains the yaml files with the parameters +- `scos_actions/configs/actions`: This folder contains the YAML files with the parameters used to initialize the actions described above. -- scos_actions/discover: This includes the code to read yaml files and make actions +- `scos_actions/discover`: This includes the code to read YAML files and make actions available to scos-sensor. -- scos_actions/hardware: This includes the signal analyzer interface and GPS interface +- `scos_actions/hardware`: This includes the signal analyzer interface and GPS interface used by the actions and the mock signal analyzer. The signal analyzer interface is intended to represent universal functionality that is common across all signal analyzers. The specific implementations of the signal analyzer interface for particular signal analyzers are provided in separate repositories like [scos-usrp](https://github.com/NTIA/scos-usrp). +- `scos_actions/signal_processing`: This contains various common signal processing +routines which are used in actions. ## Running in scos-sensor -Requires pip>=18.1 (upgrade using `python3 -m pip install --upgrade pip`) and -python>=3.7. +Requires `pip>=18.1` (upgrade using `python3 -m pip install --upgrade pip`) and +`python>=3.7`. 1. Clone scos-sensor: git clone 1. Navigate to scos-sensor: `cd scos-sensor` @@ -56,17 +61,19 @@ python>=3.7. 1. If it does not exist, create env file while in the root scos-sensor directory: `cp env.template ./env` 1. In env file, change `BASE_IMAGE=ubuntu:18.04` (at the bottom of the file) -1. Set `MOCK_SIGAN` and `MOCK_SIGAN_RANDOM` equal to 1 in docker-compose.yml +1. Set `MOCK_SIGAN` and `MOCK_SIGAN_RANDOM` equal to 1 in `docker-compose.yml` 1. Get environment variables: `source ./env` 1. Build and start containers: `docker-compose up -d --build --force-recreate` -If scos-actions is installed to scos-sensor as a plugin, the following three +If scos-actions is installed to scos-sensor as a plugin, the following parameterized actions are offered for testing using a mock signal analyzer; their -parameters are defined in scos_actions/configs/actions. +parameters are defined in `scos_actions/configs/actions`. -- test_multi_frequency_iq_action -- test_single_frequency_iq_action -- test_single_frequency_m4s_action +- `test_multi_frequency_iq_action` +- `test_multi_frequency_y_factor_action` +- `test_single_frequency_iq_action` +- `test_single_frequency_m4s_action` +- `test_single_frequency_y_factor_action` ## Development @@ -94,38 +101,38 @@ python3 -m pip install --upgrade pip # upgrade to pip>=18.1 python3 -m pip install -r requirements-dev.txt ``` -#### Using pip-tools +#### Using `pip-tools` It is recommended to keep direct dependencies in a separate file. The direct -dependencies are in the requirements.in and requirements-dev.in files. Then pip-tools +dependencies are in the `requirements.in` and `requirements-dev.in` files. Then `pip-tools` can be used to generate files with all the dependencies and transitive dependencies -(sub-dependencies). The files containing all the dependencies are in requirements.txt -and requirements-dev.txt. Run the following in the virtual environment to install -pip-tools. +(sub-dependencies). The files containing all the dependencies are in `requirements.txt` +and `requirements-dev.txt`. Run the following in the virtual environment to install +`pip-tools`. ```bash python -m pip install pip-tools ``` -To update requirements.txt after modifying requirements.in: +To update `requirements.txt` after modifying `requirements.in`: ```bash pip-compile requirements.in ``` -To update requirements-dev.txt after modifying requirements.in or requirements-dev.in: +To update `requirements-dev.txt` after modifying `requirements.in` or `requirements-dev.in`: ```bash pip-compile requirements-dev.in ``` -Use pip-sync to match virtual environment to requirements-dev.txt: +Use `pip-sync` to match virtual environment to `requirements-dev.txt`: ```bash pip-sync requirements.txt requirements-dev.txt ``` -For more information about pip-tools, see +For more information about `pip-tools`, see ### Running Tests From e70ccbc37809210f6d124ee982e548ebd9868803 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 1 Aug 2022 17:24:59 -0600 Subject: [PATCH 077/157] Update test calibration actions --- .../test_multi_frequency_y_factor_action.yml | 20 +++++++------------ .../test_single_frequency_y_factor_action.yml | 8 ++++---- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/scos_actions/configs/actions/test_multi_frequency_y_factor_action.yml b/scos_actions/configs/actions/test_multi_frequency_y_factor_action.yml index d830f4c5..023126a0 100644 --- a/scos_actions/configs/actions/test_multi_frequency_y_factor_action.yml +++ b/scos_actions/configs/actions/test_multi_frequency_y_factor_action.yml @@ -5,23 +5,17 @@ y_factor_cal: temp_sensor_idx: 1 # Index of temperature sensor in preselector # Sigan Settings gain: 40 - sample_rate: 15.36e6 + sample_rate: 14e6 duration_ms: 1000 - nskip: 15.36e4 + nskip: 0 frequency: - - 700.5e6 - - 709e6 - - 731.5e6 - - 739e6 - - 751e6 - - 763e6 - - 772e6 - - 782e6 - - 793e6 - - 802e6 + - 3555e6 + - 3565e6 + - 3575e6 + - 3585e6 # IIR Filter Settings iir_apply: True iir_rp_dB: 0.1 iir_rs_dB: 40 - iir_cutoff_Hz: 7.68e6 + iir_cutoff_Hz: 5e6 iir_width_Hz: 8e3 \ No newline at end of file diff --git a/scos_actions/configs/actions/test_single_frequency_y_factor_action.yml b/scos_actions/configs/actions/test_single_frequency_y_factor_action.yml index d28cedc0..db22d38e 100644 --- a/scos_actions/configs/actions/test_single_frequency_y_factor_action.yml +++ b/scos_actions/configs/actions/test_single_frequency_y_factor_action.yml @@ -5,13 +5,13 @@ y_factor_cal: temp_sensor_idx: 1 # Index of temperature sensor in preselector # Sigan Settings gain: 40 - sample_rate: 15.36e6 + sample_rate: 14e6 duration_ms: 1000 - nskip: 15.36e4 - frequency: 739e6 + nskip: 0 + frequency: 3555e6 # IIR Filter Settings iir_apply: True iir_rp_dB: 0.1 iir_rs_dB: 40 - iir_cutoff_Hz: 7.68e6 + iir_cutoff_Hz: 5e6 iir_width_Hz: 8e3 \ No newline at end of file From 5bd90dd187e916a1ed2e5ab3a7836aa5f396aacb Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 2 Aug 2022 08:46:06 -0600 Subject: [PATCH 078/157] Fix parameter errors --- scos_actions/actions/calibrate_y_factor.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 4586e689..7d61ac1f 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -198,10 +198,13 @@ def calibrate(self, params): # Get parameters from action config cal_source_idx = get_parameter(CAL_SOURCE_IDX, params) temp_sensor_idx = get_parameter(TEMP_SENSOR_IDX, params) - iir_apply = get_parameter(IIR_APPLY, params) fft_size = get_parameter(FFT_SIZE, params) nffts = get_parameter(NUM_FFTS, params) nskip = get_parameter(NUM_SKIP, params) + if self.iir_apply is not False: + iir_apply = get_parameter(IIR_APPLY, params) + else: + iir_apply = False fft_window = get_fft_window(self.fft_window_type, fft_size) fft_acf = get_fft_window_correction(fft_window, 'amplitude') @@ -244,6 +247,8 @@ def calibrate(self, params): ) else: iir_sos = self.iir_sos + cutoff_Hz = self.iir_cutoff_Hz + width_Hz = self.iir_width_Hz logger.debug("Applying IIR filter to IQ captures") noise_on_data = sosfilt(iir_sos, noise_on_measurement_result["data"]) noise_off_data = sosfilt(iir_sos, noise_off_measurement_result["data"]) From 290b4a8459e9352ff5015ab862f184c376c01d9f Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 2 Aug 2022 09:12:30 -0600 Subject: [PATCH 079/157] Fix ENBW when not filtering --- scos_actions/actions/calibrate_y_factor.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 7d61ac1f..3ba6745c 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -249,10 +249,12 @@ def calibrate(self, params): iir_sos = self.iir_sos cutoff_Hz = self.iir_cutoff_Hz width_Hz = self.iir_width_Hz + enbw_hz_td = (cutoff_Hz + width_Hz) * 2. # Roughly based on IIR filter logger.debug("Applying IIR filter to IQ captures") noise_on_data = sosfilt(iir_sos, noise_on_measurement_result["data"]) noise_off_data = sosfilt(iir_sos, noise_off_measurement_result["data"]) else: + enbw_hz_td = 11.607e6 # For RSA 507A #TODO REMOVE logger.debug('Skipping IIR filtering') noise_on_data = noise_on_measurement_result["data"] noise_off_data = noise_off_measurement_result["data"] @@ -271,10 +273,6 @@ def calibrate(self, params): # Y-Factor enbw_hz = get_fft_enbw(fft_window, sample_rate) - # TODO Parameterize ENBW - # ENBW should differ based on whether or not IIR filtering is used - # enbw_hz_td = 11.607e6 # For RSA 507A - enbw_hz_td = (cutoff_Hz + width_Hz) * 2. # Roughly based on IIR filter enr_linear = get_linear_enr(cal_source_idx) temp_k, temp_c, _ = get_temperature(temp_sensor_idx) From eec26dc1de5a1984b8be604aa73a59c051c5f8ae Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 2 Aug 2022 09:17:01 -0600 Subject: [PATCH 080/157] Debug in mW --- scos_actions/signal_processing/calibration.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py index 77b7a97a..616b671d 100644 --- a/scos_actions/signal_processing/calibration.py +++ b/scos_actions/signal_processing/calibration.py @@ -48,14 +48,14 @@ def y_factor( noise figure and gain, both in dB, from the Y-factor method. """ if logger.isEnabledFor(logging.DEBUG): - mean_on_watts = np.mean(pwr_noise_on_watts) - mean_off_watts = np.mean(pwr_noise_off_watts) - mean_on_dBm = convert_watts_to_dBm(mean_on_watts) - mean_off_dBm = convert_watts_to_dBm(mean_off_watts) + mean_on_mwatts = np.mean(pwr_noise_on_watts) * 1e3 + mean_off_mwatts = np.mean(pwr_noise_off_watts) * 1e3 + mean_on_dBm = convert_linear_to_dB(mean_on_mwatts) + mean_off_dBm = convert_linear_to_dB(mean_off_mwatts) logger.debug(f"ENR: {convert_linear_to_dB(enr_linear)} dB") logger.debug(f"ENBW: {enbw_hz} Hz") - logger.debug(f"Mean power on: {mean_on_watts:.2f} W = {mean_on_dBm:.2f} dBm") - logger.debug(f"Mean power off: {mean_off_watts:.2f} W = {mean_off_dBm:.2f} dBm") + logger.debug(f"Mean power on: {mean_on_mwatts:.2f} mW = {mean_on_dBm:.2f} dBm") + logger.debug(f"Mean power off: {mean_off_mwatts:.2f} mW = {mean_off_dBm:.2f} dBm") y = pwr_noise_on_watts / pwr_noise_off_watts noise_factor = enr_linear / (y - 1.0) gain_watts = pwr_noise_on_watts / ( From 1583402cdfead164d4628001105910730c3de56f Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 2 Aug 2022 12:20:07 -0600 Subject: [PATCH 081/157] Debugging --- scos_actions/actions/calibrate_y_factor.py | 11 +++++++---- scos_actions/signal_processing/fft.py | 10 ++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 3ba6745c..adb31c8e 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -260,15 +260,15 @@ def calibrate(self, params): noise_off_data = noise_off_measurement_result["data"] # Get power values in time domain - td_on_watts = calculate_power_watts(noise_on_data.copy()) / 2. # Divide by 2 for RF/baseband conversion - td_off_watts = calculate_power_watts(noise_off_data.copy()) / 2. + td_on_watts = calculate_power_watts(noise_on_data) / 2. # Divide by 2 for RF/baseband conversion + td_off_watts = calculate_power_watts(noise_off_data) / 2. # Get mean power FFT results fft_on_watts = self.apply_mean_fft( - noise_on_data.copy(), fft_size, fft_window, nffts, fft_acf + noise_on_data, fft_size, fft_window, nffts, fft_acf ) fft_off_watts = self.apply_mean_fft( - noise_off_data.copy(), fft_size, fft_window, nffts, fft_acf + noise_off_data, fft_size, fft_window, nffts, fft_acf ) # Y-Factor @@ -320,8 +320,11 @@ def apply_mean_fft( # TESTING SCALING complex_fft /= 2 # RF/baseband conversion complex_fft *= fft_window_cf # Window correction + logger.debug(f"Scaled FFT: {complex_fft[0,:5]}") power_fft = calculate_power_watts(complex_fft) + logger.debug(f"Power FFT: {power_fft[0,:5]}") mean_result = apply_power_detector(power_fft, self.fft_detector) + logger.debug(f"Mean result shape: {mean_result.shape}") return mean_result @property diff --git a/scos_actions/signal_processing/fft.py b/scos_actions/signal_processing/fft.py index 8b464885..d80e25a3 100644 --- a/scos_actions/signal_processing/fft.py +++ b/scos_actions/signal_processing/fft.py @@ -63,6 +63,7 @@ def get_fft( :return: The transformed input, scaled based on the specified normalization mode. """ + logger.debug("Computing FFTs") # Get num_ffts for default case: as many as possible if num_ffts <= 0: logger.info("Number of FFTs not specified. Using as many as possible.") @@ -79,18 +80,23 @@ def get_fft( # Resize time data for FFTs time_data = np.reshape(time_data[: num_ffts * fft_size], (num_ffts, fft_size)) + logger.debug(f"Num. FFTs: {num_ffts}, FFT Size: {fft_size}, Data shape: {time_data.shape}") # Apply the FFT window if provided if fft_window is not None: logger.debug("Applying window before FFT") - time_data = ne.evaluate("time_data*fft_window") + logger.debug(f"Time data: {time_data[0,:5]}, window: {fft_window[:5]}") + ne.evaluate("time_data*fft_window", out=time_data) + logger.debug("After windowing:") + logger.debug(f"Time data: {time_data[0,:5]}") # Take the FFT complex_fft = sp_fft(time_data, norm=norm, workers=workers) # Shift the frequencies if desired (only along second axis) if shift: - complex_fft = np.fft.fftshift(complex_fft, axes=(1,)) + logger.debug("Shifting zero-frequency component of FFT to center") + complex_fft = np.fft.fftshift(complex_fft) #, axes=(1,)) return complex_fft From 6b06e576273542a7a4e0af47c9a6e63ae8b6c92f Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 2 Aug 2022 12:31:19 -0600 Subject: [PATCH 082/157] fix typeerror --- scos_actions/signal_processing/fft.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scos_actions/signal_processing/fft.py b/scos_actions/signal_processing/fft.py index d80e25a3..2f7e6fcd 100644 --- a/scos_actions/signal_processing/fft.py +++ b/scos_actions/signal_processing/fft.py @@ -85,6 +85,8 @@ def get_fft( # Apply the FFT window if provided if fft_window is not None: logger.debug("Applying window before FFT") + if time_data.dtype is np.complex64: + time_data = time_data.astype(np.complex128) logger.debug(f"Time data: {time_data[0,:5]}, window: {fft_window[:5]}") ne.evaluate("time_data*fft_window", out=time_data) logger.debug("After windowing:") From 242b0e95efb8d3190022c53a4bbd27fa90ce261d Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 2 Aug 2022 12:55:40 -0600 Subject: [PATCH 083/157] Fix windowing data type error --- scos_actions/signal_processing/fft.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scos_actions/signal_processing/fft.py b/scos_actions/signal_processing/fft.py index 2f7e6fcd..2ebdf784 100644 --- a/scos_actions/signal_processing/fft.py +++ b/scos_actions/signal_processing/fft.py @@ -85,12 +85,11 @@ def get_fft( # Apply the FFT window if provided if fft_window is not None: logger.debug("Applying window before FFT") - if time_data.dtype is np.complex64: - time_data = time_data.astype(np.complex128) logger.debug(f"Time data: {time_data[0,:5]}, window: {fft_window[:5]}") - ne.evaluate("time_data*fft_window", out=time_data) + time_data = ne.evaluate("time_data*fft_window") logger.debug("After windowing:") logger.debug(f"Time data: {time_data[0,:5]}") + logger.debug(f"Windowed data type and shape {time_data.dtype}, {time_data.shape}") # Take the FFT complex_fft = sp_fft(time_data, norm=norm, workers=workers) From d4a5f1329a619db6ae91e8231d7f5482dd87a33a Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 2 Aug 2022 13:32:40 -0600 Subject: [PATCH 084/157] debugging power detectors --- scos_actions/signal_processing/fft.py | 6 ------ scos_actions/signal_processing/power_analysis.py | 3 +++ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/scos_actions/signal_processing/fft.py b/scos_actions/signal_processing/fft.py index 2ebdf784..6e5e0944 100644 --- a/scos_actions/signal_processing/fft.py +++ b/scos_actions/signal_processing/fft.py @@ -84,19 +84,13 @@ def get_fft( # Apply the FFT window if provided if fft_window is not None: - logger.debug("Applying window before FFT") - logger.debug(f"Time data: {time_data[0,:5]}, window: {fft_window[:5]}") time_data = ne.evaluate("time_data*fft_window") - logger.debug("After windowing:") - logger.debug(f"Time data: {time_data[0,:5]}") - logger.debug(f"Windowed data type and shape {time_data.dtype}, {time_data.shape}") # Take the FFT complex_fft = sp_fft(time_data, norm=norm, workers=workers) # Shift the frequencies if desired (only along second axis) if shift: - logger.debug("Shifting zero-frequency component of FFT to center") complex_fft = np.fft.fftshift(complex_fft) #, axes=(1,)) return complex_fft diff --git a/scos_actions/signal_processing/power_analysis.py b/scos_actions/signal_processing/power_analysis.py index bc88fc87..4e98b57c 100644 --- a/scos_actions/signal_processing/power_analysis.py +++ b/scos_actions/signal_processing/power_analysis.py @@ -140,6 +140,7 @@ def apply_power_detector( detector_functions = [np.min, np.max, np.mean, np.median] # Get functions based on specified detector + logger.debug(f"Applying power detectors: {detectors}") if "min" in detectors: detector_functions.append(detector_functions[0]) if "max" in detectors: @@ -155,6 +156,8 @@ def apply_power_detector( rng = np.random.default_rng() result.append(data[rng.integers(0, data.shape[0], 1)][0]) del rng + logger.debug(f"Power detector input shape: {data.shape}") + logger.debug(f"Power detector output shape: {result.shape}") return np.array(result, dtype=dtype) From 572b0c3cdf7340ebd8ebb6898fa16c2eddc70543 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 2 Aug 2022 13:41:54 -0600 Subject: [PATCH 085/157] Fix array conversion --- scos_actions/signal_processing/power_analysis.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scos_actions/signal_processing/power_analysis.py b/scos_actions/signal_processing/power_analysis.py index 4e98b57c..6345f8ea 100644 --- a/scos_actions/signal_processing/power_analysis.py +++ b/scos_actions/signal_processing/power_analysis.py @@ -156,9 +156,10 @@ def apply_power_detector( rng = np.random.default_rng() result.append(data[rng.integers(0, data.shape[0], 1)][0]) del rng + result = np.array(result, dtype=dtype) logger.debug(f"Power detector input shape: {data.shape}") logger.debug(f"Power detector output shape: {result.shape}") - return np.array(result, dtype=dtype) + return result def filter_quantiles(x: np.ndarray, q_lo: float, q_hi: float) -> np.ndarray: From 3565cc12e57e8de6ac9da8a8e637b54fc3472d4f Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 2 Aug 2022 13:47:31 -0600 Subject: [PATCH 086/157] Fix detector selection --- scos_actions/signal_processing/power_analysis.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scos_actions/signal_processing/power_analysis.py b/scos_actions/signal_processing/power_analysis.py index 6345f8ea..78f68577 100644 --- a/scos_actions/signal_processing/power_analysis.py +++ b/scos_actions/signal_processing/power_analysis.py @@ -141,24 +141,23 @@ def apply_power_detector( # Get functions based on specified detector logger.debug(f"Applying power detectors: {detectors}") + applied_detectors = [] if "min" in detectors: - detector_functions.append(detector_functions[0]) + applied_detectors.append(detector_functions[0]) if "max" in detectors: - detector_functions.append(detector_functions[1]) + applied_detectors.append(detector_functions[1]) if "mean" in detectors: - detector_functions.append(detector_functions[2]) + applied_detectors.append(detector_functions[2]) if "median" in detectors: - detector_functions.append(detector_functions[3]) + applied_detectors.append(detector_functions[3]) # Apply statistical detectors - result = [d(data, axis=0) for d in detector_functions] + result = [d(data, axis=0) for d in applied_detectors] # Add sample detector result if configured if "sample" in detectors: rng = np.random.default_rng() result.append(data[rng.integers(0, data.shape[0], 1)][0]) del rng result = np.array(result, dtype=dtype) - logger.debug(f"Power detector input shape: {data.shape}") - logger.debug(f"Power detector output shape: {result.shape}") return result From ce4baaa1e5520d0f8ecdb80908b74a9803067a44 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 2 Aug 2022 14:04:07 -0600 Subject: [PATCH 087/157] Test new time domain averaging --- scos_actions/actions/calibrate_y_factor.py | 25 ++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index adb31c8e..31a7978c 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -260,8 +260,14 @@ def calibrate(self, params): noise_off_data = noise_off_measurement_result["data"] # Get power values in time domain - td_on_watts = calculate_power_watts(noise_on_data) / 2. # Divide by 2 for RF/baseband conversion - td_off_watts = calculate_power_watts(noise_off_data) / 2. + # td_on_watts = calculate_power_watts(noise_on_data) / 2. # Divide by 2 for RF/baseband conversion + # td_off_watts = calculate_power_watts(noise_off_data) / 2. + td_on_watts = self.apply_mean_td( + noise_on_data, fft_size, nffts + ) + td_off_watts = self.apply_mean_td( + noise_off_data, fft_size, nffts + ) # Get mean power FFT results fft_on_watts = self.apply_mean_fft( @@ -306,6 +312,17 @@ def calibrate(self, params): # Detail results contain only FFT version of result for now return 'Noise Figure: {}, Gain: {}'.format(fft_noise_figure, fft_gain) + def apply_mean_td(self, iqdata: ndarray, block_size: int, n_blocks: int): + # TESTING + # fft_detector can also be used for time domain + # Reshape data + import numpy as np + iq = np.reshape(iqdata[:block_size * n_blocks], (n_blocks, block_size)) + iq /= 2 # RF/baseband conversion + iq_pwr = calculate_power_watts(iq) + mean_result = apply_power_detector(iq_pwr, self.fft_detector) + return mean_result + def apply_mean_fft( self, iqdata: ndarray, fft_size: int, fft_window: ndarray, nffts: int, fft_window_cf: float ) -> ndarray: @@ -317,14 +334,10 @@ def apply_mean_fft( num_ffts=nffts, shift=True ) - # TESTING SCALING complex_fft /= 2 # RF/baseband conversion complex_fft *= fft_window_cf # Window correction - logger.debug(f"Scaled FFT: {complex_fft[0,:5]}") power_fft = calculate_power_watts(complex_fft) - logger.debug(f"Power FFT: {power_fft[0,:5]}") mean_result = apply_power_detector(power_fft, self.fft_detector) - logger.debug(f"Mean result shape: {mean_result.shape}") return mean_result @property From 0c12b4b101c13098f713743dfef05f8c77e773b5 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 3 Aug 2022 17:09:20 -0600 Subject: [PATCH 088/157] Remove FFT method --- scos_actions/actions/calibrate_y_factor.py | 112 ++++++--------------- 1 file changed, 32 insertions(+), 80 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 31a7978c..ab4bf579 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -68,8 +68,8 @@ import logging import time +import numpy as np -from numpy import ndarray from scipy.constants import Boltzmann from scipy.signal import sosfilt @@ -79,7 +79,6 @@ from scos_actions.settings import SENSOR_CALIBRATION_FILE from scos_actions.actions.interfaces.action import Action from scos_actions.utils import ParameterException, get_parameter -from scos_actions.signal_processing.fft import get_fft, get_fft_enbw, get_fft_window, get_fft_window_correction from scos_actions.signal_processing.calibration import ( get_linear_enr, @@ -106,8 +105,7 @@ # Define parameter keys FREQUENCY = "frequency" SAMPLE_RATE = "sample_rate" -FFT_SIZE = "fft_size" -NUM_FFTS = "nffts" +DURATION_MS = "duration_ms" NUM_SKIP = "nskip" IIR_APPLY = 'iir_apply' IIR_RP = 'iir_rp_dB' @@ -146,10 +144,8 @@ class YFactorCalibration(Action): def __init__(self, parameters, sigan, gps=mock_gps): logger.debug('Initializing calibration action') super().__init__(parameters, sigan, gps) - # FFT setup - # TODO: Remove these - self.fft_detector = create_power_detector("MeanDetector", ["mean"]) - self.fft_window_type = "flattop" + self.power_detector = create_power_detector("MeanDetector", ["mean"]) + # IIR Filter Setup try: self.iir_apply = get_parameter(IIR_APPLY, parameters) @@ -198,17 +194,14 @@ def calibrate(self, params): # Get parameters from action config cal_source_idx = get_parameter(CAL_SOURCE_IDX, params) temp_sensor_idx = get_parameter(TEMP_SENSOR_IDX, params) - fft_size = get_parameter(FFT_SIZE, params) - nffts = get_parameter(NUM_FFTS, params) + sample_rate = get_parameter(SAMPLE_RATE, params) + duration_ms = get_parameter(DURATION_MS, params) + num_samples = int(sample_rate * duration_ms * 1e-3) nskip = get_parameter(NUM_SKIP, params) if self.iir_apply is not False: iir_apply = get_parameter(IIR_APPLY, params) else: iir_apply = False - - fft_window = get_fft_window(self.fft_window_type, fft_size) - fft_acf = get_fft_window_correction(fft_window, 'amplitude') - num_samples = fft_size * nffts # Set noise diode on logger.debug('Setting noise diode on') @@ -249,46 +242,26 @@ def calibrate(self, params): iir_sos = self.iir_sos cutoff_Hz = self.iir_cutoff_Hz width_Hz = self.iir_width_Hz - enbw_hz_td = (cutoff_Hz + width_Hz) * 2. # Roughly based on IIR filter + enbw_hz = (cutoff_Hz + width_Hz) * 2. # Roughly based on IIR filter logger.debug("Applying IIR filter to IQ captures") noise_on_data = sosfilt(iir_sos, noise_on_measurement_result["data"]) noise_off_data = sosfilt(iir_sos, noise_off_measurement_result["data"]) else: - enbw_hz_td = 11.607e6 # For RSA 507A #TODO REMOVE + enbw_hz = 11.607e6 # For RSA 507A #TODO REMOVE logger.debug('Skipping IIR filtering') noise_on_data = noise_on_measurement_result["data"] noise_off_data = noise_off_measurement_result["data"] # Get power values in time domain - # td_on_watts = calculate_power_watts(noise_on_data) / 2. # Divide by 2 for RF/baseband conversion - # td_off_watts = calculate_power_watts(noise_off_data) / 2. - td_on_watts = self.apply_mean_td( - noise_on_data, fft_size, nffts - ) - td_off_watts = self.apply_mean_td( - noise_off_data, fft_size, nffts - ) - - # Get mean power FFT results - fft_on_watts = self.apply_mean_fft( - noise_on_data, fft_size, fft_window, nffts, fft_acf - ) - fft_off_watts = self.apply_mean_fft( - noise_off_data, fft_size, fft_window, nffts, fft_acf - ) + pwr_on_watts = self.get_mean_power(noise_on_data) + pwr_off_watts = self.get_mean_power(noise_off_data) # Y-Factor - enbw_hz = get_fft_enbw(fft_window, sample_rate) enr_linear = get_linear_enr(cal_source_idx) temp_k, temp_c, _ = get_temperature(temp_sensor_idx) - # New method - td_noise_figure, td_gain = y_factor( - td_on_watts, td_off_watts, enr_linear, enbw_hz_td, temp_k - ) - # Old method for comparison - fft_noise_figure, fft_gain = y_factor( - fft_on_watts, fft_off_watts, enr_linear, enbw_hz, temp_k + noise_figure, gain = y_factor( + pwr_on_watts, pwr_off_watts, enr_linear, enbw_hz, temp_k ) # Don't update the sensor calibration while testing @@ -304,69 +277,48 @@ def calibrate(self, params): # Debugging noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) logger.debug(f'Noise floor: {noise_floor_dBm:.2f} dBm') - logger.debug(f'Noise Figure (FFT): {fft_noise_figure:.2f} dB') - logger.debug(f'Gain (FFT): {fft_gain:.2f} dB') - logger.debug(f'Noise figure (TD): {td_noise_figure:.2f} dB') - logger.debug(f"Gain (TD): {td_gain:.2f} dB") + logger.debug(f'Noise figure: {noise_figure:.2f} dB') + logger.debug(f"Gain: {gain:.2f} dB") # Detail results contain only FFT version of result for now - return 'Noise Figure: {}, Gain: {}'.format(fft_noise_figure, fft_gain) + return 'Noise Figure: {}, Gain: {}'.format(noise_figure, gain) - def apply_mean_td(self, iqdata: ndarray, block_size: int, n_blocks: int): - # TESTING - # fft_detector can also be used for time domain + def get_mean_power(self, iqdata: np.ndarray) -> np.ndarray: # Reshape data - import numpy as np - iq = np.reshape(iqdata[:block_size * n_blocks], (n_blocks, block_size)) - iq /= 2 # RF/baseband conversion - iq_pwr = calculate_power_watts(iq) - mean_result = apply_power_detector(iq_pwr, self.fft_detector) - return mean_result - - def apply_mean_fft( - self, iqdata: ndarray, fft_size: int, fft_window: ndarray, nffts: int, fft_window_cf: float - ) -> ndarray: - complex_fft = get_fft( - time_data=iqdata, - fft_size=fft_size, - norm="forward", - fft_window=fft_window, - num_ffts=nffts, - shift=True - ) - complex_fft /= 2 # RF/baseband conversion - complex_fft *= fft_window_cf # Window correction - power_fft = calculate_power_watts(complex_fft) - mean_result = apply_power_detector(power_fft, self.fft_detector) - return mean_result + # iq = np.reshape(iqdata[:block_size * n_blocks], (n_blocks, block_size)) + iqdata /= 2 # RF/baseband conversion + iq_pwr = calculate_power_watts(iqdata) + # mean_result = apply_power_detector(iq_pwr, self.power_detector) + # return mean_result + return iq_pwr @property def description(self): # Get parameters; they may be single values or lists frequencies = get_parameter(FREQUENCY, self.parameters) - nffts = get_parameter(NUM_FFTS, self.parameters) - fft_size = get_parameter(FFT_SIZE, self.parameters) + # nffts = get_parameter(NUM_FFTS, self.parameters) + # fft_size = get_parameter(FFT_SIZE, self.parameters) # Convert parameter lists to strings if needed if isinstance(frequencies, list): frequencies = utils.list_to_string( [f / 1e6 for f in get_parameter(FREQUENCY, self.parameters)] ) - if isinstance(nffts, list): - nffts = utils.list_to_string(get_parameter(NUM_FFTS, self.parameters)) - if isinstance(fft_size, list): - fft_size = utils.list_to_string(get_parameter(FFT_SIZE, self.parameters)) + # if isinstance(nffts, list): + # nffts = utils.list_to_string(get_parameter(NUM_FFTS, self.parameters)) + # if isinstance(fft_size, list): + # fft_size = utils.list_to_string(get_parameter(FFT_SIZE, self.parameters)) acq_plan = ( f"Performs a y-factor calibration at frequencies: " - f"{frequencies}, nffts:{nffts}, fft_size: {fft_size}\n" + # f"{frequencies}, nffts:{nffts}, fft_size: {fft_size}\n" ) definitions = { "name": self.name, "frequencies": frequencies, "acquisition_plan": acq_plan, - "fft_size": fft_size, - "nffts": nffts, + # "fft_size": fft_size, + # "nffts": nffts, } # __doc__ refers to the module docstring at the top of the file return __doc__.format(**definitions) From 37ac3f1e6f2fb09d4aea85fd0915312d2d4be881 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 4 Aug 2022 08:54:37 -0600 Subject: [PATCH 089/157] Temporarily disable docstring formatting --- scos_actions/actions/calibrate_y_factor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index ab4bf579..3feb9f85 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -321,7 +321,7 @@ def description(self): # "nffts": nffts, } # __doc__ refers to the module docstring at the top of the file - return __doc__.format(**definitions) + return __doc__ #.format(**definitions) def test_required_components(self): """Fail acquisition if a required component is not available.""" From d47eff96c120e3d4f4f49d54b4268f257965a067 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 4 Aug 2022 10:42:09 -0600 Subject: [PATCH 090/157] Test calculation in dB-domain --- scos_actions/actions/calibrate_y_factor.py | 3 --- scos_actions/signal_processing/calibration.py | 17 +++++++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 3feb9f85..0ed0be9b 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -115,9 +115,6 @@ CAL_SOURCE_IDX = 'cal_source_idx' TEMP_SENSOR_IDX = 'temp_sensor_idx' -# TODO: Should calibration source index and temperature sensor number -# be required parameters? - class YFactorCalibration(Action): """Perform a single- or stepped-frequency Y-factor calibration. diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py index 616b671d..fe41d72a 100644 --- a/scos_actions/signal_processing/calibration.py +++ b/scos_actions/signal_processing/calibration.py @@ -47,20 +47,25 @@ def y_factor( :return: A tuple (noise_figure, gain) containing the calculated noise figure and gain, both in dB, from the Y-factor method. """ + mean_on_dBm = convert_watts_to_dBm(np.mean(pwr_noise_on_watts)) + mean_off_dBm = convert_watts_to_dBm(np.mean(pwr_noise_off_watts)) if logger.isEnabledFor(logging.DEBUG): - mean_on_mwatts = np.mean(pwr_noise_on_watts) * 1e3 - mean_off_mwatts = np.mean(pwr_noise_off_watts) * 1e3 - mean_on_dBm = convert_linear_to_dB(mean_on_mwatts) - mean_off_dBm = convert_linear_to_dB(mean_off_mwatts) logger.debug(f"ENR: {convert_linear_to_dB(enr_linear)} dB") logger.debug(f"ENBW: {enbw_hz} Hz") - logger.debug(f"Mean power on: {mean_on_mwatts:.2f} mW = {mean_on_dBm:.2f} dBm") - logger.debug(f"Mean power off: {mean_off_mwatts:.2f} mW = {mean_off_dBm:.2f} dBm") + logger.debug(f"Mean power on: {mean_on_dBm:.2f} dBm") + logger.debug(f"Mean power off: {mean_off_dBm:.2f} dBm") y = pwr_noise_on_watts / pwr_noise_off_watts + y_dBcalc = convert_dB_to_linear(mean_on_dBm - mean_off_dBm) + logger.debug(f"Y (linear calc): {y}") + logger.debug(f"Y (dB calc): {y_dBcalc}") noise_factor = enr_linear / (y - 1.0) + nf_dbcalc = convert_linear_to_dB(enr_linear / (y_dBcalc - 1.0)) + logger.debug(f"Noise figure (dB calc): {nf_dbcalc}") gain_watts = pwr_noise_on_watts / ( Boltzmann * temp_kelvins * enbw_hz * (enr_linear + noise_factor) ) + gain_dbcalc = mean_on_dBm - convert_watts_to_dBm(Boltzmann * temp_kelvins * enbw_hz * (enr_linear + noise_factor)) + logger.debug(f"Gain (dB calc): {gain_dbcalc}") # Get mean values from arrays and convert to dB noise_figure = convert_linear_to_dB(np.mean(noise_factor)) gain = convert_linear_to_dB(np.mean(gain_watts)) From 9fa66a0ab9a3b876820973c3c314a05623b51ffd Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 4 Aug 2022 10:54:18 -0600 Subject: [PATCH 091/157] separate noise factor and noise figure --- scos_actions/signal_processing/calibration.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py index fe41d72a..36bb694a 100644 --- a/scos_actions/signal_processing/calibration.py +++ b/scos_actions/signal_processing/calibration.py @@ -59,12 +59,13 @@ def y_factor( logger.debug(f"Y (linear calc): {y}") logger.debug(f"Y (dB calc): {y_dBcalc}") noise_factor = enr_linear / (y - 1.0) - nf_dbcalc = convert_linear_to_dB(enr_linear / (y_dBcalc - 1.0)) + noise_factor_dbcalc = enr_linear / (y_dBcalc - 1.0) + nf_dbcalc = convert_linear_to_dB(noise_factor_dbcalc) logger.debug(f"Noise figure (dB calc): {nf_dbcalc}") gain_watts = pwr_noise_on_watts / ( Boltzmann * temp_kelvins * enbw_hz * (enr_linear + noise_factor) ) - gain_dbcalc = mean_on_dBm - convert_watts_to_dBm(Boltzmann * temp_kelvins * enbw_hz * (enr_linear + noise_factor)) + gain_dbcalc = mean_on_dBm - convert_watts_to_dBm(Boltzmann * temp_kelvins * enbw_hz * (enr_linear + noise_factor_dbcalc)) logger.debug(f"Gain (dB calc): {gain_dbcalc}") # Get mean values from arrays and convert to dB noise_figure = convert_linear_to_dB(np.mean(noise_factor)) From dbaa8bb0c72a72a2f4692dd2cd97e1177742cc14 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 4 Aug 2022 11:17:57 -0600 Subject: [PATCH 092/157] Remove old y-factor code --- scos_actions/signal_processing/calibration.py | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py index 36bb694a..c18699a4 100644 --- a/scos_actions/signal_processing/calibration.py +++ b/scos_actions/signal_processing/calibration.py @@ -47,30 +47,25 @@ def y_factor( :return: A tuple (noise_figure, gain) containing the calculated noise figure and gain, both in dB, from the Y-factor method. """ - mean_on_dBm = convert_watts_to_dBm(np.mean(pwr_noise_on_watts)) - mean_off_dBm = convert_watts_to_dBm(np.mean(pwr_noise_off_watts)) + # mean_on_dBm = convert_watts_to_dBm(np.mean(pwr_noise_on_watts)) + # mean_off_dBm = convert_watts_to_dBm(np.mean(pwr_noise_off_watts)) + # Testing averaging before vs. after y-factor + # This version: average after + mean_on_dBm = convert_watts_to_dBm(pwr_noise_on_watts) + mean_off_dBm = convert_watts_to_dBm(pwr_noise_off_watts) if logger.isEnabledFor(logging.DEBUG): logger.debug(f"ENR: {convert_linear_to_dB(enr_linear)} dB") logger.debug(f"ENBW: {enbw_hz} Hz") logger.debug(f"Mean power on: {mean_on_dBm:.2f} dBm") logger.debug(f"Mean power off: {mean_off_dBm:.2f} dBm") - y = pwr_noise_on_watts / pwr_noise_off_watts - y_dBcalc = convert_dB_to_linear(mean_on_dBm - mean_off_dBm) - logger.debug(f"Y (linear calc): {y}") - logger.debug(f"Y (dB calc): {y_dBcalc}") + # y = pwr_noise_on_watts / pwr_noise_off_watts + y = convert_dB_to_linear(mean_on_dBm - mean_off_dBm) noise_factor = enr_linear / (y - 1.0) - noise_factor_dbcalc = enr_linear / (y_dBcalc - 1.0) - nf_dbcalc = convert_linear_to_dB(noise_factor_dbcalc) - logger.debug(f"Noise figure (dB calc): {nf_dbcalc}") - gain_watts = pwr_noise_on_watts / ( - Boltzmann * temp_kelvins * enbw_hz * (enr_linear + noise_factor) - ) - gain_dbcalc = mean_on_dBm - convert_watts_to_dBm(Boltzmann * temp_kelvins * enbw_hz * (enr_linear + noise_factor_dbcalc)) - logger.debug(f"Gain (dB calc): {gain_dbcalc}") + gain_dB = convert_watts_to_dBm(np.mean(pwr_noise_on_watts)) - convert_watts_to_dBm(Boltzmann * temp_kelvins * enbw_hz * (enr_linear + noise_factor)) # Get mean values from arrays and convert to dB noise_figure = convert_linear_to_dB(np.mean(noise_factor)) - gain = convert_linear_to_dB(np.mean(gain_watts)) - return noise_figure, gain + # gain = convert_linear_to_dB(np.mean(gain_dB)) + return noise_figure, gain_dB def get_linear_enr(cal_source_idx: int = None) -> float: From 89607c625f0c0701666f64a497796c99bd1859f8 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 4 Aug 2022 16:17:05 -0600 Subject: [PATCH 093/157] Add shields matching scos-tekrsa --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index a40fef0a..309bab78 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # NTIA/ITS SCOS Actions Plugin +![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/NTIA/scos-actions?display_name=tag&sort=semver) +![GitHub all releases](https://img.shields.io/github/downloads/NTIA/scos-actions/total) +![GitHub issues](https://img.shields.io/github/issues/NTIA/scos-actions) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) This repository contains common actions and interfaces to be re-used by scos-sensor From d4a83e98f1277c3874c811cb65413c96e7c86067 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 4 Aug 2022 17:44:57 -0600 Subject: [PATCH 094/157] Add pip-tools dev requirement --- requirements-dev.in | 1 + requirements-dev.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-dev.in b/requirements-dev.in index baeada75..02662e27 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,5 +1,6 @@ -rrequirements.txt +pip-tools>=6.6.2, <7.0 pre-commit>=2.0, <3.0 pytest>=7.0, <8.0 tox>=3.0, <=4.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index af59dbed..c6691867 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -61,6 +61,8 @@ packaging==21.3 # numexpr # pytest # tox +pip-tools==6.8.0 + # via -r requirements-dev.in platformdirs==2.5.1 # via virtualenv pluggy==1.0.0 From 8b5f2e24ce76f973278af6005f8e0ab9638b65bc Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 4 Aug 2022 18:13:57 -0600 Subject: [PATCH 095/157] README cleanup --- README.md | 302 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 180 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 309bab78..86739294 100644 --- a/README.md +++ b/README.md @@ -5,20 +5,20 @@ ![GitHub issues](https://img.shields.io/github/issues/NTIA/scos-actions) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -This repository contains common actions and interfaces to be re-used by scos-sensor -plugins. See the [scos-sensor README]( +This repository contains common actions and interfaces to be re-used by SCOS Sensor +plugins. See the [SCOS Sensor documentation]( https://github.com/NTIA/scos-sensor/blob/master/README.md) -for more information about scos-sensor, especially the [Architecture]( +for more information about SCOS Sensor, especially the [Architecture]( https://github.com/NTIA/scos-sensor/blob/master/README.md#architecture ) and the [Actions and Hardware Support]( https://github.com/NTIA/scos-sensor/blob/master/README.md#actions-and-hardware-support -) sections which explain how scos-actions is used in the scos-sensor plugin +) sections which explain how SCOS Actions is used in the SCOS plugin architecture. ## Table of Contents - [Overview of Repo Structure](#overview-of-repo-structure) -- [Running in scos-sensor](#running-in-scos-sensor) +- [Running in SCOS Sensor](#running-in-scos-sensor) - [Development](#development) - [License](#license) - [Contact](#contact) @@ -48,27 +48,89 @@ architecture. - `scos_actions/signal_processing`: This contains various common signal processing routines which are used in actions. -## Running in scos-sensor - -Requires `pip>=18.1` (upgrade using `python3 -m pip install --upgrade pip`) and -`python>=3.7`. - -1. Clone scos-sensor: git clone -1. Navigate to scos-sensor: `cd scos-sensor` -1. In scos-sensor/src/requirements.txt, comment out the following line: - `scos_usrp @ git+ https://github.com/NTIA/scos-usrp@master#egg=scos_usrp` -1. Make sure `scos_actions` dependency is added to `scos-sensor/src/requirements.txt`. - If you are using a different branch than master, change master in the following line - to the branch you are using: - `scos_actions @ git+https://github.com/NTIA/scos-actions@master#egg=scos_actions` -1. If it does not exist, create env file while in the root scos-sensor directory: - `cp env.template ./env` -1. In env file, change `BASE_IMAGE=ubuntu:18.04` (at the bottom of the file) -1. Set `MOCK_SIGAN` and `MOCK_SIGAN_RANDOM` equal to 1 in `docker-compose.yml` -1. Get environment variables: `source ./env` -1. Build and start containers: `docker-compose up -d --build --force-recreate` - -If scos-actions is installed to scos-sensor as a plugin, the following +## Running in SCOS Sensor + +Requires `git`, `python>=3.7`, `pip>=18.1`, and `pip-tools>=6.6.2`. + +1. Clone `scos-sensor`: + + ```bash + git clone https://github.com/NTIA/scos-sensor.git + ``` + +1. Navigate to the cloned `scos-sensor` directory: + + ```bash + cd scos-sensor + ``` + +1. If testing locally, generate the necessary SSL certificates by running: + + ```bash + cd scripts && ./create_localhost_cert.sh + ``` + +1. While in the `scos-sensor` directory, create the `env` file by copying the template file: + + ```bash + cp env.template ./env + ``` + +1. In the newly-created `env` file, set the `BASE_IMAGE`: + + ```bash + BASE_IMAGE=ubuntu:18.04 + ``` + +1. Get environment variables: + + ```bash + source ./env + ``` + +1. In `scos-sensor/src/requirements.in`, comment out any unnecessary dependencies (such +as `scos_usrp`). + +1. Make sure the `scos_actions` dependency is present in +`scos-sensor/src/requirements.in`, and add it if needed. If you are using a different +branch than shown in `requirements.in`, edit the file to point SCOS Sensor to the correct +branch of SCOS Actions. As an example, the following line in `requirements.in` would use +SCOS Actions v2.0.0: + + ```text + scos_actions @ git+https://github.com/NTIA/scos-actions@2.0.0 + ``` + +1. Compile requirements by running: + + ```bash + cd src + pip-compile requirements.in + pip-compile requirements-dev.in + ``` + +1. Set `MOCK_SIGAN` and `MOCK_SIGAN_RANDOM` equal to 1 in `docker-compose.yml`: + + ```yaml + services: + ... + api: + ... + environment: + ... + - MOCK_SIGAN=1 + - MOCK_SIGAN_RANDOM=1 + ``` + +1. Build and start containers (and optionally, view logs): + + ```bash + docker-compose build --no-cache + docker-compose up -d --force-recreate + docker-compose logs -f + ``` + +If SCOS Actions is installed to SCOS Sensor as a plugin, the following parameterized actions are offered for testing using a mock signal analyzer; their parameters are defined in `scos_actions/configs/actions`. @@ -80,52 +142,46 @@ parameters are defined in `scos_actions/configs/actions`. ## Development -This repository is intended to be used by all scos-sensor plugins. Therefore, only +This repository is intended to be used by all SCOS Sensor plugins. Therefore, only universal actions that apply to most RF measurement systems should be added to -scos-actions. Custom actions for specific hardware should be added to plugins in -repositories supporting that specific hardware. New functionality could be added to the -[signal analyzer interface defined in this repository](scos_actions/hardware/sigan_iface.py) -if it is something that can be supported by most signal analyzers. +SCOS Actions. Custom actions for specific hardware should be added to plugins in +repositories supporting that specific hardware. New functionality should only be +added to the [signal analyzer interface defined in this repository](scos_actions/hardware/sigan_iface.py) +if the new functionality can be supported by most signal analyzers. ### Requirements and Configuration -Requires pip>=18.1 (upgrade using `python3 -m pip install --upgrade pip`) and -python>=3.7. +Requires `pip>=18.1` and `python>=3.7`. It is highly recommended that you first initialize a virtual development environment -using a tool such a `conda` or `venv`. The following commands create a virtual -environment using `venv` and install the required dependencies for development and -testing. +using a tool such as [Conda](https://docs.conda.io/en/latest/) or [venv](https://docs.python.org/3/library/venv.html#module-venv). +The following commands create a virtual environment using venv and install the required +dependencies for development and testing. ```bash -python3 -m venv ./venv -source venv/bin/activate -python3 -m pip install --upgrade pip # upgrade to pip>=18.1 -python3 -m pip install -r requirements-dev.txt +python -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip # upgrade to pip>=18.1 +python -m pip install -r requirements.txt ``` -#### Using `pip-tools` +#### Using pip-tools It is recommended to keep direct dependencies in a separate file. The direct -dependencies are in the `requirements.in` and `requirements-dev.in` files. Then `pip-tools` +dependencies are in the `requirements.in` and `requirements-dev.in` files. Then pip-tools can be used to generate files with all the dependencies and transitive dependencies -(sub-dependencies). The files containing all the dependencies are in `requirements.txt` -and `requirements-dev.txt`. Run the following in the virtual environment to install -`pip-tools`. +(sub-dependencies). The files containing all the dependencies are in `requirements.txt` and +`requirements-dev.txt`. Run the following in the virtual environment to install pip-tools: ```bash python -m pip install pip-tools ``` -To update `requirements.txt` after modifying `requirements.in`: +To update `requirements.txt` and `requirements-dev.txt` after modifying `requirements.in` +or `requirements-dev.in`: ```bash pip-compile requirements.in -``` - -To update `requirements-dev.txt` after modifying `requirements.in` or `requirements-dev.in`: - -```bash pip-compile requirements-dev.in ``` @@ -135,7 +191,7 @@ Use `pip-sync` to match virtual environment to `requirements-dev.txt`: pip-sync requirements.txt requirements-dev.txt ``` -For more information about `pip-tools`, see +For more information, see [pip-tools' documentation](https://pip-tools.readthedocs.io/en/latest). ### Running Tests @@ -163,50 +219,45 @@ tox -e coverage # check where test coverage lacks ### Committing Besides running the test suite and ensuring that all tests are passed, we also expect -all Python code that is checked in to have been run through an auto-formatter. - -This project uses a Python auto-formatter called Black. Additionally, import statement -sorting is handled by isort. +all Python code that's checked in to have been run through an auto-formatter. This project +uses a Python auto-formatter called [Black](https://github.com/psf/black). Additionally, +import statement sorting is handled by [isort](https://github.com/pycqa/isort). -There are several ways to autoformat your code before committing. First, IDE -integration with on-save hooks is very useful. Second, if you've already pip-installed -the dev requirements from the section above, you already have a utility called -`pre-commit` installed that will automate setting up this project's git pre-commit -hooks. Simply type the following *once*, and each time you make a commit, it will be -appropriately autoformatted. +There are several ways to auto-format your code before committing. First, IDE integration +with on-save hooks is very useful. Second, if you already pip-installed the development +requirements from the section above, you already have a utility called pre-commit that +will automate setting up this project's pre-commit Git hooks. Simply type the following +*once*, and each time you make a commit, it will be appropriately auto-formatted. ```bash pre-commit install ``` -You can manually run the pre-commit hooks using the following command. +You can also manually run the pre-commit hooks on the entire project: ```bash pre-commit run --all-files ``` -In addition to Black and isort, various other pre-commit tools are enabled including -markdownlint. Markdownlint will show an error message if it detects any style issues in -markdown files. See [.pre-commit-config.yaml](.pre-commit-config.yaml) for a list of -pre-commit tools enabled for this repository. +In addition to Black and isort, various other pre-commit tools are enabled including [markdownlint](https://github.com/DavidAnson/markdownlint). +See [`.pre-commit-config.yaml`](.pre-commit-config.yaml) for the list of pre-commit +tools enabled for this repository. ### Adding Actions -To expose a new action to the API, check out the available [action classes]( - scos_actions/actions/__init__.py). An *action* is a parameterized implementation of -an action class. If an existing class covers your needs, you can simply create yaml -configs and use the `init` method in `scos_actions.discover` to make these actions -available. +To expose a new action to the API, check out the available +[action classes](scos_actions/actions/__init__.py). An *action* is a parameterized +implementation of an action class. If an existing class covers your needs, you can +simply create YAML configs and use the `init` method in +[`scos_actions.discover`](scos_actions/discover/__init__.py) to make these actions available. ```python from scos_actions.discover import init from scos_usrp.hardware import gps, sigan actions = { - -"monitor_usrp": MonitorSignalAnalyzer(sigan), -"sync_gps": SyncGps(gps), - + "monitor_usrp": MonitorSignalAnalyzer(sigan), + "sync_gps": SyncGps(gps), } yaml_actions, yaml_test_actions = init(sigan=sigan, yaml_dir=ACTION_DEFINITIONS_DIR) @@ -215,21 +266,21 @@ actions.update(yaml_actions) ``` Pass the implementation of the signal analyzer interface and the directory where the -yaml files are located to the `init` method. +YAML files are located to the `init` method. If no existing action class meets your needs, see [Writing Custom Actions]( #writing-custom-actions). -#### Creating a yaml config file for an action +#### Creating a YAML config file for an action Actions can be manually initialized in `discover/__init__.py`, but an easier method for -non-developers and configuration-management software is to place a yaml file in the +non-developers and configuration-management software is to place a YAML file in the `configs/actions` directory which contains the action class name and parameter definitions. -The file name can be anything. Files must end in .yml. +The file name can be anything. File extensions must be `.yml`. -The action initialization logic parses all yaml files in this directory and registers +The action initialization logic parses all YAML files in this directory and registers the requested actions in the API. Let's look at an example. @@ -238,7 +289,7 @@ Let's look at an example. Let's say we want to make an instance of the `SingleFrequencyFftAcquisition`. -First, create a new yaml file in the +First, create a new YAML file in the `scos_actions/configs/actions` directory. In this example we're going to create an acquisition for the LTE 700 C band downlink, so we'll call it `acquire_700c_dl.yml`. @@ -270,19 +321,25 @@ up [actions/acquire_single_freq_fft.py]( the class to see what parameters are available and what units to use, etc. ```python -class SingleFrequencyFftAcquisition(SingleFrequencyTimeDomainIqAcquisition): - """Perform m4s detection over requested number of single-frequency FFTs. +class SingleFrequencyFftAcquisition(MeasurementAction): + """Perform M4S detection over requested number of single-frequency FFTs. - :param parameters: The dictionary of parameters needed for the action and the - signal analyzer. - - The action will set any matching attributes found in the signal analyzer object. - The following parameters are required by the action: + The action will set any matching attributes found in the signal + analyzer object. The following parameters are required by the action: name: name of the action frequency: center frequency in Hz fft_size: number of points in FFT (some 2^n) nffts: number of consecutive FFTs to pass to detector + + For the parameters required by the signal analyzer, see the + documentation from the Python package for the signal analyzer being + used. + + :param parameters: The dictionary of parameters needed for the + action and the signal analyzer. + :param sigan: Instance of SignalAnalyzerInterface. + """ ``` Then look at the docstring for the signal analyzer class being used. This example will @@ -301,10 +358,10 @@ class MockSignalAnalyzer(SignalAnalyzerInterface): """ ``` -Lastly, simply modify the yaml file to define any required parameters from the action -and signal analyzer. Note that the sigan parameter is a special parameter that will get -passed in separately when the action is initialized from the yaml. Therefore, it does -not need to be defined in the yaml file. +Lastly, simply modify the YAML file to define any required parameters from the action +and signal analyzer. Note that the `sigan` parameter is a special parameter that will get +passed in separately when the action is initialized from the YAML. Therefore, it does +not need to be defined in the YAML file. ```yaml # File: acquire_700c_dl.yml @@ -322,29 +379,30 @@ You're done. #### Writing Custom Actions -"Actions" are one of the main concepts used by [scos-sensor]( +"Actions" are one of the main concepts used by [SCOS Sensor]( https://github.com/NTIA/scos-sensor). At a high level, they are the things that the sensor owner wants the sensor to be able to *do*. At a lower level, they are simply Python classes with a special method `__call__`. Actions use [Django Signals]( https://docs.djangoproject.com/en/3.1/topics/signals/) to provide data and results to scos-sensor. -Start by looking at the [Action base class](scos_actions/actions/interfaces/action.py). +Start by looking at the [`Action` base class](scos_actions/actions/interfaces/action.py). It includes some logic to parse a description and summary out of the action class's docstring, and a `__call__` method that must be overridden. A new custom action can inherit from the existing action classes to reuse and build -upon existing functionality. For example, the `SingleFrequencyFftAcquisition` and -`SteppedFrequencyTimeDomainIqAcquisition` classes inherit from the -`SingleFrequencyTimeDomainIqAcquisition` class. +upon existing functionality. A [`MeasurementAction` base class](scos_actions/actions/interfaces/measurement_action.py), +which inherits from the `Action` class, is also useful for building new actions. +For example, [`SingleFrequencyTimeDomainIqAcquisition`](scos_actions/actions/acquire_single_freq_tdomain_iq.py) +inherits from `MeasurementAction`, while [`SteppedFrequencyTimeDomainIqAcquisition`](scos_actions/actions/acquire_stepped_freq_tdomain_iq.py) +inherits from `SingleFrequencyTimeDomainIqAcquisition`. Depending on the type of action, a signal should be sent upon action completion. This -enables scos-sensor to do something with the results of the action. This could range -from storing measurement data to recycling a docker container or to fixing an unhealthy +enables SCOS Sensor to do something with the results of the action. This could range +from storing measurement data to recycling a Docker container or to fixing an unhealthy connection to the signal analyzer. You can see the available signals in -[scos_actions/actions/interfaces/signals.py]( - scos_actions/actions/interfaces/signals.py). The following signals are currently -offered: +[`scos_actions/actions/interfaces/signals.py`](scos_actions/actions/interfaces/signals.py). +The following signals are currently offered: - `measurement_action_completed` - signal expects task_id, data, and metadata - `location_action_completed` - signal expects latitude and longitude @@ -354,22 +412,22 @@ is healthy New signals can be added. However, corresponding signal handlers must be added to scos-sensor to receive the signals and process the results. -##### Adding custom action to scos-actions +##### Adding custom action to SCOS Actions -A custom action meant to be re-used by other plugins can live in scos-actions. It can -be instantiated using a yaml file, or directly in the `actions` dictionary in the -`discover/__init__.py` module. This can be done in scos-actions with a mock signal +A custom action meant to be re-used by other plugins can live in SCOS Actions. It can +be instantiated using a YAML file, or directly in the `actions` dictionary in the +`discover/__init__.py` module. This can be done in SCOS Actions with a mock signal analyzer. Plugins supporting other hardware would need to import the action from -scos-actions. Then it can be instantiated in that plugin’s actions dictionary in its -discover module, or in a yaml file living in that plugin (as long as its discover -module includes the required code to parse the yaml files). +SCOS Actions. Then it can be instantiated in that plugin’s actions dictionary in its +discover module, or in a YAML file living in that plugin (as long as its discover +module includes the required code to parse the YAML files). ##### Adding system or hardware specific custom action In the repository that provides the plugin to support the hardware being used, add the action to the `actions` dictionary in the `discover/__init__.py` file. Optionally, -initialize the action using a yaml file by importing the yaml initialization code from -scos-actions. For an example of this, see [Adding Actions subsection](#adding-actions) +initialize the action using a YAML file by importing the YAML initialization code from +SCOS Actions. For an example of this, see the [Adding Actions subsection](#adding-actions) above. ### Supporting a Different Signal Analyzer @@ -380,28 +438,28 @@ another signal analyzer with a Python API. - Create a new repository called `scos-[signal analyzer name]`. - Create a new virtual environment and activate it: - `python3 -m venv ./venv && source venv/bin/activate`. - Upgrade pip: `python3 -m pip install --upgrade pip`. -- In the new repository, add this scos-actions repository as a dependency and create a + `python -m venv ./venv && source venv/bin/activate`. + Upgrade pip: `python -m pip install --upgrade pip`. +- In the new repository, add this repository as a dependency and create a class that inherits from the [SignalAnalyzerInterface](scos_actions/hardware/sigan_iface.py) abstract class. Add properties or class variables for the parameters needed to configure the signal analyzer. -- Create .yml files with the parameters needed to run the actions imported from +- Create YAML files with the parameters needed to run the actions imported from scos-actions using the new signal analyzer. Put them in the new repository in `configs/actions`. This should contain the parameters needed by the action as well as the signal analyzer settings based on which properties or class variables were implemented in the signal analyzer class in the previous step. The measurement actions - in scos-actions are configured to check if any yaml parameters are available as - attributes in the signal analyzer object, and to set them to the given yaml value if + in SCOS Actions are configured to check if any YAML parameters are available as + attributes in the signal analyzer object, and to set them to the given YAML value if available. For example, if the new signal analyzer class has a bandwidth property, - simply add a bandwidth parameter to the yaml file. Alternatively, you can create + simply add a bandwidth parameter to the YAML file. Alternatively, you can create custom actions that are unique to the hardware. See [Adding Actions](#adding-actions) subsection above. - In the new repository, add a `discover/__init__.py` file. This should contain a dictionary called `actions` with a key of action name and a value of action object. You can use the [init()](scos_actions/discover/__init__.py) and/or the [load_from_yaml()](scos_actions/discover/yaml.py) methods provided in this repository - to look for yaml files and initialize actions. These methods allow you to pass your + to look for YAML files and initialize actions. These methods allow you to pass your new signal analyzer object to the action's constructor. You can use the existing action classes [defined in this repository](scos_actions/actions/__init__.py) or [create custom actions](#writing-custom-actions). If the signal analyzer supports @@ -425,10 +483,10 @@ The final step would be to add a `setup.py` to allow for installation of the new repository as a Python package. You can use the [setup.py](setup.py) in this repository as a reference. You can find more information about Python packaging [here]( https://packaging.python.org/tutorials/packaging-projects/). Then add the new -repository as a dependency to [scos-sensor requirements.txt]( +repository as a dependency to [SCOS Sensor's requirements.txt]( https://github.com/NTIA/scos-sensor/blob/master/src/requirements.txt) using the following format: -` @ git+@#egg=`. If +` @ git+@`. If specific drivers are required for your signal analyzer, you can attempt to link to them within the package or create a docker image with the necessary files. You can host the docker image as a [GitHub package]( From b9912c584305b91428cd4e4a0578427c82244e88 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 5 Aug 2022 09:52:12 -0600 Subject: [PATCH 096/157] Remove possibly ambiguous windowing behavior --- scos_actions/signal_processing/fft.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/scos_actions/signal_processing/fft.py b/scos_actions/signal_processing/fft.py index 6e5e0944..a5390b13 100644 --- a/scos_actions/signal_processing/fft.py +++ b/scos_actions/signal_processing/fft.py @@ -104,9 +104,6 @@ def get_fft_window(window_type: str, window_length: int) -> np.ndarray: bartlett, flattop, parzen, bohman, blackmanharris, nuttall, barthann, cosine, exponential, tukey, and taylor. - If an invalid window type is specified, a boxcar (rectangular) - window will be used instead. - :param window_type: A string supported by scipy.signal.get_window. Only windows which do not require additional parameters are supported. Whitespace and capitalization are ignored. @@ -122,14 +119,7 @@ def get_fft_window(window_type: str, window_length: int) -> np.ndarray: window_type = "hann" # Get window samples - try: - window = get_window(window_type, window_length) - except ValueError: - logger.debug( - "Error generating FFT window. Attempting to" - + " use a rectangular window..." - ) - window = get_window("boxcar", window_length) + window = get_window(window_type, window_length, fftbins=True) # Return the window return window From 48f1e9a5aacf53ce8b55fefd907a4f9d0e797a44 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 5 Aug 2022 14:11:24 -0600 Subject: [PATCH 097/157] Update interface for better reuse --- scos_actions/hardware/sigan_iface.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py index 0d17a88e..0ad16083 100644 --- a/scos_actions/hardware/sigan_iface.py +++ b/scos_actions/hardware/sigan_iface.py @@ -1,5 +1,6 @@ import copy from abc import ABC, abstractmethod +from scos_actions.calibration import Calibration from scos_actions.settings import sensor_calibration from scos_actions.settings import sigan_calibration from scos_actions.utils import convert_string_to_millisecond_iso_format @@ -32,7 +33,7 @@ def __init__(self): self.sigan_calibration_data = copy.deepcopy(self.DEFAULT_SIGAN_CALIBRATION) @property - def last_calibration_time(self): + def last_calibration_time(self) -> str: """Returns the last calibration time from calibration data.""" return convert_string_to_millisecond_iso_format( sensor_calibration.calibration_datetime @@ -40,34 +41,32 @@ def last_calibration_time(self): @property @abstractmethod - def is_available(self): + def is_available(self) -> bool: pass @abstractmethod def acquire_time_domain_samples( - self, num_samples, num_samples_skip=0, retries=5, gain_adjust=True + self, num_samples: int, num_samples_skip: int = 0, retries: int = 5, gain_adjust: bool = True ) -> dict: - """Acquires time domain IQ samples - :type num_samples: integer - :param num_samples: Number of samples to acquire + """ + Acquire time domain IQ samples - :type num_samples_skip: integer + :param num_samples: Number of samples to acquire :param num_samples_skip: Number of samples to skip - - :type retries: integer :param retries: Maximum number of retries on failure - - :rtype: dictionary containing data, sample_rate, frequency, capture_time, etc + :param gain_adjust: If True, scale IQ samples based on calibration data. + :return: dictionary containing data, sample_rate, frequency, capture_time, etc """ pass @property @abstractmethod - def healthy(self): + def healthy(self) -> bool: + """Perform a health check by collecting IQ samples.""" pass - def recompute_calibration_data(self, cal_args): - """Set the calibration data based on the currently tuning""" + def recompute_calibration_data(self, cal_args: Calibration) -> None: + """Set the calibration data based on the current tuning""" # Try and get the sensor calibration data self.sensor_calibration_data = self.DEFAULT_SENSOR_CALIBRATION.copy() From a3ed1a58c7ea55742c25421a56215eb2f8a20c3d Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 5 Aug 2022 14:13:31 -0600 Subject: [PATCH 098/157] Comment abstract is_available method --- scos_actions/hardware/sigan_iface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py index 0ad16083..8b41ced7 100644 --- a/scos_actions/hardware/sigan_iface.py +++ b/scos_actions/hardware/sigan_iface.py @@ -42,6 +42,7 @@ def last_calibration_time(self) -> str: @property @abstractmethod def is_available(self) -> bool: + """Returns True if sigan is initialized and ready for measurements.""" pass @abstractmethod From f853a77403f789929b20634f41977ee921897848 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Sat, 6 Aug 2022 11:19:29 -0600 Subject: [PATCH 099/157] Create pyproject.toml --- pyproject.toml | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..7ffd6068 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,75 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "scos_actions" +dynamic = ["version"] +description = "Base plugin providing common actions and interfaces to be re-used by SCOS Sensor plugins." +readme = "README.md" +requires-python = ">=3.8" +license = { file = "LICENSE.md" } + +authors = [ + { name = "The Institute for Telecommunication Sciences" }, + { name="Anthony Romaniello", email="aromaniello@ntia.gov"} +] + +keywords = [ + "scos", "sdr", "spectrum-analyzer", "spectrum", + "analyzer", "spectrum analyzer", "scos-sensor", "scos sensor", + "spectrum monitoring", "monitoring", "spectrum management", "docker", + "linux", "software defined radio", "radio" +] + +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Intended Audience :: Telecommunications Industry", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Environment :: Plugins", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", +] + +dependencies = [ + "django>3.2.14,<4.0", + # its_preselector @ ntia/preselector@1.0.0 not included + "numexpr>=2.8.3,<3.0", + "numpy>=1.22.0,<2.0", + "python-dateutil>=2.0,<3.0", + "ruamel.yaml>=0.15,<1.0", + "scipy>=1.6.0,<2.0", + # ntia/sigmf@multi-recording-archive not included +] + +[project.optional-dependencies] +dev = [ + "pre-commit>=2.20.0,<3.0", + "pytest>=7.1.2,<8.0", + "pytest-cov>=3.0.0,<4.0", + "tox>=3.0,<4.0", + "twine>=4.0.1,<5.0", +] + +[project.urls] +"Repository" = "https://github.com/NTIA/scos-actions" +"Bug Tracker" = "https://github.com/NTIA/scos-actions/issues" +"SCOS Sensor" = "https://github.com/NTIA/scos-sensor" +"NTIA GitHub" = "https://github.com/NTIA" +"ITS Website" = "https://its.ntia.gov" + +[tool.hatch.version] +path = "src/scos_actions/__about__.py" + +[tool.hatch.build] +skip-excluded-dirs = true + +[tool.hatch.build.targets.wheel] +packages = ["src/scos_actions"] + +[tool.hatch.build.targets.sdist] From 4fcda017571b5cd75160e853faa57298dc7fce5c Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Sat, 6 Aug 2022 11:21:30 -0600 Subject: [PATCH 100/157] Update minimum python to 3.8 --- .pre-commit-config.yaml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 67568c2e..f6d622b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ default_language_version: - python: python3.7 + python: python3.8 repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.1.0 diff --git a/tox.ini b/tox.ini index 3b17e86b..8e900618 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37,py38,py39,py310 +envlist = py38,py39,py310 skip_missing_interpreters = True skipsdist = True From 87d2042da63509e44cf6a1e8ca525af82cac09db Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Sat, 6 Aug 2022 11:21:45 -0600 Subject: [PATCH 101/157] Add dynamic versioning --- scos_actions/__about__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 scos_actions/__about__.py diff --git a/scos_actions/__about__.py b/scos_actions/__about__.py new file mode 100644 index 00000000..21014090 --- /dev/null +++ b/scos_actions/__about__.py @@ -0,0 +1 @@ +VERSION = "2.0.0" From 61ef6c336ce4379f5d5a59d93cb1e1d267c02c80 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Sat, 6 Aug 2022 11:21:53 -0600 Subject: [PATCH 102/157] Update file paths --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7ffd6068..ff84e0fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,12 +64,12 @@ dev = [ "ITS Website" = "https://its.ntia.gov" [tool.hatch.version] -path = "src/scos_actions/__about__.py" +path = "scos_actions/__about__.py" [tool.hatch.build] skip-excluded-dirs = true [tool.hatch.build.targets.wheel] -packages = ["src/scos_actions"] +packages = ["scos_actions"] [tool.hatch.build.targets.sdist] From 95846bd8252fc5d8a616dcd7ae4b8d6c494fd1ee Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Sat, 6 Aug 2022 11:21:56 -0600 Subject: [PATCH 103/157] Delete setup.py --- setup.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index fdebfe51..00000000 --- a/setup.py +++ /dev/null @@ -1,35 +0,0 @@ -import os - -import setuptools - -with open("README.md", "r", encoding="utf8") as fh: - long_description = fh.read() - - -repo_root = os.path.dirname(os.path.realpath(__file__)) -requirements_path = repo_root + "/requirements.in" -install_requires = [] # Examples: ["gunicorn", "docutils>=0.3", "lxml==0.5a7"] -if os.path.isfile(requirements_path): - with open(requirements_path) as f: - install_requires = f.read().splitlines() - -setuptools.setup( - name="scos_actions", - version="2.0.0", - author="The Institute for Telecommunication Sciences", - # author_email="author@example.com", - description="Base actions and hardware support library for scos-sensor", - license="LICENSE.md", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/NTIA/scos-actions", - packages=setuptools.find_packages(), - classifiers=[ - "Programming Language :: Python :: 3", - "License :: Public Domain", - "Operating System :: OS Independent", - ], - python_requires=">=3.7", - install_requires=install_requires, - package_data={"scos_actions": ["configs/actions/*.yml", "configs/*.json"]}, -) From 85e6b762e4217b51bbe7fc993c76d4df42d7c817 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 8 Aug 2022 16:01:26 -0600 Subject: [PATCH 104/157] Remove debug messages for testing --- scos_actions/signal_processing/calibration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py index c18699a4..9c84f595 100644 --- a/scos_actions/signal_processing/calibration.py +++ b/scos_actions/signal_processing/calibration.py @@ -56,8 +56,8 @@ def y_factor( if logger.isEnabledFor(logging.DEBUG): logger.debug(f"ENR: {convert_linear_to_dB(enr_linear)} dB") logger.debug(f"ENBW: {enbw_hz} Hz") - logger.debug(f"Mean power on: {mean_on_dBm:.2f} dBm") - logger.debug(f"Mean power off: {mean_off_dBm:.2f} dBm") + # logger.debug(f"Mean power on: {mean_on_dBm:.2f} dBm") + # logger.debug(f"Mean power off: {mean_off_dBm:.2f} dBm") # y = pwr_noise_on_watts / pwr_noise_off_watts y = convert_dB_to_linear(mean_on_dBm - mean_off_dBm) noise_factor = enr_linear / (y - 1.0) From 5d71d1143975674a9266c977559dff7ea460e3f0 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 8 Aug 2022 16:05:57 -0600 Subject: [PATCH 105/157] Move averaging for testing --- scos_actions/signal_processing/calibration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py index 9c84f595..eccdce86 100644 --- a/scos_actions/signal_processing/calibration.py +++ b/scos_actions/signal_processing/calibration.py @@ -60,10 +60,10 @@ def y_factor( # logger.debug(f"Mean power off: {mean_off_dBm:.2f} dBm") # y = pwr_noise_on_watts / pwr_noise_off_watts y = convert_dB_to_linear(mean_on_dBm - mean_off_dBm) - noise_factor = enr_linear / (y - 1.0) + noise_factor = np.mean(enr_linear / (y - 1.0)) gain_dB = convert_watts_to_dBm(np.mean(pwr_noise_on_watts)) - convert_watts_to_dBm(Boltzmann * temp_kelvins * enbw_hz * (enr_linear + noise_factor)) # Get mean values from arrays and convert to dB - noise_figure = convert_linear_to_dB(np.mean(noise_factor)) + noise_figure = convert_linear_to_dB(noise_factor) # gain = convert_linear_to_dB(np.mean(gain_dB)) return noise_figure, gain_dB From 56589e6cc13787f872e7f34ed18d977c79008d38 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 8 Aug 2022 16:15:05 -0600 Subject: [PATCH 106/157] Cleanup y-factor --- scos_actions/signal_processing/calibration.py | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py index eccdce86..96d87dc1 100644 --- a/scos_actions/signal_processing/calibration.py +++ b/scos_actions/signal_processing/calibration.py @@ -31,9 +31,9 @@ def y_factor( """ Perform Y-Factor calculations of noise figure and gain. - Noise factor and linear gain are computed element-wise from - the input arrays using the Y-Factor method. The linear values - are then averaged and converted to dB. + Noise factor and linear gain are computed from the input + arrays using the Y-Factor method. The linear values are + then averaged and converted to dB. :param pwr_noise_on_watts: Array of power values, in Watts, recorded with the calibration noise source on. @@ -47,25 +47,18 @@ def y_factor( :return: A tuple (noise_figure, gain) containing the calculated noise figure and gain, both in dB, from the Y-factor method. """ - # mean_on_dBm = convert_watts_to_dBm(np.mean(pwr_noise_on_watts)) - # mean_off_dBm = convert_watts_to_dBm(np.mean(pwr_noise_off_watts)) - # Testing averaging before vs. after y-factor - # This version: average after - mean_on_dBm = convert_watts_to_dBm(pwr_noise_on_watts) - mean_off_dBm = convert_watts_to_dBm(pwr_noise_off_watts) + mean_on_dBm = convert_watts_to_dBm(np.mean(pwr_noise_on_watts)) + mean_off_dBm = convert_watts_to_dBm(np.mean(pwr_noise_off_watts)) if logger.isEnabledFor(logging.DEBUG): logger.debug(f"ENR: {convert_linear_to_dB(enr_linear)} dB") logger.debug(f"ENBW: {enbw_hz} Hz") - # logger.debug(f"Mean power on: {mean_on_dBm:.2f} dBm") - # logger.debug(f"Mean power off: {mean_off_dBm:.2f} dBm") - # y = pwr_noise_on_watts / pwr_noise_off_watts + logger.debug(f"Mean power on: {mean_on_dBm:.2f} dBm") + logger.debug(f"Mean power off: {mean_off_dBm:.2f} dBm") y = convert_dB_to_linear(mean_on_dBm - mean_off_dBm) - noise_factor = np.mean(enr_linear / (y - 1.0)) + noise_factor = enr_linear / (y - 1.0) gain_dB = convert_watts_to_dBm(np.mean(pwr_noise_on_watts)) - convert_watts_to_dBm(Boltzmann * temp_kelvins * enbw_hz * (enr_linear + noise_factor)) - # Get mean values from arrays and convert to dB - noise_figure = convert_linear_to_dB(noise_factor) - # gain = convert_linear_to_dB(np.mean(gain_dB)) - return noise_figure, gain_dB + noise_figure_dB = convert_linear_to_dB(noise_factor) + return noise_figure_dB, gain_dB def get_linear_enr(cal_source_idx: int = None) -> float: From d01d5a7768c7e062ce414ad5936f66931ac08883 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 8 Aug 2022 16:29:03 -0600 Subject: [PATCH 107/157] Test switch to official sigmf 1.0.0 --- requirements-dev.txt | 3 ++- requirements.txt | 3 ++- scos_actions/metadata/sigmf_builder.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c6691867..cfe89639 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -101,7 +101,8 @@ ruamel-yaml-clib==0.2.6 # ruamel-yaml scipy==1.7.3 # via -r requirements.txt -sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive +# sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive +sigmf==1.0.0 # via -r requirements.txt six==1.16.0 # via diff --git a/requirements.txt b/requirements.txt index 3d11c041..9ba9bbb6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,7 +40,8 @@ ruamel-yaml-clib==0.2.6 # via ruamel-yaml scipy==1.7.3 # via -r requirements.in -sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive +sigmf==1.0.0 +# sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive # via -r requirements.in six==1.16.0 # via diff --git a/scos_actions/metadata/sigmf_builder.py b/scos_actions/metadata/sigmf_builder.py index 7011983b..e1eed6cb 100644 --- a/scos_actions/metadata/sigmf_builder.py +++ b/scos_actions/metadata/sigmf_builder.py @@ -3,7 +3,7 @@ from sigmf import SigMFFile GLOBAL_INFO = { - "core:version": "0.0.2", + "core:version": "v1.0.0", "core:extensions": { "ntia-algorithm": "v1.0.0", "ntia-core": "v1.0.0", From 18252c5d27135131a0abc971abb5611200cf5461 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 8 Aug 2022 16:48:05 -0600 Subject: [PATCH 108/157] Update requirements --- requirements-dev.txt | 2 +- requirements.in | 4 ++-- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index cfe89639..c58e498e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,7 +22,7 @@ charset-normalizer==2.0.12 # requests distlib==0.3.4 # via virtualenv -django==3.2.12 +django==3.2.15 # via -r requirements.txt filelock==3.6.0 # via diff --git a/requirements.in b/requirements.in index 06accdbc..34e089db 100644 --- a/requirements.in +++ b/requirements.in @@ -1,8 +1,8 @@ -django>=3.0, <4.0 +django>=3.2.14, <4.0 numpy>=1.0, <2.0 python-dateutil>=2.0, < 3.0 ruamel.yaml>=0.1, <1.0 scipy>=1.6.0, <2.0 numexpr>=2.8.3, <3.0 -SigMF @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive +sigmf==1.0.0 its-preselector @ git+https://github.com/NTIA/Preselector@1.0.0#egg=its-preselector diff --git a/requirements.txt b/requirements.txt index 9ba9bbb6..55fe8297 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ certifi==2021.10.8 # via requests charset-normalizer==2.0.12 # via requests -django==3.2.12 +django==3.2.15 # via -r requirements.in idna==3.3 # via requests From 0da13cee4db70dd8546d78e3d117c8e6f6c2d473 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 8 Aug 2022 18:34:38 -0600 Subject: [PATCH 109/157] Pull sensor ENBW from calibration in non-IIR case --- scos_actions/actions/calibrate_y_factor.py | 24 +++++++++++++--------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 0ed0be9b..96fbd89d 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -240,11 +240,14 @@ def calibrate(self, params): cutoff_Hz = self.iir_cutoff_Hz width_Hz = self.iir_width_Hz enbw_hz = (cutoff_Hz + width_Hz) * 2. # Roughly based on IIR filter + # TODO: Verify this is an appropriate way to specify ENBW logger.debug("Applying IIR filter to IQ captures") noise_on_data = sosfilt(iir_sos, noise_on_measurement_result["data"]) noise_off_data = sosfilt(iir_sos, noise_off_measurement_result["data"]) else: - enbw_hz = 11.607e6 # For RSA 507A #TODO REMOVE + # Get ENBW from sensor calibration + enbw_hz = sensor_calibration["enbw_sensor"] + logger.debug(f"Got sensor ENBW: {enbw_hz} Hz") logger.debug('Skipping IIR filtering') noise_on_data = noise_on_measurement_result["data"] noise_off_data = noise_off_measurement_result["data"] @@ -261,15 +264,15 @@ def calibrate(self, params): pwr_on_watts, pwr_off_watts, enr_linear, enbw_hz, temp_k ) - # Don't update the sensor calibration while testing - # sensor_calibration.update( - # params, - # utils.get_datetime_str_now(), - # gain, - # noise_figure, - # temp_c, - # SENSOR_CALIBRATION_FILE, - # ) + # Update sensor calibration with results + sensor_calibration.update( + params, + utils.get_datetime_str_now(), + gain, + noise_figure, + temp_c, + SENSOR_CALIBRATION_FILE, + ) # Debugging noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) @@ -291,6 +294,7 @@ def get_mean_power(self, iqdata: np.ndarray) -> np.ndarray: @property def description(self): + # TODO: Update # Get parameters; they may be single values or lists frequencies = get_parameter(FREQUENCY, self.parameters) # nffts = get_parameter(NUM_FFTS, self.parameters) From b168e903dfadc455a5998921aa05eac399b19697 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 8 Aug 2022 19:15:26 -0600 Subject: [PATCH 110/157] Don't allow multiple filters for multi-channel cal --- scos_actions/actions/calibrate_y_factor.py | 124 +++++++++------------ 1 file changed, 54 insertions(+), 70 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 96fbd89d..cc4331d2 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -19,10 +19,11 @@ r"""Perform a Y-Factor Calibration. Supports calibration of gain and noise figure for one or more channels. For each center frequency, sets the preselector to the noise diode path, turns -noise diode on, performs a mean FFT measurement, turns the noise diode off and -performs another mean FFT measurement. The mean power on and mean power off +noise diode on, performs a mean power measurement, turns the noise diode off and +performs another mean power measurement. The mean power on and mean power off data are used to compute the noise figure and gain. For each measurement, the -mean detector is applied over {nffts} {fft_size}-pt FFTs at {frequencies} MHz. +mean detector is applied over {num_samples} samples at {frequencies} MHz. +Mean power is calculated in the time domain{filtering_suffix}. # {name} @@ -34,32 +35,10 @@ ## Time-domain processing First, the ${nffts} \times {fft_size}$ continuous samples are acquired from -the signal analyzer. If specified, a voltage scaling factor is applied to the complex -time-domain signals. Then, the data is reshaped into a ${nffts} \times -{fft_size}$ matrix: +the signal analyzer. If specified, an IIR lowpass filter is used to filter +the complex time-domain samples before mean power calculations are performed. -$$ -\begin{{pmatrix}} -a_{{1,1}} & a_{{1,2}} & \cdots & a_{{1,fft\_size}} \\\\ -a_{{2,1}} & a_{{2,2}} & \cdots & a_{{2,fft\_size}} \\\\ -\vdots & \vdots & \ddots & \vdots \\\\ -a_{{nffts,1}} & a_{{nfts,2}} & \cdots & a_{{nfts,fft\_size}} \\\\ -\end{{pmatrix}} -$$ - -where $a_{{i,j}}$ is a complex time-domain sample. - -At that point, a Flat Top window, defined as - -$$w(n) = &0.2156 - 0.4160 \cos{{(2 \pi n / M)}} + 0.2781 \cos{{(4 \pi n / M)}} - - &0.0836 \cos{{(6 \pi n / M)}} + 0.0069 \cos{{(8 \pi n / M)}}$$ - -where $M = {fft_size}$ is the number of points in the window, is applied to -each row of the matrix. - -## Frequency-domain processing - -### To-do: add details of FFT processing +{filter_description} ## Y-Factor Method @@ -86,7 +65,6 @@ y_factor, ) from scos_actions.signal_processing.power_analysis import ( - apply_power_detector, calculate_power_watts, create_power_detector, ) @@ -128,7 +106,6 @@ class YFactorCalibration(Action): fft_size: number of points in FFT (some 2^n) nffts: number of consecutive FFTs to pass to detector - For the parameters required by the signal analyzer, see the documentation from the Python package for the signal analyzer being used. @@ -150,10 +127,11 @@ def __init__(self, parameters, sigan, gps=mock_gps): logger.info("Config parameter 'iir_apply' not provided. " + "No IIR filtering will be used during calibration.") self.iir_apply = False + + if isinstance(self.iir_apply, list): + raise ParameterException("Only one set of IIR filter parameters may be specified.") if self.iir_apply is True: - # If any parameters are multiply-specified, generate the filter - # in the calibration loop instead of here. self.iir_rp_dB = get_parameter(IIR_RP, parameters) self.iir_rs_dB = get_parameter(IIR_RS, parameters) self.iir_cutoff_Hz = get_parameter(IIR_CUTOFF, parameters) @@ -164,9 +142,8 @@ def __init__(self, parameters, sigan, gps=mock_gps): self.iir_sos = generate_elliptic_iir_low_pass_filter( self.iir_rp_dB, self.iir_rs_dB, self.iir_cutoff_Hz, self.iir_width_Hz, self.sample_rate ) - self.regenerate_iir = False else: - self.regenerate_iir = True + raise ParameterException("Only one set of IIR filter parameters may be specified (including sample rate).") def __call__(self, schedule_entry_json, task_id): """This is the entrypoint function called by the scheduler.""" @@ -195,10 +172,6 @@ def calibrate(self, params): duration_ms = get_parameter(DURATION_MS, params) num_samples = int(sample_rate * duration_ms * 1e-3) nskip = get_parameter(NUM_SKIP, params) - if self.iir_apply is not False: - iir_apply = get_parameter(IIR_APPLY, params) - else: - iir_apply = False # Set noise diode on logger.debug('Setting noise diode on') @@ -225,25 +198,14 @@ def calibrate(self, params): assert sample_rate == noise_off_measurement_result["sample_rate"], "Sample rate mismatch" # Apply IIR filtering to both captures if configured - if iir_apply: - if self.regenerate_iir: - logger.debug("Generating IIR filter") - rp_dB = get_parameter(IIR_RP, params) - rs_dB = get_parameter(IIR_RS, params) - cutoff_Hz = get_parameter(IIR_CUTOFF, params) - width_Hz = get_parameter(IIR_WIDTH, params) - iir_sos = generate_elliptic_iir_low_pass_filter( - rp_dB, rs_dB, cutoff_Hz, width_Hz, sample_rate - ) - else: - iir_sos = self.iir_sos - cutoff_Hz = self.iir_cutoff_Hz - width_Hz = self.iir_width_Hz + if self.iir_apply: + cutoff_Hz = self.iir_cutoff_Hz + width_Hz = self.iir_width_Hz enbw_hz = (cutoff_Hz + width_Hz) * 2. # Roughly based on IIR filter # TODO: Verify this is an appropriate way to specify ENBW logger.debug("Applying IIR filter to IQ captures") - noise_on_data = sosfilt(iir_sos, noise_on_measurement_result["data"]) - noise_off_data = sosfilt(iir_sos, noise_off_measurement_result["data"]) + noise_on_data = sosfilt(self.iir_sos, noise_on_measurement_result["data"]) + noise_off_data = sosfilt(self.iir_sos, noise_off_measurement_result["data"]) else: # Get ENBW from sensor calibration enbw_hz = sensor_calibration["enbw_sensor"] @@ -253,8 +215,8 @@ def calibrate(self, params): noise_off_data = noise_off_measurement_result["data"] # Get power values in time domain - pwr_on_watts = self.get_mean_power(noise_on_data) - pwr_off_watts = self.get_mean_power(noise_off_data) + pwr_on_watts = self.get_td_power(noise_on_data) + pwr_off_watts = self.get_td_power(noise_off_data) # Y-Factor enr_linear = get_linear_enr(cal_source_idx) @@ -283,33 +245,54 @@ def calibrate(self, params): # Detail results contain only FFT version of result for now return 'Noise Figure: {}, Gain: {}'.format(noise_figure, gain) - def get_mean_power(self, iqdata: np.ndarray) -> np.ndarray: + def get_td_power(self, iqdata: np.ndarray) -> np.ndarray: # Reshape data - # iq = np.reshape(iqdata[:block_size * n_blocks], (n_blocks, block_size)) iqdata /= 2 # RF/baseband conversion iq_pwr = calculate_power_watts(iqdata) - # mean_result = apply_power_detector(iq_pwr, self.power_detector) - # return mean_result return iq_pwr @property def description(self): - # TODO: Update + #TODO: provide num_samples # Get parameters; they may be single values or lists frequencies = get_parameter(FREQUENCY, self.parameters) - # nffts = get_parameter(NUM_FFTS, self.parameters) - # fft_size = get_parameter(FFT_SIZE, self.parameters) + duration_ms = get_parameter(DURATION_MS, self.parameters) + sample_rate = get_parameter(SAMPLE_RATE, self.parameters) + + if isinstance(duration_ms, list) and not isinstance(sample_rate, list): + sample_rate = sample_rate * np.ones_like(duration_ms) + sample_rate = sample_rate + elif isinstance(sample_rate, list) and not isinstance(duration_ms, list): + duration_ms = duration_ms * np.ones_like(sample_rate) + duration_ms = duration_ms + + num_samples = duration_ms * sample_rate * 1e-3 + if len(num_samples) != 1: + num_samples = num_samples.tolist() + else: + num_samples = int(num_samples) + + # TODO: generate blank if no filtering, else filter details + if self.iir_apply is True: + filtering_suffix = ", after applying an IIR lowpass filter to the complex time-domain samples" + filter_description = ( + """ + ### Filtering + Optionally, IQ samples can be filtered using an elliptic IIR filter before + performing the rest of the time-domain Y-factor calculations. The filter + design produces the lowest order digital filter which loses no more than + {gpass} + """ + ) + else: + filter_description = "No filter is applied to the input samples before Y-factor calculations" # Convert parameter lists to strings if needed if isinstance(frequencies, list): frequencies = utils.list_to_string( [f / 1e6 for f in get_parameter(FREQUENCY, self.parameters)] ) - # if isinstance(nffts, list): - # nffts = utils.list_to_string(get_parameter(NUM_FFTS, self.parameters)) - # if isinstance(fft_size, list): - # fft_size = utils.list_to_string(get_parameter(FFT_SIZE, self.parameters)) - + acq_plan = ( f"Performs a y-factor calibration at frequencies: " # f"{frequencies}, nffts:{nffts}, fft_size: {fft_size}\n" @@ -318,10 +301,11 @@ def description(self): "name": self.name, "frequencies": frequencies, "acquisition_plan": acq_plan, - # "fft_size": fft_size, - # "nffts": nffts, + "num_samples": num_samples, + } # __doc__ refers to the module docstring at the top of the file + # TODO uncomment fornat below return __doc__ #.format(**definitions) def test_required_components(self): From 0199ccd264882391b15fe13bc82fd03997457f95 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 8 Aug 2022 20:28:46 -0600 Subject: [PATCH 111/157] Add dynamic action description --- scos_actions/actions/calibrate_y_factor.py | 119 +++++++++++++-------- 1 file changed, 74 insertions(+), 45 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index cc4331d2..0a91e9c2 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -21,28 +21,51 @@ For each center frequency, sets the preselector to the noise diode path, turns noise diode on, performs a mean power measurement, turns the noise diode off and performs another mean power measurement. The mean power on and mean power off -data are used to compute the noise figure and gain. For each measurement, the -mean detector is applied over {num_samples} samples at {frequencies} MHz. -Mean power is calculated in the time domain{filtering_suffix}. +data are used to compute the noise figure and gain. Mean power is calculated in the +time domain{filtering_suffix}. # {name} ## Signal analyzer setup and sample acquisition -Each time this task runs, the following process is followed: +Each time this task runs, the following process is followed to take measurements +separately with the noise diode off and on: {acquisition_plan} ## Time-domain processing -First, the ${nffts} \times {fft_size}$ continuous samples are acquired from -the signal analyzer. If specified, an IIR lowpass filter is used to filter -the complex time-domain samples before mean power calculations are performed. +{filtering_description} -{filter_description} +Next, mean power calculations are performed. Sample amplitudes are divided by two +to account for the power difference between RF and complex baseband samples. Then, +power is calculated element-wise from the complex time-domain samples. The power of +each sample is defined by the square of the magnitude of the complex sample, divided by +the system impedance, which is taken to be 50 Ohms. ## Y-Factor Method -### To-do: add details of Y-Factor method +The mean power for the noise diode on and off captures are calculated by taking the +mean of each array of power samples. Next, the Y-factor is calculated by: + +$$ y = P_{on} / P_{off} $$ + +Where $P_{on}$ is the mean power measured with the noise diode on, and $P_{off}$ +is the mean power measured with the noise diode off. The linear noise factor is then +calculated by: + +$$ NF = \frac{ENR}{y - 1} $$ + +Where $ENR$ is the excess noise ratio, in linear units, of the noise diode used for +the power measurements. Next, the linear gain is calculated by: + +$$ G = \frac{P_{on}}{k_B T B_{eq} (ENR + NF)} $$ + +Where $k_B$ is Boltzmann's constant, $T$ is the calibration temperature in Kelvins, +and $B_{eq}$ is the sensor's equivalent noise bandwidth. Finally, the noise factor +and linear gain are converted to noise figure $F_N$ and decibel gain $G_{dB}$: + +$$ G_{dB} = 10 \log_{10}(G) $$ +$$ F_N = 10 \log_{10}(NF) $$ """ import logging @@ -148,19 +171,17 @@ def __init__(self, parameters, sigan, gps=mock_gps): def __call__(self, schedule_entry_json, task_id): """This is the entrypoint function called by the scheduler.""" self.test_required_components() - iteration_params = utils.get_iterable_parameters(self.parameters) + self.iteration_params = utils.get_iterable_parameters(self.parameters) detail = '' # Run calibration routine - for i, p in enumerate(iteration_params): + for i, p in enumerate(self.iteration_params): if i == 0: detail += self.calibrate(p) else: detail += os.linesep + self.calibrate(p) - return detail - def calibrate(self, params): # Configure signal analyzer super().configure_sigan(params) @@ -207,21 +228,20 @@ def calibrate(self, params): noise_on_data = sosfilt(self.iir_sos, noise_on_measurement_result["data"]) noise_off_data = sosfilt(self.iir_sos, noise_off_measurement_result["data"]) else: + logger.debug('Skipping IIR filtering') # Get ENBW from sensor calibration enbw_hz = sensor_calibration["enbw_sensor"] logger.debug(f"Got sensor ENBW: {enbw_hz} Hz") - logger.debug('Skipping IIR filtering') noise_on_data = noise_on_measurement_result["data"] noise_off_data = noise_off_measurement_result["data"] - # Get power values in time domain - pwr_on_watts = self.get_td_power(noise_on_data) - pwr_off_watts = self.get_td_power(noise_off_data) + # Get power values in time domain (division by 2 for RF/baseband conversion) + pwr_on_watts = calculate_power_watts(noise_on_data / 2.) + pwr_off_watts = calculate_power_watts(noise_off_data / 2.) # Y-Factor enr_linear = get_linear_enr(cal_source_idx) temp_k, temp_c, _ = get_temperature(temp_sensor_idx) - noise_figure, gain = y_factor( pwr_on_watts, pwr_off_watts, enr_linear, enbw_hz, temp_k ) @@ -245,15 +265,8 @@ def calibrate(self, params): # Detail results contain only FFT version of result for now return 'Noise Figure: {}, Gain: {}'.format(noise_figure, gain) - def get_td_power(self, iqdata: np.ndarray) -> np.ndarray: - # Reshape data - iqdata /= 2 # RF/baseband conversion - iq_pwr = calculate_power_watts(iqdata) - return iq_pwr - @property def description(self): - #TODO: provide num_samples # Get parameters; they may be single values or lists frequencies = get_parameter(FREQUENCY, self.parameters) duration_ms = get_parameter(DURATION_MS, self.parameters) @@ -272,41 +285,57 @@ def description(self): else: num_samples = int(num_samples) - # TODO: generate blank if no filtering, else filter details if self.iir_apply is True: + pb_edge = self.iir_cutoff_Hz / 1e6 + sb_edge = (self.iir_cutoff_Hz + self.iir_width_Hz) / 1e6 filtering_suffix = ", after applying an IIR lowpass filter to the complex time-domain samples" filter_description = ( """ ### Filtering - Optionally, IQ samples can be filtered using an elliptic IIR filter before + The acquired samples are then filtered using an elliptic IIR filter before performing the rest of the time-domain Y-factor calculations. The filter design produces the lowest order digital filter which loses no more than - {gpass} + {self.iir_rp_dB} dB in the passband and has at least {self.iir_rs_dB} dB attenuation + in the stopband. The filter has a defined passband edge at {pb_edge} MHz + and a stopband edge at {sb_edge} MHz. From this filter design, second-order filter + coefficients are generated in order to minimize numerical precision errors + when filtering the time domain samples. The filtering function is implemented + as a series of second-order filters with direct-form II transposed structure. + + ### Power Calculation """ ) else: - filter_description = "No filter is applied to the input samples before Y-factor calculations" - - # Convert parameter lists to strings if needed - if isinstance(frequencies, list): - frequencies = utils.list_to_string( - [f / 1e6 for f in get_parameter(FREQUENCY, self.parameters)] + filtering_suffix = "" + filter_description = "" + + acquisition_plan = "" + acq_plan_template = "The signal analyzer is tuned to {center_frequency:.2f} MHz and the following parameters are set:\n" + acq_plan_template += "{parameters}" + acq_plan_template += "Then, acquire samples for {duration_ms} ms.\n" + + used_keys = [FREQUENCY, DURATION_MS, "name"] + for params in self.iteration_params: + parameters = "" + for name, value in params.items(): + if name not in used_keys: + parameters += f"{name} = {value}\n" + acquisition_plan += acq_plan_template.format( + **{ + "center_frequency": params[FREQUENCY] / 1e6, + "parameters": parameters, + "duration_ms": params[DURATION_MS], + } ) - - acq_plan = ( - f"Performs a y-factor calibration at frequencies: " - # f"{frequencies}, nffts:{nffts}, fft_size: {fft_size}\n" - ) + definitions = { "name": self.name, - "frequencies": frequencies, - "acquisition_plan": acq_plan, - "num_samples": num_samples, - + "filtering_suffix": filtering_suffix, + "filtering_description": filter_description, + "acquisition_plan": acquisition_plan, } # __doc__ refers to the module docstring at the top of the file - # TODO uncomment fornat below - return __doc__ #.format(**definitions) + return __doc__ .format(**definitions) def test_required_components(self): """Fail acquisition if a required component is not available.""" From 9692a7b73f7fc2a9314d5aeb04a473350a94637c Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 8 Aug 2022 20:30:04 -0600 Subject: [PATCH 112/157] Update variable names for consistency --- .../actions/acquire_stepped_freq_tdomain_iq.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py index f0ad0167..b7fd0fac 100644 --- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py +++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py @@ -80,11 +80,10 @@ class SteppedFrequencyTimeDomainIqAcquisition(SingleFrequencyTimeDomainIqAcquisi def __init__(self, parameters, sigan, gps=mock_gps): super().__init__(parameters=parameters, sigan=sigan, gps=gps) - self.sorted_measurement_parameters = [] num_center_frequencies = len(parameters[FREQUENCY]) # Create iterable parameter set - self.sorted_measurement_parameters = utils.get_iterable_parameters(parameters) + self.iterable_params = utils.get_iterable_parameters(parameters) self.sigan = sigan # make instance variable to allow mocking self.num_center_frequencies = num_center_frequencies @@ -94,7 +93,7 @@ def __call__(self, schedule_entry_json, task_id): self.test_required_components() for recording_id, measurement_params in enumerate( - self.sorted_measurement_parameters, start=1 + self.iterable_params, start=1 ): start_time = utils.get_datetime_str_now() self.configure(measurement_params) @@ -135,7 +134,7 @@ def description(self): acq_plan_template += "{parameters}" acq_plan_template += "Then, acquire samples for {duration_ms} ms.\n" - for measurement_params in self.sorted_measurement_parameters: + for measurement_params in self.iterable_params: parameters = "" for name, value in measurement_params.items(): if name not in used_keys: @@ -148,7 +147,7 @@ def description(self): } ) - durations = [v[DURATION_MS] for v in self.sorted_measurement_parameters] + durations = [v[DURATION_MS] for v in self.iterable_params] min_duration_ms = np.sum(durations) defs = { @@ -157,7 +156,7 @@ def description(self): "center_frequencies": ", ".join( [ "{:.2f} MHz".format(param[FREQUENCY] / 1e6) - for param in self.sorted_measurement_parameters + for param in self.iterable_params ] ), "acquisition_plan": acquisition_plan, From eb6dd61d14490b441f93306fc9ee1e25628e785c Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 8 Aug 2022 20:40:12 -0600 Subject: [PATCH 113/157] Update minimum scipy version --- requirements-dev.txt | 46 ++++++++++++++++++++++++++------------------ requirements.in | 4 ++-- requirements.txt | 11 +++-------- 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c58e498e..ead52b82 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements-dev.in @@ -8,8 +8,12 @@ asgiref==3.5.0 # via # -r requirements.txt # django +atomicwrites==1.4.1 + # via pytest attrs==21.4.0 # via pytest +build==0.8.0 + # via pip-tools certifi==2021.10.8 # via # -r requirements.txt @@ -20,6 +24,14 @@ charset-normalizer==2.0.12 # via # -r requirements.txt # requests +click==8.1.3 + # via pip-tools +colorama==0.4.5 + # via + # build + # click + # pytest + # tox distlib==0.3.4 # via virtualenv django==3.2.15 @@ -34,13 +46,6 @@ idna==3.3 # via # -r requirements.txt # requests -importlib-metadata==4.11.2 - # via - # pluggy - # pre-commit - # pytest - # tox - # virtualenv iniconfig==1.1.1 # via pytest its-preselector @ git+https://github.com/NTIA/Preselector@1.0.0 @@ -58,9 +63,12 @@ numpy==1.21.5 packaging==21.3 # via # -r requirements.txt + # build # numexpr # pytest # tox +pep517==0.13.0 + # via build pip-tools==6.8.0 # via -r requirements-dev.in platformdirs==2.5.1 @@ -99,16 +107,14 @@ ruamel-yaml-clib==0.2.6 # via # -r requirements.txt # ruamel-yaml -scipy==1.7.3 +scipy==1.9.0 # via -r requirements.txt -# sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive sigmf==1.0.0 # via -r requirements.txt six==1.16.0 # via # -r requirements.txt # python-dateutil - # sigmf # tox # virtualenv sqlparse==0.4.2 @@ -120,14 +126,12 @@ toml==0.10.2 # pre-commit # tox tomli==2.0.1 - # via pytest + # via + # build + # pep517 + # pytest tox==3.24.5 # via -r requirements-dev.in -typing-extensions==4.1.1 - # via - # -r requirements.txt - # asgiref - # importlib-metadata urllib3==1.26.8 # via # -r requirements.txt @@ -136,5 +140,9 @@ virtualenv==20.13.3 # via # pre-commit # tox -zipp==3.7.0 - # via importlib-metadata +wheel==0.37.1 + # via pip-tools + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/requirements.in b/requirements.in index 34e089db..98ff4ef3 100644 --- a/requirements.in +++ b/requirements.in @@ -1,8 +1,8 @@ django>=3.2.14, <4.0 -numpy>=1.0, <2.0 +numpy>=1.0, <1.23.0 python-dateutil>=2.0, < 3.0 ruamel.yaml>=0.1, <1.0 -scipy>=1.6.0, <2.0 +scipy>=1.8.0, <2.0 numexpr>=2.8.3, <3.0 sigmf==1.0.0 its-preselector @ git+https://github.com/NTIA/Preselector@1.0.0#egg=its-preselector diff --git a/requirements.txt b/requirements.txt index 55fe8297..90025c98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -38,18 +38,13 @@ ruamel-yaml==0.17.21 # via -r requirements.in ruamel-yaml-clib==0.2.6 # via ruamel-yaml -scipy==1.7.3 +scipy==1.9.0 # via -r requirements.in sigmf==1.0.0 -# sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive # via -r requirements.in six==1.16.0 - # via - # python-dateutil - # sigmf + # via python-dateutil sqlparse==0.4.2 # via django -typing-extensions==4.1.1 - # via asgiref urllib3==1.26.8 # via requests From 8c32461aa5fdad75cb901d12e04074cf55eb705b Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 11:55:22 -0600 Subject: [PATCH 114/157] Fix conditional in description method --- scos_actions/actions/calibrate_y_factor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 0a91e9c2..8a0f6e4e 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -280,7 +280,8 @@ def description(self): duration_ms = duration_ms num_samples = duration_ms * sample_rate * 1e-3 - if len(num_samples) != 1: + + if isinstance(num_samples, np.ndarray) and len(num_samples) != 1: num_samples = num_samples.tolist() else: num_samples = int(num_samples) From de887d05f00cf7f19660ee3a5dedf1d40a396ad3 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 12:00:38 -0600 Subject: [PATCH 115/157] Move iterable parameter generation to constructor --- scos_actions/actions/calibrate_y_factor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 8a0f6e4e..bb7ae45d 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -141,6 +141,7 @@ class YFactorCalibration(Action): def __init__(self, parameters, sigan, gps=mock_gps): logger.debug('Initializing calibration action') super().__init__(parameters, sigan, gps) + self.iteration_params = utils.get_iterable_parameters(parameters) self.power_detector = create_power_detector("MeanDetector", ["mean"]) # IIR Filter Setup @@ -171,7 +172,6 @@ def __init__(self, parameters, sigan, gps=mock_gps): def __call__(self, schedule_entry_json, task_id): """This is the entrypoint function called by the scheduler.""" self.test_required_components() - self.iteration_params = utils.get_iterable_parameters(self.parameters) detail = '' # Run calibration routine From 9538469656baefbde511b657e6e00fd56e530a71 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 12:07:09 -0600 Subject: [PATCH 116/157] Fix docstring LaTeX formatting --- scos_actions/actions/calibrate_y_factor.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index bb7ae45d..be6c5508 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -47,25 +47,25 @@ The mean power for the noise diode on and off captures are calculated by taking the mean of each array of power samples. Next, the Y-factor is calculated by: -$$ y = P_{on} / P_{off} $$ +$$ y = P_{{on}} / P_{{off}} $$ -Where $P_{on}$ is the mean power measured with the noise diode on, and $P_{off}$ +Where $P_{{on}}$ is the mean power measured with the noise diode on, and $P_{{off}}$ is the mean power measured with the noise diode off. The linear noise factor is then calculated by: -$$ NF = \frac{ENR}{y - 1} $$ +$$ NF = \frac{{ENR}}{{y - 1}} $$ Where $ENR$ is the excess noise ratio, in linear units, of the noise diode used for the power measurements. Next, the linear gain is calculated by: -$$ G = \frac{P_{on}}{k_B T B_{eq} (ENR + NF)} $$ +$$ G = \frac{{P_{{on}}}}{{k_B T B_{{eq}} (ENR + NF)}} $$ Where $k_B$ is Boltzmann's constant, $T$ is the calibration temperature in Kelvins, -and $B_{eq}$ is the sensor's equivalent noise bandwidth. Finally, the noise factor -and linear gain are converted to noise figure $F_N$ and decibel gain $G_{dB}$: +and $B_{{eq}}$ is the sensor's equivalent noise bandwidth. Finally, the noise factor +and linear gain are converted to noise figure $F_N$ and decibel gain $G_{{dB}}$: -$$ G_{dB} = 10 \log_{10}(G) $$ -$$ F_N = 10 \log_{10}(NF) $$ +$$ G_{{dB}} = 10 \log_{{10}}(G) $$ +$$ F_N = 10 \log_{{10}}(NF) $$ """ import logging From 8552e94906f3db8fc5fa5ce6d4ec22de0506eb75 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 12:29:16 -0600 Subject: [PATCH 117/157] Suppress warnings on sigan configuration --- scos_actions/actions/calibrate_y_factor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index be6c5508..06135eec 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -184,7 +184,10 @@ def __call__(self, schedule_entry_json, task_id): def calibrate(self, params): # Configure signal analyzer - super().configure_sigan(params) + sigan_params = params.copy() + for k in [IIR_APPLY, IIR_RP, IIR_RS, IIR_CUTOFF, IIR_WIDTH, CAL_SOURCE_IDX, TEMP_SENSOR_IDX]: + sigan_params.pop(k) + super().configure_sigan(sigan_params) # Get parameters from action config cal_source_idx = get_parameter(CAL_SOURCE_IDX, params) From 91a60c64d3161b1f487d1be95624e59741f7601f Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 12:34:28 -0600 Subject: [PATCH 118/157] Make sigan instance variable --- scos_actions/actions/calibrate_y_factor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 06135eec..b6209b49 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -141,6 +141,7 @@ class YFactorCalibration(Action): def __init__(self, parameters, sigan, gps=mock_gps): logger.debug('Initializing calibration action') super().__init__(parameters, sigan, gps) + self.sigan = sigan self.iteration_params = utils.get_iterable_parameters(parameters) self.power_detector = create_power_detector("MeanDetector", ["mean"]) @@ -233,7 +234,7 @@ def calibrate(self, params): else: logger.debug('Skipping IIR filtering') # Get ENBW from sensor calibration - enbw_hz = sensor_calibration["enbw_sensor"] + enbw_hz = self.sigan.sensor_calibration_data["enbw_sensor"] logger.debug(f"Got sensor ENBW: {enbw_hz} Hz") noise_on_data = noise_on_measurement_result["data"] noise_off_data = noise_off_measurement_result["data"] From fdea3626fd27627dcda3abfa025ec6d515223594 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 12:43:29 -0600 Subject: [PATCH 119/157] Add support for non-iir cal in sigan config --- scos_actions/actions/calibrate_y_factor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index b6209b49..8efc755c 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -186,8 +186,12 @@ def __call__(self, schedule_entry_json, task_id): def calibrate(self, params): # Configure signal analyzer sigan_params = params.copy() - for k in [IIR_APPLY, IIR_RP, IIR_RS, IIR_CUTOFF, IIR_WIDTH, CAL_SOURCE_IDX, TEMP_SENSOR_IDX]: - sigan_params.pop(k) + for k in [IIR_RP, IIR_RS, IIR_CUTOFF, IIR_WIDTH, CAL_SOURCE_IDX, TEMP_SENSOR_IDX]: + try: + sigan_params.pop(k) + except KeyError: + continue + super().configure_sigan(sigan_params) # Get parameters from action config From 0b544ac233632df511de351bec46041bf0550fd3 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 12:52:05 -0600 Subject: [PATCH 120/157] Attempt recompute calibration data to get ENBW --- scos_actions/actions/calibrate_y_factor.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 8efc755c..405b0bbe 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -186,12 +186,14 @@ def __call__(self, schedule_entry_json, task_id): def calibrate(self, params): # Configure signal analyzer sigan_params = params.copy() - for k in [IIR_RP, IIR_RS, IIR_CUTOFF, IIR_WIDTH, CAL_SOURCE_IDX, TEMP_SENSOR_IDX]: + # Suppress warnings during sigan configuration + for k in [DURATION_MS, NUM_SKIP, IIR_APPLY, IIR_RP, IIR_RS, IIR_CUTOFF, IIR_WIDTH, CAL_SOURCE_IDX, TEMP_SENSOR_IDX]: try: sigan_params.pop(k) except KeyError: continue - + # sigan_params also used as calibration args for getting sensor ENBW + # if no IIR filtering is applied. super().configure_sigan(sigan_params) # Get parameters from action config @@ -238,6 +240,9 @@ def calibrate(self, params): else: logger.debug('Skipping IIR filtering') # Get ENBW from sensor calibration + calibration_args = [sigan_params[k] for k in sigan_params.keys()] + logger.debug(f"Using calibration args: {calibration_args}") + self.sigan.recompute_calibration_data(calibration_args) enbw_hz = self.sigan.sensor_calibration_data["enbw_sensor"] logger.debug(f"Got sensor ENBW: {enbw_hz} Hz") noise_on_data = noise_on_measurement_result["data"] From a4e7e056bcb28576f53b1c3312036312be7c4aa6 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 13:07:27 -0600 Subject: [PATCH 121/157] Temporary fix for Tek cal --- scos_actions/actions/calibrate_y_factor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 405b0bbe..609d4e1f 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -240,7 +240,8 @@ def calibrate(self, params): else: logger.debug('Skipping IIR filtering') # Get ENBW from sensor calibration - calibration_args = [sigan_params[k] for k in sigan_params.keys()] + # TODO: Remove this hard-coded fix + calibration_args = [sigan_params[k] for k in [SAMPLE_RATE, FREQUENCY, "reference_level"]] logger.debug(f"Using calibration args: {calibration_args}") self.sigan.recompute_calibration_data(calibration_args) enbw_hz = self.sigan.sensor_calibration_data["enbw_sensor"] From 8e4176508e0cffd0ac08fbccf153d2acf6287149 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 13:22:52 -0600 Subject: [PATCH 122/157] Debug get cal parameters --- scos_actions/actions/calibrate_y_factor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 609d4e1f..daaec8f3 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -243,6 +243,8 @@ def calibrate(self, params): # TODO: Remove this hard-coded fix calibration_args = [sigan_params[k] for k in [SAMPLE_RATE, FREQUENCY, "reference_level"]] logger.debug(f"Using calibration args: {calibration_args}") + test_cal_args = sensor_calibration.calibration_parameters + logger.debug(f"Testing calibration args: {test_cal_args}, {type(test_cal_args)}") self.sigan.recompute_calibration_data(calibration_args) enbw_hz = self.sigan.sensor_calibration_data["enbw_sensor"] logger.debug(f"Got sensor ENBW: {enbw_hz} Hz") From e033abfad0ba05b13808eb1bdbba7ff69faac7ce Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 13:38:09 -0600 Subject: [PATCH 123/157] Make ENBW lookup get cal args dynamically --- scos_actions/actions/calibrate_y_factor.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index daaec8f3..0fec3773 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -240,12 +240,10 @@ def calibrate(self, params): else: logger.debug('Skipping IIR filtering') # Get ENBW from sensor calibration - # TODO: Remove this hard-coded fix - calibration_args = [sigan_params[k] for k in [SAMPLE_RATE, FREQUENCY, "reference_level"]] - logger.debug(f"Using calibration args: {calibration_args}") - test_cal_args = sensor_calibration.calibration_parameters - logger.debug(f"Testing calibration args: {test_cal_args}, {type(test_cal_args)}") - self.sigan.recompute_calibration_data(calibration_args) + cal_params = sensor_calibration.calibration_parameters + cal_args = [sigan_params[k] for k in cal_params] + logger.debug(f"Looking up sensor ENBW based on calibration args: {[f'{k} : {v}' for k, v in zip(cal_params, cal_args)]}") + self.sigan.recompute_calibration_data(cal_args) enbw_hz = self.sigan.sensor_calibration_data["enbw_sensor"] logger.debug(f"Got sensor ENBW: {enbw_hz} Hz") noise_on_data = noise_on_measurement_result["data"] From f091d04a648436975f204f6a34acdcf03bfa2753 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 13:51:42 -0600 Subject: [PATCH 124/157] Clean up ENBW debugging --- scos_actions/actions/calibrate_y_factor.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 0fec3773..cef19c48 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -240,12 +240,9 @@ def calibrate(self, params): else: logger.debug('Skipping IIR filtering') # Get ENBW from sensor calibration - cal_params = sensor_calibration.calibration_parameters - cal_args = [sigan_params[k] for k in cal_params] - logger.debug(f"Looking up sensor ENBW based on calibration args: {[f'{k} : {v}' for k, v in zip(cal_params, cal_args)]}") + cal_args = [sigan_params[k] for k in sensor_calibration.calibration_parameters] self.sigan.recompute_calibration_data(cal_args) enbw_hz = self.sigan.sensor_calibration_data["enbw_sensor"] - logger.debug(f"Got sensor ENBW: {enbw_hz} Hz") noise_on_data = noise_on_measurement_result["data"] noise_off_data = noise_off_measurement_result["data"] From 635b007462a66915e6be4fe7d3870b3e2fd78f0d Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 14:21:59 -0600 Subject: [PATCH 125/157] Revert to NTIA SigMF repo --- requirements-dev.txt | 3 ++- requirements.in | 2 +- requirements.txt | 6 ++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index ead52b82..efce6781 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -109,12 +109,13 @@ ruamel-yaml-clib==0.2.6 # ruamel-yaml scipy==1.9.0 # via -r requirements.txt -sigmf==1.0.0 +sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive # via -r requirements.txt six==1.16.0 # via # -r requirements.txt # python-dateutil + # sigmf # tox # virtualenv sqlparse==0.4.2 diff --git a/requirements.in b/requirements.in index 98ff4ef3..4d5d2b69 100644 --- a/requirements.in +++ b/requirements.in @@ -4,5 +4,5 @@ python-dateutil>=2.0, < 3.0 ruamel.yaml>=0.1, <1.0 scipy>=1.8.0, <2.0 numexpr>=2.8.3, <3.0 -sigmf==1.0.0 +sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive its-preselector @ git+https://github.com/NTIA/Preselector@1.0.0#egg=its-preselector diff --git a/requirements.txt b/requirements.txt index 90025c98..9dc4c04e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,10 +40,12 @@ ruamel-yaml-clib==0.2.6 # via ruamel-yaml scipy==1.9.0 # via -r requirements.in -sigmf==1.0.0 +sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive # via -r requirements.in six==1.16.0 - # via python-dateutil + # via + # python-dateutil + # sigmf sqlparse==0.4.2 # via django urllib3==1.26.8 From c84ea5c6e8bae6ac6fef3c388d6d9612d6613445 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 14:49:19 -0600 Subject: [PATCH 126/157] Revert sigmf core version number --- scos_actions/metadata/sigmf_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/metadata/sigmf_builder.py b/scos_actions/metadata/sigmf_builder.py index e1eed6cb..63233b9e 100644 --- a/scos_actions/metadata/sigmf_builder.py +++ b/scos_actions/metadata/sigmf_builder.py @@ -3,7 +3,7 @@ from sigmf import SigMFFile GLOBAL_INFO = { - "core:version": "v1.0.0", + "core:version": "v0.0.2", "core:extensions": { "ntia-algorithm": "v1.0.0", "ntia-core": "v1.0.0", From 578e68399638b0131334f0da40a32b67dd011435 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 16:02:15 -0600 Subject: [PATCH 127/157] Debug filter/no-filter gain calculation --- scos_actions/actions/calibrate_y_factor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index cef19c48..5a54335a 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -242,7 +242,9 @@ def calibrate(self, params): # Get ENBW from sensor calibration cal_args = [sigan_params[k] for k in sensor_calibration.calibration_parameters] self.sigan.recompute_calibration_data(cal_args) - enbw_hz = self.sigan.sensor_calibration_data["enbw_sensor"] + # TODO: Return this to be pulled from sensor cal file + # enbw_hz = self.sigan.sensor_calibration_data["enbw_sensor"] + enbw_hz = 11.607e6 noise_on_data = noise_on_measurement_result["data"] noise_off_data = noise_off_measurement_result["data"] From 48a95269e58540aa5a2dec996a8e56dae64b6b9a Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 16:56:43 -0600 Subject: [PATCH 128/157] Loosen cap on numpy version --- requirements.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.in b/requirements.in index 4d5d2b69..356627e0 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,5 @@ django>=3.2.14, <4.0 -numpy>=1.0, <1.23.0 +numpy>=1.22, <2.0 python-dateutil>=2.0, < 3.0 ruamel.yaml>=0.1, <1.0 scipy>=1.8.0, <2.0 From 26563099fec3b2308380fead52445ee161ab96ad Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 9 Aug 2022 17:03:05 -0600 Subject: [PATCH 129/157] Updated locked requirements --- requirements-dev.txt | 10 +--------- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index efce6781..d1a0a13c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,8 +8,6 @@ asgiref==3.5.0 # via # -r requirements.txt # django -atomicwrites==1.4.1 - # via pytest attrs==21.4.0 # via pytest build==0.8.0 @@ -26,12 +24,6 @@ charset-normalizer==2.0.12 # requests click==8.1.3 # via pip-tools -colorama==0.4.5 - # via - # build - # click - # pytest - # tox distlib==0.3.4 # via virtualenv django==3.2.15 @@ -54,7 +46,7 @@ nodeenv==1.6.0 # via pre-commit numexpr==2.8.3 # via -r requirements.txt -numpy==1.21.5 +numpy==1.23.1 # via # -r requirements.txt # numexpr diff --git a/requirements.txt b/requirements.txt index 9dc4c04e..b408638d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ its-preselector @ git+https://github.com/NTIA/Preselector@1.0.0 # via -r requirements.in numexpr==2.8.3 # via -r requirements.in -numpy==1.21.5 +numpy==1.23.1 # via # -r requirements.in # numexpr From cb19b743d8ce3b1be4d717e43602d117c19e63ce Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 10 Aug 2022 11:23:06 -0600 Subject: [PATCH 130/157] Switch to flit backend --- pyproject.toml | 61 ++++++++--------- requirements-dev.in | 6 -- requirements-dev.txt | 141 -------------------------------------- requirements.in | 8 --- requirements.txt | 52 -------------- scos_actions/__about__.py | 1 - scos_actions/__init__.py | 5 ++ 7 files changed, 33 insertions(+), 241 deletions(-) delete mode 100644 requirements-dev.in delete mode 100644 requirements-dev.txt delete mode 100644 requirements.in delete mode 100644 requirements.txt delete mode 100644 scos_actions/__about__.py diff --git a/pyproject.toml b/pyproject.toml index ff84e0fe..38b581a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,25 +1,28 @@ [build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" +requires = ["flit_core>=3.4,<4"] +build-backend = "flit_core.buildapi" [project] -name = "scos_actions" -dynamic = ["version"] -description = "Base plugin providing common actions and interfaces to be re-used by SCOS Sensor plugins." -readme = "README.md" +name = "scos-actions" +dynamic = ["version", "description"] +readme = { file = "README.md" } requires-python = ">=3.8" license = { file = "LICENSE.md" } authors = [ { name = "The Institute for Telecommunication Sciences" }, - { name="Anthony Romaniello", email="aromaniello@ntia.gov"} +] + +maintainers = [ + { name = "Doug Boulware", email = "dboulware@ntia.gov" }, + { name = "Justin Haze", email = "jhaze@ntia.gov" }, + { name = "Anthony Romaniello", email = "aromaniello@ntia.gov" }, ] keywords = [ - "scos", "sdr", "spectrum-analyzer", "spectrum", - "analyzer", "spectrum analyzer", "scos-sensor", "scos sensor", - "spectrum monitoring", "monitoring", "spectrum management", "docker", - "linux", "software defined radio", "radio" + "SCOS", "SDR", "spectrum monitoring", "radio", "sensor", + "spectrum", "monitoring", "remote", "distributed", "sensing", + "NTIA", "ITS", "telecommunications", ] classifiers = [ @@ -38,22 +41,22 @@ classifiers = [ dependencies = [ "django>3.2.14,<4.0", - # its_preselector @ ntia/preselector@1.0.0 not included - "numexpr>=2.8.3,<3.0", - "numpy>=1.22.0,<2.0", - "python-dateutil>=2.0,<3.0", - "ruamel.yaml>=0.15,<1.0", - "scipy>=1.6.0,<2.0", - # ntia/sigmf@multi-recording-archive not included + "its_preselector @ https://github.com/NTIA/Preselector/archive/refs/tags/1.0.0.zip", + "numexpr>=2.8.3", + "numpy>=1.22.0", + "python-dateutil>=2.0", + "ruamel.yaml>=0.15", + "scipy>=1.8.0", + "sigmf @ https://github.com/NTIA/SigMF/archive/refs/heads/multi-recording-archive.zip" ] [project.optional-dependencies] dev = [ - "pre-commit>=2.20.0,<3.0", - "pytest>=7.1.2,<8.0", - "pytest-cov>=3.0.0,<4.0", - "tox>=3.0,<4.0", - "twine>=4.0.1,<5.0", + "flit>=3.4,<4", + "pre-commit>=2.20.0", + "pytest>=7.1.2", + "pytest-cov>=3.0.0", + "tox>=3.0", ] [project.urls] @@ -63,13 +66,5 @@ dev = [ "NTIA GitHub" = "https://github.com/NTIA" "ITS Website" = "https://its.ntia.gov" -[tool.hatch.version] -path = "scos_actions/__about__.py" - -[tool.hatch.build] -skip-excluded-dirs = true - -[tool.hatch.build.targets.wheel] -packages = ["scos_actions"] - -[tool.hatch.build.targets.sdist] +[tool.flit.module] +name = "scos_actions" diff --git a/requirements-dev.in b/requirements-dev.in deleted file mode 100644 index 02662e27..00000000 --- a/requirements-dev.in +++ /dev/null @@ -1,6 +0,0 @@ --rrequirements.txt - -pip-tools>=6.6.2, <7.0 -pre-commit>=2.0, <3.0 -pytest>=7.0, <8.0 -tox>=3.0, <=4.0 diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index d1a0a13c..00000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,141 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# pip-compile requirements-dev.in -# -asgiref==3.5.0 - # via - # -r requirements.txt - # django -attrs==21.4.0 - # via pytest -build==0.8.0 - # via pip-tools -certifi==2021.10.8 - # via - # -r requirements.txt - # requests -cfgv==3.3.1 - # via pre-commit -charset-normalizer==2.0.12 - # via - # -r requirements.txt - # requests -click==8.1.3 - # via pip-tools -distlib==0.3.4 - # via virtualenv -django==3.2.15 - # via -r requirements.txt -filelock==3.6.0 - # via - # tox - # virtualenv -identify==2.4.11 - # via pre-commit -idna==3.3 - # via - # -r requirements.txt - # requests -iniconfig==1.1.1 - # via pytest -its-preselector @ git+https://github.com/NTIA/Preselector@1.0.0 - # via -r requirements.txt -nodeenv==1.6.0 - # via pre-commit -numexpr==2.8.3 - # via -r requirements.txt -numpy==1.23.1 - # via - # -r requirements.txt - # numexpr - # scipy - # sigmf -packaging==21.3 - # via - # -r requirements.txt - # build - # numexpr - # pytest - # tox -pep517==0.13.0 - # via build -pip-tools==6.8.0 - # via -r requirements-dev.in -platformdirs==2.5.1 - # via virtualenv -pluggy==1.0.0 - # via - # pytest - # tox -pre-commit==2.17.0 - # via -r requirements-dev.in -py==1.11.0 - # via - # pytest - # tox -pyparsing==3.0.9 - # via - # -r requirements.txt - # packaging -pytest==7.0.1 - # via -r requirements-dev.in -python-dateutil==2.8.2 - # via -r requirements.txt -pytz==2021.3 - # via - # -r requirements.txt - # django -pyyaml==6.0 - # via pre-commit -requests==2.27.1 - # via - # -r requirements.txt - # its-preselector -ruamel-yaml==0.17.21 - # via -r requirements.txt -ruamel-yaml-clib==0.2.6 - # via - # -r requirements.txt - # ruamel-yaml -scipy==1.9.0 - # via -r requirements.txt -sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive - # via -r requirements.txt -six==1.16.0 - # via - # -r requirements.txt - # python-dateutil - # sigmf - # tox - # virtualenv -sqlparse==0.4.2 - # via - # -r requirements.txt - # django -toml==0.10.2 - # via - # pre-commit - # tox -tomli==2.0.1 - # via - # build - # pep517 - # pytest -tox==3.24.5 - # via -r requirements-dev.in -urllib3==1.26.8 - # via - # -r requirements.txt - # requests -virtualenv==20.13.3 - # via - # pre-commit - # tox -wheel==0.37.1 - # via pip-tools - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/requirements.in b/requirements.in deleted file mode 100644 index 356627e0..00000000 --- a/requirements.in +++ /dev/null @@ -1,8 +0,0 @@ -django>=3.2.14, <4.0 -numpy>=1.22, <2.0 -python-dateutil>=2.0, < 3.0 -ruamel.yaml>=0.1, <1.0 -scipy>=1.8.0, <2.0 -numexpr>=2.8.3, <3.0 -sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive -its-preselector @ git+https://github.com/NTIA/Preselector@1.0.0#egg=its-preselector diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index b408638d..00000000 --- a/requirements.txt +++ /dev/null @@ -1,52 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# pip-compile requirements.in -# -asgiref==3.5.0 - # via django -certifi==2021.10.8 - # via requests -charset-normalizer==2.0.12 - # via requests -django==3.2.15 - # via -r requirements.in -idna==3.3 - # via requests -its-preselector @ git+https://github.com/NTIA/Preselector@1.0.0 - # via -r requirements.in -numexpr==2.8.3 - # via -r requirements.in -numpy==1.23.1 - # via - # -r requirements.in - # numexpr - # scipy - # sigmf -packaging==21.3 - # via numexpr -pyparsing==3.0.9 - # via packaging -python-dateutil==2.8.2 - # via -r requirements.in -pytz==2021.3 - # via django -requests==2.27.1 - # via its-preselector -ruamel-yaml==0.17.21 - # via -r requirements.in -ruamel-yaml-clib==0.2.6 - # via ruamel-yaml -scipy==1.9.0 - # via -r requirements.in -sigmf @ git+https://github.com/NTIA/SigMF.git@multi-recording-archive - # via -r requirements.in -six==1.16.0 - # via - # python-dateutil - # sigmf -sqlparse==0.4.2 - # via django -urllib3==1.26.8 - # via requests diff --git a/scos_actions/__about__.py b/scos_actions/__about__.py deleted file mode 100644 index 21014090..00000000 --- a/scos_actions/__about__.py +++ /dev/null @@ -1 +0,0 @@ -VERSION = "2.0.0" diff --git a/scos_actions/__init__.py b/scos_actions/__init__.py index e69de29b..eb0b7909 100644 --- a/scos_actions/__init__.py +++ b/scos_actions/__init__.py @@ -0,0 +1,5 @@ +"""The base plugin providing common actions and interfaces for SCOS Sensor plugins. + +Refer to the SCOS Actions README file for detailed usage information. +""" +__version__ = "2.0.0" From 93c2f752d388ee733df979002eba20f2cf091b1f Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 10 Aug 2022 11:23:22 -0600 Subject: [PATCH 131/157] Remove VS code configs --- .vscode/launch.json | 16 ---------------- .vscode/settings.json | 13 ------------- 2 files changed, 29 deletions(-) delete mode 100644 .vscode/launch.json delete mode 100644 .vscode/settings.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 5dba2172..00000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - - { - "name": "Python: Current File", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "integratedTerminal" - } - ] -} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index e1ae4439..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "python.pythonPath": "venv/bin/python", - "python.testing.pytestArgs": ["."], - "python.testing.unittestEnabled": false, - "python.testing.nosetestsEnabled": false, - "python.testing.pytestEnabled": true, - "python.formatting.provider": "black", - "[markdown]": { - "editor.rulers": [ - 88 - ] - } -} From 62ed08baf0d9d0e7d9096e1e037e112f3e1f1611 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 10 Aug 2022 11:24:58 -0600 Subject: [PATCH 132/157] Fix README specification in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 38b581a6..020ca7ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "flit_core.buildapi" [project] name = "scos-actions" dynamic = ["version", "description"] -readme = { file = "README.md" } +readme = "README.md" requires-python = ">=3.8" license = { file = "LICENSE.md" } From abb465063fb876dbd3f3853ff44a6a88196d497b Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 10 Aug 2022 11:37:40 -0600 Subject: [PATCH 133/157] Updated readme for flit --- README.md | 84 +++++++++++++++---------------------------------------- 1 file changed, 22 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 86739294..ee974a19 100644 --- a/README.md +++ b/README.md @@ -151,48 +151,36 @@ if the new functionality can be supported by most signal analyzers. ### Requirements and Configuration -Requires `pip>=18.1` and `python>=3.7`. - -It is highly recommended that you first initialize a virtual development environment -using a tool such as [Conda](https://docs.conda.io/en/latest/) or [venv](https://docs.python.org/3/library/venv.html#module-venv). -The following commands create a virtual environment using venv and install the required -dependencies for development and testing. +Set up a development environment using a tool like [Conda](https://docs.conda.io/en/latest/) +or [venv](https://docs.python.org/3/library/venv.html#module-venv), with `python>=3.8`. Then, +from the cloned directory, install the development dependencies by running: ```bash -python -m venv .venv -source .venv/bin/activate -python -m pip install --upgrade pip # upgrade to pip>=18.1 -python -m pip install -r requirements.txt +pip install .[dev] ``` -#### Using pip-tools - -It is recommended to keep direct dependencies in a separate file. The direct -dependencies are in the `requirements.in` and `requirements-dev.in` files. Then pip-tools -can be used to generate files with all the dependencies and transitive dependencies -(sub-dependencies). The files containing all the dependencies are in `requirements.txt` and -`requirements-dev.txt`. Run the following in the virtual environment to install pip-tools: +This will install the project itself, along with development dependencies for pre-commit +hooks, building distributions, and running tests. Set up pre-commit, which runs auto-formatting +and code-checking automatically when you make a commit, by running: ```bash -python -m pip install pip-tools +pre-commit install ``` -To update `requirements.txt` and `requirements-dev.txt` after modifying `requirements.in` -or `requirements-dev.in`: +The pre-commit tool will auto-format Python code using [Black](https://github.com/psf/black) +and [isort](https://github.com/pycqa/isort). Other pre-commit hooks are also enabled, and +can be found in [`.pre-commit-config.yaml`](.pre-commit-config.yaml). -```bash -pip-compile requirements.in -pip-compile requirements-dev.in -``` +### Building New Releases -Use `pip-sync` to match virtual environment to `requirements-dev.txt`: +This project uses [flit](https://github.com/pypa/flit) as a backend. To build a new release +(both wheel and sdist/tarball), first update the version number in +[`scos_actions/__init__.py`](scos_actions/__init__.py), then run: ```bash -pip-sync requirements.txt requirements-dev.txt +flit build ``` -For more information, see [pip-tools' documentation](https://pip-tools.readthedocs.io/en/latest). - ### Running Tests Ideally, you should add a test to cover any new feature that you add. If you've done @@ -200,11 +188,10 @@ that, then running the included test suite is the easiest way to check that ever is working. In any case, all tests should be run after making any local modifications to ensure that you haven't caused a regression. -scos-actions uses [pytest](https://docs.pytest.org/en/stable/) for testing. - -[tox](https://tox.readthedocs.io/en/latest/) is a tool that can run all available tests -in a virtual environment against all supported versions of Python. Running `pytest` -directly is faster but running `tox` is a more thorough test. +The `scos_actions` package is tested using the [pytest](https://docs.pytest.org/en/stable/) +framework. Additionally, [tox](https://tox.readthedocs.io/en/latest/) is used to run all +available tests in a virtual environment against all supported versions of Python. +Running `pytest` directly is faster but running `tox` is a more thorough test. The following commands can be used to run tests. Note, for tox to run with all Python versions listed in tox.ini, all those versions must be installed on your system. @@ -216,33 +203,6 @@ tox --recreate # if you change `requirements.txt` tox -e coverage # check where test coverage lacks ``` -### Committing - -Besides running the test suite and ensuring that all tests are passed, we also expect -all Python code that's checked in to have been run through an auto-formatter. This project -uses a Python auto-formatter called [Black](https://github.com/psf/black). Additionally, -import statement sorting is handled by [isort](https://github.com/pycqa/isort). - -There are several ways to auto-format your code before committing. First, IDE integration -with on-save hooks is very useful. Second, if you already pip-installed the development -requirements from the section above, you already have a utility called pre-commit that -will automate setting up this project's pre-commit Git hooks. Simply type the following -*once*, and each time you make a commit, it will be appropriately auto-formatted. - -```bash -pre-commit install -``` - -You can also manually run the pre-commit hooks on the entire project: - -```bash -pre-commit run --all-files -``` - -In addition to Black and isort, various other pre-commit tools are enabled including [markdownlint](https://github.com/DavidAnson/markdownlint). -See [`.pre-commit-config.yaml`](.pre-commit-config.yaml) for the list of pre-commit -tools enabled for this repository. - ### Adding Actions To expose a new action to the API, check out the available @@ -445,7 +405,7 @@ another signal analyzer with a Python API. abstract class. Add properties or class variables for the parameters needed to configure the signal analyzer. - Create YAML files with the parameters needed to run the actions imported from - scos-actions using the new signal analyzer. Put them in the new repository in + `scos_actions` using the new signal analyzer. Put them in the new repository in `configs/actions`. This should contain the parameters needed by the action as well as the signal analyzer settings based on which properties or class variables were implemented in the signal analyzer class in the previous step. The measurement actions @@ -500,4 +460,4 @@ See [LICENSE](LICENSE.md). ## Contact -For technical questions about scos-actions, contact Justin Haze, jhaze@ntia.gov +For technical questions about SCOS Actions, contact Justin Haze, jhaze@ntia.gov From 908531e2da1644fa78ef04144ca2d2b679e3df7d Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 10 Aug 2022 13:06:38 -0600 Subject: [PATCH 134/157] Update python version for pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 67568c2e..f6d622b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ default_language_version: - python: python3.7 + python: python3.8 repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.1.0 From e9beefb60cf593e2abc9eeb8034d81b662fc0ed9 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 10 Aug 2022 13:06:56 -0600 Subject: [PATCH 135/157] Removed old VS Code configs --- .vscode/launch.json | 16 ---------------- .vscode/settings.json | 13 ------------- 2 files changed, 29 deletions(-) delete mode 100644 .vscode/launch.json delete mode 100644 .vscode/settings.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 5dba2172..00000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - - { - "name": "Python: Current File", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "integratedTerminal" - } - ] -} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index e1ae4439..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "python.pythonPath": "venv/bin/python", - "python.testing.pytestArgs": ["."], - "python.testing.unittestEnabled": false, - "python.testing.nosetestsEnabled": false, - "python.testing.pytestEnabled": true, - "python.formatting.provider": "black", - "[markdown]": { - "editor.rulers": [ - 88 - ] - } -} From 2820de71bf4e9562e2d7ce367a25fc5c7c02b578 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 10 Aug 2022 13:08:35 -0600 Subject: [PATCH 136/157] Remove py37 from Tox config --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3b17e86b..8e900618 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37,py38,py39,py310 +envlist = py38,py39,py310 skip_missing_interpreters = True skipsdist = True From 6fa2f175a85d8bc13ad59b23bb42fb7a4f846174 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 10 Aug 2022 13:12:16 -0600 Subject: [PATCH 137/157] Increment version to 3.0.0 --- scos_actions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/__init__.py b/scos_actions/__init__.py index eb0b7909..f6584813 100644 --- a/scos_actions/__init__.py +++ b/scos_actions/__init__.py @@ -2,4 +2,4 @@ Refer to the SCOS Actions README file for detailed usage information. """ -__version__ = "2.0.0" +__version__ = "3.0.0" From 79a3c6021b6ae2968097f2875408a75ad073474c Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 11 Aug 2022 14:41:07 -0600 Subject: [PATCH 138/157] Update Django minimum version to 3.2.15 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 020ca7ec..3a33ff10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ classifiers = [ ] dependencies = [ - "django>3.2.14,<4.0", + "django>3.2.15,<4.0", "its_preselector @ https://github.com/NTIA/Preselector/archive/refs/tags/1.0.0.zip", "numexpr>=2.8.3", "numpy>=1.22.0", From 43fc126391dfea9b04c07648f486df2bd777c882 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 11 Aug 2022 14:50:44 -0600 Subject: [PATCH 139/157] Allow Django==3.2.15 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3a33ff10..f9bbf656 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ classifiers = [ ] dependencies = [ - "django>3.2.15,<4.0", + "django>=3.2.15,<4.0", "its_preselector @ https://github.com/NTIA/Preselector/archive/refs/tags/1.0.0.zip", "numexpr>=2.8.3", "numpy>=1.22.0", From 14d6c85309e356733033f67043e0ebc0cea7d5f7 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 11 Aug 2022 15:55:44 -0600 Subject: [PATCH 140/157] Add type check for FFT parameters --- scos_actions/signal_processing/fft.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scos_actions/signal_processing/fft.py b/scos_actions/signal_processing/fft.py index a5390b13..63c260f0 100644 --- a/scos_actions/signal_processing/fft.py +++ b/scos_actions/signal_processing/fft.py @@ -64,11 +64,26 @@ def get_fft( normalization mode. """ logger.debug("Computing FFTs") + # Make sure num_ffts and fft_size are integers + if isinstance(fft_size, int) and isinstance(num_ffts, int): + pass + else: + if isinstance(fft_size, float) and fft_size == int(fft_size): + fft_size = int(fft_size) + else: + raise ValueError("fft_size must be an integer.") + if isinstance(num_ffts, float) and num_ffts == int(num_ffts): + num_ffts = int(num_ffts) + else: + raise ValueError("num_ffts must be an integer.") + # Get num_ffts for default case: as many as possible if num_ffts <= 0: logger.info("Number of FFTs not specified. Using as many as possible.") num_ffts = int(len(time_data) // fft_size) - logger.info(f"Number of FFTs set to {num_ffts} based on specified FFT size {fft_size}") + logger.info( + f"Number of FFTs set to {num_ffts} based on specified FFT size {fft_size}" + ) # Determine if truncation will occur and raise a warning if so if len(time_data) != fft_size * num_ffts: @@ -80,7 +95,9 @@ def get_fft( # Resize time data for FFTs time_data = np.reshape(time_data[: num_ffts * fft_size], (num_ffts, fft_size)) - logger.debug(f"Num. FFTs: {num_ffts}, FFT Size: {fft_size}, Data shape: {time_data.shape}") + logger.debug( + f"Num. FFTs: {num_ffts}, FFT Size: {fft_size}, Data shape: {time_data.shape}" + ) # Apply the FFT window if provided if fft_window is not None: @@ -91,7 +108,7 @@ def get_fft( # Shift the frequencies if desired (only along second axis) if shift: - complex_fft = np.fft.fftshift(complex_fft) #, axes=(1,)) + complex_fft = np.fft.fftshift(complex_fft) # , axes=(1,)) return complex_fft From 9825be606e8e594663f58d40288a035ac42b6f79 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Thu, 11 Aug 2022 17:37:52 -0600 Subject: [PATCH 141/157] Update FFT debug --- scos_actions/signal_processing/fft.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scos_actions/signal_processing/fft.py b/scos_actions/signal_processing/fft.py index 63c260f0..39c4ecd1 100644 --- a/scos_actions/signal_processing/fft.py +++ b/scos_actions/signal_processing/fft.py @@ -68,14 +68,14 @@ def get_fft( if isinstance(fft_size, int) and isinstance(num_ffts, int): pass else: - if isinstance(fft_size, float) and fft_size == int(fft_size): + if fft_size == int(fft_size): fft_size = int(fft_size) else: - raise ValueError("fft_size must be an integer.") - if isinstance(num_ffts, float) and num_ffts == int(num_ffts): + raise ValueError(f"fft_size must be an integer, not {type(fft_size)}.") + if num_ffts == int(num_ffts): num_ffts = int(num_ffts) else: - raise ValueError("num_ffts must be an integer.") + raise ValueError(f"num_ffts must be an integer, not {type(num_ffts)}.") # Get num_ffts for default case: as many as possible if num_ffts <= 0: From a56d2e0576c0cc7350a2952f9d2b0f13807ef701 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 12 Aug 2022 10:14:06 -0600 Subject: [PATCH 142/157] Switch Flit to Hatchling --- README.md | 21 +++++++++++++++------ pyproject.toml | 21 +++++++++++++-------- scos_actions/__init__.py | 4 ---- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ee974a19..e8c7cb86 100644 --- a/README.md +++ b/README.md @@ -160,8 +160,8 @@ pip install .[dev] ``` This will install the project itself, along with development dependencies for pre-commit -hooks, building distributions, and running tests. Set up pre-commit, which runs auto-formatting -and code-checking automatically when you make a commit, by running: +hooks, building distributions, and running tests. Set up pre-commit, which runs +auto-formatting and code-checking automatically when you make a commit, by running: ```bash pre-commit install @@ -173,12 +173,21 @@ can be found in [`.pre-commit-config.yaml`](.pre-commit-config.yaml). ### Building New Releases -This project uses [flit](https://github.com/pypa/flit) as a backend. To build a new release -(both wheel and sdist/tarball), first update the version number in -[`scos_actions/__init__.py`](scos_actions/__init__.py), then run: +This project uses [Hatchling](https://github.com/pypa/hatch/tree/master/backend) as a +backend. Hatchling makes versioning and building new releases easy. The package version can +be updated easily by using any of the following commands. ```bash -flit build +hatchling version major # 1.0.0 -> 2.0.0 +hatchling version minor # 1.0.0 -> 1.1.0 +hatchling version micro # 1.0.0 -> 1.0.1 +hatchling version "X.X.X" # 1.0.0 -> X.X.X +``` + +To build a new release (both wheel and sdist/tarball), run: + +```bash +hatchling build ``` ### Running Tests diff --git a/pyproject.toml b/pyproject.toml index f9bbf656..f73ea6d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,11 @@ [build-system] -requires = ["flit_core>=3.4,<4"] -build-backend = "flit_core.buildapi" +requires = ["hatchling"] +build-backend = "hatchling.build" [project] name = "scos-actions" -dynamic = ["version", "description"] +dynamic = ["version"] +description = "The base plugin providing common actions and interfaces for SCOS Sensor plugins" readme = "README.md" requires-python = ">=3.8" license = { file = "LICENSE.md" } @@ -41,22 +42,23 @@ classifiers = [ dependencies = [ "django>=3.2.15,<4.0", - "its_preselector @ https://github.com/NTIA/Preselector/archive/refs/tags/1.0.0.zip", + "its_preselector @ git+https://github.com/NTIA/Preselector@1.0.0", "numexpr>=2.8.3", "numpy>=1.22.0", "python-dateutil>=2.0", "ruamel.yaml>=0.15", "scipy>=1.8.0", - "sigmf @ https://github.com/NTIA/SigMF/archive/refs/heads/multi-recording-archive.zip" + "sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive", ] [project.optional-dependencies] dev = [ - "flit>=3.4,<4", + "hatchling>=1.6.0,<2.0", "pre-commit>=2.20.0", "pytest>=7.1.2", "pytest-cov>=3.0.0", "tox>=3.0", + "twine>=4.0.1,<5.0", ] [project.urls] @@ -66,5 +68,8 @@ dev = [ "NTIA GitHub" = "https://github.com/NTIA" "ITS Website" = "https://its.ntia.gov" -[tool.flit.module] -name = "scos_actions" +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.version] +path = "scos_actions/__init__.py" diff --git a/scos_actions/__init__.py b/scos_actions/__init__.py index f6584813..528787cf 100644 --- a/scos_actions/__init__.py +++ b/scos_actions/__init__.py @@ -1,5 +1 @@ -"""The base plugin providing common actions and interfaces for SCOS Sensor plugins. - -Refer to the SCOS Actions README file for detailed usage information. -""" __version__ = "3.0.0" From 5bfc401ff73f3d0706c93911286f482c569e2376 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 17 Aug 2022 15:10:58 -0600 Subject: [PATCH 143/157] Removed unused, commented code fragment --- scos_actions/signal_processing/fft.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scos_actions/signal_processing/fft.py b/scos_actions/signal_processing/fft.py index 39c4ecd1..531220e9 100644 --- a/scos_actions/signal_processing/fft.py +++ b/scos_actions/signal_processing/fft.py @@ -106,9 +106,9 @@ def get_fft( # Take the FFT complex_fft = sp_fft(time_data, norm=norm, workers=workers) - # Shift the frequencies if desired (only along second axis) + # Shift the frequencies if desired if shift: - complex_fft = np.fft.fftshift(complex_fft) # , axes=(1,)) + complex_fft = np.fft.fftshift(complex_fft) return complex_fft From 0a41351020c7dc1f7d57689788c84ba03f183f2a Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Wed, 17 Aug 2022 16:22:54 -0600 Subject: [PATCH 144/157] Remove hardcoded ENBW --- scos_actions/actions/calibrate_y_factor.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 5a54335a..9559c05b 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -233,7 +233,6 @@ def calibrate(self, params): cutoff_Hz = self.iir_cutoff_Hz width_Hz = self.iir_width_Hz enbw_hz = (cutoff_Hz + width_Hz) * 2. # Roughly based on IIR filter - # TODO: Verify this is an appropriate way to specify ENBW logger.debug("Applying IIR filter to IQ captures") noise_on_data = sosfilt(self.iir_sos, noise_on_measurement_result["data"]) noise_off_data = sosfilt(self.iir_sos, noise_off_measurement_result["data"]) @@ -242,9 +241,7 @@ def calibrate(self, params): # Get ENBW from sensor calibration cal_args = [sigan_params[k] for k in sensor_calibration.calibration_parameters] self.sigan.recompute_calibration_data(cal_args) - # TODO: Return this to be pulled from sensor cal file - # enbw_hz = self.sigan.sensor_calibration_data["enbw_sensor"] - enbw_hz = 11.607e6 + enbw_hz = self.sigan.sensor_calibration_data["enbw_sensor"] noise_on_data = noise_on_measurement_result["data"] noise_off_data = noise_off_measurement_result["data"] From e7165ed3a3033e018ca0c18d1aa9ba82486d4882 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 19 Aug 2022 12:58:04 -0600 Subject: [PATCH 145/157] Fixed Python version in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e8c7cb86..f8cec247 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ routines which are used in actions. ## Running in SCOS Sensor -Requires `git`, `python>=3.7`, `pip>=18.1`, and `pip-tools>=6.6.2`. +Requires `git`, `python>=3.8`, `pip>=18.1`, and `pip-tools>=6.6.2`. 1. Clone `scos-sensor`: From 3b0b750a4a18bdfdc24aeb6442f16208fd718bef Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 19 Aug 2022 13:02:48 -0600 Subject: [PATCH 146/157] Remove redundant documentation --- README.md | 97 +++++++++---------------------------------------------- 1 file changed, 16 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index f8cec247..5327993e 100644 --- a/README.md +++ b/README.md @@ -50,89 +50,24 @@ routines which are used in actions. ## Running in SCOS Sensor -Requires `git`, `python>=3.8`, `pip>=18.1`, and `pip-tools>=6.6.2`. +Refer to the [SCOS Sensor documentation](https://github.com/NTIA/scos-sensor#readme) for +detailed instructions. To run SCOS Actions in SCOS Sensor with a mock signal analyzer, +set `MOCK_SIGAN` and `MOCK_SIGAN_RANDOM` equal to 1 in `docker-compose.yml` before +starting SCOS Sensor: -1. Clone `scos-sensor`: - - ```bash - git clone https://github.com/NTIA/scos-sensor.git - ``` - -1. Navigate to the cloned `scos-sensor` directory: - - ```bash - cd scos-sensor - ``` - -1. If testing locally, generate the necessary SSL certificates by running: - - ```bash - cd scripts && ./create_localhost_cert.sh - ``` - -1. While in the `scos-sensor` directory, create the `env` file by copying the template file: - - ```bash - cp env.template ./env - ``` - -1. In the newly-created `env` file, set the `BASE_IMAGE`: - - ```bash - BASE_IMAGE=ubuntu:18.04 - ``` - -1. Get environment variables: - - ```bash - source ./env - ``` - -1. In `scos-sensor/src/requirements.in`, comment out any unnecessary dependencies (such -as `scos_usrp`). - -1. Make sure the `scos_actions` dependency is present in -`scos-sensor/src/requirements.in`, and add it if needed. If you are using a different -branch than shown in `requirements.in`, edit the file to point SCOS Sensor to the correct -branch of SCOS Actions. As an example, the following line in `requirements.in` would use -SCOS Actions v2.0.0: - - ```text - scos_actions @ git+https://github.com/NTIA/scos-actions@2.0.0 - ``` - -1. Compile requirements by running: - - ```bash - cd src - pip-compile requirements.in - pip-compile requirements-dev.in - ``` - -1. Set `MOCK_SIGAN` and `MOCK_SIGAN_RANDOM` equal to 1 in `docker-compose.yml`: - - ```yaml - services: +```yaml +services: + ... + api: + ... + environment: ... - api: - ... - environment: - ... - - MOCK_SIGAN=1 - - MOCK_SIGAN_RANDOM=1 - ``` - -1. Build and start containers (and optionally, view logs): - - ```bash - docker-compose build --no-cache - docker-compose up -d --force-recreate - docker-compose logs -f - ``` - -If SCOS Actions is installed to SCOS Sensor as a plugin, the following -parameterized actions are offered for testing using a mock signal analyzer; their -parameters are defined in `scos_actions/configs/actions`. + - MOCK_SIGAN=1 + - MOCK_SIGAN_RANDOM=1 +``` + +The following parameterized actions are offered for testing using a mock signal analyzer; +their parameters are defined in `scos_actions/configs/actions`. - `test_multi_frequency_iq_action` - `test_multi_frequency_y_factor_action` From 99ad2a5afc60afc764089116f32969197dbd3803 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 19 Aug 2022 13:07:37 -0600 Subject: [PATCH 147/157] Removed y-factor test action configs --- .../test_multi_frequency_y_factor_action.yml | 21 ------------------- .../test_single_frequency_y_factor_action.yml | 17 --------------- 2 files changed, 38 deletions(-) delete mode 100644 scos_actions/configs/actions/test_multi_frequency_y_factor_action.yml delete mode 100644 scos_actions/configs/actions/test_single_frequency_y_factor_action.yml diff --git a/scos_actions/configs/actions/test_multi_frequency_y_factor_action.yml b/scos_actions/configs/actions/test_multi_frequency_y_factor_action.yml deleted file mode 100644 index 023126a0..00000000 --- a/scos_actions/configs/actions/test_multi_frequency_y_factor_action.yml +++ /dev/null @@ -1,21 +0,0 @@ -y_factor_cal: - name: test_multi_frequency_y_factor_action - # Preselector configuration - cal_source_idx: 0 # Index of calibration source in preselector - temp_sensor_idx: 1 # Index of temperature sensor in preselector - # Sigan Settings - gain: 40 - sample_rate: 14e6 - duration_ms: 1000 - nskip: 0 - frequency: - - 3555e6 - - 3565e6 - - 3575e6 - - 3585e6 - # IIR Filter Settings - iir_apply: True - iir_rp_dB: 0.1 - iir_rs_dB: 40 - iir_cutoff_Hz: 5e6 - iir_width_Hz: 8e3 \ No newline at end of file diff --git a/scos_actions/configs/actions/test_single_frequency_y_factor_action.yml b/scos_actions/configs/actions/test_single_frequency_y_factor_action.yml deleted file mode 100644 index db22d38e..00000000 --- a/scos_actions/configs/actions/test_single_frequency_y_factor_action.yml +++ /dev/null @@ -1,17 +0,0 @@ -y_factor_cal: - name: test_single_frequency_y_factor_action - # Preselector configuration - cal_source_idx: 0 # Index of calibration source in preselector - temp_sensor_idx: 1 # Index of temperature sensor in preselector - # Sigan Settings - gain: 40 - sample_rate: 14e6 - duration_ms: 1000 - nskip: 0 - frequency: 3555e6 - # IIR Filter Settings - iir_apply: True - iir_rp_dB: 0.1 - iir_rs_dB: 40 - iir_cutoff_Hz: 5e6 - iir_width_Hz: 8e3 \ No newline at end of file From 0976a88aa7b2a445385829ed799701ebe32b5256 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 19 Aug 2022 15:47:45 -0600 Subject: [PATCH 148/157] Add bin_size 0 handling --- scos_actions/signal_processing/apd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scos_actions/signal_processing/apd.py b/scos_actions/signal_processing/apd.py index f6c77392..4600bf8d 100644 --- a/scos_actions/signal_processing/apd.py +++ b/scos_actions/signal_processing/apd.py @@ -30,6 +30,7 @@ def get_apd( :param time_data: Input complex baseband IQ samples. :param bin_size_dB: Amplitude granularity, in dB, for estimating the APD. If not specified, the APD will not be downsampled (default behavior). + Setting this to zero will also result in no downsampling. :return: A tuple (p, a) of NumPy arrays, where p contains the APD probabilities, and a contains the APD amplitudes. """ @@ -42,7 +43,7 @@ def get_apd( # Convert amplitudes from V to dBV all_amps = convert_linear_to_dB(all_amps) - if bin_size_dB is None: + if bin_size_dB is None or bin_size_dB == 0: # No downsampling a = np.sort(all_amps) p = 1 - ((np.arange(len(a)) + 1) / len(a)) From 868448cb6f0108b5937ce2459d23077fed189b63 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 19 Aug 2022 16:04:30 -0600 Subject: [PATCH 149/157] Added APD tests --- scos_actions/tests/test_apd.py | 78 ++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 scos_actions/tests/test_apd.py diff --git a/scos_actions/tests/test_apd.py b/scos_actions/tests/test_apd.py new file mode 100644 index 00000000..dc42226c --- /dev/null +++ b/scos_actions/tests/test_apd.py @@ -0,0 +1,78 @@ +import numpy as np +import pytest + +from scos_actions.signal_processing import apd + +rng = np.random.default_rng() + + +@pytest.fixture +def example_iq_data(): + """ + Generate complex samples, with real and imaginary parts + being independent normally distributed random variables, + with mean zero and variance 1/2. + """ + n_samps = 10000 + std_dev = np.sqrt(2) / 2.0 + samps = rng.normal(0, std_dev, n_samps) + 1j * rng.normal(0, std_dev, n_samps) + return samps + + +def test_get_apd_nan_handling(): + # All zero amplitudes should be converted to NaN + # Peak amplitude 0 count should be replaced with NaN + zero_amps = np.zeros(10) * (1 + 1j) + p, a = apd.get_apd(zero_amps) + assert np.isnan(p[-1]) + assert all(np.isnan(a)) + + +def test_get_apd_no_downsample(example_iq_data): + bin_sizes = [None, 0] + for bin_size in bin_sizes: + apd_result = apd.get_apd(example_iq_data, bin_size) + assert isinstance(apd_result, tuple) + assert len(apd_result) == 2 + assert all(isinstance(x, np.ndarray) for x in apd_result) + assert all(len(x) == len(example_iq_data) for x in apd_result) + p, a = apd_result + assert not any(x == 0 for x in a) + np.testing.assert_equal(a, np.real(a)) + assert all(a[i] <= a[i + 1] for i in range(len(a) - 1)) + assert max(p) < 1 + assert min(p) > 0 + assert all(p[i + 1] <= p[i] for i in range(len(p) - 2)) + assert np.isnan(p[-1]) + + +def test_get_apd_downsample(example_iq_data): + bin_sizes = [0.5, 0.25, 0.15] + for bin_size in bin_sizes: + p, a = apd.get_apd(example_iq_data, bin_size) + assert len(p) == len(a) + assert len(p) < len(example_iq_data) + assert not any(x == 0 for x in a) + np.testing.assert_equal(a, np.real(a)) + assert all(a[i] <= a[i + 1] for i in range(len(a) - 1)) + np.testing.assert_allclose( + np.diff(a), np.ones(len(a) - 1) * bin_size, rtol=1e-14, atol=0 + ) + assert max(p) < 1 + assert min(p) > 0 + assert all(p[i + 1] <= p[i] for i in range(len(p) - 2)) + assert np.isnan(p[-1]) + + +def test_sample_ccdf(): + example_ccdf_bins = np.arange(0, 51, 1) + example_ccdf_data = np.linspace(0, 50, 50) + ccdf = apd.sample_ccdf(example_ccdf_data, example_ccdf_bins, density=False) + ccdf_d = apd.sample_ccdf(example_ccdf_data, example_ccdf_bins, density=True) + assert len(ccdf) == len(example_ccdf_bins) + assert len(ccdf_d) == len(example_ccdf_bins) + assert isinstance(ccdf, np.ndarray) + assert isinstance(ccdf_d, np.ndarray) + np.testing.assert_equal(ccdf_d, ccdf / len(example_ccdf_data)) + assert all(ccdf[i + 1] <= ccdf[i] for i in range(len(ccdf) - 1)) + assert all(ccdf_d[i + 1] <= ccdf_d[i] for i in range(len(ccdf_d) - 1)) From f964d61881bb5b935660fc6581f0c27103ec87e5 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Fri, 19 Aug 2022 16:34:28 -0600 Subject: [PATCH 150/157] Consolidated configure_sigan --- scos_actions/actions/interfaces/action.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/scos_actions/actions/interfaces/action.py b/scos_actions/actions/interfaces/action.py index 7037bef7..6abb7cba 100644 --- a/scos_actions/actions/interfaces/action.py +++ b/scos_actions/actions/interfaces/action.py @@ -37,26 +37,19 @@ def __init__(self, parameters, sigan=mock_sigan, gps=mock_gps): self.gps = gps self.sensor_definition = capabilities['sensor'] - def configure(self, measurement_params): + def configure(self, measurement_params: dict): self.configure_sigan(measurement_params) self.configure_preselector(measurement_params) - def configure_sigan(self, measurement_params): - if isinstance(measurement_params, list): - for item in measurement_params: - self.configure_sigan_with_dictionary(item) - - elif isinstance(measurement_params, dict): - self.configure_sigan_with_dictionary(measurement_params) - - def configure_sigan_with_dictionary(self, dictionary): - for key, value in dictionary.items(): + def configure_sigan(self, measurement_params: dict): + for key, value in measurement_params.items(): if hasattr(self.sigan, key): + logger.debug(f"Applying setting to sigan: {key}: {value}") setattr(self.sigan, key, value) else: - logger.warning(f"radio does not have attribute {key}") + logger.warning(f"Sigan does not have attribute {key}") - def configure_preselector(self, measurement_params): + def configure_preselector(self, measurement_params: dict): if self.PRESELECTOR_PATH_KEY in measurement_params: path = measurement_params[self.PRESELECTOR_PATH_KEY] preselector.set_state(path) From 4ab0465aee0c5ae0ea81ed0f1acc1553124862e9 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 22 Aug 2022 10:49:39 -0600 Subject: [PATCH 151/157] Parameterized sortby key in iter_params --- scos_actions/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scos_actions/utils.py b/scos_actions/utils.py index 0ba45dfe..bdfff9f7 100644 --- a/scos_actions/utils.py +++ b/scos_actions/utils.py @@ -41,7 +41,7 @@ def load_from_json(fname): logger.exception("Unable to load JSON file {}".format(fname)) -def get_iterable_parameters(parameters: dict): +def get_iterable_parameters(parameters: dict, sortby: str = "frequency"): """ Convert parameter dictionary into iterable list. @@ -58,10 +58,12 @@ def get_iterable_parameters(parameters: dict): to use the same gain value for all measurements in a stepped-frequency acquisition. - The output list is automatically sorted by frequency, but it can - be manually resorted by any key if desired. + The output list is automatically sorted by the key provided as the + ``sortby`` parameter. By default, ``sortby`` is "frequency". :param parameters: The parameter dictionary, as loaded by the action. + :param sortby: The key to sort the resulting list by, in ascending order. + Defaults to "frequency". :return: An iterable list of parameter dictionaries based on the input. If only single values are given for all parameters in the input, a list will still be returned, containing a single dictionary. @@ -90,7 +92,7 @@ def get_iterable_parameters(parameters: dict): raise ParameterException(msg) # Construct iterable parameter mapping result = [dict(zip(params, v)) for v in zip(*params.values())] - result.sort(key=lambda param: param["frequency"]) + result.sort(key=lambda param: param[sortby]) return result From 5317394015fe1962aa7039d7f7b1bb5d44ea0c3c Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 22 Aug 2022 11:10:51 -0600 Subject: [PATCH 152/157] Added more tests for utils --- scos_actions/tests/test_utils.py | 127 ++++++++++++++++++++++++++++--- 1 file changed, 116 insertions(+), 11 deletions(-) diff --git a/scos_actions/tests/test_utils.py b/scos_actions/tests/test_utils.py index 90cc971a..9f8297d6 100644 --- a/scos_actions/tests/test_utils.py +++ b/scos_actions/tests/test_utils.py @@ -1,15 +1,120 @@ -import unittest +import pytest from scos_actions import utils +import datetime +from dateutil import tz +import numpy as np -class MyTestCase(unittest.TestCase): - def test_get_parameters(self): - parameters = {"name": 'test_params', 'frequency': [100,200,300], 'gain': [0,10,40], 'sample_rate': [1, 2,3]} - iteration_params = utils.get_iterable_parameters(parameters) - self.assertEqual(3, len(iteration_params)) - self.assertEqual(iteration_params[0]['frequency'], 100) - self.assertEqual(iteration_params[0]['gain'], 0) - self.assertEqual(iteration_params[0]['sample_rate'], 1) +@pytest.fixture +def valid_params_no_lists(): + return {"name": "valid_params_no_list", "sample_rate": 14e6, "frequency": 770e6} -if __name__ == '__main__': - unittest.main() +def test_get_datetime_str_now(): + datetime_str = utils.get_datetime_str_now() + assert len(datetime_str) == 24 + assert all(datetime_str[i] == "-" for i in [4, 7]) + assert datetime_str[10] == "T" + assert all(datetime_str[i] == ":" for i in [13, 16]) + assert datetime_str[19] == "." + assert datetime_str[-1] == "Z" + +def test_parse_datetime_iso_format_str(): + tstamp = utils.get_datetime_str_now() + parsed = utils.parse_datetime_iso_format_str(tstamp) + assert type(parsed) is datetime.datetime + # 2022-08-22T16:22:11.833Z + assert type(parsed.year) is int + assert parsed.year == int(tstamp[:4]) + assert type(parsed.month) is int + assert parsed.month == int(tstamp[5:7]) + assert type(parsed.day) is int + assert parsed.day == int(tstamp[8:10]) + assert type(parsed.hour) is int + assert parsed.hour == int(tstamp[11:13]) + assert type(parsed.minute) is int + assert parsed.minute == int(tstamp[14:16]) + assert type(parsed.second) is int + assert parsed.second == int(tstamp[17:19]) + assert type(parsed.microsecond) is int + assert parsed.microsecond == int(tstamp[20:23] + "000") + assert type(parsed.tzinfo) is tz.tz.tzutc + +# def test_get_parameters(): +# parameters = {"name": 'test_params', 'frequency': [100,200,300], 'gain': [0,10,40], 'sample_rate': [1, 2,3]} +# iteration_params = utils.get_iterable_parameters(parameters) +# assert len(iteration_params) == 3 +# assert iteration_params[0]['frequency'] == 100 +# assert iteration_params[0]['gain'] == 0 +# assert iteration_params[0]['sample_rate'] == 1 + +def test_get_iterable_parameters_no_lists(valid_params_no_lists): + i_params = utils.get_iterable_parameters(valid_params_no_lists) + assert type(i_params) is list + assert len(i_params) == 1 + with pytest.raises(KeyError): + _ = utils.get_iterable_parameters(valid_params_no_lists, "gain") + assert not any("name" in x.keys() for x in i_params) + valid_params_no_lists.pop("name") + assert all(i_params[0][k] == valid_params_no_lists[k] for k in valid_params_no_lists.keys()) + +def test_get_iterable_parameters_all_lists(): + all_lists = { + "name": "params_all_lists", + "sample_rate": [14e6, 28e6, 56e6], + "frequency": [720e6, 710e6, 700e6] + } + i_params = utils.get_iterable_parameters(all_lists) + assert type(i_params) is list + assert len(i_params) == 3 + assert not any("name" in x.keys() for x in i_params) + minf = min(all_lists["frequency"]) + maxf = max(all_lists["frequency"]) + assert i_params[0]["frequency"] == minf + assert i_params[0]["sample_rate"] == all_lists["sample_rate"][all_lists["frequency"].index(minf)] + assert i_params[-1]["frequency"] == maxf + assert i_params[-1]["sample_rate"] == all_lists["sample_rate"][all_lists["frequency"].index(maxf)] + +def test_get_iterable_parameters_some_lists(): + some_lists = { + "name": "some_lists", + "sample_rate": 14e6, + "frequency": [720e6, 705e6, 1000e6, 10], + "gain": [1, 2, 3, 4] + } + i_params = utils.get_iterable_parameters(some_lists) + assert type(i_params) is list + assert len(i_params) == 4 + assert not any("name" in x.keys() for x in i_params) + minf = min(some_lists["frequency"]) + maxf = max(some_lists["frequency"]) + aminf = some_lists["frequency"].index(minf) + amaxf = some_lists["frequency"].index(maxf) + assert i_params[0]["frequency"] == minf + assert i_params[0]["sample_rate"] == some_lists["sample_rate"] + assert i_params[0]["gain"] == some_lists["gain"][aminf] + assert i_params[-1]["frequency"] == maxf + assert i_params[-1]["sample_rate"] == some_lists["sample_rate"] + assert i_params[-1]["gain"] == some_lists["gain"][amaxf] + +def test_get_iterable_parameters_incompatible_lists(): + incompat_lists = { + "name": "incompatible_lists", + "sample_rate": [14e6, 28e6, 56e6], + "frequency": 700e6, + "gain": [1, 2] + } + with pytest.raises(utils.ParameterException): + _ = utils.get_iterable_parameters(incompat_lists) + +def test_list_to_string(): + ex_list = [1, 2.0, "testing"] + test_str = utils.list_to_string(ex_list) + assert type(test_str) is str + assert test_str == "1,2.0,testing" + +def test_get_parameter(valid_params_no_lists): + valid_key = "sample_rate" + invalid_key = "attenuation" + assert utils.get_parameter(valid_key, valid_params_no_lists) == 14e6 + with pytest.raises(utils.ParameterException): + utils.get_parameter(invalid_key, valid_params_no_lists) From b279187718d26eeabf845a147c5d077eaebc6f4d Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 22 Aug 2022 11:45:16 -0600 Subject: [PATCH 153/157] Added set_first attributes to configure_sigan --- scos_actions/actions/interfaces/action.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scos_actions/actions/interfaces/action.py b/scos_actions/actions/interfaces/action.py index 6abb7cba..86205040 100644 --- a/scos_actions/actions/interfaces/action.py +++ b/scos_actions/actions/interfaces/action.py @@ -42,6 +42,14 @@ def configure(self, measurement_params: dict): self.configure_preselector(measurement_params) def configure_sigan(self, measurement_params: dict): + # List of attributes which must be set first + set_first = ["preamp_enable"] + for k in set_first: + if k in measurement_params.keys() and hasattr(self.sigan, k): + logger.debug(f"Applying setting to sigan: {k}: {measurement_params[k]}") + setattr(self.sigan, k, measurement_params[k]) + measurement_params.pop(k) + # Set remaining attributes for key, value in measurement_params.items(): if hasattr(self.sigan, key): logger.debug(f"Applying setting to sigan: {key}: {value}") From 7d25facd275a1c15d9c11b6f123eced49e197544 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 22 Aug 2022 11:46:32 -0600 Subject: [PATCH 154/157] Removed old unused test code --- scos_actions/tests/test_utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scos_actions/tests/test_utils.py b/scos_actions/tests/test_utils.py index 9f8297d6..6f8a1043 100644 --- a/scos_actions/tests/test_utils.py +++ b/scos_actions/tests/test_utils.py @@ -39,14 +39,6 @@ def test_parse_datetime_iso_format_str(): assert parsed.microsecond == int(tstamp[20:23] + "000") assert type(parsed.tzinfo) is tz.tz.tzutc -# def test_get_parameters(): -# parameters = {"name": 'test_params', 'frequency': [100,200,300], 'gain': [0,10,40], 'sample_rate': [1, 2,3]} -# iteration_params = utils.get_iterable_parameters(parameters) -# assert len(iteration_params) == 3 -# assert iteration_params[0]['frequency'] == 100 -# assert iteration_params[0]['gain'] == 0 -# assert iteration_params[0]['sample_rate'] == 1 - def test_get_iterable_parameters_no_lists(valid_params_no_lists): i_params = utils.get_iterable_parameters(valid_params_no_lists) assert type(i_params) is list From d1ba989cc5115a6b8cec6d8cf74dedc69b34d5e9 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 22 Aug 2022 16:36:28 -0600 Subject: [PATCH 155/157] Added IIR filter ENBW calculation --- scos_actions/actions/calibrate_y_factor.py | 172 ++++++++++++-------- scos_actions/signal_processing/filtering.py | 161 +++++++++++++++--- 2 files changed, 244 insertions(+), 89 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 9559c05b..4e13abd1 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -69,52 +69,52 @@ """ import logging +import os import time -import numpy as np +import numpy as np from scipy.constants import Boltzmann from scipy.signal import sosfilt from scos_actions import utils -from scos_actions.hardware import gps as mock_gps -from scos_actions.settings import sensor_calibration -from scos_actions.settings import SENSOR_CALIBRATION_FILE from scos_actions.actions.interfaces.action import Action -from scos_actions.utils import ParameterException, get_parameter - +from scos_actions.hardware import gps as mock_gps +from scos_actions.settings import SENSOR_CALIBRATION_FILE, sensor_calibration from scos_actions.signal_processing.calibration import ( get_linear_enr, get_temperature, y_factor, ) +from scos_actions.signal_processing.filtering import ( + generate_elliptic_iir_low_pass_filter, + get_iir_enbw, +) from scos_actions.signal_processing.power_analysis import ( calculate_power_watts, create_power_detector, ) - from scos_actions.signal_processing.unit_conversion import convert_watts_to_dBm -from scos_actions.signal_processing.filtering import generate_elliptic_iir_low_pass_filter - -import os +from scos_actions.utils import ParameterException, get_parameter logger = logging.getLogger(__name__) -RF_PATH = 'rf_path' -NOISE_DIODE_ON = {RF_PATH: 'noise_diode_on'} -NOISE_DIODE_OFF = {RF_PATH: 'noise_diode_off'} +RF_PATH = "rf_path" +NOISE_DIODE_ON = {RF_PATH: "noise_diode_on"} +NOISE_DIODE_OFF = {RF_PATH: "noise_diode_off"} # Define parameter keys FREQUENCY = "frequency" SAMPLE_RATE = "sample_rate" DURATION_MS = "duration_ms" NUM_SKIP = "nskip" -IIR_APPLY = 'iir_apply' -IIR_RP = 'iir_rp_dB' -IIR_RS = 'iir_rs_dB' -IIR_CUTOFF = 'iir_cutoff_Hz' -IIR_WIDTH = 'iir_width_Hz' -CAL_SOURCE_IDX = 'cal_source_idx' -TEMP_SENSOR_IDX = 'temp_sensor_idx' +IIR_APPLY = "iir_apply" +IIR_GPASS = "iir_gpass_dB" +IIR_GSTOP = "iir_gstop_dB" +IIR_PB_EDGE = "iir_pb_edge_Hz" +IIR_SB_EDGE = "iir_sb_edge_Hz" +IIR_RESP_FREQS = "iir_num_response_frequencies" +CAL_SOURCE_IDX = "cal_source_idx" +TEMP_SENSOR_IDX = "temp_sensor_idx" class YFactorCalibration(Action): @@ -139,7 +139,7 @@ class YFactorCalibration(Action): """ def __init__(self, parameters, sigan, gps=mock_gps): - logger.debug('Initializing calibration action') + logger.debug("Initializing calibration action") super().__init__(parameters, sigan, gps) self.sigan = sigan self.iteration_params = utils.get_iterable_parameters(parameters) @@ -149,32 +149,56 @@ def __init__(self, parameters, sigan, gps=mock_gps): try: self.iir_apply = get_parameter(IIR_APPLY, parameters) except ParameterException: - logger.info("Config parameter 'iir_apply' not provided. " - + "No IIR filtering will be used during calibration.") + logger.info( + "Config parameter 'iir_apply' not provided. " + + "No IIR filtering will be used during calibration." + ) self.iir_apply = False if isinstance(self.iir_apply, list): - raise ParameterException("Only one set of IIR filter parameters may be specified.") - + raise ParameterException( + "Only one set of IIR filter parameters may be specified." + ) + if self.iir_apply is True: - self.iir_rp_dB = get_parameter(IIR_RP, parameters) - self.iir_rs_dB = get_parameter(IIR_RS, parameters) - self.iir_cutoff_Hz = get_parameter(IIR_CUTOFF, parameters) - self.iir_width_Hz = get_parameter(IIR_WIDTH, parameters) + self.iir_gpass_dB = get_parameter(IIR_GPASS, parameters) + self.iir_gstop_dB = get_parameter(IIR_GSTOP, parameters) + self.iir_pb_edge_Hz = get_parameter(IIR_PB_EDGE, parameters) + self.iir_sb_edge_Hz = get_parameter(IIR_SB_EDGE, parameters) + self.iir_num_response_frequencies = get_parameter( + IIR_RESP_FREQS, parameters + ) self.sample_rate = get_parameter(SAMPLE_RATE, parameters) - if not any([isinstance(v, list) for v in [self.iir_rp_dB, self.iir_rs_dB, self.iir_cutoff_Hz, self.iir_width_Hz, self.sample_rate]]): + if not any( + [ + isinstance(v, list) + for v in [ + self.iir_gpass_dB, + self.iir_gstop_dB, + self.iir_pb_edge_Hz, + self.iir_sb_edge_Hz, + self.sample_rate, + ] + ] + ): # Generate single filter ahead of calibration loop self.iir_sos = generate_elliptic_iir_low_pass_filter( - self.iir_rp_dB, self.iir_rs_dB, self.iir_cutoff_Hz, self.iir_width_Hz, self.sample_rate + self.iir_gpass_dB, + self.iir_gstop_dB, + self.iir_pb_edge_Hz, + self.iir_sb_edge_Hz, + self.sample_rate, ) else: - raise ParameterException("Only one set of IIR filter parameters may be specified (including sample rate).") + raise ParameterException( + "Only one set of IIR filter parameters may be specified (including sample rate)." + ) def __call__(self, schedule_entry_json, task_id): """This is the entrypoint function called by the scheduler.""" self.test_required_components() - detail = '' - + detail = "" + # Run calibration routine for i, p in enumerate(self.iteration_params): if i == 0: @@ -187,7 +211,17 @@ def calibrate(self, params): # Configure signal analyzer sigan_params = params.copy() # Suppress warnings during sigan configuration - for k in [DURATION_MS, NUM_SKIP, IIR_APPLY, IIR_RP, IIR_RS, IIR_CUTOFF, IIR_WIDTH, CAL_SOURCE_IDX, TEMP_SENSOR_IDX]: + for k in [ + DURATION_MS, + NUM_SKIP, + IIR_APPLY, + IIR_GPASS, + IIR_GSTOP, + IIR_PB_EDGE, + IIR_SB_EDGE, + CAL_SOURCE_IDX, + TEMP_SENSOR_IDX, + ]: try: sigan_params.pop(k) except KeyError: @@ -203,11 +237,11 @@ def calibrate(self, params): duration_ms = get_parameter(DURATION_MS, params) num_samples = int(sample_rate * duration_ms * 1e-3) nskip = get_parameter(NUM_SKIP, params) - + # Set noise diode on - logger.debug('Setting noise diode on') + logger.debug("Setting noise diode on") super().configure_preselector(NOISE_DIODE_ON) - time.sleep(.25) + time.sleep(0.25) # Get noise diode on IQ logger.debug("Acquiring IQ samples with noise diode ON") @@ -217,37 +251,42 @@ def calibrate(self, params): sample_rate = noise_on_measurement_result["sample_rate"] # Set noise diode off - logger.debug('Setting noise diode off') + logger.debug("Setting noise diode off") self.configure_preselector(NOISE_DIODE_OFF) - time.sleep(.25) + time.sleep(0.25) # Get noise diode off IQ - logger.debug('Acquiring IQ samples with noise diode OFF') + logger.debug("Acquiring IQ samples with noise diode OFF") noise_off_measurement_result = self.sigan.acquire_time_domain_samples( num_samples, num_samples_skip=nskip, gain_adjust=False ) - assert sample_rate == noise_off_measurement_result["sample_rate"], "Sample rate mismatch" + assert ( + sample_rate == noise_off_measurement_result["sample_rate"] + ), "Sample rate mismatch" # Apply IIR filtering to both captures if configured if self.iir_apply: - cutoff_Hz = self.iir_cutoff_Hz - width_Hz = self.iir_width_Hz - enbw_hz = (cutoff_Hz + width_Hz) * 2. # Roughly based on IIR filter + # Estimate of IIR filter ENBW does NOT account for passband ripple in sensor transfer function! + enbw_hz = get_iir_enbw( + self.iir_sos, self.iir_num_response_frequencies, sample_rate + ) logger.debug("Applying IIR filter to IQ captures") noise_on_data = sosfilt(self.iir_sos, noise_on_measurement_result["data"]) noise_off_data = sosfilt(self.iir_sos, noise_off_measurement_result["data"]) else: - logger.debug('Skipping IIR filtering') + logger.debug("Skipping IIR filtering") # Get ENBW from sensor calibration - cal_args = [sigan_params[k] for k in sensor_calibration.calibration_parameters] + cal_args = [ + sigan_params[k] for k in sensor_calibration.calibration_parameters + ] self.sigan.recompute_calibration_data(cal_args) enbw_hz = self.sigan.sensor_calibration_data["enbw_sensor"] noise_on_data = noise_on_measurement_result["data"] noise_off_data = noise_off_measurement_result["data"] # Get power values in time domain (division by 2 for RF/baseband conversion) - pwr_on_watts = calculate_power_watts(noise_on_data / 2.) - pwr_off_watts = calculate_power_watts(noise_off_data / 2.) + pwr_on_watts = calculate_power_watts(noise_on_data / 2.0) + pwr_off_watts = calculate_power_watts(noise_off_data / 2.0) # Y-Factor enr_linear = get_linear_enr(cal_source_idx) @@ -268,12 +307,12 @@ def calibrate(self, params): # Debugging noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz) - logger.debug(f'Noise floor: {noise_floor_dBm:.2f} dBm') - logger.debug(f'Noise figure: {noise_figure:.2f} dB') + logger.debug(f"Noise floor: {noise_floor_dBm:.2f} dBm") + logger.debug(f"Noise figure: {noise_figure:.2f} dB") logger.debug(f"Gain: {gain:.2f} dB") - + # Detail results contain only FFT version of result for now - return 'Noise Figure: {}, Gain: {}'.format(noise_figure, gain) + return "Noise Figure: {}, Gain: {}".format(noise_figure, gain) @property def description(self): @@ -289,7 +328,7 @@ def description(self): duration_ms = duration_ms * np.ones_like(sample_rate) duration_ms = duration_ms - num_samples = duration_ms * sample_rate * 1e-3 + num_samples = duration_ms * sample_rate * 1e-3 if isinstance(num_samples, np.ndarray) and len(num_samples) != 1: num_samples = num_samples.tolist() @@ -297,25 +336,22 @@ def description(self): num_samples = int(num_samples) if self.iir_apply is True: - pb_edge = self.iir_cutoff_Hz / 1e6 - sb_edge = (self.iir_cutoff_Hz + self.iir_width_Hz) / 1e6 filtering_suffix = ", after applying an IIR lowpass filter to the complex time-domain samples" - filter_description = ( - """ + filter_description = f""" ### Filtering The acquired samples are then filtered using an elliptic IIR filter before performing the rest of the time-domain Y-factor calculations. The filter design produces the lowest order digital filter which loses no more than - {self.iir_rp_dB} dB in the passband and has at least {self.iir_rs_dB} dB attenuation - in the stopband. The filter has a defined passband edge at {pb_edge} MHz - and a stopband edge at {sb_edge} MHz. From this filter design, second-order filter - coefficients are generated in order to minimize numerical precision errors - when filtering the time domain samples. The filtering function is implemented - as a series of second-order filters with direct-form II transposed structure. + {self.iir_gpass_dB} dB in the passband and has at least {self.iir_gstop_dB} + dB attenuation in the stopband. The filter has a defined passband edge at + {self.iir_pb_edge_Hz / 1e6} MHz and a stopband edge at {self.iir_sb_edge_Hz / 1e6} + MHz. From this filter design, second-order filter coefficients are generated in + order to minimize numerical precision errors when filtering the time domain samples. + The filtering function is implemented as a series of second-order filters with direct- + form II transposed structure. ### Power Calculation """ - ) else: filtering_suffix = "" filter_description = "" @@ -338,7 +374,7 @@ def description(self): "duration_ms": params[DURATION_MS], } ) - + definitions = { "name": self.name, "filtering_suffix": filtering_suffix, @@ -346,12 +382,10 @@ def description(self): "acquisition_plan": acquisition_plan, } # __doc__ refers to the module docstring at the top of the file - return __doc__ .format(**definitions) + return __doc__.format(**definitions) def test_required_components(self): """Fail acquisition if a required component is not available.""" if not self.sigan.is_available: msg = "acquisition failed: signal analyzer required but not available" raise RuntimeError(msg) - - diff --git a/scos_actions/signal_processing/filtering.py b/scos_actions/signal_processing/filtering.py index 6a581f62..9222af13 100644 --- a/scos_actions/signal_processing/filtering.py +++ b/scos_actions/signal_processing/filtering.py @@ -1,44 +1,65 @@ import logging +from multiprocessing.sharedctypes import Value +from typing import Tuple, Union + +import numexpr as ne import numpy as np -from scipy.signal import ellip, ellipord, kaiserord, firwin +from scipy.signal import ellip, ellipord, firwin, kaiserord, sos2zpk, sosfreqz + +from scos_actions.signal_processing.unit_conversion import convert_linear_to_dB logger = logging.getLogger(__name__) + def generate_elliptic_iir_low_pass_filter( - rp_dB: float, - rs_dB: float, - cutoff_Hz: float, - width_Hz: float, - sample_rate_Hz: float, + gpass_dB: float, + gstop_dB: float, + pb_edge_Hz: float, + sb_edge_Hz: float, + sample_rate_Hz: float, ) -> np.ndarray: """ Generate an elliptic IIR low pass filter. + This method generates a second-order sections representation of + the lowest order digital elliptic filter which loses no more than + ``gpass_dB`` dB in the passband and has at least ``gstop_dB`` + attenuation in the stopband. The passband and stopband are defined + by their edge frequencies, ``pb_edge_Hz`` and ``sb_edge_Hz``. + Apply this filter to data using scipy.signal.sosfilt or scipy.signal.sosfiltfilt (for forwards-backwards filtering). - :param rp_dB: Maximum passband ripple below unity gain, in dB. - :param rs_dB: Minimum stopband attenuation, in dB. - :param cutoff_Hz: Filter cutoff frequency, in Hz. - :param width_Hz: Passband-to-stopband transition width, in Hz. + :param gpass_dB: Maximum passband ripple below unity gain, in dB. + :param gstop_dB: Minimum stopband attenuation, in dB. + :param pb_edge_Hz: Filter passband edge frequency, in Hz. + :param sb_edge_Hz: Filter stopband edge frequency, in Hz. :param sample_rate_Hz: Sampling rate, in Hz. :return: Second-order sections representation of the IIR filter. """ - ord, wn = ellipord(cutoff_Hz, cutoff_Hz + width_Hz, rp_dB, rs_dB, False, sample_rate_Hz) - sos = ellip(ord, rp_dB, rs_dB, wn, 'lowpass', False, 'sos', sample_rate_Hz) - logger.debug(f'Generated low-pass IIR filter with order {ord}.') + if sb_edge_Hz <= pb_edge_Hz: + raise ValueError( + f"Stopband edge frequency {sb_edge_Hz} Hz is not greater than passband" + + f"edge frequency {pb_edge_Hz} Hz." + ) + ord, wn = ellipord( + pb_edge_Hz, sb_edge_Hz, gpass_dB, gstop_dB, False, sample_rate_Hz + ) + sos = ellip(ord, gpass_dB, gstop_dB, wn, "lowpass", False, "sos", sample_rate_Hz) + logger.debug(f"Generated low-pass IIR filter with order {ord}.") + print(f"FILTER ORDER: {ord}") return sos def generate_fir_low_pass_filter( - attenuation_dB: float, - width_Hz: float, - cutoff_Hz: float, - sample_rate_Hz: float + attenuation_dB: float, width_Hz: float, cutoff_Hz: float, sample_rate_Hz: float ) -> np.ndarray: """ Generate a FIR low pass filter using the Kaiser window method. + This method computes the coefficients of a finite impulse + response filter, with linear phase, + Apply this filter to data using scipy.signal.lfilter or scipy.signal.filtfilt (for forwards-backwards filtering). In either case, use the coefficients output by this method as @@ -51,6 +72,106 @@ def generate_fir_low_pass_filter( :return: Coeffiecients of the FIR low pass filter. """ ord, beta = kaiserord(attenuation_dB, width_Hz / (0.5 * sample_rate_Hz)) - taps = firwin(ord + 1, cutoff_Hz, width_Hz, ('kaiser', beta), 'lowpass', True, fs=sample_rate_Hz) - logger.debug(f"Generated Type {'I' if ord % 2 == 0 else 'II'} low-pass FIR filter with order {ord} and length {ord + 1}.") - return taps \ No newline at end of file + taps = firwin( + ord + 1, + cutoff_Hz, + width_Hz, + ("kaiser", beta), + "lowpass", + True, + fs=sample_rate_Hz, + ) + logger.debug( + f"Generated Type {'I' if ord % 2 == 0 else 'II'} low-pass FIR filter with order {ord} and length {ord + 1}." + ) + return taps + + +def get_iir_frequency_response( + sos: np.ndarray, worN: Union[int, np.ndarray], sample_rate_Hz: float +) -> Tuple[np.ndarray, np.ndarray]: + """ + Get the frequency response of an IIR filter. + + :param sos: Second-order sections representation of the IIR filter. + :param worN: If a single integer, then compute at that many frequencies. + If an array is supplied, it should be the frequencies at which to + compute the frequency response (in Hz). + :param sample_rate_Hz: Sampling rate, in Hz. + :return: A tuple containing two NumPy arrays. The first is the array of + frequencies, in Hz, for which the frequency response was calculated. + The second is the array containing the frequency response values, which + are complex values in linear units. + """ + w, h = sosfreqz(sos, worN, whole=True, fs=sample_rate_Hz) + return w, h + + +def get_iir_phase_response( + sos: np.ndarray, worN: Union[int, np.ndarray], sample_rate_Hz: float +) -> Tuple[np.ndarray, np.ndarray]: + """ + Get the phase response of an IIR filter. + + :param sos: Second-order sections representation of the IIR filter. + :param worN: If a single integer, then compute at that many frequencies. + If an array is supplied, it should be the frequencies at which to + compute the phase response (in Hz). + :param sample_rate_Hz: Sampling rate, in Hz. + :return: A tuple containing two NumPy arrays. The first is the array of + frequencies, in Hz, for which the phase response was calculated. + The second is the array containing the phase response values, in radians. + """ + w, h = sosfreqz(sos, worN, whole=False, fs=sample_rate_Hz) + angles = np.unwrap(np.angle(h)) + return w, angles + + +def get_iir_enbw( + sos: np.ndarray, worN: Union[int, np.ndarray], sample_rate_Hz: float +) -> float: + """ + Get the equivalent noise bandwidth of an IIR filter. + + :param sos: Second-order sections representation of the IIR filter. + :param worN: If a single integer, then compute at that many frequencies. + If an array is supplied, it should be the frequencies at which to + compute the frequency response (in Hz) to estimate the ENBW. The frequencies + should span from ``-sample_rate_Hz / 2`` to ``+sample_rate_Hz / 2``. + :param sample_rate_Hz: Sampling rate, in Hz. + :return: The equivalent noise bandwidth of the input filter, in Hz. + """ + if isinstance(worN, float) and worN.is_integer(): + worN = int(worN) + if isinstance(worN, int): + worN = np.linspace(-sample_rate_Hz / 2, sample_rate_Hz / 2, num=worN) + if not isinstance(worN, np.ndarray): + raise TypeError(f"Parameter worN must be int or np.ndarray, not {type(worN)}.") + if min(worN) < -sample_rate_Hz / 2 or max(worN) > sample_rate_Hz / 2: + raise ValueError( + "Supplied frequency values must fall within +/- Nyquist frequency at baseband." + ) + logger.debug( + f"Calculating filter ENBW using a frequency response of {len(worN)} points " + + f"from {min(worN)} Hz to {max(worN)} Hz." + ) + w, h = get_iir_frequency_response(sos, worN, sample_rate_Hz) + dw = np.mean(np.diff(w)) + h = np.abs(h) ** 2.0 + enbw = np.sum(h) * (dw / h.max()) + return enbw + + +def is_stable(sos: np.ndarray) -> bool: + """ + Check IIR filter stability using Z-plane analysis. + + An IIR filter is stable if its poles lie within the + unit circle on the Z-plane. + + :param sos: Second-order sections representation of the IIR filter. + :return: True if the filter is stable, False if not. + """ + _, poles, _ = sos2zpk(sos) + stable = all([True if p < 1 else False for p in np.square(np.abs(poles))]) + return stable From 6a15e6704f0aa0cdebd76f8e6b9f6c9e3c344586 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Mon, 22 Aug 2022 17:15:24 -0600 Subject: [PATCH 156/157] Improved error message --- scos_actions/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scos_actions/utils.py b/scos_actions/utils.py index bdfff9f7..c02e8683 100644 --- a/scos_actions/utils.py +++ b/scos_actions/utils.py @@ -111,5 +111,7 @@ def get_parameter(p: str, params: dict): :raises ParameterException: If p is not a key in params. """ if p not in params: - raise ParameterException(f"{p} missing from measurement parameters.") + raise ParameterException( + f"{p} missing from measurement parameters." + + f"Available parameters: {params}") return params[p] From f18bc3f26821bab4805658d2c6060205c9f01184 Mon Sep 17 00:00:00 2001 From: Anthony Romaniello Date: Tue, 23 Aug 2022 09:49:34 -0600 Subject: [PATCH 157/157] Removed configure fix which is no longer needed --- scos_actions/actions/interfaces/action.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scos_actions/actions/interfaces/action.py b/scos_actions/actions/interfaces/action.py index 86205040..6abb7cba 100644 --- a/scos_actions/actions/interfaces/action.py +++ b/scos_actions/actions/interfaces/action.py @@ -42,14 +42,6 @@ def configure(self, measurement_params: dict): self.configure_preselector(measurement_params) def configure_sigan(self, measurement_params: dict): - # List of attributes which must be set first - set_first = ["preamp_enable"] - for k in set_first: - if k in measurement_params.keys() and hasattr(self.sigan, k): - logger.debug(f"Applying setting to sigan: {k}: {measurement_params[k]}") - setattr(self.sigan, k, measurement_params[k]) - measurement_params.pop(k) - # Set remaining attributes for key, value in measurement_params.items(): if hasattr(self.sigan, key): logger.debug(f"Applying setting to sigan: {key}: {value}")