diff --git a/.gitignore b/.gitignore index c192a97a4..ec023447e 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,7 @@ lib/ /sampling/truth_data *.pyc report.xml + +# Rubocop output +/.rubocop +/.rubocop-http* \ No newline at end of file diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 000000000..29a76b130 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,25 @@ + +AllCops: + Exclude: + - 'build/**/*' + - 'documentation/**/*' + - 'national/**/*' + - 'postprocessing/**/*' + - 'samples/**/*' + - 'sampling/**/*' + - 'test/**/*' + - 'ymls/**/*' + NewCops: enable + +inherit_from: + - http://s3.amazonaws.com/openstudio-resources/styles/rubocop.yml + +# custom edits to rubocop checks +Metrics/AbcSize: + Max: 200 + +Metrics/BlockLength: + Max: 100 + +Style/FrozenStringLiteralComment: + Enabled: false \ No newline at end of file diff --git a/README.md b/README.md index 1585fa95f..e7a923a1e 100644 --- a/README.md +++ b/README.md @@ -62,19 +62,32 @@ This is needed if you are a developer making changes to `openstudio-standards` o 7. Contents: `require "C:/openstudio-3.8.0/Ruby/openstudio.rb"` Modify `3.8.0` to the version you installed. 8. Save it here: `C:/Ruby32-x64/lib/ruby/site_ruby/openstudio.rb` -4. `gem install bundler` This installs the `bundler` ruby gem. +4. `gem install bundler:2.4.10` This installs the version of bundler needed for ComStock. 5. Install [Git](https://git-scm.com/). 6. Install [GitHub desktop](https://desktop.github.com/) or another GUI that makes Git easier to use. 7. Clone the [ComStock source code](https://github.com/NREL/ComStock.git) using GitHub desktop (easier) or Git (harder). 8. Run all commands below from the top level `/ComStock` directory -13. `mkdir .custom_gems` This makes a temp directory to install required gems inside. -13. `copy /Y .\resources\Gemfile .\.custom_gems\Gemfile` This copies the Gemfile to the temp directory. -13. `gem install bundler:2.4.10` This installs the version of bundler needed by OpenStudio. -13. `bundle _2.4.10_ install --path "C:/GitRepos/ComStock/.custom_gems" --gemfile "C:/GitRepos/ComStock/.custom_gems/Gemfile" --without test` This will install all ruby gems necessary to develop this code. -14. If running simulations locally, install [BuildStock Batch](https://buildstockbatch.readthedocs.io/en/stable/installation.html#local) -15. Add the following additional Python packages into your `buildstockbatch` environment: +9. `mkdir .custom_gems` This makes a temp directory to install required gems inside. +10. `copy /Y .\resources\Gemfile .\.custom_gems\Gemfile` This copies the Gemfile to the temp directory. +11. `bundle _2.4.10_ install --path "C:/GitRepos/ComStock/.custom_gems" --gemfile "C:/GitRepos/ComStock/.custom_gems/Gemfile" --without test` This will install all ruby gems necessary to develop this code. +12. If running simulations locally, install [BuildStock Batch](https://buildstockbatch.readthedocs.io/en/stable/installation.html#local) +13. Add the following additional Python packages into your `buildstockbatch` environment: ```bash conda activate buildstockbatch pip install GHEDesigner==1.0 pip install NREL-PySAM==4.2.0 ``` + +## Measure Tests +1. Follow the developer installation instructions above. In particular, `GHEDesigner==1.0` must be installed with `pip install GHEDesigner==1.0`. +2. Run the measure tests with the rake command: + - `bundle exec rake unit_tests:all_tests` to run all measure tests. + - `bundle exec rake unit_tests:reporting_measure_tests` to run all reporting measure tests. + - `bundle exec rake unit_tests:workflow_measure_tests` to run all workflow measure tests. + - `bundle exec rake unit_tests:upgrade_measure_tests` to run all `upgrade_` measure tests. + +## Running Rubocop +1. Follow the developer installation instructions above +2. navigate to the `resources/` directory +3. run `bundle exec rake rubocop:show` in terminal +4. review results by opening the `rubocop-results.html` in the `.rubocop/` directory \ No newline at end of file diff --git a/Rakefile b/Rakefile index f2958ffcf..5df231b21 100644 --- a/Rakefile +++ b/Rakefile @@ -1,48 +1,89 @@ -require 'rake' require 'rake/testtask' -require 'minitest/reporters' # Require the gem +require 'minitest/reporters' +require 'parallel' +require 'rubocop/rake_task' -# Configure the JUnit reporter - -desc 'Perform tasks related to unit tests' +desc 'Run measure tests' namespace :unit_tests do - desc 'Run measure tests' - Rake::TestTask.new('measure_tests') do |t| - MEASURETESTS_PATH = "test/measure_tests.txt" - if File.exist?(MEASURETESTS_PATH) - # load test files from file. - full_file_list = FileList.new(File.readlines(MEASURETESTS_PATH).map(&:chomp)) - full_file_list.select! { |item| item.include?('rb')} - p full_file_list + desc 'Run all measure tests' + task :all_tests => [:measure_tests, :workflow_measure_tests, :upgrade_measure_tests] do + puts 'Running all measure tests:' + end + desc 'Run reporting measure tests' + task :reporting_measure_tests do + puts 'Running all reporting measure tests:' + # load test files from file + file_list = FileList.new(File.readlines('test/reporting_measure_tests.txt').map(&:chomp)) + file_list.select! { |item| item.include?('rb') && File.exist?(item) } + Parallel.each(file_list, in_processes: Parallel.processor_count) do |file| + puts "Running test #{file} in process #{Process.pid}" + load file + true + rescue StandardError => e + puts "Error in #{file}: #{e.message}" + Minitest::Reporters.reporter.report_error(file, e) + false end - t.test_files = full_file_list - p(full_file_list) - t.verbose = false - t.warning = false end - Rake::TestTask.new('resource_measure_tests') do |t| - RESOURCE_MEASURETESTS_PATH = "test/resource_measure_tests.txt" - if File.exist?(RESOURCE_MEASURETESTS_PATH) - # load test files from file. - full_file_list = FileList.new(File.readlines(RESOURCE_MEASURETESTS_PATH).map(&:chomp)) - full_file_list.select! { |item| item.include?('rb') && File.exist?(item) } - p full_file_list + desc 'Run workflow measure tests' + task :workflow_measure_tests do + puts 'Running all workflow measure tests:' + # load test files from file + file_list = FileList.new(File.readlines('test/workflow_measure_tests.txt').map(&:chomp)) + file_list.select! { |item| item.include?('rb') && !item.include?('upgrade') && File.exist?(item) } + Parallel.each(file_list, in_processes: Parallel.processor_count) do |file| + puts "Running test #{file} in process #{Process.pid}" + load file + true + rescue StandardError => e + puts "Error in #{file}: #{e.message}" + Minitest::Reporters.reporter.report_error(file, e) + false end - t.test_files = full_file_list.select do |file| - begin - # Try to load the file to check for syntax errors - load file - true - rescue Exception => e - puts "Error in #{file}: #{e.message}" - Minitest::Reporters.reporter.report_error(file, e) - false - end + end + + desc 'Run upgrade measure tests' + task :upgrade_measure_tests do + puts 'Running all upgrade measure tests:' + # load test files from file + file_list = FileList.new(File.readlines('test/upgrade_measure_tests.txt').map(&:chomp)) + file_list.select! { |item| item.include?('rb') && item.include?('upgrade') && File.exist?(item) } + Parallel.each(file_list, in_processes: Parallel.processor_count) do |file| + puts "Running test #{file} in process #{Process.pid}" + load file + true + rescue StandardError => e + puts "Error in #{file}: #{e.message}" + Minitest::Reporters.reporter.report_error(file, e) + false end - p(full_file_list) - t.verbose = false - t.warning = false + end +end + +desc 'Check the code for style consistency' +RuboCop::RakeTask.new(:rubocop) do |t| + # Make a folder for the output + out_dir = '.rubocop' + FileUtils.mkdir_p(out_dir) + # Output both XML (CheckStyle format) and HTML + t.options = ["--out=#{out_dir}/rubocop-results.xml", '--format=h', "--out=#{out_dir}/rubocop-results.html", '--format=offenses', "--out=#{out_dir}/rubocop-summary.txt"] + t.requires = ['rubocop/formatter/checkstyle_formatter'] + t.formatters = ['RuboCop::Formatter::CheckstyleFormatter'] + # don't abort rake on failure + t.fail_on_error = false +end + +desc 'Show the rubocop output in a web browser' +task 'rubocop:show' => [:rubocop] do + link = "#{__dir__}/.rubocop/rubocop-results.html" + case RbConfig::CONFIG['host_os'] + when /mswin/, /mingw/, /cygwin/ + system "start #{link}" + when /darwin/ + system "open #{link}" + when /linux/, /bsd/ + system "xdg-open #{link}" end end diff --git a/documentation/reference_doc/4_2_meta.tex b/documentation/reference_doc/4_2_meta.tex index 9f02f27ee..4aaffa6d5 100644 --- a/documentation/reference_doc/4_2_meta.tex +++ b/documentation/reference_doc/4_2_meta.tex @@ -96,4 +96,7 @@ \subsection{Space Type Ratios} \subsection{Weather Data} ComStock can be run with two different types of weather data: typical meteorological year (TMY3) and actual meteorological year (AMY). AMY data is the data measured during a specific year, taken from weather stations such as those at airports. Because these data are from a particular calendar year, weather patterns that span large areas, such as nationwide heat waves, are captured in the data across multiple locations. Therefore, these weather patterns are captured in the outputs of ComStock. This is important for use cases where coordinated weather patterns influence loads, such as peak load impacts for bulk power grid planning. TMY3 data, in contrast, take the ``most typical'' weather for each calendar month from a 30-year historical record and stitch these months together to form a complete year. The advantage of this method is that the weather data is less influenced by an extremely hot or cold year. However, this approach does not capture wide-area weather patterns, as the month of data used varies from location to location. For a more in-depth discussion of AMY and TMY3 weather data, see \cite{eulp_final_report}. -For geographic granularity, ComStock currently uses one weather file for each county in the United States. For counties with no weather data available (generally sparsely populated rural areas), data from the nearest weather station in the same climate zone are used. See \citep{eulp_final_report} for a more in-depth discussion of the weather data sources, cleaning process, and assignment assumptions. \ No newline at end of file +For geographic granularity, ComStock currently uses one weather file for each county in the United States. For counties with no weather data available (generally sparsely populated rural areas), data from the nearest weather station in the same climate zone are used. See \citep{eulp_final_report} for a more in-depth discussion of the weather data sources, cleaning process, and assignment assumptions. + +\subsection{Soil Properties} +Soil thermal conductivity and undisturbed ground temperature are location-dependent properties that are required in the ComStock model by several goethermal heat pump upgrade measures. Therefore, these properties are part of the ComStock sampling workflow and are stored as additionl properties in the building models, which can then be used by downstream measures. Soil thermal conductivity distributions by climate zone were dervied from a dataset produced by the Southern Methodist University Geothermal Lab, and are shown in Table \ref{fig:soil_conductivity} (\cite{smu_soil_conductivity}). The soil thermal conductivity values range from 0.5 to 2.6 W/m-K. Average undisturbed ground temperatures by climate zone were derived from a 2014 Oklahoma State University study and are shown in Table \ref{tab:undisturbed_ground_temp} (\cite{xing2014}). \ No newline at end of file diff --git a/documentation/reference_doc/4_7_plug_and_process.tex b/documentation/reference_doc/4_7_plug_and_process.tex index 51a16eb5e..6cf875559 100644 --- a/documentation/reference_doc/4_7_plug_and_process.tex +++ b/documentation/reference_doc/4_7_plug_and_process.tex @@ -66,7 +66,7 @@ \subsection{Kitchen Equipment} ComStock uses published data to create representative probability distributions of commercial cooking equipment counts, by building type, for both gas and electric appliances. Additionally, the equipment distributions are scaled by area to represent the non-linear scaling suggested in the literature. Although other building types likely include some degree of cooking equipment as well, such as larger offices (\cite{eia2012cbecs}), the current implementation of ComStock only includes cooking equipment in the previously-mentioned six building types plus quick service restaurants found in strip malls. -Commercial kitchens can contain electric or gas cooking equipment, or a mix of both. The prevalence of gas and electric fuel types for each equipment type and the rated powers used in ComStock are derived from a DOE study (\cite{goetzler_commercial_appliances}). These are shown in Table~\ref{tab:kitchen_prev_and_power}. +Commercial kitchens can contain electric or gas cooking equipment, or a mix of both. The prevalence of gas and electric fuel types for each equipment type used in ComStock are derived from a DOE study (\cite{goetzler_commercial_appliances}). ComStock requires rated input power values and fractions of radiant, latent, and lost heat for gas and electric kitchen equipment. These values are primarily derived from the ASHRAE Fundamentals Handbook (\cite{ashrae2017}) after comparisons with other kitchen equipment studies and commercially available products. More details about how these values were determined can be found in the End Use Savings Shapes documentation (\cite{nrel89130}).The assumptions used in ComStock for prevalence, rated input power, and fractions radiant, latent, and lost for gas and electric appliances are shown in Table~\ref{tab:kitchen_prev_and_power}. \input{tables/kitchen_prev_and_power} diff --git a/documentation/reference_doc/6_AppendixA.tex b/documentation/reference_doc/6_AppendixA.tex index c80e749dd..a4f44c2f0 100644 --- a/documentation/reference_doc/6_AppendixA.tex +++ b/documentation/reference_doc/6_AppendixA.tex @@ -87,3 +87,4 @@ \input{tables/kitchen_cook_counts} +\input{tables/undisturbed_ground_temp} \ No newline at end of file diff --git a/documentation/reference_doc/7_AppendixB.tex b/documentation/reference_doc/7_AppendixB.tex index 4b69b6905..f57ed9a06 100644 --- a/documentation/reference_doc/7_AppendixB.tex +++ b/documentation/reference_doc/7_AppendixB.tex @@ -460,4 +460,12 @@ \centering \includegraphics[width=1.0\textwidth]{figures/refrigeration_LTnew.png} \caption[Compressor performance for large, new, low temperature compressors]{Power and capacity values as a function of suction and discharge temperature for large, new, low-temperature compressors.} \label{fig:refrig_lt_new} -\end{figure} \ No newline at end of file +\end{figure} + +\begin{figure} [ht!] + \includegraphics[width=0.8\textwidth]{figures/soil_conductivity.png} + \centering + \caption[Soil thermal conductivity distributions by climate zone]{Soil thermal conductivity distributions by climate zone.} + \label{fig:soil_conductivity} +\end{figure} + \ No newline at end of file diff --git a/documentation/reference_doc/bibliography.bib b/documentation/reference_doc/bibliography.bib index 0ca7b09d8..378ced561 100644 --- a/documentation/reference_doc/bibliography.bib +++ b/documentation/reference_doc/bibliography.bib @@ -584,7 +584,7 @@ @misc{trane_foundation } @misc{carrier_economiser, - title = {Carrier Economi$er}, + title = {Carrier Economizer}, author = {{Carrier}}, year = 2023, } @@ -690,7 +690,7 @@ @misc{zip_to_util } @misc{tract_to_zip, - author = {{HUD PD&R}}, + author = {{HUD PD\&R}}, institution = {U.S. Department of Housing and Urban Development Office of Policy Development and Research}, title = {HUD USPS ZIP CODE CROSSWALK FILES}, year = 2023, @@ -714,4 +714,37 @@ @misc{atus2018 year = 2018, howpublished = {https://www.bls.gov/tus/}, note = {Accessed: 2023-11-27} +} + +@book{ashrae2017, + title = {ASHRAE Handbook - Fundamentals}, + publisher = {American Society of Heating, Refrigerating, and Air-Conditioning Engineers, Inc.}, + year = {2017}, + address = {Atlanta} +} + +@techreport{nrel89130, + author = {Marlena Praprost}, + title = {End-Use Savings Shapes Measure Documentation: Electric Cooking Equipment}, + institution = {National Renewable Energy Laboratory}, + year = {2024}, + type = {Technical Report}, + number = {89130}, + url = {https://www.nrel.gov/docs/fy24osti/89130.pdf} +} + +@online{smu_soil_conductivity, + author = {{Dedman College of Humanities and Sciences; Roy M Huffington Department of Earth Sciences}}, + title = {SMU Geothermal Lab | Data and Maps | Temperature Maps}, + year = {2023}, + url = {https://www.smu.edu/dedman/academics/departments/earth-sciences/research/geothermallab/datamaps/temperaturemaps}, + note = {Accessed: 15 December 2023} +} + +@phdthesis{xing2014, + author = {L. Xing}, + title = {Estimations of Undisturbed Ground Temperatures Using Numerical and Analytical Modeling}, + school = {Oklahoma State University}, + address = {Stillwater, Oklahoma}, + year = {2014} } \ No newline at end of file diff --git a/documentation/reference_doc/figures/soil_conductivity.png b/documentation/reference_doc/figures/soil_conductivity.png new file mode 100644 index 000000000..dac4f6b42 Binary files /dev/null and b/documentation/reference_doc/figures/soil_conductivity.png differ diff --git a/documentation/reference_doc/main.tex b/documentation/reference_doc/main.tex index 2d12c25b2..f4924efc8 100644 --- a/documentation/reference_doc/main.tex +++ b/documentation/reference_doc/main.tex @@ -8,6 +8,7 @@ \usepackage[section]{placeins} \usepackage{pdflscape} \usepackage{soul} +\usepackage{biblatex} \hypersetup{colorlinks=true} % ----------------------------------- @@ -23,6 +24,8 @@ \author{Amy LeBar} \author{Janghyun Kim} \author{Lauren Klun} +\author{Eric Ringold} +\author{Wenyi Kuang} \affil{National Renewable Energy Laboratory} \fancypagestyle{plain}{} diff --git a/documentation/reference_doc/tables/kitchen_prev_and_power.tex b/documentation/reference_doc/tables/kitchen_prev_and_power.tex index 4f4bdf37e..3cea7ef3c 100644 --- a/documentation/reference_doc/tables/kitchen_prev_and_power.tex +++ b/documentation/reference_doc/tables/kitchen_prev_and_power.tex @@ -1,16 +1,95 @@ -\begin{table}[h] -\small -\centering -\caption[Cooking Equipment Fuel Type Prevelance and Rater Power]{Cooking Equipment Fuel Type Prevelance and Rater Power} -\label{tab:kitchen_prev_and_power} -\begin{tabular}{|l|l|l|l|l|} -\hline -\textbf{Appliance} & \textbf{\% Gas} & \textbf{\% Electric} & \textbf{Rated Energy - Gas (BTU/hr)} & \textbf{Rated Energy - Electricity (kW)} \\ \hline -Broiler & 0.91 & 0.09 & 88000 & 11 \\ \hline -Fryers & 0.58 & 0.42 & 164000 & 22 \\ \hline -Griddles & 0.5 & 0.5 & 70000 & 12 \\ \hline -Ovens & 0.55 & 0.45 & 56000 & 15 \\ \hline -Ranges & 0.91 & 0.09 & 179000 & 12 \\ \hline -Steamers & 0.33 & 0.67 & 210000 & 24 \\ \hline -\end{tabular} -\end{table} \ No newline at end of file +% Please add the following required packages to your document preamble: +% \usepackage{multirow} +% \usepackage{graphicx} +\begin{table}[] + \caption[Cooking Equipment Fuel Type Prevelance and Rater Power]{Cooking Equipment Fuel Type Prevelance and Rater Power} + \label{tab:kitchen_prev_and_power} + \resizebox{\columnwidth}{!}{% + \begin{tabular}{|l|ll|ll|ll|ll|ll|} + \hline + \multicolumn{1}{|c|}{\multirow{2}{*}{\textbf{Appliance}}} & + \multicolumn{2}{c|}{\textbf{Fuel Prevalence Fraction}} & + \multicolumn{2}{c|}{\textbf{Rated Power}} & + \multicolumn{2}{c|}{\textbf{Fraction Radiant}} & + \multicolumn{2}{c|}{\textbf{Fraction Latent}} & + \multicolumn{2}{c|}{\textbf{Fraction Lost}} \\ \cline{2-11} + \multicolumn{1}{|c|}{} & + \multicolumn{1}{l|}{\textbf{Gas}} & + \textbf{Electric} & + \multicolumn{1}{l|}{\textbf{Gas (Btu/h)}} & + \textbf{Electric (kW)} & + \multicolumn{1}{l|}{\textbf{Gas}} & + \textbf{Electric} & + \multicolumn{1}{l|}{\textbf{Gas}} & + \textbf{Electric} & + \multicolumn{1}{l|}{\textbf{Gas}} & + \textbf{Electric} \\ \hline + Broiler & + \multicolumn{1}{l|}{0.91} & + 0.09 & + \multicolumn{1}{l|}{96,000} & + 10.8 & + \multicolumn{1}{l|}{0.12} & + 0.35 & + \multicolumn{1}{l|}{0.1} & + 0.1 & + \multicolumn{1}{l|}{0.68} & + 0.45 \\ \hline + Griddle & + \multicolumn{1}{l|}{0.58} & + 0.42 & + \multicolumn{1}{l|}{90,000} & + 17.1 & + \multicolumn{1}{l|}{0.18} & + 0.39 & + \multicolumn{1}{l|}{0.1} & + 0.1 & + \multicolumn{1}{l|}{0.62} & + 0.41 \\ \hline + Fryer & + \multicolumn{1}{l|}{0.5} & + 0.5 & + \multicolumn{1}{l|}{80,000} & + 14 & + \multicolumn{1}{l|}{0.23} & + 0.36 & + \multicolumn{1}{l|}{0.1} & + 0.1 & + \multicolumn{1}{l|}{0.57} & + 0.44 \\ \hline + Oven & + \multicolumn{1}{l|}{0.55} & + 0.45 & + \multicolumn{1}{l|}{44,000} & + 12.1 & + \multicolumn{1}{l|}{0.08} & + 0.22 & + \multicolumn{1}{l|}{0.1} & + 0.1 & + \multicolumn{1}{l|}{0.72} & + 0.58 \\ \hline + Range & + \multicolumn{1}{l|}{0.91} & + 0.09 & + \multicolumn{1}{l|}{145,000} & + 21 & + \multicolumn{1}{l|}{0.11} & + 0.1 & + \multicolumn{1}{l|}{0.1} & + 0.1 & + \multicolumn{1}{l|}{0.69} & + 0.7 \\ \hline + Steamer & + \multicolumn{1}{l|}{0.33} & + 0.67 & + \multicolumn{1}{l|}{200,000} & + 27 & + \multicolumn{1}{l|}{0.1} & + 0.1 & + \multicolumn{1}{l|}{0.1} & + 0.1 & + \multicolumn{1}{l|}{0.7} & + 0.7 \\ \hline + \end{tabular}% + } + \end{table} \ No newline at end of file diff --git a/documentation/reference_doc/tables/undisturbed_ground_temp.tex b/documentation/reference_doc/tables/undisturbed_ground_temp.tex new file mode 100644 index 000000000..e0d57c182 --- /dev/null +++ b/documentation/reference_doc/tables/undisturbed_ground_temp.tex @@ -0,0 +1,28 @@ +% Please add the following required packages to your document preamble: +% \usepackage{graphicx} +\begin{table}[] + \centering + \caption[Average Undisturbed Ground Temperature by Climate Zone]{Average Undisturbed Ground Temperature by Climate Zone} + \label{tab:undisturbed_ground_temp} + \begin{tabular}{|l|l|} + \hline + 2012 IECC Climate zone & Annual average undisturbed ground temperature (C) \\ \hline + 1A & 25.9 \\ \hline + 2A & 20.9 \\ \hline + 2B & 25 \\ \hline + 3A & 17.9 \\ \hline + 3B & 19.7 \\ \hline + 3C & 17 \\ \hline + 4A & 14.7 \\ \hline + 4B & 16.3 \\ \hline + 4C & 13.3 \\ \hline + 5A & 11.5 \\ \hline + 5B & 12.9 \\ \hline + 6A & 9 \\ \hline + 6B & 9.3 \\ \hline + 7A & 7 \\ \hline + 7AK & 5.4 \\ \hline + 7B & 6.5 \\ \hline + 8AK & 2.3 \\ \hline + \end{tabular}% +\end{table} \ No newline at end of file diff --git a/postprocessing/comstockpostproc/comstock.py b/postprocessing/comstockpostproc/comstock.py index df2ff0a20..e6d0dd6ec 100644 --- a/postprocessing/comstockpostproc/comstock.py +++ b/postprocessing/comstockpostproc/comstock.py @@ -13,6 +13,7 @@ import polars as pl import re import datetime +import pytest from comstockpostproc.naming_mixin import NamingMixin from comstockpostproc.units_mixin import UnitsMixin @@ -226,6 +227,7 @@ def __init__(self, s3_base_dir, comstock_run_name, comstock_run_version, comstoc self.cached_parquet.append((upgrade_id, file_path)) #cached_parquet is a list of parquets used to export and reload logger.info(f'Exporting to: {file_path}') self.data = self.reorder_data_columns(self.data) + self._sightGlass_metadata_check(self.data) self.data.write_parquet(file_path) up_lazyframes.append(pl.scan_parquet(file_path)) @@ -2879,3 +2881,65 @@ def export_data_and_enumeration_dictionary(self): file_path = os.path.abspath(os.path.join(self.output_dir, file_name)) logger.info(f'Exporting enumeration dictionary to: {file_path}') enum_dictionary.write_csv(file_path, separator='\t') + + + def _sightGlass_metadata_check(self, row_segment: pl.DataFrame): + # Check that the metadata columns are present in the data + # when the columns are in memory + err_log = "" + + #check no na values in any columns + if row_segment.null_count().pipe(sum).item() > 0: + err_log += 'Null values found in data\n' + for c in row_segment.columns: + if c.startswith("out.qoi.") or c.startswith("out.utility_bills.") or c.startswith('applicability.upgrade_add_pvwatts'): + continue + if row_segment[c].null_count() > 0: + err_log += f'Column {c} has null values\n' + + SIGHTGLASS_REQUIRED_COLS = [self.BLDG_ID, self.META_IDX, self.UPGRADE_ID, + self.BLDG_WEIGHT, self.UPGRADE_APPL, self.FLR_AREA] + + for col in SIGHTGLASS_REQUIRED_COLS: + if col not in row_segment.columns: + err_log += f'{col} not found in data, which is needed for sightglass\n' + + #Skip pattern, may need delete later: + pattern = r'out\.electricity\.total\.[a-zA-Z]{3}\.energy_consumption' + + for c in row_segment.columns: + if re.search('[^a-z0-9._]', c): + # (f'Column {c} violates name rules: may only contain . _ 0-9 lowercaseletters (no spaces)') + err_log += f'Column {c} violates name rules: may only contain . _ 0-9 lowercaseletters (no spaces)\n' + + #Actually that's the perfect case to use regex to check the summary. + TOTAL_PATTERN = r'out\.([a-zA-Z_]+)\.total\.energy_consumption\.\.kwh' + ENDUSE_PATTERN = r'out\.([a-zA-Z_]+)\.(?!total)([a-zA-Z_]+)\.energy_consumption\.\.kwh' + MONTH_PATTERN = r'out\.electricity\.total\.([a-zA-Z]{3})\.energy_consumption' + + #Find the sum of total culmns for each type fuels, and for each fuel type find the sum of different + #enduse columns. And record them in a dictionary like: {fuel_type: total_energy} + fuel_total, end_use_total, month_total = {}, {}, {} + for c in row_segment.columns: + if re.match(TOTAL_PATTERN, c): + fuel_type = re.match(TOTAL_PATTERN, c).group(1) + fuel_total[fuel_type] = row_segment[c].sum() + elif re.match(ENDUSE_PATTERN, c): + fuel_type = re.match(ENDUSE_PATTERN, c).group(1) + end_use_total[fuel_type] = end_use_total.get(fuel_type, 0) + row_segment[c].sum() + elif re.match(MONTH_PATTERN, c): + month = re.match(MONTH_PATTERN, c).group(1) + month_total[month] = row_segment[c].sum() + + logger.info(f"Fuel total: {fuel_total}, Enduse total: {end_use_total}, Month total: {month_total}") + # Check that the total site energy is the sum of the fuel totals + for fuel, total in end_use_total.items(): + if not total == pytest.approx(fuel_total[fuel], rel=0.001): + err_log += f'Fuel total for {fuel} does not match sum of enduse columns\n' + if not sum(fuel_total.values()) == pytest.approx(row_segment[self.ANN_TOT_ENGY_KBTU].sum(), rel=0.001): + err_log += 'Site total does not match sum of fuel totals\n' + if not sum(month_total.values()) == pytest.approx(row_segment[self.ANN_TOT_ELEC_KBTU].sum(), rel=0.01): + err_log += 'Electricity total does not match sum of month totals\n' + + if err_log: + raise ValueError(err_log) \ No newline at end of file diff --git a/postprocessing/comstockpostproc/naming_mixin.py b/postprocessing/comstockpostproc/naming_mixin.py index b226d0916..3d2c467ff 100644 --- a/postprocessing/comstockpostproc/naming_mixin.py +++ b/postprocessing/comstockpostproc/naming_mixin.py @@ -201,7 +201,7 @@ class NamingMixin(): 'out.emissions.natural_gas..co2e_kg', 'out.emissions.fuel_oil..co2e_kg', 'out.emissions.propane..co2e_kg', - 'out.emissions.electricity.lrmer_95_decarb_by_2035_15_2023_start..co2e_kg' + 'out.emissions.electricity.lrmer_mid_case_15_2023_start..co2e_kg' ] # QOI COLS diff --git a/resources/Gemfile b/resources/Gemfile index 50d5d9072..0d60cf2a4 100644 --- a/resources/Gemfile +++ b/resources/Gemfile @@ -1,22 +1,23 @@ source 'http://rubygems.org' -ruby "~> 3.2.0" +ruby '~> 3.2.0' # Bug in addressable to 2.8.1 and patched version has an issue https://github.com/NREL/OpenStudio/issues/4870 gem 'addressable', '= 2.8.1' -# This removes the rubocop runtime dependency on 'json ~> 2.3'. -gem 'rubocop', :github => 'jmarrec/rubocop', :ref => '1.50.0-no_json' - -gem 'bcl', "= 0.8.0" +gem 'bcl', '= 0.8.0' gem 'openstudio-extension', '= 0.8.0' -gem 'openstudio-workflow', '= 2.4.0' +gem 'openstudio-geb', '= 0.4.0', github: 'LBNL-ETA/Openstudio-GEB-gem', ref: 'a18e023936dd5c92be7dbf354f6331f60a965828' gem 'openstudio-standards', '= 0.6.3' -#gem 'openstudio-standards', git: 'https://github.com/NREL/openstudio-standards.git', ref: 'e61d7d41a0e285cc99c7c6ea4aff1fcdc0be3599' +# gem 'openstudio-standards', git: 'https://github.com/NREL/openstudio-standards.git', ref: 'e61d7d41a0e285cc99c7c6ea4aff1fcdc0be3599' # gem 'openstudio-standards', path: "C:/GitRepos/openstudio-standards" # Use this format when testing openstudio-standards changes locally +gem 'openstudio-workflow', '= 2.4.0' + +# This removes the rubocop runtime dependency on 'json ~> 2.3'. +gem 'rubocop', github: 'jmarrec/rubocop', ref: '1.50.0-no_json' + gem 'tbd', '= 3.4.1' -gem 'openstudio-geb', '= 0.4.0',:github => 'LBNL-ETA/Openstudio-GEB-gem', :ref => 'a18e023936dd5c92be7dbf354f6331f60a965828' group :test do - gem 'minitest-reporters', '= 1.4.3' gem 'minitest-junit', '= 1.1.0' + gem 'minitest-reporters', '= 1.4.3' end diff --git a/test/measure_tests.txt b/test/reporting_measure_tests.txt similarity index 85% rename from test/measure_tests.txt rename to test/reporting_measure_tests.txt index f622472ff..6650c36ad 100644 --- a/test/measure_tests.txt +++ b/test/reporting_measure_tests.txt @@ -1,6 +1,5 @@ +measures/comstock_sensitivity_reports/tests/comstock_sensitivity_reports_test.rb +measures/emissions_reporting/tests/EmissionsReporting_Test.rb measures/qoi_report/tests/qoi_report_test.rb measures/simulation_settings_check/tests/measure_test.rb -measures/emissions_reporting/tests/EmissionsReporting_Test.rb measures/TimeseriesCSVExport/tests/Measure_Test.rb -measures/comstock_sensitivity_reports/tests/comstock_sensitivity_reports_test.rb -measures/scout_loads_summary/tests/Measure_Test.rb diff --git a/test/upgrade_measure_tests.txt b/test/upgrade_measure_tests.txt new file mode 100644 index 000000000..27b9a522d --- /dev/null +++ b/test/upgrade_measure_tests.txt @@ -0,0 +1,22 @@ +resources/measures/upgrade_advanced_rtu_control/tests/advanced_rtu_control_test.rb +resources/measures/upgrade_df_thermostat_control_load_shed/tests/df_thermostat_control_load_shed_test.rb +resources/measures/upgrade_df_thermostat_control_load_shift/tests/df_thermostat_control_load_shift_test.rb +resources/measures/upgrade_env_exterior_wall_insulation/tests/measure_test.rb +resources/measures/upgrade_env_new_aedg_windows/tests/measure_test.rb +resources/measures/upgrade_env_roof_insul_aedg/tests/env_roof_insul_aedg_test.rb +resources/measures/upgrade_env_secondary_windows/tests/measure_test.rb +resources/measures/upgrade_env_window_film/tests/env_window_film_test.rb +resources/measures/upgrade_hvac_add_heat_pump_rtu/tests/measure_test.rb +resources/measures/upgrade_hvac_console_gshp/tests/console_gthp_test.rb +resources/measures/upgrade_hvac_dcv/tests/hvac_dcv_test.rb +resources/measures/upgrade_hvac_doas_hp_minisplits/tests/hvac_doas_hp_minisplits_test.rb +resources/measures/upgrade_hvac_economizer/tests/hvac_economizer_test.rb +resources/measures/upgrade_hvac_exhaust_air_energy_or_heat_recovery/tests/hvac_exhaust_air_energy_or_heat_recovery_test.rb +resources/measures/upgrade_hvac_hydronic_gshp/tests/hydronic_gthp_test.rb +resources/measures/upgrade_hvac_multispeed_minimum_flow/tests/measure_Test.rb +resources/measures/upgrade_hvac_packaged_gshp/tests/packaged_gthp_test.rb +resources/measures/upgrade_hvac_replace_boiler_by_heatpump/tests/add_hpwh_test.rb +resources/measures/upgrade_hvac_vrf_hr_doas/tests/measure_test.rb +resources/measures/upgrade_light_led/tests/light_led_test.rb +resources/measures/upgrade_ppl_electric_kitchen_equipment/tests/kitchen_equip_measure_test.rb +resources/measures/upgrade_unoccupied_oa_controls/tests/unoccupied_oa_controls_measure_test.rb \ No newline at end of file diff --git a/test/resource_measure_tests.txt b/test/workflow_measure_tests.txt similarity index 50% rename from test/resource_measure_tests.txt rename to test/workflow_measure_tests.txt index e11d41fc3..5206f4926 100644 --- a/test/resource_measure_tests.txt +++ b/test/workflow_measure_tests.txt @@ -1,18 +1,15 @@ -resources/measures/ChangeBuildingLocation/tests/change_building_location_test.rb resources/measures/add_blinds_to_selected_windows/tests/add_blinds_to_selected_windows_test.rb resources/measures/add_hvac_nighttime_operation_variability/tests/add_hvac_nighttime_operation_variability_test.rb resources/measures/add_thermostat_setpoint_variability/tests/add_thermostat_setpoint_variability_test.rb resources/measures/adjust_occupancy_schedule/tests/adjust_occupancy_schedule_test.rb -resources/measures/upgrade_env_roof_insul_aedg/tests/env_roof_insul_aedg_test.rb -resources/measures/set_exterior_lighting_template/tests/measure_test.rb -resources/measures/set_wall_template/tests/measure_test.rb -resources/measures/upgrade_env_new_aedg_windows/tests/measure_test.rb +resources/measures/ChangeBuildingLocation/tests/change_building_location_test.rb resources/measures/create_bar_from_building_type_ratios/tests/create_bar_from_building_type_ratios_test.rb resources/measures/create_typical_building_from_model/tests/create_typical_building_from_model_test.rb resources/measures/create_typical_building_from_model/tests/test_hvac.rb resources/measures/fault_hvac_economizer_changeover_temperature/tests/measure_test.rb resources/measures/fault_hvac_economizer_damper_stuck/tests/measure_test.rb resources/measures/hardsize_model/tests/measure_test.rb +resources/measures/hvac_nighttime_oa_controls/tests/new_measure_test.rb" resources/measures/prototype_space_type_assignment/tests/prototype_space_type_assignment_test.rb resources/measures/replace_baseline_windows/tests/replace_baseline_windows_test.rb resources/measures/set_electric_equipment_bpr/tests/set_electric_equipment_bpr_test.rb @@ -29,27 +26,4 @@ resources/measures/set_service_water_heating_fuel/tests/measure_test.rb resources/measures/set_service_water_heating_template/tests/measure_test.rb resources/measures/set_space_type_load_subcategories/tests/set_space_type_load_subcategories_test.rb resources/measures/set_wall_template/tests/measure_test.rb -resources/measures/simulation_settings/tests/measure_test.rb -resources/measures/upgrade_advanced_rtu_control/tests/advanced_rtu_control_test.rb -resources/measures/upgrade_df_thermostat_control_load_shed/tests/df_thermostat_control_load_shed_test.rb -resources/measures/upgrade_df_thermostat_control_load_shift/tests/df_thermostat_control_load_shift_test.rb -resources/measures/upgrade_env_exterior_wall_insulation/tests/measure_test.rb -resources/measures/upgrade_env_new_aedg_windows/tests/env_new_aedg_windows_test.rb -resources/measures/upgrade_env_new_aedg_windows/tests/measure_test.rb -resources/measures/upgrade_env_roof_insul_aedg/tests/env_roof_insul_aedg_test.rb -resources/measures/upgrade_env_secondary_windows/tests/measure_test.rb -resources/measures/upgrade_env_window_film/tests/env_window_film_test.rb -resources/measures/upgrade_hvac_add_heat_pump_rtu/tests/measure_test.rb -resources/measures/upgrade_hvac_console_gshp/tests/console_gthp_test.rb -resources/measures/upgrade_hvac_dcv/tests/hvac_dcv_test.rb -resources/measures/upgrade_hvac_doas_hp_minisplits/tests/hvac_doas_hp_minisplits_test.rb -resources/measures/upgrade_hvac_economizer/tests/hvac_economizer_test.rb -resources/measures/upgrade_hvac_exhaust_air_energy_or_heat_recovery/tests/hvac_exhaust_air_energy_or_heat_recovery_test.rb -resources/measures/upgrade_hvac_hydronic_gshp/tests/hydronic_gthp_test.rb -resources/measures/upgrade_hvac_multispeed_minimum_flow/tests/measure_Test.rb -resources/measures/upgrade_hvac_packaged_gshp/tests/packaged_gthp_test.rb -resources/measures/upgrade_hvac_replace_boiler_by_heatpump/tests/add_hpwh_test.rb -resources/measures/upgrade_hvac_vrf_hr_doas/tests/measure_test.rb -resources/measures/upgrade_light_led/tests/light_led_test.rb -resources/measures/upgrade_ppl_electric_kitchen_equipment/tests/kitchen_equip_measure_test.rb -resources/measures/upgrade_unoccupied_oa_controls/tests/unoccupied_oa_controls_measure_test.rb \ No newline at end of file +resources/measures/simulation_settings/tests/measure_test.rb \ No newline at end of file