From 8cc8943bbe1df176391d0850fea87431f87da19d Mon Sep 17 00:00:00 2001 From: Adam Erispaha Date: Sun, 6 Oct 2019 17:01:05 -0500 Subject: [PATCH 1/7] rough draft of [CURVES] parser --- swmmio/core.py | 23 +++++++++++-- swmmio/defs/section_headers.json | 3 +- swmmio/defs/sectionheaders.py | 4 +-- .../data/model_full_features_network_xy.inp | 28 ++++++++++++++++ swmmio/tests/test_model_elements.py | 12 +++++-- swmmio/utils/dataframes.py | 32 +++++++++++++++++++ 6 files changed, 95 insertions(+), 7 deletions(-) diff --git a/swmmio/core.py b/swmmio/core.py index 9a95249..49a5907 100644 --- a/swmmio/core.py +++ b/swmmio/core.py @@ -7,7 +7,8 @@ import math from swmmio.utils import spatial from swmmio.utils import functions -from swmmio.utils.dataframes import create_dataframeINP, create_dataframeRPT, get_link_coords +from swmmio.utils.dataframes import create_dataframeINP, create_dataframeRPT, get_link_coords, \ + create_dataframe_multi_index from swmmio.defs.config import * from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH, MODEL_FULL_FEATURES_XY import warnings @@ -540,6 +541,7 @@ def __init__(self, file_path): self._subareas_df = None self._infiltration_df = None self._headers = None + self._curves_df = None SWMMIOFile.__init__(self, file_path) # run the superclass init @@ -559,6 +561,7 @@ def __init__(self, file_path): '[SUBAREAS]', '[INFILTRATION]', '[COORDINATES]' + '[CURVES]' ] def save(self, target_path=None): @@ -924,7 +927,23 @@ def polygons(self): @polygons.setter def polygons(self, df): """Set inp.polygons DataFrame.""" - self._polygons_df = df + self._curves_df = df + + @property + def curves(self): + """ + get/set curves section of model + :return: multi-index dataframe of model curves + """ + if self._curves_df is not None: + return self._curves_df + self._curves_df = create_dataframe_multi_index(self.path, '[CURVES]', comment_cols=False) + return self._curves_df + + @curves.setter + def curves(self, df): + """Set inp.polygons DataFrame.""" + self._curves_df = df def drop_invalid_model_elements(inp): diff --git a/swmmio/defs/section_headers.json b/swmmio/defs/section_headers.json index 2ea897d..583100a 100644 --- a/swmmio/defs/section_headers.json +++ b/swmmio/defs/section_headers.json @@ -42,7 +42,8 @@ "[Polygons]": ["Name", "X", "Y"], "[REPORT]": ["Param", "Status"], "[TAGS]": ["ElementType", "Name", "Tag"], - "[MAP]": ["x1", "y1", "x2", "y2"] + "[MAP]": ["x1", "y1", "x2", "y2"], + "[CURVES]": ["Name", "Type", "X-Value", "Y-Value"] }, "infiltration_cols": { "HORTON": ["Subcatchment", "MaxRate", "MinRate", "Decay", "DryTime", "MaxInfil"], diff --git a/swmmio/defs/sectionheaders.py b/swmmio/defs/sectionheaders.py index 1c80d18..252cc5b 100644 --- a/swmmio/defs/sectionheaders.py +++ b/swmmio/defs/sectionheaders.py @@ -26,8 +26,8 @@ '[Polygons]':'Name X Y', '[REPORT]':'Param Status', '[TAGS]':'ElementType Name Tag', - '[MAP]': 'x1 y1 x2 y2' - #'[CURVES]':'Name Type X-Value Y-Value', + '[MAP]': 'x1 y1 x2 y2', + '[CURVES]': 'Name Type X-Value Y-Value', #'[TIMESERIES]':'Name Date Time Value' } diff --git a/swmmio/tests/data/model_full_features_network_xy.inp b/swmmio/tests/data/model_full_features_network_xy.inp index e57b6cd..fb37346 100644 --- a/swmmio/tests/data/model_full_features_network_xy.inp +++ b/swmmio/tests/data/model_full_features_network_xy.inp @@ -137,8 +137,36 @@ J1 FLOW "" FLOW 1.0 1 1 [CURVES] ;;Name Type X-Value Y-Value ;;-------------- ---------- ---------- ---------- +;control curve +control-curve Control 0 0 +control-curve 1 0.5 +control-curve 10 1.0 +; +diversion-curve Diversion 0 0 +diversion-curve 1 10 +; P1_Curve Pump4 0 10 P1_Curve 5 20 +; +P2_Curve Pump4 0 10 +P2_Curve 1 20 +P2_Curve 3 30 +; +;tenk sotrage curve +tenk-sturage Shape 0 0 +tenk-sturage 1 10 +tenk-sturage 2 30 +tenk-sturage 3 60 +tenk-sturage 10 60 +; +storage-curve Storage 0 0 +storage-curve 1 100 +storage-curve 10 100 +; +tidal-curve Tidal 0 1 +tidal-curve 6 1.5 +tidal-curve 12 2 +tidal-curve 23 1 [TIMESERIES] ;;Name Date Time Value diff --git a/swmmio/tests/test_model_elements.py b/swmmio/tests/test_model_elements.py index 2f7138f..a033a95 100644 --- a/swmmio/tests/test_model_elements.py +++ b/swmmio/tests/test_model_elements.py @@ -1,13 +1,14 @@ -from swmmio.tests.data import MODEL_FULL_FEATURES__NET_PATH +from swmmio.tests.data import MODEL_FULL_FEATURES_XY from swmmio import Model from swmmio.elements import ModelSection from swmmio.utils import functions import pytest + @pytest.fixture def test_model(): - return Model(MODEL_FULL_FEATURES__NET_PATH) + return Model(MODEL_FULL_FEATURES_XY) def test_model_section(test_model): @@ -31,3 +32,10 @@ def test_complete_headers(test_model): '[SYMBOLS]' ] assert (all(section in headers['headers'] for section in sections_in_inp)) + + +def test_get_set_curves(test_model): + + curves = test_model.inp.curves + + print (curves) diff --git a/swmmio/utils/dataframes.py b/swmmio/utils/dataframes.py index 950d050..0edb63a 100644 --- a/swmmio/utils/dataframes.py +++ b/swmmio/utils/dataframes.py @@ -81,6 +81,38 @@ def create_dataframeINP(inp_path, section='[CONDUITS]', ignore_comments=True, return df.rename(index=str) +def create_dataframe_multi_index(inp_path, section='[CURVES]', ignore_comments=True, + comment_str=';', comment_cols=True, headers=None): + # find all the headers and their defs (section title with cleaned one-liner column headers) + headerdefs = funcs.complete_inp_headers(inp_path) + # create temp file with section isolated from inp file + tempfilepath = txt.extract_section_from_inp(inp_path, section, + headerdefs=headerdefs, + ignore_comments=ignore_comments, + skipheaders=True) + + headerlist = HEADERS['inp_sections'][section] + + with open(tempfilepath, 'r') as f: + data = [] + for line in f.readlines(): + items = line.strip().split() + if len(items) == 3: + items = [items[0], None, items[1], items[2]] + if len(items) == 4: + data.append(items) + + # df = pd.read_csv(tempfilepath, header=None, delim_whitespace=True, skiprows=[0], + # index_col=[0,1], names=headerlist, comment=comment_str) + + df = pd.DataFrame(data=data, columns=headerlist) + df = df.set_index(['Name', 'Type']) + + + # os.startfile(tempfilepath) + os.remove(tempfilepath) + return df + def get_link_coords(row, nodexys, verticies): """for use in an df.apply, to get coordinates of a conduit/link """ From c89ce09e10a592ed2bc0d458e696d428ec1d25b5 Mon Sep 17 00:00:00 2001 From: Adam Erispaha Date: Fri, 25 Oct 2019 16:41:32 -0400 Subject: [PATCH 2/7] update networkx, other deps --- requirements.txt | 8 ++++---- setup.py | 7 ++++--- swmmio/__init__.py | 2 +- swmmio/tests/test_dataframes.py | 4 ++-- swmmio/tests/test_model_elements.py | 2 +- swmmio/utils/functions.py | 11 +++-------- 6 files changed, 15 insertions(+), 19 deletions(-) diff --git a/requirements.txt b/requirements.txt index 88a62bf..f22df0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ # Build dependencies #python pytest -pillow==6.0.0 -numpy==1.16.4 -pandas==0.24.2 +pillow>=6.2.0 +numpy>=1.16.4 +pandas>=0.24.2 pyshp==2.1.0 geojson==2.4.1 -networkx==2.1 +networkx>=2.4 # Run dependencies diff --git a/setup.py b/setup.py index 4148383..94d1475 100644 --- a/setup.py +++ b/setup.py @@ -32,11 +32,12 @@ def get_description(): AUTHOR_EMAIL = 'aerispaha@gmail.com' install_requires = [ - 'Pillow==6.0.0', - 'numpy==1.16.4', - 'pandas==0.24.2', + 'Pillow>=6.2.0', + 'numpy>=1.16.4', + 'pandas>=0.24.2', 'pyshp==2.1.0', 'geojson==2.4.1', + 'networkx>=2.4', ] tests_require = [ diff --git a/swmmio/__init__.py b/swmmio/__init__.py index 0b7b8b4..1858f1f 100644 --- a/swmmio/__init__.py +++ b/swmmio/__init__.py @@ -5,7 +5,7 @@ '''Python SWMM Input/Output Tools''' -VERSION_INFO = (0, 3, 6, 'dev0') +VERSION_INFO = (0, 3, 6, 'dev1') __version__ = '.'.join(map(str, VERSION_INFO)) __author__ = 'Adam Erispaha' __copyright__ = 'Copyright (c) 2016' diff --git a/swmmio/tests/test_dataframes.py b/swmmio/tests/test_dataframes.py index 072a79a..81b6966 100644 --- a/swmmio/tests/test_dataframes.py +++ b/swmmio/tests/test_dataframes.py @@ -87,7 +87,7 @@ def test_model_to_networkx(): assert (G['J2']['J3']['C2.1']['Length'] == 666) assert (G['J1']['J2']['C1:C2']['Length'] == 244.63) - assert (round(G.node['J2']['InvertElev'], 3) == 13.0) + assert (round(G.nodes['J2']['InvertElev'], 3) == 13.0) links = m.links() assert(len(links) == len(G.edges())) @@ -139,4 +139,4 @@ def pumps_old_method(model): pumps_old_method = pumps_old_method(m) pumps = m.pumps() - assert(pumps_old_method.equals(pumps)) \ No newline at end of file + assert(pumps_old_method.equals(pumps)) diff --git a/swmmio/tests/test_model_elements.py b/swmmio/tests/test_model_elements.py index a033a95..0ecbf03 100644 --- a/swmmio/tests/test_model_elements.py +++ b/swmmio/tests/test_model_elements.py @@ -1,4 +1,4 @@ -from swmmio.tests.data import MODEL_FULL_FEATURES_XY +from swmmio.tests.data import MODEL_FULL_FEATURES_XY, MODEL_FULL_FEATURES__NET_PATH from swmmio import Model from swmmio.elements import ModelSection from swmmio.utils import functions diff --git a/swmmio/utils/functions.py b/swmmio/utils/functions.py index c8042ad..a77ab95 100644 --- a/swmmio/utils/functions.py +++ b/swmmio/utils/functions.py @@ -2,6 +2,7 @@ from collections import deque import pandas as pd from swmmio.tests.data import MODEL_FULL_FEATURES_INVALID +import networkx as nx def random_alphanumeric(n=6): @@ -16,11 +17,6 @@ def model_to_networkx(model, drop_cycles=True): Networkx MultiDiGraph representation of the model ''' from geojson import Point, LineString - try: - import networkx as nx - except ImportError: - raise ImportError('networkx module needed. get this package here: ', - 'https://pypi.python.org/pypi/networkx') def multidigraph_from_edges(edges, source, target): ''' @@ -52,8 +48,7 @@ def multidigraph_from_edges(edges, source, target): model.rpt.path, "Node Inflow Summary")[inflow_cols] nodes = nodes.join(flows) - conduits = model.conduits() - links = pd.concat([conduits, model.orifices(), model.weirs(), model.pumps()], sort=True) + links = model.links() links['facilityid'] = links.index # create a nx.MultiDiGraph from the combined model links, add node data, set CRS @@ -66,7 +61,7 @@ def multidigraph_from_edges(edges, source, target): G[u][v][k]['geometry'] = LineString(coords) for n, coords in G.nodes(data='coords'): if coords: - G.node[n]['geometry'] = Point(coords[0]) + G.nodes[n]['geometry'] = Point(coords[0]) if drop_cycles: # remove cycles From 23beb2382cbd23cc6a44e6b87e61b0eb3e6a818c Mon Sep 17 00:00:00 2001 From: Adam Erispaha Date: Fri, 25 Oct 2019 16:49:59 -0400 Subject: [PATCH 3/7] add autorelease appveyor logic --- appveyor.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 9d4c5ee..cd52d6e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -36,3 +36,16 @@ test_script: # Please this command locally: # 'autopep8 -i -a -r .'" # } + +on_success: + - > + IF "%APPVEYOR_REPO_BRANCH%" == "master" + ( + IF "%APPVEYOR_REPO_TAG%" == "true" + ( + %PYTHON%\\python.exe -m pip install wheel + %PYTHON%\\python.exe setup.py bdist_wheel + %PYTHON%\\python.exe -m pip install twine && + %PYTHON%\\python.exe -m twine upload dist/* --repository-url %REPO% -u %PYPI_USERNAME% -p %PYPI_PASSWORD% + ) + ) From bcc414354606de1cc931b4f80f3a53be7267e041 Mon Sep 17 00:00:00 2001 From: Adam Erispaha Date: Fri, 25 Oct 2019 16:57:53 -0400 Subject: [PATCH 4/7] drop py2.7 from CI --- .travis.yml | 1 - appveyor.yml | 24 +----------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index c017b54..122ac9f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "2.7" - "3.6" # Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs diff --git a/appveyor.yml b/appveyor.yml index cd52d6e..7a26fc1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,19 +1,9 @@ -#matrix: -# fast_finish: true - -#branches: -# only: -# - master -# - /dev-.*/ - environment: matrix: # For Python versions available on Appveyor, see # http://www.appveyor.com/docs/installed-software#python - # The list here is complete (excluding Python 2.6, which - # isn't covered by this document) at the time of writing. - - PYTHON: "C:\\Python27" - PYTHON: "C:\\Python36" + - PYTHON: "C:\\Python37" install: - "%PYTHON%\\python setup.py develop" @@ -22,20 +12,8 @@ install: build: off test_script: - - "%PYTHON%\\Scripts\\pytest %APPVEYOR_BUILD_FOLDER%" - # Asserting pep8 formatting checks (using autopep8 tool) - # - ps: | - # $output = C:\\Python36\\Scripts\\autopep8 -d --recursive . - # if($output) - # { - # echo $output; - # $host.SetShouldExit(1) - # Write-Host "autopep8 failed: - # Please this command locally: - # 'autopep8 -i -a -r .'" - # } on_success: - > From f8c80bd49a993fb76565c825d0bc818f8be2c990 Mon Sep 17 00:00:00 2001 From: Adam Erispaha Date: Fri, 25 Oct 2019 17:01:50 -0400 Subject: [PATCH 5/7] travis simpler config --- .travis.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 122ac9f..723e2f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,8 @@ language: python python: - "3.6" + - "3.7" -# Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs -matrix: - include: - - python: 3.7 - dist: xenial - sudo: true # command to install dependencies install: - pip install -r requirements.txt From 8c0a20e6ffffab3242736947d6e99733e527869c Mon Sep 17 00:00:00 2001 From: Adam Erispaha Date: Fri, 25 Oct 2019 17:10:44 -0400 Subject: [PATCH 6/7] default pypi server --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 7a26fc1..732a5f7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,6 +24,6 @@ on_success: %PYTHON%\\python.exe -m pip install wheel %PYTHON%\\python.exe setup.py bdist_wheel %PYTHON%\\python.exe -m pip install twine && - %PYTHON%\\python.exe -m twine upload dist/* --repository-url %REPO% -u %PYPI_USERNAME% -p %PYPI_PASSWORD% + %PYTHON%\\python.exe -m twine upload dist/* -u %PYPI_USERNAME% -p %PYPI_PASSWORD% ) ) From 9f83c67634c73358cc9e7538865534699de39eb5 Mon Sep 17 00:00:00 2001 From: Adam Erispaha Date: Fri, 25 Oct 2019 17:13:55 -0400 Subject: [PATCH 7/7] update appv --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 732a5f7..b154209 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,6 +1,6 @@ environment: matrix: - # For Python versions available on Appveyor, see + # For Python versions available on Appveyor, see # http://www.appveyor.com/docs/installed-software#python - PYTHON: "C:\\Python36" - PYTHON: "C:\\Python37"