Skip to content

Commit 925bc9b

Browse files
binh-vubinh-vu
authored
Fix continuous integration (#32)
* fix tests * update ci to run test
1 parent 1a8b2d2 commit 925bc9b

24 files changed

+261
-39
lines changed

‎.github/workflows/build.sh

+94
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
# Description: builds Python's wheels.
6+
# The script needs yum or apt
7+
#
8+
# Envionment Arguments: (handled by `args.py`)
9+
# PYTHON_HOME: the path to the Python installation, which will be used to build the wheels for.
10+
# Empty if you want to use multiple Pythons by providing PYTHON_HOMES. This has the highest priority. If set, we won't consider PYTHON_HOMES and PYTHON_VERSIONS
11+
# PYTHON_HOMES: comma-separated directories that either contains Python installations or are Python installations.
12+
# PYTHON_VERSIONS: versions of Python separated by comma if you want to restricted to specific versions.
13+
# Arguments:
14+
# -t <target>: target platform. See https://doc.rust-lang.org/nightly/rustc/platform-support.html
15+
16+
export PATH=$EXTRA_PATH:$PATH
17+
18+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
19+
20+
while getopts t: flag
21+
do
22+
case "${flag}" in
23+
t) target=${OPTARG};;
24+
esac
25+
done
26+
27+
if [ -z "$target" ]
28+
then
29+
echo "target is not set (-t <target>). See more: https://doc.rust-lang.org/nightly/rustc/platform-support.html"
30+
exit 1
31+
fi
32+
33+
echo "::group::Setup build tools"
34+
# ##############################################
35+
# to build rocksdb, we need CLang and LLVM
36+
echo "Install CLang and LLVM"
37+
if ! command -v yum &> /dev/null
38+
then
39+
# debian
40+
apt update
41+
apt install -y clang-11
42+
else
43+
# centos
44+
# https://developers.redhat.com/blog/2018/07/07/yum-install-gcc7-clang#
45+
yum install -y llvm-toolset-7
46+
source /opt/rh/llvm-toolset-7/enable
47+
fi
48+
49+
# ##############################################
50+
echo "Install Rust"
51+
if ! command -v cargo &> /dev/null
52+
then
53+
# install rust and cargo
54+
curl --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
55+
source $HOME/.cargo/env
56+
else
57+
echo "Rust is already installed."
58+
rustup show
59+
fi
60+
61+
if [ ! -d $(rustc --print target-libdir --target "$target" ) ]
62+
then
63+
rustup target add $target;
64+
fi
65+
66+
echo "::endgroup::"
67+
echo
68+
69+
echo "::group::Discovering Python"
70+
IFS=',' read -a PYTHON_HOMES <<< $(MINIMUM_PYTHON_VERSION=3.8 python $SCRIPT_DIR/pydiscovery.py)
71+
if [ ${#PYTHON_HOMES[@]} -eq 0 ]; then
72+
echo "No Python found. Did you forget to set any environment variable PYTHON_HOME or PYTHON_HOMES?"
73+
else
74+
for PYTHON_HOME in "${PYTHON_HOMES[@]}"
75+
do
76+
echo "Found $PYTHON_HOME"
77+
done
78+
fi
79+
echo "::endgroup::"
80+
echo
81+
82+
for PYTHON_HOME in "${PYTHON_HOMES[@]}"
83+
do
84+
echo "::group::Building for Python $PYTHON_HOME"
85+
86+
echo "Run: $PYTHON_HOME/bin/pip install maturin"
87+
"$PYTHON_HOME/bin/pip" install maturin
88+
89+
echo "Run: $PYTHON_HOME/bin/maturin build -r -o dist -i $PYTHON_HOME/bin/python --target $target"
90+
"$PYTHON_HOME/bin/maturin" build -r -o dist -i "$PYTHON_HOME/bin/python" --target $target
91+
92+
echo "::endgroup::"
93+
echo
94+
done

‎.github/workflows/ci.yml

+59-9
Original file line numberDiff line numberDiff line change
@@ -6,48 +6,98 @@ on:
66

77
jobs:
88
linux:
9+
strategy:
10+
matrix:
11+
platform:
12+
- target: x86_64-unknown-linux-gnu
13+
image: quay.io/pypa/manylinux2014_x86_64:latest
14+
run_test: true
15+
# - target: i686-unknown-linux-gnu
16+
# image: quay.io/pypa/manylinux2014_i686:latest
917
runs-on: ubuntu-latest
1018
steps:
1119
- uses: actions/checkout@v3
12-
- name: Run test
13-
run: cargo test --no-default-features --features pyo3/auto-initialize
14-
- uses: messense/maturin-action@v1
20+
- name: Build wheels
21+
run: |
22+
docker run --rm -w /project -v $(pwd):/project \
23+
-e EXTRA_PATH=/opt/python/cp38-cp38/bin \
24+
-e PYTHON_HOMES=/opt/python \
25+
-e CARGO_NET_GIT_FETCH_WITH_CLI=false \
26+
${{ matrix.platform.image }} \
27+
bash /project/.github/workflows/build.sh -t ${{ matrix.platform.target }}
28+
- name: Prepare to run test
29+
if: matrix.platform.run_test == true
30+
uses: actions/setup-python@v4
1531
with:
16-
manylinux: auto
17-
command: build
18-
args: --release -o dist
32+
python-version: 3.8
33+
- name: Run test
34+
if: matrix.platform.run_test == true
35+
run: |
36+
pip install dist/*cp38*.whl
37+
pip install pytest
38+
mv python/drepr python/drepr2
39+
pytest -xvs python/tests
1940
- name: Upload wheels
2041
uses: actions/upload-artifact@v2
2142
with:
2243
name: wheels
2344
path: dist
2445

2546
windows:
47+
strategy:
48+
matrix:
49+
python: ["3.8", "3.9", "3.10"]
2650
runs-on: windows-latest
2751
steps:
2852
- uses: actions/checkout@v3
53+
- name: Set up Python
54+
uses: actions/setup-python@v4
55+
with:
56+
python-version: ${{ matrix.python }}
2957
- name: Run test
3058
run: cargo test --no-default-features --features pyo3/auto-initialize
3159
- uses: messense/maturin-action@v1
3260
with:
3361
command: build
34-
args: --release --no-sdist -o dist
62+
args: --release --no-sdist -o dist -i python
63+
# - name: Run test
64+
# if: matrix.python == '3.8'
65+
# run: |
66+
# ls dist
67+
# bash -c 'pwd; pip install dist/*cp38*.whl'
68+
# pip install pytest
69+
# mv python/drepr python/drepr2
70+
# pytest -xvs python/tests
3571
- name: Upload wheels
3672
uses: actions/upload-artifact@v2
3773
with:
3874
name: wheels
3975
path: dist
4076

4177
macos:
78+
strategy:
79+
matrix:
80+
python: ["3.8", "3.9", "3.10"]
4281
runs-on: macos-latest
4382
steps:
4483
- uses: actions/checkout@v3
84+
- name: Set up Python
85+
uses: actions/setup-python@v4
86+
with:
87+
python-version: ${{ matrix.python }}
4588
- name: Run test
4689
run: cargo test --no-default-features --features pyo3/auto-initialize
4790
- uses: messense/maturin-action@v1
4891
with:
4992
command: build
50-
args: --release --no-sdist -o dist --universal2
93+
args: --release --no-sdist -o dist
94+
- name: Run test
95+
if: matrix.python == '3.8'
96+
run: |
97+
pip install dist/*cp38*.whl
98+
pip install pytest
99+
mv python/drepr python/drepr2
100+
pytest -xvs python/tests
51101
- name: Upload wheels
52102
uses: actions/upload-artifact@v2
53103
with:
@@ -57,7 +107,7 @@ jobs:
57107
release:
58108
name: Release
59109
runs-on: ubuntu-latest
60-
if: "startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/master')"
110+
if: "startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/heads/dev-ci')"
61111
needs: [macos, windows, linux]
62112
steps:
63113
- uses: actions/download-artifact@v2

‎.github/workflows/pydiscovery.py

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
from __future__ import print_function
2+
import os, subprocess, re
3+
4+
5+
homes = []
6+
if "PYTHON_HOME" in os.environ:
7+
homes.append(os.environ["PYTHON_HOME"])
8+
elif "PYTHON_HOMES" in os.environ:
9+
lst = os.environ["PYTHON_HOMES"].split(",")
10+
for path in lst:
11+
if os.path.exists(os.path.join(path, "bin", "python")):
12+
# is the python directory
13+
homes.append(path)
14+
else:
15+
for subpath in os.listdir(path):
16+
if os.path.exists(os.path.join(path, subpath, "bin", "python")):
17+
homes.append(os.path.join(path, subpath))
18+
19+
if "PYTHON_VERSIONS" in os.environ:
20+
versions = os.environ["PYTHON_VERSIONS"].split(",")
21+
filtered_homes = []
22+
for home in homes:
23+
output = (
24+
subprocess.check_output([os.path.join(home, "bin", "python"), "-V"])
25+
.decode()
26+
.strip()
27+
)
28+
for version in versions:
29+
m = re.match("Python ([\d\.)]+)", output)
30+
assert m is not None
31+
pyversion = m.group(1)
32+
if pyversion.startswith(version):
33+
filtered_homes.append(home)
34+
break
35+
36+
homes = filtered_homes
37+
38+
39+
if "MINIMUM_PYTHON_VERSION" in os.environ:
40+
minimum_version = [int(d) for d in os.environ["MINIMUM_PYTHON_VERSION"].split(".")]
41+
filtered_homes = []
42+
for home in homes:
43+
output = (
44+
subprocess.check_output([os.path.join(home, "bin", "python"), "-V"])
45+
.decode()
46+
.strip()
47+
)
48+
m = re.match(r"Python ([\d\.)]+)", output)
49+
assert m is not None
50+
pyversion = m.group(1).split(".")
51+
52+
if all(
53+
int(pyversion[i]) >= minimum_version[i] for i in range(len(minimum_version))
54+
):
55+
filtered_homes.append(home)
56+
57+
homes = filtered_homes
58+
59+
print(",".join(homes))
60+
exit(0)

‎.vscode/launch.json

+8-7
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,16 @@
55
"version": "0.2.0",
66
"configurations": [
77
{
8-
"name": "Python: Current File",
8+
"name": "pytest",
99
"type": "python",
1010
"request": "launch",
11-
"program": "${file}",
12-
"console": "integratedTerminal",
13-
"justMyCode": true,
14-
"env": {
15-
"RUST_BACKTRACE": "1"
16-
}
11+
"module": "pytest",
12+
"args": [
13+
"-xvs",
14+
"python/tests",
15+
// "python/tests/drepr/outputs/test_get_data_as_ndarray.py::test_get_prop_as_ndarray"
16+
],
17+
"justMyCode": true
1718
}
1819
]
1920
}

‎pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name = "drepr"
33
version = "2.10.0"
44
description = "Data Representation Language for Reading Heterogeneous Datasets"
55
readme = "README.md"
6-
requires-python = ">=3.7"
6+
requires-python = ">=3.8"
77
license = {file = "LICENSE"}
88
authors = [{name = "Binh Vu", email = "[email protected]"}]
99

‎python/drepr/__init__.py

+2-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
# from drepr.version import __version__
2-
# from drepr.engine import execute, FileOutput, MemoryOutput, OutputFormat
1+
from drepr.engine import execute, FileOutput, MemoryOutput, OutputFormat
32
# from drepr.graph_deprecated import Graph
4-
# from drepr.models import DRepr, DReprBuilder
5-
# from drepr import outputs
6-
# from drepr import models
3+
from drepr.models import DRepr, DReprBuilder

‎python/drepr/engine.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ def execute(
111111
if isinstance(output, MemoryOutput) and output.format == OutputFormat.GraphPy:
112112
class2nodes = {}
113113
for u in ds_model.sm.iter_class_nodes():
114-
class2nodes[u.node_id] = result["class2nodes"][
114+
class2nodes[u.node_id] = result[
115115
engine_model.sm_node_idmap[u.node_id]
116116
]
117117
return class2nodes

‎python/drepr/models/parse_v2/sm_parser.py

+8
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,14 @@ def parse(cls, sm: dict) -> SemanticModel:
119119
f"{trace1}\nParsing data type")
120120
data_type = DataType(data_type)
121121

122+
# normalize value's type (e.g., ruamel.yaml read float into ScalarFloat)
123+
if isinstance(value, str):
124+
value = str(value)
125+
elif isinstance(value, int):
126+
value = int(value)
127+
elif isinstance(value, float):
128+
value = float(value)
129+
122130
node = LiteralNode(node_id=f"lnode:{len(nodes)}", value=value, data_type=data_type)
123131
nodes[node.node_id] = node
124132
edges[len(edges)] = Edge(len(edges), class_id, node.node_id, predicate)

‎python/drepr/models/sm.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ class DataNode:
4242
@dataclass
4343
class LiteralNode:
4444
node_id: str
45-
value: str
45+
# you should rely on data_type to get the type of value right. The parser may be wrong about it.
46+
value: Union[str, int, float]
4647
data_type: Optional[DataType] = None
4748

4849

‎python/drepr/outputs/__init__.py

-5
Original file line numberDiff line numberDiff line change
@@ -1,5 +0,0 @@
1-
from .array_backend.array_backend import ArrayBackend
2-
from .graph_backend.graph_backend import GraphBackend
3-
from .namespace import Namespace
4-
from .base_output_sm import FCondition
5-
from .prop_data_ndarray import PropDataNDArray

‎python/tests/conftest.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
def get_examples_dir():
88
testdir = Path(os.path.abspath(__file__)).parent
9-
return testdir.parent / "examples"
9+
return testdir.parent.parent / "examples"
1010

1111

1212
@pytest.fixture()

‎python/tests/drepr/models/__init__.py

Whitespace-only changes.

‎python/tests/drepr/engine/__init__.py ‎python/tests/engine/__init__.py

File renamed without changes.

‎python/tests/drepr/engine/conftest.py ‎python/tests/engine/conftest.py

File renamed without changes.

‎python/tests/drepr/engine/test_complete_description.py ‎python/tests/engine/test_complete_description.py

File renamed without changes.

‎python/tests/drepr/__init__.py ‎python/tests/models/__init__.py

File renamed without changes.

‎python/tests/drepr/models/conftest.py ‎python/tests/models/conftest.py

File renamed without changes.

‎python/tests/drepr/models/test_drepr.py ‎python/tests/models/test_drepr.py

File renamed without changes.

‎python/tests/drepr/outputs/__init__.py ‎python/tests/outputs/__init__.py

File renamed without changes.

‎python/tests/drepr/outputs/conftest.py ‎python/tests/outputs/conftest.py

+10-5
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,20 @@
44

55
import pytest
66

7-
from drepr.outputs.array_backend.array_backend import ArrayBackend
7+
try:
8+
from drepr.outputs.array_backend.array_backend import ArrayBackend
9+
except ModuleNotFoundError:
10+
ArrayBackend = None
11+
812
from drepr.outputs.graph_backend.graph_backend import GraphBackend
913

1014

1115
def get_backends(dataset_dir: Path):
12-
return [
13-
ArrayBackend.from_drepr(str(dataset_dir / "model.yml"), str(dataset_dir / "resource.json")),
14-
GraphBackend.from_drepr(str(dataset_dir / "model.yml"), str(dataset_dir / "resource.json"))
15-
]
16+
backends = []
17+
if ArrayBackend is not None:
18+
backends.append(ArrayBackend.from_drepr(str(dataset_dir / "model.yml"), str(dataset_dir / "resource.json")))
19+
backends.append(GraphBackend.from_drepr(str(dataset_dir / "model.yml"), str(dataset_dir / "resource.json")))
20+
return backends
1621

1722

1823
@pytest.fixture()

0 commit comments

Comments
 (0)