Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor: use pathlib #1948

Open
wants to merge 22 commits into
base: main
Choose a base branch
from
Open

refactor: use pathlib #1948

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
7e18a8e
refactor(bootstrap obv ci): use pathlib
hulkoba May 30, 2024
12b897c
refactor(azur ci utils): use pathlib
hulkoba May 30, 2024
f7e057a
refactor(ci register): use pathlib
hulkoba May 30, 2024
8f43a2a
refactor(ci skeleton): use pathlib
hulkoba May 30, 2024
d2b239e
refactor(cli): use pathlib
hulkoba May 30, 2024
9d352f7
refactor(conf feedstock): use pathlib
hulkoba May 30, 2024
e6788d3
refactor(feedstock io): use pathlib
hulkoba May 30, 2024
7f707b9
refactor(feedstock tokens): use pathlib
hulkoba May 30, 2024
3cc7975
refactor(feedstocks): use pathlib
hulkoba May 30, 2024
85a4414
refactor(github): use pathlib
hulkoba May 30, 2024
abe0937
refactor(lint recipe): use pathlib
hulkoba May 30, 2024
e7d347c
refactor(utils): use pathlib
hulkoba May 30, 2024
c604e79
refactor(build locally): use pathlib
hulkoba May 30, 2024
4f0dca5
refactor(conftest): use pathlib
hulkoba May 30, 2024
e07e8ce
refactor(test cli): use pathlib
hulkoba May 30, 2024
9e329f6
refactor(test conf feedstock): use pathlib
hulkoba May 30, 2024
ec8d5da
refactor(test feedstock io): use pathlib
hulkoba May 30, 2024
356503e
refactor(test feedstock tokens): use pathlib
hulkoba May 30, 2024
171ad9e
refactor(test lint recipe): use pathlib
hulkoba May 30, 2024
425fed4
refactor(rever): use pathlib
hulkoba May 30, 2024
ef40b60
add news entry for pathlib refactor
hulkoba May 30, 2024
19e578d
fix: resolve linting errors in pathlib refactor
AlbaHerrerias Jun 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions bootstrap-obvious-ci-and-miniconda.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,9 @@

"""
import argparse
import os
from pathlib import Path
import platform
import subprocess
import sys

try:
from urllib.request import urlretrieve
Expand Down Expand Up @@ -85,22 +84,22 @@ def main(
else:
raise ValueError("Unsupported operating system.")

if not os.path.exists(basename):
if not Path(basename).exists():
print("Downloading from {}".format(URL))
urlretrieve(URL, basename)
else:
print("Using cached version of {}".format(URL))

# Install with powershell.
if os.path.exists(target_dir):
if Path(target_dir).exists():
raise IOError("Installation directory already exists")
subprocess.check_call(cmd)

if not os.path.isdir(target_dir):
if not Path(target_dir).is_dir():
raise RuntimeError("Failed to install miniconda :(")

if install_obvci:
conda_path = os.path.join(target_dir, bin_dir, "conda")
conda_path = Path(target_dir, bin_dir, "conda")
subprocess.check_call(
Comment on lines 93 to 103
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand why we are using several Path instantiations here when it's all target_dir. We should have something like:

target_dir = Path(target_dir)
if target_dir.exists():
  ...
if not target_dir.is_dir():
  ...
if install_obvci:
  conda_path = target_dir / bin_dir / "conda"

and so on. I'm afraid this happens in more areas of the PR, so a careful review is advised imo.

[
conda_path,
Expand Down
3 changes: 2 additions & 1 deletion conda_smithy/azure_ci_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
from pathlib import Path
import typing
import warnings

Expand Down Expand Up @@ -41,7 +42,7 @@ def __init__(

try:
with open(
os.path.expanduser("~/.conda-smithy/azure.token"), "r"
Copy link
Member

@wolfv wolfv Jul 23, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about Path("~/.conda-smithy/azure.token").expanduser().read_text().strip()?

Path("~/.conda-smithy/azure.token").expanduser(), "r"
) as fh:
self.token = fh.read().strip()
if not self.token:
Expand Down
11 changes: 6 additions & 5 deletions conda_smithy/ci_register.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#!/usr/bin/env python
import os
from pathlib import Path
import requests
import time
import sys
Expand All @@ -14,7 +15,7 @@
# https://circleci.com/api/v1/project/:username/:project/envvar?circle-token=:token

try:
with open(os.path.expanduser("~/.conda-smithy/circle.token"), "r") as fh:
with open(Path("~/.conda-smithy/circle.token").expanduser(), "r") as fh:
circle_token = fh.read().strip()
if not circle_token:
raise ValueError()
Expand All @@ -25,7 +26,7 @@
)

try:
with open(os.path.expanduser("~/.conda-smithy/appveyor.token"), "r") as fh:
with open(Path("~/.conda-smithy/appveyor.token").expanduser(), "r") as fh:
appveyor_token = fh.read().strip()
if not appveyor_token:
raise ValueError()
Expand All @@ -36,7 +37,7 @@
)

try:
with open(os.path.expanduser("~/.conda-smithy/drone.token"), "r") as fh:
with open(Path("~/.conda-smithy/drone.token").expanduser(), "r") as fh:
drone_token = fh.read().strip()
if not drone_token:
raise ValueError()
Expand All @@ -51,7 +52,7 @@
except KeyError:
try:
with open(
os.path.expanduser("~/.conda-smithy/anaconda.token"), "r"
Path("~/.conda-smithy/anaconda.token").expanduser(), "r"
) as fh:
anaconda_token = fh.read().strip()
if not anaconda_token:
Expand Down Expand Up @@ -91,7 +92,7 @@ def travis_headers():
"Content-Type": "application/json",
"Travis-API-Version": "3",
}
travis_token = os.path.expanduser("~/.conda-smithy/travis.token")
travis_token = Path("~/.conda-smithy/travis.token").expanduser()
try:
with open(travis_token, "r") as fh:
token = fh.read().strip()
Expand Down
30 changes: 13 additions & 17 deletions conda_smithy/ci_skeleton.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,20 @@
added to conda-forge's queue.
"""

import os
from pathlib import Path
import sys

from .configure_feedstock import make_jinja_env


def _render_template(template_file, env, forge_dir, config):
"""Renders the template"""
template = env.get_template(
os.path.basename(template_file) + ".ci-skel.tmpl"
)
target_fname = os.path.join(forge_dir, template_file)
print("Generating " + target_fname, file=sys.stderr)
template_file_name = Path(template_file).name
template = env.get_template(template_file_name + ".ci-skel.tmpl")
target_fname = Path(forge_dir, template_file)
print("Generating ", target_fname, file=sys.stderr)
new_file_contents = template.render(**config)
os.makedirs(os.path.dirname(target_fname), exist_ok=True)
target_fname.parent.mkdir(parents=True, exist_ok=True)
with open(target_fname, "w") as fh:
fh.write(new_file_contents)

Expand All @@ -37,18 +36,16 @@ def _insert_into_gitignore(
):
"""Places gitignore contents into gitignore."""
# get current contents
fname = os.path.join(feedstock_directory, ".gitignore")
print("Updating " + fname)
if os.path.isfile(fname):
fname = Path(feedstock_directory, ".gitignore")
print("Updating ", fname.name)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
print("Updating ", fname.name)
print("Updating", fname.name)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure, isn't .name going to give you only the file name (not the whole path as previously)?

I think it should be print("Updating", fname).

if fname.is_file():
with open(fname, "r") as f:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This can be written as fname.read_text() or fname.read_binary().

s = f.read()
before, _, s = s.partition(prefix)
_, _, after = s.partition(suffix)
else:
before = after = ""
dname = os.path.dirname(fname)
if dname:
os.makedirs(dname, exist_ok=True)
fname.parent.mkdir(parents=True, exist_ok=True)
new = prefix + GITIGNORE_ADDITIONAL + suffix
# write out the file
with open(fname, "w") as f:
Expand All @@ -60,7 +57,7 @@ def generate(
package_name="pkg", feedstock_directory=".", recipe_directory="recipe"
):
"""Generates the CI skeleton."""
forge_dir = os.path.abspath(feedstock_directory)
forge_dir = Path(feedstock_directory).resolve()
env = make_jinja_env(forge_dir)
config = dict(
package_name=package_name,
Expand All @@ -69,8 +66,7 @@ def generate(
)
# render templates
_render_template("conda-forge.yml", env, forge_dir, config)
_render_template(
os.path.join(recipe_directory, "meta.yaml"), env, forge_dir, config
)
recipe_file_name = str(Path(recipe_directory, "meta.yaml"))
_render_template(recipe_file_name, env, forge_dir, config)
# update files which may exist with other content
_insert_into_gitignore(feedstock_directory=feedstock_directory)
51 changes: 25 additions & 26 deletions conda_smithy/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import os
import logging
from pathlib import Path
import subprocess
import sys
import time
Expand Down Expand Up @@ -28,20 +28,19 @@


def default_feedstock_config_path(feedstock_directory):
return os.path.join(feedstock_directory, "conda-forge.yml")
return str(Path(feedstock_directory, "conda-forge.yml"))


def generate_feedstock_content(target_directory, source_recipe_dir):
target_directory = os.path.abspath(target_directory)
target_directory = Path(target_directory).resolve()
recipe_dir = "recipe"
target_recipe_dir = os.path.join(target_directory, recipe_dir)
target_recipe_dir = Path(target_directory, recipe_dir)
target_recipe_dir.mkdir(parents=True, exist_ok=True)

if not os.path.exists(target_recipe_dir):
os.makedirs(target_recipe_dir)
# If there is a source recipe, copy it now to the right dir
if source_recipe_dir:
try:
configure_feedstock.copytree(source_recipe_dir, target_recipe_dir)
configure_feedstock.copytree(source_recipe_dir, str(target_recipe_dir))
except Exception as e:
import sys

Expand All @@ -50,16 +49,16 @@ def generate_feedstock_content(target_directory, source_recipe_dir):
).with_traceback(sys.exc_info()[2])

forge_yml = default_feedstock_config_path(target_directory)
if not os.path.exists(forge_yml):
if not Path(forge_yml).exists():
with feedstock_io.write_file(forge_yml) as fh:
fh.write("{}")

# merge in the existing configuration in the source recipe directory
forge_yml_recipe = os.path.join(source_recipe_dir, "conda-forge.yml")
forge_yml_recipe = Path(source_recipe_dir, "conda-forge.yml")
yaml = YAML()
if os.path.exists(forge_yml_recipe):
if forge_yml_recipe.exists():
feedstock_io.remove_file(
os.path.join(target_recipe_dir, "conda-forge.yml")
target_recipe_dir.joinpath("conda-forge.yml")
)
try:
with open(forge_yml_recipe, "r") as fp:
Expand Down Expand Up @@ -114,7 +113,7 @@ def __init__(self, parser):

def __call__(self, args):
# check some error conditions
if args.recipe_directory and not os.path.isdir(args.recipe_directory):
if args.recipe_directory and not Path(args.recipe_directory).is_dir():
raise IOError(
"The source recipe directory should be the directory of the "
"conda-recipe you want to build a feedstock for. Got {}".format(
Expand All @@ -135,7 +134,7 @@ def __call__(self, args):
__version__
)

os.makedirs(feedstock_directory)
Path(feedstock_directory).mkdir(parents=True)
subprocess.check_call(["git", "init"], cwd=feedstock_directory)
generate_feedstock_content(feedstock_directory, args.recipe_directory)
subprocess.check_call(
Expand Down Expand Up @@ -224,7 +223,7 @@ def __init__(self, parser):
scp = self.subcommand_parser
scp.add_argument(
"--feedstock_directory",
default=feedstock_io.get_repo_root(os.getcwd()) or os.getcwd(),
default=feedstock_io.get_repo_root(Path.cwd()) or Path.cwd(),
help="The directory of the feedstock git repository.",
)
scp.add_argument(
Expand Down Expand Up @@ -473,7 +472,7 @@ def __init__(self, parser):
scp = self.subcommand_parser
scp.add_argument(
"--feedstock_directory",
default=feedstock_io.get_repo_root(os.getcwd()) or os.getcwd(),
default=feedstock_io.get_repo_root(Path.cwd()) or Path.cwd(),
help="The directory of the feedstock git repository.",
)
scp.add_argument(
Expand Down Expand Up @@ -502,7 +501,7 @@ def __call__(self, args):
from conda_smithy import azure_ci_utils

owner = args.user or args.organization
repo = os.path.basename(os.path.abspath(args.feedstock_directory))
repo = Path(args.feedstock_directory).resolve().name

config = azure_ci_utils.AzureConfig(
org_or_user=owner, project_name=args.project_name
Expand Down Expand Up @@ -537,7 +536,7 @@ def __init__(self, parser):
scp = self.subcommand_parser
scp.add_argument(
"--feedstock_directory",
default=feedstock_io.get_repo_root(os.getcwd()) or os.getcwd(),
default=feedstock_io.get_repo_root(Path.cwd()) or Path.cwd(),
help="The directory of the feedstock git repository.",
)
scp.add_argument(
Expand Down Expand Up @@ -608,13 +607,13 @@ def __init__(self, parser):
)
scp = self.subcommand_parser
scp.add_argument("--conda-forge", action="store_true")
scp.add_argument("recipe_directory", default=[os.getcwd()], nargs="*")
scp.add_argument("recipe_directory", default=[Path.cwd()], nargs="*")

def __call__(self, args):
all_good = True
for recipe in args.recipe_directory:
lints, hints = lint_recipe.main(
os.path.join(recipe),
Path(recipe),
conda_forge=args.conda_forge,
return_hints=True,
)
Expand Down Expand Up @@ -682,7 +681,7 @@ def __init__(self, parser):
scp = self.subcommand_parser
scp.add_argument(
"--feedstock-directory",
default=os.getcwd(),
default=Path.cwd(),
help="The directory of the feedstock git repository.",
dest="feedstock_directory",
)
Expand Down Expand Up @@ -755,7 +754,7 @@ def __init__(self, parser):
scp = self.subcommand_parser
scp.add_argument(
"--feedstock_directory",
default=feedstock_io.get_repo_root(os.getcwd()) or os.getcwd(),
default=feedstock_io.get_repo_root(Path.cwd()) or Path.cwd(),
help="The directory of the feedstock git repository.",
)
scp.add_argument(
Expand All @@ -780,7 +779,7 @@ def __call__(self, args):
)

owner = args.user or args.organization
repo = os.path.basename(os.path.abspath(args.feedstock_directory))
repo = Path(args.feedstock_directory).resolve().name

if not args.unique_token_per_provider:
generate_and_write_feedstock_token(owner, repo)
Expand Down Expand Up @@ -831,7 +830,7 @@ def __init__(self, parser):
scp = self.subcommand_parser
scp.add_argument(
"--feedstock_directory",
default=feedstock_io.get_repo_root(os.getcwd()) or os.getcwd(),
default=feedstock_io.get_repo_root(Path.cwd()) or Path.cwd(),
help="The directory of the feedstock git repository.",
)
scp.add_argument(
Expand Down Expand Up @@ -897,7 +896,7 @@ def __call__(self, args):
drone_endpoints = [drone_default_endpoint]

owner = args.user or args.organization
repo = os.path.basename(os.path.abspath(args.feedstock_directory))
repo = Path(args.feedstock_directory).resolve().name

if args.token_repo is None:
token_repo = (
Expand Down Expand Up @@ -978,7 +977,7 @@ def __init__(self, parser):
scp = self.subcommand_parser
scp.add_argument(
"--feedstock_directory",
default=feedstock_io.get_repo_root(os.getcwd()) or os.getcwd(),
default=feedstock_io.get_repo_root(Path.cwd()) or Path.cwd(),
help="The directory of the feedstock git repository.",
)
scp.add_argument(
Expand Down Expand Up @@ -1031,7 +1030,7 @@ def __call__(self, args):
from conda_smithy.anaconda_token_rotation import rotate_anaconda_token

owner = args.user or args.organization
repo = os.path.basename(os.path.abspath(args.feedstock_directory))
repo = Path(args.feedstock_directory).resolve().name

if args.feedstock_config is None:
args.feedstock_config = default_feedstock_config_path(
Expand Down
Loading