Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ENH] add new UI #313

Merged
merged 32 commits into from
May 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
c2f95a6
refactor and deal with simple visbility
Remi-Gau May 20, 2024
69a273a
add protocol page
Remi-Gau May 21, 2024
c6920d2
display activity in protocol page
Remi-Gau May 21, 2024
db86e44
fix issue new instance form
Remi-Gau May 21, 2024
1371a63
fix path activity
Remi-Gau May 21, 2024
3a477c7
style items and deal with sliders items
Remi-Gau May 22, 2024
a87de83
add min and max value for int and float
Remi-Gau May 22, 2024
4337acc
add min max to float and refactor ui
Remi-Gau May 22, 2024
cf5c94c
refactor
Remi-Gau May 22, 2024
1c98007
basic upload
Remi-Gau May 23, 2024
569c710
enable upload
Remi-Gau May 23, 2024
4beb48d
refactor
Remi-Gau May 23, 2024
79af006
package
Remi-Gau May 24, 2024
f273fa4
reorganize items description and details
Remi-Gau May 24, 2024
a06b4e4
update details items
Remi-Gau May 24, 2024
4e2346d
update visbility handling
Remi-Gau May 25, 2024
91e0a21
upload bold json
Remi-Gau May 25, 2024
af5f34b
adapt size
Remi-Gau May 25, 2024
0f4df82
refactor ui
Remi-Gau May 27, 2024
b51104f
refactor tests
Remi-Gau May 27, 2024
3519a09
refactor
Remi-Gau May 27, 2024
07d6ba3
generate and download methods
Remi-Gau May 27, 2024
c399b81
reformat jinja templates
Remi-Gau May 28, 2024
a9f7a5a
add section about preprocessing
Remi-Gau May 28, 2024
4bb563f
refactor
Remi-Gau May 28, 2024
31eb4dd
move ui into new repo
Remi-Gau May 28, 2024
4fc2719
update precommit
Remi-Gau May 28, 2024
f1440a5
Merge remote-tracking branch 'origin/main' into ui
Remi-Gau May 28, 2024
0184233
update all schemas
Remi-Gau May 28, 2024
312e831
update precommit
Remi-Gau May 28, 2024
0041650
drop reproschema-ui
Remi-Gau May 28, 2024
e279d5d
fix tests
Remi-Gau May 28, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .flake8
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
[flake8]
exclude =
.git,
./env
**/env
ecobidas/_version.py

docstring-convention = numpy
max-line-length = 150
max_complexity = 15
max_function_length = 150
max_parameters_amount = 10
max_returns_amount = 4
max_returns_amount = 7

# ----------------------- errors to include / exclude -----------------------

Expand Down
4 changes: 0 additions & 4 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,3 @@
[submodule "reproschema-py"]
path = reproschema-py
url = https://github.com/Remi-Gau/reproschema-py.git
[submodule "reproschema-ui"]
path = reproschema-ui
url = https://github.com/Remi-Gau/reproschema-ui.git
datalad-url = https://github.com/Remi-Gau/reproschema-ui.git
8 changes: 8 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,14 @@ repos:
# additional_dependencies: [types-all, pandas-stubs]
# args: [--config-file=pyproject.toml]

# Check formatting of CSS and HTML
# prettier: https://prettier.io/
- repo: https://github.com/pre-commit/mirrors-prettier
rev: v4.0.0-alpha.8
hooks:
- id: prettier
types_or: [css, html, json]

# Check that Python code complies with PEP8 guidelines
# flake8 uses pydocstyle to check docstrings: https://flake8.pycqa.org/en/latest/
# flake8-docstrings: https://pypi.org/project/flake8-docstrings/
Expand Down
13 changes: 6 additions & 7 deletions .vscode/extensions.json
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
{
"recommendations": [
"ms-python.python",
"sourcery.sourcery",
"ms-python.vscode-pylance"
],
"unwantedRecommendations": [
]
"recommendations": [
"ms-python.python",
"sourcery.sourcery",
"ms-python.vscode-pylance"
],
"unwantedRecommendations": []
}
2 changes: 1 addition & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
{
"restructuredtext.pythonRecommendation.disabled": true
"restructuredtext.pythonRecommendation.disabled": true
}
2 changes: 1 addition & 1 deletion cobidas_schema
Submodule cobidas_schema updated 790 files
2 changes: 1 addition & 1 deletion ecobidas/cli.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import sys
from typing import Sequence
from collections.abc import Sequence

from loguru import logger
from rich_argparse import RichHelpFormatter
Expand Down
3 changes: 1 addition & 2 deletions ecobidas/create_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,11 +85,10 @@ def create_schema(

item_info = get_item_info(this_item)

item_info["id"] = f"{activity_idx:.0f}.{item_idx:.0f}"

print_item_info(activity_idx, item_idx, item_info)

item = define_new_item(item_info)

item.write(os.path.join(activity_path, "items"))

activity.append_item(item)
Expand Down
26 changes: 23 additions & 3 deletions ecobidas/download_tsv.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Download the content of the different google spreadsheet in the inputs folder."""
import ast
import json
from pathlib import Path

Expand All @@ -19,19 +20,19 @@ def download_spreadsheet(schema: str, output_dir: Path = None) -> None:
spreadsheets_info = get_spreadsheets_info()

# Initialize lists to store data
google_IDs = []
google_ids = []
subfolders = []
output_filenames = []

# Parse the file
for spreadsheet, values in spreadsheets_info.items():
if spreadsheet.startswith(schema):
google_IDs.append(values["google_id"])
google_ids.append(values["google_id"])
subfolders.append(values["dir"])
output_filenames.append(values["basename"])

# Iterate through entries and download spreadsheets
for google_id, subfolder, output_filename in zip(google_IDs, subfolders, output_filenames):
for google_id, subfolder, output_filename in zip(google_ids, subfolders, output_filenames):

output_folder = output_dir / subfolder
output_folder.mkdir(exist_ok=True, parents=True)
Expand Down Expand Up @@ -95,6 +96,25 @@ def validate_downloaded_file(file: str | Path) -> None:
f"\nThe following columns are missing from the data dictionary: {sorted(extra_columns)}"
)

invalid_vis = []
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

issue (complexity): Consider simplifying the new validation logic for visibility values.

The new code introduces additional complexity that may not be necessary. Here are some points to consider:

  1. Introduction of ast Module: The new code introduces the ast module and a new function is_valid_python. While this might be necessary for specific validation, it adds additional complexity. The original code did not require this module, and its introduction should be justified with a clear need.

  2. New Validation Logic: The new validation logic for visibility values adds a loop and a function call that were not present in the original code. This increases the cognitive load for anyone reading or maintaining the code.

  3. Error Handling: The new is_valid_python function includes error handling for SyntaxError and ValueError, which adds more branches and potential points of failure in the code.

  4. Logging: The new code adds additional logging for invalid visibility values. While logging is generally good for debugging, it adds more lines of code and complexity.

  5. Code Structure: The new code introduces a new function is_valid_python and modifies the main function. This changes the structure of the code, making it less straightforward than the original.

Consider simplifying the code to maintain readability and ease of maintenance. Here is a less complex way to achieve the same functionality:

#!/usr/bin/env python3
"""Download the content of the different google spreadsheet in the inputs folder."""
import json
from pathlib import Path

import pandas as pd
import requests
from loguru import logger

from ecobidas.utils import get_input_dir, get_spreadsheets_info

# Function to download spreadsheet
def download_spreadsheet(schema: str, output_dir: Path = None) -> None:
    if output_dir is None:
        output_dir = get_input_dir()
    spreadsheets_info = get_spreadsheets_info()
    # Initialize lists to store data
    google_ids = []
    subfolders = []
    output_filenames = []

    # Parse the file
    for spreadsheet, values in spreadsheets_info.items():
        if spreadsheet.startswith(schema):
            google_ids.append(values["google_id"])
            subfolders.append(values["dir"])
            output_filenames.append(values["basename"])

    # Iterate through entries and download spreadsheets
    for google_id, subfolder, output_filename in zip(google_ids, subfolders, output_filenames):
        output_folder = output_dir / subfolder
        output_folder.mkdir(exist_ok=True, parents=True)
        output_file = output_folder / f"{output_filename}.tsv"

        logger.info(
            f"\nDownloading GoogleSheet https://docs.google.com/spreadsheets/d/{google_id} "
            f"\nfor {subfolder}/{output_filename} spreadsheet "
            f"\nto {output_file}"
        )

def validate_downloaded_file(file: Path) -> None:
    df = pd.read_csv(file, sep='\t')
    columns = set(df.columns)
    data_dictionary = get_data_dictionary()

    required_keys = set(data_dictionary['required'])
    recommended_keys = set(data_dictionary['recommended'])

    if missing_keys := required_keys - columns:
        logger.error(f"\nThe following expected columns are missing: {sorted(missing_keys)}")
    if missing_keys := recommended_keys - columns:
        logger.warning(f"\nThe following recommended columns are missing: {sorted(missing_keys)}")
    if extra_columns := columns - set(data_dictionary.keys()):
        logger.info(
            f"\nThe following columns are missing from the data dictionary: {sorted(extra_columns)}"
        )

    invalid_vis = [vis for vis in df.visibility.values if not is_valid_python(vis)]
    if invalid_vis:
        logger.warning(f"\nThe following visibility values are not valid Python:\n{invalid_vis}")

def is_valid_python(code: str) -> bool:
    try:
        ast.parse(code)
    except (SyntaxError, ValueError):
        return False
    return True

def main() -> None:
    # validates files
    tsv_files = get_input_dir().glob("*/*.tsv")
    for file in tsv_files:
        validate_downloaded_file(file)

if __name__ == "__main__":
    main()

This refactored version reduces complexity by keeping the is_valid_python function simple and focused, using list comprehensions to simplify the validation logic, and ensuring that the main structure of the code remains clear and easy to follow.

visibility = df.visibility.values
for vis in visibility:
if not is_valid_python(vis):
invalid_vis.append(vis)
if invalid_vis:
logger.warning(f"\nThe following visibility are not valid python:\n{invalid_vis}")


def is_valid_python(code):
try:
ast.parse(code)
except SyntaxError:
return False
except ValueError as exc:
print(f"{exc}: {code}")
return True
return True


def main() -> None:
# validates files
Expand Down
22 changes: 0 additions & 22 deletions ecobidas/generate_landing_page.py

This file was deleted.

Loading
Loading