Skip to content

Commit

Permalink
πŸ”– Prepare project for release and publishing (#19)
Browse files Browse the repository at this point in the history
* Fix license classifier

* Rebrand from `codebase` to `epitaxy`

* Simplify `CaseInsensitivePathCompleter` using `expanduser`

* Simplify `parse_extensions()` function

* Add type hints to utilities and filter functions
  • Loading branch information
OLILHR authored Aug 21, 2024
1 parent 50627a6 commit 939f7b1
Show file tree
Hide file tree
Showing 18 changed files with 214 additions and 232 deletions.
8 changes: 4 additions & 4 deletions .codebaseignore.example β†’ .epitaxyignore.example
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ README.md
# "node_modules/",
# # Python
# "*.pyc",
# # codebase-specific files
# ".codebaseignore",
# ".codebaseignore.example",
# "codebase.md",
# # epitaxy-specific files
# ".epitaxyignore",
# ".epitaxyignore.example",
# "epitaxy.md",
# ]
6 changes: 3 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,6 @@ htmlcov/
.DS_Store
Thumbs.db

# coodebase-specific files
.codebaseignore
codebase.md
# epitaxy-specific files
.epitaxyignore
epitaxy.md
30 changes: 15 additions & 15 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,40 +1,40 @@
<div align="center">

<img width="75%" src="codebase.svg" alt="codebase.svg"><br>
<img width="85%" src="https://raw.githubusercontent.com/OLILHR/epitaxy/main/epitaxy.svg" alt="epitaxy.svg"><br>

<p>🧊 data consolidation.</p>
<p>🧊 codebase consolidation.</p>

![PyPI status badge](https://img.shields.io/pypi/v/alloy?labelColor=30363D&color=fccccc)
![Unittests status badge](https://github.com/OLILHR/codebase/workflows/Unittests/badge.svg)
![Coverage status badge](https://github.com/OLILHR/codebase/workflows/Coverage/badge.svg)
![Pylint status badge](https://github.com/OLILHR/codebase/workflows/Linting/badge.svg)
![Formatting status badge](https://github.com/OLILHR/codebase/workflows/Formatting/badge.svg)
![PyPI status badge](https://img.shields.io/pypi/v/epitaxy?labelColor=30363D&color=fccccc)
![Unittests status badge](https://github.com/OLILHR/epitaxy/workflows/Unittests/badge.svg)
![Coverage status badge](https://github.com/OLILHR/epitaxy/workflows/Coverage/badge.svg)
![Pylint status badge](https://github.com/OLILHR/epitaxy/workflows/Linting/badge.svg)
![Formatting status badge](https://github.com/OLILHR/epitaxy/workflows/Formatting/badge.svg)

</div>

## ℹ️ Installation

```sh
$ pip install codebase
$ pip install epitaxy
```

> [!NOTE]
> It is generally recommended to add a `.codebaseignore` file to the root directory of the codebase you'd like to consolidate.
> All files, folders and file extensions specified in `.codebaseignore` will be excluded from the output file.
> Please refer to the `.codebaseignore.example` for suggestions regarding what to include in `.codebaseignore`.
> It is generally recommended to add an `.epitaxyignore` file to the root directory of the codebase you'd like to consolidate.
> All files, folders and file extensions specified in `.epitaxyignore` will be excluded from the output file.
> Please refer to the `.epitaxyignore.example` for suggestions regarding what to include in `.epitaxyignore`.
To execute the script, simply run

```sh
$ codebase
$ epitaxy
```

and follow the prompts by providing an input directory, an output file destination and optional filters.

Alternatively, the script can also be executed using a single command with the appropriate flags:
Alternatively, the script can be executed using a single command with the appropriate flags:

```sh
$ codebase -i <input_path> -o <output_path> -f <(optional) filters>
$ epitaxy -i <input_path> -o <output_path> -f <(optional) filters>
```

For further information, run `$ codebase --help`.
For further information, run `$ epitaxy --help`.
1 change: 0 additions & 1 deletion codebase.svg

This file was deleted.

1 change: 1 addition & 0 deletions epitaxy.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
File renamed without changes.
File renamed without changes.
41 changes: 20 additions & 21 deletions codebase/filter.py β†’ epitaxy/filter.py
Original file line number Diff line number Diff line change
@@ -1,31 +1,32 @@
import os
from typing import Any, Callable, List, Optional


def skip_ignore_list_comments(file_path):
ignore_list = []
def skip_ignore_list_comments(file_path: str) -> List[str]:
ignore_list: List[str] = []
with open(file_path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if line and not line.startswith("#"): # ignore comments in .codebaseignore and DEFAULT_IGNORE_LIST
if line and not line.startswith("#"): # ignore comments in .epitaxyignore and DEFAULT_IGNORE_LIST
ignore_list.append(line)
return ignore_list


def read_codebaseignore(project_root, extension_filter):
def read_epitaxyignore(project_root: str, extension_filter: Optional[List[str]]) -> Callable[[str], bool]:
"""
Excludes all files, extensions and directories specified in .codebaseignore, located inside the root directory.
Excludes all files, extensions and directories specified in .epitaxyignore, located inside the root directory.
"""
codebaseignore = os.path.join(project_root, ".codebaseignore")
epitaxyignore = os.path.join(project_root, ".epitaxyignore")
default_ignore_list = DEFAULT_IGNORE_LIST.copy()

ignore_list = []
if os.path.exists(codebaseignore):
with open(codebaseignore, "r", encoding="utf-8") as f:
ignore_list: List[str] = []
if os.path.exists(epitaxyignore):
with open(epitaxyignore, "r", encoding="utf-8") as f:
ignore_list = [line.strip() for line in f if line.strip() and not line.startswith("#")]

default_ignore_list.extend(ignore_list)

def exclude_files(file_path):
def exclude_files(file_path: str) -> bool:
file_path = file_path.replace(os.sep, "/")

if extension_filter:
Expand Down Expand Up @@ -58,29 +59,27 @@ def exclude_files(file_path):
return exclude_files


def filter_extensions(file_path, extensions):
def filter_extensions(file_path: str, extensions: Optional[List[str]]) -> bool:
"""
Optional filter to include only certain provided extensions in the consolidated markdown file. If no extensions are
provided, all files are considered except files, extensions and directories that are explicitly excluded in the
specified .codebaseignore file, located inside the root directory.
specified .epitaxyignore file, located inside the root directory.
"""
if not extensions:
return True
_, file_extension = os.path.splitext(file_path)
return file_extension[1:] in extensions


def parse_extensions(_csx, _param, value):
def parse_extensions(_csx: Any, _param: Any, value: Optional[List[str]]) -> Optional[List[str]]:
"""
Converts a comma-separated string of file extensions into a list of individual extensions, which - in turn - is
parsed to the main function to filter files during the consolidation process.
"""
if not value:
return None
return [ext.strip() for item in value for ext in item.split(",")]
return [ext.strip() for item in value for ext in item.split(",")] if value else None


DEFAULT_IGNORE_LIST = [
DEFAULT_IGNORE_LIST: List[str] = [
".cache/",
".coverage",
"dist/",
Expand All @@ -94,8 +93,8 @@ def parse_extensions(_csx, _param, value):
"node_modules/",
# Python
"*.pyc",
# codebase-specific files
".codebaseignore",
".codebaseignore.example",
"codebase.md",
# epitaxy-specific files
".epitaxyignore",
".epitaxyignore.example",
"epitaxy.md",
]
105 changes: 46 additions & 59 deletions codebase/main.py β†’ epitaxy/main.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import logging
import os
from dataclasses import dataclass
from typing import Any, Iterable, List, Optional

import click
from prompt_toolkit import prompt
from prompt_toolkit.completion import Completer, Completion
from prompt_toolkit.document import Document

from .filter import parse_extensions
from .utilities import NoMatchingExtensionError, consolidate
Expand All @@ -15,17 +17,13 @@
_logger = logging.getLogger(__name__)
_logger.setLevel(GLOBAL_LOG_LEVEL)

MAX_FILE_SIZE = 1024 * 1024 * 10 # 10 MB
MAX_FILE_SIZE: int = 1024 * 1024 * 10 # 10 MB


def get_project_root():
"""
Required for input/output path prompts to display the project root as default path.
"""

current_dir = os.path.abspath(os.getcwd())
def get_project_root() -> str:
current_dir: str = os.path.abspath(os.getcwd())

root_indicators = [
root_indicators: List[str] = [
".git",
"package.json",
"pdm.lock",
Expand All @@ -45,50 +43,44 @@ def get_project_root():
@dataclass
class CaseInsensitivePathCompleter(Completer):
only_directories: bool = False
expanduser: bool = True

def get_completions(self, document, complete_event):
text = document.text_before_cursor
if len(text) == 0:
def get_completions(self, document: Document, complete_event: Any) -> Iterable[Completion]:
text: str = os.path.expanduser(document.text_before_cursor)
if not text:
return

directory = os.path.dirname(text)
prefix = os.path.basename(text)
directory: str = os.path.dirname(text)
prefix: str = os.path.basename(text)

if os.path.isabs(text):
full_directory = os.path.abspath(directory)
else:
full_directory = os.path.abspath(os.path.join(os.getcwd(), directory))
full_directory: str = os.path.abspath(directory)

try:
suggestions = os.listdir(full_directory)
suggestions: List[str] = os.listdir(full_directory)
except OSError:
return

for suggestion in suggestions:
if suggestion.lower().startswith(prefix.lower()):
if self.only_directories and not os.path.isdir(os.path.join(full_directory, suggestion)):
continue
completion = suggestion[len(prefix) :]
display = suggestion
yield Completion(completion, start_position=0, display=display)
yield Completion(suggestion[len(prefix) :], start_position=0, display=suggestion)


def path_prompt(message, default, exists=False):
def path_prompt(message: str, default: str, exists: bool = False) -> str:
"""
Enables basic shell features, like relative path suggestion and autocompletion, for CLI prompts.
Required by prompt_toolkit to enable basic shell features for CLI prompts, like path suggestion and autocompletion.
"""
path_completer = CaseInsensitivePathCompleter()

if not default.endswith(os.path.sep):
default += os.path.sep

while True:
path = prompt(f"{message} ", default=default, completer=path_completer)
full_path = os.path.abspath(os.path.expanduser(path))
path: str = prompt(f"{message} ", default=default, completer=path_completer)
full_path: str = os.path.abspath(os.path.expanduser(path))
if not exists or os.path.exists(full_path):
return full_path
print(f"πŸ”΄ {full_path} DOES NOT EXIST.")
logging.error("πŸ”΄ %s DOES NOT EXIST.", full_path)


@click.command()
Expand All @@ -100,26 +92,23 @@ def path_prompt(message, default, exists=False):
"extension_filter",
callback=parse_extensions,
multiple=True,
help="enables optional filtering by extensions, for instance: -f py,json", # markdown contains only .py/.json files
help="enables optional filtering by extensions, for instance: -f py,json",
)
# pylint: disable=too-many-locals
def generate_markdown(input_path, output_path, extension_filter):
no_flags_provided = input_path is None and output_path is None and not extension_filter
project_root = get_project_root()

if input_path is None:
input_path = path_prompt("πŸ“ INPUT PATH OF YOUR TARGET DIRECTORY -", default=project_root, exists=True)
else:
input_path = os.path.abspath(input_path)

if output_path is None:
output_path = path_prompt("πŸ“ OUTPUT PATH FOR THE MARKDOWN FILE -", default=project_root)
else:
output_path = os.path.abspath(output_path)
def generate_markdown(
input_path: Optional[str], output_path: Optional[str], extension_filter: Optional[List[str]]
) -> None:
no_flags_provided: bool = input_path is None and output_path is None and not extension_filter
project_root: str = get_project_root()

input_path = input_path or path_prompt(
"πŸ“ INPUT PATH OF YOUR TARGET DIRECTORY -", default=project_root, exists=True
)
output_path = output_path or path_prompt("πŸ“ OUTPUT PATH FOR THE MARKDOWN FILE -", default=project_root)

extensions = extension_filter
extensions: Optional[List[str]] = extension_filter
if no_flags_provided:
extensions_input = click.prompt(
extensions_input: str = click.prompt(
"πŸ”Ž OPTIONAL FILTER FOR SPECIFIC EXTENSIONS (COMMA-SEPARATED)",
default="",
show_default=False,
Expand All @@ -134,31 +123,29 @@ def generate_markdown(input_path, output_path, extension_filter):
input_path, extensions
)
except NoMatchingExtensionError:
_logger.error("\n⚠️ NO FILES MATCH THE SPECIFIED EXTENSION(S) - PLEASE REVIEW YOUR .codebaseignore FILE.")
_logger.error("\n⚠️ NO FILES MATCH THE SPECIFIED EXTENSION(S) - PLEASE REVIEW YOUR .epitaxyignore FILE.")
_logger.error("πŸ”΄ NO MARKDOWN FILE GENERATED.\n")
return

if len(markdown_content.encode("utf-8")) > MAX_FILE_SIZE:
_logger.error(
"\n" + "πŸ”΄ GENERATED CONTENT EXCEEDS 10 MB. CONSIDER ADDING LARGER FILES TO YOUR .codebaseignore."
)
_logger.error("\n" + "πŸ”΄ GENERATED CONTENT EXCEEDS 10 MB. CONSIDER ADDING LARGER FILES TO YOUR .epitaxyignore.")
return

codebase = os.path.join(output_path, "codebase.md")
epitaxy: str = os.path.join(output_path, "epitaxy.md")

os.makedirs(output_path, exist_ok=True)
with open(codebase, "w", encoding="utf-8") as f:
with open(epitaxy, "w", encoding="utf-8") as f:
f.write(markdown_content)

codebase_size = os.path.getsize(codebase)
if codebase_size < 1024:
file_size = f"{codebase_size} bytes"
elif codebase_size < 1024 * 1024:
file_size = f"{codebase_size / 1024:.2f} KB"
epitaxy_size: int = os.path.getsize(epitaxy)
if epitaxy_size < 1024:
file_size: str = f"{epitaxy_size} bytes"
elif epitaxy_size < 1024 * 1024:
file_size = f"{epitaxy_size / 1024:.2f} KB"
else:
file_size = f"{codebase_size / (1024 * 1024):.2f} MB"
file_size = f"{epitaxy_size / (1024 * 1024):.2f} MB"

file_type_distribution = " ".join(
file_type_distribution: str = " ".join(
f".{file_type} ({percentage:.0f}%)" for file_type, percentage in type_distribution
)

Expand All @@ -178,7 +165,7 @@ def generate_markdown(input_path, output_path, extension_filter):
+ "\n"
+ "πŸͺ™ TOKEN COUNT: %d"
+ "\n",
codebase,
epitaxy,
file_size,
file_count,
file_type_distribution,
Expand All @@ -187,7 +174,7 @@ def generate_markdown(input_path, output_path, extension_filter):
)


# to run the script during local development, either execute $ python -m codebase
# or install codebase locally via `pdm install` and simply run $ codebase
# to run the script during local development, either execute $ python -m epitaxy
# or install epitaxy locally via `pdm install` and simply run $ epitaxy
if __name__ == "__main__":
generate_markdown.main(standalone_mode=False)
2 changes: 1 addition & 1 deletion codebase/py.typed β†’ epitaxy/py.typed
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
# py.typed marks codebase as PEP561 compatible.
# py.typed marks epitaxy as PEP561 compatible.
# https://mypy.readthedocs.io/en/stable/installed_packages.html#creating-pep-561-compatible-packages
Loading

0 comments on commit 939f7b1

Please sign in to comment.