forked from ucbepic/docetl
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'ucbepic:main' into main
- Loading branch information
Showing
64 changed files
with
5,803 additions
and
2,441 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -15,6 +15,7 @@ workloads/* | |
*pytest_cache* | ||
*ruff_cache* | ||
motion-old* | ||
venv/ | ||
|
||
# dependencies | ||
website/node_modules | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,19 +1,50 @@ | ||
import importlib.metadata | ||
from docetl.operations.cluster import ClusterOperation | ||
from docetl.operations.code_operations import CodeFilterOperation, CodeMapOperation, CodeReduceOperation | ||
from docetl.operations.equijoin import EquijoinOperation | ||
from docetl.operations.filter import FilterOperation | ||
from docetl.operations.gather import GatherOperation | ||
from docetl.operations.map import MapOperation | ||
from docetl.operations.reduce import ReduceOperation | ||
from docetl.operations.resolve import ResolveOperation | ||
from docetl.operations.split import SplitOperation | ||
from docetl.operations.sample import SampleOperation | ||
from docetl.operations.unnest import UnnestOperation | ||
|
||
|
||
mapping = { | ||
"cluster": ClusterOperation, | ||
"code_filter": CodeFilterOperation, | ||
"code_map": CodeMapOperation, | ||
"code_reduce": CodeReduceOperation, | ||
"equijoin": EquijoinOperation, | ||
"filter": FilterOperation, | ||
"gather": GatherOperation, | ||
"map": MapOperation, | ||
"reduce": ReduceOperation, | ||
"resolve": ResolveOperation, | ||
"split": SplitOperation, | ||
"sample": SampleOperation, | ||
"unnest": UnnestOperation, | ||
} | ||
|
||
def get_operation(operation_type: str): | ||
"""Loads a single operation by name""" | ||
try: | ||
entrypoint = importlib.metadata.entry_points(group="docetl.operation")[ | ||
operation_type | ||
] | ||
except KeyError as e: | ||
return entrypoint.load() | ||
except KeyError: | ||
if operation_type in mapping: | ||
return mapping[operation_type] | ||
raise KeyError(f"Unrecognized operation {operation_type}") | ||
return entrypoint.load() | ||
|
||
def get_operations(): | ||
"""Load all available operations and return them as a dictionary""" | ||
return { | ||
operations = mapping.copy() | ||
operations.update({ | ||
op.name: op.load() | ||
for op in importlib.metadata.entry_points(group="docetl.operation") | ||
} | ||
}) | ||
return operations |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.