diff --git a/repo2docker/app.py b/repo2docker/app.py index 41e831e2..14a94924 100755 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -153,6 +153,7 @@ def _default_log_level(self): contentproviders.Hydroshare, contentproviders.Swhid, contentproviders.CKAN, + contentproviders.Meca, contentproviders.Mercurial, contentproviders.Git, ], diff --git a/repo2docker/contentproviders/__init__.py b/repo2docker/contentproviders/__init__.py index a0a2e019..7dbc6c67 100755 --- a/repo2docker/contentproviders/__init__.py +++ b/repo2docker/contentproviders/__init__.py @@ -4,6 +4,7 @@ from .figshare import Figshare from .git import Git from .hydroshare import Hydroshare +from .meca import Meca from .mercurial import Mercurial from .swhid import Swhid from .zenodo import Zenodo diff --git a/repo2docker/contentproviders/meca.py b/repo2docker/contentproviders/meca.py new file mode 100644 index 00000000..60ff05e3 --- /dev/null +++ b/repo2docker/contentproviders/meca.py @@ -0,0 +1,146 @@ +import hashlib +import os +import shutil +import tempfile +import xml.etree.ElementTree as ET +from hashlib import md5 +from os import path +from urllib.parse import urlparse, urlunparse +from zipfile import ZipFile, is_zipfile + +from requests import Session + +from .base import ContentProvider + + +def get_hashed_slug(url, changes_with_content): + """Returns a unique slug that is invariant to query parameters in the url""" + parsed_url = urlparse(url) + stripped_url = urlunparse( + (parsed_url.scheme, parsed_url.netloc, parsed_url.path, "", "", "") + ) + + return "meca-" + md5(f"{stripped_url}-{changes_with_content}".encode()).hexdigest() + + +def fetch_zipfile(session, url, dst_dir): + resp = session.get(url, headers={"accept": "application/zip"}, stream=True) + resp.raise_for_status() + + dst_filename = path.join(dst_dir, "meca.zip") + with open(dst_filename, "wb") as dst: + for chunk in resp.iter_content(chunk_size=128): + dst.write(chunk) + + return dst_filename + + +def extract_validate_and_identify_bundle(zip_filename, dst_dir): + if not os.path.exists(zip_filename): + raise RuntimeError("Downloaded MECA bundle not found") + + if not is_zipfile(zip_filename): + raise RuntimeError("MECA bundle is not a zip file") + + with ZipFile(zip_filename, "r") as zip_ref: + zip_ref.extractall(dst_dir) + + try: + manifest = path.join(dst_dir, "manifest.xml") + if not os.path.exists(manifest): + raise RuntimeError("MECA bundle is missing manifest.xml") + article_source_dir = "bundle/" + + tree = ET.parse(manifest) + root = tree.getroot() + + bundle_instance = root.findall( + "{*}item[@item-type='article-source-directory']/{*}instance" + ) + for attr in bundle_instance[0].attrib: + if attr.endswith("href"): + article_source_dir = bundle_instance[0].get(attr) + + return True, path.join(dst_dir, article_source_dir) + except: + return False, dst_dir + + +class Meca(ContentProvider): + """A repo2docker content provider for MECA bundles""" + + def __init__(self): + super().__init__() + self.session = Session() + self.session.headers.update( + { + "user-agent": f"repo2docker MECA", + } + ) + + def detect(self, spec, ref=None, extra_args=None): + """`spec` contains a faux protocol of http[s]+meca for detection purposes + and we assume `spec` trusted as a reachable MECA bundle from an allowed origin + (binderhub RepoProvider class is already checking for this). + + An other HEAD check in made here in order to get the content-length header + """ + is_local_file = False + if spec.endswith(".meca.zip") and os.path.isfile(spec): + url = os.path.abspath(spec) + is_local_file = True + with open(url, "rb") as f: + file_hash = hashlib.blake2b() + while chunk := f.read(8192): + file_hash.update(chunk) + changes_with_content = file_hash.hexdigest() + else: + parsed = urlparse(spec) + if not parsed.scheme.endswith("+meca"): + return None + parsed = parsed._replace(scheme=parsed.scheme[:-5]) + url = urlunparse(parsed) + + headers = self.session.head(url).headers + changes_with_content = headers.get("ETag") or headers.get("Content-Length") + + self.hashed_slug = get_hashed_slug(url, changes_with_content) + + return {"url": url, "slug": self.hashed_slug, "is_local_file": is_local_file} + + def fetch(self, spec, output_dir, yield_output=False): + hashed_slug = spec["slug"] + url = spec["url"] + is_local_file = spec["is_local_file"] + + yield f"Creating temporary directory.\n" + with tempfile.TemporaryDirectory() as tmpdir: + yield f"Temporary directory created at {tmpdir}.\n" + + if is_local_file: + yield f"Found MECA Bundle {url}.\n" + zip_filename = url + else: + yield f"Fetching MECA Bundle {url}.\n" + zip_filename = fetch_zipfile(self.session, url, tmpdir) + + yield f"Extracting MECA Bundle {zip_filename}.\n" + is_meca, bundle_dir = extract_validate_and_identify_bundle( + zip_filename, tmpdir + ) + + if not is_meca: + yield f"This doesn't look like a meca bundle, extracting everything.\n" + + yield f"Copying MECA Bundle at {bundle_dir} to {output_dir}.\n" + files = os.listdir(bundle_dir) + for f in files: + shutil.move(os.path.join(bundle_dir, f), output_dir) + + yield f"Removing temporary directory.\n" + + yield f"MECA Bundle {hashed_slug} fetched and unpacked.\n" + + @property + def content_id(self): + return self.hashed_slug