diff --git a/Dockerfile b/Dockerfile index 7c0a3256..07deef82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ FROM python:3.10-slim-bookworm AS build -RUN mkdir -p /binaries/ +RUN mkdir -p /calcus/binaries/ COPY scripts /calcus/scripts -COPY bin /binaries/xtb +COPY bin /calcus/binaries/xtb RUN python /calcus/scripts/extract_xtb.py RUN apt update && apt install build-essential gcc libxm4 libgl1 libmagic1 -y @@ -16,7 +16,7 @@ RUN pip install -r /calcus/requirements.txt FROM python:3.10-slim-bookworm AS calcus_user -COPY --from=0 /binaries/ /binaries/ +COPY --from=0 /calcus/binaries/ /calcus/binaries/ ARG CALCUS_VERSION_HASH ENV CALCUS_VERSION_HASH=${CALCUS_VERSION_HASH} @@ -26,14 +26,14 @@ ENV CALCUS_KEY_HOME "/calcus/keys" ENV CALCUS_TEST_SCR_HOME "/calcus/scratch/scr" ENV CALCUS_TEST_KEY_HOME "/calcus/scratch/keys" -ENV EBROOTORCA "/binaries/orca" -ENV GAUSS_EXEDIR "/binaries/g16" -ENV XTB4STDAHOME "/binaries/xtb" -ENV XTBPATH "/binaries/xtb/xtb:$XTB4STDAHOME" -ENV STDAHOME "/binaries/xtb" +ENV EBROOTORCA "/calcus/binaries/orca" +ENV GAUSS_EXEDIR "/calcus/binaries/g16" +ENV XTB4STDAHOME "/calcus/binaries/xtb" +ENV XTBPATH "/calcus/binaries/xtb/xtb:$XTB4STDAHOME" +ENV STDAHOME "/calcus/binaries/xtb" ENV PATH=$PATH:$XTB4STDAHOME/xtb/bin:$XTB4STDAHOME:$EBROOTORCA:$GAUSS_EXEDIR -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/binaries/orca:/usr/lib/openmpi/ +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/calcus/binaries/orca:/usr/lib/openmpi/ ENV PYTHONUNBUFFERED 1 diff --git a/cloud-compose.yml b/cloud-compose.yml index aea2170a..1a370752 100644 --- a/cloud-compose.yml +++ b/cloud-compose.yml @@ -1,5 +1,3 @@ -version: "3.4" - services: redis: image: redis:latest diff --git a/dev-compose.yml b/dev-compose.yml index 97161909..ed93b4f9 100644 --- a/dev-compose.yml +++ b/dev-compose.yml @@ -1,5 +1,3 @@ -version: "3.4" - services: redis: image: redis:latest diff --git a/docker-compose.yml b/docker-compose.yml index 49a417e6..3bcd4b0c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3.4" - services: redis: image: redis:latest diff --git a/docker/slurm/Dockerfile b/docker/slurm/Dockerfile index a109e2af..4a9c899e 100644 --- a/docker/slurm/Dockerfile +++ b/docker/slurm/Dockerfile @@ -11,11 +11,11 @@ RUN yes "clustertest" | passwd slurm ENV EBROOTORCA "/home/slurm/orca" ENV GAUSS_EXEDIR "/home/slurm/g16" -ENV XTB4STDAHOME "/binaries/xtb/" +ENV XTB4STDAHOME "/calcus/binaries/xtb/" ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"/home/slurm/orca" ENV PATH=$PATH:"/home/slurm/xtb:/home/slurm/g16:/home/slurm/orca:/home/slurm/other:/home/slurm/openmpi" -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/binaries/orca:/usr/lib/openmpi/ +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/calcus/binaries/orca:/usr/lib/openmpi/ ADD slurm.conf /etc/slurm-llnl/ ADD cgroup.conf /etc/slurm-llnl/ diff --git a/frontend/calculation_unittest.py b/frontend/calculation_unittest.py index 1bc21aea..b05f2da3 100644 --- a/frontend/calculation_unittest.py +++ b/frontend/calculation_unittest.py @@ -7,8 +7,8 @@ from django.core.management import call_command from frontend import tasks -from .libxyz import * -from .models import * +from .libxyz import get_angle, parse_xyz_from_text +from .models import np from .gen_calc import gen_calc from .tasks import run_calc from .calcusliveserver import SCR_DIR diff --git a/frontend/calcusliveserver.py b/frontend/calcusliveserver.py index 53326ab4..ce433c6a 100644 --- a/frontend/calcusliveserver.py +++ b/frontend/calcusliveserver.py @@ -39,10 +39,18 @@ from django.contrib.staticfiles.testing import StaticLiveServerTestCase from django.core.management import call_command -from django.contrib.auth.models import User -from .models import * -from .environment_variables import * +from .models import ( + Calculation, + CalculationOrder, + Ensemble, + Molecule, + Project, + ResearchGroup, + settings, + User, +) +from .environment_variables import CALCUS_KEY_HOME, CALCUS_SCR_HOME tests_dir = os.path.join("/".join(__file__.split("/")[:-1]), "tests/") dir_path = os.path.dirname(os.path.realpath(__file__)) @@ -1773,7 +1781,7 @@ def get_name_presets(self): def select_preset(self, name): self.wait_for_ajax() - element = WebDriverWait(self.driver, 2).until( + WebDriverWait(self.driver, 2).until( EC.presence_of_element_located( (By.XPATH, f"//*[@id='presets']/option[text()='{name}']") ) @@ -2173,7 +2181,7 @@ def subscribe(self, plan, email, card_number, length="month"): if length == "year": switch = WebDriverWait(self.driver, 5).until( EC.presence_of_element_located( - (By.CSS_SELECTOR, f"[for=sub_length_switch]") + (By.CSS_SELECTOR, "[for=sub_length_switch]") ) ) switch.click() diff --git a/frontend/cloud_job.py b/frontend/cloud_job.py index 9caa1c1f..2c3392c7 100644 --- a/frontend/cloud_job.py +++ b/frontend/cloud_job.py @@ -4,7 +4,7 @@ from django.conf import settings -from .environment_variables import * +from .environment_variables import IS_TEST from .helpers import job_triage if settings.IS_CLOUD: diff --git a/frontend/cluster_daemon.py b/frontend/cluster_daemon.py index be4090ea..67ecf694 100644 --- a/frontend/cluster_daemon.py +++ b/frontend/cluster_daemon.py @@ -79,8 +79,8 @@ class ConnectionCodes: os.environ.setdefault("DJANGO_SETTINGS_MODULE", "calcus.settings") django.setup() -from frontend.models import * -from frontend.environment_variables import * +from frontend.models import Calculation, ClusterAccess, ErrorCodes +from frontend.environment_variables import CALCUS_KEY_HOME, CALCUS_SCR_HOME from frontend import tasks diff --git a/frontend/context.py b/frontend/context.py index d070f7b7..a4a6cafb 100644 --- a/frontend/context.py +++ b/frontend/context.py @@ -29,6 +29,6 @@ def default(request): try: d["SUBSCRIPTION_DATA"] = constants.SUBSCRIPTION_DATA[version] except (NameError, AttributeError): - logger.error(f"Could not get subscription data") + logger.error("Could not get subscription data") return d diff --git a/frontend/decorators.py b/frontend/decorators.py index bc8ec52d..78c07b27 100644 --- a/frontend/decorators.py +++ b/frontend/decorators.py @@ -17,7 +17,7 @@ along with this program. If not, see . """ -from django.http import HttpResponse, HttpResponseRedirect +from django.http import HttpResponseRedirect def superuser_required(func): diff --git a/frontend/environment_variables.py b/frontend/environment_variables.py index f03eb3d1..e3fc2dba 100644 --- a/frontend/environment_variables.py +++ b/frontend/environment_variables.py @@ -57,4 +57,4 @@ MEM = int(PAL) * STACKSIZE EBROOTORCA = os.environ.get("EBROOTORCA", "") -MULTIWFN_DIR = os.environ.get("MULTIWFN_DIR", "/binaries/xtb/") +MULTIWFN_DIR = os.environ.get("MULTIWFN_DIR", "/calcus/binaries/xtb/") diff --git a/frontend/forms.py b/frontend/forms.py index 4ce5f33f..6d9621fb 100644 --- a/frontend/forms.py +++ b/frontend/forms.py @@ -71,13 +71,13 @@ def save(self, commit=True): if commit: user.save() - r = ResourceAllocation.objects.create( + ResourceAllocation.objects.create( code=get_random_string(), redeemer=user, allocation_seconds=settings.FREE_DEFAULT_COMP_SECONDS, note=ResourceAllocation.NEW_ACCOUNT, ) - proj = Project.objects.create(name="My Main Project", author=user) + Project.objects.create(name="My Main Project", author=user) return user @@ -119,17 +119,17 @@ def validate_access_code(self): classes = ClassGroup.objects.filter(access_code=code) if classes.count() == 0: - raise ValidationError(f"No group found with such access code") + raise ValidationError("No group found with such access code") elif classes.count() > 1: logger.error(f"Multiple groups found with access code {code}!") - raise ValidationError(f"Internal error") + raise ValidationError("Internal error") elif classes.count() == 1: self.student_class = classes.first() else: logger.error( f"Unexpected number of groups found with access code {code}: {classes.count()}!" ) - raise ValidationError(f"Internal error") + raise ValidationError("Internal error") def save(self, commit=True): user = super().save(commit=False) @@ -188,13 +188,13 @@ def save(self, commit=True): if commit: user.save() - r = ResourceAllocation.objects.create( + ResourceAllocation.objects.create( code=get_random_string(), redeemer=user, allocation_seconds=settings.TRIAL_DEFAULT_COMP_SECONDS, note=ResourceAllocation.TRIAL, ) - proj = Project.objects.create(name="My Main Project", author=user) + Project.objects.create(name="My Main Project", author=user) return user @@ -239,7 +239,7 @@ def save(self, commit=True): if commit: user.save() - r = ResourceAllocation.objects.create( + ResourceAllocation.objects.create( code=get_random_string(), redeemer=user, allocation_seconds=( diff --git a/frontend/gen_calc.py b/frontend/gen_calc.py index 2c107128..09362478 100644 --- a/frontend/gen_calc.py +++ b/frontend/gen_calc.py @@ -17,7 +17,17 @@ along with this program. If not, see . """ -from .models import * +from .models import ( + BasicStep, + Calculation, + CalculationOrder, + Ensemble, + Molecule, + Parameters, + Project, + Structure, + os, +) from .tasks import generate_xyz_structure TESTS_DIR = os.path.join("/".join(__file__.split("/")[:-1]), "tests/") diff --git a/frontend/gen_data_documentation.py b/frontend/gen_data_documentation.py index c6abdf66..4a0065be 100644 --- a/frontend/gen_data_documentation.py +++ b/frontend/gen_data_documentation.py @@ -17,7 +17,7 @@ along with this program. If not, see . """ -from constants import * +from .constants import SOFTWARE_BASIS_SETS, SOFTWARE_METHODS TABLE_TEMPLATE = """ diff --git a/frontend/helpers.py b/frontend/helpers.py index 3918cfca..c94e1972 100644 --- a/frontend/helpers.py +++ b/frontend/helpers.py @@ -23,7 +23,7 @@ from django.conf import settings from xkcdpass import xkcd_password as xp -from .constants import * +from .constants import ATOMIC_NUMBER, ATOMIC_SYMBOL, BOHR_VAL full_alphabet = string.ascii_letters + string.digits diff --git a/frontend/libxyz.py b/frontend/libxyz.py index b48fb6ce..9afb1b3e 100644 --- a/frontend/libxyz.py +++ b/frontend/libxyz.py @@ -18,12 +18,16 @@ """ import json -import numpy as np import periodictable import copy -from .constants import * - from hashlib import md5 +import numpy as np +from numpy.linalg import expm, norm +from numpy import cross, eye + + +from .constants import ATOMIC_NUMBER + # Structure of xyz: # [, [x, y, z]] @@ -375,7 +379,7 @@ def create_derivative(base_xyz, sub_xyz): anchor = [i for i in range(len(sub_xyz)) if sub_xyz[i][0] == "He"] if len(anchor) != 1: - print("Invalid substituent: {}".format(sub)) + print("Invalid substituent") return anchor = anchor[0] @@ -383,7 +387,7 @@ def create_derivative(base_xyz, sub_xyz): conn = get_neighbors_lists(sub_xyz, He_radius=1.4) anchor_neigh = conn[anchor] if len(anchor_neigh) != 1: - print("Could not find proper substitution bond: {}".format(sub)) + print("Could not find proper substitution bond") return sub_neigh = anchor_neigh[0] diff --git a/frontend/management/commands/add_resources.py b/frontend/management/commands/add_resources.py index cd8a5d05..4d73c40d 100755 --- a/frontend/management/commands/add_resources.py +++ b/frontend/management/commands/add_resources.py @@ -1,9 +1,6 @@ -import os from django.core.management.base import BaseCommand -from frontend.models import * -from frontend.environment_variables import * -from frontend.tasks import run_calc +from frontend.models import ResourceAllocation, User from frontend.helpers import get_random_string diff --git a/frontend/management/commands/check_su.py b/frontend/management/commands/check_su.py index 9131d4d5..e07a35fe 100755 --- a/frontend/management/commands/check_su.py +++ b/frontend/management/commands/check_su.py @@ -18,7 +18,7 @@ """ from django.core.management.base import BaseCommand -from frontend.models import * +from frontend.models import User, os su_email = os.environ.get("CALCUS_SU_EMAIL", "") diff --git a/frontend/management/commands/init_static_obj.py b/frontend/management/commands/init_static_obj.py index 2e4ae6ed..6bfd74be 100755 --- a/frontend/management/commands/init_static_obj.py +++ b/frontend/management/commands/init_static_obj.py @@ -25,7 +25,15 @@ filepath = pathlib.Path(__file__).parent.resolve() showcase_dir = os.path.join(filepath.parent.parent, "showcases") -from frontend.models import * +from frontend.models import ( + BasicStep, + Example, + Parameters, + Recipe, + ShowcaseEnsemble, + ShowcaseProperty, + Structure, +) from frontend.libxyz import format_xyz, parse_multixyz_from_file try: diff --git a/frontend/management/commands/refresh_cached_info.py b/frontend/management/commands/refresh_cached_info.py index e60ff5da..71f65489 100755 --- a/frontend/management/commands/refresh_cached_info.py +++ b/frontend/management/commands/refresh_cached_info.py @@ -17,10 +17,8 @@ along with this program. If not, see . """ -import glob -import os from django.core.management.base import BaseCommand -from frontend.models import * +from frontend.models import Project, Profile class Command(BaseCommand): @@ -62,7 +60,7 @@ def handle(self, *args, **options): running += 1 else: completed += 1 - if o.ensemble != None: + if o.ensemble is not None: mol = o.ensemble.parent_molecule if mol is not None: dnum[mol.id] += o.calculation_set.count() diff --git a/frontend/management/commands/reschedule_calc.py b/frontend/management/commands/reschedule_calc.py index b0117dae..e17e501d 100755 --- a/frontend/management/commands/reschedule_calc.py +++ b/frontend/management/commands/reschedule_calc.py @@ -1,9 +1,6 @@ -import glob -import os from django.core.management.base import BaseCommand -from frontend.models import * -from frontend.environment_variables import * +from frontend.models import Calculation from frontend.cloud_job import submit_cloud_job diff --git a/frontend/management/commands/run_calc.py b/frontend/management/commands/run_calc.py index 81d324ef..1cf9902d 100755 --- a/frontend/management/commands/run_calc.py +++ b/frontend/management/commands/run_calc.py @@ -1,8 +1,7 @@ import os from django.core.management.base import BaseCommand -from frontend.models import * -from frontend.environment_variables import * +from frontend.models import Calculation from frontend.tasks import run_calc diff --git a/frontend/management/commands/wipe.py b/frontend/management/commands/wipe.py index e59df47d..fc897978 100755 --- a/frontend/management/commands/wipe.py +++ b/frontend/management/commands/wipe.py @@ -17,10 +17,9 @@ along with this program. If not, see . """ -import glob import os from django.core.management.base import BaseCommand -from frontend.models import * +from frontend.models import Calculation, Molecule, Project try: os.environ["CALCUS_TEST"] diff --git a/frontend/models.py b/frontend/models.py index 0e1d71cd..76f0ee5e 100644 --- a/frontend/models.py +++ b/frontend/models.py @@ -36,7 +36,14 @@ from hashid_field import HashidAutoField, BigHashidAutoField -from .constants import * +from .constants import ( + HARTREE_FVAL, + HARTREE_TO_KCAL_F, + NMR_REGRESSIONS, + R_CONSTANT_HARTREE, + TEMP, + decimal, +) from .helpers import get_random_readable_code, job_triage from .environment_variables import PAL diff --git a/frontend/tasks.py b/frontend/tasks.py index 4cfd71ef..cf51b477 100644 --- a/frontend/tasks.py +++ b/frontend/tasks.py @@ -81,11 +81,54 @@ def wrapper(*args, **kwargs): from ccinput.wrapper import generate_calculation from ccinput.utilities import get_solvent -from .libxyz import * -from .models import * -from .helpers import * -from .constants import HARTREE_TO_EV -from .environment_variables import * +from .libxyz import ( + format_xyz, + parse_multixyz_from_file, + parse_xyz_from_file, + parse_xyz_from_text, +) +from .models import ( + BasicStep, + Calculation, + CalculationFrame, + CalculationOrder, + Ensemble, + FlowchartOrder, + Molecule, + PAL, + Project, + Property, + ResourceAllocation, + Step, + Structure, + Subscription, + User, +) + +from .constants import ( + ATOMIC_NUMBER, + ATOMIC_SYMBOL, + HARTREE_FVAL, + HARTREE_TO_KCAL_F, + LOWERCASE_ATOMIC_SYMBOLS, + HARTREE_TO_EV, + ErrorCodes, + MAX_COMMAND_ATTEMPT_COUNT, +) +from .helpers import ( + clean_xyz, + get_number_of_electrons, + get_random_string, + get_xyz_from_Gaussian_input, +) +from .environment_variables import ( + IS_TEST, + CALCUS_SCR_HOME, + CALCUS_CACHE_HOME, + EBROOTORCA, + MEM, + MULTIWFN_DIR, +) from .cloud_job import submit_cloud_job import traceback @@ -301,7 +344,7 @@ def testing_delay_local(res): for i in range(wait): time.sleep(1) if res.is_aborted(): - logger.info(f"Stopping calculation after loading the cache") + logger.info("Stopping calculation after loading the cache") return ErrorCodes.JOB_CANCELLED return ErrorCodes.SUCCESS @@ -311,6 +354,8 @@ def testing_delay_remote(calc_id): """ Same as `testing_delay_local`, but for remote calculations. """ + pid = int(threading.get_ident()) + wait = int(os.environ.get("CACHE_POST_WAIT", "0")) for i in range(wait): if pid in kill_sig: @@ -332,7 +377,7 @@ def testing_delay_cloud(calc_id): time.sleep(1) calc = Calculation.objects.get(id=calc_id) if calc.status == 3: - logger.info(f"Stopping calculation after loading the cache") + logger.info("Stopping calculation after loading the cache") return ErrorCodes.JOB_CANCELLED return ErrorCodes.SUCCESS @@ -493,7 +538,7 @@ def system( t = subprocess.Popen(shlex.split(command), stdout=stream, stderr=stream) except FileNotFoundError as e: - targetstr = ",".join(glob.glob("/binaries/xtb/*")) + targetstr = ",".join(glob.glob("/calcus/binaries/xtb/*")) logger.error( f'Could not run command "{command}" - executable not found (msg: {str(e)}), os.environ is {str(os.environ)} and target dir contains {targetstr}' ) @@ -599,14 +644,14 @@ def calc_is_cached(calc): and os.getenv("CAN_USE_CACHED_LOGS") == "true" ): if calc.local and not os.path.isdir(CALCUS_CACHE_HOME): - logger.info(f"no cache") + logger.info("No cache") os.mkdir(CALCUS_CACHE_HOME) return False index = get_cache_index(calc, CALCUS_CACHE_HOME) if index == -1: - logger.info(f"not found") + logger.info("Cache not found") return False logger.info(f"Using cache index for calc {calc.id}") @@ -700,9 +745,8 @@ def generate_xyz_structure(drawing, inp, ext, scale=1): return ErrorCodes.UNIMPLEMENTED -def launch_pysis_calc(in_file, calc, files): +def launch_pysis_calc(calc, files): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - folder = f"scratch/calcus/{calc.id}" if not os.path.isdir(local_folder): os.makedirs(local_folder, exist_ok=True) @@ -715,23 +759,23 @@ def launch_pysis_calc(in_file, calc, files): out.write(calc.input_file) if not calc.local: - log_path = os.path.join(folder, "calc.out") pid = int(threading.get_ident()) conn = connections[pid] lock = locks[pid] remote_dir = remote_dirs[pid] + log_path = os.path.join(remote_dir, "calc.out") if calc.remote_id == 0: sftp_put( f"{local_folder}/calc.xyz", - os.path.join(folder, "calc.xyz"), + os.path.join(remote_dir, "calc.xyz"), conn, lock, ) if calc.input_file != "": sftp_put( f"{local_folder}/calc.inp", - os.path.join(folder, "calc.inp"), + os.path.join(remote_dir, "calc.inp"), conn, lock, ) @@ -750,7 +794,7 @@ def launch_pysis_calc(in_file, calc, files): if not calc.local: for f in files: a = sftp_get( - f"{folder}/{f}", + f"{remote_dir}/{f}", os.path.join(CALCUS_SCR_HOME, str(calc.id), f), conn, lock, @@ -772,10 +816,10 @@ def launch_pysis_calc(in_file, calc, files): return ErrorCodes.JOB_CANCELLED -def xtb_opt(in_file, calc): +def xtb_opt(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_xtb_calc(in_file, calc, ["calc.out", "xtbopt.xyz"]) + ret = launch_xtb_calc(calc, ["calc.out", "xtbopt.xyz"]) if ret != ErrorCodes.SUCCESS: return ret @@ -814,10 +858,10 @@ def xtb_opt(in_file, calc): return ErrorCodes.SUCCESS -def xtb_mo_gen(in_file, calc): +def xtb_mo_gen(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_xtb_calc(in_file, calc, ["calc.out", "molden.input", "xtbout.json"]) + ret = launch_xtb_calc(calc, ["calc.out", "molden.input", "xtbout.json"]) if ret != ErrorCodes.SUCCESS: return ret @@ -875,9 +919,8 @@ def xtb_mo_gen(in_file, calc): return ErrorCodes.SUCCESS -def launch_xtb_calc(in_file, calc, files): +def launch_xtb_calc(calc, files): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - folder = f"scratch/calcus/{calc.id}" if not os.path.isdir(local_folder): os.makedirs(local_folder, exist_ok=True) @@ -892,23 +935,24 @@ def launch_xtb_calc(in_file, calc, files): os.chdir(local_folder) if not calc.local: - log_path = os.path.join(folder, local_folder) pid = int(threading.get_ident()) conn = connections[pid] lock = locks[pid] remote_dir = remote_dirs[pid] + log_path = os.path.join(remote_dir, local_folder) + if calc.remote_id == 0: sftp_put( f"{local_folder}/in.xyz", - os.path.join(folder, "in.xyz"), + os.path.join(remote_dir, "in.xyz"), conn, lock, ) if calc.input_file != "": sftp_put( f"{local_folder}/input", - os.path.join(folder, "input"), + os.path.join(remote_dir, "input"), conn, lock, ) @@ -927,7 +971,7 @@ def launch_xtb_calc(in_file, calc, files): if not calc.local: for f in files: a = sftp_get( - f"{folder}/{f}", + f"{remote_dir}/{f}", os.path.join(CALCUS_SCR_HOME, str(calc.id), f), conn, lock, @@ -939,7 +983,7 @@ def launch_xtb_calc(in_file, calc, files): and os.getenv("CAN_USE_CACHED_LOGS") == "true" ): a = sftp_get( - f"{folder}/NOT_CONVERGED", + f"{remote_dir}/NOT_CONVERGED", os.path.join(CALCUS_SCR_HOME, str(calc.id), "NOT_CONVERGED"), conn, lock, @@ -969,24 +1013,22 @@ def launch_xtb_calc(in_file, calc, files): return ErrorCodes.JOB_CANCELLED -def mep(in_file, calc): +def mep(calc): if calc.parameters.driver == "orca": - return mep_orca(in_file, calc) + return mep_orca(calc) elif calc.parameters.driver == "pysisyphus": - return mep_pysis(in_file, calc) + return mep_pysis(calc) else: raise Exception(f"Unknown driver for MEP calculation: {calc.parameters.driver}") -def mep_orca(in_file, calc): - folder = "/".join(in_file.split("/")[:-1]) +def mep_orca(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - local = calc.local with open(os.path.join(local_folder, "calc2.xyz"), "w") as out: out.write(calc.aux_structure.xyz_structure) - ret = launch_orca_calc(in_file, calc, ["calc.out", "calc_MEP_trj.xyz"]) + ret = launch_orca_calc(calc, ["calc.out", "calc_MEP_trj.xyz"]) if ret != ErrorCodes.SUCCESS: return ret @@ -1027,10 +1069,8 @@ def mep_orca(in_file, calc): return ErrorCodes.SUCCESS -def mep_pysis(in_file, calc): - folder = "/".join(in_file.split("/")[:-1]) +def mep_pysis(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - local = calc.local with open(os.path.join(local_folder, "calc2.xyz"), "w") as out: out.write(calc.aux_structure.xyz_structure) @@ -1044,13 +1084,13 @@ def mep_pysis(in_file, calc): if calc.remote_id == 0: sftp_put( f"{local_folder}/calc2.xyz", - os.path.join(folder, "calc2.xyz"), + os.path.join(remote_dir, "calc2.xyz"), conn, lock, ) ret = launch_pysis_calc( - in_file, calc, ["calc.out", "current_geometries.trj", "splined_hei.xyz"] + calc, ["calc.out", "current_geometries.trj", "splined_hei.xyz"] ) if ret != ErrorCodes.SUCCESS: @@ -1126,10 +1166,10 @@ def mep_pysis(in_file, calc): return ErrorCodes.SUCCESS -def xtb_sp(in_file, calc): +def xtb_sp(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_xtb_calc(in_file, calc, ["calc.out"]) + ret = launch_xtb_calc(calc, ["calc.out"]) if ret != ErrorCodes.SUCCESS: return ret @@ -1161,20 +1201,19 @@ def get_or_create(params, struct): return Property.objects.create(parameters=params, parent_structure=struct) -def xtb_handle_ts(in_file, calc): +def xtb_handle_ts(calc): """Chooses the right driver for the calculation (ORCA or Pysisyphus)""" # if settings.IS_CLOUD: - return xtb_ts_pysis(in_file, calc) + return xtb_ts_pysis(calc) # else: - # return xtb_ts_orca(in_file, calc) + # return xtb_ts_orca(calc) -def xtb_ts_orca(in_file, calc): +def xtb_ts_orca(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - local = calc.local - ret = launch_orca_calc(in_file, calc, ["calc.out", "calc.xyz"]) + ret = launch_orca_calc(calc, ["calc.out", "calc.xyz"]) if ret != ErrorCodes.SUCCESS: return ret @@ -1212,11 +1251,10 @@ def xtb_ts_orca(in_file, calc): return ErrorCodes.SUCCESS -def xtb_ts_pysis(in_file, calc): +def xtb_ts_pysis(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - local = calc.local - ret = launch_pysis_calc(in_file, calc, ["calc.out", "ts_final_geometry.xyz"]) + ret = launch_pysis_calc(calc, ["calc.out", "ts_final_geometry.xyz"]) if ret != ErrorCodes.SUCCESS: return ret @@ -1254,15 +1292,15 @@ def xtb_ts_pysis(in_file, calc): return ErrorCodes.SUCCESS -def xtb_scan(in_file, calc): +def xtb_scan(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) has_scan = "scan" in calc.constraints.lower() if has_scan: - ret = launch_xtb_calc(in_file, calc, ["calc.out", "xtbscan.log"]) + ret = launch_xtb_calc(calc, ["calc.out", "xtbscan.log"]) else: - ret = launch_xtb_calc(in_file, calc, ["calc.out", "xtbopt.xyz"]) + ret = launch_xtb_calc(calc, ["calc.out", "xtbopt.xyz"]) failed = False if ret != ErrorCodes.SUCCESS: @@ -1294,8 +1332,6 @@ def xtb_scan(in_file, calc): ind += 1 inds.append(len(lines)) - min_E = 0 - """ Since we can't get the keys of items created in bulk and we can't set a reference without first creating the objects, I haven't found a way to create both the structures and properties using bulk_update. This is still >2.5 times faster than the naive approach. """ @@ -1362,10 +1398,10 @@ def xtb_scan(in_file, calc): return ret -def xtb_freq(in_file, calc): +def xtb_freq(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_xtb_calc(in_file, calc, ["calc.out", "vibspectrum", "g98.out"]) + ret = launch_xtb_calc(calc, ["calc.out", "vibspectrum", "g98.out"]) if ret != ErrorCodes.SUCCESS: return ret @@ -1486,10 +1522,10 @@ def xtb_freq(in_file, calc): return ErrorCodes.SUCCESS -def crest(in_file, calc): +def crest(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_xtb_calc(in_file, calc, ["calc.out", "crest_conformers.xyz"]) + ret = launch_xtb_calc(calc, ["calc.out", "crest_conformers.xyz"]) if ret != ErrorCodes.SUCCESS: return ret @@ -1509,7 +1545,6 @@ def crest(in_file, calc): ) return ErrorCodes.INVALID_OUTPUT - weighted_energy = 0.0 ind += 1 structures = [] properties = [] @@ -1562,9 +1597,8 @@ def crest(in_file, calc): return ErrorCodes.SUCCESS -def fast_conf(in_file, calc): +def fast_conf(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - folder = f"scratch/calcus/{calc.id}" # Remote calc_start = timezone.now() @@ -1702,9 +1736,8 @@ def clean_struct_line(line): return f"{LOWERCASE_ATOMIC_SYMBOLS[a.lower()]} {x} {y} {z}\n" -def launch_orca_calc(in_file, calc, files): +def launch_orca_calc(calc, files): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - folder = f"scratch/calcus/{calc.id}" if not os.path.isdir(local_folder): os.makedirs(local_folder, exist_ok=True) @@ -1717,17 +1750,18 @@ def launch_orca_calc(in_file, calc, files): conn = connections[pid] lock = locks[pid] remote_dir = remote_dirs[pid] + if calc.remote_id == 0: sftp_put( f"{local_folder}/calc.inp", - os.path.join(folder, "calc.inp"), + os.path.join(remote_dir, "calc.inp"), conn, lock, ) if calc.step.name == "Minimum Energy Path": sftp_put( f"{local_folder}/calc2.xyz", - os.path.join(folder, "calc2.xyz"), + os.path.join(remote_dir, "calc2.xyz"), conn, lock, ) @@ -1753,7 +1787,7 @@ def launch_orca_calc(in_file, calc, files): if not calc.local: for f in files: a = sftp_get( - f"{folder}/{f}", + f"{remote_dir}/{f}", os.path.join(CALCUS_SCR_HOME, str(calc.id), f), conn, lock, @@ -1763,7 +1797,7 @@ def launch_orca_calc(in_file, calc, files): if not cancelled and calc.parameters.software == "xtb": a = sftp_get( - f"{folder}/NOT_CONVERGED", + f"{remote_dir}/NOT_CONVERGED", os.path.join(CALCUS_SCR_HOME, str(calc.id), "NOT_CONVERGED"), conn, lock, @@ -1781,11 +1815,10 @@ def launch_orca_calc(in_file, calc, files): return ErrorCodes.JOB_CANCELLED -def orca_mo_gen(in_file, calc): +def orca_mo_gen(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) ret = launch_orca_calc( - in_file, calc, ["calc.out", "calc.molden.input"], ) @@ -1842,7 +1875,7 @@ def orca_mo_gen(in_file, calc): return ErrorCodes.SUCCESS -def orca_opt(in_file, calc): +def orca_opt(calc): lines = [ i + "\n" for i in clean_xyz(calc.structure.xyz_structure).split("\n")[2:] @@ -1861,11 +1894,11 @@ def orca_opt(in_file, calc): calc.step = BasicStep.objects.get(name="Single-Point Energy") calc.save() add_input_to_calc(calc) - return orca_sp(in_file, calc) + return orca_sp(calc) local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_orca_calc(in_file, calc, ["calc.out", "calc.xyz"]) + ret = launch_orca_calc(calc, ["calc.out", "calc.xyz"]) if ret != ErrorCodes.SUCCESS: return ret @@ -1906,10 +1939,10 @@ def orca_opt(in_file, calc): return ErrorCodes.SUCCESS -def orca_sp(in_file, calc): +def orca_sp(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_orca_calc(in_file, calc, ["calc.out"]) + ret = launch_orca_calc(calc, ["calc.out"]) if ret != ErrorCodes.SUCCESS: return ret @@ -1937,10 +1970,10 @@ def orca_sp(in_file, calc): return ErrorCodes.SUCCESS -def orca_ts(in_file, calc): +def orca_ts(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_orca_calc(in_file, calc, ["calc.out", "calc.xyz"]) + ret = launch_orca_calc(calc, ["calc.out", "calc.xyz"]) if ret != ErrorCodes.SUCCESS: return ret @@ -1979,10 +2012,10 @@ def orca_ts(in_file, calc): return ErrorCodes.SUCCESS -def orca_freq(in_file, calc): +def orca_freq(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_orca_calc(in_file, calc, ["calc.out"]) + ret = launch_orca_calc(calc, ["calc.out"]) if ret != ErrorCodes.SUCCESS: return ret @@ -2087,7 +2120,7 @@ def orca_freq(in_file, calc): assert ind < len(lines) - 1 ind += 7 - start_num = int(nums[0]) + # start_num = int(nums[0]) end_num = int(nums[-1]) vibs = [] @@ -2131,19 +2164,18 @@ def is_all_null(arr): return ErrorCodes.SUCCESS -def orca_scan(in_file, calc): +def orca_scan(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) has_scan = "scan" in calc.constraints.lower() if has_scan: ret = launch_orca_calc( - in_file, calc, ["calc.out", "calc.relaxscanact.dat", "calc.allxyz", "calc_trj.xyz"], ) else: - ret = launch_orca_calc(in_file, calc, ["calc.out", "calc.xyz"]) + ret = launch_orca_calc(calc, ["calc.out", "calc.xyz"]) if ret != ErrorCodes.SUCCESS: return ret @@ -2173,7 +2205,6 @@ def orca_scan(in_file, calc): inds.append(len(lines) + 1) - min_E = 0 for metaind, mol in enumerate(inds[:-1]): E = energies[metaind] struct = clean_xyz( @@ -2230,10 +2261,10 @@ def orca_scan(in_file, calc): return ErrorCodes.SUCCESS -def orca_nmr(in_file, calc): +def orca_nmr(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_orca_calc(in_file, calc, ["calc.out"]) + ret = launch_orca_calc(calc, ["calc.out"]) if ret != ErrorCodes.SUCCESS: return ret @@ -2307,14 +2338,12 @@ def plot_vibs(_x, PP): return val -def xtb_stda(in_file, calc): # TO OPTIMIZE +def xtb_stda(calc): # TO OPTIMIZE ww = [] TT = [] PP = [] - folder = "/".join(in_file.split("/")[:-1]) local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - local = calc.local if calc.parameters.solvent != "Vacuum": solvent_add = f"-g {get_solvent(calc.parameters.solvent, 'xtb')}" @@ -2324,7 +2353,7 @@ def xtb_stda(in_file, calc): # TO OPTIMIZE os.chdir(local_folder) ret1 = system( - f"xtb4stda {in_file} -chrg {calc.parameters.charge} {solvent_add}", + f"xtb4stda in.xyz -chrg {calc.parameters.charge} {solvent_add}", os.path.join(local_folder, "calc.out"), calc_id=calc.id, ) @@ -2333,7 +2362,7 @@ def xtb_stda(in_file, calc): # TO OPTIMIZE return ret1 ret2 = system( - "stda -xtb -e 12".format(in_file, calc.parameters.charge, solvent_add), + "stda -xtb -e 12", os.path.join(local_folder, "calc2.out"), calc_id=calc.id, ) @@ -2341,26 +2370,26 @@ def xtb_stda(in_file, calc): # TO OPTIMIZE if ret2 != ErrorCodes.SUCCESS: return ret2 - if not local: + if not calc.local: pid = int(threading.get_ident()) conn = connections[pid] lock = locks[pid] remote_dir = remote_dirs[pid] a = sftp_get( - f"{folder}/tda.dat", + f"{remote_dir}/tda.dat", os.path.join(CALCUS_SCR_HOME, str(calc.id), "tda.dat"), conn, lock, ) b = sftp_get( - f"{folder}/calc.out", + f"{remote_dir}/calc.out", os.path.join(CALCUS_SCR_HOME, str(calc.id), "calc.out"), conn, lock, ) c = sftp_get( - f"{folder}/calc2.out", + f"{remote_dir}/calc2.out", os.path.join(CALCUS_SCR_HOME, str(calc.id), "calc2.out"), conn, lock, @@ -2408,9 +2437,8 @@ def xtb_stda(in_file, calc): # TO OPTIMIZE return ErrorCodes.SUCCESS -def launch_nwchem_calc(in_file, calc, files): +def launch_nwchem_calc(calc, files): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - folder = f"scratch/calcus/{calc.id}" if not os.path.isdir(local_folder): os.makedirs(local_folder, exist_ok=True) @@ -2429,7 +2457,7 @@ def launch_nwchem_calc(in_file, calc, files): if calc.remote_id == 0: sftp_put( f"{local_folder}/calc.inp", - os.path.join(folder, "calc.inp"), + os.path.join(remote_dir, "calc.inp"), conn, lock, ) @@ -2463,7 +2491,7 @@ def launch_nwchem_calc(in_file, calc, files): if not calc.local: for f in files: a = sftp_get( - f"{folder}/{f}", + f"{remote_dir}/{f}", os.path.join(CALCUS_SCR_HOME, str(calc.id), f), conn, lock, @@ -2481,10 +2509,10 @@ def launch_nwchem_calc(in_file, calc, files): return ErrorCodes.JOB_CANCELLED -def nwchem_sp(in_file, calc): +def nwchem_sp(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_nwchem_calc(in_file, calc, ["calc.out"]) + ret = launch_nwchem_calc(calc, ["calc.out"]) if ret != ErrorCodes.SUCCESS: return ret @@ -2515,11 +2543,10 @@ def nwchem_sp(in_file, calc): return ErrorCodes.SUCCESS -def nwchem_mo_gen(in_file, calc): +def nwchem_mo_gen(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) ret = launch_nwchem_calc( - in_file, calc, ["calc.out", "in.molden"], ) @@ -2677,11 +2704,10 @@ def get_cube_from_molden(molden, orb_num): return "".join(lines) -def nwchem_esp_gen(in_file, calc): +def nwchem_esp_gen(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) ret = launch_nwchem_calc( - in_file, calc, ["calc.out", "esp.cube", "in.eld.total.cube"], ) @@ -2736,10 +2762,10 @@ def nwchem_esp_gen(in_file, calc): return ErrorCodes.SUCCESS -def nwchem_opt(in_file, calc): +def nwchem_opt(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_nwchem_calc(in_file, calc, ["calc.out"]) + ret = launch_nwchem_calc(calc, ["calc.out"]) if ret != ErrorCodes.SUCCESS: return ret @@ -2847,10 +2873,10 @@ def nwchem_opt(in_file, calc): return ErrorCodes.SUCCESS -def nwchem_freq(in_file, calc): +def nwchem_freq(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_nwchem_calc(in_file, calc, ["calc.out"]) + ret = launch_nwchem_calc(calc, ["calc.out"]) if ret != ErrorCodes.SUCCESS: return ret @@ -3100,9 +3126,8 @@ def calc_to_ccinput(calc): return inp -def launch_gaussian_calc(in_file, calc, files): +def launch_gaussian_calc(calc, files): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - folder = f"scratch/calcus/{calc.id}" if not os.path.isdir(local_folder): os.makedirs(local_folder, exist_ok=True) @@ -3119,7 +3144,7 @@ def launch_gaussian_calc(in_file, calc, files): if calc.remote_id == 0: sftp_put( f"{local_folder}/calc.com", - os.path.join(folder, "calc.com"), + os.path.join(remote_dir, "calc.com"), conn, lock, ) @@ -3138,7 +3163,7 @@ def launch_gaussian_calc(in_file, calc, files): if not calc.local: for f in files: - a = sftp_get(f"{folder}/{f}", os.path.join(local_folder, f), conn, lock) + a = sftp_get(f"{remote_dir}/{f}", os.path.join(local_folder, f), conn, lock) if not cancelled and a != ErrorCodes.SUCCESS: return a @@ -3419,10 +3444,10 @@ def parse_Hirshfeld_gaussian_charges(calc, s): prop.save() -def gaussian_sp(in_file, calc): +def gaussian_sp(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_gaussian_calc(in_file, calc, ["calc.log"]) + ret = launch_gaussian_calc(calc, ["calc.log"]) if ret != ErrorCodes.SUCCESS: return ret @@ -3444,10 +3469,10 @@ def gaussian_sp(in_file, calc): return ErrorCodes.SUCCESS -def gaussian_td(in_file, calc): +def gaussian_td(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_gaussian_calc(in_file, calc, ["calc.log"]) + ret = launch_gaussian_calc(calc, ["calc.log"]) if ret != ErrorCodes.SUCCESS: return ret @@ -3518,10 +3543,10 @@ def parse_td_dft(lines, ind): return ErrorCodes.SUCCESS -def gaussian_opt(in_file, calc): +def gaussian_opt(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_gaussian_calc(in_file, calc, ["calc.log"]) + ret = launch_gaussian_calc(calc, ["calc.log"]) if ret != ErrorCodes.SUCCESS: return ret @@ -3576,10 +3601,10 @@ def gaussian_opt(in_file, calc): return ErrorCodes.SUCCESS -def gaussian_freq(in_file, calc): +def gaussian_freq(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_gaussian_calc(in_file, calc, ["calc.log"]) + ret = launch_gaussian_calc(calc, ["calc.log"]) if ret != ErrorCodes.SUCCESS: return ret @@ -3708,10 +3733,10 @@ def gaussian_freq(in_file, calc): return ErrorCodes.SUCCESS -def gaussian_ts(in_file, calc): +def gaussian_ts(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_gaussian_calc(in_file, calc, ["calc.log"]) + ret = launch_gaussian_calc(calc, ["calc.log"]) if ret != ErrorCodes.SUCCESS: return ret @@ -3766,10 +3791,10 @@ def gaussian_ts(in_file, calc): return ErrorCodes.SUCCESS -def gaussian_scan(in_file, calc): +def gaussian_scan(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_gaussian_calc(in_file, calc, ["calc.log"]) + ret = launch_gaussian_calc(calc, ["calc.log"]) has_scan = "scan" in calc.constraints.lower() @@ -3911,10 +3936,10 @@ def gaussian_scan(in_file, calc): return ErrorCodes.SUCCESS -def gaussian_nmr(in_file, calc): +def gaussian_nmr(calc): local_folder = os.path.join(CALCUS_SCR_HOME, str(calc.id)) - ret = launch_gaussian_calc(in_file, calc, ["calc.log"]) + ret = launch_gaussian_calc(calc, ["calc.log"]) if ret != ErrorCodes.SUCCESS: return ret @@ -4899,6 +4924,7 @@ def get_calc(calc_id): f = BASICSTEP_TABLE[calc.parameters.software][calc.step.name] workdir = os.path.join(CALCUS_SCR_HOME, str(calc.id)) + os.makedirs(workdir, exist_ok=True) if calc.status == 3: # Already revoked: logger.info(f"Calc {calc_id} already revoked") @@ -4913,32 +4939,9 @@ def get_calc(calc_id): if isinstance(ret, ErrorCodes): return ret - #### - in_file = os.path.join(workdir, "in.xyz") - - if calc.status == 0: - os.makedirs(workdir, exist_ok=True) - - with open(in_file, "w") as out: - out.write(clean_xyz(calc.structure.xyz_structure)) - #### - - if not calc.local and calc.remote_id == 0: - logger.debug(f"Preparing remote folder for calc {calc_id}") - pid = int(threading.get_ident()) - conn = connections[pid] - lock = locks[pid] - remote_dir = remote_dirs[pid] - - if calc.status == 0: - direct_command(f"mkdir -p {remote_dir}", conn, lock) - sftp_put(in_file, os.path.join(remote_dir, "in.xyz"), conn, lock) - - in_file = os.path.join(remote_dir, "in.xyz") - logger.info(f"Running calc {calc_id}") try: - ret = f(in_file, calc) + ret = f(calc) except Exception as e: ret = ErrorCodes.UNKNOWN_TERMINATION traceback.print_exc() diff --git a/frontend/templates/frontend/properties/simple_nmr.html b/frontend/templates/frontend/properties/simple_nmr.html index 1ed4b528..9398327c 100644 --- a/frontend/templates/frontend/properties/simple_nmr.html +++ b/frontend/templates/frontend/properties/simple_nmr.html @@ -1,3 +1,4 @@ +{% load details_tags %}
diff --git a/frontend/templatetags/details_tags.py b/frontend/templatetags/details_tags.py index 6da1f916..6566bd10 100644 --- a/frontend/templatetags/details_tags.py +++ b/frontend/templatetags/details_tags.py @@ -18,7 +18,7 @@ """ from django import template -from frontend.models import * +from frontend.models import Property register = template.Library() diff --git a/frontend/templatetags/folder_tags.py b/frontend/templatetags/folder_tags.py index b4078da7..db5563a5 100644 --- a/frontend/templatetags/folder_tags.py +++ b/frontend/templatetags/folder_tags.py @@ -18,7 +18,6 @@ """ from django import template -from frontend.models import * register = template.Library() diff --git a/frontend/test_calculations.py b/frontend/test_calculations.py index 041bb8cb..4442f038 100644 --- a/frontend/test_calculations.py +++ b/frontend/test_calculations.py @@ -23,8 +23,8 @@ from .calculation_unittest import CalculationUnitTest from .calcusliveserver import tests_dir -from .models import * -from .libxyz import * +from .models import Calculation, Property +from .libxyz import get_angle, parse_xyz_from_file class XtbCalculationTests(CalculationUnitTest): @@ -575,16 +575,6 @@ def test_sp_HF_CPCM(self): def test_sp_DFT(self): self.assertTrue(self.run_test(theory_level="DFT", method="M062X")) - def test_sp_DFT_SMD18(self): - self.assertTrue( - self.run_test( - theory_level="DFT", - method="M062X", - solvation_model="SMD", - solvent="Methanol", - ) - ) - def test_sp_DFT_SMD18(self): self.assertTrue( self.run_test( @@ -824,7 +814,7 @@ def test_DFT_multiple_pop(self): self.assertIn("Hirshfeld:", prop.charges) self.assertIn("CM5:", prop.charges) - def test_DFT_multiple_pop(self): + def test_DFT_multiple_pop2(self): self.assertTrue( self.run_test(theory_level="DFT", method="M062X", specifications="pop(esp)") ) diff --git a/frontend/test_fingerprint.py b/frontend/test_fingerprint.py index 1c892b7e..ad4a713a 100644 --- a/frontend/test_fingerprint.py +++ b/frontend/test_fingerprint.py @@ -22,7 +22,7 @@ from django.test import TestCase from .tasks import write_mol, gen_fingerprint -from .models import * +from .models import Structure tests_dir = os.path.join("/".join(__file__.split("/")[:-1]), "tests/") diff --git a/frontend/test_md5.py b/frontend/test_md5.py index 6d3060ea..210232a3 100644 --- a/frontend/test_md5.py +++ b/frontend/test_md5.py @@ -17,12 +17,10 @@ along with this program. If not, see . """ -import os -from .models import * from .gen_calc import gen_param from django.core.management import call_command -from django.test import TestCase, Client +from django.test import TestCase class ParametersMd5Tests(TestCase): diff --git a/frontend/test_models.py b/frontend/test_models.py index b2bc5a22..a7ccc242 100644 --- a/frontend/test_models.py +++ b/frontend/test_models.py @@ -23,7 +23,14 @@ from django.test import TransactionTestCase from django.db import close_old_connections, connection -from .models import * +from .models import ( + ResearchGroup, + ResourceAllocation, + Subscription, + User, + settings, + timezone, +) def redeem(obj, user, stall=0): @@ -37,7 +44,7 @@ def tearDown(self): def test_base_redeem(self): u1 = User.objects.create_user(email="U1@test.com", password="1234") - u2 = User.objects.create_user(email="U2@test.com", password="1234") + User.objects.create_user(email="U2@test.com", password="1234") alloc = ResourceAllocation.objects.create(code="AAAA", allocation_seconds=10) diff --git a/frontend/test_selenium.py b/frontend/test_selenium.py index 1a5ef1b2..b32bb7cf 100644 --- a/frontend/test_selenium.py +++ b/frontend/test_selenium.py @@ -25,14 +25,26 @@ from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.support.ui import Select + +# from selenium.webdriver.support.ui import Select from selenium.webdriver.support.select import Select from django.conf import settings -from .models import * -from .libxyz import * +from .models import ( + Calculation, + CalculationOrder, + Ensemble, + Molecule, + Parameters, + Project, + Property, + ResearchGroup, + Structure, + User, +) +from .libxyz import get_angle, np, parse_xyz_from_file, parse_xyz_from_text from .calcusliveserver import CalcusLiveServer, tests_dir diff --git a/frontend/test_units.py b/frontend/test_units.py index 68ffe073..f36a872c 100644 --- a/frontend/test_units.py +++ b/frontend/test_units.py @@ -17,7 +17,6 @@ along with this program. If not, see . """ -from .models import * from .helpers import get_xyz_from_Gaussian_input from django.test import TestCase diff --git a/frontend/test_views.py b/frontend/test_views.py index f843dfe0..a57d1233 100644 --- a/frontend/test_views.py +++ b/frontend/test_views.py @@ -28,8 +28,21 @@ from .gen_calc import gen_calc from .calcusliveserver import SCR_DIR -from .constants import * -from .models import * +from .models import ( + CalculationOrder, + Ensemble, + HARTREE_FVAL, + HARTREE_TO_KCAL_F, + Molecule, + Parameters, + Project, + Property, + ResearchGroup, + Structure, + User, + np, + timezone, +) tests_dir = os.path.join("/".join(__file__.split("/")[:-1]), "tests/") diff --git a/frontend/test_xyz.py b/frontend/test_xyz.py index c106bbc8..94b9b90d 100644 --- a/frontend/test_xyz.py +++ b/frontend/test_xyz.py @@ -21,7 +21,14 @@ from django.test import TestCase -from .libxyz import * +from .libxyz import ( + equivalent_atoms, + morgan_hashz_numbering, + morgan_numbering, + np, + parse_xyz_from_file, + reorder_xyz, +) tests_dir = os.path.join("/".join(__file__.split("/")[:-1]), "tests/") @@ -43,7 +50,6 @@ def test_morgan_algorithm2(self): def test_morgan_hashz(self): xyz = parse_xyz_from_file(os.path.join(tests_dir, "ethanol.xyz")) - REF = [54, 23, 23, 23, 57, 24, 24, 32, 15] indices = morgan_hashz_numbering(xyz) groups = [[0], [1, 2, 3], [4], [5, 6], [7], [8]] diff --git a/frontend/tests/find_empty_caches.py b/frontend/tests/find_empty_caches.py index 87b91623..5956c7dc 100644 --- a/frontend/tests/find_empty_caches.py +++ b/frontend/tests/find_empty_caches.py @@ -1,6 +1,5 @@ import glob import os -import shutil KEEP_FILES = [ "calc.inp", diff --git a/frontend/views.py b/frontend/views.py index 35723f11..44ef3559 100644 --- a/frontend/views.py +++ b/frontend/views.py @@ -27,7 +27,6 @@ import basis_set_exchange import numpy as np import tempfile -import json import ccinput from datetime import datetime import base64, gzip @@ -39,7 +38,6 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.backends import default_backend -from django import template from django.shortcuts import render, redirect from django.http import HttpResponse, HttpResponseRedirect, JsonResponse from django.views.decorators.csrf import csrf_exempt @@ -47,13 +45,12 @@ from django.views import generic from django.utils import timezone from django.contrib.auth.decorators import login_required -from django.contrib.auth.models import AnonymousUser, User +from django.contrib.auth.models import AnonymousUser from django.contrib.auth import login, update_session_auth_hash, authenticate from django.utils.datastructures import MultiValueDictKeyError from django.contrib import messages from django.contrib.auth.forms import PasswordChangeForm from django.template.loader import get_template -from django.shortcuts import render from .forms import ( @@ -88,8 +85,6 @@ ShowcaseEnsemble, FlowchartOrder, ResourceAllocation, - BatchCalcOrder, - BatchCalculation, ) from .tasks import ( dispatcher, @@ -115,8 +110,15 @@ from .cloud_job import send_gcloud_task, record_event_analytics from frontend import tasks -from .decorators import superuser_required -from .constants import * +from .constants import ( + ATOMIC_NUMBER, + ErrorCodes, + HARTREE_FVAL, + HARTREE_TO_KCAL_F, + MAX_FOLDER_DEPTH, + full_tos, + trial_tos, +) from .libxyz import ( parse_xyz_from_text, equivalent_atoms, @@ -124,7 +126,7 @@ create_derivative, format_xyz, ) -from .environment_variables import * +from .environment_variables import CALCUS_KEY_HOME, CALCUS_SCR_HOME, IS_TEST from .helpers import ( get_xyz_from_Gaussian_input, get_random_string, @@ -1547,7 +1549,7 @@ def parse_parameters(request, parameters_dict, is_flowchart=None, verify=False): else: try: project_set = request.user.project_set.filter(name=project) - except Profile.DoesNotExist: + except Project.DoesNotExist: return "No such project" if len(project_set) != 1: @@ -1591,7 +1593,7 @@ def save_preset(request): return HttpResponse("No preset name") params = Parameters.objects.create(**_params) - preset = Preset.objects.create(name=preset_name, author=request.user, params=params) + Preset.objects.create(name=preset_name, author=request.user, params=params) return HttpResponse("Preset created") @@ -2488,12 +2490,12 @@ def get_default_name(): # Since the charge/multiplicity keywords are considered when detecting molecules, # all the structures of a given molecule must necessarily have the same charge/multiplicity __params = _params.copy() - __params["charge"] = ( - struct.properties.first().parameters.charge - ) - __params["multiplicity"] = ( - struct.properties.first().parameters.multiplicity - ) + __params[ + "charge" + ] = struct.properties.first().parameters.charge + __params[ + "multiplicity" + ] = struct.properties.first().parameters.multiplicity unique_params[_mol_name] = Parameters.objects.create( **__params ) @@ -3638,9 +3640,9 @@ def uvvis(request, pk): return HttpResponse(status=404) response = HttpResponse(prop.uvvis, content_type="text/csv") - response["Content-Disposition"] = ( - f"attachment; filename=uvvis_{prop.parent_structure.id}.csv" - ) + response[ + "Content-Disposition" + ] = f"attachment; filename=uvvis_{prop.parent_structure.id}.csv" return response @@ -3888,9 +3890,9 @@ def nmr(request): content += f"{-(shift[3] + 0.001)},{0}\n" response = HttpResponse(content, content_type="text/csv") - response["Content-Disposition"] = ( - f"attachment; filename=nmr_{clean_filename(e.name)}.csv" - ) + response[ + "Content-Disposition" + ] = f"attachment; filename=nmr_{clean_filename(e.name)}.csv" return response @@ -3906,9 +3908,9 @@ def ir_spectrum(request, pk): if prop.ir_spectrum != "": response = HttpResponse(prop.ir_spectrum, content_type="text/csv") - response["Content-Disposition"] = ( - f"attachment; filename=ir_{prop.parent_structure.id}.csv" - ) + response[ + "Content-Disposition" + ] = f"attachment; filename=ir_{prop.parent_structure.id}.csv" return response else: return HttpResponse(status=204) @@ -4378,9 +4380,9 @@ def download_all_logs(request, pk): ) response = HttpResponse(mem.getvalue(), content_type="application/zip") - response["Content-Disposition"] = ( - f'attachment; filename="{order.molecule_name}_order_{pk}.zip"' - ) + response[ + "Content-Disposition" + ] = f'attachment; filename="{order.molecule_name}_order_{pk}.zip"' return response diff --git a/generate_env.py b/generate_env.py index 3def528a..4fca0cf7 100755 --- a/generate_env.py +++ b/generate_env.py @@ -176,8 +176,8 @@ def get_env_backup_path(): } override_list = { - "g16": [" - ${CALCUS_GAUSSIAN}:/binaries/g16"], - "orca": [" - ${CALCUS_ORCA}:/binaries/orca"], + "g16": [" - ${CALCUS_GAUSSIAN}:/calcus/binaries/g16"], + "orca": [" - ${CALCUS_ORCA}:/calcus/binaries/orca"], } software_paths = "" diff --git a/scripts/extract_xtb.py b/scripts/extract_xtb.py index 8a792dcb..2d8049b0 100644 --- a/scripts/extract_xtb.py +++ b/scripts/extract_xtb.py @@ -2,13 +2,18 @@ import subprocess import shlex -if not os.path.isdir("/binaries/xtb/xtb"): - subprocess.call(shlex.split("tar xvfJ xtb_suite.tar.xz"), cwd="/binaries/xtb/") +if not os.path.isdir("/calcus/binaries/xtb/xtb"): subprocess.call( - shlex.split("chmod +x crest stda xtb4stda xtb/bin/xtb"), cwd="/binaries/xtb/" + shlex.split("tar xvfJ xtb_suite.tar.xz"), cwd="/calcus/binaries/xtb/" ) - os.remove("/binaries/xtb/xtb_suite.tar.xz") -if not os.path.isdir("/binaries/xtb/Multiwfn"): - subprocess.call(shlex.split("tar xvfj Multiwfn.tar.bz2"), cwd="/binaries/xtb/") - subprocess.call(shlex.split("chmod +x Multiwfn"), cwd="/binaries/xtb/") - os.remove("/binaries/xtb/Multiwfn.tar.bz2") + subprocess.call( + shlex.split("chmod +x crest stda xtb4stda xtb/bin/xtb"), + cwd="/calcus/binaries/xtb/", + ) + os.remove("/calcus/binaries/xtb/xtb_suite.tar.xz") +if not os.path.isdir("/calcus/binaries/xtb/Multiwfn"): + subprocess.call( + shlex.split("tar xvfj Multiwfn.tar.bz2"), cwd="/calcus/binaries/xtb/" + ) + subprocess.call(shlex.split("chmod +x Multiwfn"), cwd="/calcus/binaries/xtb/") + os.remove("/calcus/binaries/xtb/Multiwfn.tar.bz2") diff --git a/test-compose.yml b/test-compose.yml index ba7bc82d..ff9b3d9d 100644 --- a/test-compose.yml +++ b/test-compose.yml @@ -1,5 +1,3 @@ -version: "3.4" - services: redis: image: redis:7.2-rc2