Commit 43a9a36c authored by Martino Bertoni's avatar Martino Bertoni 🌋
Browse files

hpc module can be used in lite version ('paramiko' added to requirements)

parent af86fed4
Pipeline #734 passed with stages
in 10 minutes and 36 seconds
......@@ -63,6 +63,7 @@ From: centos
pip install autologging
pip install scipy
pip install sqlalchemy
pip install paramiko
# unittest utils
pip install pytest
pip install mock
......
......@@ -22,7 +22,7 @@ from chemicalchecker.util.hpc import HPC
class ChemicalChecker():
"""Explore the Chemical Checker."""
def __init__(self, cc_root):
def __init__(self, cc_root=None):
"""Initialize the Chemical Checker.
If the CC_ROOT directory is empty a skeleton of CC is initialized.
......@@ -33,17 +33,21 @@ class ChemicalChecker():
cc_root(str): The Chemical Checker root directory. It's version
dependendent.
"""
self.cc_root = cc_root
if not cc_root:
self.cc_root = Config().PATH.CC_ROOT
else:
self.cc_root = cc_root
self._basic_molsets = ['reference', 'full']
self._datasets = set()
self._molsets = set(self._basic_molsets)
self.__log.debug("ChemicalChecker with root: %s", cc_root)
if not os.path.isdir(cc_root):
self.__log.debug("ChemicalChecker with root: %s", self.cc_root)
if not os.path.isdir(self.cc_root):
self.__log.warning("Empty root directory, creating dataset dirs")
for molset in self._basic_molsets:
for dataset in Dataset.get():
ds = dataset.code
new_dir = os.path.join(cc_root, molset, ds[:1], ds[:2], ds)
new_dir = os.path.join(
self.cc_root, molset, ds[:1], ds[:2], ds)
self._datasets.add(ds)
self.__log.debug("Creating %s", new_dir)
original_umask = os.umask(0)
......@@ -51,7 +55,7 @@ class ChemicalChecker():
os.umask(original_umask)
else:
# if the directory exists get molsets and datasets
paths = glob(os.path.join(cc_root, '*', '*', '*', '*'))
paths = glob(os.path.join(self.cc_root, '*', '*', '*', '*'))
self._molsets = set(x.split('/')[-4] for x in paths)
self._datasets = set(x.split('/')[-1] for x in paths)
self._molsets = sorted(list(self._molsets))
......
......@@ -3,3 +3,4 @@ from .logging.our_logging import *
from autologging import logged
from .logging.profilehooks import profile
from .config import Config
from .hpc import HPC
......@@ -7,6 +7,7 @@ import time
import pickle
import tarfile
import datetime
import paramiko
import numpy as np
from chemicalchecker.util import logged
......@@ -55,12 +56,6 @@ fi
"""Initialize the SGE object.
"""
try:
import paramiko
self.paramiko = paramiko
except ImportError:
raise ImportError("requires paramiko " +
"http://www.paramiko.org/")
self.host = config.HPC.host
self.queue = None
self.statusFile = None
......@@ -73,10 +68,10 @@ fi
self.conn_params["password"] = config.HPC.password
if not dry_run:
try:
ssh = self.paramiko.SSHClient()
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect(self.host, **self.conn_params)
except self.paramiko.SSHException as sshException:
except paramiko.SSHException as sshException:
self.__log.warning(
"Unable to establish SSH connection: %s" % sshException)
finally:
......@@ -192,7 +187,7 @@ fi
time.sleep(15)
try:
ssh = self.paramiko.SSHClient()
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect(self.host, **self.conn_params)
stdin, stdout, stderr = ssh.exec_command(
......@@ -207,7 +202,7 @@ fi
self.job_id = self.job_id.rstrip()
self.__log.debug(self.job_id)
except self.paramiko.SSHException as sshException:
except paramiko.SSHException as sshException:
self.__log.warning(
"Unable to establish SSH connection: %s" % sshException)
finally:
......@@ -308,7 +303,7 @@ fi
if self.status_id == STARTED:
try:
ssh = self.paramiko.SSHClient()
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect(self.host, **self.conn_params)
stdin, stdout, stderr = ssh.exec_command(
......@@ -323,7 +318,7 @@ fi
self.status_id = DONE
with open(self.statusFile, "w") as f:
f.write(self.status_id)
except self.paramiko.SSHException as sshException:
except paramiko.SSHException as sshException:
self.__log.warning(
"Unable to establish SSH connection: %s" % sshException)
finally:
......
......@@ -6,6 +6,7 @@ import uuid
import time
import pickle
import tarfile
import paramiko
import numpy as np
from chemicalchecker.util import logged
......@@ -53,12 +54,6 @@ fi
"""Initialize the SLURM object.
"""
try:
import paramiko
self.paramiko = paramiko
except ImportError:
raise ImportError("requires paramiko " +
"http://www.paramiko.org/")
self.host = config.HPC.host
self.queue = None
self.statusFile = None
......@@ -71,17 +66,17 @@ fi
self.conn_params["password"] = config.HPC.password
if not dry_run:
try:
ssh_config = self.paramiko.SSHConfig()
ssh_config = paramiko.SSHConfig()
user_config_file = os.path.expanduser("~/.ssh/config")
if os.path.exists(user_config_file):
with open(user_config_file) as f:
ssh_config.parse(f)
cfg = ssh_config.lookup(self.host)
ssh = self.paramiko.SSHClient()
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect(cfg['hostname'], username=cfg[
'user'], key_filename=cfg['identityfile'][0])
except self.paramiko.SSHException as sshException:
except paramiko.SSHException as sshException:
self.__log.warning(
"Unable to establish SSH connection: %s" % sshException)
finally:
......@@ -193,13 +188,13 @@ fi
time.sleep(2)
try:
ssh_config = self.paramiko.SSHConfig()
ssh_config = paramiko.SSHConfig()
user_config_file = os.path.expanduser("~/.ssh/config")
if os.path.exists(user_config_file):
with open(user_config_file) as f:
ssh_config.parse(f)
cfg = ssh_config.lookup(self.host)
ssh = self.paramiko.SSHClient()
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect(cfg['hostname'], username=cfg[
'user'], key_filename=cfg['identityfile'][0])
......@@ -215,7 +210,7 @@ fi
self.job_id = self.job_id.rstrip()
self.__log.debug(self.job_id)
except self.paramiko.SSHException as sshException:
except paramiko.SSHException as sshException:
self.__log.warning(
"Unable to establish SSH connection: %s" % sshException)
finally:
......@@ -315,13 +310,13 @@ fi
if self.status_id == STARTED:
try:
ssh_config = self.paramiko.SSHConfig()
ssh_config = paramiko.SSHConfig()
user_config_file = os.path.expanduser("~/.ssh/config")
if os.path.exists(user_config_file):
with open(user_config_file) as f:
ssh_config.parse(f)
cfg = ssh_config.lookup(self.host)
ssh = self.paramiko.SSHClient()
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect(cfg['hostname'], username=cfg[
'user'], key_filename=cfg['identityfile'][0])
......@@ -337,7 +332,7 @@ fi
self.status_id = DONE
with open(self.statusFile, "w") as f:
f.write(self.status_id)
except self.paramiko.SSHException as sshException:
except paramiko.SSHException as sshException:
self.__log.warning(
"Unable to establish SSH connection: %s" % sshException)
finally:
......
......@@ -19,7 +19,8 @@ requirements = [
'networkx',
'autologging',
'scipy',
'sqlalchemy'
'sqlalchemy',
'paramiko'
]
setup_requirements = ['pytest-runner']
......
......@@ -3,7 +3,7 @@ import pytest
import unittest
import functools
from chemicalchecker.util.hpc import HPC
from chemicalchecker.util import HPC
from chemicalchecker.util import Config
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment