Rename pyenv to micromamba submodule and add tests
This commit is contained in:
parent
bf8bce2fb1
commit
33abe472d4
8 changed files with 119 additions and 80 deletions
75
src/glicid_spawner/micromamba.py
Normal file
75
src/glicid_spawner/micromamba.py
Normal file
|
@ -0,0 +1,75 @@
|
||||||
|
"""Micromamba environments module."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from operator import itemgetter
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
MICROMAMBA_ROOT = Path('/micromamba') # default micromamba root location
|
||||||
|
|
||||||
|
GLOBAL_USER = 'operator'
|
||||||
|
GLOBAL_EXCLUDED = 'glicid-jupyterhub'
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MicromambaEnv:
|
||||||
|
"""Generic micromamba environment."""
|
||||||
|
|
||||||
|
scope: str
|
||||||
|
name: str
|
||||||
|
path: str
|
||||||
|
|
||||||
|
|
||||||
|
def _envs(folder, excluded=None) -> list:
|
||||||
|
"""List micromamba environments."""
|
||||||
|
# Convert excluded as as list
|
||||||
|
if excluded is None:
|
||||||
|
excluded = []
|
||||||
|
elif isinstance(excluded, str):
|
||||||
|
excluded = [excluded]
|
||||||
|
|
||||||
|
# Get micromamba envs as pathlib.Path
|
||||||
|
envs = MICROMAMBA_ROOT / folder / 'envs'
|
||||||
|
|
||||||
|
if not envs.exists():
|
||||||
|
return []
|
||||||
|
|
||||||
|
return sorted(
|
||||||
|
[
|
||||||
|
(env.name, str(env))
|
||||||
|
for env in envs.iterdir()
|
||||||
|
if env.is_dir() and env.name not in excluded
|
||||||
|
],
|
||||||
|
key=itemgetter(0),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _envs_user(username: str) -> list:
|
||||||
|
"""Micromamba environment(s) available to the user."""
|
||||||
|
return [MicromambaEnv('USER', name, path) for name, path in _envs(username)]
|
||||||
|
|
||||||
|
|
||||||
|
def _envs_team(username: str) -> list:
|
||||||
|
"""Micromamba environment(s) available to the user's team.
|
||||||
|
|
||||||
|
Warning
|
||||||
|
-------
|
||||||
|
At the moment micromamba team environments is not available on GLiCID.
|
||||||
|
|
||||||
|
"""
|
||||||
|
teams = [] # FIXME: pull user teams list from groups
|
||||||
|
return [
|
||||||
|
MicromambaEnv('TEAM', name, path) for team in teams for name, path in _envs(f'teams/{team}')
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _envs_global() -> list:
|
||||||
|
"""Micromamba environment(s) available globally."""
|
||||||
|
return [
|
||||||
|
MicromambaEnv('GLOBAL', name, path)
|
||||||
|
for name, path in _envs(GLOBAL_USER, excluded=GLOBAL_EXCLUDED)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def get_envs(username: str) -> list:
|
||||||
|
"""List of all the micromamba environment available to a user."""
|
||||||
|
return _envs_user(username) + _envs_team(username) + _envs_global()
|
|
@ -1,74 +0,0 @@
|
||||||
"""Python environment module."""
|
|
||||||
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from operator import itemgetter
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class PyEnv:
|
|
||||||
"""Python environment generic class."""
|
|
||||||
|
|
||||||
name: str
|
|
||||||
path: str
|
|
||||||
|
|
||||||
def __eq__(self, item):
|
|
||||||
return self.path == str(item)
|
|
||||||
|
|
||||||
|
|
||||||
class UserPyEnv(PyEnv):
|
|
||||||
"""User python environment."""
|
|
||||||
|
|
||||||
scope = 'USER'
|
|
||||||
|
|
||||||
|
|
||||||
class TeamPyEnv(PyEnv):
|
|
||||||
"""Team python environment."""
|
|
||||||
|
|
||||||
scope = 'TEAM'
|
|
||||||
|
|
||||||
|
|
||||||
class GlobalPyEnv(PyEnv):
|
|
||||||
"""Global python environment."""
|
|
||||||
|
|
||||||
scope = 'GLOBAL'
|
|
||||||
|
|
||||||
|
|
||||||
def _micromamba_path(path: str) -> Path:
|
|
||||||
"""Micromamba path environments locations."""
|
|
||||||
return Path(f'/micromamba/{path}/envs')
|
|
||||||
|
|
||||||
|
|
||||||
def _micromamba_envs(path, excluded=None) -> list:
|
|
||||||
"""List micromamba environment list."""
|
|
||||||
if excluded is None:
|
|
||||||
excluded = []
|
|
||||||
elif isinstance(excluded, str):
|
|
||||||
excluded = [excluded]
|
|
||||||
|
|
||||||
envs = _micromamba_path(path)
|
|
||||||
return sorted(
|
|
||||||
[(f.name, f.absolute()) for f in envs.iterdir() if f.is_dir() and f.name not in excluded],
|
|
||||||
key=itemgetter(0),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def get_pyenv_user(username: str) -> list:
|
|
||||||
"""List of python environment available to the user."""
|
|
||||||
return [UserPyEnv(*env) for env in _micromamba_envs(username)]
|
|
||||||
|
|
||||||
|
|
||||||
def get_pyenv_team(username: str) -> list:
|
|
||||||
"""List of python environment available to the user's team."""
|
|
||||||
teams = [] # FIXME: pull user teams list from groups
|
|
||||||
return [TeamPyEnv(*env) for team in teams for env in _micromamba_envs(f'teams/{team}')]
|
|
||||||
|
|
||||||
|
|
||||||
def get_pyenv_global() -> list:
|
|
||||||
"""List of python environment available globally."""
|
|
||||||
return [GlobalPyEnv(*env) for env in _micromamba_envs('operator', excluded='glicid-jupyterhub')]
|
|
||||||
|
|
||||||
|
|
||||||
def get_pyenv(username: str) -> list:
|
|
||||||
"""List of all the python environment available to a user."""
|
|
||||||
return get_pyenv_user(username) + get_pyenv_team(username) + get_pyenv_global()
|
|
|
@ -3,7 +3,7 @@
|
||||||
from batchspawner import SlurmSpawner
|
from batchspawner import SlurmSpawner
|
||||||
from jinja2 import Environment, PackageLoader, select_autoescape
|
from jinja2 import Environment, PackageLoader, select_autoescape
|
||||||
|
|
||||||
from .pyenv import get_pyenv
|
from .micromamba import get_envs
|
||||||
from .resources import CPU, GPU, RAM
|
from .resources import CPU, GPU, RAM
|
||||||
|
|
||||||
|
|
||||||
|
@ -20,16 +20,14 @@ class GlicidSpawner(SlurmSpawner):
|
||||||
|
|
||||||
return template.render(
|
return template.render(
|
||||||
username=self.user.name,
|
username=self.user.name,
|
||||||
python_envs=get_pyenv(self.user.name),
|
python_envs=get_envs(self.user.name),
|
||||||
cpu_available=CPU,
|
cpu_available=CPU,
|
||||||
ram_available=RAM,
|
ram_available=RAM,
|
||||||
gpu_available=GPU,
|
gpu_available=GPU,
|
||||||
)
|
)
|
||||||
|
|
||||||
def options_from_form(self, formdata) -> dict:
|
def options_from_form(self, formdata) -> dict:
|
||||||
options = {}
|
"""Export options from form."""
|
||||||
options['pyenv'] = formdata['python-env'][0]
|
|
||||||
|
|
||||||
# Index of user resources choices
|
# Index of user resources choices
|
||||||
i_cpu = int(formdata['cpu'][0])
|
i_cpu = int(formdata['cpu'][0])
|
||||||
i_ram = int(formdata['ram'][0])
|
i_ram = int(formdata['ram'][0])
|
||||||
|
@ -42,8 +40,12 @@ class GlicidSpawner(SlurmSpawner):
|
||||||
)
|
)
|
||||||
|
|
||||||
# Export options
|
# Export options
|
||||||
|
options = {}
|
||||||
|
|
||||||
|
options['pyenv'] = formdata['python-env'][0]
|
||||||
|
|
||||||
options['cpus-per-task'] = CPU[i_cpu].description
|
options['cpus-per-task'] = CPU[i_cpu].description
|
||||||
options['mem'] = RAM[i_ram].description + 'GB'
|
options['mem'] = RAM[i_ram].description.replace(' ', '')
|
||||||
|
|
||||||
if i_gpu:
|
if i_gpu:
|
||||||
options['gres'] = 'gpu:' + GPU[i_gpu].description
|
options['gres'] = 'gpu:' + GPU[i_gpu].description
|
||||||
|
|
0
tests/data/micromamba/global/envs/baz/.gitkeep
Normal file
0
tests/data/micromamba/global/envs/baz/.gitkeep
Normal file
0
tests/data/micromamba/global/envs/qux/.gitkeep
Normal file
0
tests/data/micromamba/global/envs/qux/.gitkeep
Normal file
0
tests/data/micromamba/john-doe/envs/bar/.gitkeep
Normal file
0
tests/data/micromamba/john-doe/envs/bar/.gitkeep
Normal file
0
tests/data/micromamba/john-doe/envs/foo/.gitkeep
Normal file
0
tests/data/micromamba/john-doe/envs/foo/.gitkeep
Normal file
36
tests/test_micromamba.py
Normal file
36
tests/test_micromamba.py
Normal file
|
@ -0,0 +1,36 @@
|
||||||
|
"""Test python environment module."""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from glicid_spawner import micromamba
|
||||||
|
|
||||||
|
MICROMAMBA_ROOT = Path(__file__).parent / 'data' / 'micromamba'
|
||||||
|
|
||||||
|
|
||||||
|
def test_micromamba_envs_getter(monkeypatch):
|
||||||
|
"""Test micromamba envs getter."""
|
||||||
|
monkeypatch.setattr(micromamba, 'MICROMAMBA_ROOT', MICROMAMBA_ROOT)
|
||||||
|
monkeypatch.setattr(micromamba, 'GLOBAL_USER', 'global')
|
||||||
|
monkeypatch.setattr(micromamba, 'GLOBAL_EXCLUDED', 'qux')
|
||||||
|
|
||||||
|
# User with micromamba envs
|
||||||
|
envs = micromamba.get_envs('john-doe')
|
||||||
|
|
||||||
|
assert len(envs) == 3
|
||||||
|
assert [env.scope for env in envs] == ['USER', 'USER', 'GLOBAL']
|
||||||
|
assert [env.name for env in envs] == ['bar', 'foo', 'baz']
|
||||||
|
assert [env.path for env in envs] == [
|
||||||
|
str(MICROMAMBA_ROOT / 'john-doe' / 'envs' / 'bar'),
|
||||||
|
str(MICROMAMBA_ROOT / 'john-doe' / 'envs' / 'foo'),
|
||||||
|
str(MICROMAMBA_ROOT / 'global' / 'envs' / 'baz'),
|
||||||
|
]
|
||||||
|
|
||||||
|
# User without micromamba envs, only global envs listed
|
||||||
|
envs = micromamba.get_envs('jane-smith')
|
||||||
|
|
||||||
|
assert len(envs) == 1
|
||||||
|
assert [env.scope for env in envs] == ['GLOBAL']
|
||||||
|
assert [env.name for env in envs] == ['baz']
|
||||||
|
assert [env.path for env in envs] == [
|
||||||
|
str(MICROMAMBA_ROOT / 'global' / 'envs' / 'baz'),
|
||||||
|
]
|
Loading…
Add table
Reference in a new issue