From 11d878cecfec52a2e4adac7a7f7f34057449296f Mon Sep 17 00:00:00 2001 From: Benoit Seignovert Date: Tue, 20 Feb 2024 15:13:14 +0100 Subject: [PATCH] Add SLURM resources to form template --- README.md | 2 +- render/__main__.py | 53 +++- src/glicid_spawner/form.py | 54 ++-- src/glicid_spawner/templates/static/style.css | 23 +- .../templates/views/resources.jinja | 40 +-- .../templates/views/slurm.jinja | 179 +++++++++--- tests/test_form.py | 267 ++++++++++++++++-- 7 files changed, 506 insertions(+), 112 deletions(-) diff --git a/README.md b/README.md index 4ea60dc..8fb078e 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ poetry run ruff format To render the form template (with live reload): ```bash -poetry run python -m render +poetry run python -m render [--single-cluster] ``` To activate the virtual environement globally: diff --git a/render/__main__.py b/render/__main__.py index 8c7fd79..63c11dd 100644 --- a/render/__main__.py +++ b/render/__main__.py @@ -3,12 +3,16 @@ Usage: `python -m render` """ +import sys +from argparse import ArgumentParser +from pathlib import Path from traceback import format_exc from flask import Flask, render_template, request from glicid_spawner.form import TEMPLATES, options_from_form from glicid_spawner.micromamba import MicromambaEnv -from glicid_spawner.resources import CPU, GPU, RAM +from glicid_spawner.resources import CPU, MEMORY, gpu_max_duration +from glicid_spawner.slurm import gres, sinfo_from_file from livereload import Server # Dummy username and python environments @@ -18,15 +22,30 @@ ENVS = [ MicromambaEnv('USER', 'bar', f'/{USERNAME}/envs/bar'), MicromambaEnv('GLOBAL', 'baz', '/global/envs/baz'), ] + +# Dummy SLURM config +DATA = Path(__file__).parent / '..' / 'tests' / 'data' +SINFO = sinfo_from_file( + DATA / 'sinfo.txt', with_states=('idle', 'mixed', 'allocated', 'completing', 'planned') +) + +# Single vs. multi-cluster implementation +SLURM_SINGLE_CLUSTER = {'N/A': SINFO.pop('N/A')} +SLURM_MULTI_CLUSTER = SINFO +GPU_SINGLE_CLUSTER = gpu_max_duration(gres(SLURM_SINGLE_CLUSTER)) +GPU_MULTI_CLUSTER = gpu_max_duration(gres(SLURM_MULTI_CLUSTER)) + +# Format dummy options OPTIONS = { 'username': USERNAME, 'envs': ENVS, 'cpus': CPU, - 'rams': RAM, - 'gpus': GPU, + 'mems': MEMORY, + # Multi-cluster by default. See `--single-cluster` flag in CLI for single cluster config. + 'gpus': GPU_MULTI_CLUSTER, + 'sinfo': SLURM_MULTI_CLUSTER, } - # Flask app app = Flask(__name__) app.debug = True @@ -60,11 +79,35 @@ def submit(): return render_template('options.html', formdata=formdata, err=format_exc()) +@app.route('/favicon.ico') +def favicon(): + """Dummy favicon.""" + return '' + + def server_autoreload(): """Start auto-reload server.""" server = Server(app.wsgi_app) server.serve() -if __name__ == '__main__': +def cli(argv=None): + """Command line interface.""" + parser = ArgumentParser('Spawner form render') + parser.add_argument( + '--single-cluster', + action='store_true', + help='Toggle SLURM config to single cluster configuration.', + ) + + args, _ = parser.parse_known_args(argv) + + if args.single_cluster: + OPTIONS['gpus'] = GPU_SINGLE_CLUSTER + OPTIONS['sinfo'] = SLURM_SINGLE_CLUSTER + server_autoreload() + + +if __name__ == '__main__': + cli(sys.argv) diff --git a/src/glicid_spawner/form.py b/src/glicid_spawner/form.py index 300de95..aa4a87b 100644 --- a/src/glicid_spawner/form.py +++ b/src/glicid_spawner/form.py @@ -3,7 +3,8 @@ from jinja2 import Environment, PackageLoader, select_autoescape from .micromamba import get_envs -from .resources import CPU, GPU, RAM +from .resources import CPU, GPU_DEFAULTS, MEMORY, gpu_max_duration +from .slurm import gres, sinfo TEMPLATES = Environment( loader=PackageLoader('glicid_spawner'), @@ -13,12 +14,18 @@ TEMPLATES = Environment( def options_attrs(username: str) -> dict: """Form options attributes.""" + slurm_sinfo = sinfo(username) + + # Allocated 1h to any SLURM GPU resources not listed in GPUS + gpus = gpu_max_duration(gres(slurm_sinfo), unknown_default=1) + return { 'username': username, 'envs': get_envs(username), 'cpus': CPU, - 'rams': RAM, - 'gpus': GPU, + 'mems': MEMORY, + 'gpus': gpus, + 'sinfo': slurm_sinfo, } @@ -30,26 +37,37 @@ def options_form(username: str) -> str: def options_from_form(formdata) -> dict: """Export options from default form.""" - # Resources choices indexes - i_cpu = int(formdata['cpu'][0]) - i_ram = int(formdata['ram'][0]) - i_gpu = int(formdata['gpu'][0]) + # Parse form data response + env = formdata['python-env'][0] + cpu = int(formdata['cpu'][0]) + mem = int(formdata['mem'][0]) + gpu = formdata['gpu'][0] + cluster = formdata.get('cluster', [None])[0] + partition = formdata.get('partition', [None])[0] + node = formdata.get('node', [None])[0] - duration = min( - CPU[i_cpu].max_duration, - RAM[i_ram].max_duration, - GPU[i_gpu].max_duration, - ) + # Compute max duration + # If the value provided is not in the original list, runtime = 0 (except unknown GPU = 1h) + runtime = min(CPU.get(cpu, 0), MEMORY.get(mem, 0), GPU_DEFAULTS.get(gpu, 1)) # Export options options = { - 'pyenv': formdata['python-env'][0], - 'nprocs': int(CPU[i_cpu].description), - 'memory': RAM[i_ram].description.replace(' ', ''), - 'runtime': f'{duration:02d}:00:00', + 'pyenv': env, + 'nprocs': cpu, + 'memory': f'{mem}GB', + 'runtime': f'{runtime:02d}:00:00', } - if i_gpu: - options['gres'] = 'gpu:' + GPU[i_gpu].description.lower() + if gpu != 'None': + options['gres'] = f'gpu:{gpu.lower()}' + + if cluster: + options['cluster'] = cluster + + if partition: + options['partition'] = partition + + if node: + options['node'] = node return options diff --git a/src/glicid_spawner/templates/static/style.css b/src/glicid_spawner/templates/static/style.css index 01bdc34..bdcb8a6 100644 --- a/src/glicid_spawner/templates/static/style.css +++ b/src/glicid_spawner/templates/static/style.css @@ -15,10 +15,29 @@ input[type=radio]:checked+label { .panel-heading .panel-title-toggle:before { font-family: 'FontAwesome'; - content: "\\f078"; + content: "\f078"; color: lightgrey; + display: inline-block; + width: 1.5rem; } .panel-heading .panel-title-toggle.collapsed:before { - content: "\\f054"; + content: "\f054"; +} + +.hidden { + display: none; +} + +.flex-container { + display: flex; + flex-wrap: wrap; + gap: 2px; +} + +.flex-item-2 { + width: 49%; +} +.flex-item-4 { + width: 24%; } diff --git a/src/glicid_spawner/templates/views/resources.jinja b/src/glicid_spawner/templates/views/resources.jinja index 1d8382a..efae401 100644 --- a/src/glicid_spawner/templates/views/resources.jinja +++ b/src/glicid_spawner/templates/views/resources.jinja @@ -3,12 +3,13 @@
- {%- for cpu in cpus -%} + {%- for cpu, max_duration in cpus.items() -%}
- -
{% endfor -%} @@ -16,14 +17,15 @@
- +
- {%- for ram in rams -%} + {%- for mem, max_duration in mems.items() -%}
- -
{% endfor -%} @@ -33,17 +35,19 @@
- {%- for gpu in gpus -%} + {%- for gpu, max_duration in gpus.items() -%}
- -
{% endfor -%}
+
@@ -59,9 +63,9 @@ $('.resources input[type=radio]').change(function () { var cpu = $('input[name=cpu]:checked').data('max-duration'); - var ram = $('input[name=ram]:checked').data('max-duration'); + var mem = $('input[name=mem]:checked').data('max-duration'); var gpu = $('input[name=gpu]:checked').data('max-duration'); - $reservations_dropdown.text(Math.min(cpu, ram, gpu)); + $reservations_dropdown.text(Math.min(cpu, mem, gpu)); }) diff --git a/src/glicid_spawner/templates/views/slurm.jinja b/src/glicid_spawner/templates/views/slurm.jinja index e6efce4..f00fb2e 100644 --- a/src/glicid_spawner/templates/views/slurm.jinja +++ b/src/glicid_spawner/templates/views/slurm.jinja @@ -1,25 +1,23 @@ -
+
-
+
-

You can specify on which cluster/partion/node you want to start your Jupyter server:

- {# {% if slurm is not 'None' %} #} -
- -
- {%- for cluster in slurm -%} -
- + {% if 'N/A' not in sinfo %} +
+ +
+ {%- for cluster in sinfo.values() -%} +
+ @@ -27,40 +25,42 @@ {% endfor -%}
- {# {% endif %} #} + {% endif %} -
- -
+
+ +
+ {%- for cluster in sinfo.values() -%} + {%- for partition in cluster -%} +
+ + +
+ {% endfor %} + {% endfor %} +
-
- -
-
- - -
-
- - -
-
- - -
-
- -
+ + + + diff --git a/tests/test_form.py b/tests/test_form.py index 48447ac..a43417b 100644 --- a/tests/test_form.py +++ b/tests/test_form.py @@ -1,14 +1,24 @@ """Test form templates module.""" import re +from pathlib import Path from glicid_spawner import form -from glicid_spawner.form import options_form, options_from_form +from glicid_spawner.form import options_attrs, options_form, options_from_form from glicid_spawner.micromamba import MicromambaEnv +from glicid_spawner.slurm import sinfo_from_file +from pytest import fixture + +DATA = Path(__file__).parent / 'data' +SINFO = sinfo_from_file(DATA / 'sinfo.txt') + +SLURM_SINGLE_CLUSTER = {'N/A': SINFO.pop('N/A')} +SLURM_MULTI_CLUSTER = SINFO -def test_options_form(monkeypatch): - """Test options form render.""" +@fixture +def mock_python_envs(monkeypatch): + """Mock python environments list.""" monkeypatch.setattr( form, 'get_envs', @@ -19,6 +29,80 @@ def test_options_form(monkeypatch): ], ) + +@fixture +def mock_cluster(monkeypatch, mock_python_envs): + """Mock multi cluster configuration (default).""" + monkeypatch.setattr(form, 'sinfo', lambda _: SLURM_MULTI_CLUSTER) + + +@fixture +def mock_single_cluster(monkeypatch, mock_python_envs): + """Mock multi cluster configuration.""" + monkeypatch.setattr(form, 'sinfo', lambda _: SLURM_SINGLE_CLUSTER) + + +def test_options_attrs(mock_cluster): + """Test form options attributes.""" + options = options_attrs('john-doe') + + assert options['username'] == 'john-doe' + + assert [env.path for env in options['envs']] == [ + '/john-doe/envs/foo', + '/john-doe/envs/bar', + '/global/envs/baz', + ] + + cpu = options['cpus'] + + assert cpu[1] == 24 + assert cpu[24] == 1 + + mem = options['mems'] + + assert mem[4] == 24 + assert mem[96] == 1 + + gpu = options['gpus'] + + assert gpu['None'] == 24 + assert gpu['A100'] == 1 + + # Multi cluster configuration (default) + sinfo = options['sinfo'] + + assert 'N/A' not in sinfo + assert 'nautilus' in sinfo + assert 'waves' in sinfo + + node = sinfo['nautilus']['gpu']['gnode1'] + + assert node == 'gnode1' + assert node.cpu.idle == 92 + assert node.gpu.name == 'A100' + + +def test_options_attrs_single_cluster(mock_single_cluster): + """Test form options attributes in single cluster configuration.""" + options = options_attrs('john-doe') + + # Single cluster configuration + sinfo = options['sinfo'] + + assert 'N/A' in sinfo + assert 'nautilus' not in sinfo + assert 'waves' not in sinfo + + node = sinfo['N/A']['Devel']['nazare001'] + + assert node == 'nazare001' + assert node.cpu.idle == 20 + assert node.gpu.name == 'None' + + +def test_options_form_resources(mock_cluster): + """Test options form render.""" html = re.sub(r'\n\s+', ' ', options_form('john-doe')) # trim line breaks # Username @@ -31,61 +115,192 @@ def test_options_form(monkeypatch): # CPU assert ( - '' + '' in html ) - assert '' in html - assert '' in html - assert '' in html + assert '' in html + assert '' in html + assert '' in html # Memory assert ( - '' + '' in html ) - assert '' in html - assert '' in html - assert '' in html + assert '' in html + assert '' in html + assert '' in html # GPU assert ( - '' + '' + in html + ) + assert '' in html + assert ( + '' in html + ) + assert '' in html + + +def test_options_form_slurm(mock_cluster): + """Test options form render.""" + html = re.sub(r'\n\s+', ' ', options_form('john-doe')) # trim line breaks + + # Cluster (multi-cluster by default) + assert '' in html + + assert '
' in html + assert '
' in html + + assert '' in html + assert ( + '' in html + ) + + # Partitions (hidden by default for multi-cluster) + assert '