Homec4science
Diffusion scitas-job-submit (ncoudene) Browse

scitas-job-submit/scripts/elk_slurm_jobacct_async_pyslurm.pipeline
b358d0ed0a9cncoudene

Last Change
Raw File

elk_slurm_jobacct_async_pyslurm.pipeline
View Options

PUT _ingest/pipeline/slurm_squeue
{
"version": 1,
"description": "Add two properties :\n- cost_m : based on tres/billing req or alloc and time_limit\n- cost_chf\n",
"processors": [
{
"script": {
"source": "def req = ctx['tres_req'];\ndef alloc = ctx['tres_alloc'];\ndef max_time = ctx['time_limit'];\ndef bill = 0;\nif(alloc != null){\n bill = alloc['billing']\n} else if(req != null){\n bill = req['billing']\n} else {\n bill = 0\n}\n\nctx['cost_m'] = max_time*bill/60.0;",
"description": "add cost_m (hours)",
"on_failure": [
{
"append": {
"field": "ingest_status",
"value": [
"add cost_m"
]
}
}
]
}
},
{
"enrich": {
"field": "account",
"policy_name": "slurm_account_pricing_enrich",
"target_field": "temp_pricing",
"description": "link account w/ pricing",
"on_failure": [
{
"append": {
"field": "ingest_status",
"value": [
"link account w/ pricing"
]
}
}
]
}
},
{
"script": {
"source": "ctx ['pricing'] = ctx['temp_pricing']['pricing'];\nctx['account_pricing'] = ctx['pricing'];",
"description": "add account_pricing",
"on_failure": [
{
"script": {
"source": "ctx['pricing'] = 'U0';",
"description": "if accout not existent then pricing = U0"
}
}
]
}
},
{
"enrich": {
"field": "pricing",
"policy_name": "slurm_pricing_cost_enrich",
"target_field": "temp_cost",
"description": "link pricing w/ cost_chf_multi by cluster",
"on_failure": [
{
"append": {
"field": "ingest_status",
"value": [
"link pricing w/ cost_chf_multi by cluster"
]
}
}
]
}
},
{
"script": {
"source": "def cost_chf_m = 0;\ndef cluster = ctx['cluster'];\ndef temp = ctx['temp_cost']['clusters'][cluster];\nif (temp != null) {\n cost_chf_m = temp;\n}\ndef cost = cost_chf_m*ctx['cost_m'];\nctx['cost_chf'] = Math.round(cost*1000)/1000.0;\nctx['account_pricing'] = ctx['pricing'];",
"description": "add cost_chf",
"on_failure": [
{
"append": {
"field": "ingest_status",
"value": [
"add cost_chf"
]
}
}
]
}
},
{
"remove": {
"field": [
"temp_cost",
"temp_pricing",
"pricing"
],
"ignore_failure": true,
"description": "remove temporary properties"
}
}
]
}
c4science · Help