diff --git a/data-calculation/v1/README.md b/data-calculation/v1/README.md new file mode 100644 index 0000000..822e225 --- /dev/null +++ b/data-calculation/v1/README.md @@ -0,0 +1,11 @@ +### usage + +```bash +# Send data for batch processing +cat input.tar.gz |curl --data-binary @- -H "X-Hook: https://webhook.site/dce2fefa-9a72-4f76-96e5-059405a04f6c" "http://localhost:31976/v1/tree-segment" > output.json + +# When the corpus is processed, get the result +cat output.json |curl --data-binary @- "http://localhost:31976/v1/retrieve" > output.tar.gz + + ``` + diff --git a/data-calculation/v1/retrieve.ini b/data-calculation/v1/retrieve.ini new file mode 100644 index 0000000..1259407 --- /dev/null +++ b/data-calculation/v1/retrieve.ini @@ -0,0 +1,23 @@ +[use] +plugin = basics +plugin = analytics + +[JSONParse] +separator = * + +[env] +path = manifest.identifier +value = get('value') + +[exchange] +value = get('value') + +[FILELoad] +location = /tmp + +[unpack] + +[TARDump] +compress = true +manifest = env('manifest') +manifest = fix({description: 'Exemple de fichier batch pour traitement asynchrone'}) diff --git a/data-calculation/v1/tree-segment.ini b/data-calculation/v1/tree-segment.ini new file mode 100644 index 0000000..95d0cda --- /dev/null +++ b/data-calculation/v1/tree-segment.ini @@ -0,0 +1,63 @@ +[use] +plugin = basics +plugin = analytics + +[TARExtract] +compress = true + +# Création d'un identifiant unique pour le corpus reçu +[singleton] +[singleton/identify] +[singleton/env] +path = identifier +value = get('uri').replace('uid:/', '') + +# Traitement asynchnrone de tous les items du corpus +[fork] +standalone = true + +# Calcul sur le coprus +[fork/replace] +path = id +value = get('id') +path = value +value = get('value').castArray().map((item) => [].concat(item).reduce((pre, cur, idx, src) => (pre ? [ ...pre, [src[idx-1],cur]] : []), false)).flatten() + + +# Sauvegarde le résultat produit +[fork/FILESave] +location = /tmp +identifier = env('identifier') +jsonl = true + +# Signal le fin du traitment via un appel à un webhook (si il a été précisé) +[fork/swing] +test = env('headers.x-hook').startsWith('http') +[fork/swing/replace] +path = body +value = self().pick(['size', 'atime', 'mtime', 'ctime']).set('identifier', env('identifier')).set('state', 'ready') + + +[fork/swing/URLFetch] +url = env('headers.x-hook') +path = body +headers = Content-Type:application/json + +[fork/swing/debug] +text = webhook triggered + +[fork/debug] +text = process completed + + +# Retour immédiat d'un seul élement indiquant command récupére le corpus quand il sera pret +[shift] +[replace] +path = id +value = http://localhost:31976/retrieve +path = value +value = env('identifier') + +[JSONString] +indent = env('indent') +