diff --git a/data-computer/v1/base-line.ini b/data-computer/v1/base-line.ini index ecdf3db..a7814a5 100644 --- a/data-computer/v1/base-line.ini +++ b/data-computer/v1/base-line.ini @@ -5,7 +5,7 @@ post.operationId = post-v1-base-line post.description = Chargement et analyse d'un fichier corpus post.summary = Le corpus est analysé et restitué sans modification des données -post.tags.0 = data-computer +post.tags.0 = data-workflow post.requestBody.content.application/x-tar.schema.type = string post.requestBody.content.application/x-tar.schema.format = binary post.requestBody.required = true @@ -31,10 +31,6 @@ path = generator value = base-line -[use] -plugin = basics -plugin = analytics - # Step 1 (générique): Charger le fichier corpus [delegate] file = charger.cfg @@ -44,6 +40,10 @@ standalone = true logger = logger.cfg +# Step 2.0 (optionnel): Accélére le détachement du fork si l'enrichissement est lent +[fork/delegate] +file = buffer.cfg + # Step 2.1 (spécifique): Lancer un calcul sur tous les items reçus [fork/exchange] value = self().omit('uri') diff --git a/data-computer/v1/buffer.cfg b/data-computer/v1/buffer.cfg new file mode 100644 index 0000000..f8b16b5 --- /dev/null +++ b/data-computer/v1/buffer.cfg @@ -0,0 +1,26 @@ +[use] +plugin = basics + +# On sauvegarde sur disque pour accepter rapidement tous les objets en entrée +# et répondre rapidement au client que le traitmenent asynchnrone est lancé. +# +# Le "fork" se détache uniquement quand tous les objets sont "rentrés" dans le fork +# Si le traitement est plus lent que la sauvegarde sur disque +# il est nécessaire de créer un fichier temporaire +[pack] +[FILESave] +identifier = env('identifier') +location = /tmp/upload +compress = true + +[exchange] +value = get('filename') + +[FILELoad] +compress = true +location = /tmp/upload +[unpack] + +[metrics] +bucket = buffer + diff --git a/data-computer/v1/charger.cfg b/data-computer/v1/charger.cfg index c1f8093..9250763 100644 --- a/data-computer/v1/charger.cfg +++ b/data-computer/v1/charger.cfg @@ -14,3 +14,5 @@ path = identifier value = get('uri').replace('uid:/', '') +[metrics] +bucket = charger diff --git a/data-computer/v1/logger.cfg b/data-computer/v1/logger.cfg index b46e88a..6377dd9 100644 --- a/data-computer/v1/logger.cfg +++ b/data-computer/v1/logger.cfg @@ -2,6 +2,9 @@ plugin = basics plugin = analytics +[metrics] +bucket = logger + # On ne garde que la première erreur déclénchée [shift] diff --git a/data-computer/v1/recorder.cfg b/data-computer/v1/recorder.cfg index f768491..7811144 100644 --- a/data-computer/v1/recorder.cfg +++ b/data-computer/v1/recorder.cfg @@ -2,6 +2,9 @@ plugin = basics plugin = analytics +[metrics] +bucket = recorder + # Step 2.2 (générique): Création d'un fichier résulat standard [TARDump] compress = true @@ -34,10 +37,10 @@ # Step 2.4.3 (faculatif) : Ajouter une trace dans log [swing/debug] -text = webhook triggered +text = WebHook triggered # Step 2.5 (faculatif) : Ajouter une trace dans log [debug] -text = process completed +text = Process completed diff --git a/data-workflow/v1/base-line.ini b/data-workflow/v1/base-line.ini index e3bb2ff..a7814a5 100644 --- a/data-workflow/v1/base-line.ini +++ b/data-workflow/v1/base-line.ini @@ -31,10 +31,6 @@ path = generator value = base-line -[use] -plugin = basics -plugin = analytics - # Step 1 (générique): Charger le fichier corpus [delegate] file = charger.cfg @@ -44,6 +40,10 @@ standalone = true logger = logger.cfg +# Step 2.0 (optionnel): Accélére le détachement du fork si l'enrichissement est lent +[fork/delegate] +file = buffer.cfg + # Step 2.1 (spécifique): Lancer un calcul sur tous les items reçus [fork/exchange] value = self().omit('uri') diff --git a/data-workflow/v1/buffer.cfg b/data-workflow/v1/buffer.cfg index a56c958..f8b16b5 100644 --- a/data-workflow/v1/buffer.cfg +++ b/data-workflow/v1/buffer.cfg @@ -21,4 +21,6 @@ location = /tmp/upload [unpack] +[metrics] +bucket = buffer diff --git a/data-workflow/v1/charger.cfg b/data-workflow/v1/charger.cfg index c1f8093..9250763 100644 --- a/data-workflow/v1/charger.cfg +++ b/data-workflow/v1/charger.cfg @@ -14,3 +14,5 @@ path = identifier value = get('uri').replace('uid:/', '') +[metrics] +bucket = charger diff --git a/data-workflow/v1/conditormetrie.cfg b/data-workflow/v1/conditormetrie.cfg index 1a0d9ac..401058a 100644 --- a/data-workflow/v1/conditormetrie.cfg +++ b/data-workflow/v1/conditormetrie.cfg @@ -6,7 +6,6 @@ path = number2labelDR value = fix({"01": "DR01 Ile-de-France Villejuif","02": "DR02 Paris-Centre","04": "DR04 Ile-de-France Gif-sur-Yvette","05": "DR05 Ile-de-France Meudon","16": "DR16 Paris-Normandie","06": "DR06 Centre Est","10": "DR10 Alsace","08": "DR08 Centre Limousin Poitou Charente","17": "DR17 Bretagne et Pays de la Loire","18": "DR18 Hauts-de-France","07": "DR07 Rhône Auvergne","11": "DR11 Alpes","12": "DR12 Provence et Corse","20": "DR20 Côte d'Azur","13": "DR13 Occitanie Est","14": "DR14 Occitanie Ouest","15": "DR15 Aquitaine"}) - [assign] # Récupère electronicPublicationDate et publicationDate # Prend la plus ancienne (= la plus petite) diff --git a/data-workflow/v1/conditormetrie.ini b/data-workflow/v1/conditormetrie.ini index 65fc99c..6f64081 100644 --- a/data-workflow/v1/conditormetrie.ini +++ b/data-workflow/v1/conditormetrie.ini @@ -31,13 +31,6 @@ path = generator value = conditormetrie -path = language -value = en - -[use] -plugin = basics -plugin = analytics - # Step 1 (générique): Charger le fichier corpus [delegate] file = charger.cfg diff --git a/data-workflow/v1/logger.cfg b/data-workflow/v1/logger.cfg index b46e88a..6377dd9 100644 --- a/data-workflow/v1/logger.cfg +++ b/data-workflow/v1/logger.cfg @@ -2,6 +2,9 @@ plugin = basics plugin = analytics +[metrics] +bucket = logger + # On ne garde que la première erreur déclénchée [shift] diff --git a/data-workflow/v1/recorder.cfg b/data-workflow/v1/recorder.cfg index 66e4702..7811144 100644 --- a/data-workflow/v1/recorder.cfg +++ b/data-workflow/v1/recorder.cfg @@ -2,6 +2,9 @@ plugin = basics plugin = analytics +[metrics] +bucket = recorder + # Step 2.2 (générique): Création d'un fichier résulat standard [TARDump] compress = true