diff --git a/Makefile b/Makefile index 03612f2..f44fcc4 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ python: ## Launch PYTHON daemon for one directory, eg make python ./domains-classifier WORKING_DIR="$(filter-out $@,$(MAKECMDGOALS))" docker-compose up --no-deps --force-recreate python -nodejs: ## Launch NODEJS daemon for one directory, eg make nodejs./base-line +nodejs: ## Launch NODEJS daemon for one directory, eg make nodejs ./base-line WORKING_DIR="$(filter-out $@,$(MAKECMDGOALS))" docker-compose up --no-deps --force-recreate nodejs reset: ## Reset files permissions for all directory diff --git a/base-line-nodejs/examples.http b/base-line-nodejs/examples.http new file mode 100755 index 0000000..82f655a --- /dev/null +++ b/base-line-nodejs/examples.http @@ -0,0 +1,13 @@ +# These examples can be used directly in VSCode, using REST Client extension (humao.rest-client) + +# Parse un flux JSON, et le returne sans modification +POST https://base-line.services.inist.fr/v1/echo/json?indent=true HTTP/1.1 +Content-Type: application/json + +[ + { "value": "à l'école" }, + { "value": "où" } +] +### + + diff --git a/base-line-nodejs/index.js b/base-line-nodejs/index.js new file mode 100755 index 0000000..58d13b3 --- /dev/null +++ b/base-line-nodejs/index.js @@ -0,0 +1,15 @@ +var accents = require('remove-accents'); + +const dropAccent = (data, feed, ctx) => { + if (ctx.isLast()) { + return feed.close(); + } + const defval = ctx.getParam('valeurParDefaut', 'n/a') + const curval = data.value || defval; + data.value = accents.remove(curval); + return feed.send(data); +}; + +exports.default = { + dropAccent, +}; diff --git a/base-line-nodejs/package.json b/base-line-nodejs/package.json new file mode 100755 index 0000000..0986261 --- /dev/null +++ b/base-line-nodejs/package.json @@ -0,0 +1,8 @@ +{ + "name": "@ezs/local", + "version": "1.0.0", + "main": "index.js", + "dependencies": { + "remove-accents": "^0.4.2" + } +} diff --git a/base-line-nodejs/v1/no-accent.ini b/base-line-nodejs/v1/no-accent.ini new file mode 100755 index 0000000..6d1c649 --- /dev/null +++ b/base-line-nodejs/v1/no-accent.ini @@ -0,0 +1,33 @@ +# OpenAPI Documentation - JSON format (dot notation) +mimeType = application/json + +post.operationId = post-v1-no-accent +post.description = Exemple d'utilisatino d'une fonction adhoc +post.summary = Récupération à partir d'un tableau d'objets avec à minima un champ value contenant une chaine de caractère +post.tags.0 = base-line-nodejs +post.requestBody.content.application/json.example.0.id = 1 +post.requestBody.content.application/json.example.0.value = école +post.requestBody.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.requestBody.required = true +post.responses.default.content.application/json.example.0.id = 1 +post.responses.default.content.application/json.example.0.value = ecole +post.responses.default.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.responses.default.description = Les accents dans le champ value sont supprimés +post.parameters.0.description = Indenter le JSON résultant +post.parameters.0.in = query +post.parameters.0.name = indent +post.parameters.0.schema.type = boolean + +[use] +plugin = @ezs/basics +plugin = @ezs/local + +[JSONParse] +separator = * + +[dropAccent] +defValue = Pas de valeur + +[dump] +indent = env('indent') + diff --git a/base-line-nodejs/v1/no-accent2.ini b/base-line-nodejs/v1/no-accent2.ini new file mode 100755 index 0000000..856be80 --- /dev/null +++ b/base-line-nodejs/v1/no-accent2.ini @@ -0,0 +1,41 @@ +# OpenAPI Documentation - JSON format (dot notation) +mimeType = application/json + +post.operationId = post-v1-no-accent +post.description = Exemple d'utilisatino d'une fonction adhoc +post.summary = Récupération à partir d'un tableau d'objets avec à minima un champ value contenant une chaine de caractère +post.tags.0 = base-line-nodejs +post.requestBody.content.application/json.example.0.id = 1 +post.requestBody.content.application/json.example.0.value = école +post.requestBody.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.requestBody.required = true +post.responses.default.content.application/json.example.0.id = 1 +post.responses.default.content.application/json.example.0.value = ecole +post.responses.default.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.responses.default.description = Les accents dans le champ value sont supprimés +post.parameters.0.description = Indenter le JSON résultant +post.parameters.0.in = query +post.parameters.0.name = indent +post.parameters.0.schema.type = boolean + +[use] +plugin = @ezs/basics +plugin = @ezs/analytics +plugin = @ezs/local + +[JSONParse] +separator = * + +# expand is the easiest way to optimize your script. It allows to parallelize batch processing +[expand] +path = value +size = 100 +# in production mode, uncomment the following line +# cache = boost + +[expand/dropAccent] +defValue = Pas de valeur + +[dump] +indent = env('indent') + diff --git a/base-line-python/examples.http b/base-line-python/examples.http new file mode 100755 index 0000000..516e186 --- /dev/null +++ b/base-line-python/examples.http @@ -0,0 +1,11 @@ +# These examples can be used directly in VSCode, using REST Client extension (humao.rest-client) + +# Parse un flux JSON, et le returne sans modification +POST http://localhost:31976/v1/no-accent?indent=true HTTP/1.1 +Content-Type: application/json + +[ + { "value": "é" }, + { "value": "à" } +] +### diff --git a/base-line-python/requirements.txt b/base-line-python/requirements.txt new file mode 100755 index 0000000..051b14c --- /dev/null +++ b/base-line-python/requirements.txt @@ -0,0 +1 @@ +unidecode diff --git a/base-line-python/v1/no-accent.ini b/base-line-python/v1/no-accent.ini new file mode 100755 index 0000000..df12654 --- /dev/null +++ b/base-line-python/v1/no-accent.ini @@ -0,0 +1,35 @@ +# OpenAPI Documentation - JSON format (dot notation) +mimeType = application/json + +post.operationId = post-v1-no-accent +post.description = Exemple d'utilisation d'une fonction adhoc +post.summary = Récupération à partir d'un tableau d'objets avec à minima un champ value contenant une chaine de caractère +post.tags.0 = base-line-python +post.requestBody.content.application/json.example.0.id = 1 +post.requestBody.content.application/json.example.0.value = école +post.requestBody.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.requestBody.required = true +post.responses.default.content.application/json.example.0.id = 1 +post.responses.default.content.application/json.example.0.value = ecole +post.responses.default.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.responses.default.description = Les accents dans le champ value sont supprimés +post.parameters.0.description = Indenter le JSON résultant +post.parameters.0.in = query +post.parameters.0.name = indent +post.parameters.0.schema.type = boolean + +[use] +plugin = @ezs/spawn +plugin = @ezs/analytics +plugin = @ezs/basics + +[JSONParse] +separator = * + +[exec] +# command should be executable ! +command = ./v1/no-accent.py + +[JSONString] +indent = env('indent') + diff --git a/base-line-python/v1/no-accent.py b/base-line-python/v1/no-accent.py new file mode 100755 index 0000000..cae593c --- /dev/null +++ b/base-line-python/v1/no-accent.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import json +import sys +from unidecode import unidecode + +for line in sys.stdin: + data=json.loads(line) + data['value']=unidecode(data['value']) + sys.stdout.write(json.dumps(data)) + sys.stdout.write('\n') + diff --git a/base-line-python/v1/no-accent2.ini b/base-line-python/v1/no-accent2.ini new file mode 100755 index 0000000..d207c2d --- /dev/null +++ b/base-line-python/v1/no-accent2.ini @@ -0,0 +1,42 @@ +# OpenAPI Documentation - JSON format (dot notation) +mimeType = application/json + +post.operationId = post-v1-no-accent +post.description = Exemple d'utilisation d'une fonction adhoc +post.summary = Récupération à partir d'un tableau d'objets avec à minima un champ value contenant une chaine de caractère +post.tags.0 = base-line-python +post.requestBody.content.application/json.example.0.id = 1 +post.requestBody.content.application/json.example.0.value = école +post.requestBody.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.requestBody.required = true +post.responses.default.content.application/json.example.0.id = 1 +post.responses.default.content.application/json.example.0.value = ecole +post.responses.default.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.responses.default.description = Les accents dans le champ value sont supprimés +post.parameters.0.description = Indenter le JSON résultant +post.parameters.0.in = query +post.parameters.0.name = indent +post.parameters.0.schema.type = boolean + +[use] +plugin = @ezs/spawn +plugin = @ezs/analytics +plugin = @ezs/basics + +[JSONParse] +separator = * + +# expand is the easiest way to optimize your script. It allows to parallelize batch processing +[expand] +path = value +size = 100 +# in production mode, uncomment the following line +# cache = boost + +[expand/exec] +# command should be executable ! +command = ./v1/no-accent.py + +[JSONString] +indent = env('indent') + diff --git a/base-line/v1/no-accent.ini b/base-line/v1/no-accent.ini new file mode 100644 index 0000000..bd0e63e --- /dev/null +++ b/base-line/v1/no-accent.ini @@ -0,0 +1,33 @@ +# OpenAPI Documentation - JSON format (dot notation) +mimeType = application/json + +post.operationId = post-v1-no-accent +post.description = Exemple d'utilisatino d'une fonction adhoc +post.summary = Récupération à partir d'un tableau d'objets avec à minima un champ value contenant une chaine de caractère +post.tags.0 = base-line +post.requestBody.content.application/json.example.0.id = 1 +post.requestBody.content.application/json.example.0.value = école +post.requestBody.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.requestBody.required = true +post.responses.default.content.application/json.example.0.id = 1 +post.responses.default.content.application/json.example.0.value = ecole +post.responses.default.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.responses.default.description = Les accents dans le champ value sont supprimés +post.parameters.0.description = Indenter le JSON résultant +post.parameters.0.in = query +post.parameters.0.name = indent +post.parameters.0.schema.type = boolean + +[use] +plugin = @ezs/basics + +[JSONParse] +separator = * + +[assign] +path = value +value = get('value').deburr() + +[dump] +indent = env('indent') + diff --git a/base-line/v1/no-accent2.ini b/base-line/v1/no-accent2.ini new file mode 100644 index 0000000..46a95ae --- /dev/null +++ b/base-line/v1/no-accent2.ini @@ -0,0 +1,40 @@ +# OpenAPI Documentation - JSON format (dot notation) +mimeType = application/json + +post.operationId = post-v1-no-accent +post.description = Exemple d'utilisatino d'une fonction adhoc +post.summary = Récupération à partir d'un tableau d'objets avec à minima un champ value contenant une chaine de caractère +post.tags.0 = base-line +post.requestBody.content.application/json.example.0.id = 1 +post.requestBody.content.application/json.example.0.value = école +post.requestBody.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.requestBody.required = true +post.responses.default.content.application/json.example.0.id = 1 +post.responses.default.content.application/json.example.0.value = ecole +post.responses.default.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.responses.default.description = Les accents dans le champ value sont supprimés +post.parameters.0.description = Indenter le JSON résultant +post.parameters.0.in = query +post.parameters.0.name = indent +post.parameters.0.schema.type = boolean + +[use] +plugin = @ezs/basics + +[JSONParse] +separator = * + +# expand is the easiest way to optimize your script. It allows to parallelize batch processing +[expand] +path = value +size = 100 +# in production mode, uncomment the following line +# cache = boost + +[expand/assign] +path = value +value = get('value').deburr() + +[dump] +indent = env('indent') + diff --git a/biblio-tools/examples.http b/biblio-tools/examples.http index b8c8dc7..38fbefc 100644 --- a/biblio-tools/examples.http +++ b/biblio-tools/examples.http @@ -117,3 +117,32 @@ ] ### +# wos/fecth +POST http://localhost:31976/v1/wos/fetch?indent=true HTTP/1.1 +Content-Type: application/json + +[ + { + "query": "0", + "value": "XX" + }, + { + "query" :"I", + "value": { + "databaseId": "WOK", + "usrQuery": "TS=(cadmium)", + "count": "5", + "firstRecord": "1" + } + }, + { + "query" :"II", + "value": { + "databaseId": "WOK", + "usrQuery": "TS=(cadmium)", + "count": "5", + "firstRecord": "6" + } + } +] +### diff --git a/biblio-tools/swagger.json b/biblio-tools/swagger.json index aa0aa70..c5ee886 100755 --- a/biblio-tools/swagger.json +++ b/biblio-tools/swagger.json @@ -1,6 +1,6 @@ { "info": { - "version": "1.3.4" + "version": "1.3.6" }, "tags": [ { diff --git a/biblio-tools/v1/unpaywall/corhal.ini b/biblio-tools/v1/unpaywall/corhal.ini index c157be2..8ed030e 100644 --- a/biblio-tools/v1/unpaywall/corhal.ini +++ b/biblio-tools/v1/unpaywall/corhal.ini @@ -39,6 +39,9 @@ path = token value = env('UNPAYWALL_API_KEY') +[assign] +path = value +value = get('value').toLower() # Pour traiter par lot on groupe puis on utilise spawn qui crée un buffer par lot [group] @@ -85,7 +88,7 @@ # On créé l'object attendu par combine { id, value } pour faire le mapping [spawn/combine/replace] path = id -value = get('doi') +value = get('doi').toLower() path = value.is_oa value = get('is_oa') diff --git a/biblio-tools/v1/unpaywall/expand.ini b/biblio-tools/v1/unpaywall/expand.ini index fc7ebaa..2d92f34 100644 --- a/biblio-tools/v1/unpaywall/expand.ini +++ b/biblio-tools/v1/unpaywall/expand.ini @@ -39,6 +39,9 @@ path = token value = env('UNPAYWALL_API_KEY') +[assign] +path = value +value = get('value').toLower() # Pour traiter par lot on groupe puis on utilise spawn qui crée un buffer par lot [group] @@ -84,7 +87,7 @@ # On créé l'object attendu par combine { id, value } pour faire le mapping [spawn/combine/replace] path = id -value = get('doi') +value = get('doi').toLower() path = value.is_oa value = get('is_oa') @@ -98,7 +101,6 @@ path = value.has_repository_copy value = get('has_repository_copy', 'n/a') - # On nettoie l'objet en supprimant les champs temporaires [spawn/exchange] value = self().omit('bufferID') @@ -107,7 +109,6 @@ [assign] path = value value = get('value.value', self.value) - # On marque les réponses qui n'ont pas abouties [swing] test = has('value.is_oa') diff --git a/biblio-tools/v1/unpaywall/is_oa.ini b/biblio-tools/v1/unpaywall/is_oa.ini index 12cf392..af83306 100644 --- a/biblio-tools/v1/unpaywall/is_oa.ini +++ b/biblio-tools/v1/unpaywall/is_oa.ini @@ -39,6 +39,9 @@ path = token value = env('UNPAYWALL_API_KEY') +[assign] +path = value +value = get('value').toLower() # Pour traiter par lot on groupe puis on utilise spawn qui crée un buffer par lot [group] @@ -84,7 +87,7 @@ # On créé l'object attendu par combine { id, value } pour faire le mapping [spawn/combine/replace] path = id -value = get('doi') +value = get('doi').toLower() path = value value = get('is_oa', 'n/a') diff --git a/biblio-tools/v1/wos/fetch.ini b/biblio-tools/v1/wos/fetch.ini new file mode 100644 index 0000000..cb293f9 --- /dev/null +++ b/biblio-tools/v1/wos/fetch.ini @@ -0,0 +1,75 @@ +# OpenAPI Documentation - JSON format (dot notation) +mimeType = application/json + +post.operationId = post-v1-wos-search +post.description = Recherche et récuperation dans le Web of science +post.summary = Récupération de notices WOS à partir d'un tableau de requetes +post.tags.0 = biblio-tools +post.requestBody.content.application/json.example.0.value.databaseId = WOK +post.requestBody.content.application/json.example.0.value.usrQuery = TS=(cadmium) +post.requestBody.content.application/json.example.0.value.count = 5 +post.requestBody.content.application/json.example.0.value.firstRecord = 1 +post.requestBody.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.requestBody.required = true +post.parameters.0.description = Indenter le JSON résultant +post.parameters.0.in = query +post.parameters.0.name = indent +post.parameters.0.schema.type = boolean + +# Necessary plugins +[use] +plugin = @ezs/basics +plugin = @ezs/analytics + +# Flow configuration +[JSONParse] +separator = * + +[env] +path = indent +value = true + +path = token +value = env('WOS_API_KEY') + +[expand] +path = value +size = 1 +cacheName = wos-fetch + +[expand/URLFetch] +path = value +target = value +url = https://wos-api.clarivate.com/api/wos/ +json = true +timeout = 60000 +noerror = true +header = accept: application/json +header = env('token').prepend('X-ApiKey:') +header = Content-Type: application/json + +[expand/replace] +path = id +value = get('id') +path = value +value = get('value.Data.Records.records.REC', 'n/a') + +[replace] +path = id +value = self().omit('value') +path = value +value = get('value', 'n/a') + +[exploding] +id = id +value = value + +[assign] +path = id.value +value = get('value') + +[exchange] +value = get('id') + +[dump] +indent = env('indent', false) diff --git a/docker-compose.yml b/docker-compose.yml index 6617ecc..bba8bdd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: shell: - image: inistcnrs/lodex-workers-python:3.0.6 + image: inistcnrs/lodex-workers-python:4.0.10 environment: - http_proxy - https_proxy @@ -9,7 +9,7 @@ - .:/app command: bash python: - image: inistcnrs/lodex-workers-python:3.0.6 + image: inistcnrs/lodex-workers-python:4.0.10 environment: - http_proxy - https_proxy @@ -18,7 +18,7 @@ volumes: - ./${WORKING_DIR}:/app/public nodejs: - image: inistcnrs/lodex-workers:9.0.5 + image: inistcnrs/lodex-workers:9.2.1 environment: - http_proxy - https_proxy