diff --git a/conf/conf.json b/conf/conf.json new file mode 100644 index 0000000..6cdc651 --- /dev/null +++ b/conf/conf.json @@ -0,0 +1,5 @@ +{ + "halApi":"https://api.archives-ouvertes.fr/search?", + "proxy":"https://proxyout.inist.fr:8080", + "max_retry":2 +} \ No newline at end of file diff --git a/index.js b/index.js index 54096c1..9e097a2 100644 --- a/index.js +++ b/index.js @@ -3,7 +3,10 @@ const fetch = require('node-fetch'); const fs = require('graceful-fs'); const xmlFormatter = require('xml-formatter'); +const HttpsProxyAgent = require('https-proxy-agent'); + const utils = require('./lib/utils.js'); +const config = require('./conf/conf.json'); const business = {}; @@ -66,8 +69,16 @@ */ function requestByQuery (docObject, query, cursorMark, retryCount, cb) { const target = cursorMark ? utils.getScrollUrl(query, cursorMark) : query.href; + const fetchOpts = { + headers: { + 'Content-Type': 'application/json' + } + }; + if (config.proxy !== '') { + fetchOpts.headers.agent = new HttpsProxyAgent(config.proxy); + } return fetch(target, { - headers: { 'Content-Type': 'application/json' } + fetchOpts }).then((res) => res.json()).then(async function (res) { const nextCursorMark = res.nextCursorMark; const docs = res.response.docs; @@ -114,7 +125,7 @@ requestByQuery(docObject, query, nextCursorMark, 0, cb); } }).catch(function (err) { - if (retryCount < 2) { + if (retryCount < config.max_retry) { retryCount++; requestByQuery(docObject, query, cursorMark, retryCount, cb); } else { diff --git a/lib/utils.js b/lib/utils.js index c4277f1..b419668 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -2,6 +2,7 @@ const { URLSearchParams } = require('url'); const query = require('../conf/query.json'); +const config = require('../conf/conf.json'); const utils = {}; /** @@ -15,7 +16,7 @@ sort: query.sort, cursorMark: query.cursorMark }; - const url = new URL('http://api.archives-ouvertes.fr/search?' + new URLSearchParams(newQuery).toString()); + const url = new URL(config.halApi + new URLSearchParams(newQuery).toString()); query.fq.forEach(fq => { url.searchParams.append('fq', fq); }); diff --git a/package.json b/package.json index c01f4b2..47b1085 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "dependencies": { "chai": "^3.4.0", "graceful-fs": "^4.2.8", + "https-proxy-agent": "^5.0.0", "mocha": "^2.3.3" }, "devDependencies": {