'use strict'; const { URLSearchParams } = require('url'); const query = require('../conf/query.json'); const config = require('../conf/conf.json'); const HttpProxyAgent = require('http-proxy-agent'); const HttpsProxyAgent = require('https-proxy-agent'); const utils = {}; /** * getQueryUrl : build query URL from conf/query.json file */ utils.getQueryUrl = function () { const newQuery = { fl: query.fl, q: query.q, rows: query.rows, sort: query.sort, cursorMark: query.cursorMark }; const url = new URL(config.halApi + new URLSearchParams(newQuery).toString()); query.fq.forEach(fq => { url.searchParams.append('fq', fq); }); return url; }; /** * getDateYYYYMMDD : return date format YYYYMMDD * @param {Date} date */ utils.getDateYYYYMMDD = function (date) { const yyyy = date.getFullYear().toString(); const mm = (date.getMonth() + 1).toString(); // getMonth() is zero-based const dd = date.getDate().toString(); return yyyy + (mm[1] ? mm : '0' + mm[0]) + (dd[1] ? dd : '0' + dd[0]); }; /** * writeFile : write file once 'drain' to avoid backpressuring * @param {WriteStream} writer * @param {String} data: xml-tei content */ utils.writeFile = (writer, data) => { return new Promise((resolve) => { if (!writer.write(data)) { writer.once('drain', resolve); } else { resolve(); } }); }; /** * padFolderName: change folder name from '1' to '001' * @param {String} output : folder name */ utils.padFolderName = function (output) { return (new Array(3).join('0') + output.level1).slice(-3) + '/' + (new Array(3).join('0') + output.level2).slice(-3); }; /** * getScrollUrl: build scroll url from last returned cursorMark * @param {String} halQueryUrl * @param {String} cursorMark */ utils.getScrollUrl = function (halQueryUrl, cursorMark) { const href = halQueryUrl.href.split('?'); const urlBase = href[0]; const sp = new URLSearchParams(href[1]); sp.delete('cursorMark'); sp.append('cursorMark', cursorMark); return `${urlBase}/?${sp.toString()}`; }; utils.getHarvestPath = function (path) { return (process.env.CORPUSES_ROOT ? process.env.CORPUSES_ROOT + '/' + path : path); }; utils.getProxyAgent = function () { // Get proxy env vars const httpProxy = process.env.HTTP_PROXY ? process.env.HTTP_PROXY : process.env.http_proxy; const httpsProxy = process.env.HTTPS_PROXY ? process.env.HTTPS_PROXY : process.env.https_proxy; if (httpProxy) { return new HttpProxyAgent(httpProxy); } else if (httpsProxy) { return new HttpsProxyAgent(httpsProxy); } else return false; }; module.exports = utils;