Newer
Older
li-harvest-hal / lib / utils.js
'use strict';

const { URLSearchParams } = require('url');
const query = require('../config/query.json');
const config = require('../config/config');
const { getProxyHttpAgent } = require('proxy-http-agent');
const utils = {};

/**
 * getQueryUrl : build query URL from config/query.json file
 */
utils.getQueryUrl = function () {
  const newQuery = {
    fl: query.fl,
    q: query.q,
    rows: query.rows,
    sort: query.sort,
    cursorMark: query.cursorMark
  };
  const url = new URL(config.halApi + new URLSearchParams(newQuery).toString());
  query.fq.forEach(fq => {
    url.searchParams.append('fq', fq);
  });
  return url;
};

/**
 * getDateYYYYMMDD : return date format YYYYMMDD
 * @param  {Date} date
 */
utils.getDateYYYYMMDD = function (date) {
  const yyyy = date.getFullYear().toString();
  const mm = (date.getMonth() + 1).toString(); // getMonth() is zero-based
  const dd = date.getDate().toString();
  return yyyy + (mm[1] ? mm : '0' + mm[0]) + (dd[1] ? dd : '0' + dd[0]);
};

/**
 * writeFile : write file once 'drain' to avoid backpressuring
 * @param  {WriteStream} writer
 * @param  {String} data: xml-tei content
 */
utils.writeFile = (writer, data) => {
  return new Promise((resolve) => {
    if (!writer.write(data)) {
      writer.once('drain', resolve);
    } else {
      resolve();
    }
  });
};

/**
 * padFolderName: change folder name from '1' to '001'
 * @param  {String} output : folder name
 */
utils.padFolderName = function (output) {
  return (new Array(3).join('0') + output.level1).slice(-3) + '/' + (new Array(3).join('0') + output.level2).slice(-3);
};

/**
 * getScrollUrl: build scroll url from last returned cursorMark
 * @param  {String} halQueryUrl
 * @param  {String} cursorMark
 */
utils.getScrollUrl = function (halQueryUrl, cursorMark) {
  const href = halQueryUrl.href.split('?');
  const urlBase = href[0];
  const sp = new URLSearchParams(href[1]);
  sp.delete('cursorMark');
  sp.append('cursorMark', cursorMark);
  return `${urlBase}/?${sp.toString()}`;
};

utils.getProxyAgent = function () {
  // Get proxy env vars
  const httpProxy = process.env.HTTP_PROXY ? process.env.HTTP_PROXY : process.env.http_proxy;
  const httpsProxy = process.env.HTTPS_PROXY ? process.env.HTTPS_PROXY : process.env.https_proxy;

  if (httpProxy) {
    return getProxyHttpAgent({
      proxy: httpProxy,
      rejectUnauthorized: false
    });
  } else if (httpsProxy) {
    return getProxyHttpAgent({
      proxy: httpsProxy,
      rejectUnauthorized: false
    });
  } else return false;
};

module.exports = utils;