diff --git a/index.js b/index.js index d3643d5..7e2ac4e 100644 --- a/index.js +++ b/index.js @@ -2,19 +2,14 @@ const fetch = require('node-fetch'); const fs = require('graceful-fs'); -const path= require('path'); -const xmlFormatter = require('xml-formatter'); - +const path = require('path'); const utils = require('./lib/utils'); const config = require('./config/config'); const business = {}; - let outputPath, harvestPath; - let processedFileCount = 0; let writtenFilesCount = 0; - const output = { level1: 1, level2: 1 @@ -39,7 +34,7 @@ 'to', utils.getDateYYYYMMDD(new Date(dateEnd)) ].filter(Boolean).join('-'); - + harvestPath = path.join(process.env.CORPUSES_ROOT, 'hal', folderName); if (fs.existsSync(harvestPath)) { @@ -120,8 +115,7 @@ fs.mkdirSync(outputPath, { recursive: true }); } const writeStream = fs.createWriteStream(outputPath + '/' + docs[i].halId_s + '.xml', { flags: 'a' }); - const xmlContent = xmlFormatter(docs[i].label_xml); - await utils.writeFile(writeStream, xmlContent); + await utils.writeFile(writeStream, docs[i].label_xml); writeStream.end(); writeStream.on('finish', function () { writtenFilesCount++; @@ -135,7 +129,7 @@ requestByQuery(docObject, query, nextCursorMark, 0, cb); } }).catch(function (err) { - if (retryCount < config.max_retry) { + if (retryCount < config.maxRetry) { retryCount++; requestByQuery(docObject, query, cursorMark, retryCount, cb); } else { diff --git a/package.json b/package.json index 60eb0e9..6e533e9 100644 --- a/package.json +++ b/package.json @@ -15,33 +15,24 @@ "author": "mounir .b", "license": "GNU", "dependencies": { - "chai": "^3.4.0", - "fetch-with-proxy": "^3.0.1", - "graceful-fs": "^4.2.8", - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", - "mocha": "^2.3.3", - "proxy-http-agent": "^1.0.1" - }, - "devDependencies": { "async": "^3.2.0", "chai": "^3.4.0", - "commander": "^4.1.1", - "csv-parser": "^2.3.3", + "graceful-fs": "^4.2.8", + "mocha": "^2.3.3", + "proxy-http-agent": "^1.0.1", + "lodash": "^4.17.15", + "node-fetch": "^2.6.0", + "stream-json": "^1.3.3", + "xmlsplit": "^1.2.8", + "fast-xml-parser": "^3.21.1" + }, + "devDependencies": { "eslint": "^7.32.0", "eslint-config-semistandard": "^16.0.0", "eslint-config-standard": "^16.0.3", "eslint-plugin-import": "^2.24.2", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^5.1.0", - "eslint-plugin-standard": "^5.0.0", - "fast-xml-parser": "^3.21.1", - "lodash": "^4.17.15", - "mocha": "^2.3.3", - "node-fetch": "^2.6.0", - "proxy-agent": "^3.1.1", - "stream-json": "^1.3.3", - "xml-formatter": "^2.5.1", - "xmlsplit": "^1.2.8" + "eslint-plugin-standard": "^5.0.0" } }