diff --git a/.gitignore b/.gitignore index 7e231d0..c165e18 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules/ *.gz +data.json diff --git a/index.mjs b/index.mjs index 163ce0d..a05abf8 100755 --- a/index.mjs +++ b/index.mjs @@ -3,33 +3,46 @@ import "cli-progress"; import cliProgress from "cli-progress"; +/** + * Display on stdout all notices, without suffix and prefix of an array + * @param {object[]} notices + * @param {boolean} last if these are the last notices + */ +const display = (notices, last = false) => { + const noticesStr = notices + .map(notice => JSON.stringify(notice)) + .join(",\n"); + console.log(noticesStr + (last ? "" : ",\n")); +} + const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_grey); -let notices = []; let afterKeyToken; +console.log("[\n"); + const response = await fetch("https://corhal-api.inist.fr/mergedDocuments?size=50"); -const data = /** @type {object[]} */(await response.json()); -notices = [...notices, ...data]; +let notices = /** @type {object[]} */(await response.json()); const totalCount = Number(response.headers.get("x-total-count")); let resultCount = notices.length; bar.start(totalCount, resultCount); afterKeyToken = response.headers.get("after-key-token"); -const max = 10_000_000; +const max = Math.max(10_000_000, totalCount); // const max = 300; // const max = 10_000; +// const max = 50; while (afterKeyToken && resultCount < max) { + display(notices); const response = await fetch("https://corhal-api.inist.fr/mergedDocuments?size=50"); - const data = /** @type {object[]} */(await response.json()); - notices = [...notices, ...data]; - const totalCount = response.headers.get("x-total-count"); - resultCount = notices.length; + notices = /** @type {object[]} */(await response.json()); + resultCount += notices.length; bar.update(resultCount); afterKeyToken = response.headers.get("after-key-token"); - // console.error({ resultCount, totalCount, afterKeyToken }); } bar.stop(); -console.log(JSON.stringify(notices) + "\n"); +display(notices, true); + +console.log("]\n");