#!/bin/bash debug() { cat <<< "$@" 1>&2; } NAME="/tmp/data-$RANDOM.dat" COMP=${cpt:-0} while IFS='$\n' read -r line; do COMP=`expr $COMP + 1` URI=$(echo $line|node -pe 'JSON.stringify(JSON.parse(fs.readFileSync(0)).id)') URL=$(echo $line|node -pe 'JSON.parse(fs.readFileSync(0)).value') PDF_FILE="/tmp/data-$RANDOM-$COMP.pdf" TXT_FILE="/tmp/data-$RANDOM-$COMP.txt" curl -s "${URL}" -H 'Referer;' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36' -H 'DNT: 1' --compressed -L -o "$PDF_FILE" pdftotext -q $PDF_FILE $TXT_FILE echo -n "{\"id\": ${URI}, \"value\":" cat $TXT_FILE |node -pe 'JSON.stringify(fs.readFileSync(0).toString()).concat("}")' rm $PDF_FILE $TXT_FILE 1>&2 2>/dev/null done < <(cat -)