Newer
Older
sisyphe-go / indexCorpus.sh
@Nacim Nacim on 16 Mar 2022 3 KB update go and dependencies
#!/bin/sh
# IDfr 2022/03

#Conf Part
BLUE='\033[1;36m'
NC='\033[0m'
filePattern="analyse-logs.json"
filePatternReadyForCurl="analyse-logs_curlReady.json"
elasticURL="https://elastic:secret@vd-istex-esk.intra.inist.fr"
elasticPort="9200"
kibanaPort="5601"
outCurlFolder="/applis/istex/home/sisyphe-go/out.curl"
dashboardTemplateFolder="/applis/istex/home/sisyphe-go/kibanatemplates"
genericDashboardTemplate="dashboard-generique.ndjson"
detailDashboardTemplate="dashboard-detail.ndjson"

#Help and args check
if [ -z $2 ]
	then
	echo ""
	echo  "${BLUE}IDfr${NC} - Sisyphe Corpus indexation - 2022/03"
	echo ""
	echo "${BLUE}#################${NC}"
	echo "Please add args"
	echo "${BLUE}#################${NC}"
	echo ""
	echo "Script Usage : indexCorpus.sh \$FullPath \$Type"
	echo " \$FullPath = full path to json out file"
	echo " \$Type = analysis type ${BLUE}generique|detail${NC}"
	echo ""
	echo "ex : /sisyphe-go/out/1312441234-cairn-2022-03-09/analyse-logs.json generique"
	echo ""
	exit
fi
fullPath=$1
analysisType=$2

#Check if $fullPath exists
if [ -f $fullPath ]
	then
	#extract json file name from path"
	fileName=$(echo $fullPath |  awk -F "/" '{print $NF}')
	#check if json file = $filePattern
	if [ "$fileName" = "$filePattern" ]
		then
		#get corpus name from path (remove timestamp)
		corpusName=$(echo $fullPath |  awk -F "/" '{print $(NF-1)}' | cut -d '-' -f2-  )
		#get log file folder
		fullPathFolder=$(dirname $fullPath)
		echo "${BLUE}#################${NC}"
		echo "Corpus : "$corpusName
		echo "${BLUE}#################${NC}"
		echo "Converting file for bulk"
		#create buffer folder for json convert and kibana dashboard template
		mkdir $outCurlFolder"/"$corpusName
		#convert json file to json ready for bulk
 		sed 's/{"corpusname":/{ "index" : {} }\n{"corpusname":/gi' $fullPath > $outCurlFolder"/"$corpusName"/"$filePatternReadyForCurl		
		#indexing
		echo "Sending to Index"
		curl -k --noproxy '*'  -XPOST $elasticURL":"$elasticPort"/analyse-"$corpusName"/_bulk?pretty" -H "Content-Type: application/json" --data-binary "@"$outCurlFolder"/"$corpusName"/"$filePatternReadyForCurl

		#Import Dashboard - check type first
		if [ "$analysisType" = "generique" ]
			then
			echo "Generic Analysis"		
			cat $dashboardTemplateFolder"/"$genericDashboardTemplate | sed "s/CORPUSNAMEREPLACE/"$corpusName"/" | sed "s/DASHBOARDTITLEGEN/"$corpusName" Générique/" > $outCurlFolder"/"$corpusName"/dashboardGEN.ndjson"
			echo "Creating dashboard"
			curl -k --noproxy '*'  -X POST $elasticURL":"$kibanaPort"/api/saved_objects/_import?createNewCopies=true" -H "kbn-xsrf: true" --form "file=@"$outCurlFolder"/"$corpusName"/dashboardGEN.ndjson"
		elif [ "$analysisType" = "detaille" ]
			then
			echo "Detailled dashboard"
			cat $dashboardTemplateFolder"/"$detailDashboardTemplate | sed "s/CORPUSNAMEREPLACE/"$corpusName"/" | sed "s/DASHBOARDTITLEGEN/"$corpusName" Détaillée/" > $outCurlFolder"/"$corpusName"/dashboardDET.ndjson"
			echo "Creating dashboard"
			curl -k --noproxy '*'  -X POST $elasticURL":"$kibanaPort"/api/saved_objects/_import?createNewCopies=true" -H "kbn-xsrf: true" --form "file=@"$outCurlFolder"/"$corpusName"/dashboardDET.ndjson"
		else
			echo "Unknown Analysis type : "$analysisType
		fi
	else
		echo "Unexpected file name : "$fileName" instead of "$filePattern
		exit
	fi
else
	echo "File is missing"
	exit
fi

#Removing buffer folder
rm -rf $outCurlFolder"/"$corpusName