diff --git a/docker-compose.yml b/docker-compose.yml index cc07ce8..cd5605a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,4 +17,13 @@ - $CORPUS_RESOURCES:/corpus-resources/ - $SISYPHE_OUT:/app/out/ ports: - - '6379:6379' \ No newline at end of file + - '6379:6379' + + nginx: + image: nginx:1.21.6 + volumes: + - "/work:/usr/share/nginx/html/work:ro" + - "./nginx.conf.d/nginx.conf:/etc/nginx/nginx.conf" + ports: + - 40000:80 + diff --git a/indexCorpus.sh b/indexCorpus.sh new file mode 100755 index 0000000..963acea --- /dev/null +++ b/indexCorpus.sh @@ -0,0 +1,91 @@ +#!/bin/sh +# IDfr 2022/03 + +#Conf Part +BLUE='\033[1;36m' +NC='\033[0m' +filePattern="analyse-logs.json" +filePatternReadyForCurl="analyse-logs_curlReady.json" +elasticURL="https://elastic:secret@vd-istex-esk.intra.inist.fr" +elasticPort="9200" +kibanaPort="5601" +outCurlFolder="/applis/istex/home/sisyphe-go/out.curl" +dashboardTemplateFolder="/applis/istex/home/sisyphe-go/kibanatemplates" +genericDashboardTemplate="dashboard-generique.ndjson" +detailDashboardTemplate="dashboard-detail.ndjson" + +#Help and args check +if [ -z $2 ] + then + echo "" + echo "${BLUE}IDfr${NC} - Sisyphe Corpus indexation - 2022/03" + echo "" + echo "${BLUE}#################${NC}" + echo "Please add args" + echo "${BLUE}#################${NC}" + echo "" + echo "Script Usage : indexCorpus.sh \$FullPath \$Type" + echo " \$FullPath = full path to json out file" + echo " \$Type = analysis type ${BLUE}generique|detail${NC}" + echo "" + echo "ex : /sisyphe-go/out/1312441234-cairn-2022-03-09/analyse-logs.json generique" + echo "" + exit +fi +fullPath=$1 +analysisType=$2 + +#Check if $fullPath exists +if [ -f $fullPath ] + then + #extract json file name from path" + fileName=$(echo $fullPath | awk -F "/" '{print $NF}') + #check if json file = $filePattern + if [ "$fileName" = "$filePattern" ] + then + #get corpus name from path (remove timestamp) + corpusName=$(echo $fullPath | awk -F "/" '{print $(NF-1)}' | cut -d '-' -f2- ) + #get log file folder + fullPathFolder=$(dirname $fullPath) + echo "${BLUE}#################${NC}" + echo "Corpus : "$corpusName + echo "${BLUE}#################${NC}" + echo "Converting file for bulk" + #create buffer folder for json convert and kibana dashboard template + mkdir $outCurlFolder"/"$corpusName + #convert json file to json ready for bulk + sed 's/{"corpusname":/{ "index" : {} }\n{"corpusname":/gi' $fullPath > $outCurlFolder"/"$corpusName"/"$filePatternReadyForCurl + #indexing + echo "Sending to Index" + curl -k --noproxy '*' -XPOST $elasticURL":"$elasticPort"/analyse-"$corpusName"/_bulk?pretty" -H "Content-Type: application/json" --data-binary "@"$outCurlFolder"/"$corpusName"/"$filePatternReadyForCurl + + #Import Dashboard - check type first + if [ "$analysisType" = "generique" ] + then + echo "Generic Analysis" + cat $dashboardTemplateFolder"/"$genericDashboardTemplate | sed "s/CORPUSNAMEREPLACE/"$corpusName"/" | sed "s/DASHBOARDTITLEGEN/"$corpusName" Générique/" > $outCurlFolder"/"$corpusName"/dashboardGEN.ndjson" + echo "Creating dashboard" + curl -k --noproxy '*' -X POST $elasticURL":"$kibanaPort"/api/saved_objects/_import?createNewCopies=true" -H "kbn-xsrf: true" --form "file=@"$outCurlFolder"/"$corpusName"/dashboardGEN.ndjson" + elif [ "$analysisType" = "detaille" ] + then + echo "Detailled dashboard" + cat $dashboardTemplateFolder"/"$detailDashboardTemplate | sed "s/CORPUSNAMEREPLACE/"$corpusName"/" | sed "s/DASHBOARDTITLEGEN/"$corpusName" Détaillée/" > $outCurlFolder"/"$corpusName"/dashboardDET.ndjson" + echo "Creating dashboard" + curl -k --noproxy '*' -X POST $elasticURL":"$kibanaPort"/api/saved_objects/_import?createNewCopies=true" -H "kbn-xsrf: true" --form "file=@"$outCurlFolder"/"$corpusName"/dashboardDET.ndjson" + else + echo "Unknown Analysis type : "$analysisType + fi + else + echo "Unexpected file name : "$fileName" instead of "$filePattern + exit + fi +else + echo "File is missing" + exit +fi + +#Removing buffer folder +rm -rf $outCurlFolder"/"$corpusName + + + diff --git a/kibanatemplates/dashboard-detail.ndjson b/kibanatemplates/dashboard-detail.ndjson new file mode 100644 index 0000000..6767ec3 --- /dev/null +++ b/kibanatemplates/dashboard-detail.ndjson @@ -0,0 +1,2 @@ +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"corpusname.raw: \\\"CORPUSNAMEREPLACE\\\"\",\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true,\"syncColors\":true}","panelsJSON":"[{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":162,\"w\":48,\"h\":6,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{},\"hidePanelTitles\":true},\"panelRefName\":\"panel_1\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":34,\"w\":48,\"h\":6,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{},\"hidePanelTitles\":true},\"panelRefName\":\"panel_2\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":4,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{},\"hidePanelTitles\":true},\"panelRefName\":\"panel_3\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":16,\"w\":48,\"h\":6,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{},\"hidePanelTitles\":true},\"panelRefName\":\"panel_7\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":51,\"w\":48,\"h\":4,\"i\":\"34\"},\"panelIndex\":\"34\",\"embeddableConfig\":{\"enhancements\":{},\"hidePanelTitles\":true},\"panelRefName\":\"panel_34\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":98,\"w\":48,\"h\":4,\"i\":\"59\"},\"panelIndex\":\"59\",\"embeddableConfig\":{\"enhancements\":{},\"hidePanelTitles\":true},\"panelRefName\":\"panel_59\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":4,\"w\":15,\"h\":6,\"i\":\"72\"},\"panelIndex\":\"72\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_72\"},{\"version\":\"7.14.0\",\"type\":\"search\",\"gridData\":{\"x\":21,\"y\":40,\"w\":27,\"h\":11,\"i\":\"75\"},\"panelIndex\":\"75\",\"embeddableConfig\":{\"columns\":[\"wellFormedErrors\"],\"sort\":[\"_score\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_75\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":40,\"w\":21,\"h\":11,\"i\":\"76\"},\"panelIndex\":\"76\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"true\":\"#629E51\",\"false\":\"#BF1B00\"},\"legendOpen\":true},\"table\":null},\"panelRefName\":\"panel_76\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":12,\"y\":55,\"w\":12,\"h\":13,\"i\":\"77\"},\"panelIndex\":\"77\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"false\":\"#BF1B00\",\"true\":\"#629E51\"},\"legendOpen\":true},\"table\":null},\"panelRefName\":\"panel_77\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":55,\"w\":12,\"h\":13,\"i\":\"78\"},\"panelIndex\":\"78\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"false\":\"#BF1B00\",\"true\":\"#629E51\"},\"legendOpen\":true},\"table\":null},\"panelRefName\":\"panel_78\"},{\"version\":\"7.14.0\",\"type\":\"search\",\"gridData\":{\"x\":24,\"y\":55,\"w\":24,\"h\":25,\"i\":\"79\"},\"panelIndex\":\"79\",\"embeddableConfig\":{\"columns\":[\"validationErrors\"],\"sort\":[\"_score\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_79\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":68,\"w\":12,\"h\":12,\"i\":\"80\"},\"panelIndex\":\"80\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_80\"},{\"version\":\"7.14.0\",\"type\":\"search\",\"gridData\":{\"x\":12,\"y\":68,\"w\":12,\"h\":12,\"i\":\"81\"},\"panelIndex\":\"81\",\"embeddableConfig\":{\"columns\":[\"validationSchemaErrors\"],\"sort\":[\"_score\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_81\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":10,\"w\":15,\"h\":6,\"i\":\"82\"},\"panelIndex\":\"82\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_82\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":15,\"y\":4,\"w\":33,\"h\":12,\"i\":\"83\"},\"panelIndex\":\"83\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_83\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":22,\"w\":16,\"h\":12,\"i\":\"84\"},\"panelIndex\":\"84\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_84\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":16,\"y\":22,\"w\":32,\"h\":12,\"i\":\"85\"},\"panelIndex\":\"85\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_85\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":168,\"w\":16,\"h\":6,\"i\":\"88\"},\"panelIndex\":\"88\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_88\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":174,\"w\":16,\"h\":12,\"i\":\"89\"},\"panelIndex\":\"89\",\"embeddableConfig\":{\"enhancements\":{},\"spy\":{\"mode\":{\"fill\":false,\"name\":\"table\"}},\"vis\":{\"colors\":{\"0 to 25\":\"#890F02\",\"25 to 100\":\"#E0752D\",\"100 to 200\":\"#B7DBAB\",\"200 to 20,000\":\"#508642\"},\"legendOpen\":true},\"table\":null},\"panelRefName\":\"panel_89\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":32,\"y\":174,\"w\":16,\"h\":12,\"i\":\"90\"},\"panelIndex\":\"90\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"legendOpen\":true},\"table\":null},\"panelRefName\":\"panel_90\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":16,\"y\":174,\"w\":16,\"h\":12,\"i\":\"91\"},\"panelIndex\":\"91\",\"embeddableConfig\":{\"enhancements\":{},\"spy\":{\"mode\":{\"fill\":false,\"name\":\"table\"}},\"vis\":{\"legendOpen\":true},\"table\":null},\"panelRefName\":\"panel_91\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":16,\"y\":168,\"w\":32,\"h\":6,\"i\":\"92\"},\"panelIndex\":\"92\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_92\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":80,\"w\":8,\"h\":4,\"i\":\"93\"},\"panelIndex\":\"93\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_93\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":8,\"y\":80,\"w\":8,\"h\":4,\"i\":\"94\"},\"panelIndex\":\"94\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_94\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":84,\"w\":8,\"h\":4,\"i\":\"95\"},\"panelIndex\":\"95\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_95\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":8,\"y\":84,\"w\":8,\"h\":4,\"i\":\"96\"},\"panelIndex\":\"96\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_96\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":16,\"y\":80,\"w\":8,\"h\":4,\"i\":\"97\"},\"panelIndex\":\"97\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_97\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":16,\"y\":84,\"w\":8,\"h\":4,\"i\":\"98\"},\"panelIndex\":\"98\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_98\"},{\"version\":\"7.14.0\",\"type\":\"search\",\"gridData\":{\"x\":24,\"y\":80,\"w\":24,\"h\":8,\"i\":\"99\"},\"panelIndex\":\"99\",\"embeddableConfig\":{\"columns\":[\"publicationYearError\"],\"sort\":[\"_score\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_99\"},{\"version\":\"7.14.0\",\"type\":\"search\",\"gridData\":{\"x\":0,\"y\":186,\"w\":48,\"h\":17,\"i\":\"101\"},\"panelIndex\":\"101\",\"embeddableConfig\":{\"columns\":[\"_source\"],\"sort\":[\"_score\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_101\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":138,\"w\":24,\"h\":12,\"i\":\"102\"},\"panelIndex\":\"102\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_102\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":24,\"y\":138,\"w\":24,\"h\":12,\"i\":\"104\"},\"panelIndex\":\"104\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_104\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":150,\"w\":24,\"h\":12,\"i\":\"105\"},\"panelIndex\":\"105\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_105\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":126,\"w\":24,\"h\":12,\"i\":\"106\"},\"panelIndex\":\"106\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_106\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":24,\"y\":150,\"w\":24,\"h\":12,\"i\":\"107\"},\"panelIndex\":\"107\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_107\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":32,\"y\":102,\"w\":16,\"h\":12,\"i\":\"108\"},\"panelIndex\":\"108\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"0 to 0.9\":\"#BF1B00\",\"1 to 5\":\"#EF843C\",\"6 to 10\":\"#EAB839\",\"11 to 50\":\"#629E51\",\"51 to +∞\":\"#3F6833\"},\"legendOpen\":true},\"table\":null},\"panelRefName\":\"panel_108\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":16,\"y\":114,\"w\":16,\"h\":12,\"i\":\"110\"},\"panelIndex\":\"110\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"0 to 0.9\":\"#BF1B00\",\"1 to 5\":\"#EF843C\",\"6 to 10\":\"#EAB839\",\"11 to 50\":\"#629E51\",\"51 to +∞\":\"#3F6833\",\"false\":\"#BF1B00\",\"true\":\"#508642\"},\"legendOpen\":true},\"table\":null},\"panelRefName\":\"panel_110\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":102,\"w\":16,\"h\":12,\"i\":\"112\"},\"panelIndex\":\"112\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"0 to 0.9\":\"#BF1B00\",\"1 to 5\":\"#EF843C\",\"6 to 10\":\"#EAB839\",\"11 to 50\":\"#629E51\",\"51 to +∞\":\"#3F6833\",\"false\":\"#BF1B00\",\"true\":\"#508642\"},\"legendOpen\":true},\"table\":null},\"panelRefName\":\"panel_112\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":114,\"w\":16,\"h\":12,\"i\":\"113\"},\"panelIndex\":\"113\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"0 to 0.9\":\"#BF1B00\",\"1 to 5\":\"#EF843C\",\"6 to 10\":\"#EAB839\",\"11 to 50\":\"#629E51\",\"51 to +∞\":\"#3F6833\"},\"legendOpen\":true}},\"panelRefName\":\"panel_113\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":16,\"y\":102,\"w\":16,\"h\":12,\"i\":\"114\"},\"panelIndex\":\"114\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"0 to 0.9\":\"#BF1B00\",\"1 to 5\":\"#EF843C\",\"6 to 10\":\"#EAB839\",\"11 to 50\":\"#629E51\",\"51 to +∞\":\"#3F6833\"},\"legendOpen\":true}},\"panelRefName\":\"panel_114\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":32,\"y\":114,\"w\":16,\"h\":12,\"i\":\"115\"},\"panelIndex\":\"115\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"0 to 0.9\":\"#BF1B00\",\"1 to 5\":\"#EF843C\",\"11 to 50\":\"#629E51\",\"51 to +∞\":\"#3F6833\",\"6 to 10\":\"#EAB839\",\"false\":\"#BF1B00\",\"true\":\"#508642\"},\"legendOpen\":true},\"table\":null},\"panelRefName\":\"panel_115\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":24,\"y\":126,\"w\":24,\"h\":12,\"i\":\"117\"},\"panelIndex\":\"117\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_117\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":88,\"w\":47,\"h\":10,\"i\":\"118\"},\"panelIndex\":\"118\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_118\"}]","timeRestore":false,"title":"DASHBOARDTITLEGEN","version":1},"coreMigrationVersion":"8.0.1","id":"614f0f20-c842-11eb-991a-016300219d19","migrationVersion":{"dashboard":"8.0.1"},"references":[{"id":"TITRE-PDF","name":"1:panel_1","type":"visualization"},{"id":"Titre-XML","name":"2:panel_2","type":"visualization"},{"id":"Titre-générale","name":"3:panel_3","type":"visualization"},{"id":"Titre-fichiers-corrompus","name":"7:panel_7","type":"visualization"},{"id":"Sous-titre-XML-bien-formé","name":"34:panel_34","type":"visualization"},{"id":"Information-contenus-XML","name":"59:panel_59","type":"visualization"},{"id":"visu-nbfiles","name":"72:panel_72","type":"visualization"},{"id":"search-wellformed-errors","name":"75:panel_75","type":"search"},{"id":"visu-bad-well-formed-xml","name":"76:panel_76","type":"visualization"},{"id":"visu-dtdvalid-byyear","name":"77:panel_77","type":"visualization"},{"id":"visu-dtds","name":"78:panel_78","type":"visualization"},{"id":"search-dtd-errors","name":"79:panel_79","type":"search"},{"id":"visu-validxml-byyear","name":"80:panel_80","type":"visualization"},{"id":"search-schema-errors","name":"81:panel_81","type":"search"},{"id":"visu-corpus-size","name":"82:panel_82","type":"visualization"},{"id":"visu-allfiles","name":"83:panel_83","type":"visualization"},{"id":"visu-bad-files","name":"84:panel_84","type":"visualization"},{"id":"visu-bad-ext","name":"85:panel_85","type":"visualization"},{"id":"visu-nbpdf","name":"88:panel_88","type":"visualization"},{"id":"visu-wordsperpdf","name":"89:panel_89","type":"visualization"},{"id":"visu-pdftool","name":"90:panel_90","type":"visualization"},{"id":"visu-pdf-byyear","name":"91:panel_91","type":"visualization"},{"id":"visu-pdfsize","name":"92:panel_92","type":"visualization"},{"id":"visu-bad-volnum","name":"93:panel_93","type":"visualization"},{"id":"visu-bad-eissn","name":"94:panel_94","type":"visualization"},{"id":"visu-bad-isbn","name":"95:panel_95","type":"visualization"},{"id":"visu-bad-pubyear","name":"96:panel_96","type":"visualization"},{"id":"visu-bad-doi","name":"97:panel_97","type":"visualization"},{"id":"visu-bad-issn","name":"98:panel_98","type":"visualization"},{"id":"search-pubyear-errors","name":"99:panel_99","type":"search"},{"id":"search-allinfos","name":"101:panel_101","type":"search"},{"id":"visu-eissn","name":"102:panel_102","type":"visualization"},{"id":"visu-unique-issn","name":"104:panel_104","type":"visualization"},{"id":"visu-pii","name":"105:panel_105","type":"visualization"},{"id":"visu-title","name":"106:panel_106","type":"visualization"},{"id":"visu-doi","name":"107:panel_107","type":"visualization"},{"id":"visu-nbkeywords-byyear","name":"108:panel_108","type":"visualization"},{"id":"visu-contenttype","name":"110:panel_110","type":"visualization"},{"id":"visu-hasabstract-byyear","name":"112:panel_112","type":"visualization"},{"id":"visu-refbibs-year","name":"113:panel_113","type":"visualization"},{"id":"visu-nbelembody","name":"114:panel_114","type":"visualization"},{"id":"visu-lang-byyear","name":"115:panel_115","type":"visualization"},{"id":"f87642c3-df6d-11e9-b330-2f8fc7ffcdbf","name":"117:panel_117","type":"visualization"},{"id":"caf429a0-2271-11eb-9210-7f7c2e4c1694","name":"118:panel_118","type":"visualization"}],"type":"dashboard","updated_at":"2022-03-09T12:07:12.743Z","version":"WzEyMjQsMTRd"} +{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":1,"missingRefCount":0,"missingReferences":[]} diff --git a/kibanatemplates/dashboard-generique.ndjson b/kibanatemplates/dashboard-generique.ndjson new file mode 100644 index 0000000..422fa58 --- /dev/null +++ b/kibanatemplates/dashboard-generique.ndjson @@ -0,0 +1,2 @@ +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"corpusname: \\\"CORPUSNAMEREPLACE\\\"\",\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true,\"syncColors\":true}","panelsJSON":"[{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":43,\"w\":48,\"h\":7,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":106,\"w\":48,\"h\":7,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":4,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":4,\"w\":16,\"h\":8,\"i\":\"4\"},\"panelIndex\":\"4\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":12,\"w\":16,\"h\":8,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":20,\"w\":48,\"h\":7,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":27,\"w\":16,\"h\":8,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":16,\"y\":27,\"w\":32,\"h\":16,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":35,\"w\":16,\"h\":8,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_10\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":16,\"y\":4,\"w\":32,\"h\":16,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_11\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":50,\"w\":12,\"h\":8,\"i\":\"12\"},\"panelIndex\":\"12\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_12\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":113,\"w\":48,\"h\":6,\"i\":\"14\"},\"panelIndex\":\"14\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_14\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":66,\"w\":24,\"h\":12,\"i\":\"15\"},\"panelIndex\":\"15\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"1.1\":\"#BF1B00\",\"1.2\":\"#EA6460\",\"1.3\":\"#9AC48A\",\"1.5\":\"#629E51\",\"1.6\":\"#3F6833\",\"1.7\":\"#447EBC\"},\"legendOpen\":true}},\"panelRefName\":\"panel_15\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":24,\"y\":119,\"w\":24,\"h\":16,\"i\":\"18\"},\"panelIndex\":\"18\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_18\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":24,\"y\":66,\"w\":24,\"h\":12,\"i\":\"28\"},\"panelIndex\":\"28\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"legendOpen\":true}},\"panelRefName\":\"panel_28\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":90,\"w\":48,\"h\":16,\"i\":\"29\"},\"panelIndex\":\"29\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_29\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":119,\"w\":24,\"h\":16,\"i\":\"30\"},\"panelIndex\":\"30\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"false\":\"#BF1B00\",\".sgm\":\"#EF843C\"},\"legendOpen\":true}},\"panelRefName\":\"panel_30\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":12,\"y\":54,\"w\":36,\"h\":4,\"i\":\"32\"},\"panelIndex\":\"32\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_32\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":12,\"y\":50,\"w\":36,\"h\":4,\"i\":\"33\"},\"panelIndex\":\"33\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_33\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":58,\"w\":12,\"h\":8,\"i\":\"34\"},\"panelIndex\":\"34\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_34\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":12,\"y\":58,\"w\":36,\"h\":8,\"i\":\"35\"},\"panelIndex\":\"35\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_35\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":78,\"w\":24,\"h\":12,\"i\":\"36\"},\"panelIndex\":\"36\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"200 to 20000\":\"#508642\",\"0 to 25\":\"#BF1B00\",\"25 to 100\":\"#E0752D\",\"100 to 200\":\"#9AC48A\"},\"legendOpen\":true}},\"panelRefName\":\"panel_36\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":135,\"w\":48,\"h\":24,\"i\":\"38\"},\"panelIndex\":\"38\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_38\"}]","timeRestore":false,"title":"DASHBOARDTITLEGEN","version":1},"coreMigrationVersion":"8.0.1","id":"96ffaab0-eb9e-11eb-a7fd-451cae8e85dd","migrationVersion":{"dashboard":"8.0.1"},"references":[{"id":"TITRE-PDF","name":"1:panel_1","type":"visualization"},{"id":"Titre-XML","name":"2:panel_2","type":"visualization"},{"id":"Titre-générale","name":"3:panel_3","type":"visualization"},{"id":"Nb-fichiers","name":"4:panel_4","type":"visualization"},{"id":"Taille-corpus","name":"6:panel_6","type":"visualization"},{"id":"Titre-fichiers-corrompus","name":"7:panel_7","type":"visualization"},{"id":"Nb-Fichier-corrompus","name":"8:panel_8","type":"visualization"},{"id":"Extensions-fichiers-corrompus","name":"9:panel_9","type":"visualization"},{"id":"Taille-Fichiers-Corrompus","name":"10:panel_10","type":"visualization"},{"id":"Répartition-de-tous-les-fichiers-slash-Extension","name":"11:panel_11","type":"visualization"},{"id":"Nb-de-PDF","name":"12:panel_12","type":"visualization"},{"id":"Nb-d'XML","name":"14:panel_14","type":"visualization"},{"id":"Répartition-versions-de-PDF","name":"15:panel_15","type":"visualization"},{"id":"Repartition-des-DTD","name":"18:panel_18","type":"visualization"},{"id":"Répartition-logiciel-création-PDF","name":"28:panel_28","type":"visualization"},{"id":"Répartition-PDF-par-ratio-mots-slash-page","name":"29:panel_29","type":"visualization"},{"id":"Repartition-XML-bien-ampersand-mal-formés","name":"30:panel_30","type":"visualization"},{"id":"Nombre-de-mots-PDF","name":"32:panel_32","type":"visualization"},{"id":"Répartition-poids-des-PDF","name":"33:panel_33","type":"visualization"},{"id":"Nombre-total-de-pages-PDF","name":"34:panel_34","type":"visualization"},{"id":"Répartition-par-nombre-de-page","name":"35:panel_35","type":"visualization"},{"id":"Nombre-de-mots-par-pdf-slash-pages","name":"36:panel_36","type":"visualization"},{"id":"liste","name":"38:panel_38","type":"visualization"}],"type":"dashboard","updated_at":"2022-03-09T12:07:12.743Z","version":"WzEyNDQsMTRd"} +{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":1,"missingRefCount":0,"missingReferences":[]} diff --git a/kibanatemplates/dashboard.ndjson b/kibanatemplates/dashboard.ndjson new file mode 100644 index 0000000..857be8a --- /dev/null +++ b/kibanatemplates/dashboard.ndjson @@ -0,0 +1,2 @@ +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"corpusname: \\\"CORPUSNAMEREPLACE\\\"\",\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true,\"syncColors\":true}","panelsJSON":"[{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":43,\"w\":48,\"h\":7,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":106,\"w\":48,\"h\":7,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":4,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":4,\"w\":16,\"h\":8,\"i\":\"4\"},\"panelIndex\":\"4\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":12,\"w\":16,\"h\":8,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":20,\"w\":48,\"h\":7,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":27,\"w\":16,\"h\":8,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":16,\"y\":27,\"w\":32,\"h\":16,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":35,\"w\":16,\"h\":8,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_10\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":16,\"y\":4,\"w\":32,\"h\":16,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_11\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":50,\"w\":12,\"h\":8,\"i\":\"12\"},\"panelIndex\":\"12\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_12\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":113,\"w\":48,\"h\":6,\"i\":\"14\"},\"panelIndex\":\"14\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_14\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":66,\"w\":24,\"h\":12,\"i\":\"15\"},\"panelIndex\":\"15\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"1.1\":\"#BF1B00\",\"1.2\":\"#EA6460\",\"1.3\":\"#9AC48A\",\"1.5\":\"#629E51\",\"1.6\":\"#3F6833\",\"1.7\":\"#447EBC\"},\"legendOpen\":true}},\"panelRefName\":\"panel_15\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":24,\"y\":119,\"w\":24,\"h\":16,\"i\":\"18\"},\"panelIndex\":\"18\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_18\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":24,\"y\":66,\"w\":24,\"h\":12,\"i\":\"28\"},\"panelIndex\":\"28\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"legendOpen\":true}},\"panelRefName\":\"panel_28\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":90,\"w\":48,\"h\":16,\"i\":\"29\"},\"panelIndex\":\"29\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_29\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":119,\"w\":24,\"h\":16,\"i\":\"30\"},\"panelIndex\":\"30\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"false\":\"#BF1B00\",\".sgm\":\"#EF843C\"},\"legendOpen\":true}},\"panelRefName\":\"panel_30\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":12,\"y\":54,\"w\":36,\"h\":4,\"i\":\"32\"},\"panelIndex\":\"32\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_32\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":12,\"y\":50,\"w\":36,\"h\":4,\"i\":\"33\"},\"panelIndex\":\"33\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_33\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":58,\"w\":12,\"h\":8,\"i\":\"34\"},\"panelIndex\":\"34\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_34\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":12,\"y\":58,\"w\":36,\"h\":8,\"i\":\"35\"},\"panelIndex\":\"35\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_35\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":78,\"w\":24,\"h\":12,\"i\":\"36\"},\"panelIndex\":\"36\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"colors\":{\"200 to 20000\":\"#508642\",\"0 to 25\":\"#BF1B00\",\"25 to 100\":\"#E0752D\",\"100 to 200\":\"#9AC48A\"},\"legendOpen\":true}},\"panelRefName\":\"panel_36\"},{\"version\":\"7.14.0\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":135,\"w\":48,\"h\":24,\"i\":\"38\"},\"panelIndex\":\"38\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_38\"}]","timeRestore":false,"title":"DASHBOARDTITLEGEN générique","version":1},"coreMigrationVersion":"8.0.1","id":"96ffaab0-eb9e-11eb-a7fd-451cae8e85dd","migrationVersion":{"dashboard":"8.0.1"},"references":[{"id":"TITRE-PDF","name":"1:panel_1","type":"visualization"},{"id":"Titre-XML","name":"2:panel_2","type":"visualization"},{"id":"Titre-générale","name":"3:panel_3","type":"visualization"},{"id":"Nb-fichiers","name":"4:panel_4","type":"visualization"},{"id":"Taille-corpus","name":"6:panel_6","type":"visualization"},{"id":"Titre-fichiers-corrompus","name":"7:panel_7","type":"visualization"},{"id":"Nb-Fichier-corrompus","name":"8:panel_8","type":"visualization"},{"id":"Extensions-fichiers-corrompus","name":"9:panel_9","type":"visualization"},{"id":"Taille-Fichiers-Corrompus","name":"10:panel_10","type":"visualization"},{"id":"Répartition-de-tous-les-fichiers-slash-Extension","name":"11:panel_11","type":"visualization"},{"id":"Nb-de-PDF","name":"12:panel_12","type":"visualization"},{"id":"Nb-d'XML","name":"14:panel_14","type":"visualization"},{"id":"Répartition-versions-de-PDF","name":"15:panel_15","type":"visualization"},{"id":"Repartition-des-DTD","name":"18:panel_18","type":"visualization"},{"id":"Répartition-logiciel-création-PDF","name":"28:panel_28","type":"visualization"},{"id":"Répartition-PDF-par-ratio-mots-slash-page","name":"29:panel_29","type":"visualization"},{"id":"Repartition-XML-bien-ampersand-mal-formés","name":"30:panel_30","type":"visualization"},{"id":"Nombre-de-mots-PDF","name":"32:panel_32","type":"visualization"},{"id":"Répartition-poids-des-PDF","name":"33:panel_33","type":"visualization"},{"id":"Nombre-total-de-pages-PDF","name":"34:panel_34","type":"visualization"},{"id":"Répartition-par-nombre-de-page","name":"35:panel_35","type":"visualization"},{"id":"Nombre-de-mots-par-pdf-slash-pages","name":"36:panel_36","type":"visualization"},{"id":"liste","name":"38:panel_38","type":"visualization"}],"type":"dashboard","updated_at":"2022-03-09T12:07:12.743Z","version":"WzEyNDQsMTRd"} +{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":1,"missingRefCount":0,"missingReferences":[]} diff --git a/main.go b/main.go index 0edfa41..a563f01 100644 --- a/main.go +++ b/main.go @@ -158,6 +158,7 @@ fmt.Println(color.InGreen("The detailed analysis is enabled for"), color.InGreen(*configurationFolder+"/"+*corpusName+"/sisyphe-conf.json")) jsonFile, err := ioutil.ReadFile(*configurationFolder + "/" + *corpusName + "/sisyphe-conf.json") if err != nil { + fmt.Println(err) fmt.Println(color.InRed("Bad configuration file")) os.Exit(1) } diff --git a/nginx.conf.d/nginx.conf b/nginx.conf.d/nginx.conf new file mode 100644 index 0000000..dd658bf --- /dev/null +++ b/nginx.conf.d/nginx.conf @@ -0,0 +1,34 @@ + +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; + autoindex on; + autoindex_format html; +} diff --git a/struct.go b/struct.go index d5f6e99..bba59e8 100644 --- a/struct.go +++ b/struct.go @@ -61,6 +61,7 @@ Id string `json:"id"` SubjGroupType string `json:"subj-group-type"` Subject string `json:"subject"` + // Dynamic field: can include other fields } type DetailledAnalysis struct { @@ -85,15 +86,16 @@ type ConfigDetailledAnalyze struct { XML struct { - ListDTD []string `json:"dtd"` - ListXSD []string `json:"xsd"` - XPATH []MetadataConfigDetailledAnalyze `json:"metadata"` + ListDTD []string `json:"dtd"` + ListXSD []string `json:"schema"` + XPATH []MetadataConfigDetailledAnalyze `json:"metadata"` + IgnoreNamespace string `json:"ignoreNamespace"` } `json:"xml"` } type MetadataConfigDetailledAnalyze struct { Name string `json:"name"` - Type string `json:"type"` Regex string `json:"regex"` + Type string `json:"type"` Xpath []string `json:"xpath"` }