diff --git a/main.go b/main.go index 4233468..1892a38 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ "io/ioutil" "log" "os" + "os/exec" "path/filepath" "regexp" "runtime" @@ -39,7 +40,10 @@ // regex var regexMime = regexp.MustCompile(`(.*); charset=(.*)`) +// other +var logPath = "" var configDetailledAnalyze ConfigDetailledAnalyze +var importType = "generique" func GetAllFiles(dir string) { defer wg.Done() @@ -97,11 +101,9 @@ if fileData.mimetype == "application/pdf" { wg.Add(1) go ProcessPDF(&fileData, logger) - numberFiles++ } else if fileData.mimetype == "application/xml" || fileData.mimetype == "text/xml" { wg.Add(1) go ProcessXML(&fileData, logger) - numberFiles++ } else { logger.Info("") numberFiles++ @@ -110,7 +112,6 @@ if fileData.mimetype == "application/xml" || fileData.mimetype == "text/xml" { wg.Add(1) go ProcessXML(&fileData, logger) - numberFiles++ } } } @@ -136,7 +137,7 @@ // init logger sec := time.Now().Unix() - logPath := *outputPath + "/" + strconv.Itoa(int(sec)) + "-" + *corpusName + logPath = *outputPath + "/" + strconv.Itoa(int(sec)) + "-" + *corpusName log.Println(color.Green + "Begin of program with " + strconv.Itoa(runtime.NumCPU()) + " CPU" + color.Reset) log.Println("Read corpus in", *corpusPath, "and write out in", logPath) os.MkdirAll(logPath, os.ModePerm|os.ModeDir) @@ -173,6 +174,7 @@ } fmt.Println(color.InBlue(strconv.Itoa(len(configDetailledAnalyze.XML.ListDTD)) + " DTD files for analysis")) fmt.Println(color.InBlue(strconv.Itoa(len(configDetailledAnalyze.XML.ListXSD)) + " XSD files for analysis")) + importType = "detaille" } func main() { @@ -187,9 +189,14 @@ GetAllFiles(*corpusPath) wg.Wait() close(queueForConcurrent) - time.Sleep(2 * time.Second) elapsed := time.Since(start) fmt.Println("") log.Println(color.Green + "End of program with " + strconv.Itoa(numberFiles) + " files processed" + color.Reset) log.Printf("Total time %s", elapsed) + + _, err := exec.Command("/bin/bash", "indexCorpus.sh", logPath+"/analyse-logs.json", importType).CombinedOutput() + if err != nil { + fmt.Println(color.InRed("Error indexCorpus.sh")) + panic(err) + } } diff --git a/pdf.go b/pdf.go index 2e211b1..aecf5ab 100644 --- a/pdf.go +++ b/pdf.go @@ -38,6 +38,7 @@ }) } logger.Info("") + numberFiles++ } // return Message with metadata info diff --git a/util.go b/util.go index 8d8afdf..58cc8b9 100644 --- a/util.go +++ b/util.go @@ -2,7 +2,7 @@ import "strings" -func GetStringWithSuffix(s []string, e string) string { +func GetStringWithSuffixInList(s []string, e string) string { for _, a := range s { if strings.HasSuffix(a, e) { return a diff --git a/xml.go b/xml.go index da04e42..f552f7e 100644 --- a/xml.go +++ b/xml.go @@ -37,6 +37,7 @@ } else { logger.Info("") } + numberFiles++ } func CheckIfXmlIsWellFormed(xmlPath string, logger *logrus.Entry) (XMLInfo, *logrus.Entry) { @@ -137,7 +138,7 @@ // if dtd exist in xml file process only this if len(configDetailledAnalyze.XML.ListDTD) > 0 { if xmlInfo.doctype.Sysid != "" { - dtdPath := GetStringWithSuffix(configDetailledAnalyze.XML.ListDTD, xmlInfo.doctype.Sysid) + dtdPath := GetStringWithSuffixInList(configDetailledAnalyze.XML.ListDTD, xmlInfo.doctype.Sysid) if dtdPath != "" { result, _ := exec.Command("xmllint", "--dtdvalid", pathExec+"/dtd/"+dtdPath, pathXml, "--noout", "--nowarning").CombinedOutput() if string(result) == "" { diff --git a/xml_test.go b/xml_test.go index 570fae6..834ff6a 100644 --- a/xml_test.go +++ b/xml_test.go @@ -74,4 +74,4 @@ result, _ := CheckXMLValidation(xmlData.path, XMLInfo{}, contextLogger) assert.Equal(t, result.isValidAgainstSchema, false, "XML must not be valid according to the Schema") assert.Equal(t, result.validationSchemaErrors, []ErrorXML{{Message: "Schemas validity error : Element 'note': No matching global declaration available for the validation root.", Line: "3", File: "bad-xsd.xsd"}}, "Return error if xsd is not valid") -} \ No newline at end of file +}