Newer
Older
sisyphe-go / main.go
@Nacim Nacim on 6 Feb 2022 1 KB add .gitignore
package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/gabriel-vasile/mimetype"
	"github.com/schollz/progressbar/v3"
	"github.com/sirupsen/logrus"
)

type Message struct {
	corpusname string
	startAt    string
	extension  string
	path       string
	mimetype   string
	size       int64
}

var sem = make(chan struct{}, 1000)
var wg sync.WaitGroup // instanciation de notre structure WaitGroup
var numberFiles int = 0

func logger(message *Message) {
	logrus.WithFields(logrus.Fields{
		"corpusname": message.corpusname,
		"startAt":    message.startAt,
		"extension":  message.extension,
		"path":       message.path,
		"mimetype":   message.mimetype,
		"size":       message.size,
	}).Info("")
}

func getAllFiles(dir string) {
	loadFilesProgress := progressbar.Default(-1, "Load files")
	defer wg.Done()

	visit := func(path string, file os.FileInfo, err error) error {
		if file.IsDir() && path != dir {
			wg.Add(1)
			go getAllFiles(path)
			return filepath.SkipDir
		}
		if file.Mode().IsRegular() {
			loadFilesProgress.Add(1)
			mtype, err := mimetype.DetectFile(path)
			if err == nil {
				logger(&Message{
					corpusname: file.Name(),
					startAt:    file.ModTime().String(),
					extension:  mtype.Extension(),
					path:       path,
					mimetype:   mtype.String(),
					size:       file.Size(),
				})
			}

		}

		return nil
	}

	filepath.Walk(dir, visit)
}

func processPDF() {

}

func processXML() {

}

func initlogrus() {
	f, err := os.OpenFile("./file.log", os.O_WRONLY|os.O_CREATE, 0755)
	if err != nil {
		fmt.Println(err)
	}

	logrus.SetFormatter(&logrus.JSONFormatter{})

	logrus.SetOutput(f)
}

func main() {
	start := time.Now()
	fmt.Println("Read file in", os.Args[1])
	root := os.Args[1]

	initlogrus()
	// read all files
	wg.Add(1)
	getAllFiles(root)
	wg.Wait()
	elapsed := time.Since(start)
	log.Printf("Total time %s", elapsed)
}