Newer
Older
web-dumps / apil-dumps / Makefile
# To set specific directory for each crontab run
ifeq ($(strip $(Startup)),)
# use the the day of the week
#VERSION_DIR := $(shell date +%A)
# use the month
VERSION_DIR := $(shell date +%B)
else
# use no version
VERSION_DIR := .
endif

# To set ezs parameters
EZSFLAGS := --param VERSION_DIR="$(VERSION_DIR)"
# To set global  parameters
ROOT_DIR := $(shell pwd)
# To set the location and the extension of sources files (queries)
INPUT_DIR := 01-query
INPUT_EXT := txt

# To set the location and the extension of results files
OUTPUT_DIR := 05-result/$(VERSION_DIR)
OUTPUT_EXT := jsonl

# To generate all files from source directory to target directory
SOURCE_FILES := $(wildcard $(INPUT_DIR)/*.$(INPUT_EXT))
TARGET_FILES := $(patsubst $(INPUT_DIR)/%.$(INPUT_EXT), $(OUTPUT_DIR)/%.$(OUTPUT_EXT), $(SOURCE_FILES))

# Phony Rules
help:
	@grep -P '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

list: ## list all target files
	@ls -1 $(TARGET_FILES)

view: ## To view current generated files
	find $(OUTPUT_DIR)

clean-cache: ## To delete enrichment cache
	@rm -v -f $(TMPDIR)/memory

clean: ## To delete current generated files
	@rm -v -f $(TARGET_FILES)

drain: ## To delete all directories (except the source directory)
	@rm -v -Rf 02-download  03-enrichment 04-refine 05-result

watch: ## Automatically build files when they change
	while true; do \
		inotifywait -qr -e modify -e create -e delete -e move --exclude '/\.' $(INPUT_DIR); \
		make clean-cache; \
		make all; \
	done

# Rules
all: $(TARGET_FILES) ## Build all files

%: $(OUTPUT_DIR)/%.$(OUTPUT_EXT)
	@echo "$<"

02-download/$(VERSION_DIR)/%.jsonl: 01-query/%.txt
	mkdir -p $(@D)
	time ezs $(EZSFLAGS) 02-download.ini < $< > $@.crdownload
	mv $@.crdownload $@

03-enrichment/$(VERSION_DIR)/%.jsonl: 02-download/$(VERSION_DIR)/%.jsonl
	mkdir -p $(@D)
	time ezs $(EZSFLAGS) 03-enrichment.ini < $< > $@

04-refine/$(VERSION_DIR)/%.jsonl: 03-enrichment/$(VERSION_DIR)/%.jsonl
	mkdir -p $(@D)
	time ezs $(EZSFLAGS) 04-refine.ini < $< > $@

05-result/$(VERSION_DIR)/%.jsonl: 04-refine/$(VERSION_DIR)/%.jsonl
	mkdir -p $(@D)
	mv $< $@
	ls -lhag $@ |sed -re 's/^[^ ]* //' >> "$(subst .jsonl,.log,$@)"

# To prevent deleting intermediate files (for controls)
#.PRECIOUS: 02-download/$(VERSION_DIR)/%.jsonl 03-enrichment/$(VERSION_DIR)/%.jsonl

.DEFAULT_GOAL := help
.PHONY: clean drain view help list all clean-cache watch