Repository navigation
Expand file tree
/
Copy pathMakefile
More file actions
316 lines (273 loc) · 10.1 KB
/
Copy pathMakefile
File metadata and controls
316 lines (273 loc) · 10.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
# ==============================================================================
# TorchSig Project Makefile
# ==============================================================================
# --- Variables ---
PYTHON = python3
PIP = $(PYTHON) -m pip
PYTEST = $(PYTHON) -m pytest
# Directories
SRC_DIR = torchsig
TEST_DIR = tests
# Cap local xdist workers
# override with: PYTEST_NPROCS=8 make test
PYTEST_NPROCS ?= 2
# --- Phony Targets ---
# .PHONY tells Make that these are "commands" and not "files" on disk.
# This prevents conflicts if you have a folder named 'tests' or 'clean'.
.PHONY: install build verify publish test test-debug test-cov test-notebooks test-notebooks-clean \
clean-notebooks lint format fix clean docs open-docs help benchmarks benchmarks-clean check-lfs
# Default target: show help
help:
@echo "TorchSig Project Orchestration"
@echo "-----------------------------"
@echo "Usage: make [target]"
@echo ""
@echo "Targets:"
@echo " install Install dependencies and the package in editable mode"
@echo " build Builds source distribution (sdist) and wheel for PyPI"
@echo " verify Validates distribution files and lists them (pre-publish check)"
@echo " publish Uploads distribution files to PyPI"
@echo " test Run all tests in the $(TEST_DIR) directory in parallel"
@echo " test-debug Run all tests in the $(TEST_DIR) directory sequentially"
@echo " test-cov Run tests and generate a coverage report"
@echo " test-notebooks"
@echo " Executes all Jupyter notebooks to verify they run without errors."
@echo " test-notebooks-clean"
@echo " Removes stamp files created by notebook execution"
@echo " clean-notebooks"
@echo " Removes all output from executed notebooks"
@echo " lint Run static analysis (ruff check)"
@echo " format Auto-format code (ruff format)"
@echo " fix Apply automatic fixes and formatting (ruff check --fix)"
@echo " docs Builds the HTML documentation"
@echo " open-docs Opens the built documentation in the default browser"
@echo " benchmarks Runs the benchmarks"
@echo " benchmarks-clean"
@echo " Removes previous benchmark results"
@echo " check-lfs Checks for any remaining Git LFS references in the repository"
@echo " clean Remove caches and temporary test artifacts"
@echo " help Show this help message"
# --- Targets ---
# Setup
install:
@echo "Installing dependencies..."
$(PIP) install --upgrade pip
$(PIP) install -e ".[dev]"
# Build
build: .build-stamp
.build-stamp:
pip install --upgrade build
python -m build --sdist --wheel
verify: build
twine check dist/*
ls -lh dist/
publish: verify
twine check dist/*
twine upload dist/*
# Testing
TEST_MODE ?= fast
# Report configuration
JUNIT_REPORT ?= report.xml
COVERAGE_XML ?= coverage.xml
COVERAGE_TERM ?= term-missing
test:
@echo "Running tests ($(TEST_MODE) mode) with $(PYTEST_NPROCS) workers..."
$(PYTEST) \
-n $(PYTEST_NPROCS) \
--dist=loadfile \
--ignore=benchmarks \
--test-mode=$(TEST_MODE) \
--junitxml=$(JUNIT_REPORT) \
$(TEST_DIR)
test-debug:
@echo "Running tests (debug sequential)..."
$(PYTEST) \
-n 0 \
--junitxml=$(JUNIT_REPORT) \
$(TEST_DIR)
test-cov:
@echo "Running tests with coverage (parallel)..."
$(PYTEST) \
-n $(PYTEST_NPROCS) \
--dist=loadfile \
--ignore=benchmarks \
--test-mode=$(TEST_MODE) \
--junitxml=$(JUNIT_REPORT) \
--cov=$(SRC_DIR) \
--cov-report=xml:$(COVERAGE_XML) \
--cov-report=$(COVERAGE_TERM) \
$(TEST_DIR)
# test-notebooks target
HAS_GPU := $(shell python3 -c "import torch; print(int(torch.cuda.is_available()))")
ifeq ($(HAS_GPU),1)
NOTEBOOKS := $(wildcard examples/*.ipynb examples/*/*.ipynb)
else
NOTEBOOKS := $(filter-out \
examples/classifier_example.ipynb \
examples/create_dataset_example.ipynb, \
$(wildcard examples/*.ipynb examples/*/*.ipynb))
endif
EXECUTED_STAMPS := $(patsubst %.ipynb,%.executed,$(NOTEBOOKS))
test-notebooks: $(EXECUTED_STAMPS)
%.executed: %.ipynb
jupyter trust $< || { echo "❌ $< is not a valid notebook"; exit 1; }
PYTHONPATH=$(PWD) jupyter nbconvert \
--to notebook \
--execute $< \
--output $(notdir $<) \
--output-dir $(dir $<) \
--ExecutePreprocessor.startup_timeout=300 \
--ExecutePreprocessor.timeout=7200 \
|| { rm -f $@; exit 1; }
touch $@
test-notebooks-clean:
rm -f $(EXECUTED_STAMPS) .cleaned
clean-notebooks:
echo "🧽 Running nbclean on all notebooks…"
nb-clean clean $(NOTEBOOKS) || { echo "❌ nb-clean failed – see the messages above"; exit 1; }
# The *file* that tells make the cleaning has finished.
# It is touched only after the clean step succeeded.
.cleaned: clean-notebooks
touch $@
# Quality Assurance
lint:
@echo "Running Ruff check..."
$(PYTHON) -m ruff check $(SRC_DIR)
format:
@echo "Formatting with Ruff..."
$(PYTHON) -m ruff format $(SRC_DIR)
fix:
@echo "Automatically fixing lint errors..."
$(PYTHON) -m ruff check --fix $(SRC_DIR)
$(PYTHON) -m ruff format $(SRC_DIR)
# Docs
docs:
cd docs && make html && cd ..
open-docs: docs
echo "Opening docs in browser..."
xdg-open docs/build/html/index.html || \
open docs/build/html/index.html || \
start docs/build/html/index.html || \
firefox docs/build/html/index.html
# Benchmarks
BENCHMARK_FILES = \
benchmarks/benchmark_transforms_functional.py \
benchmarks/benchmark_dataset_generation.py
BENCHMARK_DIR ?= .benchmarks
BENCHMARK_OUTPUT ?= $(BENCHMARK_DIR)/benchmark_output.txt
BENCHMARK_JSON ?= $(BENCHMARK_DIR)/benchmark.json
benchmarks: ## Run performance benchmarks
@echo "Running benchmarks..."
mkdir -p $(BENCHMARK_DIR)
$(PYTEST) $(BENCHMARK_FILES) \
--benchmark-only \
--benchmark-save=$(BENCHMARK_DIR) \
--benchmark-json=$(BENCHMARK_JSON) \
--benchmark-histogram \
--benchmark-compare \
-v \
--no-cov 2>&1 | tee $(BENCHMARK_OUTPUT)
benchmarks-clean: ## Remove benchmark data
@echo "Cleaning benchmark data..."
rm -rf $(BENCHMARK_DIR)
# Maintenance
clean:
@echo "Cleaning up..."
# Remove Python cache files
find . -type d -name "__pycache__" -exec rm -rf {} +
find . -type f -name "*.pyc" -delete
# Remove pytest and coverage artifacts
rm -rf .pytest_cache .coverage htmlcov
# Remove temporary directories created by file-handler tests
# (Adjust the pattern if your tests use a different tmp naming convention)
rm -fr "/tmp/pytest-of-$(USER)"
@echo "Cleanup complete."
check-lfs:
@echo "🔍 Checking branch: $(shell git rev-parse --abbrev-ref HEAD) (post filter-repo)"
@LFS_FOUND=0; \
# 1. Check current .gitattributes \
printf "\n1. Checking current .gitattributes...\n"; \
if [ -f .gitattributes ]; then \
if grep -q "filter=lfs" .gitattributes; then \
printf "\033[0;31m ❌ Current .gitattributes contains LFS rules\033[0m\n"; \
LFS_FOUND=1; \
else \
printf "\033[0;32m ✅ Current .gitattributes has no LFS rules\033[0m\n"; \
fi; \
else \
printf "\033[0;32m ✅ No .gitattributes file exists\033[0m\n"; \
fi; \
# 2. Check .gitattributes history \
printf "\n2. Checking .gitattributes history...\n"; \
if git log --all --oneline -- .gitattributes >/dev/null 2>&1; then \
printf " Found .gitattributes in history...\n"; \
git log --all --pretty=format:'%H' -- .gitattributes | while read -r commit; do \
if git show "$$commit:.gitattributes" 2>/dev/null | grep -q "filter=lfs"; then \
SHORT_COMMIT=$${commit:0:7}; \
printf "\033[0;31m ❌ LFS rule found in .gitattributes (commit: $$SHORT_COMMIT)\033[0m\n"; \
LFS_FOUND=1; \
fi; \
done; \
else \
printf "\033[0;32m ✅ No .gitattributes in history\033[0m\n"; \
fi; \
# 3. Check for LFS pointers in current files \
printf "\n3. Checking current files for LFS pointers...\n"; \
git ls-tree -r HEAD --name-only 2>/dev/null | while read -r file; do \
if [ "$$file" = ".gitattributes" ]; then continue; fi; \
blob=$(git rev-parse "HEAD:$$file" 2>/dev/null); \
if [ -n "$$blob" ]; then \
content=$(git cat-file -p "$$blob" 2>/dev/null | head -1); \
if [[ "$$content" == _* ]]; then \
printf "\033[0;31m ❌ LFS pointer found in current file: $$file\033[0m\n"; \
LFS_FOUND=1; \
fi; \
fi; \
done; \
if [ "$$LFS_FOUND" -eq 0 ]; then \
printf "\033[0;32m ✅ No LFS pointers in current files\033[0m\n"; \
fi; \
# 4. Check Git LFS cache \
printf "\n4. Checking Git LFS cache...\n"; \
if [ -d .git/lfs/objects ]; then \
LFS_OBJECTS=$$(find .git/lfs/objects -type f 2>/dev/null | wc -l); \
if [ "$$LFS_OBJECTS" -gt 0 ]; then \
printf "\033[0;31m ❌ Found $$LFS_OBJECTS LFS objects in .git/lfs/objects\033[0m\n"; \
LFS_FOUND=1; \
else \
printf "\033[0;32m ✅ No LFS objects in cache\033[0m\n"; \
fi; \
else \
printf "\033[0;32m ✅ No LFS cache directory\033[0m\n"; \
fi; \
# 5. Check Git config \
printf "\n5. Checking Git config...\n"; \
if git config --get-regexp 'filter.lfs' >/dev/null 2>&1; then \
printf "\033[0;31m ❌ LFS filter found in Git config\033[0m\n"; \
LFS_FOUND=1; \
git config --get-regexp 'filter.lfs' | while read -r key value; do \
printf "\033[0;31m $$key=$$value\033[0m\n"; \
done; \
else \
printf "\033[0;32m ✅ No LFS in Git config\033[0m\n"; \
fi; \
# Final result \
printf "\n=== Summary for $(shell git rev-parse --abbrev-ref HEAD) (post filter-repo) ===\n"; \
if [ "$$LFS_FOUND" -eq 0 ]; then \
printf "\033[0;32m✅ No active Git LFS objects or references found\033[0m\n"; \
printf "\033[0;32m (History may have had LFS, but it's been cleaned)\033[0m\n"; \
exit 0; \
else \
printf "\033[0;31m❌ Active Git LFS references still exist\033[0m\n"; \
printf "\033[1;33m To fix:\033[0m\n"; \
printf "\033[1;33m 1. Remove LFS config: git config --remove-section filter.lfs\033[0m\n"; \
printf "\033[1;33m 2. Prune LFS cache: git lfs prune\033[0m\n"; \
printf "\033[1;33m 3. If needed, run: git filter-repo --invert-paths --path .gitattributes\033[0m\n"; \
exit 1; \
fi
# ==============================================================================
# Notes:
# - Use 'make install' when first setting up the repo.
# - Use 'make test' before every commit.
# - Use 'make clean' if you encounter weird filesystem issues.
# ==============================================================================