-
-
Notifications
You must be signed in to change notification settings - Fork 64
Expand file tree
/
Copy pathMakefile
More file actions
231 lines (199 loc) · 8.89 KB
/
Copy pathMakefile
File metadata and controls
231 lines (199 loc) · 8.89 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
# define configurable variables. May need to be adapted for your environment.
CurrentDir := $(shell pwd)
UV := uv
PY_UV ?= python3.14
VENV_GUI_CPU := .venv-gui-cpu
VENV_GUI_CUDA := .venv-gui-cuda
VENV_CLI_CUDA := .venv-cli-cuda
VENV_TEST_WHL := .venv-test-whl
PYINSTALLER_VENV := $(VENV_GUI_CPU)
PYTHON := $(VENV_GUI_CPU)/bin/python
PYINSTALLER := $(PYINSTALLER_VENV)/bin/pyinstaller
GPU_BACKEND := auto # set to cu128, cu130 etc. to force a specific cuda version. Maybe rocm also works, that hasn't been tested.
BUILD_DIR := dist/
DIR_ICONS := icons
UI_DIR := ui_files/
PACKED_TRANSLATION_DATA := pcleaner/data/translation_generated_files
RC_OUTPUT_DIR := pcleaner/gui/rc_generated_files/
UI_OUTPUT_DIR := pcleaner/gui/ui_generated_files/
UIC_COMPILER := $(VENV_GUI_CPU)/bin/pyside6-uic
I18N_LUPDATE := $(VENV_GUI_CPU)/bin/pyside6-lupdate
I18N_COMPILER := $(VENV_GUI_CPU)/bin/pyside6-lrelease
BLACK_LINE_LENGTH := 100
BLACK_TARGET_DIR := pcleaner/
BLACK_EXCLUDE_PATTERN := "^$(UI_OUTPUT_DIR).*|^pcleaner/comic_text_detector/.*"
LANGUAGES := $(shell python -c "import sys; sys.path.append('.'); from pcleaner.gui.supported_languages import supported_languages; lang_codes = list(supported_languages().keys()); lang_codes.remove('en_US'); print(' '.join(lang_codes))")
PYTHON_VERSION = $(shell $(PYTHON) -c "import sys; print('{}.{}'.format(sys.version_info[0], sys.version_info[1]))")
PYINSTALLER_SITE = $(PYINSTALLER_VENV)/lib/python$(PYTHON_VERSION)/site-packages
export UV_EXCLUDE_NEWER := 10 days
run-gui-cpu:
# Check that the venv exists, then launch.
@if [ ! -d $(VENV_GUI_CPU) ]; then echo "Virtual environment not found. Please run 'make uv-sync-gui-cpu' first."; exit 1; fi
$(VENV_GUI_CPU)/bin/python -m pcleaner.gui.launcher
# print supported languages
print-translated-languages:
@echo $(LANGUAGES)
fresh-install: clean-build build install
refresh-assets: build-icon-cache compile-ui refresh-i18n compile-i18n
uv-sync-gui-cpu:
rm -rf $(VENV_GUI_CPU)
$(UV) venv --python $(PY_UV) $(VENV_GUI_CPU)
$(UV) pip install --python $(VENV_GUI_CPU)/bin/python \
--torch-backend cpu \
--group runtime-base \
--group runtime-gui \
--group runtime-dbus \
--group dev-tools \
--group runtime-torch
uv-sync-gui-cuda:
rm -rf $(VENV_GUI_CUDA)
$(UV) venv --python $(PY_UV) $(VENV_GUI_CUDA)
$(UV) pip install --python $(VENV_GUI_CUDA)/bin/python \
--torch-backend $(GPU_BACKEND) \
--group runtime-base \
--group runtime-gui \
--group runtime-dbus \
--group dev-tools \
--group runtime-torch
uv-sync-cli-cuda:
rm -rf $(VENV_CLI_CUDA)
$(UV) venv --python $(PY_UV) $(VENV_CLI_CUDA)
$(UV) pip install --python $(VENV_CLI_CUDA)/bin/python \
--torch-backend $(GPU_BACKEND) \
--group runtime-base \
--group runtime-dbus \
--group dev-tools \
--group runtime-torch
sync-setup-cfg:
$(PYTHON) tools/sync_setup_cfg.py
# Normal build target. Use the setup-cli-gui.cfg configuration.
build: compile-ui sync-setup-cfg
cp setup-cli-gui.cfg setup.cfg
$(UV) build --out-dir $(BUILD_DIR)
rm setup.cfg
# CLI-only build target. Use the setup-cli.cfg configuration.
build-cli: sync-setup-cfg
cp setup-cli.cfg setup.cfg
$(UV) build --out-dir $(BUILD_DIR)
rm setup.cfg
build-both: build build-cli
# clean-build target
clean-build:
rm -rf $(BUILD_DIR)
# Refresh localization files for the source language, en_US only.
# For some of the i18n, getting Linguist to recognize the strings with enough context added was too
# big of a pain in the ass, so here are python functions that extract the data structures and parse
# out the relevant strings, then create fake code for Linguist to ingest.
refresh-i18n:
mkdir -p translations
PYTHONPATH=/home/corbin/Repos/PanelCleaner $(PYTHON) translations/profile_extractor.py
PYTHONPATH=/home/corbin/Repos/PanelCleaner $(PYTHON) translations/process_steps_extractor.py
$(I18N_LUPDATE) -extensions .py,.ui -no-recursive pcleaner pcleaner/gui pcleaner/gui/CustomQ ui_files \
translations/profile_strings.py translations/process_strings.py -source-language en_US -target-language en_US -ts translations/PanelCleaner_source.ts
# Generate .ts files for each language if they don't already exist.
#Typically handled by Crowdin now, do not overwrite existing files in Crowdin (unless they're fubar).
generate-ts:
$(foreach lang, $(LANGUAGES), \
if [ ! -f translations/PanelCleaner_$(lang).ts ]; then \
echo "Generating TS file for: $(lang)"; \
$(I18N_LUPDATE) -extensions .py,.ui -no-recursive pcleaner pcleaner/gui pcleaner/gui/CustomQ ui_files \
translations/profile_strings.py translations/process_strings.py -source-language en_US -target-language $(lang) -ts translations/PanelCleaner_$(lang).ts; \
fi;)
# Compile localization files for each language, then update the QRC file and compile it.
compile-i18n: generate-ts
$(foreach lang, $(LANGUAGES), $(I18N_COMPILER) translations/PanelCleaner_$(lang).ts -qm $(PACKED_TRANSLATION_DATA)/PanelCleaner_$(lang).qm;)
# compile .ui files
compile-ui:
for file in $(UI_DIR)*.ui; do \
basename=`basename $$file .ui`; \
$(UIC_COMPILER) $$file -o $(UI_OUTPUT_DIR)ui_$$basename.py; \
done
# run build_icon_cache.py
build-icon-cache:
$(PYTHON) $(DIR_ICONS)/build_icon_cache.py
$(PYTHON) $(DIR_ICONS)/copy_from_dark_to_light.py
# install target
install:
$(UV) pip install --python $(PYTHON) $(BUILD_DIR)*.whl
# create a temporary venv and install the newest wheel from dist/
test-whl-install:
rm -rf $(VENV_TEST_WHL)
$(UV) venv $(VENV_TEST_WHL)
NEWEST_WHL="$$(find $(BUILD_DIR) -maxdepth 1 -type f -name '*.whl' -printf '%T@ %p\n' | sort -nr | head -n 1 | cut -d' ' -f2-)"; \
if [ -z "$$NEWEST_WHL" ]; then \
echo "No wheel found in $(BUILD_DIR)"; \
exit 1; \
fi; \
echo "Installing $$NEWEST_WHL into $(VENV_TEST_WHL)"; \
$(UV) pip install --python $(VENV_TEST_WHL)/bin/python "$$NEWEST_WHL"
# clean target
clean:
rm -rf build
rm -rf dist
rm -rf dist-elf/PanelCleaner
rm -rf .pytest_cache
rm -rf pcleaner.egg-info
rm -rf AUR/panelcleaner/pkg
rm -rf AUR/panelcleaner/src
rm -rf AUR/panelcleaner/*.tar.gz
rm -rf AUR/panelcleaner/*.tar.zst
# format the code
black-format:
find $(BLACK_TARGET_DIR) -type f -name '*.py' | grep -Ev $(BLACK_EXCLUDE_PATTERN) | xargs black --line-length $(BLACK_LINE_LENGTH)
release: confirm
$(VENV_GUI_CPU)/bin/python -m twine upload $(BUILD_DIR)*
build-elf:
$(PYINSTALLER) pcleaner/main.py \
--paths "${PYINSTALLER_SITE}" \
--onedir --noconfirm --clean --workpath=build --distpath=dist-elf --windowed \
--name="PanelCleaner" \
--copy-metadata=filelock \
--copy-metadata=huggingface-hub \
--copy-metadata=numpy \
--copy-metadata=packaging \
--copy-metadata=pyyaml \
--copy-metadata=pillow \
--copy-metadata=regex \
--copy-metadata=requests \
--copy-metadata=safetensors \
--copy-metadata=tokenizers \
--copy-metadata=tqdm \
--copy-metadata=torch \
--copy-metadata=manga_ocr \
--collect-data=torch \
--collect-data=unidic_lite \
--hidden-import=scipy.signal \
--add-data "${PYINSTALLER_SITE}/manga_ocr:manga_ocr/" \
--collect-data pcleaner
# Note on adding data for manga_ocr. For some reason PyInstaller stopped including the source files for manga_ocr,
# which transformers needs to run, so we manually tell it to include it as data. Hacky but necessary, for now.
# This stupid thing refuses to collect data, so do it manually:
@echo "Copying data files..."
mkdir -p dist-elf/PanelCleaner/_internal/pcleaner
cp -r pcleaner/data dist-elf/PanelCleaner/_internal/pcleaner/
@echo "Purging __pycache__ directories..."
@find dist-elf/PanelCleaner/_internal/pcleaner -type d -name "__pycache__"
@echo "Confiming deletion of __pycache__ directories..."
@find dist-elf/PanelCleaner/_internal/pcleaner -type d -name "__pycache__" -exec rm -rf {} \; || true
# The cpu-limited install of torch now cuts out most of them, but one or two might still be lurking here,
# so clean them up just in case.
@echo "Purging CUDA related files from _internal directory..."
@find dist-elf/PanelCleaner/_internal -type f \( \
-name 'libtorch_cuda.so' -o \
-name 'libc10_cuda.so' -o \
-name 'libcusparse.so*' -o \
-name 'libcurand.so*' -o \
-name 'libcudnn.so*' -o \
-name 'libcublasLt.so*' -o \
-name 'libcublas.so*' -o \
-name 'libcupti.so*' -o \
-name 'libcufft.so*' -o \
-name 'libcudart.so*' -o \
-name 'libnv*' -o \
-name 'libnccl.so*' \
\) -exec rm -rf {} \;
requirements-tested-gui-cpu:
# These are the packages that are tested on Linux and used for the Flatpak build.
@if [ ! -x $(VENV_GUI_CPU)/bin/python ]; then echo "Virtual environment not found. Please run 'make uv-sync-gui-cpu' first."; exit 1; fi
$(UV) pip freeze --no-cache --python $(VENV_GUI_CPU)/bin/python > requirements_tested.txt
.PHONY: confirm clean build build-cli build-both install test-whl-install fresh-install release refresh-i18n compile-i18n compile-ui build-icon-cache refresh-assets black-format pip-install-torch-no-cuda build-elf build-app-image uv-sync-gui-cpu uv-sync-gui-cuda uv-sync-cli-cuda sync-setup-cfg requirements-tested-gui-cpu