diff --git a/.tekton/lightspeed-stack-pull-request.yaml b/.tekton/lightspeed-stack-pull-request.yaml index 268679c3b..3c6658da1 100644 --- a/.tekton/lightspeed-stack-pull-request.yaml +++ b/.tekton/lightspeed-stack-pull-request.yaml @@ -34,7 +34,35 @@ spec: - name: build-source-image value: 'true' - name: prefetch-input - value: '[{"type": "rpm", "path": "."}, {"type": "pip", "path": ".", "allow_binary": "true", "requirements_files": ["requirements.x86_64.txt", "requirements.aarch64.txt", "requirements.hermetic.txt", "requirements.torch.txt"]}]' + # no source available: torch, faiss-cpu + # hermeto prefetch problems: uv, pip, jiter, tiktoken, + # to accelerate build:numpy, scipy, pandas, pillow, scikit_learn + # those need cmake to build: pyarrow + # those need cargo to build: jiter, tiktoken, cryptography, fastuuid, hf_xet, maturin, pydantic_core, rpds_py, safetensors, tokenizers + value: | + [ + { + "type": "rpm", + "path": "." + }, + { + "type": "pip", + "path": ".", + "requirements_files": [ + "requirements.x86_64.txt", + "requirements.aarch64.txt", + "requirements.hermetic.txt", + "requirements.torch.txt" + ], + "requirements_build_files": ["requirements-build.txt"], + "binary": { + "packages": "torch,faiss-cpu,uv,pip,jiter,tiktoken,numpy,scipy,pandas,pillow,scikit_learn,pyarrow,cryptography,fastuuid,hf_xet,maturin,pydantic_core,rpds_py,safetensors,tokenizers", + "os": "linux", + "arch": "x86_64,aarch64", + "py_version": "312" + } + } + ] - name: hermetic value: 'true' - name: dockerfile diff --git a/.tekton/lightspeed-stack-push.yaml b/.tekton/lightspeed-stack-push.yaml index bec184957..e25588a45 100644 --- a/.tekton/lightspeed-stack-push.yaml +++ b/.tekton/lightspeed-stack-push.yaml @@ -31,7 +31,30 @@ spec: - name: build-source-image value: 'true' - name: prefetch-input - value: '[{"type": "rpm", "path": "."}, {"type": "pip", "path": ".", "allow_binary": "true", "requirements_files": ["requirements.x86_64.txt", "requirements.aarch64.txt", "requirements.hermetic.txt", "requirements.torch.txt"]}]' + value: | + [ + { + "type": "rpm", + "path": "." + }, + { + "type": "pip", + "path": ".", + "requirements_files": [ + "requirements.x86_64.txt", + "requirements.aarch64.txt", + "requirements.hermetic.txt", + "requirements.torch.txt" + ], + "requirements_build_files": ["requirements-build.txt"], + "binary": { + "packages": "torch,faiss-cpu,uv,pip,jiter,tiktoken,numpy,scipy,pandas,pillow,scikit_learn,pyarrow,cryptography,fastuuid,hf_xet,maturin,pydantic_core,rpds_py,safetensors,tokenizers", + "os": "linux", + "arch": "x86_64,aarch64", + "py_version": "312" + } + } + ] - name: hermetic value: 'true' - name: dockerfile diff --git a/Containerfile b/Containerfile index c35cc3ad0..6632706d1 100644 --- a/Containerfile +++ b/Containerfile @@ -31,7 +31,10 @@ COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR # PIP_FIND_LINKS=/cachi2/output/deps/pip # PIP_NO_INDEX=true RUN if [ -f /cachi2/cachi2.env ]; then \ - . /cachi2/cachi2.env && uv venv --seed --no-index --find-links ${PIP_FIND_LINKS} && . .venv/bin/activate && pip install --no-index --find-links ${PIP_FIND_LINKS} -r requirements.$(uname -m).txt -r requirements.torch.txt; \ + . /cachi2/cachi2.env && \ + uv venv --seed --no-index --find-links ${PIP_FIND_LINKS} && \ + . .venv/bin/activate && \ + pip install --no-index --find-links ${PIP_FIND_LINKS} -r requirements.$(uname -m).txt -r requirements.torch.txt; \ else \ uv sync --locked --no-dev --group llslibdev; \ fi diff --git a/Makefile b/Makefile index 2dd9fc58a..f2c845ba5 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +SHELL := /bin/bash + ARTIFACT_DIR := $(if $(ARTIFACT_DIR),$(ARTIFACT_DIR),tests/test_results) PATH_TO_PLANTUML := ~/bin @@ -114,6 +116,9 @@ konflux-requirements: ## generate hermetic requirements.*.txt file for konflux b ./scripts/remove_torch_deps.sh requirements.x86_64.txt ./scripts/remove_torch_deps.sh requirements.aarch64.txt echo "torch==${TORCH_VERSION}" | uv pip compile - -o requirements.torch.txt --generate-hashes --python-version 3.12 --torch-backend cpu --emit-index-url --no-deps --index-url https://download.pytorch.org/whl/cpu --refresh + uv run pybuild-deps compile --output-file=requirements-build.txt \ + <(grep -v "^faiss-cpu" requirements.hermetic.txt) \ + <(grep -v "^faiss-cpu" requirements.x86_64.txt) help: ## Show this help screen @echo 'Usage: make ... ' diff --git a/pyproject.toml b/pyproject.toml index fd6028e38..5a7eba49b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,6 +122,7 @@ dev = [ "openapi-to-md>=0.1.0b2", "pytest-subtests>=0.14.2", "bandit>=1.8.6", + "pybuild-deps>=0.5.0", ] llslibdev = [ # To check llama-stack API provider dependecies: diff --git a/requirements-build.txt b/requirements-build.txt new file mode 100644 index 000000000..c5affb330 --- /dev/null +++ b/requirements-build.txt @@ -0,0 +1,306 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pybuild-deps compile --output-file=requirements-build.txt /dev/fd/62 /dev/fd/63 +# +beniget==0.4.2.post1 + # via pythran +calver==2025.10.20 + # via trove-classifiers +cffi==2.0.0 + # via cryptography +coherent-licensed==0.5.2 + # via + # importlib-metadata + # zipp +cython==3.1.8 + # via scipy +cython==3.2.4 + # via + # asyncpg + # lxml + # numpy + # pandas + # pyarrow + # pyyaml + # scikit-learn + # sqlalchemy +dunamai==1.25.0 + # via uv-dynamic-versioning +expandvars==1.1.2 + # via + # frozenlist + # propcache + # yarl +flit-core==3.12.0 + # via + # aiosqlite + # click + # coherent-licensed + # idna + # jinja2 + # markdown-it-py + # mdurl + # packaging + # pathspec + # pip + # pyproject-metadata + # threadpoolctl + # typing-extensions + # wheel +gast==0.6.0 + # via + # beniget + # pythran +hatch-fancy-pypi-readme==25.1.0 + # via + # attrs + # httpcore + # httpx + # jsonschema + # llama-stack-client + # openai + # pydantic +hatch-vcs==0.5.0 + # via + # attrs + # filelock + # fsspec + # jsonschema + # jsonschema-specifications + # referencing + # scikit-build-core + # termcolor + # urllib3 +hatchling==1.26.3 + # via + # hatch-fancy-pypi-readme + # llama-stack-client + # openai +hatchling==1.28.0 + # via + # annotated-types + # attrs + # dnspython + # expandvars + # filelock + # fsspec + # hatch-fancy-pypi-readme + # hatch-vcs + # httpcore + # httpx + # jsonschema + # jsonschema-specifications + # mcp + # opentelemetry-api + # opentelemetry-exporter-otlp + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-instrumentation + # opentelemetry-proto + # opentelemetry-sdk + # opentelemetry-semantic-conventions + # pydantic + # pydantic-settings + # pygments + # python-multipart + # referencing + # scikit-build-core + # starlette + # termcolor + # typing-inspection + # urllib3 + # uv-dynamic-versioning + # uvicorn +jinja2==3.1.6 + # via uv-dynamic-versioning +markupsafe==3.0.3 + # via jinja2 +maturin==1.11.2 + # via + # cryptography + # fastuuid + # hf-xet + # jiter + # pydantic-core + # rpds-py + # safetensors + # tokenizers + # uv +meson-python==0.18.0 + # via + # numpy + # pandas + # scikit-learn + # scipy +meson==1.10.0 + # via + # meson-python + # pandas +numpy==2.2.6 + # via + # pandas + # pyarrow + # scikit-learn + # scipy +packaging==25.0 + # via + # hatchling + # meson-python + # scikit-build-core + # setuptools-scm +pathspec==1.0.1 + # via + # hatchling + # scikit-build-core +pdm-backend==2.4.6 + # via + # annotated-doc + # fastapi +pkgconfig==1.5.5 + # via aiohttp +pluggy==1.6.0 + # via hatchling +ply==3.11 + # via pythran +poetry-core==2.2.1 + # via + # aiohappyeyeballs + # dunamai + # litellm + # pkgconfig + # rich + # rsa + # tomlkit +pybind11==3.0.1 + # via + # pillow + # scipy +pycparser==2.23 + # via cffi +pyproject-metadata==0.10.0 + # via meson-python +pythran==0.18.1 + # via scipy +scikit-build-core==0.11.6 + # via pybind11 +scipy==1.16.3 + # via scikit-learn +semantic-version==2.10.0 + # via setuptools-rust +setuptools-rust==1.12.0 + # via + # maturin + # tiktoken +setuptools-scm==7.1.0 + # via python-dateutil +setuptools-scm==9.2.2 + # via + # anyio + # hatch-vcs + # httpx-sse + # importlib-metadata + # pluggy + # pyarrow + # setuptools-rust + # sniffio + # tqdm + # urllib3 + # zipp +tomlkit==0.13.3 + # via uv-dynamic-versioning +trove-classifiers==2025.12.1.14 + # via hatchling +typing-extensions==4.15.0 + # via setuptools-scm +uv-dynamic-versioning==0.12.0 + # via mcp +versioneer==0.29 + # via pandas +wheel==0.45.1 + # via + # authlib + # cachetools + # fire + # httpx-sse + # litellm + # meson + # pandas + # python-dateutil + # sentence-transformers + # tiktoken + # tqdm + # tzdata + +# The following packages are considered to be unsafe in a requirements file: +setuptools==80.9.0 + # via + # aiohttp + # aiosignal + # anyio + # asyncpg + # authlib + # blobfile + # cachetools + # calver + # certifi + # cffi + # charset-normalizer + # cryptography + # dill + # distro + # emoji + # fire + # frozenlist + # googleapis-common-protos + # greenlet + # httpx-sse + # importlib-metadata + # joblib + # llama-stack + # lxml + # markupsafe + # maturin + # meson + # mpmath + # multidict + # multiprocess + # networkx + # pathspec + # pillow + # pluggy + # polyleven + # prometheus-client + # prompt-toolkit + # propcache + # psutil + # pyarrow + # pyasn1 + # pyasn1-modules + # pycryptodomex + # pyjwt + # python-dateutil + # python-dotenv + # pythran + # pyyaml + # regex + # semver + # sentence-transformers + # setuptools-rust + # setuptools-scm + # sniffio + # sqlalchemy + # sse-starlette + # tiktoken + # tqdm + # tree-sitter + # trl + # trove-classifiers + # tzdata + # versioneer + # wrapt + # xxhash + # yarl + # zipp diff --git a/uv.lock b/uv.lock index 90327ad9a..10e656bdc 100644 --- a/uv.lock +++ b/uv.lock @@ -1253,6 +1253,7 @@ dev = [ { name = "build" }, { name = "mypy" }, { name = "openapi-to-md" }, + { name = "pybuild-deps" }, { name = "pydocstyle" }, { name = "pylint" }, { name = "pyright" }, @@ -1330,6 +1331,7 @@ dev = [ { name = "build", specifier = ">=1.2.2.post1" }, { name = "mypy", specifier = ">=1.16.0" }, { name = "openapi-to-md", specifier = ">=0.1.0b2" }, + { name = "pybuild-deps", specifier = ">=0.5.0" }, { name = "pydocstyle", specifier = ">=6.3.0" }, { name = "pylint", specifier = ">=3.3.7" }, { name = "pyright", specifier = ">=1.1.401" }, @@ -2134,6 +2136,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ac/82/8b9b97bba2e3576a340f93b044a3a3a09841170ab4c1eb0d5c93469fd32f/pillow-12.1.0-cp313-cp313t-win_arm64.whl", hash = "sha256:80941e6d573197a0c28f394753de529bb436b1ca990ed6e765cf42426abc39f8", size = 2454547, upload-time = "2026-01-02T09:12:18.704Z" }, ] +[[package]] +name = "pip" +version = "25.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/6e/74a3f0179a4a73a53d66ce57fdb4de0080a8baa1de0063de206d6167acc2/pip-25.3.tar.gz", hash = "sha256:8d0538dbbd7babbd207f261ed969c65de439f6bc9e5dbd3b3b9a77f25d95f343", size = 1803014, upload-time = "2025-10-25T00:55:41.394Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/3c/d717024885424591d5376220b5e836c2d5293ce2011523c9de23ff7bf068/pip-25.3-py3-none-any.whl", hash = "sha256:9655943313a94722b7774661c21049070f6bbb0a1516bf02f7c8d5d9201514cd", size = 1778622, upload-time = "2025-10-25T00:55:39.247Z" }, +] + +[[package]] +name = "pip-tools" +version = "7.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "build" }, + { name = "click" }, + { name = "pip" }, + { name = "pyproject-hooks" }, + { name = "setuptools" }, + { name = "wheel" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/79/d149fb40bc425ad9defcb8ff73c65088bbc36a84b1825e035397d1c40624/pip_tools-7.5.2.tar.gz", hash = "sha256:2d64d72da6a044da1110257d333960563d7a4743637e8617dd2610ae7b82d60f", size = 164815, upload-time = "2025-11-12T22:46:12.627Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/c1/61aef9517201b43cc20f4a5c9339a072644cbaf0e9ce4e4970c2a105f766/pip_tools-7.5.2-py3-none-any.whl", hash = "sha256:2fe16db727bbe5bf28765aeb581e792e61be51fc275545ef6725374ad720a1ce", size = 66905, upload-time = "2025-11-12T22:46:11.374Z" }, +] + [[package]] name = "platformdirs" version = "4.5.1" @@ -2387,6 +2415,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, ] +[[package]] +name = "pybuild-deps" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "pip-tools" }, + { name = "requests" }, + { name = "xdg" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/33/ed89d0113158801aa111178f7b7671e336bf89005eeed8a1eeca4d14a0ff/pybuild_deps-0.5.0.tar.gz", hash = "sha256:fa488db42cc53f93926ccb55ef56fb300fbd7769d31a56ebc7f83f11e28aeac8", size = 26352, upload-time = "2025-03-15T18:28:46.224Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/d6/b632fc0440417a886fc5e066c1d7a0c1a6e13d024181cd60fbca9d2001e8/pybuild_deps-0.5.0-py3-none-any.whl", hash = "sha256:4cc5b8634b5aac371755a7ff33da1f47cf528938e419c1fb943cc95a8c3337e7", size = 29284, upload-time = "2025-03-15T18:28:45.029Z" }, +] + [[package]] name = "pycparser" version = "2.23" @@ -3564,6 +3607,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, ] +[[package]] +name = "wheel" +version = "0.45.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/98/2d9906746cdc6a6ef809ae6338005b3f21bb568bea3165cfc6a243fdc25c/wheel-0.45.1.tar.gz", hash = "sha256:661e1abd9198507b1409a20c02106d9670b2576e916d58f520316666abca6729", size = 107545, upload-time = "2024-11-23T00:18:23.513Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/2c/87f3254fd8ffd29e4c02732eee68a83a1d3c346ae39bc6822dcbcb697f2b/wheel-0.45.1-py3-none-any.whl", hash = "sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248", size = 72494, upload-time = "2024-11-23T00:18:21.207Z" }, +] + [[package]] name = "wrapt" version = "1.17.3" @@ -3593,6 +3645,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, ] +[[package]] +name = "xdg" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2a/b9/0e6e6f19fb75cf5e1758f4f33c1256738f718966700cffc0fde2f966218b/xdg-6.0.0.tar.gz", hash = "sha256:24278094f2d45e846d1eb28a2ebb92d7b67fc0cab5249ee3ce88c95f649a1c92", size = 3453, upload-time = "2023-02-27T19:27:44.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/54/3516c1cf349060fc3578686d271eba242f10ec00b4530c2985af9faac49b/xdg-6.0.0-py3-none-any.whl", hash = "sha256:df3510755b4395157fc04fc3b02467c777f3b3ca383257397f09ab0d4c16f936", size = 3855, upload-time = "2023-02-27T19:27:42.151Z" }, +] + [[package]] name = "xxhash" version = "3.6.0"