From 942d1551244c893c771cc8b1a21d08ee0923a8c7 Mon Sep 17 00:00:00 2001 From: YongZhuIntel Date: Thu, 11 Sep 2025 16:57:16 +0800 Subject: [PATCH 1/2] Fix PSIRT Vulnerability - Dependency Confusion in oneccl_bind_pt package --- docker/llm/serving/cpu/docker/Dockerfile | 2 +- .../deepspeed_autotp_fastapi_quickstart.md | 2 +- .../QLoRA-FineTuning/alpaca-qlora/README.md | 2 +- .../Self-Speculation/baichuan2/README.md | 2 +- .../Self-Speculation/llama2/README.md | 2 +- .../Self-Speculation/llama3/Dockerfile | 106 ++++++++++++++++++ .../Self-Speculation/llama3/README.md | 2 +- .../Self-Speculation/mistral/README.md | 2 +- .../GPU/Deepspeed-AutoTP-FastAPI/README.md | 2 +- .../GPU/LLM-Finetuning/HF-PEFT/README.md | 2 +- .../example/GPU/LLM-Finetuning/LoRA/README.md | 2 +- .../LoRA/chatglm_finetune/README.md | 2 +- .../GPU/LLM-Finetuning/QA-LoRA/README.md | 2 +- .../QLoRA/alpaca-qlora/README.md | 2 +- .../GPU/LLM-Finetuning/ReLora/README.md | 2 +- .../GPU/Pipeline-Parallel-Inference/README.md | 2 +- .../GPU/Pipeline-Parallel-Serving/README.md | 2 +- 17 files changed, 122 insertions(+), 16 deletions(-) create mode 100644 python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama3/Dockerfile diff --git a/docker/llm/serving/cpu/docker/Dockerfile b/docker/llm/serving/cpu/docker/Dockerfile index 29fa58c7482..7da76bbd2c3 100644 --- a/docker/llm/serving/cpu/docker/Dockerfile +++ b/docker/llm/serving/cpu/docker/Dockerfile @@ -75,7 +75,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ pip install Jinja2==3.1.3 && \ pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cpu && \ pip install intel-extension-for-pytorch==2.2.0 && \ - pip install oneccl_bind_pt==2.2.0 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/cn/ && \ + pip install oneccl_bind_pt==2.2.0 --index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/cn/ && \ pip install transformers==4.36.2 && \ # Install vllm dependencies pip install --upgrade fastapi && \ diff --git a/docs/mddocs/Quickstart/deepspeed_autotp_fastapi_quickstart.md b/docs/mddocs/Quickstart/deepspeed_autotp_fastapi_quickstart.md index 0fa9888b60c..70185b86446 100644 --- a/docs/mddocs/Quickstart/deepspeed_autotp_fastapi_quickstart.md +++ b/docs/mddocs/Quickstart/deepspeed_autotp_fastapi_quickstart.md @@ -20,7 +20,7 @@ conda create -n llm python=3.11 conda activate llm # below command will install intel_extension_for_pytorch==2.1.10+xpu as default pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ -pip install oneccl_bind_pt==2.1.100 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ +pip install oneccl_bind_pt==2.1.100 --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ # configures OneAPI environment variables source /opt/intel/oneapi/setvars.sh pip install git+https://github.com/microsoft/DeepSpeed.git@ed8aed5 diff --git a/python/llm/example/CPU/QLoRA-FineTuning/alpaca-qlora/README.md b/python/llm/example/CPU/QLoRA-FineTuning/alpaca-qlora/README.md index 899f7d8bd6c..dc1cc6f7ad1 100644 --- a/python/llm/example/CPU/QLoRA-FineTuning/alpaca-qlora/README.md +++ b/python/llm/example/CPU/QLoRA-FineTuning/alpaca-qlora/README.md @@ -53,7 +53,7 @@ python ./alpaca_qlora_finetuning_cpu.py \ ```bash # need to run the alpaca stand-alone version first # for using mpirun -pip install oneccl_bind_pt --extra-index-url https://developer.intel.com/ipex-whl-stable +pip install oneccl_bind_pt --index-url https://developer.intel.com/ipex-whl-stable ``` 2. modify conf in `finetune_one_node_two_sockets.sh` and run diff --git a/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/baichuan2/README.md b/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/baichuan2/README.md index ece64c342ff..26a3ba91dbf 100644 --- a/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/baichuan2/README.md +++ b/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/baichuan2/README.md @@ -69,7 +69,7 @@ To accelerate speculative decoding on CPU, optionally, you can install our valid ```bash python -m pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cpu python -m pip install intel-extension-for-pytorch==2.2.0 -python -m pip install oneccl_bind_pt==2.2.0 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/us/ +python -m pip install oneccl_bind_pt==2.2.0 --index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/us/ # if there is any installation problem for oneccl_binding, you can also find suitable index url at "https://pytorch-extension.intel.com/release-whl/stable/cpu/cn/" or "https://developer.intel.com/ipex-whl-stable-cpu" according to your environment. # Install other dependencies diff --git a/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama2/README.md b/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama2/README.md index e5702c1286b..04750429ff2 100644 --- a/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama2/README.md +++ b/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama2/README.md @@ -104,7 +104,7 @@ To accelerate speculative decoding on CPU, you can install our validated version # Install IPEX 2.2.0+cpu python -m pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cpu python -m pip install intel-extension-for-pytorch==2.2.0 -python -m pip install oneccl_bind_pt==2.2.0 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/us/ +python -m pip install oneccl_bind_pt==2.2.0 --index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/us/ # if there is any installation problem for oneccl_binding, you can also find suitable index url at "https://pytorch-extension.intel.com/release-whl/stable/cpu/cn/" or "https://developer.intel.com/ipex-whl-stable-cpu" according to your environment. # Update transformers diff --git a/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama3/Dockerfile b/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama3/Dockerfile new file mode 100644 index 00000000000..7da76bbd2c3 --- /dev/null +++ b/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama3/Dockerfile @@ -0,0 +1,106 @@ +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# Stage 1: Build stage to handle file preparation +FROM ubuntu:22.04 as build + +# Copy the files to the build image +COPY ./start-notebook.sh /llm/ +COPY ./model_adapter.py.patch /llm/ +COPY ./vllm_offline_inference.py /llm/ +COPY ./payload-1024.lua /llm/ +COPY ./start-vllm-service.sh /llm/ +COPY ./benchmark_vllm_throughput.py /llm/ +COPY ./start-fastchat-service.sh /llm/ + +# Stage 2: Final image that only includes necessary runtime artifacts +FROM ubuntu:22.04 + +# Copy the scripts from the build stage +COPY --from=build /llm /llm/ + +ARG http_proxy +ARG https_proxy +# Disable pip's cache behavior +ARG PIP_NO_CACHE_DIR=false +ARG DEBIAN_FRONTEND=noninteractive + +ENV PYTHONUNBUFFERED=1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + # Install basic utilities + libunwind8-dev vim less \ + # Version control and download tools + git curl wget \ + # add-apt-repository requires gnupg, gpg-agent, software-properties-common + gnupg gpg-agent software-properties-common \ + # Install performance testing tool, NUMA (Non-Uniform Memory Access) support, and patch tool + wrk numactl patch && \ +# Install Python 3.11 + # Add Python 3.11 PPA repository + add-apt-repository ppa:deadsnakes/ppa -y && \ + # Install Python 3.11 and related packages + apt-get update && apt-get install -y --no-install-recommends python3.11 python3-pip python3.11-dev python3-wheel python3.11-distutils && \ + # Remove the original /usr/bin/python3 symbolic link + rm /usr/bin/python3 && \ + # Create a symbolic link pointing to Python 3.11 at /usr/bin/python3 + ln -s /usr/bin/python3.11 /usr/bin/python3 && \ + # Create a symbolic link pointing to /usr/bin/python3 at /usr/bin/python + ln -s /usr/bin/python3 /usr/bin/python && \ +# Download and install pip, install FastChat from source requires PEP 660 support + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py && \ + rm get-pip.py && \ +# Install Basic Python utilities + pip install --upgrade requests argparse urllib3 && \ +# Download ipex-llm-tutorial + pip install --upgrade jupyterlab && \ + git clone https://github.com/intel-analytics/ipex-llm-tutorial /llm/ipex-llm-tutorial && \ + chmod +x /llm/start-notebook.sh && \ +# Download all-in-one benchmark + git clone https://github.com/intel-analytics/IPEX-LLM && \ + cp -r ./IPEX-LLM/python/llm/dev/benchmark/ /llm/benchmark && \ +# Copy chat.py script + pip install --upgrade colorama && \ + cp -r ./IPEX-LLM/python/llm/portable-zip/ /llm/portable-zip && \ +# Install all-in-one dependencies + pip install --upgrade omegaconf && \ + pip install --upgrade pandas && \ +# Install ipex-llm + pip uninstall -y setuptools && \ + pip install --pre --upgrade ipex-llm[serving] && \ + # Fix Trivy CVE Issues + pip install Jinja2==3.1.3 transformers==4.36.2 gradio==4.19.2 cryptography==42.0.4 && \ + # Fix CVE-2024-22195 + pip install Jinja2==3.1.3 && \ + pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cpu && \ + pip install intel-extension-for-pytorch==2.2.0 && \ + pip install oneccl_bind_pt==2.2.0 --index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/cn/ && \ + pip install transformers==4.36.2 && \ +# Install vllm dependencies + pip install --upgrade fastapi && \ + pip install --upgrade "uvicorn[standard]" && \ +# Add Qwen support + pip install --upgrade transformers_stream_generator einops && \ +# Fix Qwen model adapter in fastchat + patch /usr/local/lib/python3.11/dist-packages/fastchat/model/model_adapter.py < /llm/model_adapter.py.patch && \ +# Copy vLLM-Serving + cp -r ./IPEX-LLM/python/llm/example/CPU/vLLM-Serving/ /llm/vLLM-Serving && \ + rm -rf ./IPEX-LLM && \ +# Fix vllm service + pip install pydantic==1.10.11 && \ +# Install vllm + apt-get install -y g++ gcc-12 g++-12 libnuma-dev && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 && \ + git clone https://github.com/vllm-project/vllm.git && \ + cd ./vllm && \ + git checkout v0.6.6.post1 && \ + pip install cmake==3.31.6 wheel packaging ninja "setuptools-scm>=8" numpy && \ + pip uninstall -y intel-extension-for-pytorch && \ + pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu && \ + VLLM_TARGET_DEVICE=cpu python3 setup.py install && \ + pip install ray triton==3.1.0 && \ +# Clean up unnecessary files to reduce image size + rm -rf /var/lib/apt/lists/* + +WORKDIR /llm/ diff --git a/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama3/README.md b/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama3/README.md index 84a0df2b932..01f214730fa 100644 --- a/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama3/README.md +++ b/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama3/README.md @@ -81,7 +81,7 @@ To accelerate speculative decoding on CPU, you can install our validated version # Install IPEX 2.2.0+cpu python -m pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cpu python -m pip install intel-extension-for-pytorch==2.2.0 -python -m pip install oneccl_bind_pt==2.2.0 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/us/ +python -m pip install oneccl_bind_pt==2.2.0 --index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/us/ # if there is any installation problem for oneccl_binding, you can also find suitable index url at "https://pytorch-extension.intel.com/release-whl/stable/cpu/cn/" or "https://developer.intel.com/ipex-whl-stable-cpu" according to your environment. # Update transformers diff --git a/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/mistral/README.md b/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/mistral/README.md index b58007a66a7..5d43aae9f6c 100644 --- a/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/mistral/README.md +++ b/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/mistral/README.md @@ -90,7 +90,7 @@ To accelerate speculative decoding on CPU, you can install our validated version # Install IPEX 2.2.0+cpu python -m pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cpu python -m pip install intel-extension-for-pytorch==2.2.0 -python -m pip install oneccl_bind_pt==2.2.0 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/us/ +python -m pip install oneccl_bind_pt==2.2.0 --index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/us/ # if there is any installation problem for oneccl_binding, you can also find suitable index url at "https://pytorch-extension.intel.com/release-whl/stable/cpu/cn/" or "https://developer.intel.com/ipex-whl-stable-cpu" according to your environment. # Update transformers diff --git a/python/llm/example/GPU/Deepspeed-AutoTP-FastAPI/README.md b/python/llm/example/GPU/Deepspeed-AutoTP-FastAPI/README.md index 437c23d484a..a84072fc743 100644 --- a/python/llm/example/GPU/Deepspeed-AutoTP-FastAPI/README.md +++ b/python/llm/example/GPU/Deepspeed-AutoTP-FastAPI/README.md @@ -15,7 +15,7 @@ conda create -n llm python=3.11 conda activate llm # below command will install intel_extension_for_pytorch==2.1.10+xpu as default pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ -pip install oneccl_bind_pt==2.1.100 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ +pip install oneccl_bind_pt==2.1.100 --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ # configures OneAPI environment variables source /opt/intel/oneapi/setvars.sh pip install git+https://github.com/microsoft/DeepSpeed.git@ed8aed5 diff --git a/python/llm/example/GPU/LLM-Finetuning/HF-PEFT/README.md b/python/llm/example/GPU/LLM-Finetuning/HF-PEFT/README.md index b0078690509..e1cbcf6beb6 100644 --- a/python/llm/example/GPU/LLM-Finetuning/HF-PEFT/README.md +++ b/python/llm/example/GPU/LLM-Finetuning/HF-PEFT/README.md @@ -17,7 +17,7 @@ pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-exte pip install transformers==4.45.0 "trl<0.12.0" datasets pip install bitsandbytes==0.45.1 scipy pip install fire peft==0.10.0 -pip install oneccl_bind_pt==2.1.100 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ # necessary to run distributed finetuning +pip install oneccl_bind_pt==2.1.100 --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ # necessary to run distributed finetuning ``` ### 2. Configures OneAPI environment variables diff --git a/python/llm/example/GPU/LLM-Finetuning/LoRA/README.md b/python/llm/example/GPU/LLM-Finetuning/LoRA/README.md index 0c5dfc1095e..db87a07d5c2 100644 --- a/python/llm/example/GPU/LLM-Finetuning/LoRA/README.md +++ b/python/llm/example/GPU/LLM-Finetuning/LoRA/README.md @@ -15,7 +15,7 @@ pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-exte pip install transformers==4.45.0 "trl<0.12.0" datasets pip install fire peft==0.10.0 pip install bitsandbytes==0.45.1 scipy -pip install oneccl_bind_pt==2.1.100 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ # necessary to run distributed finetuning +pip install oneccl_bind_pt==2.1.100 --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ # necessary to run distributed finetuning ``` ### 2. Configures OneAPI environment variables diff --git a/python/llm/example/GPU/LLM-Finetuning/LoRA/chatglm_finetune/README.md b/python/llm/example/GPU/LLM-Finetuning/LoRA/chatglm_finetune/README.md index cad1665acbc..77d55667ef7 100644 --- a/python/llm/example/GPU/LLM-Finetuning/LoRA/chatglm_finetune/README.md +++ b/python/llm/example/GPU/LLM-Finetuning/LoRA/chatglm_finetune/README.md @@ -21,7 +21,7 @@ pip install "deepspeed==0.13.1" pip install "mpi4py>=3.1.5" # below command will install intel_extension_for_pytorch==2.1.10+xpu as default pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ -pip install oneccl_bind_pt==2.1.100 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ +pip install oneccl_bind_pt==2.1.100 --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ ``` ### 2. Configures OneAPI Environment Variables diff --git a/python/llm/example/GPU/LLM-Finetuning/QA-LoRA/README.md b/python/llm/example/GPU/LLM-Finetuning/QA-LoRA/README.md index 08be0bc8748..9cc311d2923 100644 --- a/python/llm/example/GPU/LLM-Finetuning/QA-LoRA/README.md +++ b/python/llm/example/GPU/LLM-Finetuning/QA-LoRA/README.md @@ -15,7 +15,7 @@ pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-exte pip install transformers==4.45.0 "trl<0.12.0" datasets pip install fire peft==0.10.0 pip install bitsandbytes==0.45.1 scipy -pip install oneccl_bind_pt==2.1.100 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ # necessary to run distributed finetuning +pip install oneccl_bind_pt==2.1.100 --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ # necessary to run distributed finetuning ``` ### 2. Configures OneAPI environment variables diff --git a/python/llm/example/GPU/LLM-Finetuning/QLoRA/alpaca-qlora/README.md b/python/llm/example/GPU/LLM-Finetuning/QLoRA/alpaca-qlora/README.md index c859ca5a7a0..8e7a326ec32 100644 --- a/python/llm/example/GPU/LLM-Finetuning/QLoRA/alpaca-qlora/README.md +++ b/python/llm/example/GPU/LLM-Finetuning/QLoRA/alpaca-qlora/README.md @@ -19,7 +19,7 @@ conda activate llm pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ pip install transformers==4.36.1 datasets pip install fire peft==0.10.0 accelerate==0.23.0 -pip install oneccl_bind_pt==2.1.100 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ # necessary to run distributed finetuning +pip install oneccl_bind_pt==2.1.100 --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ # necessary to run distributed finetuning pip install bitsandbytes scipy # configures OneAPI environment variables source /opt/intel/oneapi/setvars.sh # necessary to run before installing deepspeed diff --git a/python/llm/example/GPU/LLM-Finetuning/ReLora/README.md b/python/llm/example/GPU/LLM-Finetuning/ReLora/README.md index 77a252d2c70..aed2c04d111 100644 --- a/python/llm/example/GPU/LLM-Finetuning/ReLora/README.md +++ b/python/llm/example/GPU/LLM-Finetuning/ReLora/README.md @@ -15,7 +15,7 @@ pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-exte pip install transformers==4.45.0 "trl<0.12.0" datasets pip install fire peft==0.10.0 pip install bitsandbytes==0.45.1 scipy -pip install oneccl_bind_pt==2.1.100 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ # necessary to run distributed finetuning +pip install oneccl_bind_pt==2.1.100 --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ # necessary to run distributed finetuning ``` ### 2. Configures OneAPI environment variables diff --git a/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md b/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md index c350be36263..decc0e10382 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md +++ b/python/llm/example/GPU/Pipeline-Parallel-Inference/README.md @@ -48,7 +48,7 @@ conda create -n llm python=3.11 conda activate llm # below command will install intel_extension_for_pytorch==2.1.10+xpu as default pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ -pip install oneccl_bind_pt==2.1.100 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ +pip install oneccl_bind_pt==2.1.100 --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ ``` ### 2. Run pipeline parallel inference on multiple GPUs diff --git a/python/llm/example/GPU/Pipeline-Parallel-Serving/README.md b/python/llm/example/GPU/Pipeline-Parallel-Serving/README.md index 32745ceaf92..dcb999c46b9 100644 --- a/python/llm/example/GPU/Pipeline-Parallel-Serving/README.md +++ b/python/llm/example/GPU/Pipeline-Parallel-Serving/README.md @@ -36,7 +36,7 @@ conda create -n llm python=3.11 conda activate llm # below command will install intel_extension_for_pytorch==2.1.10+xpu as default pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ -pip install oneccl_bind_pt==2.1.100 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ +pip install oneccl_bind_pt==2.1.100 --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ # configures OneAPI environment variables source /opt/intel/oneapi/setvars.sh pip install mpi4py fastapi uvicorn openai From 42a440e26ad567190a6b3358aa8ecddb1638d5ad Mon Sep 17 00:00:00 2001 From: YongZhuIntel Date: Thu, 11 Sep 2025 17:01:00 +0800 Subject: [PATCH 2/2] update --- .../Self-Speculation/llama3/Dockerfile | 106 ------------------ 1 file changed, 106 deletions(-) delete mode 100644 python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama3/Dockerfile diff --git a/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama3/Dockerfile b/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama3/Dockerfile deleted file mode 100644 index 7da76bbd2c3..00000000000 --- a/python/llm/example/CPU/Speculative-Decoding/Self-Speculation/llama3/Dockerfile +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (C) 2025 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -# Stage 1: Build stage to handle file preparation -FROM ubuntu:22.04 as build - -# Copy the files to the build image -COPY ./start-notebook.sh /llm/ -COPY ./model_adapter.py.patch /llm/ -COPY ./vllm_offline_inference.py /llm/ -COPY ./payload-1024.lua /llm/ -COPY ./start-vllm-service.sh /llm/ -COPY ./benchmark_vllm_throughput.py /llm/ -COPY ./start-fastchat-service.sh /llm/ - -# Stage 2: Final image that only includes necessary runtime artifacts -FROM ubuntu:22.04 - -# Copy the scripts from the build stage -COPY --from=build /llm /llm/ - -ARG http_proxy -ARG https_proxy -# Disable pip's cache behavior -ARG PIP_NO_CACHE_DIR=false -ARG DEBIAN_FRONTEND=noninteractive - -ENV PYTHONUNBUFFERED=1 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - # Install basic utilities - libunwind8-dev vim less \ - # Version control and download tools - git curl wget \ - # add-apt-repository requires gnupg, gpg-agent, software-properties-common - gnupg gpg-agent software-properties-common \ - # Install performance testing tool, NUMA (Non-Uniform Memory Access) support, and patch tool - wrk numactl patch && \ -# Install Python 3.11 - # Add Python 3.11 PPA repository - add-apt-repository ppa:deadsnakes/ppa -y && \ - # Install Python 3.11 and related packages - apt-get update && apt-get install -y --no-install-recommends python3.11 python3-pip python3.11-dev python3-wheel python3.11-distutils && \ - # Remove the original /usr/bin/python3 symbolic link - rm /usr/bin/python3 && \ - # Create a symbolic link pointing to Python 3.11 at /usr/bin/python3 - ln -s /usr/bin/python3.11 /usr/bin/python3 && \ - # Create a symbolic link pointing to /usr/bin/python3 at /usr/bin/python - ln -s /usr/bin/python3 /usr/bin/python && \ -# Download and install pip, install FastChat from source requires PEP 660 support - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3 get-pip.py && \ - rm get-pip.py && \ -# Install Basic Python utilities - pip install --upgrade requests argparse urllib3 && \ -# Download ipex-llm-tutorial - pip install --upgrade jupyterlab && \ - git clone https://github.com/intel-analytics/ipex-llm-tutorial /llm/ipex-llm-tutorial && \ - chmod +x /llm/start-notebook.sh && \ -# Download all-in-one benchmark - git clone https://github.com/intel-analytics/IPEX-LLM && \ - cp -r ./IPEX-LLM/python/llm/dev/benchmark/ /llm/benchmark && \ -# Copy chat.py script - pip install --upgrade colorama && \ - cp -r ./IPEX-LLM/python/llm/portable-zip/ /llm/portable-zip && \ -# Install all-in-one dependencies - pip install --upgrade omegaconf && \ - pip install --upgrade pandas && \ -# Install ipex-llm - pip uninstall -y setuptools && \ - pip install --pre --upgrade ipex-llm[serving] && \ - # Fix Trivy CVE Issues - pip install Jinja2==3.1.3 transformers==4.36.2 gradio==4.19.2 cryptography==42.0.4 && \ - # Fix CVE-2024-22195 - pip install Jinja2==3.1.3 && \ - pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cpu && \ - pip install intel-extension-for-pytorch==2.2.0 && \ - pip install oneccl_bind_pt==2.2.0 --index-url https://pytorch-extension.intel.com/release-whl/stable/cpu/cn/ && \ - pip install transformers==4.36.2 && \ -# Install vllm dependencies - pip install --upgrade fastapi && \ - pip install --upgrade "uvicorn[standard]" && \ -# Add Qwen support - pip install --upgrade transformers_stream_generator einops && \ -# Fix Qwen model adapter in fastchat - patch /usr/local/lib/python3.11/dist-packages/fastchat/model/model_adapter.py < /llm/model_adapter.py.patch && \ -# Copy vLLM-Serving - cp -r ./IPEX-LLM/python/llm/example/CPU/vLLM-Serving/ /llm/vLLM-Serving && \ - rm -rf ./IPEX-LLM && \ -# Fix vllm service - pip install pydantic==1.10.11 && \ -# Install vllm - apt-get install -y g++ gcc-12 g++-12 libnuma-dev && \ - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 && \ - git clone https://github.com/vllm-project/vllm.git && \ - cd ./vllm && \ - git checkout v0.6.6.post1 && \ - pip install cmake==3.31.6 wheel packaging ninja "setuptools-scm>=8" numpy && \ - pip uninstall -y intel-extension-for-pytorch && \ - pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu && \ - VLLM_TARGET_DEVICE=cpu python3 setup.py install && \ - pip install ray triton==3.1.0 && \ -# Clean up unnecessary files to reduce image size - rm -rf /var/lib/apt/lists/* - -WORKDIR /llm/