diff --git a/.github/workflows/docker-build-ascend-a2.yml b/.github/workflows/docker-build-ascend-a2.yml index 78592edc403..452c47e58ad 100644 --- a/.github/workflows/docker-build-ascend-a2.yml +++ b/.github/workflows/docker-build-ascend-a2.yml @@ -14,10 +14,11 @@ on: env: QUAY_REPO: quay.io/ascend/verl + QUAY_TEMP_REPO: quay.io/yezib/verltest jobs: # ============================================================ - # 第一阶段:分架构构建并推送 digest(main 镜像) + # 第一阶段:分架构构建并推送 digest 到临时仓库(main 镜像) # ============================================================ build-push-digest: name: build-${{ matrix.tag }} @@ -79,18 +80,19 @@ jobs: driver: docker-container use: true - - name: Login to Quay.io + # Build 阶段只推送到临时仓库,只需登录临时仓库 + - name: Login to Quay Temp uses: docker/login-action@v3 with: registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} + username: ${{ secrets.QUAY_USERNAME_TMP }} + password: ${{ secrets.QUAY_PASSWORD_TMP }} - name: Clean Docker cache before build run: | docker system prune -a -f --volumes || true - - name: Build and push by digest (main image) + - name: Build and push by digest to temp repo (main image) uses: docker/build-push-action@v6 id: build_main with: @@ -98,13 +100,20 @@ jobs: platforms: ${{ matrix.arch }} file: ./docker/ascend/Dockerfile.ascend_9.1.0_a2 push: true - tags: ${{ env.QUAY_REPO }}:tmp-a2-${{ matrix.tag }} + outputs: type=image,name=${{ env.QUAY_TEMP_REPO }},push-by-digest=true,name-canonical=true,push=true provenance: false cache-from: type=gha,scope=ascend-a2-${{ matrix.tag }} cache-to: type=gha,mode=max,scope=ascend-a2-${{ matrix.tag }} build-args: | BUILDKIT_INLINE_CACHE=1 + # Quay.io 约 1 小时自动清理未打 tag 的 digest,构建后立即打临时 tag 防止 GC + - name: Tag digest to prevent GC + run: | + docker buildx imagetools create \ + -t "${{ env.QUAY_TEMP_REPO }}:tmp-a2-${{ matrix.tag }}" \ + "${{ env.QUAY_TEMP_REPO }}@${{ steps.build_main.outputs.digest }}" + - name: Export digest (main image) run: | mkdir -p ${{ runner.temp }}/digests @@ -120,7 +129,7 @@ jobs: retention-days: 1 # ============================================================ - # 第一阶段(v0.8.0):分架构构建并推送 digest + # 第一阶段(v0.8.0):分架构构建并推送 digest 到临时仓库 # ============================================================ build-push-digest-v080: name: build-v0.8.0-${{ matrix.tag }} @@ -163,18 +172,18 @@ jobs: driver: docker-container use: true - - name: Login to Quay.io + - name: Login to Quay Temp uses: docker/login-action@v3 with: registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} + username: ${{ secrets.QUAY_USERNAME_TMP }} + password: ${{ secrets.QUAY_PASSWORD_TMP }} - name: Clean Docker cache before build run: | docker system prune -a -f --volumes || true - - name: Build and push by digest (v0.8.0 image) + - name: Build and push by digest to temp repo (v0.8.0 image) uses: docker/build-push-action@v6 id: build_v080 with: @@ -182,13 +191,19 @@ jobs: platforms: ${{ matrix.arch }} file: ./docker/ascend/Dockerfile.ascend_9.0.0_a2_v0.8.0 push: true - tags: ${{ env.QUAY_REPO }}:tmp-a2-v080-${{ matrix.tag }} + outputs: type=image,name=${{ env.QUAY_TEMP_REPO }},push-by-digest=true,name-canonical=true,push=true provenance: false cache-from: type=gha,scope=ascend-a2-v080-${{ matrix.tag }} cache-to: type=gha,mode=max,scope=ascend-a2-v080-${{ matrix.tag }} build-args: | BUILDKIT_INLINE_CACHE=1 + - name: Tag digest to prevent GC + run: | + docker buildx imagetools create \ + -t "${{ env.QUAY_TEMP_REPO }}:tmp-a2-v080-${{ matrix.tag }}" \ + "${{ env.QUAY_TEMP_REPO }}@${{ steps.build_v080.outputs.digest }}" + - name: Export digest (v0.8.0 image) run: | mkdir -p ${{ runner.temp }}/digests-v080 @@ -218,7 +233,9 @@ jobs: pattern: digests-main-* merge-multiple: true - - name: Login to Quay.io + # Merge 阶段只推送到正式仓库,只需登录正式仓库 + # 临时仓库需为 Public,否则匿名读取 digest 会 401 + - name: Login to Quay.io (Official) uses: docker/login-action@v3 with: registry: quay.io @@ -230,21 +247,31 @@ jobs: - name: Merge and push multi-arch image (main) env: - IMAGE: ${{ env.QUAY_REPO }} + SOURCE_IMAGE: ${{ env.QUAY_TEMP_REPO }} + TARGET_IMAGE: ${{ env.QUAY_REPO }} run: | - DIGESTS=$(printf "$IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests-main | sed 's/^main-//')) + DIGESTS=$(printf "$SOURCE_IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests-main | sed 's/^main-//')) echo "Main image digests: $DIGESTS" docker buildx imagetools create \ - -t "$IMAGE:${{ needs.build-push-digest.outputs.tag }}" \ + -t "$TARGET_IMAGE:${{ needs.build-push-digest.outputs.tag }}" \ $DIGESTS - - name: Delete temporary tags (main) + + # Cleanup 阶段单独登录临时仓库,不影响前面的 imagetools create + - name: Login to Quay Temp for cleanup + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME_TMP }} + password: ${{ secrets.QUAY_PASSWORD_TMP }} + + - name: Delete temporary tags from temp repo (main) run: | curl -sL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz | tar -xz crane chmod +x crane - ./crane auth login quay.io -u "${{ secrets.QUAY_USERNAME }}" -p "${{ secrets.QUAY_PASSWORD }}" - echo "Deleting temporary main tags..." - ./crane delete ${{ env.QUAY_REPO }}:tmp-a2-amd64 || true - ./crane delete ${{ env.QUAY_REPO }}:tmp-a2-arm64 || true + echo "Deleting temporary tags from temp repo..." + ./crane delete ${{ env.QUAY_TEMP_REPO }}:tmp-a2-amd64 || true + ./crane delete ${{ env.QUAY_TEMP_REPO }}:tmp-a2-arm64 || true + # ============================================================ # 第二阶段(v0.8.0):合并多架构 manifest # ============================================================ @@ -260,7 +287,7 @@ jobs: pattern: digests-v080-* merge-multiple: true - - name: Login to Quay.io + - name: Login to Quay.io (Official) uses: docker/login-action@v3 with: registry: quay.io @@ -272,20 +299,26 @@ jobs: - name: Merge and push multi-arch image (v0.8.0) env: - IMAGE: ${{ env.QUAY_REPO }} + SOURCE_IMAGE: ${{ env.QUAY_TEMP_REPO }} + TARGET_IMAGE: ${{ env.QUAY_REPO }} run: | - DIGESTS=$(printf "$IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests-v080 | sed 's/^v080-//')) + DIGESTS=$(printf "$SOURCE_IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests-v080 | sed 's/^v080-//')) echo "v0.8.0 image digests: $DIGESTS" docker buildx imagetools create \ - -t "$IMAGE:v0.8.0-cann9.0.0-torch_npu2.9.0.post2-910b-ubuntu22.04-py3.11-vllm" \ + -t "$TARGET_IMAGE:v0.8.0-cann9.0.0-torch_npu2.9.0.post2-910b-ubuntu22.04-py3.11-vllm" \ $DIGESTS - - name: Delete temporary tags (v0.8.0) + + - name: Login to Quay Temp for cleanup + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME_TMP }} + password: ${{ secrets.QUAY_PASSWORD_TMP }} + + - name: Delete temporary tags from temp repo (v0.8.0) run: | curl -sL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz | tar -xz crane chmod +x crane - ./crane auth login quay.io -u "${{ secrets.QUAY_USERNAME }}" -p "${{ secrets.QUAY_PASSWORD }}" - echo "Deleting temporary v0.8.0 tags..." - ./crane delete ${{ env.QUAY_REPO }}:tmp-a2-v080-amd64 || true - ./crane delete ${{ env.QUAY_REPO }}:tmp-a2-v080-arm64 || true - - + echo "Deleting temporary v0.8.0 tags from temp repo..." + ./crane delete ${{ env.QUAY_TEMP_REPO }}:tmp-a2-v080-amd64 || true + ./crane delete ${{ env.QUAY_TEMP_REPO }}:tmp-a2-v080-arm64 || true \ No newline at end of file diff --git a/.github/workflows/docker-build-ascend-a3.yml b/.github/workflows/docker-build-ascend-a3.yml index aff03c4a4d9..c35261f6ff6 100644 --- a/.github/workflows/docker-build-ascend-a3.yml +++ b/.github/workflows/docker-build-ascend-a3.yml @@ -14,10 +14,11 @@ on: env: QUAY_REPO: quay.io/ascend/verl + QUAY_TEMP_REPO: quay.io/yezib/verltest jobs: # ============================================================ - # 第一阶段:分架构构建并推送 digest(main 镜像) + # 第一阶段:分架构构建并推送 digest 到临时仓库(main 镜像) # ============================================================ build-push-digest: name: build-${{ matrix.tag }} @@ -79,18 +80,19 @@ jobs: driver: docker-container use: true - - name: Login to Quay.io + # Build 阶段只推送到临时仓库,只需登录临时仓库 + - name: Login to Quay Temp uses: docker/login-action@v3 with: registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} + username: ${{ secrets.QUAY_USERNAME_TMP }} + password: ${{ secrets.QUAY_PASSWORD_TMP }} - name: Clean Docker cache before build run: | docker system prune -a -f --volumes || true - - name: Build and push by digest (main image) + - name: Build and push by digest to temp repo (main image) uses: docker/build-push-action@v6 id: build_main with: @@ -98,13 +100,20 @@ jobs: platforms: ${{ matrix.arch }} file: ./docker/ascend/Dockerfile.ascend_9.1.0_a3 push: true - tags: ${{ env.QUAY_REPO }}:tmp-a3-${{ matrix.tag }} + outputs: type=image,name=${{ env.QUAY_TEMP_REPO }},push-by-digest=true,name-canonical=true,push=true provenance: false cache-from: type=gha,scope=ascend-a3-${{ matrix.tag }} cache-to: type=gha,mode=max,scope=ascend-a3-${{ matrix.tag }} build-args: | BUILDKIT_INLINE_CACHE=1 + # Quay.io 约 1 小时自动清理未打 tag 的 digest,构建后立即打临时 tag 防止 GC + - name: Tag digest to prevent GC + run: | + docker buildx imagetools create \ + -t "${{ env.QUAY_TEMP_REPO }}:tmp-${{ matrix.tag }}" \ + "${{ env.QUAY_TEMP_REPO }}@${{ steps.build_main.outputs.digest }}" + - name: Export digest (main image) run: | mkdir -p ${{ runner.temp }}/digests @@ -120,7 +129,7 @@ jobs: retention-days: 1 # ============================================================ - # 第一阶段(v0.8.0):分架构构建并推送 digest + # 第一阶段(v0.8.0):分架构构建并推送 digest 到临时仓库 # ============================================================ build-push-digest-v080: name: build-v0.8.0-${{ matrix.tag }} @@ -163,18 +172,19 @@ jobs: driver: docker-container use: true - - name: Login to Quay.io + # Build 阶段只推送到临时仓库,只需登录临时仓库 + - name: Login to Quay Temp uses: docker/login-action@v3 with: registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} + username: ${{ secrets.QUAY_USERNAME_TMP }} + password: ${{ secrets.QUAY_PASSWORD_TMP }} - name: Clean Docker cache before build run: | docker system prune -a -f --volumes || true - - name: Build and push by digest (v0.8.0 image) + - name: Build and push by digest to temp repo (v0.8.0 image) uses: docker/build-push-action@v6 id: build_v080 with: @@ -182,13 +192,19 @@ jobs: platforms: ${{ matrix.arch }} file: ./docker/ascend/Dockerfile.ascend_9.0.0_a3_v0.8.0 push: true - tags: ${{ env.QUAY_REPO }}:tmp-a3-v080-${{ matrix.tag }} + outputs: type=image,name=${{ env.QUAY_TEMP_REPO }},push-by-digest=true,name-canonical=true,push=true provenance: false cache-from: type=gha,scope=ascend-a3-v080-${{ matrix.tag }} cache-to: type=gha,mode=max,scope=ascend-a3-v080-${{ matrix.tag }} build-args: | BUILDKIT_INLINE_CACHE=1 + - name: Tag digest to prevent GC + run: | + docker buildx imagetools create \ + -t "${{ env.QUAY_TEMP_REPO }}:tmp-v080-${{ matrix.tag }}" \ + "${{ env.QUAY_TEMP_REPO }}@${{ steps.build_v080.outputs.digest }}" + - name: Export digest (v0.8.0 image) run: | mkdir -p ${{ runner.temp }}/digests-v080 @@ -218,7 +234,9 @@ jobs: pattern: digests-main-* merge-multiple: true - - name: Login to Quay.io + # Merge 阶段只推送到正式仓库,只需登录正式仓库 + # 临时仓库需为 Public,否则匿名读取 digest 会 401 + - name: Login to Quay.io (Official) uses: docker/login-action@v3 with: registry: quay.io @@ -230,21 +248,30 @@ jobs: - name: Merge and push multi-arch image (main) env: - IMAGE: ${{ env.QUAY_REPO }} + SOURCE_IMAGE: ${{ env.QUAY_TEMP_REPO }} + TARGET_IMAGE: ${{ env.QUAY_REPO }} run: | - DIGESTS=$(printf "$IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests-main | sed 's/^main-//')) + DIGESTS=$(printf "$SOURCE_IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests-main | sed 's/^main-//')) echo "Main image digests: $DIGESTS" docker buildx imagetools create \ - -t "$IMAGE:${{ needs.build-push-digest.outputs.tag }}" \ + -t "$TARGET_IMAGE:${{ needs.build-push-digest.outputs.tag }}" \ $DIGESTS - - name: Delete temporary tags (main) + + # Cleanup 阶段单独登录临时仓库,不影响前面的 imagetools create + - name: Login to Quay Temp for cleanup + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME_TMP }} + password: ${{ secrets.QUAY_PASSWORD_TMP }} + + - name: Delete temporary tags from temp repo (main) run: | curl -sL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz | tar -xz crane chmod +x crane - ./crane auth login quay.io -u "${{ secrets.QUAY_USERNAME }}" -p "${{ secrets.QUAY_PASSWORD }}" - echo "Deleting temporary main tags..." - ./crane delete ${{ env.QUAY_REPO }}:tmp-a3-amd64 || true - ./crane delete ${{ env.QUAY_REPO }}:tmp-a3-arm64 || true + echo "Deleting temporary tags from temp repo..." + ./crane delete ${{ env.QUAY_TEMP_REPO }}:tmp-amd64 || true + ./crane delete ${{ env.QUAY_TEMP_REPO }}:tmp-arm64 || true # ============================================================ # 第二阶段(v0.8.0):合并多架构 manifest @@ -261,7 +288,7 @@ jobs: pattern: digests-v080-* merge-multiple: true - - name: Login to Quay.io + - name: Login to Quay.io (Official) uses: docker/login-action@v3 with: registry: quay.io @@ -273,18 +300,26 @@ jobs: - name: Merge and push multi-arch image (v0.8.0) env: - IMAGE: ${{ env.QUAY_REPO }} + SOURCE_IMAGE: ${{ env.QUAY_TEMP_REPO }} + TARGET_IMAGE: ${{ env.QUAY_REPO }} run: | - DIGESTS=$(printf "$IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests-v080 | sed 's/^v080-//')) + DIGESTS=$(printf "$SOURCE_IMAGE@sha256:%s " $(ls ${{ runner.temp }}/digests-v080 | sed 's/^v080-//')) echo "v0.8.0 image digests: $DIGESTS" docker buildx imagetools create \ - -t "$IMAGE:v0.8.0-cann9.0.0-torch_npu2.9.0.post2-a3-ubuntu22.04-py3.11-vllm" \ + -t "$TARGET_IMAGE:v0.8.0-cann9.0.0-torch_npu2.9.0.post2-a3-ubuntu22.04-py3.11-vllm" \ $DIGESTS - - name: Delete temporary tags (v0.8.0) + + - name: Login to Quay Temp for cleanup + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME_TMP }} + password: ${{ secrets.QUAY_PASSWORD_TMP }} + + - name: Delete temporary tags from temp repo (v0.8.0) run: | curl -sL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz | tar -xz crane chmod +x crane - ./crane auth login quay.io -u "${{ secrets.QUAY_USERNAME }}" -p "${{ secrets.QUAY_PASSWORD }}" - echo "Deleting temporary v0.8.0 tags..." - ./crane delete ${{ env.QUAY_REPO }}:tmp-a3-v080-amd64 || true - ./crane delete ${{ env.QUAY_REPO }}:tmp-a3-v080-arm64 || true + echo "Deleting temporary v0.8.0 tags from temp repo..." + ./crane delete ${{ env.QUAY_TEMP_REPO }}:tmp-v080-amd64 || true + ./crane delete ${{ env.QUAY_TEMP_REPO }}:tmp-v080-arm64 || true \ No newline at end of file diff --git a/docs/ascend_tutorial/dev_guide/performance/perf_tuning_on_ascend.rst b/docs/ascend_tutorial/dev_guide/performance/perf_tuning_on_ascend.rst index 56b5573865b..e96fa921e4b 100644 --- a/docs/ascend_tutorial/dev_guide/performance/perf_tuning_on_ascend.rst +++ b/docs/ascend_tutorial/dev_guide/performance/perf_tuning_on_ascend.rst @@ -119,7 +119,6 @@ Megatron 的融合算子集成在 MindSpeed 中,需要添加特定参数开启 :: +actor_rollout_ref.actor.megatron.override_transformer_config.use_flash_attn=True - ++actor_rollout_ref.ref.megatron.override_transformer_config.use_flash_attn=True 2. **RotaryMul** ::