diff --git a/.github/workflows/201-user-sdpt-controller-adhoc.yaml b/.github/workflows/201-user-sdpt-controller-adhoc.yaml index 24a014f46a62..4d3334580b80 100644 --- a/.github/workflows/201-user-sdpt-controller-adhoc.yaml +++ b/.github/workflows/201-user-sdpt-controller-adhoc.yaml @@ -40,14 +40,9 @@ on: default: "" type: string description: 'Add props to settings.txt, e.g. "state.saveStatePeriod, 300". Newline is "\n"' - crypto-bench-merkle-db-java-args: - required: true - default: "-XX:+UseZGC -XX:+ZGenerational -XX:ZAllocationSpikeTolerance=2 -XX:ConcGCThreads=14 -XX:ZMarkStackSpaceLimit=12g -XX:MaxDirectMemorySize=24g -Xmx90g" - description: "JMH javaargs" - type: string crypto-bench-merkle-db-test-params: required: true - default: "-p maxKey=500000000 -p numRecords=100000 -p keySize=24 -p recordSize=1024 -p numFiles=60" + default: "-p maxKey=500000000 -p numRecords=100000 -p keySize=24 -p recordSize=1024 -p numFiles=6000" description: "Test params" type: string duration-minutes: @@ -84,14 +79,14 @@ jobs: uses: pandaswhocode/initialize-github-job@5dbfbc4329ceab48a57d983325dadbd3ec8123e9 # v1.2.2 with: checkout: "true" - checkout-ref: "${{ inputs.ref || github.ref }}" + checkout-ref: "${{ github.ref }}" checkout-token: "${{ secrets.GITHUB_TOKEN }}" - name: Verify Tag id: verify-tag env: PROMOTED_GREP_PATTERN: "build-(.{5})" # Pattern to match build tags (e.g., build-12345) - CHECK_TAG: "${{ inputs.ref || github.ref }}" + CHECK_TAG: "${{ github.ref }}" run: | # Ensure the tag that we received is a build tag # if the tag is a build tag then we can parse out the build number (the last 5 characters) @@ -115,7 +110,7 @@ jobs: name: Extract Java Version uses: ./.github/workflows/854-call-extract-jdk-version.yaml with: - ref: ${{ inputs.ref || github.ref }} + ref: ${{ github.ref }} secrets: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -123,7 +118,7 @@ jobs: name: Extract CITR Vars uses: ./.github/workflows/855-call-extract-citr-vars.yaml with: - ref: ${{ inputs.ref || github.ref }} + ref: ${{ github.ref }} secrets: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -131,7 +126,7 @@ jobs: name: Extract Chewie Properties uses: ./.github/workflows/862-call-get-chewie-properties.yaml with: - ref: ${{ inputs.ref || github.ref }} + ref: ${{ github.ref }} secrets: checkout-token: ${{ secrets.GITHUB_TOKEN }} @@ -197,6 +192,7 @@ jobs: if: ${{ needs.verify-tag.result == 'success' || inputs.disable-notifications == true }} with: ref: "${{ inputs.ref || github.ref }}" + bnref: "${{ needs.extract-citr-vars.outputs.bn-release-version }}" solo-version: "${{ needs.extract-citr-vars.outputs.adhoc-solo-version }}" nlg-accounts: "${{ inputs.nlg-accounts }}" nlg-time: "${{ inputs.nlg-time }}" @@ -207,6 +203,7 @@ jobs: duration-minutes: ${{ fromJSON(needs.verify-citr-vars.outputs.request-duration) }} chewie-request-timeout: ${{ fromJSON(needs.verify-citr-vars.outputs.request-timeout) }} java-version: "${{ needs.extract-jdk-version.outputs.jdk-version }}" + nlg-version: "${{ needs.extract-jdk-version.outputs.nlg-version }}" secrets: slack-report-webhook: ${{ secrets.SLACK_CITR_DETAILED_REPORTS_WEBHOOK }} github-token: ${{ secrets.GITHUB_TOKEN }} @@ -226,7 +223,7 @@ jobs: uses: pandaswhocode/initialize-github-job@5dbfbc4329ceab48a57d983325dadbd3ec8123e9 # v1.2.2 with: checkout: "true" - checkout-ref: "${{ inputs.ref || github.ref }}" + checkout-ref: "${{ github.ref }}" checkout-token: "${{ secrets.GH_ACCESS_TOKEN }}" checkout-fetch-depth: "0" @@ -296,7 +293,7 @@ jobs: SLACK_BOT_TOKEN: ${{ secrets.SLACK_CITR_BOT_TOKEN }} run: | # Extract the commit and user information from the build tag - COMMIT=$(git rev-list -n 1 "${{ inputs.ref || github.ref }}") + COMMIT=$(git rev-list -n 1 "${{ github.ref }}") EMAIL=$(git log -1 --pretty=format:'%ae' "${COMMIT}") AUTHOR=$(git log -1 --pretty=format:'%an' "${COMMIT}") echo "commit=${COMMIT}" >> "${GITHUB_OUTPUT}" @@ -340,7 +337,7 @@ jobs: RUN_TEST_RESULT: ${{ needs.run-single-day-performance-test.outputs.result }} TAG_LABEL: "Tag SDPT Passing" TAG_RESULT: ${{ needs.tag-sdpt-result.result }} - SOURCE_TAG_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ inputs.ref || github.ref }} + SOURCE_TAG_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ inputs.ref }} COMMIT_AUTHOR: ${{ needs.get-commit-info.outputs.commit-author }} WORKFLOW_RUN_ID: ${{ github.run_id }} WORKFLOW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} @@ -376,7 +373,7 @@ jobs: uses: pandaswhocode/initialize-github-job@5dbfbc4329ceab48a57d983325dadbd3ec8123e9 # v1.2.2 with: checkout: "true" - checkout-ref: "${{ inputs.ref || github.ref }}" + checkout-ref: "${{ github.ref }}" checkout-token: "${{ secrets.GH_ACCESS_TOKEN }}" checkout-fetch-depth: "0" setup-gomplate: "true" @@ -472,7 +469,7 @@ jobs: TAG_LABEL: "Tag SDPT Passing" TAG_RESULT: ${{ needs.tag-sdpt-result.result }} COMMIT_INFO_RESULT: ${{ needs.get-commit-info.result }} - SOURCE_TAG_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ inputs.ref || github.ref }} + SOURCE_TAG_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ inputs.ref }} COMMIT_AUTHOR: ${{ needs.get-commit-info.outputs.commit-author }} SLACK_USER_ID: ${{ needs.get-commit-info.outputs.slack-user-id }} WORKFLOW_RUN_ID: ${{ github.run_id }} diff --git a/.github/workflows/202-user-sdlt-controller-adhoc.yaml b/.github/workflows/202-user-sdlt-controller-adhoc.yaml index 3b96c2b24f12..3fff72ce4bc8 100644 --- a/.github/workflows/202-user-sdlt-controller-adhoc.yaml +++ b/.github/workflows/202-user-sdlt-controller-adhoc.yaml @@ -86,14 +86,14 @@ jobs: uses: pandaswhocode/initialize-github-job@5dbfbc4329ceab48a57d983325dadbd3ec8123e9 # v1.2.2 with: checkout: "true" - checkout-ref: "${{ inputs.ref || github.ref }}" + checkout-ref: "${{ github.ref }}" checkout-token: "${{ secrets.GITHUB_TOKEN }}" - name: Verify Tag id: verify-tag env: PROMOTED_GREP_PATTERN: "build-(.{5})" # Pattern to match build tags (e.g., build-12345) - CHECK_TAG: "${{ inputs.ref || github.ref }}" + CHECK_TAG: "${{ github.ref }}" run: | # Ensure the tag that we received is a build tag # if the tag is a build tag then we can parse out the build number (the last 5 characters) @@ -117,7 +117,7 @@ jobs: name: Extract Java Version uses: ./.github/workflows/854-call-extract-jdk-version.yaml with: - ref: ${{ inputs.ref || github.ref }} + ref: ${{ github.ref }} secrets: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -125,7 +125,7 @@ jobs: name: Extract CITR Vars uses: ./.github/workflows/855-call-extract-citr-vars.yaml with: - ref: ${{ inputs.ref || github.ref }} + ref: ${{ github.ref }} secrets: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -133,7 +133,7 @@ jobs: name: Extract Chewie Properties uses: ./.github/workflows/862-call-get-chewie-properties.yaml with: - ref: ${{ inputs.ref || github.ref }} + ref: ${{ github.ref }} secrets: checkout-token: ${{ secrets.GITHUB_TOKEN }} @@ -199,6 +199,7 @@ jobs: if: ${{ needs.verify-tag.result == 'success' || inputs.disable-notifications == true }} with: ref: "${{ inputs.ref || github.ref }}" + bnref: "${{ needs.extract-citr-vars.outputs.bn-release-version }}" solo-version: "${{ needs.extract-citr-vars.outputs.adhoc-solo-version }}" nlg-accounts: "${{ inputs.nlg-accounts }}" nlg-time: "${{ inputs.nlg-time }}" @@ -208,6 +209,7 @@ jobs: duration-minutes: ${{ fromJSON(needs.verify-citr-vars.outputs.request-duration) }} chewie-request-timeout: ${{ fromJSON(needs.verify-citr-vars.outputs.request-timeout) }} java-version: "${{ needs.extract-jdk-version.outputs.jdk-version }}" + nlg-version: "${{ needs.extract-jdk-version.outputs.nlg-version }}" secrets: slack-report-webhook: ${{ secrets.SLACK_CITR_DETAILED_REPORTS_WEBHOOK }} github-token: ${{ secrets.GITHUB_TOKEN }} @@ -227,7 +229,7 @@ jobs: uses: pandaswhocode/initialize-github-job@5dbfbc4329ceab48a57d983325dadbd3ec8123e9 # v1.2.2 with: checkout: "true" - checkout-ref: "${{ inputs.ref || github.ref }}" + checkout-ref: "${{ github.ref }}" checkout-token: "${{ secrets.GH_ACCESS_TOKEN }}" checkout-fetch-depth: "0" @@ -294,7 +296,7 @@ jobs: id: find-commit-author-slack continue-on-error: true env: - BUILD_TAG: ${{ inputs.ref || github.ref }} + BUILD_TAG: ${{ github.ref }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_CITR_BOT_TOKEN }} run: | # Extract the commit and user information from the build tag @@ -334,7 +336,7 @@ jobs: - name: Build Slack Payload Message id: payload env: - HEADER_TEXT: ":tada: SDLT - Single Day Longevity Test (${{ inputs.ref || github.ref }}) Passed" + HEADER_TEXT: ":tada: SDLT - Single Day Longevity Test (${{ github.ref }}) Passed" STATUS_TEXT: "*SDLT Job Succeeded. Test panel status below*" VERIFY_TAG_LABEL: "Verify Tag for SDLT" VERIFY_TAG_RESULT: ${{ needs.verify-tag.result }} @@ -342,7 +344,7 @@ jobs: RUN_TEST_RESULT: ${{ needs.run-single-day-longevity-test.outputs.result }} TAG_LABEL: "Tag SDLT Passing" TAG_RESULT: ${{ needs.tag-sdlt-result.result }} - SOURCE_TAG_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ inputs.ref || github.ref }} + SOURCE_TAG_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref }} COMMIT_AUTHOR: ${{ needs.get-commit-info.outputs.commit-author }} WORKFLOW_RUN_ID: ${{ github.run_id }} WORKFLOW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} @@ -378,7 +380,7 @@ jobs: uses: pandaswhocode/initialize-github-job@5dbfbc4329ceab48a57d983325dadbd3ec8123e9 # v1.2.2 with: checkout: "true" - checkout-ref: "${{ inputs.ref || github.ref }}" + checkout-ref: "${{ github.ref }}" checkout-token: "${{ secrets.GH_ACCESS_TOKEN }}" checkout-fetch-depth: "0" setup-gomplate: "true" @@ -465,7 +467,7 @@ jobs: - name: Build Slack Payload Message id: payload env: - HEADER_TEXT: ":grey_exclamation: SDLT - Single Day Longevity Test (${{ inputs.ref || github.ref }}) Failed" + HEADER_TEXT: ":grey_exclamation: SDLT - Single Day Longevity Test (${{ github.ref }}) Failed" STATUS_TEXT: "*SDLT Job Resulted in failure. See status below.*" VERIFY_TAG_LABEL: "Verify Tag for SDLT" VERIFY_TAG_RESULT: ${{ needs.verify-tag.result }} @@ -474,7 +476,7 @@ jobs: TAG_LABEL: "Tag SDLT Passing" TAG_RESULT: ${{ needs.tag-sdlt-result.result }} COMMIT_INFO_RESULT: ${{ needs.get-commit-info.result }} - SOURCE_TAG_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ inputs.ref || github.ref }} + SOURCE_TAG_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref }} COMMIT_AUTHOR: ${{ needs.get-commit-info.outputs.commit-author }} SLACK_USER_ID: ${{ needs.get-commit-info.outputs.slack-user-id }} WORKFLOW_RUN_ID: ${{ github.run_id }} diff --git a/.github/workflows/221-disp-sdpt-controller.yaml b/.github/workflows/221-disp-sdpt-controller.yaml index 91cf079e9a90..1d051a52a645 100644 --- a/.github/workflows/221-disp-sdpt-controller.yaml +++ b/.github/workflows/221-disp-sdpt-controller.yaml @@ -125,6 +125,7 @@ jobs: if: ${{ needs.verify-tag.result == 'success' }} with: ref: "${{ github.ref }}" + bnref: "${{ needs.extract-citr-vars.outputs.bn-release-version }}" solo-version: "${{ needs.extract-citr-vars.outputs.citr-solo-version }}" nlg-accounts: "100000000" nlg-time: "330" @@ -135,6 +136,7 @@ jobs: duration-minutes: 1320 # 20 hours + 2 for post-mortem chewie-request-timeout: ${{ fromJSON(needs.verify-citr-vars.outputs.request-timeout) }} java-version: "${{ needs.extract-jdk-version.outputs.jdk-version }}" + nlg-version: "${{ needs.extract-jdk-version.outputs.nlg-version }}" secrets: slack-report-webhook: ${{ secrets.SLACK_CITR_DETAILED_REPORTS_WEBHOOK }} github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/222-disp-sdlt-controller.yaml b/.github/workflows/222-disp-sdlt-controller.yaml index 8cb77ed558c2..8f0acaf32144 100644 --- a/.github/workflows/222-disp-sdlt-controller.yaml +++ b/.github/workflows/222-disp-sdlt-controller.yaml @@ -126,6 +126,7 @@ jobs: if: ${{ needs.verify-tag.result == 'success' }} with: ref: "${{ github.ref }}" + bnref: "${{ needs.extract-citr-vars.outputs.bn-release-version }}" solo-version: "${{ needs.extract-citr-vars.outputs.citr-solo-version }}" nlg-accounts: "100000000" nlg-time: "720" @@ -135,6 +136,7 @@ jobs: duration-minutes: 1080 # 16 hours + 2 for post-mortem chewie-request-timeout: ${{ fromJSON(needs.verify-citr-vars.outputs.request-timeout) }} java-version: "${{ needs.extract-jdk-version.outputs.jdk-version }}" + nlg-version: "${{ needs.extract-jdk-version.outputs.nlg-version }}" secrets: slack-report-webhook: ${{ secrets.SLACK_CITR_DETAILED_REPORTS_WEBHOOK }} github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/831-call-single-day-performance-test.yaml b/.github/workflows/831-call-single-day-performance-test.yaml index 10f37cb89226..da695839ba5e 100644 --- a/.github/workflows/831-call-single-day-performance-test.yaml +++ b/.github/workflows/831-call-single-day-performance-test.yaml @@ -11,7 +11,7 @@ on: type: string solo-version: required: true - default: "v0.41.0" + default: "v0.84.1" description: "Solo Version:" type: string nlg-accounts: #'-R -c 32 -a 100000000 -T 1000 -n 100000 -S hot -p 50 -tt 1m @@ -48,6 +48,16 @@ on: required: true description: "Expected duration of all tests in minutes" type: number + bnref: + description: "BN version" + type: string + required: true + default: "" + nlg-version: + required: true + default: "0.14.3" + description: "NLG Version:" + type: string chewie-request-timeout: required: true description: "Chewie request timeout in seconds" @@ -55,7 +65,6 @@ on: java-version: description: "Java JDK Version:" type: string - required: false secrets: slack-report-webhook: required: true @@ -85,10 +94,7 @@ env: GS_ROOT_HTTPS: https://perf.analytics.eng.hashgraph.io/ephemeral/test_runs GRAFANA_SYSADM_DAL: https://grafana.lat.ope.eng.hashgraph.io/d/rYdddlPWk/node-exporter-single-node?orgId=1&timezone=utc&var-datasource=P5C24830FD6F1746E&var-job=performance-dallas-node-exporter GRAFANA_SYSADM_CHI: https://grafana.lat.ope.eng.hashgraph.io/d/rYdddlPWk/node-exporter-single-node?orgId=1&timezone=utc&var-datasource=P631E2BA859A814B3&var-job=performance-chicago-node-exporter - - #Release version from jFrog - NLG_VERSION: 0.14.3 - + NUM_OF_BLOCK_NODES: 3 #Thresholds for performance #benchmarks #CryptoTransfer HCS NftTransfer SmartContract HeliSwap CryptoBenchMerkleDb @@ -114,7 +120,7 @@ jobs: uses: ./.github/workflows/861-call-get-test-config.yaml with: test-type: "sdpt" - ref: ${{ inputs.ref }} + ref: ${{ github.ref }} secrets: checkout-token: ${{ secrets.github-token }} @@ -122,7 +128,7 @@ jobs: needs: get-chewie-jwt uses: ./.github/workflows/860-call-validate-chewie-jwt.yaml with: - ref: ${{ inputs.ref }} + ref: ${{ github.ref }} chewie-expiration: ${{ needs.get-chewie-jwt.outputs.expiration }} secrets: chewie-jwt: ${{ needs.get-chewie-jwt.outputs.jwt }} @@ -191,9 +197,22 @@ jobs: uses: pandaswhocode/initialize-github-job@5dbfbc4329ceab48a57d983325dadbd3ec8123e9 # v1.2.2 with: checkout: "true" - checkout-ref: "${{ inputs.ref }}" + checkout-ref: "${{ github.ref }}" checkout-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Checkout target CN Code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: target_consensus_node + ref: ${{ inputs.ref }} + + - name: Checkout target BN Code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: "hiero-ledger/hiero-block-node" + path: target_block_node + ref: ${{ inputs.bnref }} + - name: Set Kubernetes Cluster id: set-k8s-cluster run: | @@ -209,7 +228,15 @@ jobs: - name: Install KubeCtl uses: step-security/setup-kubectl@2edbf6aff97d814e9dc52827498ac51fe972e6d0 # v4.0.0 with: - version: v1.33.0 + version: v1.35.0 + + - name: kubectl wrapper + run: | + kubectl=$(which kubectl) + sudo mv ${kubectl} ${kubectl}.orig + sudo cp "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt ${kubectl} + sudo chmod a+rx ${kubectl} + which kubectl - name: Install Teleport Client uses: teleport-actions/setup@176c25dfcd19cd31a252f275d579822b243e7b9c # v1.0.6 @@ -276,7 +303,7 @@ jobs: echo Setup procedure ... cat /etc/*elease sudo apt-get update - sudo apt-get install -y net-tools iputils-ping node-typescript + sudo apt-get install -qq -y net-tools iputils-ping node-typescript coreutils jq xxd openssl - name: Create report dir shell: bash @@ -304,21 +331,43 @@ jobs: echo "inputs.add-app-props=${{ inputs.add-app-props }}" | tee -a "${GITHUB_STEP_SUMMARY}" "${{ github.workspace }}"/version_run.txt echo "inputs.add-settings=${{ inputs.add-settings }}" | tee -a "${GITHUB_STEP_SUMMARY}" "${{ github.workspace }}"/version_run.txt echo "inputs.nlg-test=${{ env.run_NLG_Test }}" | tee -a "${GITHUB_STEP_SUMMARY}" "${{ github.workspace }}"/version_run.txt + echo "inputs.NLGDparams=${{ inputs.NLGDparams }}" | tee -a "${GITHUB_STEP_SUMMARY}" "${{ github.workspace }}"/version_run.txt echo "JOB_URL=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | tee -a "${GITHUB_STEP_SUMMARY}" "${{ github.workspace }}"/version_run.txt - name: Set hederahash sha value id: hederaHash run: | - hederahash="$(git rev-parse HEAD)" + hederahash="$(git -C target_consensus_node rev-parse HEAD)" echo "sha=${hederahash}" >> "${GITHUB_OUTPUT}" echo "hederahash=${hederahash}" | tee -a "${GITHUB_STEP_SUMMARY}" "${{ github.workspace }}"/version_run.txt - name: Compile - run: ./gradlew clean assemble + run: | + set +e + set +x + cd "${{ github.workspace }}"/target_consensus_node + + ./gradlew clean assemble + + cd "${{ github.workspace }}" + + - name: Compile BN + run: | + cd "${{ github.workspace }}"/target_block_node + ./gradlew app:distTar + + shortversion=$(echo "${{ inputs.bnref }}" | sed -e 's/^v\(.*\)/\1/g') + cd ./block-node/app/build/distributions/ + rm -rf app-${shortversion}/plugins app-${shortversion}.tar + tar cf app-${shortversion}.tar app-${shortversion} + + cd "${{ github.workspace }}" - name: Build Validator run: | + cd "${{ github.workspace }}"/target_consensus_node ./gradlew --no-daemon :hedera-state-validator:shadowJar + cd "${{ github.workspace }}" - name: Setup Node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 @@ -373,16 +422,21 @@ jobs: # Generate the init-containers-values7.yaml inline cp "${{ github.workspace }}"/.github/workflows/support/citr/init-containers-values7.yaml init-containers-values7.yaml + # Generate the block-node-values.yaml inline + cp "${{ github.workspace }}"/.github/workflows/support/citr/block-node-values.yaml block-node-values.yaml + # Update Taskfile values sed -i -e "s@%HOME%@${HOME}@g" -e "s@%SOLO_NAMESPACE%@${{ needs.acquire-kubernetes-resources.outputs.namespace }}@g" \ - -e "s@%HEDERA_SERVICES_ROOT%@${curdir}@g" Taskfile.yml + -e "s@%HEDERA_SERVICES_ROOT%@${curdir}/target_consensus_node@g" Taskfile.yml # Update Init Containers values NETWORK_ID="${{ needs.acquire-kubernetes-resources.outputs.network-id }}" sed -i -e "s@%NETWORK_ID%@${NETWORK_ID}@g" init-containers-values7.yaml + sed -i -e "s@%NETWORK_ID%@${NETWORK_ID}@g" block-node-values.yaml NETWORK_OWNER="${{ needs.acquire-kubernetes-resources.outputs.owner }}" sed -i -e "s@%NETWORK_OWNER%@${NETWORK_OWNER}@g" init-containers-values7.yaml + sed -i -e "s@%NETWORK_OWNER%@${NETWORK_OWNER}@g" block-node-values.yaml # Generate the application.properties file inline cp "${{ github.workspace }}"/.github/workflows/support/citr/application.properties.txt application.properties @@ -409,18 +463,143 @@ jobs: export NODE_COPY_CONCURRENT=1 task - task_code=${?} + + - name: Add BlockNodes + run: | + set +x + set +e + cd "${{ github.workspace}}"/solo + + mv block-node-values.yaml block-node-values.yaml.orig + export BLOCK_NODE_VERSION="${{ inputs.bnref }}" #"${{ env.BlockNode_version }}" + #export CONSENSUS_NODE_VERSION=${{ inputs.ref }} + + #dedicate top-3 nodes for BN + NETWORK_ID="${{ needs.acquire-kubernetes-resources.outputs.network-id}}" + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt get nodes --show-labels | grep "network-id=${NETWORK_ID}," | grep 'consensus-node' | sort -n -k 1 | awk '{print $1}' | head -n 1 | grep -n -E '^' > firstNode.txt + + for i in $(seq 1 1 ${{ env.NUM_OF_BLOCK_NODES }}) + do + #create PVC values.yaml from template + for pvc_name in application-state-storage archive-storage logging-storage live-storage plugins-storage + do + sed -e "s@%NODE_ID%@${i}@g" -e "s@%PVC_NAME%@${pvc_name}@g" "${{ github.workspace }}"/.github/workflows/support/citr/block-node-pvc.yaml > block-node-pvc_${pvc_name}_${i}.yaml + echo "kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" create -f block-node-pvc_${pvc_name}_${i}.yaml" + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" create -f block-node-pvc_${pvc_name}_${i}.yaml + done + sleep 10 + done + + bnindex=1 + BNNODE=$(grep "${bnindex}:" firstNode.txt | awk -F : '{print $2}') + + sed -i -e "s@%SOLO_NAMESPACE%@${{ needs.acquire-kubernetes-resources.outputs.namespace }}@g" block-node-values.yaml.orig + + #BN node 1 + sed -e "s@%NODE_ID%@1@g" -e "s@%BNNODE%@${BNNODE}@g" -e "s@%ANOTHER_BN_1%@block-node-2@g" -e "s@%ANOTHER_BN_2%@block-node-3@g" block-node-values.yaml.orig > block-node-values1.yaml + cp block-node-values1.yaml block-node-values.yaml; + export BN_MAPPING="node1=1,node2=2,node3=3,node4=3,node5=1,node6=2,node7=2" + task solo:block:add + + #BN node 2 + sed -e "s@%NODE_ID%@2@g" -e "s@%BNNODE%@${BNNODE}@g" -e "s@%ANOTHER_BN_1%@block-node-1@g" -e "s@%ANOTHER_BN_2%@block-node-3@g" block-node-values.yaml.orig > block-node-values2.yaml + cp block-node-values2.yaml block-node-values.yaml; + export BN_MAPPING="node1=3,node2=1,node3=1,node4=2,node5=3,node6=3,node7=1" + task solo:block:add + + #BN node 3 + sed -e "s@%NODE_ID%@3@g" -e "s@%BNNODE%@${BNNODE}@g" -e "s@%ANOTHER_BN_1%@block-node-1@g" -e "s@%ANOTHER_BN_2%@block-node-2@g" block-node-values.yaml.orig > block-node-values3.yaml + cp block-node-values3.yaml block-node-values.yaml; + export BN_MAPPING="node1=2,node2=2,node3=3,node4=1,node5=1,node6=1,node7=3" + task solo:block:add + + sleep 30 + + - name: Deploy CNs + run: | + set +x + set +e + cd "${{ github.workspace}}"/solo + task installCNs + + - name: Generate RSARoster + run: | + genesisFile=$(find $HOME/.solo -type f -name 'genesis-network.json') + bash "${{ github.workspace }}"/.github/workflows/support/citr/genesisNetworkBNRSARoster.sh -i ${genesisFile} -o "${{ github.workspace}}"/solo/rsa-bootstrap-roster.json + jq . "${{ github.workspace}}"/solo/rsa-bootstrap-roster.json + + - name: Update BlockNodes + run: | + ls "${{ github.workspace }}"/target_block_node + ls "${{ github.workspace }}"/target_block_node/block-node/app/build/distributions/ + bnTar=$(ls "${{ github.workspace }}"/target_block_node/block-node/app/build/distributions/app-*.tar) + bnVersion=$(basename $bnTar | sed -e 's/.tar//g') + echo "$bnTar=$bnTar" + for pod in `sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep 'block-node' | awk '{print $1}'` + do + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" cp ${bnTar} ${pod}:/tmp/BN.tar + done + + for pod in `sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep 'block-node' | awk '{print $1}'` + do + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- bash -c "jpid=\$(ps -aef | grep -w java | grep -v grep | awk '{print \$2}'); cat /proc/\$jpid/environ | strings | grep -v -E 'JAVA_OPTS|JAVA_TOOL_OPTIONS'> /tmp/env.sh" + done + + - name: Update RSA BlockNodes + run: | + + for pod in `sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep 'block-node' | awk '{print $1}'` + do + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- bash -c "mkdir -p /opt/hiero/block-node/application-state" + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" cp "${{ github.workspace}}"/solo/rsa-bootstrap-roster.json \ + ${pod}:/opt/hiero/block-node/application-state/rsa-bootstrap-roster.json + done + + - name: Temp down BlockNodes + run: | + for pod in $(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep 'block-node' | awk '{print $1}') + do + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- bash -c "ps -aef | grep java | grep -v grep | awk '{print \$2}' | xargs kill -9" + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- bash -c "cd /opt/hiero/block-node/; tar xf /tmp/BN.tar --overwrite" + done + + - name: Start BlockNodes + run: | + bnTar=$(ls "${{ github.workspace }}"/target_block_node/block-node/app/build/distributions/app-*.tar) + bnVersion=$(basename $bnTar | sed -e 's/.tar//g') + for pod in $(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep 'block-node' | awk '{print $1}') + do + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- \ + bash -c "find /opt/hiero/block-node/data -type f -exec rm -f {} \;" >/dev/null 2>&1 + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- \ + bash -c "rm -rf /opt/hiero/block-node/logs/blocknode.log*" >/dev/null 2>&1 + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- \ + bash -c ". /tmp/env.sh; export BLOCK_NODE_SERVER_MAX_MESSAGE_SIZE_BYTES=300000000; export JAVA_OPTS='-Xms48G -Xmx52G -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="/tmp/dump.hprof"'; \ + JAVA_TOOL_OPTIONS='-Djava.util.logging.config.file=/opt/hiero/block-node/logs/config/logging.properties -Dmetrics.exporter.openmetrics.http.hostname=0.0.0.0 -Dmetrics.exporter.openmetrics.http.port=16007 -Dmetrics.exporter.openmetrics.http.path=/metrics'; \ + nohup bash /opt/hiero/block-node/${bnVersion}/bin/app > /opt/hiero/block-node/logs/log.txt 2>&1 &" + done + + - name: Start CNs + run: | + set +x + set +e + cd "${{ github.workspace}}"/solo + task start + + - name: Solo logs + if: ${{ !cancelled() && always() }} + env: + NAMESPACE_ALIAS: ${{ inputs.test-asset }} + CONTEXT: hashgraph.teleport.sh-${{ steps.set-k8s-cluster.outputs.kubernetes-cluster }} + run: | + set +x + set +e + cd "${{ github.workspace}}"/solo task solo:node:logs - if [[ ${task_code} -ne 0 ]] - then - echo "ERROR! log:" - tail -n 100 "${HOME}"/.solo/logs/solo.log - fi sleep 30 sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods cd "${{ github.workspace }}" - mkdir solo_deploy cp -r "${HOME}"/.solo/logs solo_deploy/ sh "${{ github.workspace }}"/.github/workflows/support/citr/getClusterErrors.sh "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" @@ -428,8 +607,6 @@ jobs: gcloud --no-user-output-enabled storage cp --recursive solo_deploy "${{ env.GS_ROOT_DIR }}/${{ steps.set-hcn-version.outputs.run-hcn-version }}_${{ needs.acquire-kubernetes-resources.outputs.namespace }}_${{needs.acquire-kubernetes-resources.outputs.network-id }}_${{ github.run_number }}"/solo_deploy echo Done: solo deploy logs in "${{ env.GS_ROOT_HTTPS }}/${{ steps.set-hcn-version.outputs.run-hcn-version }}_${{ needs.acquire-kubernetes-resources.outputs.namespace }}_${{needs.acquire-kubernetes-resources.outputs.network-id }}_${{ github.run_number }}"/solo_deploy - exit "${task_code}" - - name: Move to Github Workspace run: cd "${{ github.workspace }}" @@ -474,7 +651,7 @@ jobs: set +x set +e cd "${{ github.workspace }}"/solo - /usr/bin/helm upgrade --install --set appName=nlg nlg oci://artifacts.hashgraph.io/load-generator-helm-release-local/network-load-generator --version "${{ env.NLG_VERSION }}" --values nlg-values.yaml -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" + /usr/bin/helm upgrade --install --set appName=nlg nlg oci://artifacts.hashgraph.io/load-generator-helm-release-local/network-load-generator --version "${{ inputs.nlg-version }}" --values nlg-values.yaml -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" sleep 180 sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods nlgpod="$(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep nlg-network-load-generator| awk '{print $1}')" @@ -487,7 +664,7 @@ jobs: run: | set +x set +e - nlgpod=`sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep nlg-network-load-generator | awk '{print $1}'` + nlgpod=$(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep nlg-network-load-generator | awk '{print $1}') sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${nlgpod} -- bash -c "rm -rf /opt/java/openjdk" sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${nlgpod} -- bash -c "\ apt update; \ @@ -502,7 +679,7 @@ jobs: set +x set +e sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" \ - cp ${{ github.workspace }}/hedera-state-validator/build/libs/hedera-state-validator-*-SNAPSHOT-all.jar network-node1-0:/tmp/ + cp ${{ github.workspace }}/target_consensus_node/hedera-state-validator/build/libs/hedera-state-validator-*-SNAPSHOT-all.jar network-node1-0:/tmp/ sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" \ exec -it network-node1-0 -c root-container -- bash -c "mkdir /opt/hgcapp/services-hedera/HapiApp2.0/data/saved/validation.tmp" @@ -515,7 +692,7 @@ jobs: set +x set +e export TERM=vt100 - cd "${{ github.workspace }}"/platform-sdk/swirlds-benchmarks + cd "${{ github.workspace }}"/target_consensus_node/platform-sdk/swirlds-benchmarks ulimit -n 16000 ../../gradlew jmhJar @@ -620,6 +797,12 @@ jobs: ${run_NLGDebugparams} -cp /app/lib/*:\$(ls -1 /app/network-load-generator-*.jar) com.hedera.benchmark.${{ env.run_NLG_Test }} \ ${NLGargs} -tt 10s > client.log 2>&1 &" + - name: Start BlockNode Cleaner + run: | + set +x + set +e + nohup sh "${{ github.workspace }}"/.github/workflows/support/citr/blockNodeClean.sh "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" > ${{ github.workspace }}/report/blockNodeClean.log 2>&1 & + - name: Wait for account creation run: | set +x @@ -787,7 +970,7 @@ jobs: - name: Checkout Code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - ref: ${{ inputs.ref }} + ref: ${{ github.ref }} - name: Print parameters run: | @@ -796,7 +979,15 @@ jobs: - name: Install KubeCtl uses: step-security/setup-kubectl@2edbf6aff97d814e9dc52827498ac51fe972e6d0 # v4.0.0 with: - version: v1.33.0 + version: v1.35.0 + + - name: kubectl wrapper + run: | + kubectl=$(which kubectl) + sudo mv ${kubectl} ${kubectl}.orig + sudo cp "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt ${kubectl} + sudo chmod a+rx ${kubectl} + which kubectl - name: Install Teleport Client uses: teleport-actions/setup@176c25dfcd19cd31a252f275d579822b243e7b9c # v1.0.6 @@ -858,6 +1049,12 @@ jobs: run: | mkdir "${{ github.workspace }}"/report + - name: Start BlockNode Cleaner + run: | + set +x + set +e + nohup sh "${{ github.workspace }}"/.github/workflows/support/citr/blockNodeClean.sh "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" > ${{ github.workspace }}/report/blockNodeClean.log 2>&1 & + - name: Wait for NLG Transfer run: | set +x @@ -970,7 +1167,7 @@ jobs: - name: Checkout Code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - ref: ${{ inputs.ref }} + ref: ${{ github.ref }} - name: Print parameters run: | @@ -979,7 +1176,15 @@ jobs: - name: Install KubeCtl uses: step-security/setup-kubectl@2edbf6aff97d814e9dc52827498ac51fe972e6d0 # v4.0.0 with: - version: v1.33.0 + version: v1.35.0 + + - name: kubectl wrapper + run: | + kubectl=$(which kubectl) + sudo mv ${kubectl} ${kubectl}.orig + sudo cp "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt ${kubectl} + sudo chmod a+rx ${kubectl} + which kubectl - name: Install Teleport Client uses: teleport-actions/setup@176c25dfcd19cd31a252f275d579822b243e7b9c # v1.0.6 @@ -1308,7 +1513,7 @@ jobs: - name: Checkout Code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - ref: ${{ inputs.ref }} + ref: ${{ github.ref }} - name: Print parameters run: | @@ -1317,7 +1522,15 @@ jobs: - name: Install KubeCtl uses: step-security/setup-kubectl@2edbf6aff97d814e9dc52827498ac51fe972e6d0 # v4.0.0 with: - version: v1.33.0 + version: v1.35.0 + + - name: kubectl wrapper + run: | + kubectl=$(which kubectl) + sudo mv ${kubectl} ${kubectl}.orig + sudo cp "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt ${kubectl} + sudo chmod a+rx ${kubectl} + which kubectl - name: Install Teleport Client uses: teleport-actions/setup@176c25dfcd19cd31a252f275d579822b243e7b9c # v1.0.6 @@ -1365,7 +1578,7 @@ jobs: run: | set +x set +e - kubectl -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" \ + KUBECTL_TIMEOUT=60m kubectl -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" \ exec -it network-node1-0 -c root-container -- bash -c "cd /opt/hgcapp/services-hedera/HapiApp2.0; bash checkValidatorNode.sh 0" 2>&1 | tee "${{ github.workspace }}"/report/validator_status.log sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" \ cp network-node1-0:/opt/hgcapp/services-hedera/HapiApp2.0/data/saved/validation.tmp/validator.log "${{ github.workspace }}"/report/validator.log @@ -1503,6 +1716,7 @@ jobs: - performance-SmartContractLoadTest - performance-post-test-validation - single-day-performance-test-result + if: ${{ (needs.performance-tests-start.result != 'success' || needs.performance-NftTransferLoadTest-watch1.result != 'success' || needs.performance-NftTransferLoadTest-report.result != 'success' || diff --git a/.github/workflows/832-call-execute-performance-test.yaml b/.github/workflows/832-call-execute-performance-test.yaml index 3a1604627370..5c68f6f985ef 100644 --- a/.github/workflows/832-call-execute-performance-test.yaml +++ b/.github/workflows/832-call-execute-performance-test.yaml @@ -68,6 +68,7 @@ env: GS_ROOT_HTTPS: https://perf.analytics.eng.hashgraph.io/ephemeral/test_runs #Release version from jFrog NLG_VERSION: 0.14.3 + java-version: "25.0.2" defaults: run: @@ -91,7 +92,7 @@ jobs: uses: pandaswhocode/initialize-github-job@5dbfbc4329ceab48a57d983325dadbd3ec8123e9 # v1.2.2 with: checkout: "true" - checkout-ref: "${{ inputs.ref }}" + checkout-ref: "${{ github.ref }}" checkout-token: "${{ secrets.GITHUB_TOKEN }}" - name: Validate Input Parameters @@ -169,7 +170,15 @@ jobs: - name: Install KubeCtl uses: step-security/setup-kubectl@2edbf6aff97d814e9dc52827498ac51fe972e6d0 # v4.0.0 with: - version: v1.33.0 + version: v1.35.0 + + - name: kubectl wrapper + run: | + kubectl=$(which kubectl) + sudo mv ${kubectl} ${kubectl}.orig + sudo cp "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt ${kubectl} + sudo chmod a+rx ${kubectl} + which kubectl - name: Install Teleport Client uses: teleport-actions/setup@176c25dfcd19cd31a252f275d579822b243e7b9c # v1.0.6 @@ -204,7 +213,7 @@ jobs: uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 with: distribution: temurin - java-version: "${{ inputs.java-version }}" + java-version: "${{ env.java-version }}" - name: Install Task uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0 @@ -235,9 +244,9 @@ jobs: run_NLGDparams="" # e.g. -Dbenchmark.maxtps=8000 run_NLGDebugparams="-Dorg.slf4j.simpleLogger.defaultLogLevel=debug" # e.g. -Dorg.slf4j.simpleLogger.defaultLogLevel=debug - nlgpod=`sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${K8S_NAMESPACE}" get pods | grep nlg-network-load-generator| awk '{print $1}'` + nlgpod=$(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${K8S_NAMESPACE}" get pods | grep nlg-network-load-generator| awk '{print $1}') - n=`expr "${NLG_ACCOUNTS}" / 1000` + n=$(expr "${NLG_ACCOUNTS}" / 1000) NLG_c=32 NLGargs="-metrics -R -c ${NLG_c} -a ${NLG_ACCOUNTS}" @@ -250,7 +259,7 @@ jobs: esac run_NLG_Time=${NLG_TIME} - run_NLG_Time3=`expr ${run_NLG_Time} / 3` + run_NLG_Time3=$(expr ${run_NLG_Time} / 3) case "${NLG_TEST}" in CryptoTransferLoadTest) run_NLG_Time=${run_NLG_Time3};; @@ -268,6 +277,12 @@ jobs: ${run_NLGDebugparams} -cp /app/lib/*:\$(ls -1 /app/network-load-generator-*.jar) com.hedera.benchmark.${NLG_TEST} \ ${NLGargs} -tt ${run_NLG_Time}m > client.log 2>&1 &" + - name: Start BlockNode Cleaner + run: | + set +x + set +e + nohup sh "${{ github.workspace }}"/.github/workflows/support/citr/blockNodeClean.sh "${{ inputs.k8s-namespace }}" > ${{ github.workspace }}/report/blockNodeClean.log 2>&1 & + - name: Wait for test completion env: K8S_NAMESPACE: ${{ inputs.k8s-namespace }} @@ -298,12 +313,12 @@ jobs: sleep 60 - nlgpod=`sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${K8S_NAMESPACE}" get pods | grep nlg-network-load-generator| awk '{print $1}'` + nlgpod=$(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${K8S_NAMESPACE}" get pods | grep nlg-network-load-generator| awk '{print $1}') sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${K8S_NAMESPACE}" exec ${nlgpod} -c nlg -- bash -c "tail /app/client.log" counter=0 - start_time=`date +%s` + start_time=$(date +%s) max_counter=340 #max_counter=330 mins of max test time + 10 min waiting time @@ -315,9 +330,9 @@ jobs: do sleep 60 - current_time=`date +%s` - counter=`expr "${current_time}" - "${start_time}"` - counter=`expr "${counter}" \/ 60` + current_time=$(date +%s) + counter=$(expr "${current_time}" - "${start_time}") + counter=$(expr "${counter}" \/ 60) sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${K8S_NAMESPACE}" exec ${nlgpod} -c nlg -- bash -c "tail /app/client.log" > client.log tail -n 1 client.log diff --git a/.github/workflows/833-call-single-day-longevity-test.yaml b/.github/workflows/833-call-single-day-longevity-test.yaml index 5ace04ad03d5..3511b555f1ac 100644 --- a/.github/workflows/833-call-single-day-longevity-test.yaml +++ b/.github/workflows/833-call-single-day-longevity-test.yaml @@ -11,7 +11,7 @@ on: type: string solo-version: required: true - default: "v0.41.0" + default: "v0.84.1" description: "Solo Version:" type: string nlg-accounts: #'-R -c 32 -a 100000000 -T 1000 -n 100000 -S hot -p 50 -tt 1m @@ -43,6 +43,16 @@ on: required: true description: "Expected duration of all tests in minutes" type: number + bnref: + description: "BN version" + type: string + required: true + default: "" + nlg-version: + required: true + default: "0.14.3" + description: "NLG Version:" + type: string chewie-request-timeout: required: true description: "Chewie request timeout in seconds" @@ -50,7 +60,6 @@ on: java-version: description: "Java JDK Version:" type: string - required: false secrets: slack-report-webhook: required: true @@ -80,9 +89,7 @@ env: GS_ROOT_HTTPS: https://perf.analytics.eng.hashgraph.io/ephemeral/test_runs GRAFANA_SYSADM_DAL: https://grafana.lat.ope.eng.hashgraph.io/d/rYdddlPWk/node-exporter-single-node?orgId=1&timezone=utc&var-datasource=P5C24830FD6F1746E&var-job=performance-dallas-node-exporter GRAFANA_SYSADM_CHI: https://grafana.lat.ope.eng.hashgraph.io/d/rYdddlPWk/node-exporter-single-node?orgId=1&timezone=utc&var-datasource=P631E2BA859A814B3&var-job=performance-chicago-node-exporter - - #Release version from jFrog - NLG_VERSION: 0.14.3 + NUM_OF_BLOCK_NODES: 3 jobs: get-chewie-jwt: @@ -95,7 +102,7 @@ jobs: uses: ./.github/workflows/861-call-get-test-config.yaml with: test-type: "sdlt" - ref: ${{ inputs.ref }} + ref: ${{ github.ref }} secrets: checkout-token: ${{ secrets.github-token }} @@ -104,7 +111,7 @@ jobs: - get-chewie-jwt uses: ./.github/workflows/860-call-validate-chewie-jwt.yaml with: - ref: ${{ inputs.ref }} + ref: ${{ github.ref }} chewie-expiration: ${{ needs.get-chewie-jwt.outputs.expiration }} secrets: chewie-jwt: ${{ needs.get-chewie-jwt.outputs.jwt }} @@ -171,13 +178,21 @@ jobs: uses: pandaswhocode/initialize-github-job@5dbfbc4329ceab48a57d983325dadbd3ec8123e9 # v1.2.2 with: checkout: "true" - checkout-ref: "${{ inputs.ref }}" + checkout-ref: "${{ github.ref }}" checkout-token: "${{ secrets.GITHUB_TOKEN }}" - setup-java: "true" - java-distribution: "temurin" - java-version: "${{ inputs.java-version }}" - setup-gradle: "true" - gradle-version: "wrapper" + + - name: Checkout target CN Code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: target_consensus_node + ref: ${{ inputs.ref }} + + - name: Checkout target BN Code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: "hiero-ledger/hiero-block-node" + path: target_block_node + ref: ${{ inputs.bnref }} - name: Set Kubernetes Cluster id: set-k8s-cluster @@ -194,7 +209,15 @@ jobs: - name: Install KubeCtl uses: step-security/setup-kubectl@2edbf6aff97d814e9dc52827498ac51fe972e6d0 # v4.0.0 with: - version: v1.33.0 + version: v1.35.0 + + - name: kubectl wrapper + run: | + kubectl=$(which kubectl) + sudo mv ${kubectl} ${kubectl}.orig + sudo cp "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt ${kubectl} + sudo chmod a+rx ${kubectl} + which kubectl - name: Install Teleport Client uses: teleport-actions/setup@176c25dfcd19cd31a252f275d579822b243e7b9c # v1.0.6 @@ -235,6 +258,17 @@ jobs: which helm helm version + - name: Setup Java + uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1 + with: + distribution: temurin + java-version: "${{ inputs.java-version }}" + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@4d9f0ba0025fe599b4ebab900eb7f3a1d93ef4c2 # v5.0.0 + with: + gradle-version: wrapper + - name: Install Task uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0 with: @@ -250,7 +284,7 @@ jobs: echo Setup procedure ... cat /etc/*elease sudo apt-get update - sudo apt-get install -y net-tools iputils-ping node-typescript + sudo apt-get install -qq -y net-tools iputils-ping node-typescript coreutils jq xxd openssl - name: Create report dir shell: bash @@ -284,16 +318,37 @@ jobs: - name: Set hederahash sha value id: hederahash run: | - hederahash="$(git rev-parse HEAD)" + hederahash="$(git -C target_consensus_node rev-parse HEAD)" echo "sha=${hederahash}" >> "${GITHUB_OUTPUT}" echo "hederahash=${hederahash}" | tee -a "${GITHUB_STEP_SUMMARY}" "${{ github.workspace }}"/version_run.txt - name: Compile - run: ./gradlew clean assemble + run: | + set +e + set +x + cd "${{ github.workspace }}"/target_consensus_node + + ./gradlew clean assemble + + cd "${{ github.workspace }}" + + - name: Compile BN + run: | + cd "${{ github.workspace }}"/target_block_node + ./gradlew app:distTar + + shortversion=$(echo "${{ inputs.bnref }}" | sed -e 's/^v\(.*\)/\1/g') + cd ./block-node/app/build/distributions/ + rm -rf app-${shortversion}/plugins app-${shortversion}.tar + tar cf app-${shortversion}.tar app-${shortversion} + + cd "${{ github.workspace }}" - name: Build Validator run: | + cd "${{ github.workspace }}"/target_consensus_node ./gradlew --no-daemon :hedera-state-validator:shadowJar + cd "${{ github.workspace }}" - name: Setup Node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 @@ -347,15 +402,21 @@ jobs: # Generate the init-containers-values7.yaml inline cp "${{ github.workspace }}"/.github/workflows/support/citr/init-containers-values7.yaml init-containers-values7.yaml + # Generate the block-node-values.yaml inline + cp "${{ github.workspace }}"/.github/workflows/support/citr/block-node-values.yaml block-node-values.yaml + # Update Taskfile values sed -i -e "s@%HOME%@${HOME}@g" -e "s@%SOLO_NAMESPACE%@${{ needs.acquire-kubernetes-resources.outputs.namespace }}@g" \ - -e "s@%HEDERA_SERVICES_ROOT%@${curdir}@g" Taskfile.yml + -e "s@%HEDERA_SERVICES_ROOT%@${curdir}/target_consensus_node@g" Taskfile.yml # Update Init Containers values NETWORK_ID="${{ needs.acquire-kubernetes-resources.outputs.network-id }}" sed -i -e "s@%NETWORK_ID%@${NETWORK_ID}@g" init-containers-values7.yaml + sed -i -e "s@%NETWORK_ID%@${NETWORK_ID}@g" block-node-values.yaml + NETWORK_OWNER="${{ needs.acquire-kubernetes-resources.outputs.owner }}" sed -i -e "s@%NETWORK_OWNER%@${NETWORK_OWNER}@g" init-containers-values7.yaml + sed -i -e "s@%NETWORK_OWNER%@${NETWORK_OWNER}@g" block-node-values.yaml # Generate the application.properties file inline cp "${{ github.workspace }}"/.github/workflows/support/citr/application.properties.txt application.properties @@ -382,18 +443,144 @@ jobs: export NODE_COPY_CONCURRENT=1 task - task_code=${?} + + - name: Add BlockNodes + run: | + set +x + set +e + cd "${{ github.workspace}}"/solo + + mv block-node-values.yaml block-node-values.yaml.orig + export BLOCK_NODE_VERSION="${{ inputs.bnref }}" #"${{ env.BlockNode_version }}" + #export CONSENSUS_NODE_VERSION=${{ inputs.ref }} + + #dedicate top-3 nodes for BN + NETWORK_ID="${{ needs.acquire-kubernetes-resources.outputs.network-id }}" + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt get nodes --show-labels | grep "network-id=${NETWORK_ID}," | grep 'consensus-node' | sort -n -k 1 | awk '{print $1}' | head -n 1 | grep -n -E '^' > firstNode.txt + + for i in $(seq 1 1 ${{ env.NUM_OF_BLOCK_NODES }}) + do + #create PVC values.yaml from template + for pvc_name in application-state-storage archive-storage logging-storage live-storage plugins-storage + do + sed -e "s@%NODE_ID%@${i}@g" -e "s@%PVC_NAME%@${pvc_name}@g" "${{ github.workspace }}"/.github/workflows/support/citr/block-node-pvc.yaml > block-node-pvc_${pvc_name}_${i}.yaml + echo "kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" create -f block-node-pvc_${pvc_name}_${i}.yaml" + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" create -f block-node-pvc_${pvc_name}_${i}.yaml + done + sleep 10 + done + + bnindex=1 + BNNODE=$(grep "${bnindex}:" firstNode.txt | awk -F : '{print $2}') + + sed -i -e "s@%SOLO_NAMESPACE%@${{ needs.acquire-kubernetes-resources.outputs.namespace }}@g" block-node-values.yaml.orig + + #BN node 1 + sed -e "s@%NODE_ID%@1@g" -e "s@%BNNODE%@${BNNODE}@g" -e "s@%ANOTHER_BN_1%@block-node-2@g" -e "s@%ANOTHER_BN_2%@block-node-3@g" block-node-values.yaml.orig > block-node-values1.yaml + cp block-node-values1.yaml block-node-values.yaml; + export BN_MAPPING="node1=1,node2=2,node3=3,node4=3,node5=1,node6=2,node7=2" + task solo:block:add + + #BN node 2 + sed -e "s@%NODE_ID%@2@g" -e "s@%BNNODE%@${BNNODE}@g" -e "s@%ANOTHER_BN_1%@block-node-1@g" -e "s@%ANOTHER_BN_2%@block-node-3@g" block-node-values.yaml.orig > block-node-values2.yaml + cp block-node-values2.yaml block-node-values.yaml; + export BN_MAPPING="node1=3,node2=1,node3=1,node4=2,node5=3,node6=3,node7=1" + task solo:block:add + + #BN node 3 + sed -e "s@%NODE_ID%@3@g" -e "s@%BNNODE%@${BNNODE}@g" -e "s@%ANOTHER_BN_1%@block-node-1@g" -e "s@%ANOTHER_BN_2%@block-node-2@g" block-node-values.yaml.orig > block-node-values3.yaml + cp block-node-values3.yaml block-node-values.yaml; + export BN_MAPPING="node1=2,node2=2,node3=3,node4=1,node5=1,node6=1,node7=3" + task solo:block:add + + sleep 30 + + - name: Deploy CNs + run: | + set +x + set +e + cd "${{ github.workspace}}"/solo + task installCNs + + - name: Generate RSARoster + run: | + genesisFile=$(find $HOME/.solo -type f -name 'genesis-network.json') + bash "${{ github.workspace }}"/.github/workflows/support/citr/genesisNetworkBNRSARoster.sh -i ${genesisFile} -o "${{ github.workspace}}"/solo/rsa-bootstrap-roster.json + jq . "${{ github.workspace}}"/solo/rsa-bootstrap-roster.json + + - name: Update BlockNodes + run: | + ls "${{ github.workspace }}"/target_block_node + ls "${{ github.workspace }}"/target_block_node/block-node/app/build/distributions/ + bnTar=$(ls "${{ github.workspace }}"/target_block_node/block-node/app/build/distributions/app-*.tar) + bnVersion=$(basename $bnTar | sed -e 's/.tar//g') + echo "$bnTar=$bnTar" + for pod in `sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep 'block-node' | awk '{print $1}'` + do + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" cp ${bnTar} ${pod}:/tmp/BN.tar + done + + for pod in `sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep 'block-node' | awk '{print $1}'` + do + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- bash -c "jpid=\$(ps -aef | grep -w java | grep -v grep | awk '{print \$2}'); cat /proc/\$jpid/environ | strings | grep -v -E 'JAVA_OPTS|JAVA_TOOL_OPTIONS'> /tmp/env.sh" + done + + - name: Update RSA BlockNodes + run: | + + for pod in `sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep 'block-node' | awk '{print $1}'` + do + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- bash -c "mkdir -p /opt/hiero/block-node/application-state" + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" cp "${{ github.workspace}}"/solo/rsa-bootstrap-roster.json \ + ${pod}:/opt/hiero/block-node/application-state/rsa-bootstrap-roster.json + done + + - name: Temp down BlockNodes + run: | + for pod in $(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep 'block-node' | awk '{print $1}') + do + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- bash -c "ps -aef | grep java | grep -v grep | awk '{print \$2}' | xargs kill -9" + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- bash -c "cd /opt/hiero/block-node/; tar xf /tmp/BN.tar --overwrite" + done + + - name: Start BlockNodes + run: | + bnTar=$(ls "${{ github.workspace }}"/target_block_node/block-node/app/build/distributions/app-*.tar) + bnVersion=$(basename $bnTar | sed -e 's/.tar//g') + for pod in $(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep 'block-node' | awk '{print $1}') + do + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- \ + bash -c "find /opt/hiero/block-node/data -type f -exec rm -f {} \;" >/dev/null 2>&1 + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- \ + bash -c "rm -rf /opt/hiero/block-node/logs/blocknode.log*" >/dev/null 2>&1 + sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${pod} -- \ + bash -c ". /tmp/env.sh; export BLOCK_NODE_SERVER_MAX_MESSAGE_SIZE_BYTES=300000000; export JAVA_OPTS='-Xms48G -Xmx52G -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="/tmp/dump.hprof"'; \ + JAVA_TOOL_OPTIONS='-Djava.util.logging.config.file=/opt/hiero/block-node/logs/config/logging.properties -Dmetrics.exporter.openmetrics.http.hostname=0.0.0.0 -Dmetrics.exporter.openmetrics.http.port=16007 -Dmetrics.exporter.openmetrics.http.path=/metrics'; \ + nohup bash /opt/hiero/block-node/${bnVersion}/bin/app > /opt/hiero/block-node/logs/log.txt 2>&1 &" + done + + - name: Start CNs + run: | + set +x + set +e + cd "${{ github.workspace}}"/solo + task start + + - name: Solo logs + if: ${{ !cancelled() && always() }} + env: + NAMESPACE_ALIAS: ${{ inputs.test-asset }} + CONTEXT: hashgraph.teleport.sh-${{ steps.set-k8s-cluster.outputs.kubernetes-cluster }} + REF: ${{ inputs.ref }} + run: | + set +x + set +e + cd "${{ github.workspace}}"/solo task solo:node:logs - if [[ ${task_code} -ne 0 ]] - then - echo "ERROR! log:" - tail -n 100 "${HOME}"/.solo/logs/solo.log - fi sleep 30 sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods cd "${{ github.workspace }}" - mkdir solo_deploy cp -r "${HOME}"/.solo/logs solo_deploy/ sh "${{ github.workspace }}"/.github/workflows/support/citr/getClusterErrors.sh "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" @@ -401,8 +588,6 @@ jobs: gcloud --no-user-output-enabled storage cp --recursive solo_deploy "${{ env.GS_ROOT_DIR }}/${REF}_${{ needs.acquire-kubernetes-resources.outputs.namespace }}_${{needs.acquire-kubernetes-resources.outputs.network-id }}_${{ github.run_number }}"/solo_deploy echo Done: solo deploy logs in "${{ env.GS_ROOT_HTTPS }}/${REF}_${{ needs.acquire-kubernetes-resources.outputs.namespace }}_${{needs.acquire-kubernetes-resources.outputs.network-id }}_${{ github.run_number }}"/solo_deploy - exit "${task_code}" - - name: Move to Github Workspace run: cd "${{ github.workspace }}" @@ -440,7 +625,7 @@ jobs: set +x set +e cd "${{ github.workspace }}"/solo - /usr/bin/helm upgrade --install --set appName=nlg nlg oci://artifacts.hashgraph.io/load-generator-helm-release-local/network-load-generator --version "${{ env.NLG_VERSION }}" --values nlg-values.yaml -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" + /usr/bin/helm upgrade --install --set appName=nlg nlg oci://artifacts.hashgraph.io/load-generator-helm-release-local/network-load-generator --version "${{ inputs.nlg-version }}" --values nlg-values.yaml -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" sleep 180 sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods nlgpod="$(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep nlg-network-load-generator| awk '{print $1}')" @@ -453,7 +638,7 @@ jobs: run: | set +x set +e - nlgpod=`sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep nlg-network-load-generator | awk '{print $1}'` + nlgpod=$(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep nlg-network-load-generator | awk '{print $1}') sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${nlgpod} -- bash -c "rm -rf /opt/java/openjdk" sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${nlgpod} -- bash -c "\ apt update; \ @@ -463,33 +648,12 @@ jobs: apt update; \ apt install -y temurin-25-jdk" - - name: Prepare NLG communication properties for CN - run: | - set +x - set +e - NLG_IP=$(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" describe pod nlg-network-load-generator | grep -E '^IP:' | awk '{print $2}') - - cat < block-nodes.json - { - "nodes": [ - { "address": "${NLG_IP}", "port": 8080, "streamingPort": 8080, "servicePort":-1, "priority": 0 } - ] - } - EOF - - NofNodes=$(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep 'network-node' | wc -l) - - for i in $(seq 1 1 ${NofNodes}) - do - sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" cp ./block-nodes.json network-node${i}-0:/opt/hgcapp/services-hedera/HapiApp2.0/data/config/block-nodes.json - done - - name: Deploy Validator run: | set +x set +e sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" \ - cp "${{ github.workspace }}"/hedera-state-validator/build/libs/hedera-state-validator-*-SNAPSHOT-all.jar network-node1-0:/tmp/ + cp "${{ github.workspace }}"/target_consensus_node/hedera-state-validator/build/libs/hedera-state-validator-*-SNAPSHOT-all.jar network-node1-0:/tmp/ sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" \ exec -it network-node1-0 -c root-container -- bash -c "mkdir /opt/hgcapp/services-hedera/HapiApp2.0/data/saved/validation.tmp" @@ -510,13 +674,13 @@ jobs: - name: Start Longevity test run: | run_NLGDebugparams="-Dorg.slf4j.simpleLogger.defaultLogLevel=debug" # e.g. -Dorg.slf4j.simpleLogger.defaultLogLevel=debug - nlgpod=`sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep nlg-network-load-generator| awk '{print $1}'` - n=`expr ${{ inputs.nlg-accounts }} / 1000` + nlgpod=$(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep nlg-network-load-generator| awk '{print $1}') + n=$(expr ${{ inputs.nlg-accounts }} / 1000) NLG_c=32 test=LongevityLoadTest - NLGargs="-K ECDSA -L -R -c ${NLG_c} -a ${{ inputs.nlg-accounts }} -n ${n} -T 1000 -S hot -p 50" + NLGargs="-K ECDSA -R -c ${NLG_c} -a ${{ inputs.nlg-accounts }} -n ${n} -T 1000 -S hot -p 50" echo kubectl -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${nlgpod} -c nlg -- bash -c "nohup /usr/bin/env java -Xmx30g \ ${run_NLGDebugparams} -cp /app/lib/*:\$(ls -1 /app/network-load-generator-*.jar) com.hedera.benchmark.${test} \ @@ -526,6 +690,7 @@ jobs: ${run_NLGDebugparams} -cp /app/lib/*:\$(ls -1 /app/network-load-generator-*.jar) com.hedera.benchmark.${test} \ ${NLGargs} -tt ${{ inputs.nlg-time }}m > client.log 2>&1 &" sleep 30 + - id: matrix run: | runs=$(((${{ inputs.nlg-time }}+${{ env.TIMEOUT_6H_LIMIT }}-1)/${{ env.TIMEOUT_6H_LIMIT }})) @@ -559,10 +724,23 @@ jobs: K8S_CLUSTER="${{ needs.acquire-kubernetes-resources.outputs.fqdn }}" echo "kubernetes-cluster=${K8S_CLUSTER}" >> "${GITHUB_OUTPUT}" + - name: Checkout Code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.ref }} + - name: Install KubeCtl uses: step-security/setup-kubectl@2edbf6aff97d814e9dc52827498ac51fe972e6d0 # v4.0.0 with: - version: v1.33.0 + version: v1.35.0 + + - name: kubectl wrapper + run: | + kubectl=$(which kubectl) + sudo mv ${kubectl} ${kubectl}.orig + sudo cp "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt ${kubectl} + sudo chmod a+rx ${kubectl} + which kubectl - name: Install Teleport Client uses: teleport-actions/setup@176c25dfcd19cd31a252f275d579822b243e7b9c # v1.0.6 @@ -589,7 +767,7 @@ jobs: - name: Checkout Code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - ref: ${{ inputs.ref }} + ref: ${{ github.ref }} - name: Create report dir shell: bash @@ -630,7 +808,7 @@ jobs: sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${nlgpod} -c nlg -- bash -c "tail /app/client.log" counter=0 - start_time=`date +%s` + start_time=$(date +%s) max_counter=${{ env.TIMEOUT_6H_LIMIT }} ec=2 @@ -703,10 +881,23 @@ jobs: K8S_CLUSTER="${{ needs.acquire-kubernetes-resources.outputs.fqdn }}" echo "kubernetes-cluster=${K8S_CLUSTER}" >> "${GITHUB_OUTPUT}" + - name: Checkout Code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.ref }} + - name: Install KubeCtl uses: step-security/setup-kubectl@2edbf6aff97d814e9dc52827498ac51fe972e6d0 # v4.0.0 with: - version: v1.33.0 + version: v1.35.0 + + - name: kubectl wrapper + run: | + kubectl=$(which kubectl) + sudo mv ${kubectl} ${kubectl}.orig + sudo cp "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt ${kubectl} + sudo chmod a+rx ${kubectl} + which kubectl - name: Install Teleport Client uses: teleport-actions/setup@176c25dfcd19cd31a252f275d579822b243e7b9c # v1.0.6 @@ -743,7 +934,7 @@ jobs: - name: Checkout Code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - ref: ${{ inputs.ref }} + ref: ${{ github.ref }} - name: Create report dir shell: bash @@ -763,7 +954,7 @@ jobs: run: | set +x set +e - kubectl -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" \ + KUBECTL_TIMEOUT=60m kubectl -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" \ exec -it network-node1-0 -c root-container -- bash -c "cd /opt/hgcapp/services-hedera/HapiApp2.0; bash checkValidatorNode.sh 0" 2>&1 | tee "${{ github.workspace }}"/report/validator_status.log sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" \ cp network-node1-0:/opt/hgcapp/services-hedera/HapiApp2.0/data/saved/validation.tmp/validator.log "${{ github.workspace }}"/report/validator.log @@ -842,9 +1033,6 @@ jobs: gcloud --no-user-output-enabled storage cp --recursive report "${{ env.GS_ROOT_DIR }}/${REF}_${{ needs.acquire-kubernetes-resources.outputs.namespace }}_${{needs.acquire-kubernetes-resources.outputs.network-id }}_${{ github.run_number }}/report/longevity" echo Done: see results in "${{ env.GS_ROOT_HTTPS }}/${REF}_${{ needs.acquire-kubernetes-resources.outputs.namespace }}_${{needs.acquire-kubernetes-resources.outputs.network-id }}_${{ github.run_number }}/report/longevity" - #echo "Truncating logs for next test..." - #sh "${{ github.workspace }}"/.github/workflows/support/citr/resetCNlogs.sh "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" - # Send this output back to the caller workflow longevity-test-reconnect: name: Reconnect test @@ -863,10 +1051,23 @@ jobs: K8S_CLUSTER=${{ needs.acquire-kubernetes-resources.outputs.fqdn }} echo "kubernetes-cluster=${K8S_CLUSTER}" >> "${GITHUB_OUTPUT}" + - name: Checkout Code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.ref }} + - name: Install KubeCtl uses: step-security/setup-kubectl@2edbf6aff97d814e9dc52827498ac51fe972e6d0 # v4.0.0 with: - version: v1.33.0 + version: v1.35.0 + + - name: kubectl wrapper + run: | + kubectl=$(which kubectl) + sudo mv ${kubectl} ${kubectl}.orig + sudo cp "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt ${kubectl} + sudo chmod a+rx ${kubectl} + which kubectl - name: Install Teleport Client uses: teleport-actions/setup@176c25dfcd19cd31a252f275d579822b243e7b9c # v1.0.6 @@ -903,7 +1104,7 @@ jobs: - name: Checkout Code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - ref: ${{ inputs.ref }} + ref: ${{ github.ref }} - name: Create report dir shell: bash @@ -913,13 +1114,13 @@ jobs: - name: Start Longevity test run: | run_NLGDebugparams="-Dorg.slf4j.simpleLogger.defaultLogLevel=debug" # e.g. -Dorg.slf4j.simpleLogger.defaultLogLevel=debug - nlgpod=`sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep nlg-network-load-generator | awk '{print $1}'` - n=`expr ${{ inputs.nlg-accounts }} / 1000` + nlgpod=$(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep nlg-network-load-generator | awk '{print $1}') + n=$(expr ${{ inputs.nlg-accounts }} / 1000) NLG_c=32 test=LongevityLoadTest - NLGargs="-K ECDSA -L -R -c ${NLG_c} -a ${{ inputs.nlg-accounts }} -n ${n} -T 1000 -S hot -p 50" + NLGargs="-K ECDSA -R -c ${NLG_c} -a ${{ inputs.nlg-accounts }} -n ${n} -T 1000 -S hot -p 50" echo kubectl -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${nlgpod} -c nlg -- bash -c "nohup /usr/bin/env java -Xmx30g \ ${run_NLGDebugparams} -cp /app/lib/*:\$(ls -1 /app/network-load-generator-*.jar) com.hedera.benchmark.${test} \ @@ -945,7 +1146,7 @@ jobs: mkdir -p "${{ github.workspace }}"/report/reconnect bash "${{ github.workspace }}"/.github/workflows/support/citr/profileReconnectLoopK8s.sh "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" ${downtime} ${warmtime} ${NofLoops} 2>&1 | tee "${{ github.workspace }}"/report/reconnect/reconnect.log - nlgpod=`sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep nlg-network-load-generator | awk '{print $1}'` + nlgpod=$(sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" get pods | grep nlg-network-load-generator | awk '{print $1}') kubectl -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${nlgpod} -c nlg -- bash -c "tail -n 20 /app/client.log" kubectl -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" exec ${nlgpod} -c nlg -- bash -c "ps -aef" | grep 'com.hedera.benchmark' result=$? @@ -966,7 +1167,7 @@ jobs: run: | set +x set +e - kubectl -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" \ + KUBECTL_TIMEOUT=60m kubectl -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" \ exec -it network-node1-0 -c root-container -- bash -c "cd /opt/hgcapp/services-hedera/HapiApp2.0; bash checkValidatorNode.sh 0" 2>&1 | tee "${{ github.workspace }}"/report/reconnect/validator_status.log sh "${{ github.workspace }}"/.github/workflows/support/citr/kubectlt -n "${{ needs.acquire-kubernetes-resources.outputs.namespace }}" \ cp network-node1-0:/opt/hgcapp/services-hedera/HapiApp2.0/data/saved/validation.tmp/validator.log "${{ github.workspace }}"/report/reconnect/validator.log diff --git a/.github/workflows/855-call-extract-citr-vars.yaml b/.github/workflows/855-call-extract-citr-vars.yaml index 35e980b35b89..ad36b98f1e01 100644 --- a/.github/workflows/855-call-extract-citr-vars.yaml +++ b/.github/workflows/855-call-extract-citr-vars.yaml @@ -33,7 +33,8 @@ on: value: ${{ jobs.extract-citr-vars.outputs.bn-mirror-node-version }} json-rpc-relay-version: value: ${{ jobs.extract-citr-vars.outputs.json-rpc-relay-version }} - + nlg-version: + value: ${{ jobs.extract-citr-vars.outputs.nlg-version }} permissions: contents: read @@ -56,6 +57,8 @@ jobs: block-node-version: ${{ steps.get-citr-vars.outputs.block-node-version }} bn-mirror-node-version: ${{ steps.get-citr-vars.outputs.bn-mirror-node-version }} json-rpc-relay-version: ${{ steps.get-citr-vars.outputs.json-rpc-relay-version }} + nlg-version: ${{ steps.get-citr-vars.outputs.nlg-version }} + steps: - name: Initialize Job uses: pandaswhocode/initialize-github-job@5dbfbc4329ceab48a57d983325dadbd3ec8123e9 # v1.2.2 @@ -100,6 +103,7 @@ jobs: "block-node-version" "bn-mirror-node-version" "json-rpc-relay-version" + "nlg-version" ) # Loop through expected keys and check if they are set in GITHUB_OUTPUT diff --git a/.github/workflows/support/chewie/AdHoc16-sdlt.json b/.github/workflows/support/chewie/AdHoc16-sdlt.json new file mode 100644 index 000000000000..b7c03d91edc5 --- /dev/null +++ b/.github/workflows/support/chewie/AdHoc16-sdlt.json @@ -0,0 +1,44 @@ +{ + "id": 8, + "status": "approved", + "status_url": "/api/v1/compute/allocation/16", + "namespace": "chewie-solo-sdlt-n16", + "cluster_fqdn": "k8s.pft.dal.lat.ope.eng.hashgraph.io", + "expires_at": "2027-04-17T12:00:00Z", + "instances": [ + { + "group": "cn-nodes", + "spec": { + "quantity": 9, + "resources": { "cpu": 48, "memory": 393216 } + }, + "labels": { + "solo.hashgraph.io/role": "consensus-node", + "solo.hashgraph.io/owner": "adhoc-performance-test", + "solo.hashgraph.io/network-id": "16" + }, + "tolerations": [ + { "key": "solo.hashgraph.io/role", "operator": "Equal", "value": "consensus-node", "effect": "NoSchedule" }, + { "key": "solo.hashgraph.io/owner", "operator": "Equal", "value": "adhoc-performance-test", "effect": "NoSchedule" }, + { "key": "solo.hashgraph.io/network-id", "operator": "Equal", "value": "16", "effect": "NoSchedule" } + ] + }, + { + "group": "aux-nodes", + "spec": { + "quantity": 1, + "resources": { "cpu": 48, "memory": 393216 } + }, + "labels": { + "solo.hashgraph.io/role": "auxiliary-services", + "solo.hashgraph.io/owner": "adhoc-performance-test", + "solo.hashgraph.io/network-id": "16" + }, + "tolerations": [ + { "key": "solo.hashgraph.io/role", "operator": "Equal", "value": "auxiliary-services", "effect": "NoSchedule" }, + { "key": "solo.hashgraph.io/owner", "operator": "Equal", "value": "adhoc-performance-test", "effect": "NoSchedule" }, + { "key": "solo.hashgraph.io/network-id", "operator": "Equal", "value": "16", "effect": "NoSchedule" } + ] + } + ] +} diff --git a/.github/workflows/support/chewie/AdHoc16-sdpt.json b/.github/workflows/support/chewie/AdHoc16-sdpt.json new file mode 100644 index 000000000000..312af0d0e94b --- /dev/null +++ b/.github/workflows/support/chewie/AdHoc16-sdpt.json @@ -0,0 +1,44 @@ +{ + "id": 4, + "status": "approved", + "status_url": "/api/v1/compute/allocation/16", + "namespace": "chewie-solo-sdpt-n16", + "cluster_fqdn": "k8s.pft.dal.lat.ope.eng.hashgraph.io", + "expires_at": "2027-04-17T12:00:00Z", + "instances": [ + { + "group": "cn-nodes", + "spec": { + "quantity": 9, + "resources": { "cpu": 48, "memory": 393216 } + }, + "labels": { + "solo.hashgraph.io/role": "consensus-node", + "solo.hashgraph.io/owner": "adhoc-performance-test", + "solo.hashgraph.io/network-id": "16" + }, + "tolerations": [ + { "key": "solo.hashgraph.io/role", "operator": "Equal", "value": "consensus-node", "effect": "NoSchedule" }, + { "key": "solo.hashgraph.io/owner", "operator": "Equal", "value": "adhoc-performance-test", "effect": "NoSchedule" }, + { "key": "solo.hashgraph.io/network-id", "operator": "Equal", "value": "16", "effect": "NoSchedule" } + ] + }, + { + "group": "aux-nodes", + "spec": { + "quantity": 1, + "resources": { "cpu": 48, "memory": 393216 } + }, + "labels": { + "solo.hashgraph.io/role": "auxiliary-services", + "solo.hashgraph.io/owner": "adhoc-performance-test", + "solo.hashgraph.io/network-id": "16" + }, + "tolerations": [ + { "key": "solo.hashgraph.io/role", "operator": "Equal", "value": "auxiliary-services", "effect": "NoSchedule" }, + { "key": "solo.hashgraph.io/owner", "operator": "Equal", "value": "adhoc-performance-test", "effect": "NoSchedule" }, + { "key": "solo.hashgraph.io/network-id", "operator": "Equal", "value": "16", "effect": "NoSchedule" } + ] + } + ] +} diff --git a/.github/workflows/support/citr/.citr-env b/.github/workflows/support/citr/.citr-env index 9c5a2bfa74f3..fc4adfb0b204 100644 --- a/.github/workflows/support/citr/.citr-env +++ b/.github/workflows/support/citr/.citr-env @@ -1,10 +1,11 @@ -citr-solo-version=0.79.1 -adhoc-solo-version=0.79.1 +citr-solo-version=v0.84.1 +adhoc-solo-version=v0.84.1 adhoc-xts-solo-version=0.79.1 -cutover-solo-version=0.84.1 +cutover-solo-version=v0.84.1 citr-mirror-node-version=0.161.0 tck-version=v0.12.3 js-sdk-version=v2.85.0 -block-node-version=v0.35.1 +block-node-version=v0.40.0-rc1 bn-mirror-node-version=v0.157.0 json-rpc-relay-version=v0.78.0 +nlg-version=0.14.3 diff --git a/.github/workflows/support/citr/Taskfile.citr.yml b/.github/workflows/support/citr/Taskfile.citr.yml index 90bcffc2a454..4a3d2ac73bfe 100644 --- a/.github/workflows/support/citr/Taskfile.citr.yml +++ b/.github/workflows/support/citr/Taskfile.citr.yml @@ -36,7 +36,6 @@ tasks: - task: "install:kubectl:darwin" - task: "install:kubectl:linux" - task: "install" - - task: "start" install: desc: solo init, solo cluster-ref create, solo node keys, solo network deploy @@ -47,8 +46,14 @@ tasks: - task: "solo:cluster-ref:connect" - task: "solo:deployment:create" - task: "solo:deployment:add-cluster" + + installCNs: + desc: solo init, solo cluster-ref create, solo node keys, solo network deploy + cmds: - task: "solo:keys" - task: "solo:network:deploy" + - task: "solo:node:setup" + #- task: "start" destroy: cmds: @@ -106,6 +111,7 @@ tasks: - echo "MIRROR_NODE_DEPLOY_EXTRA_FLAGS=${MIRROR_NODE_DEPLOY_EXTRA_FLAGS}" - echo "EXPLORER_DEPLOY_EXTRA_FLAGS=${EXPLORER_DEPLOY_EXTRA_FLAGS}" - echo "NODE_SETUP_EXTRA_FLAGS=${NODE_SETUP_EXTRA_FLAGS}" + - echo "BLOCK_NODE_MESSAGE_SIZE=${BLOCK_NODE_MESSAGE_SIZE} - touch {{ .var_check_file }} readme: @@ -249,11 +255,10 @@ tasks: export CONSENSUS_NODE_FLAG='--release-tag {{.CONSENSUS_NODE_VERSION}}' fi if [[ "{{ .solo_ge_0440 }}" == "true" ]]; then - SOLO_HOME_DIR=${SOLO_HOME_DIR} solo consensus network deploy --deployment "${SOLO_DEPLOYMENT}" --node-aliases {{.node_identifiers}} ${CONSENSUS_NODE_FLAG} ${VALUES_FLAG} ${SETTINGS_FLAG} ${LOG4J2_FLAG} ${APPLICATION_PROPERTIES_FLAG} ${APPLICATION_ENVIRONMENT_FLAG} ${GENESIS_THROTTLES_FLAG} ${DEBUG_NODE_FLAG} ${SOLO_CHARTS_DIR_FLAG} ${LOAD_BALANCER_FLAG} ${NETWORK_DEPLOY_EXTRA_FLAGS} -q --dev + SOLO_HOME_DIR=${SOLO_HOME_DIR} solo consensus network deploy --deployment "${SOLO_DEPLOYMENT}" --node-aliases {{.node_identifiers}} ${CONSENSUS_NODE_FLAG} ${VALUES_FLAG} ${SETTINGS_FLAG} ${LOG4J2_FLAG} ${APPLICATION_PROPERTIES_FLAG} ${APPLICATION_ENVIRONMENT_FLAG} ${GENESIS_THROTTLES_FLAG} ${DEBUG_NODE_FLAG} ${SOLO_CHARTS_DIR_FLAG} ${LOAD_BALANCER_FLAG} ${NETWORK_DEPLOY_EXTRA_FLAGS} ${BLOCK_NODE_MESSAGE_SIZE} -q --dev else - SOLO_HOME_DIR=${SOLO_HOME_DIR} solo network deploy --deployment "${SOLO_DEPLOYMENT}" --node-aliases {{.node_identifiers}} ${CONSENSUS_NODE_FLAG} ${VALUES_FLAG} ${SETTINGS_FLAG} ${LOG4J2_FLAG} ${APPLICATION_PROPERTIES_FLAG} ${APPLICATION_ENVIRONMENT_FLAG} ${GENESIS_THROTTLES_FLAG} ${DEBUG_NODE_FLAG} ${SOLO_CHARTS_DIR_FLAG} ${LOAD_BALANCER_FLAG} ${NETWORK_DEPLOY_EXTRA_FLAGS} -q --dev + SOLO_HOME_DIR=${SOLO_HOME_DIR} solo network deploy --deployment "${SOLO_DEPLOYMENT}" --node-aliases {{.node_identifiers}} ${CONSENSUS_NODE_FLAG} ${VALUES_FLAG} ${SETTINGS_FLAG} ${LOG4J2_FLAG} ${APPLICATION_PROPERTIES_FLAG} ${APPLICATION_ENVIRONMENT_FLAG} ${GENESIS_THROTTLES_FLAG} ${DEBUG_NODE_FLAG} ${SOLO_CHARTS_DIR_FLAG} ${LOAD_BALANCER_FLAG} ${NETWORK_DEPLOY_EXTRA_FLAGS} ${BLOCK_NODE_MESSAGE_SIZE} -q --dev fi - - task: "solo:node:setup" solo:node:setup: silent: true @@ -623,3 +628,17 @@ tasks: kubectl exec -it {{.postgres_container_name}} \ -n {{.postgres_database_namespace}} \ -- /bin/bash /tmp/init.sh "{{.postgres_username}}" "{{.postgres_readonly_username}}" "{{.postgres_readonly_password}}" + + solo:block:add: + silent: true + desc: solo block node add + cmds: + - | + SOLO_HOME_DIR=${SOLO_HOME_DIR} solo block node add --deployment "${SOLO_DEPLOYMENT}" --cluster-ref ${CLUSTER_REF} ${SOLO_CHARTS_DIR_FLAG} ${BLOCK_NODE_ADD_EXTRA_FLAGS} ${BLOCK_NODE_MESSAGE_SIZE} -q --dev + + solo:block:destroy: + silent: true + desc: solo block node destroy + cmds: + - | + SOLO_HOME_DIR=${SOLO_HOME_DIR} solo block node destroy --deployment "${SOLO_DEPLOYMENT}" --cluster-ref ${CLUSTER_REF} --force -q --dev || true diff --git a/.github/workflows/support/citr/Taskfile.yml b/.github/workflows/support/citr/Taskfile.yml index 1ace56191374..0880144a1bbb 100644 --- a/.github/workflows/support/citr/Taskfile.yml +++ b/.github/workflows/support/citr/Taskfile.yml @@ -9,7 +9,7 @@ vars: env: SOLO_NETWORK_SIZE: 7 SOLO_NAMESPACE: %SOLO_NAMESPACE% - CONSENSUS_NODE_VERSION: 0.74.0-0 + CONSENSUS_NODE_VERSION: 0.76.0-0 VALUES_FLAG: "--values-file {{.USER_WORKING_DIR}}/init-containers-values7.yaml" SETTINGS_FLAG: "--settings-txt {{.USER_WORKING_DIR}}/settings.txt" SOLO_HOME: "{{.solo_home_override_dir}}" @@ -18,6 +18,7 @@ env: APPLICATION_ENVIRONMENT_FLAG: "--application-env {{.USER_WORKING_DIR}}/application.environment" HEDERA_SERVICES_ROOT: "%HEDERA_SERVICES_ROOT%" LOCAL_BUILD_FLAG: "--local-build-path {{.HEDERA_SERVICES_ROOT}}/hedera-node/data" + NETWORK_DEPLOY_EXTRA_FLAGS: "--wraps --tss" #ENABLE_EXPLORER_TLS_FLAG: "--enable-hedera-explorer-tls" ENABLE_EXPLORER_INGRESS: "--enable-ingress" @@ -32,3 +33,8 @@ env: SOLO_CLUSTER_SETUP_NAMESPACE: solo-setup SOLO_CLUSTER_NAME: solo-cluster-name MIRROR_RELEASE_NAME: release_name + BLOCK_NODE_ADD_EXTRA_FLAGS: --release-tag {{.CONSENSUS_NODE_VERSION}} --values-file {{.USER_WORKING_DIR}}/block-node-values.yaml --priority-mapping {{.BN_MAPPING}} + SOLO_LOG_LEVEL: debug + BLOCK_NODE_MESSAGE_SIZE: "--block-node-message-size-soft-limit-bytes 300000000 --block-node-message-size-hard-limit-bytes 300000000" + BLOCK_STREAM_STREAM_MODE: BOTH + BLOCK_STREAM_WRITER_MODE: FILE_AND_GRPC diff --git a/.github/workflows/support/citr/application.environment.txt b/.github/workflows/support/citr/application.environment.txt index f78353ec84e5..13355a239600 100644 --- a/.github/workflows/support/citr/application.environment.txt +++ b/.github/workflows/support/citr/application.environment.txt @@ -1 +1,2 @@ MALLOC_ARENA_MAX=4 +TSS_LIB_WRAPS_ARTIFACTS_PATH=/opt/hgcapp/services-hedera/HapiApp2.0/data/keys/wraps diff --git a/.github/workflows/support/citr/application.properties.txt b/.github/workflows/support/citr/application.properties.txt index c86240b0acfa..4498642237e7 100644 --- a/.github/workflows/support/citr/application.properties.txt +++ b/.github/workflows/support/citr/application.properties.txt @@ -8,10 +8,6 @@ networkAdmin.exportCandidateRoster=true addressBook.useRosterLifecycle=true nodes.gossipFqdnRestricted=false -#enable Latency Metrics -blockNode.globalCoolDownSeconds=1 -blockNode.basicNodeCoolDownSeconds=1 -blockNode.extendedNodeCoolDownSeconds=1 -blockNode.connectionStallThresholdMillis=5000 +blockStream.blockNodeConnectionFileDir=/opt/hgcapp/services-hedera/HapiApp2.0/data/config staking.periodMins=360 diff --git a/.github/workflows/support/citr/block-node-pvc.yaml b/.github/workflows/support/citr/block-node-pvc.yaml new file mode 100644 index 000000000000..979cefae43b3 --- /dev/null +++ b/.github/workflows/support/citr/block-node-pvc.yaml @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: Apache-2.0 +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: %PVC_NAME%-block-node-%NODE_ID%-0 +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 128Mi diff --git a/.github/workflows/support/citr/block-node-values.yaml b/.github/workflows/support/citr/block-node-values.yaml new file mode 100644 index 000000000000..b0e2162cd7f8 --- /dev/null +++ b/.github/workflows/support/citr/block-node-values.yaml @@ -0,0 +1,176 @@ +# SPDX-License-Identifier: Apache-2.0 +replicas: 1 + +resources: + limits: + cpu: 8 + memory: 54Gi + requests: + cpu: 8 + memory: 54Gi + +nodeSelector: + #solo.hashgraph.io/role: "auxiliary-services"consensus-node + solo.hashgraph.io/role: "consensus-node" + solo.hashgraph.io/owner: "%NETWORK_OWNER%" + solo.hashgraph.io/network-id: "%NETWORK_ID%" + kubernetes.io/hostname: "%BNNODE%" + +tolerations: + - key: "solo.hashgraph.io/role" + operator: "Equal" + #value: "auxiliary-services" + value: "consensus-node" + effect: "NoSchedule" + - key: "solo.hashgraph.io/owner" + operator: "Equal" + value: "%NETWORK_OWNER%" + effect: "NoSchedule" + - key: "solo.hashgraph.io/network-id" + operator: "Equal" + value: "%NETWORK_ID%" + effect: "NoSchedule" + - key: "kubernetes.io/hostname" + operator: "Equal" + value: "%BNNODE%" + +image: + repository: ghcr.io/hiero-ledger/hiero-block-node-solo-dev + +plugins: + names: "facility-messaging,block-access-service,health,server-status,stream-publisher,stream-subscriber,verification,blocks-file-historic,blocks-file-recent,backfill,roster-bootstrap-rsa" + repositories: + - id: besu-maven + url: https://artifacts.hashgraph.io/artifactory/hyperledger-besu-maven-external/ + +blockNode: + # if blank will use same as AppVersion of chart. + version: "" + config: + #BLOCK_NODE_EARLIEST_MANAGED_BLOCK: "1000" + JAVA_OPTS: '-Xms48G -Xmx52G -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="/tmp/dump.hprof"' + BLOCK_NODE_SERVER_MAX_MESSAGE_SIZE_BYTES: "300000000" + SERVER_MAX_MESSAGE_SIZE_BYTES: "300000000" + BACKFILL_BLOCK_NODE_SOURCES_PATH: "/opt/hiero/block-node/backfill/block-node-sources.json" + BACKFILL_FETCH_BATCH_SIZE: "10" + backfill: + ## The path to the block-node-sources.json file must match the path property in + ## blockNode.config.BACKFILL_BLOCK_NODE_SOURCES_PATH + path: "/opt/hiero/block-node/backfill" + filename: "block-node-sources.json" + sources: + # note that this is an example of another (source) BN on the same K8 cluster + # different namespace ..svc.cluster.local + - address: "%ANOTHER_BN_1%" + port: 40840 + priority: 1 + - address: "%ANOTHER_BN_2%" + port: 40840 + priority: 2 + persistence: + archive: + create: false + # name of the volume within the statefulset + volumeName: "archive-storage" + # Name of the subPath in the volume to mount to mountPath in the container + subPath: "archive-data" + # should match PERSISTENCE_STORAGE_ARCHIVE_ROOT_PATH, leave as is for default. + mountPath: "/opt/hiero/block-node/data/historic" + size: 800Gi + # Optionally add a storage class name if needed + storageClass: "local-path" + # Optionally specify an existing PVC name instead of creating via volumeClaimTemplate + existingClaim: "archive-storage-block-node-%NODE_ID%-0" + logging: + create: false + # name of the volume within the statefulset + volumeName: "logging-storage" + mountPath: "/opt/hiero/block-node/logs" + size: 2Gi + # Optionally add a storage class name if needed + storageClass: "local-path" + # Optionally specify an existing PVC name instead of creating via volumeClaimTemplate + existingClaim: "logging-storage-block-node-%NODE_ID%-0" + live: + create: false + # name of the volume within the statefulset + volumeName: "live-storage" + # Name of the subPath in the volume to mount to mountPath in the container + subPath: "live-data" + # should match PERSISTENCE_STORAGE_LIVE_ROOT_PATH, leave as is for default. + mountPath: "/opt/hiero/block-node/data/live" + size: 20Gi + # Optionally add a storage class name if needed + storageClass: "local-path" + # Optionally specify an existing PVC name instead of creating via volumeClaimTemplate + existingClaim: "live-storage-block-node-%NODE_ID%-0" + application-state: + create: false + # name of the volume within the statefulset + volumeName: "application-state-storage" + mountPath: "/opt/hiero/block-node/application-state" + size: 1Gi + # Optionally add a storage class name if needed + storageClass: "local-path" + # Optionally specify an existing PVC name instead of creating via volumeClaimTemplate + existingClaim: "application-state-storage-block-node-%NODE_ID%-0" + plugins: + create: false + # name of the volume within the statefulset + volumeName: "plugins-storage" + size: 1Gi + # Optionally add a storage class name if needed + # storageClass: "your-storage-class" + # Optionally specify an existing PVC name instead of creating via volumeClaimTemplate + existingClaim: "plugins-storage-block-node-%NODE_ID%-0" + + health: + readiness: + endpoint: "/healthz/readyz" + initialDelaySeconds: 60 + timeoutSeconds: 30 + liveness: + endpoint: "/healthz/livez" + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 30 + successThreshold: 1 + failureThreshold: 99999999999 + + logs: + # Available Levels are (from most verbose to least verbose): + # ALL FINEST FINER FINE CONFIG INFO WARNING SEVERE OFF + level: "INFO" + configMountPath: "/opt/hiero/block-node/logs/config" + loggingProperties: + org.hiero.block.level: "FINEST" + io.helidon.webserver.level: "INFO" + io.helidon.webserver.access.level: "INFO" + io.helidon.config.level: "SEVERE" + io.helidon.security.level: "INFO" + io.helidon.common.level: "INFO" + handlers: "java.util.logging.ConsoleHandler, java.util.logging.FileHandler" + java.util.logging.ConsoleHandler.level: "FINE" + java.util.logging.ConsoleHandler.formatter: "java.util.logging.SimpleFormatter" + java.util.logging.FileHandler.pattern: "/opt/hiero/block-node/logs/blocknode.log" + java.util.logging.FileHandler.append: "true" + java.util.logging.FileHandler.limit: "0" + java.util.logging.FileHandler.count: "1" + java.util.logging.FileHandler.level: "FINE" + java.util.logging.FileHandler.formatter: "java.util.logging.SimpleFormatter" + ################################################################################ + # SimpleFormatter single-line format configuration + ################################################################################ + # The format syntax uses java.util.Formatter. + # The parameters are: + # %1$ - date/time (java.util.Date) + # %2$ - source (usually class and method) + # %3$ - logger?s name + # %4$ - log level + # %5$ - log message + # %6$ - throwable trace + java.util.logging.SimpleFormatter.format: "%TF %/dev/null 2>&1 & + done + + wait + sleep 3600 +done \ No newline at end of file diff --git a/.github/workflows/support/citr/block_node_env.sh b/.github/workflows/support/citr/block_node_env.sh new file mode 100644 index 000000000000..5b0948df6de1 --- /dev/null +++ b/.github/workflows/support/citr/block_node_env.sh @@ -0,0 +1,148 @@ + + + + +MINIO_CONSOLE_IP=`cat get_svc.txt | grep 'minio-console' | awk '{print $3}'` +MINIO_IP=`cat get_svc.txt | grep -E '^minio[ \t]' | awk '{print $3}'` + + +cat << EOF | awk '{print "export "$0}' > block_node.env +MINIO_CONSOLE_PORT_9090_TCP_ADDR=$MINIO_CONSOLE_IP +MINIO_CONSOLE_PORT_9090_TCP_PORT=9090 +MINIO_SERVICE_PORT=80 +MINIO_PORT=tcp://$MINIO_IP:80 +MINIO_CONSOLE_PORT_9090_TCP_PROTO=tcp +MINIO_CONSOLE_PORT=tcp://$MINIO_CONSOLE_IP:9090 +MINIO_CONSOLE_SERVICE_PORT=9090 +MINIO_PORT_80_TCP_ADDR=$MINIO_IP +MINIO_PORT_80_TCP_PORT=80 +MINIO_CONSOLE_PORT_9090_TCP=tcp://$MINIO_CONSOLE_IP:9090 +MINIO_PORT_80_TCP_PROTO=tcp +MINIO_PORT_80_TCP=tcp://$MINIO_IP:80 +MINIO_CONSOLE_SERVICE_PORT_HTTP_CONSOLE=9090 +MINIO_SERVICE_PORT_HTTP_MINIO=80 +MINIO_SERVICE_HOST=$MINIO_IP +MINIO_CONSOLE_SERVICE_HOST=$MINIO_CONSOLE_IP +PROMETHEUS_ENDPOINT_ENABLED=false +PROMETHEUS_ENDPOINT_PORT_NUMBER=16007 +SERVER_PORT=40840 +BN_WORKDIR=/opt/hiero/block-node +JAVA_TOOL_OPTIONS=-Djava.util.logging.config.file=/opt/hiero/block-node/logs/config/logging.properties +KUBERNETES_PORT_443_TCP_PORT=443 +KUBERNETES_PORT_443_TCP_PROTO=tcp +KUBERNETES_SERVICE_PORT_HTTPS=443 +KUBERNETES_SERVICE_PORT=443 +LANG=C.UTF-8 +LC_ALL=C.UTF-8 +MESSAGING_BLOCK_NOTIFICATION_QUEUE_SIZE=16 +EOF + +for i in $(seq 1 1 7) +do + + haproxy_node_svc=`cat get_svc.txt | grep -E "haproxy-node${i}-svc" | awk '{print $3}'` + network_node_svc=`cat get_svc.txt | grep -E "network-node${i}-svc" | awk '{print $3}'` + envoy_proxy_node_svc=`cat get_svc.txt | grep -E "envoy-proxy-node${i}-svc" | awk '{print $3}'` + + cat << EOF | awk '{print "export "$0}' >> block_node.env +NETWORK_NODE${i}_SVC_PORT_50111_TCP=tcp://${network_node_svc}:50111 +NETWORK_NODE${i}_SVC_PORT_9090_TCP=tcp://${network_node_svc}:9090 +NETWORK_NODE${i}_SVC_PORT_50212_TCP=tcp://${network_node_svc}:50212 +NETWORK_NODE${i}_SVC_SERVICE_PORT_GRPC_NON_TLS=50211 +ENVOY_PROXY_NODE${i}_SVC_SERVICE_HOST=${envoy_proxy_node_svc} +ENVOY_PROXY_NODE${i}_SVC_PORT_8080_TCP_ADDR=${envoy_proxy_node_svc} +HAPROXY_NODE${i}_SVC_SERVICE_HOST=${haproxy_node_svc} +ENVOY_PROXY_NODE${i}_SVC_PORT_9090_TCP_ADDR=${envoy_proxy_node_svc} +HAPROXY_NODE${i}_SVC_PORT_9090_TCP_ADDR=${haproxy_node_svc} +NETWORK_NODE${i}_SVC_SERVICE_PORT_PROMETHEUS=9090 +ENVOY_PROXY_NODE${i}_SVC_PORT_8080_TCP_PORT=8080 +HAPROXY_NODE${i}_SVC_PORT_50211_TCP_ADDR=${haproxy_node_svc} +NETWORK_NODE${i}_SVC_SERVICE_PORT_GOSSIP=50111 +HAPROXY_NODE${i}_SVC_PORT_50212_TCP_ADDR=${haproxy_node_svc} +ENVOY_PROXY_NODE${i}_SVC_PORT_8080_TCP_PROTO=tcp +HAPROXY_NODE${i}_SVC_SERVICE_PORT_NON_TLS_GRPC_CLIENT_PORT=50211 +ENVOY_PROXY_NODE${i}_SVC_PORT_9090_TCP_PORT=9090 +ENVOY_PROXY_NODE${i}_SVC_PORT_9090_TCP_PROTO=tcp +HAPROXY_NODE${i}_SVC_PORT_50211_TCP_PORT=50211 +HAPROXY_NODE${i}_SVC_PORT_9090_TCP_PORT=9090 +HAPROXY_NODE${i}_SVC_PORT_50211_TCP_PROTO=tcp +HAPROXY_NODE${i}_SVC_PORT_50212_TCP_PORT=50212 +ENVOY_PROXY_NODE${i}_SVC_SERVICE_PORT=8080 +ENVOY_PROXY_NODE${i}_SVC_PORT=tcp://${envoy_proxy_node_svc}:8080 +HAPROXY_NODE${i}_SVC_PORT_9090_TCP_PROTO=tcp +HAPROXY_NODE${i}_SVC_SERVICE_PORT=50211 +HAPROXY_NODE${i}_SVC_PORT=tcp://${haproxy_node_svc}:50211 +HAPROXY_NODE${i}_SVC_PORT_50212_TCP_PROTO=tcp +ENVOY_PROXY_NODE${i}_SVC_PORT_8080_TCP=tcp://${envoy_proxy_node_svc}:8080 +ENVOY_PROXY_NODE${i}_SVC_PORT_9090_TCP=tcp://${envoy_proxy_node_svc}:9090 +HAPROXY_NODE${i}_SVC_PORT_50211_TCP=tcp://${haproxy_node_svc}:50211 +HAPROXY_NODE${i}_SVC_PORT_9090_TCP=tcp://${haproxy_node_svc}:9090 +HAPROXY_NODE${i}_SVC_PORT_50212_TCP=tcp://${haproxy_node_svc}:50212 +NETWORK_NODE${i}_SVC_SERVICE_PORT_GRPC_TLS=50212 +NETWORK_NODE${i}_SVC_SERVICE_HOST=${network_node_svc} +NETWORK_NODE${i}_SVC_PORT_50111_TCP_ADDR=${network_node_svc} +NETWORK_NODE${i}_SVC_PORT_9090_TCP_ADDR=${network_node_svc} +NETWORK_NODE${i}_SVC_PORT_50211_TCP_ADDR=${network_node_svc} +NETWORK_NODE${i}_SVC_PORT_50212_TCP_ADDR=${network_node_svc} +NETWORK_NODE${i}_SVC_PORT_50111_TCP_PORT=50111 +NETWORK_NODE${i}_SVC_PORT_50211_TCP_PORT=50211 +NETWORK_NODE${i}_SVC_PORT_9090_TCP_PORT=9090 +NETWORK_NODE${i}_SVC_PORT_50111_TCP_PROTO=tcp +NETWORK_NODE${i}_SVC_PORT_9090_TCP_PROTO=tcp +NETWORK_NODE${i}_SVC_PORT_50212_TCP_PORT=50212 +ENVOY_PROXY_NODE${i}_SVC_SERVICE_PORT_PROMETHEUS=9090 +NETWORK_NODE${i}_SVC_PORT_50211_TCP_PROTO=tcp +HAPROXY_NODE${i}_SVC_SERVICE_PORT_PROMETHEUS=9090 +NETWORK_NODE${i}_SVC_PORT=tcp://${network_node_svc}:50111 +NETWORK_NODE${i}_SVC_PORT_50212_TCP_PROTO=tcp +NETWORK_NODE${i}_SVC_SERVICE_PORT=50111 +HAPROXY_NODE${i}_SVC_SERVICE_PORT_TLS_GRPC_CLIENT_PORT=50212 +EOF +done + +#split prio=1 and prio=2 +for i in $(seq 1 1 7) +do + + block_node_ip=`cat get_svc.txt | grep -E "block-node-${i}" | grep -v '\-p2\-' | awk '{print $3}'` + + cat << EOF | awk '{print "export "$0}' >> block_node.env +BLOCK_NODE_${i}_PORT_16007_TCP_ADDR=${block_node_ip} +BLOCK_NODE_${i}_PORT_16007_TCP_PORT=16007 +BLOCK_NODE_${i}_PORT_16007_TCP_PROTO=tcp +BLOCK_NODE_${i}_PORT_16007_TCP=tcp://${block_node_ip}:16007 +BLOCK_NODE_${i}_PORT_40840_TCP_ADDR=${block_node_ip} +BLOCK_NODE_${i}_PORT_40840_TCP_PORT=40840 +BLOCK_NODE_${i}_PORT_40840_TCP_PROTO=tcp +BLOCK_NODE_${i}_PORT_40840_TCP=tcp://${block_node_ip}:40840 +BLOCK_NODE_${i}_PORT=tcp://${block_node_ip}:40840 +BLOCK_NODE_${i}_SERVICE_HOST=${block_node_ip} +BLOCK_NODE_${i}_SERVICE_PORT_HTTP=40840 +BLOCK_NODE_${i}_SERVICE_PORT_METRICS=16007 +BLOCK_NODE_${i}_SERVICE_PORT=40840 +EOF +done + + +for i in $(seq 1 1 7) +do + + block_node_ip=`cat get_svc.txt | grep -E "block-node-${i}" | grep '\-p2\-' | awk '{print $3}'` + + j=`expr $i + 7` + cat << EOF | awk '{print "export "$0}' >> block_node.env +BLOCK_NODE_${j}_PORT_16007_TCP_ADDR=${block_node_ip} +BLOCK_NODE_${j}_PORT_16007_TCP_PORT=16007 +BLOCK_NODE_${j}_PORT_16007_TCP_PROTO=tcp +BLOCK_NODE_${j}_PORT_16007_TCP=tcp://${block_node_ip}:16007 +BLOCK_NODE_${j}_PORT_40840_TCP_ADDR=${block_node_ip} +BLOCK_NODE_${j}_PORT_40840_TCP_PORT=40840 +BLOCK_NODE_${j}_PORT_40840_TCP_PROTO=tcp +BLOCK_NODE_${j}_PORT_40840_TCP=tcp://${block_node_ip}:40840 +BLOCK_NODE_${j}_PORT=tcp://${block_node_ip}:40840 +BLOCK_NODE_${j}_SERVICE_HOST=${block_node_ip} +BLOCK_NODE_${j}_SERVICE_PORT_HTTP=40840 +BLOCK_NODE_${j}_SERVICE_PORT_METRICS=16007 +BLOCK_NODE_${j}_SERVICE_PORT=40840 +EOF +done \ No newline at end of file diff --git a/.github/workflows/support/citr/cronClean.sh b/.github/workflows/support/citr/cronClean.sh index 7eb6188881ad..aff22dd55a08 100644 --- a/.github/workflows/support/citr/cronClean.sh +++ b/.github/workflows/support/citr/cronClean.sh @@ -14,11 +14,12 @@ wait minioIP=`kubectl -n ${NAMESPACE} describe pod minio-pool-1-0 | grep -E '^IP[\:]' | awk '{print $NF}'` nlgpod=`kubectl -n ${NAMESPACE} get pods | grep nlg-network-load-generator | awk '{print $1}'` + kubectl -n ${NAMESPACE} get secret minio-secrets -o yaml | grep 'config.env' | awk '{print $NF}' | base64 --decode >/tmp/.$$.tmp.s username=`grep MINIO_ROOT_USER /tmp/.$$.tmp.s | awk -F = '{print $NF}'` userpwd=`grep MINIO_ROOT_PASSWORD /tmp/.$$.tmp.s | awk -F = '{print $NF}'` -kubectl -n ${NAMESPACE} cp /tmp/mc ${nlgpod}:/app/mc +kubectl -n ${NAMESPACE} exec ${nlgpod} -- bash -c "if [ ! -f ./mc ]; then curl https://dl.min.io/client/mc/release/linux-amd64/mc -o mc; fi" kubectl -n ${NAMESPACE} exec ${nlgpod} -- bash -c "chmod a+rx ./mc" kubectl -n ${NAMESPACE} exec ${nlgpod} -- bash -c "./mc alias set myminio http://${minioIP}:9000 ${username} ${userpwd}" kubectl -n ${NAMESPACE} exec ${nlgpod} -- bash -c "./mc rm --older-than 0d1h0s --recursive --versions --force myminio/solo-streams/ >/dev/null 2>&1" diff --git a/.github/workflows/support/citr/genesisNetworkBNRSARoster.sh b/.github/workflows/support/citr/genesisNetworkBNRSARoster.sh new file mode 100644 index 000000000000..470541740b69 --- /dev/null +++ b/.github/workflows/support/citr/genesisNetworkBNRSARoster.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# +# genesisNetworkBNRSARoster.sh +# +# Convert a hiero-consensus-node genesis-network.json into the +# hiero-block-node rsa-bootstrap-roster.json shape. +# +# Input (CN): nodeMetadata[].rosterEntry.{nodeId, gossipCaCertificate} +# where gossipCaCertificate is a base64-encoded X.509 DER cert. +# +# Output (BN): nodeAddress[].{RSAPubKey, nodeId} +# where RSAPubKey is the lowercase-hex SubjectPublicKeyInfo +# (PKIX) DER bytes of the cert's public key. +# +# The script makes no assumptions about its own location or the current +# working directory. The caller passes the absolute path to the +# genesis-network.json via -i; the output path (optional) goes via -o. + +set -euo pipefail +# Locale-fixed for deterministic byte handling in tr/sort/openssl. +export LC_ALL=C + +usage() { + cat <<'EOF' +Usage: genesisNetworkBNRSARoster.sh -i [-o ] [-h] + +Converts a CN genesis-network.json into a BN rsa-bootstrap-roster.json. + +Options: + -i Absolute path to the genesis-network.json file (required). + -o Path to write rsa-bootstrap-roster.json. Default: stdout. + -h Show this help. + +Required tools: jq, openssl, xxd, base64. +EOF +} + +# --- arg parsing ------------------------------------------------------------ + +INPUT="" +OUTPUT="" + +while getopts ":i:o:h" opt; do + case "$opt" in + i) INPUT="$OPTARG" ;; + o) OUTPUT="$OPTARG" ;; + h) usage; exit 0 ;; + :) echo "Error: -$OPTARG requires an argument" >&2; usage >&2; exit 2 ;; + \?) echo "Error: unknown option -$OPTARG" >&2; usage >&2; exit 2 ;; + esac +done +shift $((OPTIND - 1)) +if [[ $# -gt 0 ]]; then + echo "Error: unexpected positional argument(s): $*" >&2 + usage >&2 + exit 2 +fi + +if [[ -z "$INPUT" ]]; then + echo "Error: -i is required (absolute path to genesis-network.json)" >&2 + usage >&2 + exit 2 +fi + +if [[ "$INPUT" != /* ]]; then + echo "Error: -i must be an absolute path, got: $INPUT" >&2 + exit 2 +fi +if [[ -n "$OUTPUT" && "$OUTPUT" != /* ]]; then + echo "Error: -o must be an absolute path, got: $OUTPUT" >&2 + exit 2 +fi + +# --- pre-flight checks ------------------------------------------------------ + +for bin in jq openssl xxd base64; do + if ! command -v "$bin" >/dev/null 2>&1; then + echo "Error: required tool '$bin' not found in PATH" >&2 + exit 1 + fi +done + +if [[ ! -r "$INPUT" ]]; then + echo "Error: cannot read input file: $INPUT" >&2 + exit 1 +fi + +if ! jq -e . "$INPUT" >/dev/null 2>&1; then + echo "Error: $INPUT is not valid JSON" >&2 + exit 1 +fi + +node_count="$(jq '.nodeMetadata | if type=="array" then length else -1 end' "$INPUT")" +if [[ "$node_count" -lt 0 ]]; then + echo "Error: $INPUT does not contain a 'nodeMetadata' array" >&2 + exit 1 +fi +if [[ "$node_count" -eq 0 ]]; then + echo "Error: 'nodeMetadata' array in $INPUT is empty" >&2 + exit 1 +fi + +# --- output staging --------------------------------------------------------- + +# Stage output to a temp file and atomically move on success so a partial +# failure never leaves a half-written file in place. +tmp_out="" +cleanup() { + if [[ -n "$tmp_out" && -f "$tmp_out" ]]; then rm -f "$tmp_out"; fi + return 0 # ensure EXIT trap never propagates a non-zero status +} +trap cleanup EXIT + +if [[ -n "$OUTPUT" ]]; then + out_dir="$(dirname "$OUTPUT")" + if [[ ! -d "$out_dir" ]]; then + echo "Error: output directory does not exist: $out_dir" >&2 + exit 1 + fi + tmp_out="$(mktemp "${OUTPUT}.XXXXXX")" +fi + +# --- conversion ------------------------------------------------------------- + +# Extract (nodeId, base64 gossipCaCertificate) pairs, TAB-separated. +# `tojson` on the cert is safer than raw selection (no risk of embedded tabs +# from the b64 alphabet — which can't contain tabs — but kept defensive). +pairs="$(jq -r ' + .nodeMetadata[] + | .rosterEntry + | [ (.nodeId|tostring), .gossipCaCertificate ] + | @tsv +' "$INPUT")" + +# RSA encryption OID prefix in DER SubjectPublicKeyInfo. +# SEQUENCE { SEQUENCE { OID 1.2.840.113549.1.1.1, NULL }, BIT STRING { ... } } +# The first ~30 bytes are fixed for an RSA SPKI; the OID itself appears at: +# 06 09 2a 86 48 86 f7 0d 01 01 01 +RSA_OID_HEX="06092a864886f70d010101" + +entries_json="[]" +idx=0 +while IFS=$'\t' read -r node_id cert_b64; do + [[ -z "$node_id" ]] && continue + idx=$((idx + 1)) + + if [[ -z "$cert_b64" || "$cert_b64" == "null" ]]; then + echo "Error: nodeMetadata[$((idx-1))] (nodeId=$node_id) has empty gossipCaCertificate" >&2 + exit 1 + fi + + # base64 cert -> X.509 DER -> PEM SPKI -> DER SPKI -> lowercase hex + if ! pubkey_hex="$( + printf '%s' "$cert_b64" \ + | base64 -d 2>/dev/null \ + | openssl x509 -inform DER -pubkey -noout 2>/dev/null \ + | openssl pkey -pubin -outform DER 2>/dev/null \ + | xxd -p \ + | tr -d '\n' + )"; then + echo "Error: openssl pipeline failed for nodeMetadata[$((idx-1))] (nodeId=$node_id)" >&2 + exit 1 + fi + + if [[ -z "$pubkey_hex" ]]; then + echo "Error: empty public key for nodeMetadata[$((idx-1))] (nodeId=$node_id)" >&2 + exit 1 + fi + + # Sanity check: confirm it's an RSA SPKI by looking for the OID. + if [[ "$pubkey_hex" != *"$RSA_OID_HEX"* ]]; then + echo "Error: nodeId=$node_id public key is not RSA (rsaEncryption OID not found in SPKI)" >&2 + exit 1 + fi + + entries_json="$(jq -c \ + --arg pk "$pubkey_hex" \ + --argjson id "$node_id" \ + '. + [{RSAPubKey: $pk, nodeId: $id}]' \ + <<<"$entries_json")" +done < <(printf '%s\n' "$pairs") + +# Sort by nodeId ascending for stable, input-order-independent output. +result="$(jq -n --argjson entries "$entries_json" \ + '{nodeAddress: ($entries | sort_by(.nodeId))}')" + +# --- write ------------------------------------------------------------------ + +if [[ -n "$OUTPUT" ]]; then + printf '%s\n' "$result" > "$tmp_out" + mv -f "$tmp_out" "$OUTPUT" + tmp_out="" # disarm cleanup; mv already consumed it + echo "Wrote $(jq '.nodeAddress | length' <<<"$result") entries to $OUTPUT" >&2 +else + printf '%s\n' "$result" +fi diff --git a/.github/workflows/support/citr/getBlockNodeErrors.sh b/.github/workflows/support/citr/getBlockNodeErrors.sh new file mode 100644 index 000000000000..58db9d4d9428 --- /dev/null +++ b/.github/workflows/support/citr/getBlockNodeErrors.sh @@ -0,0 +1,25 @@ +NAMESPACE=$1 +BN_LOG=/opt/hiero/block-node/logs +TOOLDIR=`dirname $0` + +if [ ! -d podlog_${NAMESPACE} ] +then + mkdir podlog_${NAMESPACE} +fi + +for i in `sh ${TOOLDIR}/kubectlt -n ${NAMESPACE} get pods | grep 'block-node-' | awk '{print $1}'` +do + sh ${TOOLDIR}/kubectlt -n ${NAMESPACE} exec -it ${i} -- \ + bash -c "grep -h -i -E 'error|exception|warn|severe' $BN_LOG/*.log" | grep -v error_prone_annotations > podlog_${NAMESPACE}/${i}-errors.log +done + +cat podlog_${NAMESPACE}/*block-node*-errors.log | grep -v -E 'exception[\=]null' |\ +perl -ne 'if (/^\d{4}[\-]\d{2}[\-]\d{2}\s+\d{2}[\:]\d{2}[\:]\d{2}[\.]\d+\s+\d+\s+(.*)$/) {print "$1\n";} else {print;}' | perl -pne '~s/\d+/N/g' | sort | uniq -c | sort -n -k 1 -r |\ +grep -v 'error_prone_annotations' > podlog_${NAMESPACE}/error_summary_blocknodes.txt +log_ + +for i in `sh ${TOOLDIR}/kubectlt -n ${NAMESPACE} get pods | grep 'block-node-' | awk '{print $1}'` +do + mkdir podlog_${NAMESPACE}/${i}_logs + sh ${TOOLDIR}/kubectlt -n ${NAMESPACE} cp ${i}:$BN_LOG podlog_${NAMESPACE}/${i}_logs +done diff --git a/.github/workflows/support/citr/getClusterErrors.sh b/.github/workflows/support/citr/getClusterErrors.sh index aade50c0e997..76dd26f06ce5 100644 --- a/.github/workflows/support/citr/getClusterErrors.sh +++ b/.github/workflows/support/citr/getClusterErrors.sh @@ -41,3 +41,5 @@ do sh ${TOOLDIR}/kubectlt -n ${NAMESPACE} cp -c root-container network-node${i}-0:${NODE_ROOT}/settingsUsed.txt podlog_${NAMESPACE}/network-node${i}_logs/config/settingsUsed.txt done + +sh $TOOLDIR/getBlockNodeErrors.sh ${NAMESPACE} \ No newline at end of file diff --git a/.github/workflows/support/citr/getLastBlock.sh b/.github/workflows/support/citr/getLastBlock.sh new file mode 100644 index 000000000000..5717000d21e5 --- /dev/null +++ b/.github/workflows/support/citr/getLastBlock.sh @@ -0,0 +1,3 @@ +f=buffer_latestBlockOpened +p=`head -3000 /opt/hgcapp/services-hedera/HapiApp2.0/data/stats/MainNetStats*.csv | awk "/${f/\//\\\/}:/ {print NR + 1;}"` +awk -F, -v OFS='\t' "{print \$${p}}" /opt/hgcapp/services-hedera/HapiApp2.0/data/stats/MainNetStats*.csv | tail -n 1 \ No newline at end of file diff --git a/.github/workflows/support/citr/init-containers-values7.yaml b/.github/workflows/support/citr/init-containers-values7.yaml index e1e9eedd7705..1b4731bd94fe 100644 --- a/.github/workflows/support/citr/init-containers-values7.yaml +++ b/.github/workflows/support/citr/init-containers-values7.yaml @@ -129,7 +129,7 @@ defaults: memory: 256Gi extraEnv: - name: JAVA_OPTS - value: "-XX:+UnlockExperimentalVMOptions -XX:+UseZGC -XX:ZAllocationSpikeTolerance=2 -XX:ConcGCThreads=14 -XX:ZMarkStackSpaceLimit=16g -XX:MaxDirectMemorySize=64g -XX:MetaspaceSize=100M -XX:+ZGenerational -Xlog:gc*:gc.log --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --enable-native-access=ALL-UNNAMED --sun-misc-unsafe-memory-access=allow -Dio.netty.tryReflectionSetAccessible=true" + value: "-XX:+UnlockExperimentalVMOptions -XX:+UseZGC -XX:ZAllocationSpikeTolerance=2 -XX:ConcGCThreads=14 -XX:ZMarkStackSpaceLimit=16g -XX:MaxDirectMemorySize=64g -XX:MetaspaceSize=100M -XX:+ZGenerational -Xlog:gc*:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp/dump.hprof --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED -Dio.netty.tryReflectionSetAccessible=true -Dnet.sourceforge.cobertura.datafile=/tmp/cobertura.ser" - name: JAVA_HEAP_MIN value: "32g" - name: JAVA_HEAP_MAX diff --git a/.github/workflows/support/citr/kubectlt b/.github/workflows/support/citr/kubectlt index 6bbdd948491e..dfac276e9e5d 100644 --- a/.github/workflows/support/citr/kubectlt +++ b/.github/workflows/support/citr/kubectlt @@ -1,16 +1,29 @@ #/usr/bin/sh -f -KBCTL=kubectl +KBCTL=$(which kubectl.orig) +timeout="10m" +if [ "$KUBECTL_TIMEOUT" != "" ] +then + timeout=$KUBECTL_TIMEOUT +fi -timeout --signal=KILL --preserve-status --foreground 10m $KBCTL "$@" >/tmp/stdout.$$ 2>/tmp/stderr.$$ +timeout --signal=KILL --preserve-status --foreground $timeout $KBCTL "$@" >/tmp/stdout.$$ 2>/tmp/stderr.$$ result=$? if [ $result -ne 137 ] then - cat /tmp/stdout.$$ > /dev/stdout - cat /tmp/stderr.$$ > /dev/stderr - rm -f /tmp/stdout.$$ /tmp/stderr.$$ - exit $result + grep -E 'Internal error occurred|context deadline exceeded' /tmp/stdout.$$ /tmp/stderr.$$ >/dev/null + if [ $? -eq 0 ] + then + sleep 30 + rm -f /tmp/stdout.$$ /tmp/stderr.$$ + timeout --signal=KILL --preserve-status --foreground $timeout $KBCTL "$@" + else + cat /tmp/stdout.$$ > /dev/stdout + cat /tmp/stderr.$$ > /dev/stderr + rm -f /tmp/stdout.$$ /tmp/stderr.$$ + exit $result + fi else - rm -f /tmp/stdout.$$ /tmp/stderr.$$ - timeout --signal=KILL --preserve-status --foreground 10m $KBCTL "$@" + rm -f /tmp/stdout.$$ /tmp/stderr.$$ + timeout --signal=KILL --preserve-status --foreground $timeout $KBCTL "$@" fi diff --git a/.github/workflows/support/citr/local-path-storage.yaml b/.github/workflows/support/citr/local-path-storage.yaml new file mode 100644 index 000000000000..05c4f367dcb6 --- /dev/null +++ b/.github/workflows/support/citr/local-path-storage.yaml @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: Apache-2.0 +apiVersion: v1 +kind: Namespace +metadata: + name: local-path-storage + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-path-provisioner-service-account + namespace: %SOLO_NAMESPACE% + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: local-path-provisioner-role + namespace: %SOLO_NAMESPACE% +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: local-path-provisioner-role +rules: + - apiGroups: [""] + resources: + ["nodes", "persistentvolumeclaims", "configmaps", "pods", "pods/log"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: local-path-provisioner-bind + namespace: %SOLO_NAMESPACE% +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: local-path-provisioner-role +subjects: + - kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: %SOLO_NAMESPACE% + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-path-provisioner-bind +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: local-path-provisioner-role +subjects: + - kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: %SOLO_NAMESPACE% + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: local-path-provisioner + namespace: %SOLO_NAMESPACE% +spec: + replicas: 1 + selector: + matchLabels: + app: local-path-provisioner + template: + metadata: + labels: + app: local-path-provisioner + spec: + serviceAccountName: local-path-provisioner-service-account + containers: + - name: local-path-provisioner + image: rancher/local-path-provisioner:v0.0.34 + imagePullPolicy: IfNotPresent + command: + - local-path-provisioner + - --debug + - start + - --config + - /etc/config/config.json + volumeMounts: + - name: config-volume + mountPath: /etc/config/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONFIG_MOUNT_PATH + value: /etc/config/ + volumes: + - name: config-volume + configMap: + name: local-path-config + +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-path +provisioner: rancher.io/local-path +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete + +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: local-path-config + namespace: %SOLO_NAMESPACE% +data: + config.json: |- + { + "nodePathMap":[ + { + "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", + "paths":["/opt/local-path-provisioner"] + } + ] + } + setup: |- + #!/bin/sh + set -eu + mkdir -m 0777 -p "$VOL_DIR" + teardown: |- + #!/bin/sh + set -eu + rm -rf "$VOL_DIR" + helperPod.yaml: |- + apiVersion: v1 + kind: Pod + metadata: + name: helper-pod + spec: + priorityClassName: system-node-critical + tolerations: + - key: node.kubernetes.io/disk-pressure + operator: Exists + effect: NoSchedule + containers: + - name: helper-pod + image: busybox + imagePullPolicy: IfNotPresent diff --git a/.github/workflows/support/citr/log4j2.xml b/.github/workflows/support/citr/log4j2.xml index f3039275cd21..79917622ce73 100644 --- a/.github/workflows/support/citr/log4j2.xml +++ b/.github/workflows/support/citr/log4j2.xml @@ -77,12 +77,12 @@ + filePattern="output/block-node-comms-%i.log"> - %d{yyyy-MM-dd HH:mm:ss.SSS} %-5p %-4L %c{1} - [%t] %m{nolookups}%n + %d{yyyy-MM-dd HH:mm:ss.SSS} %-5p %-4L %c{1} - [%t] %m{nolookups}%n - + @@ -478,4 +478,3 @@ - diff --git a/.github/workflows/support/citr/logging.properties b/.github/workflows/support/citr/logging.properties new file mode 100644 index 000000000000..1383a0c35b84 --- /dev/null +++ b/.github/workflows/support/citr/logging.properties @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 + +# Log Level Values +# +# SEVERE: indicates a critical error or failure +# WARNING: warns of potential issues or errors +# INFO: reports normal operational information +# CONFIG: provides configuration-related information +# FINE: provides detailed debugging information +# FINER: provides finer-grained debugging information +# FINEST: provides the most detailed debugging information +.level=INFO +org.hiero.block.level=FINEST +io.helidon.level=INFO +io.helidon.webserver.level=INFO +io.helidon.config.level=SEVERE +io.helidon.security.level=INFO +io.helidon.common.level=INFO +com.sun.net.httpserver.level = WARNING +com.sun.net.httpserver.ServerImpl.level = WARNING +com.sun.net.httpserver.ExchangeImpl.level = WARNING + +################################################################################ +# Handlers configuration +################################################################################ +handlers = java.util.logging.ConsoleHandler, java.util.logging.FileHandler + +################################################################################ +# ConsoleHandler configuration +################################################################################ +java.util.logging.ConsoleHandler.level = ALL +java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter + +################################################################################ +# FileHandler properties +################################################################################ +java.util.logging.FileHandler.pattern = /opt/hiero/block-node/logs/blocknode-%g.log +java.util.logging.FileHandler.append = true +java.util.logging.FileHandler.level = ALL +java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter + +################################################################################ +# SimpleFormatter single-line format configuration +################################################################################ +java.util.logging.SimpleFormatter.format = %TF %