diff --git a/.github/actions/action.yml b/.github/actions/action.yml index 11cc3633808..0cc7c27a9ab 100644 --- a/.github/actions/action.yml +++ b/.github/actions/action.yml @@ -78,8 +78,11 @@ runs: - name: Setup python and install dependencies shell: bash -x -e -u -o pipefail {0} run: | - apt-get update - apt-get install -y python3.12 python3.12-venv uuid-runtime + for i in 1 2 3; do + apt-get update && apt-get install -y python3.12 python3.12-venv uuid-runtime && break + echo "apt attempt $i failed, retrying..." + sleep 10 + done curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin sh - name: Create run-script (unit test) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index a3cfb5f848a..35eb570296d 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -488,8 +488,11 @@ jobs: - name: Install GH CLI shell: bash -x -e -u -o pipefail {0} run: | - apt-get update - apt-get install -y gh + for i in 1 2 3; do + apt-get update && apt-get install -y gh && break + echo "apt attempt $i failed, retrying..." + sleep 10 + done - name: Download test data shell: bash @@ -502,8 +505,11 @@ jobs: - name: Install GH CLI shell: bash run: | - apt-get update - apt-get install -y gh + for i in 1 2 3; do + apt-get update && apt-get install -y gh && break + echo "apt attempt $i failed, retrying..." + sleep 10 + done - name: Get last merged PR id: cache_from diff --git a/pyproject.toml b/pyproject.toml index 2acca8b123e..a3dbde52456 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -198,7 +198,7 @@ override-dependencies = [ flash_mla = [ { git = "https://github.com/deepseek-ai/FlashMLA", rev = "9edee0c022cd0938148a18e334203b0aab43aa19" }, ] -transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", rev = "5671fd3675906cda1ade26c24a65d3dedd88eb89" } +transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", rev = "f031cf87bd054c7558b887df7bed93975456667f" } nemo-run = { git = "https://github.com/NVIDIA-NeMo/Run.git", rev = "17ae86b64d7f75653351664f5d8c9e466faede00" } emerging_optimizers = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git", rev = "v0.2.0" } fast-hadamard-transform = { git = "https://github.com/Dao-AILab/fast-hadamard-transform.git", rev = "f134af63deb2df17e1171a9ec1ea4a7d8604d5ca" } diff --git a/tests/test_utils/recipes/gb200/moe.yaml b/tests/test_utils/recipes/gb200/moe.yaml index eafcb874f4c..5a2ce4e06ec 100644 --- a/tests/test_utils/recipes/gb200/moe.yaml +++ b/tests/test_utils/recipes/gb200/moe.yaml @@ -140,12 +140,12 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM] products: - environment: [dev] - scope: [nightly] + scope: [nightly-broken] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4] products: - environment: [dev] - scope: [nightly] + scope: [nightly-broken] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router] products: @@ -165,7 +165,7 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective] products: - environment: [dev] - scope: [nightly] + scope: [nightly-broken] platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer] products: @@ -175,7 +175,7 @@ products: - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph] products: - environment: [dev] - scope: [nightly] + scope: [nightly-broken] # KNOWN_ISSUE: https://github.com/NVIDIA/Megatron-LM/issues/4353 platforms: [dgx_gb200] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon] products: @@ -211,7 +211,7 @@ products: - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [mr] + scope: [mr-broken] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed] products: diff --git a/tests/test_utils/recipes/h100/moe.yaml b/tests/test_utils/recipes/h100/moe.yaml index fc7ad97e983..6308e45f749 100644 --- a/tests/test_utils/recipes/h100/moe.yaml +++ b/tests/test_utils/recipes/h100/moe.yaml @@ -149,7 +149,7 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM] products: - environment: [dev] - scope: [mr, mr-github] + scope: [mr-broken, mr-github-broken] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4] products: @@ -161,7 +161,7 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4] products: - environment: [dev] - scope: [mr, mr-github] + scope: [mr-broken, mr-github-broken] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router] products: @@ -181,7 +181,7 @@ products: - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective] products: - environment: [dev] - scope: [mr, mr-github] + scope: [mr-broken, mr-github-broken] platforms: [dgx_h100] - environment: [lts] scope: [nightly] @@ -193,7 +193,7 @@ products: - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph] products: - environment: [dev] - scope: [mr, mr-github] + scope: [mr-broken, mr-github-broken] # KNOWN_ISSUE: https://github.com/NVIDIA/Megatron-LM/issues/4353 platforms: [dgx_h100] - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon] products: @@ -234,7 +234,7 @@ products: - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [mr, mr-github, mr-github-slim] + scope: [mr-broken, mr-github-broken, mr-github-slim-broken] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed] products: diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py index 315302e66e5..8b492dc23fe 100644 --- a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -771,6 +771,7 @@ def _run_iters_with_cuda_graph( @pytest.mark.skipif( not is_te_min_version("2.14.0"), reason="CUDA Graph with TE RNG tracker requires TE >= 2.13.0" ) +@pytest.mark.flaky_in_dev @pytest.mark.parametrize( "is_mla, offload_modules, cuda_graph_scope, activation_offload_fraction, delay_offload", [ @@ -962,6 +963,7 @@ def test_fine_grained_activation_offloading_with_cuda_graph( Utils.destroy_model_parallel() +@pytest.mark.flaky_in_dev @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for offloading tests.") @pytest.mark.skipif( not is_te_min_version("2.14.0"), reason="CUDA Graph with TE RNG tracker requires TE >= 2.13.0" diff --git a/tests/unit_tests/transformer/moe/test_paged_stashing.py b/tests/unit_tests/transformer/moe/test_paged_stashing.py index 8f4c54fcc0f..cdde5f7553f 100644 --- a/tests/unit_tests/transformer/moe/test_paged_stashing.py +++ b/tests/unit_tests/transformer/moe/test_paged_stashing.py @@ -214,6 +214,7 @@ def teardown_method(self, method): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") @pytest.mark.internal + @pytest.mark.flaky_in_dev def test_forward_backward_4_layers(self): """Test paged stashing with 4 MoE layers: ref run vs paged run match.""" if not is_hybrid_ep_available(): @@ -303,6 +304,7 @@ def teardown_method(self, method): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") @pytest.mark.internal + @pytest.mark.flaky_in_dev def test_overload_factor_and_over_budget(self): """Budget matches HybridEP setup_metadata; over_budget matches map-derived load.""" if not is_hybrid_ep_available(): diff --git a/tests/unit_tests/transformer/test_vision_cuda_graphs.py b/tests/unit_tests/transformer/test_vision_cuda_graphs.py index bfd431e67a3..a7c6cf8de65 100644 --- a/tests/unit_tests/transformer/test_vision_cuda_graphs.py +++ b/tests/unit_tests/transformer/test_vision_cuda_graphs.py @@ -403,6 +403,7 @@ def test_create_cudagraphs_multi_microbatch(self): helper.delete_cuda_graphs() + @pytest.mark.flaky_in_dev def test_create_cudagraphs_no_callables_is_noop(self): """create_cudagraphs on empty helper should not crash.""" dummy_model = torch.nn.Linear(4, 4) @@ -610,6 +611,8 @@ def test_pp2_create_cudagraphs_first_stage(self): for layer in helper.callables: assert layer.cuda_graphs == [] + @pytest.mark.flaky + @pytest.mark.flaky_in_dev @pytest.mark.skipif( not (HAVE_TE_GRAPHS and is_te_min_version("2.7.0")), reason="TE CUDA graph capture requires TransformerEngine >= 2.7.0", diff --git a/uv.lock b/uv.lock index 717477b54ae..69da4be0606 100644 --- a/uv.lock +++ b/uv.lock @@ -3345,7 +3345,7 @@ requires-dist = [ { name = "torch", specifier = ">=2.6.0" }, { name = "tqdm", marker = "extra == 'dev'" }, { name = "tqdm", marker = "extra == 'lts'" }, - { name = "transformer-engine", extras = ["core-cu13", "pytorch"], marker = "extra == 'dev'", git = "https://github.com/NVIDIA/TransformerEngine.git?rev=5671fd3675906cda1ade26c24a65d3dedd88eb89" }, + { name = "transformer-engine", extras = ["core-cu13", "pytorch"], marker = "extra == 'dev'", git = "https://github.com/NVIDIA/TransformerEngine.git?rev=f031cf87bd054c7558b887df7bed93975456667f" }, { name = "transformers", marker = "extra == 'mlm'" }, { name = "transformers", marker = "extra == 'training'" }, { name = "wandb", marker = "extra == 'mlm'" }, @@ -8357,8 +8357,8 @@ wheels = [ [[package]] name = "transformer-engine" -version = "2.12.0+5671fd36" -source = { git = "https://github.com/NVIDIA/TransformerEngine.git?rev=5671fd3675906cda1ade26c24a65d3dedd88eb89#5671fd3675906cda1ade26c24a65d3dedd88eb89" } +version = "2.14.0+f031cf87" +source = { git = "https://github.com/NVIDIA/TransformerEngine.git?rev=f031cf87bd054c7558b887df7bed93975456667f#f031cf87bd054c7558b887df7bed93975456667f" } dependencies = [ { name = "einops" }, { name = "importlib-metadata" },