V3 Performance Signal Detected by TorchBench Userbenchmark "torch-nightly" on '2.5.0.dev20240619+cu124'

[xuzhao9]

TorchBench CI has detected a performance signal or runtime regression, and bisected its result.

Control PyTorch commit: f9dae86222aaf15ea085c7774da70781bae46ff9
Control PyTorch version: 2.5.0.dev20240617+cu124

Treatment PyTorch commit: 99f042d336b53844b509406f1ecf78cb6f5e5714
Treatment PyTorch version: 2.5.0.dev20240619+cu124

Bisection result:

[
    {
        "commit1": "0f81473d7b",
        "commit1_time": "2024-06-17 13:41:15 +0000",
        "commit1_digest": {
            "name": "torch-nightly",
            "environ": {
                "pytorch_git_version": "0f81473d7b4a1bf09246410712df22541be7caf3",
                "pytorch_version": "2.5.0a0+git0f81473",
                "device": "NVIDIA A100-SXM4-40GB",
                "git_commit_hash": "0f81473d7b4a1bf09246410712df22541be7caf3"
            },
            "metrics": {
                "test_eval[doctr_det_predictor-cuda-eager]_latency": 58.109275,
                "test_eval[doctr_det_predictor-cuda-eager]_cmem": 0.998046875,
                "test_eval[doctr_det_predictor-cuda-eager]_gmem": 3.49334716796875
            }
        },
        "commit2": "e3093849e5",
        "commit2_time": "2024-06-17 14:55:32 +0000",
        "commit2_digest": {
            "name": "torch-nightly",
            "environ": {
                "pytorch_git_version": "e3093849e5530dbb93a35462d3fd248a2dd7efe0",
                "pytorch_version": "2.5.0a0+gite309384",
                "device": "NVIDIA A100-SXM4-40GB",
                "git_commit_hash": "e3093849e5530dbb93a35462d3fd248a2dd7efe0"
            },
            "metrics": {
                "test_eval[doctr_det_predictor-cuda-eager]_latency": 50.176224,
                "test_eval[doctr_det_predictor-cuda-eager]_cmem": 0.966796875,
                "test_eval[doctr_det_predictor-cuda-eager]_gmem": 3.49334716796875
            }
        }
    },
    {
        "commit1": "2a41fc0390",
        "commit1_time": "2024-06-17 15:25:09 +0000",
        "commit1_digest": {
            "name": "torch-nightly",
            "environ": {
                "pytorch_git_version": "2a41fc03903de63270d325bd1886a50faf32d7e4",
                "pytorch_version": "2.5.0a0+git2a41fc0",
                "device": "NVIDIA A100-SXM4-40GB",
                "git_commit_hash": "2a41fc03903de63270d325bd1886a50faf32d7e4"
            },
            "metrics": {
                "test_eval[doctr_det_predictor-cuda-eager]_latency": 51.359641,
                "test_eval[doctr_det_predictor-cuda-eager]_cmem": 0.9755859375,
                "test_eval[doctr_det_predictor-cuda-eager]_gmem": 3.49334716796875
            }
        },
        "commit2": "bfad0aee44",
        "commit2_time": "2024-06-17 16:26:08 +0000",
        "commit2_digest": {
            "name": "torch-nightly",
            "environ": {
                "pytorch_git_version": "bfad0aee446b710c70fddc31fca34c8d4dda1bec",
                "pytorch_version": "2.5.0a0+gitbfad0ae",
                "device": "NVIDIA A100-SXM4-40GB",
                "git_commit_hash": "bfad0aee446b710c70fddc31fca34c8d4dda1bec"
            },
            "metrics": {
                "test_eval[doctr_det_predictor-cuda-eager]_latency": 60.014735,
                "test_eval[doctr_det_predictor-cuda-eager]_cmem": 0.9912109375,
                "test_eval[doctr_det_predictor-cuda-eager]_gmem": 3.49334716796875
            }
        }
    },
    {
        "commit1": "316b729677",
        "commit1_time": "2024-06-17 16:42:43 +0000",
        "commit1_digest": {
            "name": "torch-nightly",
            "environ": {
                "pytorch_git_version": "316b7296771c87637231f9d90e2658aa1d629859",
                "pytorch_version": "2.5.0a0+git316b729",
                "device": "NVIDIA A100-SXM4-40GB",
                "git_commit_hash": "316b7296771c87637231f9d90e2658aa1d629859"
            },
            "metrics": {
                "test_eval[doctr_det_predictor-cuda-eager]_latency": 52.518858,
                "test_eval[doctr_det_predictor-cuda-eager]_cmem": 1.0302734375,
                "test_eval[doctr_det_predictor-cuda-eager]_gmem": 3.49334716796875
            }
        },
        "commit2": "73b78d1cbe",
        "commit2_time": "2024-06-17 16:44:17 +0000",
        "commit2_digest": {
            "name": "torch-nightly",
            "environ": {
                "pytorch_git_version": "73b78d1cbefda55ec1723904b68660530d7d1495",
                "pytorch_version": "2.5.0a0+git73b78d1",
                "device": "NVIDIA A100-SXM4-40GB",
                "git_commit_hash": "73b78d1cbefda55ec1723904b68660530d7d1495"
            },
            "metrics": {
                "test_eval[doctr_det_predictor-cuda-eager]_latency": 58.545019,
                "test_eval[doctr_det_predictor-cuda-eager]_cmem": 0.99609375,
                "test_eval[doctr_det_predictor-cuda-eager]_gmem": 3.49334716796875
            }
        }
    },
    {
        "commit1": "5344c41d43",
        "commit1_time": "2024-06-17 18:41:42 +0000",
        "commit1_digest": {
            "name": "torch-nightly",
            "environ": {
                "pytorch_git_version": "5344c41d431042dfb9f4a8cfb23b84cfa1352569",
                "pytorch_version": "2.5.0a0+git5344c41",
                "device": "NVIDIA A100-SXM4-40GB",
                "git_commit_hash": "5344c41d431042dfb9f4a8cfb23b84cfa1352569"
            },
            "metrics": {
                "test_eval[doctr_det_predictor-cuda-eager]_latency": 57.697015,
                "test_eval[doctr_det_predictor-cuda-eager]_cmem": 1.0390625,
                "test_eval[doctr_det_predictor-cuda-eager]_gmem": 3.49334716796875
            }
        },
        "commit2": "c172b58fe0",
        "commit2_time": "2024-06-17 18:49:15 +0000",
        "commit2_digest": {
            "name": "torch-nightly",
            "environ": {
                "pytorch_git_version": "c172b58fe01a48be758c2054d27848c3a405f54a",
                "pytorch_version": "2.5.0a0+gitc172b58",
                "device": "NVIDIA A100-SXM4-40GB",
                "git_commit_hash": "c172b58fe01a48be758c2054d27848c3a405f54a"
            },
            "metrics": {
                "test_eval[doctr_det_predictor-cuda-eager]_latency": 48.396128,
                "test_eval[doctr_det_predictor-cuda-eager]_cmem": 0.9755859375,
                "test_eval[doctr_det_predictor-cuda-eager]_gmem": 3.49334716796875
            }
        }
    },
    {
        "commit1": "dff6342a0b",
        "commit1_time": "2024-06-17 16:29:22 +0000",
        "commit1_digest": {
            "name": "torch-nightly",
            "environ": {
                "pytorch_git_version": "dff6342a0b6c70f343fdc894928d10c73dd05ae5",
                "pytorch_version": "2.5.0a0+gitdff6342",
                "device": "NVIDIA A100-SXM4-40GB",
                "git_commit_hash": "dff6342a0b6c70f343fdc894928d10c73dd05ae5"
            },
            "metrics": {
                "test_eval[doctr_det_predictor-cuda-eager]_latency": 58.352773,
                "test_eval[doctr_det_predictor-cuda-eager]_cmem": 1.0078125,
                "test_eval[doctr_det_predictor-cuda-eager]_gmem": 3.49334716796875
            }
        },
        "commit2": "95ac2d6482",
        "commit2_time": "2024-06-17 16:29:25 +0000",
        "commit2_digest": {
            "name": "torch-nightly",
            "environ": {
                "pytorch_git_version": "95ac2d648279ebc73feccf6d8eccafa4b2759de8",
                "pytorch_version": "2.5.0a0+git95ac2d6",
                "device": "NVIDIA A100-SXM4-40GB",
                "git_commit_hash": "95ac2d648279ebc73feccf6d8eccafa4b2759de8"
            },
            "metrics": {
                "test_eval[doctr_det_predictor-cuda-eager]_latency": 50.290212,
                "test_eval[doctr_det_predictor-cuda-eager]_cmem": 1.0546875,
                "test_eval[doctr_det_predictor-cuda-eager]_gmem": 3.49334716796875
            }
        }
    }
]

cc @xuzhao9