patch_vllm.py

from __future__ import annotations

import argparse
import importlib
import py_compile
import shutil
import sys
from pathlib import Path

import vllm


BACKUP_SUFFIX = ".qmask_w2a16.bak"


class PatchError(RuntimeError):
    pass


def replace_once(text: str, old: str, new: str, name: str) -> tuple[str, bool]:
    if new in text:
        print(f"  [already patched] {name}")
        return text, False

    if old not in text:
        raise PatchError(f"could not find expected source for patch: {name}")

    print(f"  [patch] {name}")
    return text.replace(old, new, 1), True


def backup_file(path: Path) -> None:
    backup = Path(str(path) + BACKUP_SUFFIX)

    if backup.exists():
        print(f"  [backup exists] {backup}")
        return

    shutil.copy2(path, backup)
    print(f"  [backup] {backup}")


def restore_file(path: Path) -> None:
    backup = Path(str(path) + BACKUP_SUFFIX)

    if not backup.exists():
        print(f"  [no backup] {path}")
        return

    shutil.copy2(backup, path)
    print(f"  [restored] {path}")


def patch_compressed_tensors_moe(path: Path) -> None:
    print(f"\nPatching {path}")

    text = path.read_text()
    original = text

    old = """from vllm.model_executor.layers.quantization.utils.quant_utils import (
    QuantKey,
    kInt4Static32GroupScale,
    kInt4StaticGroupScale,
    kInt8StaticGroupScale,
)
"""

    new = """from vllm.model_executor.layers.quantization.utils.quant_utils import (
    GroupShape,
    QuantKey,
    ScaleDesc,
    kInt4Static32GroupScale,
    kInt4StaticGroupScale,
    kInt8StaticGroupScale,
)
"""

    text, _ = replace_once(text, old, new, "import GroupShape and ScaleDesc")

    old = """        if self.num_bits == 4:
            if self.group_size == 32:
                scale = kInt4Static32GroupScale
            else:
                scale = kInt4StaticGroupScale
        elif self.num_bits == 8:
            assert self.group_size == -1
            scale = kInt8StaticGroupScale
        else:
            raise ValueError(
                "CompressedTensorsWNA16MoEMethod only supports int4 and int8 now."
            )
"""

    new = """        if self.num_bits == 2:
            if not self.symmetric:
                raise ValueError("W2A16 Humming MoE requires symmetric weights.")

            if self.strategy != QuantizationStrategy.GROUP:
                raise ValueError("W2A16 Humming MoE currently requires group quantization.")

            if self.group_size is None or self.group_size <= 0:
                raise ValueError("W2A16 Humming MoE requires a positive group_size.")

            scale = ScaleDesc(torch.float16, True, GroupShape(1, self.group_size))

        elif self.num_bits == 4:
            if self.group_size == 32:
                scale = kInt4Static32GroupScale
            else:
                scale = kInt4StaticGroupScale

        elif self.num_bits == 8:
            assert self.group_size == -1
            scale = kInt8StaticGroupScale

        else:
            raise ValueError(
                f"CompressedTensorsWNA16MoEMethod currently supports int2, int4 and int8; got int{self.num_bits}."
            )
"""

    text, _ = replace_once(text, old, new, "enable W2A16 QuantKey")

    old = "        self.is_transposed = self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM\n"

    new = (
        "        self.is_transposed = self.wna16_backend not in "
        "(WNA16MoEBackend.FLASHINFER_TRTLLM, WNA16MoEBackend.HUMMING)\n"
    )

    text, _ = replace_once(text, old, new, "use N-first layout for Humming")

    old = """        self.moe_kernel = make_wna16_moe_kernel(
            moe_quant_config=self.moe_quant_config,
            moe_config=self.moe,
            experts_cls=self.experts_cls,
            routing_tables=layer._expert_routing_tables(),
            **marlin_args,
        )
"""

    new = """        self.moe_kernel = make_wna16_moe_kernel(
            moe_quant_config=self.moe_quant_config,
            moe_config=self.moe,
            experts_cls=self.experts_cls,
            backend=self.wna16_backend,
            layer=layer,
            routing_tables=layer._expert_routing_tables(),
            **marlin_args,
        )
"""

    text, _ = replace_once(text, old, new, "pass Humming backend and layer to kernel factory")

    old = """    def get_fused_moe_quant_config(
        self, layer: torch.nn.Module
    ) -> FusedMoEQuantConfig | None:
        return make_wna16_moe_quant_config(
            w1_scale=layer.w13_weight_scale,
            w2_scale=layer.w2_weight_scale,
            group_size=self.group_size,
            num_bits=self.num_bits,
            w1_zp=getattr(layer, "w13_weight_zero_point", None),
            w2_zp=getattr(layer, "w2_weight_zero_point", None),
            gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
            gemm1_alpha=getattr(layer, "swiglu_alpha", None),
            gemm1_beta=getattr(layer, "swiglu_beta", None),
        )
"""

    new = """    def get_fused_moe_quant_config(
        self, layer: torch.nn.Module
    ) -> FusedMoEQuantConfig | None:
        if self.wna16_backend == WNA16MoEBackend.HUMMING:
            from vllm.model_executor.layers.quantization.utils.humming_utils import get_humming_moe_quant_config

            return get_humming_moe_quant_config(
                layer,
                gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
                gemm1_alpha=getattr(layer, "swiglu_alpha", None),
                gemm1_beta=getattr(layer, "swiglu_beta", None),
            )

        return make_wna16_moe_quant_config(
            w1_scale=layer.w13_weight_scale,
            w2_scale=layer.w2_weight_scale,
            group_size=self.group_size,
            num_bits=self.num_bits,
            w1_zp=getattr(layer, "w13_weight_zero_point", None),
            w2_zp=getattr(layer, "w2_weight_zero_point", None),
            gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
            gemm1_alpha=getattr(layer, "swiglu_alpha", None),
            gemm1_beta=getattr(layer, "swiglu_beta", None),
        )
"""

    text, _ = replace_once(text, old, new, "build native Humming MoE quant config")

    if text == original:
        print("  no changes needed")
        return

    backup_file(path)
    path.write_text(text)


def patch_int_wna16(path: Path) -> None:
    print(f"\nPatching {path}")

    text = path.read_text()
    original = text

    old = """    from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
    if isinstance(quant_config, AutoAWQConfig):
"""

    new = """    from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig

    if isinstance(quant_config, QuantizationArgs):
        strategy = getattr(quant_config.strategy, "value", quant_config.strategy)
        qtype = getattr(quant_config.type, "value", quant_config.type)

        return {
            "quant_method": "compressed-tensors",
            "format": "pack-quantized",
            "num_bits": quant_config.num_bits,
            "group_size": quant_config.group_size,
            "strategy": strategy,
            "symmetric": quant_config.symmetric,
            "type": qtype,
        }

    if isinstance(quant_config, AutoAWQConfig):
"""

    text, _ = replace_once(text, old, new, "support compressed-tensors QuantizationArgs in Humming adapter")

    old = """    raise TypeError(
        "Humming WNA16 checkpoint schema requires AutoAWQConfig or "
        "AutoGPTQConfig, "
        f"got {type(quant_config).__name__}."
    )
"""

    new = """    raise TypeError(
        "Humming WNA16 checkpoint schema requires QuantizationArgs, "
        "AutoAWQConfig or AutoGPTQConfig, "
        f"got {type(quant_config).__name__}."
    )
"""

    text, _ = replace_once(text, old, new, "update Humming adapter error message")

    if text == original:
        print("  no changes needed")
        return

    backup_file(path)
    path.write_text(text)


def patch_fused_humming_moe(path: Path) -> None:
    print(f"\nPatching {path}")

    text = path.read_text()
    original = text

    old = """from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
"""

    new = """from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from vllm.utils.import_utils import has_humming
"""

    text, _ = replace_once(text, old, new, "import scalar_types")

    old = """    ) -> bool:
        SUPPORTED_W_A = [
"""

    new = """    ) -> bool:
        if weight_key is not None and activation_key is None:
            scale = weight_key.scale

            is_w2a16 = (
                weight_key.dtype == scalar_types.uint2b2
                and weight_key.symmetric
                and scale.static
                and scale.group_shape.row == 1
                and scale.group_shape.col > 0
            )

            if is_w2a16:
                return True

        SUPPORTED_W_A = [
"""

    text, _ = replace_once(text, old, new, "allow symmetric grouped W2A16 Humming MoE")

    if text == original:
        print("  no changes needed")
        return

    backup_file(path)
    path.write_text(text)


def find_vllm_root() -> Path:
    path = Path(vllm.__file__).resolve().parent

    if not path.exists():
        raise PatchError(f"could not find vLLM package directory: {path}")

    return path


def get_target_files(root: Path) -> list[Path]:
    return [
        root / "model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py",
        root / "model_executor/layers/fused_moe/oracle/int_wna16.py",
        root / "model_executor/layers/fused_moe/experts/fused_humming_moe.py",
    ]


def compile_files(files: list[Path]) -> None:
    print("\nSyntax checking patched files...")

    for path in files:
        py_compile.compile(str(path), doraise=True)
        print(f"  [OK] {path}")


def restore(files: list[Path]) -> None:
    print("Restoring backups...")

    for path in files:
        restore_file(path)

    importlib.invalidate_caches()


def parse_args():
    parser = argparse.ArgumentParser(description="Patch vLLM 0.27.x for CT W2A16 Humming MoE.")
    parser.add_argument("--restore", action="store_true", help="Restore .qmask_w2a16.bak files.")
    parser.add_argument("--force", action="store_true", help="Allow patching a vLLM version other than 0.27.x.")
    return parser.parse_args()


def main():
    args = parse_args()

    version = getattr(vllm, "__version__", "unknown")
    root = find_vllm_root()
    files = get_target_files(root)

    print(f"vLLM version: {version}")
    print(f"vLLM directory: {root}")

    for path in files:
        if not path.exists():
            raise PatchError(f"required file does not exist: {path}")

    if args.restore:
        restore(files)
        print("\nRestored.")
        return

    if not str(version).startswith("0.27.") and not args.force:
        raise PatchError(
            f"this patch was written for vLLM 0.27.x, but found {version}. "
            "Use --force only if you checked the source layout."
        )

    patch_compressed_tensors_moe(files[0])
    patch_int_wna16(files[1])
    patch_fused_humming_moe(files[2])

    compile_files(files)
    importlib.invalidate_caches()

    print("\nPatch complete.")
    print("Restart all running vLLM processes before testing.")
    print("For the first test, use --moe-backend humming.")
    print(f"Backups use suffix: {BACKUP_SUFFIX}")


if __name__ == "__main__":
    try:
        main()
    except PatchError as e:
        print(f"\nERROR: {e}", file=sys.stderr)
        sys.exit(1)