Create handler.py

handler.py

@@ -0,0 +1,38 @@
+import json
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from typing import Dict, List, Any
+
+# Replace with actual GraniteMoeForCausalLM import if available
+# from granitemoe import GraniteMoeForCausalLM
+
+class EndpointHandler:
+    def __init__(self, path: str = ""):
+        self.tokenizer = AutoTokenizer.from_pretrained(path)
+        self.model = AutoModelForCausalLM.from_pretrained(
+            path,
+            torch_dtype=torch.bfloat16,
+            device_map="auto"
+        )
+        self.model.eval()
+
+    def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        inputs = data.get("inputs", "")
+        parameters = data.get("parameters", {})
+        input_ids = self.tokenizer(inputs, return_tensors="pt").input_ids.to(self.model.device)
+        max_length = parameters.get("max_length", 100)
+        temperature = parameters.get("temperature", 1.0)
+        top_p = parameters.get("top_p", 1.0)
+        do_sample = parameters.get("do_sample", True)
+        with torch.no_grad():
+            outputs = self.model.generate(
+                input_ids,
+                max_length=max_length,
+                temperature=temperature,
+                top_p=top_p,
+                do_sample=do_sample,
+                pad_token_id=self.tokenizer.pad_token_id,
+                eos_token_id=self.tokenizer.eos_token_id
+            )
+        generated_text = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+        return {"generated_text": generated_text}