Upload folder using huggingface_hub

examples/basic_inference.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+"""
+Basic inference example for Isaac Sim Robotics Qwen model.
+
+This script demonstrates how to load and use the fine-tuned model
+for Isaac Sim robotics queries.
+"""
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import argparse
+import sys
+import os
+
+def load_model(model_path, device="auto", load_in_8bit=False):
+    """
+    Load the Isaac Sim Robotics Qwen model.
+    
+    Args:
+        model_path (str): Path to the model (local or HuggingFace hub)
+        device (str): Device to load model on ("auto", "cpu", "cuda")
+        load_in_8bit (bool): Whether to use 8-bit quantization
+    
+    Returns:
+        tuple: (model, tokenizer)
+    """
+    print(f"Loading model from: {model_path}")
+    
+    # Load tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(model_path)
+    
+    # Set pad token if not present
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+    
+    # Load model
+    if load_in_8bit:
+        try:
+            model = AutoModelForCausalLM.from_pretrained(
+                model_path,
+                load_in_8bit=True,
+                device_map=device,
+                torch_dtype=torch.float16
+            )
+        except ImportError:
+            print("8-bit quantization not available. Install bitsandbytes.")
+            model = AutoModelForCausalLM.from_pretrained(
+                model_path,
+                device_map=device,
+                torch_dtype=torch.float16
+            )
+    else:
+        model = AutoModelForCausalLM.from_pretrained(
+            model_path,
+            device_map=device,
+            torch_dtype=torch.float16
+        )
+    
+    print("Model loaded successfully!")
+    return model, tokenizer
+
+def generate_response(model, tokenizer, query, max_length=1024, temperature=0.7):
+    """
+    Generate a response using the model.
+    
+    Args:
+        model: The loaded model
+        tokenizer: The loaded tokenizer
+        query (str): The input query
+        max_length (int): Maximum length of generated response
+        temperature (float): Sampling temperature
+    
+    Returns:
+        str: Generated response
+    """
+    # Format query for Qwen2.5-Coder
+    formatted_query = f"<|im_start|>user\n{query}<|im_end|>\n<|im_start|>assistant"
+    
+    # Tokenize input
+    inputs = tokenizer(formatted_query, return_tensors="pt")
+    
+    # Move to same device as model
+    device = next(model.parameters()).device
+    inputs = {k: v.to(device) for k, v in inputs.items()}
+    
+    # Generate response
+    with torch.no_grad():
+        outputs = model.generate(
+            **inputs,
+            max_length=max_length,
+            temperature=temperature,
+            do_sample=True,
+            pad_token_id=tokenizer.eos_token_id,
+            eos_token_id=tokenizer.eos_token_id
+        )
+    
+    # Decode response
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    
+    # Extract only the assistant response
+    if "<|im_start|>assistant" in response:
+        response = response.split("<|im_start|>assistant")[1].strip()
+    
+    return response
+
+def main():
+    parser = argparse.ArgumentParser(description="Isaac Sim Robotics Qwen Inference")
+    parser.add_argument(
+        "--model_path",
+        type=str,
+        default="TomBombadyl/Qwen2.5-Coder-7B-Instruct-Omni1.1",
+        help="Path to model (local or HuggingFace hub)"
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default="auto",
+        choices=["auto", "cpu", "cuda"],
+        help="Device to use for inference"
+    )
+    parser.add_argument(
+        "--load_8bit",
+        action="store_true",
+        help="Use 8-bit quantization to reduce memory usage"
+    )
+    parser.add_argument(
+        "--max_length",
+        type=int,
+        default=1024,
+        help="Maximum length of generated response"
+    )
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=0.7,
+        help="Sampling temperature"
+    )
+    parser.add_argument(
+        "--query",
+        type=str,
+        help="Query to ask (if not provided, will use interactive mode)"
+    )
+    
+    args = parser.parse_args()
+    
+    try:
+        # Load model
+        model, tokenizer = load_model(
+            args.model_path,
+            device=args.device,
+            load_in_8bit=args.load_8bit
+        )
+        
+        if args.query:
+            # Single query mode
+            response = generate_response(
+                model, tokenizer, args.query, args.max_length, args.temperature
+            )
+            print(f"\nQuery: {args.query}")
+            print(f"Response:\n{response}")
+        else:
+            # Interactive mode
+            print("\n=== Isaac Sim Robotics Qwen Interactive Mode ===")
+            print("Type 'quit' to exit")
+            print("Example queries:")
+            print("- How do I create a differential drive robot in Isaac Sim?")
+            print("- How to add a depth camera to my robot?")
+            print("- What physics parameters should I use for a manipulator?")
+            print()
+            
+            while True:
+                try:
+                    query = input("Enter your Isaac Sim question: ").strip()
+                    if query.lower() in ['quit', 'exit', 'q']:
+                        break
+                    if not query:
+                        continue
+                    
+                    print("Generating response...")
+                    response = generate_response(
+                        model, tokenizer, query, args.max_length, args.temperature
+                    )
+                    print(f"\nResponse:\n{response}\n")
+                    
+                except KeyboardInterrupt:
+                    print("\nExiting...")
+                    break
+                except Exception as e:
+                    print(f"Error generating response: {e}")
+    
+    except Exception as e:
+        print(f"Error loading model: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main() 

examples/ctransformers_usage.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+"""
+CTransformers usage example for Isaac Sim Robotics Qwen model.
+
+This script demonstrates how to use the model with CTransformers
+for lightweight, memory-efficient inference.
+"""
+
+from ctransformers import AutoModelForCausalLM
+import argparse
+import sys
+
+def load_model(model_path, model_type="qwen2", gpu_layers=0):
+    """
+    Load the Isaac Sim Robotics Qwen model using CTransformers.
+    
+    Args:
+        model_path (str): Path to the CTransformers model
+        model_type (str): Model architecture type
+        gpu_layers (int): Number of layers to offload to GPU (0 = CPU only)
+    
+    Returns:
+        AutoModelForCausalLM: Loaded model
+    """
+    print(f"Loading CTransformers model from: {model_path}")
+    print(f"Model type: {model_type}, GPU layers: {gpu_layers}")
+    
+    try:
+        model = AutoModelForCausalLM.from_pretrained(
+            model_path,
+            model_type=model_type,
+            gpu_layers=gpu_layers,
+            lib="avx2"  # Use AVX2 optimizations if available
+        )
+        print("Model loaded successfully!")
+        return model
+    except Exception as e:
+        print(f"Error loading model: {e}")
+        print("Make sure you have the CTransformers model files in the specified directory.")
+        sys.exit(1)
+
+def generate_response(model, query, max_length=1024, temperature=0.7):
+    """
+    Generate a response using the CTransformers model.
+    
+    Args:
+        model: The loaded CTransformers model
+        query (str): The input query
+        max_length (int): Maximum length of generated response
+        temperature (float): Sampling temperature
+    
+    Returns:
+        str: Generated response
+    """
+    # Format query for Qwen2.5-Coder
+    formatted_query = f"<|im_start|>user\n{query}<|im_end|>\n<|im_start|>assistant"
+    
+    # Generate response
+    response = model(
+        formatted_query,
+        max_new_tokens=max_length,
+        temperature=temperature,
+        do_sample=True,
+        stop=["<|im_end|>", "<|im_start|>"]
+    )
+    
+    # Extract only the assistant response
+    if "<|im_start|>assistant" in response:
+        response = response.split("<|im_start|>assistant")[1].strip()
+    
+    return response
+
+def main():
+    parser = argparse.ArgumentParser(description="Isaac Sim Robotics Qwen CTransformers Inference")
+    parser.add_argument(
+        "--model_path",
+        type=str,
+        default="models/ctransformers",
+        help="Path to CTransformers model directory"
+    )
+    parser.add_argument(
+        "--model_type",
+        type=str,
+        default="qwen2",
+        help="Model architecture type"
+    )
+    parser.add_argument(
+        "--gpu_layers",
+        type=int,
+        default=0,
+        help="Number of layers to offload to GPU (0 = CPU only)"
+    )
+    parser.add_argument(
+        "--max_length",
+        type=int,
+        default=1024,
+        help="Maximum length of generated response"
+    )
+    parser.add_argument(
+        "--temperature",
+        type=float,
+        default=0.7,
+        help="Sampling temperature"
+    )
+    parser.add_argument(
+        "--query",
+        type=str,
+        help="Query to ask (if not provided, will use interactive mode)"
+    )
+    
+    args = parser.parse_args()
+    
+    try:
+        # Load model
+        model = load_model(
+            args.model_path,
+            model_type=args.model_type,
+            gpu_layers=args.gpu_layers
+        )
+        
+        if args.query:
+            # Single query mode
+            response = generate_response(
+                model, args.query, args.max_length, args.temperature
+            )
+            print(f"\nQuery: {args.query}")
+            print(f"Response:\n{response}")
+        else:
+            # Interactive mode
+            print("\n=== Isaac Sim Robotics Qwen CTransformers Interactive Mode ===")
+            print("Type 'quit' to exit")
+            print("Example queries:")
+            print("- How do I create a differential drive robot in Isaac Sim?")
+            print("- How to add a depth camera to my robot?")
+            print("- What physics parameters should I use for a manipulator?")
+            print()
+            
+            while True:
+                try:
+                    query = input("Enter your Isaac Sim question: ").strip()
+                    if query.lower() in ['quit', 'exit', 'q']:
+                        break
+                    if not query:
+                        continue
+                    
+                    print("Generating response...")
+                    response = generate_response(
+                        model, query, args.max_length, args.temperature
+                    )
+                    print(f"\nResponse:\n{response}\n")
+                    
+                except KeyboardInterrupt:
+                    print("\nExiting...")
+                    break
+                except Exception as e:
+                    print(f"Error generating response: {e}")
+    
+    except Exception as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main() 

examples/isaac_sim_integration.py

@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+"""
+Isaac Sim integration example for the Isaac Sim Robotics Qwen model.
+
+This script demonstrates how to use the fine-tuned model to generate
+Isaac Sim code and integrate it into simulation workflows.
+"""
+
+import omni.isaac.core.utils.extensions as extensions
+import omni.isaac.core.utils.stage as stage_utils
+import omni.isaac.core.utils.prims as prim_utils
+import omni.isaac.core.utils.nucleus as nucleus_utils
+import omni.isaac.core.utils.semantics as semantics_utils
+import omni.isaac.core.materials as materials
+import omni.isaac.core.articulations as articulations
+import omni.isaac.core.robots as robots
+import omni.isaac.core.objects as objects
+import omni.isaac.core.utils.torch as torch_utils
+import omni.isaac.core.utils.torch.maths as torch_maths
+import omni.isaac.core.utils.torch.rotations as torch_rotations
+import omni.isaac.core.utils.torch.transforms as torch_transforms
+import omni.isaac.core.utils.torch.visualization as torch_visualization
+import omni.isaac.core.utils.torch.control as torch_control
+import omni.isaac.core.utils.torch.planning as torch_planning
+import omni.isaac.core.utils.torch.perception as torch_perception
+import omni.isaac.core.utils.torch.learning as torch_learning
+import omni.isaac.core.utils.torch.optimization as torch_optimization
+import omni.isaac.core.utils.torch.simulation as torch_simulation
+import omni.isaac.core.utils.torch.physics as torch_physics
+import omni.isaac.core.utils.torch.rendering as torch_rendering
+import omni.isaac.core.utils.torch.audio as torch_audio
+import omni.isaac.core.utils.torch.haptics as torch_haptics
+import omni.isaac.core.utils.torch.networking as torch_networking
+import omni.isaac.core.utils.torch.storage as torch_storage
+import omni.isaac.core.utils.torch.training as torch_training
+import omni.isaac.core.utils.torch.evaluation as torch_evaluation
+import omni.isaac.core.utils.torch.visualization as torch_visualization
+import omni.isaac.core.utils.torch.control as torch_control
+import omni.isaac.core.utils.torch.planning as torch_planning
+import omni.isaac.core.utils.torch.perception as torch_perception
+import omni.isaac.core.utils.torch.learning as torch_learning
+import omni.isaac.core.utils.torch.optimization as torch_optimization
+import omni.isaac.core.utils.torch.simulation as torch_simulation
+import omni.isaac.core.utils.torch.physics as torch_physics
+import omni.isaac.core.utils.torch.rendering as torch_rendering
+import omni.isaac.core.utils.torch.audio as torch_audio
+import omni.isaac.core.utils.torch.haptics as torch_haptics
+import omni.isaac.core.utils.torch.networking as torch_networking
+import omni.isaac.core.utils.torch.storage as torch_storage
+import omni.isaac.core.utils.torch.training as torch_training
+import omni.isaac.core.utils.torch.evaluation as torch_evaluation
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+import numpy as np
+import argparse
+import sys
+import os
+
+class IsaacSimRoboticsAssistant:
+    """
+    Assistant class that integrates the fine-tuned model with Isaac Sim workflows.
+    """
+    
+    def __init__(self, model_path, device="auto"):
+        """
+        Initialize the Isaac Sim Robotics Assistant.
+        
+        Args:
+            model_path (str): Path to the fine-tuned model
+            device (str): Device to use for inference
+        """
+        self.model_path = model_path
+        self.device = device
+        self.model = None
+        self.tokenizer = None
+        self.simulation_world = None
+        
+        # Load the model
+        self._load_model()
+        
+        # Initialize Isaac Sim
+        self._initialize_isaac_sim()
+    
+    def _load_model(self):
+        """Load the fine-tuned Isaac Sim robotics model."""
+        print(f"Loading Isaac Sim Robotics model from: {self.model_path}")
+        
+        try:
+            self.tokenizer = AutoTokenizer.from_pretrained(self.model_path)
+            if self.tokenizer.pad_token is None:
+                self.tokenizer.pad_token = self.tokenizer.eos_token
+            
+            self.model = AutoModelForCausalLM.from_pretrained(
+                self.model_path,
+                device_map=self.device,
+                torch_dtype=torch.float16
+            )
+            print("Model loaded successfully!")
+        except Exception as e:
+            print(f"Error loading model: {e}")
+            raise
+    
+    def _initialize_isaac_sim(self):
+        """Initialize Isaac Sim simulation environment."""
+        print("Initializing Isaac Sim...")
+        
+        try:
+            # Enable required extensions
+            extensions.enable_extension("omni.isaac.core")
+            extensions.enable_extension("omni.isaac.sim")
+            extensions.enable_extension("omni.isaac.ui")
+            extensions.enable_extension("omni.isaac.debug_draw")
+            
+            # Initialize stage
+            stage_utils.initialize_stage()
+            
+            print("Isaac Sim initialized successfully!")
+        except Exception as e:
+            print(f"Warning: Isaac Sim initialization failed: {e}")
+            print("Some features may not work without proper Isaac Sim setup.")
+    
+    def generate_robot_code(self, query):
+        """
+        Generate Isaac Sim robot code using the fine-tuned model.
+        
+        Args:
+            query (str): Description of what robot to create
+            
+        Returns:
+            str: Generated Python code for Isaac Sim
+        """
+        # Format query for the model
+        formatted_query = f"""<|im_start|>user
+Create a complete Python script for Isaac Sim 5.0 that: {query}
+
+Please provide the complete, runnable code with proper imports and error handling.
+<|im_end|>
+<|im_start|>assistant"""
+        
+        # Generate response
+        inputs = self.tokenizer(formatted_query, return_tensors="pt")
+        device = next(self.model.parameters()).device
+        inputs = {k: v.to(device) for k, v in inputs.items()}
+        
+        with torch.no_grad():
+            outputs = self.model.generate(
+                **inputs,
+                max_length=2048,
+                temperature=0.7,
+                do_sample=True,
+                pad_token_id=self.tokenizer.eos_token_id,
+                eos_token_id=self.tokenizer.eos_token_id
+            )
+        
+        response = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+        
+        # Extract only the assistant response
+        if "<|im_start|>assistant" in response:
+            response = response.split("<|im_start|>assistant")[1].strip()
+        
+        return response
+    
+    def execute_generated_code(self, code):
+        """
+        Execute the generated Isaac Sim code.
+        
+        Args:
+            code (str): Python code to execute
+            
+        Returns:
+            bool: True if execution successful, False otherwise
+        """
+        try:
+            print("Executing generated code...")
+            print("=" * 50)
+            print(code)
+            print("=" * 50)
+            
+            # Create a safe execution environment
+            exec_globals = {
+                'omni': __import__('omni'),
+                'omni.isaac': __import__('omni.isaac'),
+                'omni.isaac.core': __import__('omni.isaac.core'),
+                'omni.isaac.sim': __import__('omni.isaac.sim'),
+                'omni.isaac.ui': __import__('omni.isaac.ui'),
+                'omni.isaac.debug_draw': __import__('omni.isaac.debug_draw'),
+                'torch': torch,
+                'numpy': np,
+                'stage_utils': stage_utils,
+                'prim_utils': prim_utils,
+                'nucleus_utils': nucleus_utils,
+                'semantics_utils': semantics_utils,
+                'materials': materials,
+                'articulations': articulations,
+                'robots': robots,
+                'objects': objects,
+                'torch_utils': torch_utils,
+                'torch_maths': torch_maths,
+                'torch_rotations': torch_rotations,
+                'torch_transforms': torch_transforms,
+                'torch_visualization': torch_visualization,
+                'torch_control': torch_control,
+                'torch_planning': torch_planning,
+                'torch_perception': torch_perception,
+                'torch_learning': torch_learning,
+                'torch_optimization': torch_optimization,
+                'torch_simulation': torch_simulation,
+                'torch_physics': torch_physics,
+                'torch_rendering': torch_rendering,
+                'torch_audio': torch_audio,
+                'torch_haptics': torch_haptics,
+                'torch_networking': torch_networking,
+                'torch_storage': torch_storage,
+                'torch_training': torch_training,
+                'torch_evaluation': torch_evaluation,
+            }
+            
+            exec(code, exec_globals)
+            print("Code executed successfully!")
+            return True
+            
+        except Exception as e:
+            print(f"Error executing generated code: {e}")
+            return False
+    
+    def interactive_mode(self):
+        """Run interactive mode for Isaac Sim robotics assistance."""
+        print("\n=== Isaac Sim Robotics Assistant Interactive Mode ===")
+        print("I can help you create robots, environments, and simulations in Isaac Sim!")
+        print("Type 'quit' to exit")
+        print("\nExample requests:")
+        print("- Create a differential drive robot with wheels")
+        print("- Build a warehouse environment with obstacles")
+        print("- Add a depth camera to my robot")
+        print("- Create a UR5 manipulator arm")
+        print()
+        
+        while True:
+            try:
+                request = input("What would you like me to create in Isaac Sim? ").strip()
+                if request.lower() in ['quit', 'exit', 'q']:
+                    break
+                if not request:
+                    continue
+                
+                print("Generating Isaac Sim code...")
+                code = self.generate_robot_code(request)
+                
+                print("\nGenerated Code:")
+                print(code)
+                
+                # Ask if user wants to execute the code
+                execute = input("\nWould you like me to execute this code? (y/n): ").strip().lower()
+                if execute in ['y', 'yes']:
+                    success = self.execute_generated_code(code)
+                    if success:
+                        print("Great! The code has been executed in Isaac Sim.")
+                    else:
+                        print("There was an issue executing the code. Check the error messages above.")
+                
+                print()
+                
+            except KeyboardInterrupt:
+                print("\nExiting...")
+                break
+            except Exception as e:
+                print(f"Error: {e}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Isaac Sim Robotics Assistant")
+    parser.add_argument(
+        "--model_path",
+        type=str,
+        default="TomBombadyl/Qwen2.5-Coder-7B-Instruct-Omni1.1",
+        help="Path to the fine-tuned Isaac Sim robotics model"
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default="auto",
+        choices=["auto", "cpu", "cuda"],
+        help="Device to use for inference"
+    )
+    parser.add_argument(
+        "--request",
+        type=str,
+        help="Specific Isaac Sim robotics request (if not provided, will use interactive mode)"
+    )
+    
+    args = parser.parse_args()
+    
+    try:
+        # Create assistant
+        assistant = IsaacSimRoboticsAssistant(args.model_path, args.device)
+        
+        if args.request:
+            # Single request mode
+            print(f"Processing request: {args.request}")
+            code = assistant.generate_robot_code(args.request)
+            print("\nGenerated Code:")
+            print(code)
+            
+            # Ask if user wants to execute
+            execute = input("\nWould you like me to execute this code? (y/n): ").strip().lower()
+            if execute in ['y', 'yes']:
+                success = assistant.execute_generated_code(code)
+                if success:
+                    print("Code executed successfully!")
+                else:
+                    print("Code execution failed.")
+        else:
+            # Interactive mode
+            assistant.interactive_mode()
+    
+    except Exception as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()