code: # RAG/TAG Tiger - llm.py
  # Copyright (c) 2024 Stuart Riffle
  # github.com/stuartriffle/ragtag-tiger
  
  import os
  import torch
  from .files import *
  from .lograg import lograg, lograg_verbose, lograg_error
  from .timer import TimerUntil
  
  openai_model_default    = "gpt-3.5-turbo-instruct"
  google_model_default    = "models/text-bison-001"
  anthropic_model_default = "claude-2"
  mistral_default         = "mistral-small"
  perplexity_default      = "llama-2-70b-chat"
  replicate_default       = "mistralai/mixtral-8x7b-instruct-v0.1"
  fireworks_ai_default    = "accounts/fireworks/models/mixtral-8x7b-instruct"
  together_ai_default     = "codellama/CodeLlama-70b-Instruct-hf"
  
  default_timeout         = 180
  default_temperature     = 0.1
  default_max_tokens      = 500
  default_llm_provider    = "huggingface"
  hf_model_nicknames      = { "default": "codellama/CodeLlama-7b-Instruct-hf" }
  
  def load_llm(provider, model, server, api_key, params, global_params, verbose=False, set_service_context=True, torch_device=None):
      result = None
      streaming_supported = True
      try:
          with TimerUntil("ready"):
              all_params = global_params.copy()
              model_params = dict([param.split("=") for param in params]) if params else {}
              for k, v in model_params.items():
                  all_params[k] = v
  
              model_kwargs = {}
              for k, v in all_params.items():
                  model_kwargs[k] = float(v) if v.replace(".", "", 1).isdigit() else v
  
              temperature = float(model_kwargs.get("temperature", default_temperature))
              max_tokens = int(model_kwargs.get("max_tokens", default_max_tokens))
  
              ### OpenAI
              if provider == "openai" and not server:
                  model_name = model or openai_model_default
                  api_key = api_key or os.environ.get("OPENAI_API_KEY", "") 
                  lograg(f"OpenAI model \"{model_name}\"...")
                  from llama_index.llms import OpenAI
                  result = OpenAI(
                      model=model_name,
                      timeout=default_timeout,
                      api_key=api_key,
                      additional_kwargs=model_kwargs,
                      temperature=temperature,
                      max_tokens=max_tokens,
                      verbose=verbose)
                  
              ### OpenAI API-compatible third party server                
              elif provider == "openai" and server:
                  # Auto-populate API key and model for known providers
                  if "together.ai" in server or "together.xyz" in server:
                      api_key = api_key or os.environ.get("TOGETHERAI_API_KEY", "")
                      model = model or together_ai_default
                  if "fireworks.ai" in server: 
                      api_key = api_key or os.environ.get("FIREWORKS_API_KEY", "")
                      model = model or fireworks_ai_default
                  api_key = api_key or os.environ.get("OPENAI_API_KEY", "") 
                  model_name = model or "default"
                 
                  lograg(f"Model \"{model_name}\" on \"{server}\"...")
                  from llama_index.llms import OpenAILike
                  result = OpenAILike(
                      api_key=api_key,
                      model=model_name,
                      additional_kwargs=model_kwargs,
                      api_base=server,
                      max_iterations=100,
                      timeout=default_timeout,
                      max_tokens=max_tokens,
                      temperature=temperature,
                      verbose=verbose)
                  
              ### Google
              elif provider == "google":
                  gemini_api_key = os.environ.get("GEMINI_API_KEY", "")
                  google_api_key = os.environ.get("GOOGLE_API_KEY", "")
                  model_name = model or google_model_default
                  import google.generativeai as genai
                  genai.configure(api_key=google_api_key)
  
                  if "gemini" in str(model_name).lower():
                      lograg(f"Google Gemini model \"{model_name}\"...")
                      from llama_index.llms import Gemini
                      result = Gemini(
                          api_key=api_key or gemini_api_key,
                          model_name=model_name,
                          max_tokens=max_tokens,
                          temperature=temperature,
                          model_kwargs=model_kwargs)
                  else:
                      lograg(f"Google PaLM model \"{model_name}\"...")
                      from llama_index.llms import PaLM
                      result = PaLM(
                          api_key=api_key or google_api_key,
                          model_name=model_name,
                          generate_kwargs=model_kwargs)
                      streaming_supported = False
                  
              ### Llama.cpp
              elif provider == "llamacpp":
                  if torch.cuda.is_available():
                      # FIXME - this does nothing? Always on CPU
                      model_kwargs["n_gpu_layers"] = -1
                  lograg(f"llama.cpp model \"{cleanpath(model)}\"...")
                  from llama_index.llms import LlamaCPP
                  result = LlamaCPP(
                      model_path=model,
                      model_kwargs=model_kwargs,
                      max_new_tokens=max_tokens,
                      temperature=temperature,
                      verbose=verbose)
                  
              ### Mistral
              elif provider == "mistral":
                  api_key = api_key or os.environ.get("MISTRAL_API_KEY", None)
                  model_name = model or mistral_default
                  lograg(f"Mistral model \"{model_name}\"...")
                  from llama_index.llms import MistralAI
                  result = MistralAI(
                      api_key=api_key,
                      model=model_name,
                      max_tokens=max_tokens,
                      temperature=temperature,
                      additional_kwargs=model_kwargs)
                  
              ### Perplexity
              elif provider == "perplexity":
                  api_key = api_key or os.environ.get("PERPLEXITYAI_API_KEY", "")
                  model_name = model or perplexity_default
                  lograg(f"Perplexity model \"{model_name}\"...")
                  from llama_index.llms import Perplexity
                  result = Perplexity(
                      api_key=api_key,
                      model=model_name,
                      max_tokens=max_tokens,
                      temperature=temperature,
                      model_kwargs=model_kwargs)
                  
              ### Replicate
              elif provider == "replicate":
                  api_key = api_key or os.environ.get("REPLICATE_API_TOKEN", "")
                  model_name = model or replicate_default
                  lograg(f"Replicate model \"{model_name}\"...")
                  from llama_index.llms import Replicate
                  result = Replicate(
                      model=model_name,
                      temperature=temperature,
                      additional_kwargs=model_kwargs)
              
              ### HuggingFace
              else:
                  os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1"
                  model_desc = ""
                  model_name = model or "default"
                  if model_name in hf_model_nicknames:
                      model_desc = f" (\"{model_name}\")"
                      model_name = hf_model_nicknames[model_name]
                  lograg(f"HuggingFace model \"{model_name}\"{model_desc}...")
  
                  from llama_index.llms import HuggingFaceLLM
                  result = HuggingFaceLLM(
                      model_name=model_name,
                      model_kwargs=model_kwargs,
                      max_new_tokens=max_tokens,
                      device_map=torch_device or "auto")
                      #system_prompt=system_prompt)
  
              from llama_index import ServiceContext, set_global_service_context
              service_context = ServiceContext.from_defaults(
                  embed_model='local',
                  llm=result)
  
              if set_service_context:
                  set_global_service_context(service_context)
  
      except Exception as e: 
          lograg_error(f"failure initializing LLM: {e}", exit_code=1)
  
      return result, streaming_supported, service_context
  
  
  
  def split_llm_config(config):
      """Split an LLM from a config string of format "[alias=]provider[,model[,server[,api-key[,parameters...]]]]" into its components"""
  
      fields   = config.strip("\"' ").split(",")
      provider = fields[0].strip() if len(fields) > 0 else default_llm_provider
      model    = fields[1].strip() if len(fields) > 1 else None
      server   = fields[2].strip() if len(fields) > 2 else None
      api_key  = fields[3].strip() if len(fields) > 3 else None
      params   = fields[4:]        if len(fields) > 4 else []
  
      alias = None
      if "=" in provider:
          alias, provider = provider.split("=", 1)
          provider = provider.strip()
  
      return provider, model, server, api_key, params, alias
  
  
  def load_llm_config(config, global_params, set_service_context=True):
      """Load an LLM from a config string like "provider,model,server,api-key,param1,param2,..."""
      provider, model, server, api_key, params, _ = split_llm_config(config)
      return load_llm(provider.lower(), model, server, api_key, params, global_params, set_service_context)
  
  
apis: llama_index.llms.MistralAI llama_index.llms.Perplexity llama_index.llms.Gemini llama_index.llms.OpenAILike llama_index.ServiceContext.from_defaults llama_index.llms.Replicate llama_index.llms.OpenAI llama_index.llms.LlamaCPP llama_index.llms.PaLM llama_index.llms.HuggingFaceLLM llama_index.set_global_service_context
extract_api: [((8029, 8090), 'llama_index.ServiceContext.from_defaults', 'ServiceContext.from_defaults', ([], {'embed_model': '"""local"""', 'llm': 'result'}), "(embed_model='local', llm=result)\n", (8057, 8090), False, 'from llama_index import ServiceContext, set_global_service_context\n'), ((1986, 2158), 'llama_index.llms.OpenAI', 'OpenAI', ([], {'model': 'model_name', 'timeout': 'default_timeout', 'api_key': 'api_key', 'additional_kwargs': 'model_kwargs', 'temperature': 'temperature', 'max_tokens': 'max_tokens', 'verbose': 'verbose'}), '(model=model_name, timeout=default_timeout, api_key=api_key,\n    additional_kwargs=model_kwargs, temperature=temperature, max_tokens=\n    max_tokens, verbose=verbose)\n', (1992, 2158), False, 'from llama_index.llms import OpenAI\n'), ((8177, 8220), 'llama_index.set_global_service_context', 'set_global_service_context', (['service_context'], {}), '(service_context)\n', (8203, 8220), False, 'from llama_index import ServiceContext, set_global_service_context\n'), ((1811, 1847), 'os.environ.get', 'os.environ.get', (['"""OPENAI_API_KEY"""', '""""""'], {}), "('OPENAI_API_KEY', '')\n", (1825, 1847), False, 'import os\n'), ((3186, 3404), 'llama_index.llms.OpenAILike', 'OpenAILike', ([], {'api_key': 'api_key', 'model': 'model_name', 'additional_kwargs': 'model_kwargs', 'api_base': 'server', 'max_iterations': '(100)', 'timeout': 'default_timeout', 'max_tokens': 'max_tokens', 'temperature': 'temperature', 'verbose': 'verbose'}), '(api_key=api_key, model=model_name, additional_kwargs=\n    model_kwargs, api_base=server, max_iterations=100, timeout=\n    default_timeout, max_tokens=max_tokens, temperature=temperature,\n    verbose=verbose)\n', (3196, 3404), False, 'from llama_index.llms import OpenAILike\n'), ((2934, 2970), 'os.environ.get', 'os.environ.get', (['"""OPENAI_API_KEY"""', '""""""'], {}), "('OPENAI_API_KEY', '')\n", (2948, 2970), False, 'import os\n'), ((3684, 3720), 'os.environ.get', 'os.environ.get', (['"""GEMINI_API_KEY"""', '""""""'], {}), "('GEMINI_API_KEY', '')\n", (3698, 3720), False, 'import os\n'), ((3754, 3790), 'os.environ.get', 'os.environ.get', (['"""GOOGLE_API_KEY"""', '""""""'], {}), "('GOOGLE_API_KEY', '')\n", (3768, 3790), False, 'import os\n'), ((3918, 3957), 'google.generativeai.configure', 'genai.configure', ([], {'api_key': 'google_api_key'}), '(api_key=google_api_key)\n', (3933, 3957), True, 'import google.generativeai as genai\n'), ((2614, 2654), 'os.environ.get', 'os.environ.get', (['"""TOGETHERAI_API_KEY"""', '""""""'], {}), "('TOGETHERAI_API_KEY', '')\n", (2628, 2654), False, 'import os\n'), ((2799, 2838), 'os.environ.get', 'os.environ.get', (['"""FIREWORKS_API_KEY"""', '""""""'], {}), "('FIREWORKS_API_KEY', '')\n", (2813, 2838), False, 'import os\n'), ((4171, 4315), 'llama_index.llms.Gemini', 'Gemini', ([], {'api_key': '(api_key or gemini_api_key)', 'model_name': 'model_name', 'max_tokens': 'max_tokens', 'temperature': 'temperature', 'model_kwargs': 'model_kwargs'}), '(api_key=api_key or gemini_api_key, model_name=model_name, max_tokens\n    =max_tokens, temperature=temperature, model_kwargs=model_kwargs)\n', (4177, 4315), False, 'from llama_index.llms import Gemini\n'), ((4606, 4702), 'llama_index.llms.PaLM', 'PaLM', ([], {'api_key': '(api_key or google_api_key)', 'model_name': 'model_name', 'generate_kwargs': 'model_kwargs'}), '(api_key=api_key or google_api_key, model_name=model_name,\n    generate_kwargs=model_kwargs)\n', (4610, 4702), False, 'from llama_index.llms import PaLM\n'), ((4923, 4948), 'torch.cuda.is_available', 'torch.cuda.is_available', ([], {}), '()\n', (4946, 4948), False, 'import torch\n'), ((5215, 5342), 'llama_index.llms.LlamaCPP', 'LlamaCPP', ([], {'model_path': 'model', 'model_kwargs': 'model_kwargs', 'max_new_tokens': 'max_tokens', 'temperature': 'temperature', 'verbose': 'verbose'}), '(model_path=model, model_kwargs=model_kwargs, max_new_tokens=\n    max_tokens, temperature=temperature, verbose=verbose)\n', (5223, 5342), False, 'from llama_index.llms import LlamaCPP\n'), ((5792, 5920), 'llama_index.llms.MistralAI', 'MistralAI', ([], {'api_key': 'api_key', 'model': 'model_name', 'max_tokens': 'max_tokens', 'temperature': 'temperature', 'additional_kwargs': 'model_kwargs'}), '(api_key=api_key, model=model_name, max_tokens=max_tokens,\n    temperature=temperature, additional_kwargs=model_kwargs)\n', (5801, 5920), False, 'from llama_index.llms import MistralAI\n'), ((5557, 5596), 'os.environ.get', 'os.environ.get', (['"""MISTRAL_API_KEY"""', 'None'], {}), "('MISTRAL_API_KEY', None)\n", (5571, 5596), False, 'import os\n'), ((6387, 6511), 'llama_index.llms.Perplexity', 'Perplexity', ([], {'api_key': 'api_key', 'model': 'model_name', 'max_tokens': 'max_tokens', 'temperature': 'temperature', 'model_kwargs': 'model_kwargs'}), '(api_key=api_key, model=model_name, max_tokens=max_tokens,\n    temperature=temperature, model_kwargs=model_kwargs)\n', (6397, 6511), False, 'from llama_index.llms import Perplexity\n'), ((6142, 6184), 'os.environ.get', 'os.environ.get', (['"""PERPLEXITYAI_API_KEY"""', '""""""'], {}), "('PERPLEXITYAI_API_KEY', '')\n", (6156, 6184), False, 'import os\n'), ((6972, 7061), 'llama_index.llms.Replicate', 'Replicate', ([], {'model': 'model_name', 'temperature': 'temperature', 'additional_kwargs': 'model_kwargs'}), '(model=model_name, temperature=temperature, additional_kwargs=\n    model_kwargs)\n', (6981, 7061), False, 'from llama_index.llms import Replicate\n'), ((7661, 7791), 'llama_index.llms.HuggingFaceLLM', 'HuggingFaceLLM', ([], {'model_name': 'model_name', 'model_kwargs': 'model_kwargs', 'max_new_tokens': 'max_tokens', 'device_map': "(torch_device or 'auto')"}), "(model_name=model_name, model_kwargs=model_kwargs,\n    max_new_tokens=max_tokens, device_map=torch_device or 'auto')\n", (7675, 7791), False, 'from llama_index.llms import HuggingFaceLLM\n'), ((6731, 6772), 'os.environ.get', 'os.environ.get', (['"""REPLICATE_API_TOKEN"""', '""""""'], {}), "('REPLICATE_API_TOKEN', '')\n", (6745, 6772), False, 'import os\n')]