code: #!/usr/bin/env python3
  # -*- coding: utf-8 -*-
  
  import os
  import os.path
  import re
  import sys
  import gin
  import shutil
  import logging
  import tempfile
  import requests
  import subprocess
  
  from pathlib import Path
  from urllib.parse import urlparse
  
  from llama_index import ServiceContext, StorageContext
  from llama_index import VectorStoreIndex, SimpleDirectoryReader, SimpleKeywordTableIndex
  from llama_index import set_global_service_context
  from processors.markdown import MarkdownReader
  from processors.embedding import get_embedding
  
  logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
  logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
  
  sdr_exclude = [
      "*.rst", "*.ipynb", "*.py", "*.bat", "*.txt", "*.png", "*.jpg", "*.jpeg",
      "*.csv", "*.html", "*.js", "*.css", "*.pdf", "*.json"
  ]
  
  re_github = r"https://(?P<token>.*?)github\.com/(?P<org>[^/]+)/(?P<repo>[^/\s]+)/?(?P<type>(tree|blob)/?(?P<version>[^/\s]+)/?(?P<path>.+)?)?"
  
  
  def file_reader(file_path: str):
      return MarkdownReader().load_data(Path(file_path))
  
  
  def dir_reader(dir_path: str):
      return SimpleDirectoryReader(
          input_dir=dir_path,
          exclude=sdr_exclude,
          file_extractor={
              ".md": MarkdownReader()
          },
          recursive=True,
      ).load_data()
  
  
  def url_reader(url: str):
      logging.info(f"{url} start")
      resp = requests.get(url, timeout=300)
      if "text/" in resp.headers.get('content-type', ""):
          f = tempfile.NamedTemporaryFile(suffix=".md", delete=False)
          f.write(resp.content)
          f.close()
          docs = file_reader(f.name)
          shutil.rmtree(f.name, ignore_errors=True)
          return docs
      return []
  
  
  def github_reader(urlParse: re.Match):
      urlReGroups = urlParse.groups()
      token = urlReGroups[0]
      org = urlReGroups[1]
      repo = urlReGroups[2]
      version = urlReGroups[4]  # None|tree|blob
      branch = urlReGroups[5]  # tag_name|branch_name|commit_id
      # version == tree, path is dir; version == blob, path is file
      sub_path = "" if urlReGroups[6] == None else urlReGroups[6]
  
      if version == "blob":
          url = f'https://{token}raw.githubusercontent.com/{org}/{repo}/{branch}/{sub_path}'
          return url_reader(url)
  
      if version not in [None, "tree"]:
          return []
  
      url = f'https://{token}github.com/{org}/{repo}'
  
      if branch:
          args = ["git", "clone", "--depth", "1", "--branch", branch, url, "."]
      else:
          args = ["git", "clone", "--depth", "1", url, "."]
  
      del_not_md = '''find . -type f ! -name "*.md" | xargs rm -rf'''
      logging.info(f"{args} start")
      with tempfile.TemporaryDirectory() as tmpdirname:
          subprocess.run(args, check=True, timeout=300, cwd=tmpdirname)
          subprocess.run(del_not_md, shell=True, timeout=300, cwd=tmpdirname)
          logging.info(f"{args} ended")
          docs = dir_reader(os.path.join(tmpdirname, sub_path))
          return docs
  
  
  map_func = {
      "dir": dir_reader,
      "file": file_reader,
      "github": github_reader,
      "url": url_reader,
  }
  
  # python rag-index index_persist_path collection_path...
  # collection_path
  #     data/
  #     /data
  #     https://abc.com/xyz.md
  #     https://<token>@github.com/<org>/<repo>
  #     https://<token>@github.com/<org>/<repo>/tree/<tag_name|branch_name>/<sub_dir>
  #     https://<token>@github.com/<org>/<repo>/blob/<tag_name|branch_name|commit_id>/<sub_dir>/<file_name>.md
  if __name__ == "__main__":
      if len(sys.argv) < 3:
          sys.exit(0)
  
      # We assume that there output directory is the first argument, and the rest is input directory
      output = sys.argv[1]
      gin.parse_config_file('index.gin')
  
      # init download hugging fact model
      service_context = ServiceContext.from_defaults(
          llm=None,
          llm_predictor=None,
          embed_model=get_embedding(),
      )
  
      storage_context = StorageContext.from_defaults()
  
      set_global_service_context(service_context)
  
      documents = []
      for file_path in sys.argv[2:]:
          if os.path.isfile(file_path) and file_path.endswith(".md"):
              print(map_func["file"])
              documents.extend(map_func["file"](file_path))
          elif os.path.isdir(file_path):
              documents.extend(map_func["dir"](file_path))
          else:
              match_github = re.search(re_github, file_path)
              if match_github:
                  documents.extend(map_func["github"](match_github))
                  continue
  
              match_url = urlparse(file_path)
              if match_url.scheme and match_url.netloc:
                  documents.extend(map_func["url"](file_path))
                  continue
  
      # exclude these things from considerations.
      for doc in documents:
          doc.excluded_llm_metadata_keys = ["file_name", "content_type"]
          doc.excluded_embed_metadata_keys = ["file_name", "content_type"]
  
      try:
          embedding_index = VectorStoreIndex.from_documents(
              documents, storage_context=storage_context)
          keyword_index = SimpleKeywordTableIndex(
              documents, storage_context=storage_context)
  
          embedding_index.set_index_id("embedding")
          embedding_index.storage_context.persist(persist_dir=output)
          keyword_index.set_index_id("keyword")
          keyword_index.storage_context.persist(persist_dir=output)
      except Exception as e:
          print(str(e))
          shutil.rmtree(output, ignore_errors=True)
  
apis: llama_index.VectorStoreIndex.from_documents llama_index.set_global_service_context llama_index.StorageContext.from_defaults llama_index.SimpleKeywordTableIndex
extract_api: [((535, 594), 'logging.basicConfig', 'logging.basicConfig', ([], {'stream': 'sys.stdout', 'level': 'logging.DEBUG'}), '(stream=sys.stdout, level=logging.DEBUG)\n', (554, 594), False, 'import logging\n'), ((626, 666), 'logging.StreamHandler', 'logging.StreamHandler', ([], {'stream': 'sys.stdout'}), '(stream=sys.stdout)\n', (647, 666), False, 'import logging\n'), ((1327, 1355), 'logging.info', 'logging.info', (['f"""{url} start"""'], {}), "(f'{url} start')\n", (1339, 1355), False, 'import logging\n'), ((1367, 1397), 'requests.get', 'requests.get', (['url'], {'timeout': '(300)'}), '(url, timeout=300)\n', (1379, 1397), False, 'import requests\n'), ((2577, 2606), 'logging.info', 'logging.info', (['f"""{args} start"""'], {}), "(f'{args} start')\n", (2589, 2606), False, 'import logging\n'), ((3614, 3648), 'gin.parse_config_file', 'gin.parse_config_file', (['"""index.gin"""'], {}), "('index.gin')\n", (3635, 3648), False, 'import gin\n'), ((3853, 3883), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {}), '()\n', (3881, 3883), False, 'from llama_index import ServiceContext, StorageContext\n'), ((3889, 3932), 'llama_index.set_global_service_context', 'set_global_service_context', (['service_context'], {}), '(service_context)\n', (3915, 3932), False, 'from llama_index import set_global_service_context\n'), ((595, 614), 'logging.getLogger', 'logging.getLogger', ([], {}), '()\n', (612, 614), False, 'import logging\n'), ((1040, 1055), 'pathlib.Path', 'Path', (['file_path'], {}), '(file_path)\n', (1044, 1055), False, 'from pathlib import Path\n'), ((1466, 1521), 'tempfile.NamedTemporaryFile', 'tempfile.NamedTemporaryFile', ([], {'suffix': '""".md"""', 'delete': '(False)'}), "(suffix='.md', delete=False)\n", (1493, 1521), False, 'import tempfile\n'), ((1613, 1654), 'shutil.rmtree', 'shutil.rmtree', (['f.name'], {'ignore_errors': '(True)'}), '(f.name, ignore_errors=True)\n', (1626, 1654), False, 'import shutil\n'), ((2616, 2645), 'tempfile.TemporaryDirectory', 'tempfile.TemporaryDirectory', ([], {}), '()\n', (2643, 2645), False, 'import tempfile\n'), ((2669, 2730), 'subprocess.run', 'subprocess.run', (['args'], {'check': '(True)', 'timeout': '(300)', 'cwd': 'tmpdirname'}), '(args, check=True, timeout=300, cwd=tmpdirname)\n', (2683, 2730), False, 'import subprocess\n'), ((2739, 2806), 'subprocess.run', 'subprocess.run', (['del_not_md'], {'shell': '(True)', 'timeout': '(300)', 'cwd': 'tmpdirname'}), '(del_not_md, shell=True, timeout=300, cwd=tmpdirname)\n', (2753, 2806), False, 'import subprocess\n'), ((2815, 2844), 'logging.info', 'logging.info', (['f"""{args} ended"""'], {}), "(f'{args} ended')\n", (2827, 2844), False, 'import logging\n'), ((3473, 3484), 'sys.exit', 'sys.exit', (['(0)'], {}), '(0)\n', (3481, 3484), False, 'import sys\n'), ((4880, 4955), 'llama_index.VectorStoreIndex.from_documents', 'VectorStoreIndex.from_documents', (['documents'], {'storage_context': 'storage_context'}), '(documents, storage_context=storage_context)\n', (4911, 4955), False, 'from llama_index import VectorStoreIndex, SimpleDirectoryReader, SimpleKeywordTableIndex\n'), ((4993, 5060), 'llama_index.SimpleKeywordTableIndex', 'SimpleKeywordTableIndex', (['documents'], {'storage_context': 'storage_context'}), '(documents, storage_context=storage_context)\n', (5016, 5060), False, 'from llama_index import VectorStoreIndex, SimpleDirectoryReader, SimpleKeywordTableIndex\n'), ((1013, 1029), 'processors.markdown.MarkdownReader', 'MarkdownReader', ([], {}), '()\n', (1027, 1029), False, 'from processors.markdown import MarkdownReader\n'), ((2871, 2905), 'os.path.join', 'os.path.join', (['tmpdirname', 'sub_path'], {}), '(tmpdirname, sub_path)\n', (2883, 2905), False, 'import os\n'), ((3807, 3822), 'processors.embedding.get_embedding', 'get_embedding', ([], {}), '()\n', (3820, 3822), False, 'from processors.embedding import get_embedding\n'), ((3999, 4024), 'os.path.isfile', 'os.path.isfile', (['file_path'], {}), '(file_path)\n', (4013, 4024), False, 'import os\n'), ((4163, 4187), 'os.path.isdir', 'os.path.isdir', (['file_path'], {}), '(file_path)\n', (4176, 4187), False, 'import os\n'), ((5362, 5403), 'shutil.rmtree', 'shutil.rmtree', (['output'], {'ignore_errors': '(True)'}), '(output, ignore_errors=True)\n', (5375, 5403), False, 'import shutil\n'), ((4287, 4318), 're.search', 're.search', (['re_github', 'file_path'], {}), '(re_github, file_path)\n', (4296, 4318), False, 'import re\n'), ((4465, 4484), 'urllib.parse.urlparse', 'urlparse', (['file_path'], {}), '(file_path)\n', (4473, 4484), False, 'from urllib.parse import urlparse\n'), ((1225, 1241), 'processors.markdown.MarkdownReader', 'MarkdownReader', ([], {}), '()\n', (1239, 1241), False, 'from processors.markdown import MarkdownReader\n')]