code: # Copyright 2023 LanceDB Developers
  #
  #  Licensed under the Apache License, Version 2.0 (the "License");
  #  you may not use this file except in compliance with the License.
  #  You may obtain a copy of the License at
  #      http://www.apache.org/licenses/LICENSE-2.0
  #
  #  Unless required by applicable law or agreed to in writing, software
  #  distributed under the License is distributed on an "AS IS" BASIS,
  #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #  See the License for the specific language governing permissions and
  #  limitations under the License.
  import os
  import random
  from unittest import mock
  
  import lancedb as ldb
  import numpy as np
  import pandas as pd
  import pytest
  
  pytest.importorskip("lancedb.fts")
  tantivy = pytest.importorskip("tantivy")
  
  
  @pytest.fixture
  def table(tmp_path) -> ldb.table.LanceTable:
      db = ldb.connect(tmp_path)
      vectors = [np.random.randn(128) for _ in range(100)]
  
      nouns = ("puppy", "car", "rabbit", "girl", "monkey")
      verbs = ("runs", "hits", "jumps", "drives", "barfs")
      adv = ("crazily.", "dutifully.", "foolishly.", "merrily.", "occasionally.")
      adj = ("adorable", "clueless", "dirty", "odd", "stupid")
      text = [
          " ".join(
              [
                  nouns[random.randrange(0, 5)],
                  verbs[random.randrange(0, 5)],
                  adv[random.randrange(0, 5)],
                  adj[random.randrange(0, 5)],
              ]
          )
          for _ in range(100)
      ]
      table = db.create_table(
          "test",
          data=pd.DataFrame(
              {
                  "vector": vectors,
                  "id": [i % 2 for i in range(100)],
                  "text": text,
                  "text2": text,
                  "nested": [{"text": t} for t in text],
              }
          ),
      )
      return table
  
  
  def test_create_index(tmp_path):
      index = ldb.fts.create_index(str(tmp_path / "index"), ["text"])
      assert isinstance(index, tantivy.Index)
      assert os.path.exists(str(tmp_path / "index"))
  
  
  def test_populate_index(tmp_path, table):
      index = ldb.fts.create_index(str(tmp_path / "index"), ["text"])
      assert ldb.fts.populate_index(index, table, ["text"]) == len(table)
  
  
  def test_search_index(tmp_path, table):
      index = ldb.fts.create_index(str(tmp_path / "index"), ["text"])
      ldb.fts.populate_index(index, table, ["text"])
      index.reload()
      results = ldb.fts.search_index(index, query="puppy", limit=10)
      assert len(results) == 2
      assert len(results[0]) == 10  # row_ids
      assert len(results[1]) == 10  # _distance
  
  
  def test_create_index_from_table(tmp_path, table):
      table.create_fts_index("text")
      df = table.search("puppy").limit(10).select(["text"]).to_pandas()
      assert len(df) <= 10
      assert "text" in df.columns
  
      # Check whether it can be updated
      table.add(
          [
              {
                  "vector": np.random.randn(128),
                  "id": 101,
                  "text": "gorilla",
                  "text2": "gorilla",
                  "nested": {"text": "gorilla"},
              }
          ]
      )
  
      with pytest.raises(ValueError, match="already exists"):
          table.create_fts_index("text")
  
      table.create_fts_index("text", replace=True)
      assert len(table.search("gorilla").limit(1).to_pandas()) == 1
  
  
  def test_create_index_multiple_columns(tmp_path, table):
      table.create_fts_index(["text", "text2"])
      df = table.search("puppy").limit(10).to_pandas()
      assert len(df) == 10
      assert "text" in df.columns
      assert "text2" in df.columns
  
  
  def test_empty_rs(tmp_path, table, mocker):
      table.create_fts_index(["text", "text2"])
      mocker.patch("lancedb.fts.search_index", return_value=([], []))
      df = table.search("puppy").limit(10).to_pandas()
      assert len(df) == 0
  
  
  def test_nested_schema(tmp_path, table):
      table.create_fts_index("nested.text")
      rs = table.search("puppy").limit(10).to_list()
      assert len(rs) == 10
  
  
  def test_search_index_with_filter(table):
      table.create_fts_index("text")
      orig_import = __import__
  
      def import_mock(name, *args):
          if name == "duckdb":
              raise ImportError
          return orig_import(name, *args)
  
      # no duckdb
      with mock.patch("builtins.__import__", side_effect=import_mock):
          rs = table.search("puppy").where("id=1").limit(10)
          # test schema
          assert rs.to_arrow().drop("score").schema.equals(table.schema)
  
          rs = rs.to_list()
          for r in rs:
              assert r["id"] == 1
  
      # yes duckdb
      rs2 = table.search("puppy").where("id=1").limit(10).to_list()
      for r in rs2:
          assert r["id"] == 1
  
      assert rs == rs2
      rs = table.search("puppy").where("id=1").with_row_id(True).limit(10).to_list()
      for r in rs:
          assert r["id"] == 1
          assert r["_rowid"] is not None
  
  
  def test_null_input(table):
      table.add(
          [
              {
                  "vector": np.random.randn(128),
                  "id": 101,
                  "text": None,
                  "text2": None,
                  "nested": {"text": None},
              }
          ]
      )
      table.create_fts_index("text")
  
  
  def test_syntax(table):
      # https://github.com/lancedb/lancedb/issues/769
      table.create_fts_index("text")
      with pytest.raises(ValueError, match="Syntax Error"):
          table.search("they could have been dogs OR cats").limit(10).to_list()
  
      # these should work
  
      # terms queries
      table.search('"they could have been dogs" OR cats').limit(10).to_list()
      table.search("(they AND could) OR (have AND been AND dogs) OR cats").limit(
          10
      ).to_list()
  
      # phrase queries
      table.search("they could have been dogs OR cats").phrase_query().limit(10).to_list()
      table.search('"they could have been dogs OR cats"').limit(10).to_list()
      table.search('''"the cats OR dogs were not really 'pets' at all"''').limit(
          10
      ).to_list()
      table.search('the cats OR dogs were not really "pets" at all').phrase_query().limit(
          10
      ).to_list()
      table.search('the cats OR dogs were not really "pets" at all').phrase_query().limit(
          10
      ).to_list()
  
apis: lancedb.fts.search_index lancedb.fts.populate_index lancedb.connect
extract_api: [((716, 750), 'pytest.importorskip', 'pytest.importorskip', (['"""lancedb.fts"""'], {}), "('lancedb.fts')\n", (735, 750), False, 'import pytest\n'), ((761, 791), 'pytest.importorskip', 'pytest.importorskip', (['"""tantivy"""'], {}), "('tantivy')\n", (780, 791), False, 'import pytest\n'), ((864, 885), 'lancedb.connect', 'ldb.connect', (['tmp_path'], {}), '(tmp_path)\n', (875, 885), True, 'import lancedb as ldb\n'), ((2318, 2364), 'lancedb.fts.populate_index', 'ldb.fts.populate_index', (['index', 'table', "['text']"], {}), "(index, table, ['text'])\n", (2340, 2364), True, 'import lancedb as ldb\n'), ((2398, 2450), 'lancedb.fts.search_index', 'ldb.fts.search_index', (['index'], {'query': '"""puppy"""', 'limit': '(10)'}), "(index, query='puppy', limit=10)\n", (2418, 2450), True, 'import lancedb as ldb\n'), ((901, 921), 'numpy.random.randn', 'np.random.randn', (['(128)'], {}), '(128)\n', (916, 921), True, 'import numpy as np\n'), ((2143, 2189), 'lancedb.fts.populate_index', 'ldb.fts.populate_index', (['index', 'table', "['text']"], {}), "(index, table, ['text'])\n", (2165, 2189), True, 'import lancedb as ldb\n'), ((3096, 3145), 'pytest.raises', 'pytest.raises', (['ValueError'], {'match': '"""already exists"""'}), "(ValueError, match='already exists')\n", (3109, 3145), False, 'import pytest\n'), ((4216, 4274), 'unittest.mock.patch', 'mock.patch', (['"""builtins.__import__"""'], {'side_effect': 'import_mock'}), "('builtins.__import__', side_effect=import_mock)\n", (4226, 4274), False, 'from unittest import mock\n'), ((5261, 5308), 'pytest.raises', 'pytest.raises', (['ValueError'], {'match': '"""Syntax Error"""'}), "(ValueError, match='Syntax Error')\n", (5274, 5308), False, 'import pytest\n'), ((2889, 2909), 'numpy.random.randn', 'np.random.randn', (['(128)'], {}), '(128)\n', (2904, 2909), True, 'import numpy as np\n'), ((4922, 4942), 'numpy.random.randn', 'np.random.randn', (['(128)'], {}), '(128)\n', (4937, 4942), True, 'import numpy as np\n'), ((1266, 1288), 'random.randrange', 'random.randrange', (['(0)', '(5)'], {}), '(0, 5)\n', (1282, 1288), False, 'import random\n'), ((1313, 1335), 'random.randrange', 'random.randrange', (['(0)', '(5)'], {}), '(0, 5)\n', (1329, 1335), False, 'import random\n'), ((1358, 1380), 'random.randrange', 'random.randrange', (['(0)', '(5)'], {}), '(0, 5)\n', (1374, 1380), False, 'import random\n'), ((1403, 1425), 'random.randrange', 'random.randrange', (['(0)', '(5)'], {}), '(0, 5)\n', (1419, 1425), False, 'import random\n')]