Updated evaluation function to give tolerance for slight floating point differences

test_pretrained.ipynb

@@ -385,7 +385,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 65,
    "metadata": {},
    "outputs": [
     {
@@ -400,21 +400,24 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "How many games had at least one team with 30+ assists?\n",
-      "SELECT COUNT(*)  FROM game  WHERE ast_home >= 30 OR ast_away >= 30;\n",
-      "11305\n",
+      "What is the total number of assists by the Chicago Bulls at home?\n",
+      "SELECT SUM(ast_home) as total_assists  FROM game  WHERE team_name_home = 'Chicago Bulls';\n",
+      "45090.0\n",
       "SQLite:\n",
-      "SELECT COUNT(*) \n",
+      "SELECT SUM(ast_home) \n",
       "FROM game \n",
-      "WHERE ast_home >= 30 OR ast_away >= 30;\n",
+      "WHERE team_name_home = 'Chicago Bulls';\n",
       "\n",
-      "[(11305,)]\n",
-      "SQL matched? True\n",
+      "[(45090.0,)]\n",
+      "Statement valid? True\n",
+      "SQLite matched? False\n",
       "Result matched? True\n"
      ]
     }
    ],
    "source": [
+    "import math\n",
+    "\n",
     "def compare_result(sample_query, sample_result, query_output):\n",
     "    # Clean model output to only have the query output\n",
     "    if query_output[0:7] == \"SQLite:\":\n",
@@ -435,38 +438,77 @@
     "        sample_query = sample_query.replace(\" \", \"\").replace(\"\\n\", \"\").replace(\"\\t\", \"\")\n",
     "        query_match = (query == sample_query)\n",
     "\n",
+    "        # If the queries match, the results clearly also match\n",
+    "        if query_match:\n",
+    "            return True, True, True\n",
+    "\n",
     "        # Check if this is a multi-line query\n",
     "        if \"|\" in sample_result or \"(\" in sample_result:\n",
+    "            print(rows)\n",
+    "            # Create list of results by stripping separators and splitting on them\n",
     "            if \"(\" in sample_result:\n",
     "                sample_result = sample_result.replace(\"(\", \"\").replace(\")\", \"\")\n",
     "                result_list = sample_result.split(\",\") \n",
     "            else:\n",
     "                result_list = sample_result.split(\"|\") \n",
     "\n",
+    "            # Strip all results in list\n",
     "            for i in range(len(result_list)):\n",
     "                result_list[i] = str(result_list[i]).strip()\n",
+    "            \n",
+    "            # Loop through model result and see if it matches training example\n",
     "            result = False\n",
     "            for row in rows:\n",
     "                for r in row:\n",
-    "                    if str(r) in result_list:\n",
-    "                        return query_match, True\n",
+    "                    for res in result_list:\n",
+    "                        try:\n",
+    "                            if math.isclose(float(r), float(res), abs_tol=0.5):\n",
+    "                                return True, query_match, True\n",
+    "                        except:\n",
+    "                            if r in res or res in r:\n",
+    "                                return True, query_match, True\n",
+    "                    \n",
+    "            # Check if the model returned a sum of examples as opposed to the whole thing\n",
     "            if len(rows) == 1:\n",
     "                for r in rows[0]:\n",
     "                    if r == str(len(result_list)):\n",
-    "                        return query_match, True\n",
-    "            return query_match, result\n",
+    "                        return True, query_match, True\n",
+    "                    \n",
+    "            return True, query_match, result\n",
+    "        # Else the sample result is a single value or string\n",
     "        else:\n",
     "            print(rows)\n",
     "            result = False\n",
+    "            # Loop through model result and see if it contains the sample result\n",
     "            for row in rows:\n",
     "                for r in row:\n",
+    "                    # Check by string\n",
     "                    if str(r) in str(sample_result):\n",
-    "                        return query_match, True\n",
+    "                        try:\n",
+    "                            if math.isclose(float(r), float(sample_result), abs_tol=0.5):\n",
+    "                                return True, query_match, True\n",
+    "                        except:\n",
+    "                            return True, query_match, True\n",
+    "                    # Check by number, using try incase the cast as float fails\n",
+    "                    try:\n",
+    "                        if math.isclose(float(r), float(sample_result), abs_tol=0.5):\n",
+    "                            return True, query_match, True\n",
+    "                    except:\n",
+    "                        pass\n",
+    "\n",
+    "            # Check if the model returned a list of examples instead of a total sum (both acceptable)\n",
+    "            try:\n",
+    "                if len(rows) > 1 and len(rows) == int(sample_result):\n",
+    "                    return True, query_match, True\n",
+    "                if len(rows[0]) > 1 and rows[0][1] is not None and  len(rows[0]) == int(sample_result):\n",
+    "                    return True, query_match, True\n",
+    "            except:\n",
+    "                pass\n",
     "\n",
     "            # Compare results and return\n",
-    "            return query_match, result\n",
+    "            return True, query_match, result\n",
     "    except:\n",
-    "        return False, False\n",
+    "        return False, False, False\n",
     "\n",
     "# Obtain sample\n",
     "sample = df.sample(n=1)\n",
@@ -484,8 +526,9 @@
     "print(query_output)\n",
     "\n",
     "result = compare_result(sample[\"sql_query\"].values[0], sample[\"result\"].values[0], query_output)\n",
-    "print(\"SQL matched? \" + str(result[0]))\n",
-    "print(\"Result matched? \" + str(result[1]))"
+    "print(\"Statement valid? \" + str(result[0]))\n",
+    "print(\"SQLite matched? \" + str(result[1]))\n",
+    "print(\"Result matched? \" + str(result[2]))"
    ]
   }
  ],

train-data/sql_train.tsv

@@ -476,7 +476,7 @@ How many away games did the Chicago Bulls play in the 2022 season?	SELECT COUNT(
 How many home games did the Boston Celtics play in the 2018 season?	SELECT COUNT(*) FROM game WHERE team_name_home = 'Boston Celtics' AND season_id = '22018';	41.0
 How many home games did the Boston Celtics play in the 2020 season?	SELECT COUNT(*) FROM game WHERE team_name_home = 'Boston Celtics' AND season_id = '22020';	36.0
 What is the average number of fg_pct in home games by the Chicago Bulls?	SELECT AVG(fg_pct_home) FROM game WHERE team_name_home = 'Chicago Bulls';	0.4636694306246544
-In which season did the Los Angeles Lakers have the highest average ast at home?	SELECT season_id, AVG(ast_home) as avg_stat FROM game WHERE team_name_home = 'Los Angeles Lakers' GROUP BY season_id ORDER BY avg_stat DESC LIMIT 1;	1969.0
+In which season did the Los Angeles Lakers have the highest average ast at home?	SELECT season_id, AVG(ast_home) as avg_stat FROM game WHERE team_name_home = 'Los Angeles Lakers' GROUP BY season_id ORDER BY avg_stat DESC LIMIT 1;	41969|36.6666666666667
 What is the average number of ft_pct in home games by the Los Angeles Lakers?	SELECT AVG(ft_pct_home) FROM game WHERE team_name_home = 'Los Angeles Lakers';	0.7450706106870195
 In which season did the Golden State Warriors have the highest average reb at home?	SELECT season_id, AVG(reb_home) as avg_stat FROM game WHERE team_name_home = 'Golden State Warriors' GROUP BY season_id ORDER BY avg_stat DESC LIMIT 1;	1974.0
 How many away games did the Miami Heat play in the 1999 season?	SELECT COUNT(*) FROM game WHERE team_name_away = 'Miami Heat' AND season_id = '21999';	41.0