@@ -0,0 +1,2586 @@
+%22%22%22%0ACommon tokens.%0A%0ASomething like this is the end goal of the reverse tokenizer:%0A%0Asql_statement = SELECT(%0A    subtokens=%5B%0A        FIELD(%22table1.id%22, as=%22table1_id%22),%0A        FROM(%22public.user%22, as=%22table1%22),%0A        WHERE(%0A            subtokens=%5B%0A                EQ(%22public.user.id%22, 1)%0A            %5D%0A        )%0A    %5D%0A)%0A%0A%22%22%22%0Afrom katagawa.exceptions import MissingTokenException%0Afrom katagawa.sql import Token, Aliased%0A%0A%0Aclass Select(Token):%0A    %22%22%22%0A    The base token representing a SELECT token.%0A    %22%22%22%0A%0A    def name(self):%0A        return %22SELECT%22%0A%0A    def generate_sql(self):%0A        %22%22%22%0A        Generates the SQL for the SELECT token.%0A%0A        As this is a base token, it should return the complete SQL for a full select statement.%0A        %22%22%22%0A        # Set the SELECT as the base token.%0A        generated = %22SELECT %22%0A%0A        # Filter in the subtokens.%0A        # First, we want to append every %22FIELD%22 token.%0A        fields = self.consume_tokens(%22FIELD%22)%0A%0A        # Join the fetched fields with a %60,%60.%0A        generated += %22, %22.join(%5Bfield.generate_sql() for field in fields%5D)%0A%0A        # Get the FROM clause.%0A        generated += %22 FROM %22%0A        fr = self.consume_tokens(%22FROM%22)%0A        if len(fr) %3C 1:%0A            raise MissingTokenException(%22Missing any FROM clauses in select statement%22)%0A%0A        generated += %22, %22.join(%5Bf.generate_sql(include_from=False) for f in fr%5D)%0A%0A        # Return the generated SQL.%0A        return generated%0A%0A%0Aclass Field(Aliased):%0A    %22%22%22%0A    Represents a field in a SELECT, or similar.%0A    %22%22%22%0A    __slots__ = (%22identifier%22, %22alias%22,)%0A%0A    @property%0A    def name(self):%0A        return %22FIELD%22%0A%0A    def generate_sql(self):%0A        %22%22%22%0A        Generate the SQL for a field.%0A%0A        If no alias is specified, it will just return the column name.%0A        Otherwise, it will return the column named aliased with AS.%0A        %22%22%22%0A        if self.alias is None:%0A            return self.identifier%0A%0A        else:%0A            return %22%7B%7D AS %7B%7D%22.format(self.identifier, self.alias)%0A%0A%0Aclass From(Aliased):%0A    %22%22%22%0A    Represents a FROM in a SELECT, or similar.%0A    %22%22%22%0A    @property%0A    def name(self):%0A        return %22FROM%22%0A%0A    def generate_sql(self, include_from=True):%0A        %22%22%22%0A        Generates the SQL for a FROM clause.%0A%0A        :param include_from: Include the %60FROM%60 at the beginning? Useful for multiple tables.%0A        %22%22%22%0A        if include_from:%0A            base = %22FROM %22%0A        else:%0A            base = %22%22%0A        base += self.identifier%0A%0A        if self.alias is not None:%0A            base += %22 AS %7B%7D%22.format(self.alias)%0A%0A        return base%0A

Add common tokens.