@@ -0,0 +1,2270 @@
+#!/usr/bin/env python3%0A# -*- coding: UTF-8 -*-%0A%0A# Copyright 2017 Eddie Antonio Santos %3Ceasantos@ualberta.ca%3E%0A#%0A# Licensed under the Apache License, Version 2.0 (the %22License%22);%0A# you may not use this file except in compliance with the License.%0A# You may obtain a copy of the License at%0A#%0A#   http://www.apache.org/licenses/LICENSE-2.0%0A#%0A# Unless required by applicable law or agreed to in writing, software%0A# distributed under the License is distributed on an %22AS IS%22 BASIS,%0A# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.%0A# See the License for the specific language governing permissions and%0A# limitations under the License.%0A%0Afrom typing import Sequence, Optional%0A%0Afrom .corpus import Corpus%0Afrom .token_sequence import TokenSequence%0Afrom .token_utils import Token%0Afrom .vectors import Vectors%0Afrom .tokenize_js import tokenize%0A%0A%0Aclass SourceFile:%0A    %22%22%22%0A    A source file from the corpus. Easy access to both the token sequence, and%0A    the source tokens themselves.%0A    %22%22%22%0A    corpus: Optional%5BCorpus%5D = None%0A    vectors: Optional%5BVectors%5D = None%0A%0A    __slots__ = ('filehash', '_token_vector', '_source_tokens')%0A%0A    def __init__(self, filehash: str) -%3E None:%0A        self.filehash = filehash%0A        # I used confusing names... TokenSequence != Sequence%5BToken%5D...%0A        self._token_vector: Optional%5BTokenSequence%5D = None%0A        self._source_tokens: Optional%5BSequence%5BToken%5D%5D = None%0A%0A    @property%0A    def vector(self) -%3E TokenSequence:%0A        if self._token_vector is not None:%0A            return self._token_vector%0A        if self.vectors is None:%0A            raise Exception('forgot to assign SourceFile.vectors')%0A        # Fetch the vector.%0A        _, tokens = self.vectors%5Bself.filehash%5D%0A        self._token_vector = TokenSequence(tokens)%0A        return self.vector%0A%0A    @property%0A    def source(self) -%3E Sequence%5BToken%5D:%0A        if self._source_tokens is not None:%0A            return self._source_tokens%0A        if self.corpus is None:%0A            raise Exception('forgot to assign SourceFile.corpus')%0A        # Fetch the vector.%0A        source = self.corpus.get_source(self.filehash)%0A        self._source_tokens = tokenize(source.decode('UTF-8'))%0A        return self.source%0A%0A    def line_of_token(self, index: int) -%3E int: ...%0A

Create SourceFile class.