@@ -0,0 +1,2896 @@
+# Copyright 2020 Google LLC%0A#%0A# Licensed under the Apache License, Version 2.0 (the %22License%22);%0A# you may not use this file except in compliance with the License.%0A# You may obtain a copy of the License at%0A#%0A#      http://www.apache.org/licenses/LICENSE-2.0%0A#%0A# Unless required by applicable law or agreed to in writing, software%0A# distributed under the License is distributed on an %22AS IS%22 BASIS,%0A# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.%0A# See the License for the specific language governing permissions and%0A# limitations under the License.%0A%0Aimport json%0Aimport uuid%0Afrom pathlib import Path%0Afrom typing import Any, Callable, Dict, List%0A%0Aimport requests%0Afrom pandas import DataFrame%0A%0Afrom lib.data_source import DataSource%0A%0A%0Adef _download_arcgis(%0A    url: str, offset: int = 0, log_func: Callable%5B%5Bstr%5D, None%5D = None%0A) -%3E List%5BDict%5Bstr, Any%5D%5D:%0A    %22%22%22%0A    Recursively download all records from an ArcGIS data source respecting the maximum record%0A    transfer per request.%0A    %22%22%22%0A    url_tpl = url + %22&resultOffset=%7Boffset%7D%22%0A%0A    try:%0A        res = requests.get(url_tpl.format(offset=offset)).json()%5B%22features%22%5D%0A    except Exception as exc:%0A        if log_func:%0A            log_func(requests.get(url_tpl.format(offset=offset)).text)%0A        raise exc%0A%0A    rows = %5Brow%5B%22attributes%22%5D for row in res%5D%0A    if len(rows) == 0:%0A        return rows%0A    else:%0A        return rows + _download_arcgis(url, offset=offset + len(rows))%0A%0A%0Aclass ArcGISDataSource(DataSource):%0A    def fetch(%0A        self, output_folder: Path, cache: Dict%5Bstr, str%5D, fetch_opts: List%5BDict%5Bstr, Any%5D%5D%0A    ) -%3E Dict%5Bstr, str%5D:%0A        downloaded_files = %7B%7D%0A%0A        for idx, opts in enumerate(fetch_opts):%0A            # Base URL comes from fetch_opts%0A            url_base = opts%5B%22url%22%5D%0A%0A            # Create a deterministic file name%0A            file_path = (%0A                output_folder%0A                / %22snapshot%22%0A                / (%22%25s.%25s%22 %25 (uuid.uuid5(uuid.NAMESPACE_DNS, url_base), %22json%22))%0A            )%0A%0A            # Avoid download if the file exists and flag is set%0A            skip_existing = opts.get(%22opts%22, %7B%7D).get(%22skip_existing%22)%0A            if not skip_existing or not file_path.exists():%0A                with open(file_path, %22w%22) as fd:%0A                    json.dump(%7B%22features%22: _download_arcgis(url_base)%7D, fd)%0A%0A            # Add downloaded file to the list%0A            downloaded_files%5Bopts.get(%22name%22, idx)%5D = str(file_path.absolute())%0A%0A        return downloaded_files%0A%0A    def parse(self, sources: Dict%5Bstr, str%5D, aux: Dict%5Bstr, DataFrame%5D, **parse_opts) -%3E DataFrame:%0A        dataframes = %7B%7D%0A        for name, file_path in sources.items():%0A            with open(file_path, %22r%22) as fd:%0A                records = json.load(fd)%5B%22features%22%5D%0A                dataframes%5Bname%5D = DataFrame.from_records(records)%0A        return self.parse_dataframes(dataframes, aux, **parse_opts)%0A

Add ArcGIS data source util