# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import subprocess
import pandas as pd
from datetime import datetime


def download_manual_routeviews():
  # 2018-07-27 is the earliest scan file date
  datelist = pd.date_range(start="2018-07-27", end=datetime.today().isoformat())

  for date in datelist:
    year, month, day = date.isoformat()[:10].split("-")
    print(year, month, day)
    path = "http://data.caida.org/datasets/routing/routeviews-prefix2as/" + year + "/" + month + "/"
    # possible times are 0000 to 2200 in intervals of 200
    times = [
        "0000", "0200", "0400", "0600", "0800", "1000", "1200", "1400", "1600",
        "1800", "2000", "2200"
    ]
    for time in times:
      try:
        filename = "routeviews-rv2-" + year + month + day + "-" + time + ".pfx2as.gz"

        print(path + filename)

        subprocess.run(["wget", path + filename, "-P", "/tmp/"])
        subprocess.run([
            "gsutil", "cp", "/tmp/" + filename,
            "gs://censoredplanet_geolocation/caida/routeviews/" + filename
        ])
      except:
        pass


if __name__ == "__main__":
  download_manual_routeviews()

@@ -573,348 +573,530 @@
 e.%0A%0A
-import subprocess%0Aimport pandas as pd%0Afrom datetime import datetime%0A%0A%0Adef download_manual_routeviews():%0A  # 2018-07-27 is the earliest scan file date%0A  datelist = pd.date_range(start=%222018-07-27%22, end=datetime.today().isoformat())%0A%0A  for date in datelist:%0A    year, month, day = date.isoformat()%5B:10%5D.split(%22-%22)%0A    print(year, month, day
+from datetime import datetime%0A%0Afrom google.cloud import storage%0Aimport httpio%0Aimport pandas as pd%0Aimport requests%0A%0Aearliest_data = %222018-07-27%22%0A%0A%0Adef download_manual_routeviews():%0A  datelist = pd.date_range(%0A      start=earliest_data, end=datetime.today().isoformat())%0A  client = storage.Client()%0A  bucket = client.get_bucket(%22censoredplanet_geolocation%22)%0A%0A  for date in datelist:%0A    print(%22checking date %7B%7D%22.format(date))%0A%0A    year = str(date.year)%0A    month = str(date.month).zfill(2)%0A    day = str(date.day).zfill(2
 )%0A
+%0A
     
@@ -1167,36 +1167,44 @@
 2as/
-%22 + year + %22/%22 + month + %22/%22
+%7B%7D/%7B%7D/%22.format(%0A        year, month)
 %0A   
@@ -1462,78 +1462,147 @@
 rv2-
-%22 +
+%7B%7D%7B%7D%7B%7D-%7B%7D.pfx2as.gz%22.format(%0A           
  year
- +
+,
  month
- +
+,
  day
- + %22-%22 + time + %22.pfx2as.gz%22%0A%0A        print(path
+, time)%0A        url = path + filename%0A        cloud_filepath = %22caida/routeviews/%22
  + f
@@ -1608,17 +1608,16 @@
 filename
-)
 %0A%0A      
@@ -1622,137 +1622,233 @@
     
-subprocess.run(%5B%22wget%22, path + filename, %22-P%22, %22/tmp/%22%5D)%0A        subprocess.run(%5B%0A            %22gsutil%22, %22cp%22, %22/tmp/%22 + filename,
+# This call will fail for most urls,%0A        # since we don't know which timestamp is correct.%0A        # In that case we just move on to our next guess.%0A        f = httpio.open(url)%0A%0A        print(%22mirroring %7B%7D to %7B%7D%22.format(
 %0A   
@@ -1855,16 +1855,21 @@
         
+ url,
  %22gs://c
@@ -1898,75 +1898,217 @@
 ion/
-caida/routeviews/%22 + filename%0A        %5D)%0A      except:%0A        pass
+%22 + cloud_filepath))%0A%0A        blob = bucket.blob(cloud_filepath)%0A        blob.upload_from_file(f)%0A      except requests.exceptions.HTTPError as ex:%0A        if ex.response.status_code != 404:%0A          raise ex
 %0A%0A%0Ai

clean up bulk_upload and don't store full file in memory