#!/usr/bin/env python3

# update/oui.py - script to download and parse data from the IEEE registry
#
# Copyright (C) 2018-2019 Arthur de Jong
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA

"""This script downloads data from the IEEE web site
https://regauth.standards.ieee.org/standards-ra-web/pub/view.html
and produces data files that can be use by python-stdnum to look up
manufacturers by MAC address."""

import csv
from collections import defaultdict
from itertools import chain

import requests


# The URLs of the MA-L, MA-M and MA-S registries that are downloaded to
# construct a full list of manufacturer prefixes.
mal_url = 'http://standards-oui.ieee.org/oui/oui.csv'
mam_url = 'http://standards-oui.ieee.org/oui28/mam.csv'
mas_url = 'http://standards-oui.ieee.org/oui36/oui36.csv'


def download_csv(url):
    """Download the list from the site and provide assignment and
    organisation names."""
    response = requests.get(url)
    response.raise_for_status()
    for row in csv.DictReader(line.decode('utf-8') for line in response.iter_lines()):
        yield (
            row['Assignment'],
            row['Organization Name'].strip().replace('"', '%'))


if __name__ == '__main__':
    # download the MAC Address Block Large (MA-L) list
    toplevel = defaultdict(list)
    for a, o in download_csv(mal_url):
        toplevel[o].append(a)
    # download the MAC Address Block Medium (MA-M) and Small lists
    nested = defaultdict(dict)
    for a, o in chain(download_csv(mam_url), download_csv(mas_url)):
        nested[a[:6]][a[6:]] = o
    # Generate output
    print('# list of IEEE MAC Address Block registry entries')
    print('# %s' % mal_url)
    print('# %s' % mam_url)
    print('# %s' % mas_url)
    for a, o in sorted((tuple(sorted(a)), o) for o, a in toplevel.items()):
        if o not in ('IEEE Registration Authority', 'Private'):
            print('%s o="%s"' % (','.join(a), o))
    for a in sorted(nested.keys()):
        print('%s' % a)
        for s, o in sorted(nested[a].items()):
            if o not in ('IEEE Registration Authority', 'Private'):
                print(' %s o="%s"' % (s, o))

@@ -1753,15 +1753,126 @@
     
-yield (
+o = row%5B'Organization Name'%5D.strip().replace('%22', '%25')%0A        if o not in ('IEEE Registration Authority', 'Private'):
 %0A   
@@ -1876,24 +1876,31 @@
             
+yield (
 row%5B'Assignm
@@ -1909,71 +1909,716 @@
 t'%5D,
-%0A            row%5B'Organization Name'%5D.strip().replace('%22', '%25')
+ o)%0A%0A%0Adef join_items(items):%0A    %22%22%22Join the list of items, combining consecutive numbers.%22%22%22%0A    length = len(items%5B0%5D)%0A    items = %5Bint(b, 16) for b in items%5D%0A    first = None%0A    prev = None%0A    res = ''%0A    for item in items:%0A        if first is not None and item == prev + 1:%0A            # this item is consecutive to the previous: make a range%0A            if prev %3E first:%0A                # replace the previous value%0A                res = res%5B:-length - 1%5D%0A            res += '-%25%250%25dX' %25 length %25 item%0A            prev = item%0A        else:%0A            # this is a new item, add a new one to the list%0A            res += ',%25%250%25dX' %25 length %25 item%0A            first = prev = item%0A    return res.strip(','
 )%0A%0A%0A
@@ -2694,24 +2694,41 @@
  (MA-L) list
+ and group by org
 %0A    topleve
@@ -3183,24 +3183,61 @@
  %25 mas_url)%0A
+    # output full-length assignments%0A
     for a, o
@@ -3312,113 +3312,92 @@
     
-if o not in ('IEEE Registration Authority', 'Private'):%0A            print('%25s o=%22%25s%22' %25 (','.join(a), o))
+print('%25s o=%22%25s%22' %25 (join_items(a), o))%0A    # output assignments that are subdivided
 %0A   
@@ -3504,80 +3504,8 @@
 )):%0A
-            if o not in ('IEEE Registration Authority', 'Private'):%0A    
     

Make the IEEE OUI data more compact