# -*- coding: utf-8 -*-
"""
@author: aeidelman
"""

import os
import pandas as pd

path = '/home/sgmap/data/SIRENE'

# avantage
path_file = os.path.join(path, 
                         'sirc-266_266_13705_201606_L_P_20161010_121909418-secondtry.csv')

tab = pd.read_csv(path_file, sep=';', nrows=1000,
                  encoding='cp1252'
#                 usecols = categorical_vars + ['avant_montant_ttc']
                 )

assert all(tab.APET700.str.len() == 5)
public = tab[tab.APET700.str[:2] == '84']
print(len(public))

xxx
     
#av['year'] = av['avant_date_signature'].str[6:]
iter_csv = pd.read_csv(path_file,  sep=';',
                       encoding='cp1252',
                       iterator=True, 
                       chunksize=100000)

df = pd.concat([tab[tab.APET700.str[:2] == '84'] 
    for tab in iter_csv])

path_output = os.path.join(path, 'sirene84.csv')
df.to_csv(path_output,  sep=';',
                       encoding='cp1252',
                       iterator=True, 
                       index=False)
@@ -293,16 +293,18 @@
 ows=1000
+00
 ,%0A      
@@ -332,16 +332,30 @@
 'cp1252'
+, #iso-8859-15
 %0A#      
@@ -475,16 +475,19 @@
 ) == 5)%0A
+%0A# 
 public =
@@ -498,34 +498,44 @@
 %5Btab
-.APET700
+%5B'NJ'%5D.astype(str)
 .str%5B
-:2
+0
 %5D == '
-84
+7
 '%5D%0A
+# 
 prin
@@ -554,12 +554,8 @@
 ))%0A%0A
-xxx%0A
     
@@ -799,31 +799,39 @@
 %5Btab
-.APET700
+%5B'NJ'%5D.astype(str)
 .str%5B
-:2
+0
 %5D == '
-84
+7
 '%5D %0A
@@ -901,10 +901,13 @@
 rene
-84
+_NJ_7
 .csv

use NJ