# -*- encoding: utf-8
"""
Snippet.

AWLC: When I'm trying to diagnose failures on the transformer queue,
I save the queue contents with sqs_freezeray [1], and then I want to
go through them to analyse the failures.

Our transformer messages contain pointers to S3, not the records themselves.
This snippet gets messages from the frozen SQS output, extracts the
S3 pointers, and yields the contents of the resulting objects.

Copy + paste this into a Jupyter notebook/script to use.

Usage:

    >>> for obj in get_transformer_objects():
    ...    print(obj)
    {"sourceId": "123", "sourceName": "sierra", ...}
    {"sourceId": "456", "sourceName": "sierra", ...}
    {"sourceId": "789", "sourceName": "sierra", ...}

[1]: https://github.com/wellcometrust/dockerfiles/tree/master/sqs_freezeray

"""


def get_transformer_objects(key=None):
    import json
    import boto3

    s3 = boto3.client('s3')

    if key is None:
        resp = s3.list_objects_v2(
            Bucket='wellcomecollection-platform-infra',
            Prefix='sqs'
        )
        possible_keys = [r['Key'] for r in resp['Contents']]
        key = max(possible_keys)

    if not key.startswith('sqs/'):
        key = f'sqs/{key}'

    data = s3.get_object(
        Bucket='wellcomecollection-platform-infra',
        Key=key
    )['Body'].read()

    jl = json.loads

    for line in data.splitlines():
        s3key = jl(jl(jl(line)['Body'])['Message'])['s3key']
        s3obj = s3.get_object(
            Bucket='wellcomecollection-vhs-sourcedata',
            Key=s3key
        )
        yield s3obj['Body'].read()

@@ -496,24 +496,18 @@
 %3E%3E%3E 
-for obj in get_t
+gto = GetT
 rans
@@ -516,18 +516,17 @@
 rmer
-_o
+O
 bjects()
 :%0A  
@@ -521,17 +521,39 @@
 bjects()
-:
+%0A    %3E%3E%3E for obj in gto
 %0A    ...
@@ -567,24 +567,25 @@
 nt(obj)%0A    
+'
 %7B%22sourceId%22:
@@ -612,37 +612,39 @@
 : %22sierra%22, ...%7D
+'
 %0A    
+'
 %7B%22sourceId%22: %2245
@@ -675,21 +675,23 @@
 a%22, ...%7D
+'
 %0A    
+'
 %7B%22source
@@ -730,16 +730,17 @@
 a%22, ...%7D
+'
 %0A%0A%5B1%5D: h
@@ -820,17 +820,18 @@
 %22%0A%0A%0A
-def get_t
+class GetT
 rans
@@ -840,28 +840,26 @@
 rmer
-_o
+O
 bjects(
-key=None
+object
 ):%0A
+%0A
     
@@ -916,17 +916,75 @@
 t('s3')%0A
-%0A
+    jl = json.loads%0A%0A    def __init__(self, key=None):%0A    
     if k
@@ -1003,16 +1003,20 @@
         
+  
+  
 resp = s
@@ -1034,16 +1034,20 @@
 cts_v2(%0A
+    
         
@@ -1106,16 +1106,20 @@
         
+    
 Prefix='
@@ -1123,34 +1123,42 @@
 x='sqs'%0A        
+  
-)%0A
+  )%0A    
         possible
@@ -1206,24 +1206,28 @@
 '%5D%5D%0A        
+    
 key = max(po
@@ -1244,16 +1244,20 @@
 s)%0A%0A    
+  
+  
 if not k
@@ -1283,24 +1283,28 @@
 '):%0A        
+    
 key = f'sqs/
@@ -1315,16 +1315,25 @@
 %7D'%0A%0A    
+    self.
 data = s
@@ -1338,32 +1338,36 @@
  s3.get_object(%0A
+    
         Bucket='
@@ -1406,24 +1406,28 @@
 a',%0A        
+    
 Key=key%0A    
@@ -1426,16 +1426,20 @@
 key%0A    
+ 
+   
 )%5B'Body'
@@ -1452,33 +1452,77 @@
 ()%0A%0A    
-jl = json.loads%0A%0A
+    self.cached_s3_records = %7B%7D%0A%0A    def __iter__(self):%0A    
     for 
@@ -1548,16 +1548,20 @@
 ines():%0A
+    
         
@@ -1613,24 +1613,25 @@
 s3key'%5D%0A
+%0A
         
 s3obj = 
@@ -1622,21 +1622,145 @@
         
-s3obj
+    try:%0A                self.cached_s3_records%5Bs3key%5D%0A            except KeyError:%0A                self.cached_s3_records%5Bs3key%5D
  = s3.ge
@@ -1761,32 +1761,40 @@
  s3.get_object(%0A
+        
             Buck
@@ -1841,24 +1841,32 @@
             
+        
 Key=s3key%0A  
@@ -1871,26 +1871,24 @@
         
-)%0A
         
 yield s3
@@ -1883,19 +1883,9 @@
     
-yield s3obj
+)
 %5B'Bo
@@ -1876,28 +1876,77 @@
            )%5B'Body'%5D.read()%0A
+%0A            yield self.cached_s3_records%5Bs3key%5D%0A

Use the version with caching