#!/usr/bin/env python
# -*- encoding: utf-8

import boto3
import tqdm


OLD_TABLE = 'SourceData'
OLD_BUCKET = 'wellcomecollection-vhs-sourcedata'

NEW_TABLE = 'vhs-sourcedata-miro'
NEW_BUCKET = 'wellcomecollection-vhs-sourcedata-miro'


def all_records(dynamodb_client):
    paginator = dynamodb_client.get_paginator('scan')
    for page in paginator.paginate(TableName=OLD_TABLE):
        for item in page['Items']:
            yield item


def get_existing_records(dynamodb_client):
    """
    Generates existing Miro records from the SourceData table.
    """
    for item in tqdm.tqdm(all_records(dynamodb_client), total=2039416):
        if 'reindexShard' not in item:
            print(item)

        if item['sourceName'] != {'S': 'miro'}:
            continue
        yield item


if __name__ == '__main__':
    dynamodb_client = boto3.client('dynamodb')
    s3_client = boto3.client('s3')

    for item in get_existing_records(dynamodb_client):
        del item['sourceName']
        del item['sourceId']
        del item['reindexVersion']
        item['version']['N'] = '1'

        old_key = item['s3key']['S']
        new_key = old_key.replace('miro/', '')
        del item['s3key']

        s3_client.copy_object(
            Bucket=NEW_BUCKET,
            Key=new_key,
            CopySource={
                'Bucket': OLD_BUCKET,
                'Key': old_key
            }
        )

        item['location'] = {
            'M': {
                'namespace': {'S': NEW_BUCKET},
                'key': {'S': new_key}
            }
        }

        dynamodb_client.put_item(
            TableName=NEW_TABLE,
            Item=item
        )

@@ -38,16 +38,73 @@
  utf-8%0A%0A
+import json%0A%0Afrom botocore.exceptions import ClientError%0A
 import b
@@ -618,28 +618,28 @@
 %22%22%22%0A    for 
-item
+line
  in tqdm.tqd
@@ -644,53 +644,79 @@
 qdm(
-all_records(dynamodb_client), total=2039416):
+open('sourcedata.txt'), total=2039416):%0A        item = json.loads(line)
 %0A   
@@ -1273,16 +1273,247 @@
 3key'%5D%0A%0A
+        try:%0A            s3_client.head_object(%0A                Bucket=NEW_BUCKET,%0A                Key=new_key + 'xxx'%0A            )%0A        except ClientError as err:%0A            if err.response%5B'Error'%5D%5B'Code'%5D == '404':%0A        
         
@@ -1527,32 +1527,40 @@
 nt.copy_object(%0A
+        
             Buck
@@ -1578,32 +1578,40 @@
 ET,%0A            
+        
 Key=new_key,%0A   
@@ -1619,16 +1619,24 @@
         
+  
+      
 CopySour
@@ -1640,16 +1640,24 @@
 ource=%7B%0A
+        
         
@@ -1702,16 +1702,24 @@
         
+       
+ 
 'Key': o
@@ -1741,19 +1741,75 @@
     
-%7D%0A        )
+        %7D%0A                )%0A            else:%0A                raise
 %0A%0A  
@@ -1965,16 +1965,170 @@
      %7D%0A%0A
+        resp = dynamodb_client.get_item(%0A            TableName=NEW_TABLE,%0A            Key=%7B'id': item%5B'id'%5D%7D%0A        )%0A        if 'Item' not in resp:%0A    
         
@@ -2145,32 +2145,36 @@
 lient.put_item(%0A
+    
             Tabl
@@ -2202,16 +2202,20 @@
         
+   
+ 
 Item=ite
@@ -2216,18 +2216,192 @@
 em=item%0A        
-)
+    )%0A%0A            resp = dynamodb_client.get_item(%0A                TableName=NEW_TABLE,%0A                Key=%7B'id': item%5B'id'%5D%7D%0A            )%0A            assert 'Item' in resp
 %0A

Add some logic to support resuming