Difference between revisions of "Lytro"

From Archiveteam
Jump to navigation Jump to search
(Details about preserving additional files from the Lytro hosting shutdown which are still present on their CDN)
 
(Additional details)
Line 9: Line 9:
https://archive.org/details/lytro-hosted-partial-missing-files contains three sets of WARCs which capture many of the missing files.
https://archive.org/details/lytro-hosted-partial-missing-files contains three sets of WARCs which capture many of the missing files.


The <nowiki>lfes-not-in-ia-4.txt</nowiki> file in that item contains a list of ~1.2M URLs which are the image assets referenced in the JSON files.
The <code>lfes-not-in-ia-4.txt</code> file in that item contains a list of ~1.2M URLs which are the image assets referenced in the JSON files.


----
----
Line 31: Line 31:


  <nowiki>
  <nowiki>
>>> with open('lfe-urls-in-pictures.txt', 'r') as f:
...    lfes = f.readlines()
...
>>> len(lfes)
47045
>>> fresh = []
>>> for a in lfes:
...    b = a.strip("\"'()\n")
...    if not os.path.exists(b.split('https://')[1]):
...            fresh.append(b)
...
>>> len(fresh)
39798
>>> with open('lfes-not-in-ia-1.txt', 'w') as f:
...    f.write('\n'.join(fresh))
...
>>>
$ ~/bin/wget --ca-certificate=$HOME/Downloads/curl-7.57.0/lib/ca-bundle.crt -x --warc-file=lfes-not-in-ia-1 --warc-cdx --wait=1 --random-wait -i lfes-not-in-ia-1.txt  
$ ~/bin/wget --ca-certificate=$HOME/Downloads/curl-7.57.0/lib/ca-bundle.crt -x --warc-file=lfes-not-in-ia-1 --warc-cdx --wait=1 --random-wait -i lfes-not-in-ia-1.txt  
</nowiki>
</nowiki>
Line 62: Line 80:


  <nowiki>
  <nowiki>
>>> with open('lfe-urls-in-pictures.txt', 'r') as f:
...    lfes = f.readlines()
...
>>> len(lfes)
47045
>>> v2s = []
>>> uuids = []
>>> for a in lfes:
...    b = a.strip("\"'()\n").split('/')
...    if len(b[4]) == 36:
...            uuids.append(b[4])
...    elif b[4] == 'announce':
...            uuids.append(b[5])
...    elif b[4] == 'players':
...            pass
...    else:
...            print b
...
>>> len(uuids)
47007
>>> uuids = set(uuids)
>>> len(uuids)
10866
>>> import os
>>> for a in uuids:
...    for b in ['output.html5.json', 'output.html5_normal.json', 'output.html5_small.json', 'output.html5_tiny.json', 'player_preview.jpeg']:
...            if not os.path.exists('lfe-cdn.lytro.com/lfe/{}/v2/{}'.format(a, b)):
...                    v2s.append('https://lfe-cdn.lytro.com/lfe/{}/v2/{}'.format(a, b))
...
>>> len(v2s)
30438
>>> len(set(v2s))
30438
>>>
$ ~/bin/wget --ca-certificate=/Users/vitorio/Downloads/curl-7.57.0/lib/ca-bundle.crt -x --warc-file=lfes-not-in-ia-2 --warc-cdx --warc-max-size=1G --wait=1 --random-wait -i lfes-not-in-ia-2.txt
$ ~/bin/wget --ca-certificate=/Users/vitorio/Downloads/curl-7.57.0/lib/ca-bundle.crt -x --warc-file=lfes-not-in-ia-2 --warc-cdx --warc-max-size=1G --wait=1 --random-wait -i lfes-not-in-ia-2.txt
</nowiki>
</nowiki>
Line 72: Line 125:


  <nowiki>
  <nowiki>
>>> assets = os.listdir('lfe-cdn.lytro.com/assets')
>>> lfe = os.listdir('lfe-cdn.lytro.com/lfe')
>>> len(lfe)
10896
>>> len(assets)
34
>>> uuids = assets + lfe
>>> len(uuids)
10930
>>> uuids = [x for x in uuids if len(x) == 36]
>>> len(uuids)
10927
>>> uuids = set(uuids)
>>> len(uuids)
10894
>>> v2s = []
>>> for a in uuids:
...    for b in ['output.html5.json', 'output.html5_normal.json', 'output.html5_small.json', 'output.html5_tiny.json', 'player_preview.jpeg']:
...            if not os.path.exists('lfe-cdn.lytro.com/lfe/{}/v2/{}'.format(a, b)):
...                    v2s.append('https://lfe-cdn.lytro.com/lfe/{}/v2/{}'.format(a, b))
...
>>> len(v2s)
165
>>> with open('lfes-not-in-ia-3.txt', 'w') as f:
...    f.write('\n'.join(v2s))
...
>>>
$ ~/bin/wget --ca-certificate=/Users/vitorio/Downloads/curl-7.57.0/lib/ca-bundle.crt -x --warc-file=lfes-not-in-ia-3 --warc-cdx --warc-max-size=1G --wait=1 --random-wait -i lfes-not-in-ia-3.txt
$ ~/bin/wget --ca-certificate=/Users/vitorio/Downloads/curl-7.57.0/lib/ca-bundle.crt -x --warc-file=lfes-not-in-ia-3 --warc-cdx --warc-max-size=1G --wait=1 --random-wait -i lfes-not-in-ia-3.txt
</nowiki>
</nowiki>
Line 105: Line 186:


The filenames generated by Lytro's processes seem to be fairly unique per embed, so no point in generating URLs not represented in the JSON files.
The filenames generated by Lytro's processes seem to be fairly unique per embed, so no point in generating URLs not represented in the JSON files.
Parse all the JSON files and generate the list of image dependencies:
<nowiki>
>>> import os, nested_lookup, json, gzip
>>> urls = []
>>> assets = []
>>> for root, dirs, files in os.walk('lfe-cdn.lytro.com/lfe'):
...    for a in files:
...            if os.path.splitext(a)[1] == '.json':
...                    try:
...                            with open(os.path.join(root, a), 'r') as f:
...                                    j = json.load(f)
...                    except:
...                            print os.path.join(root, a)
...                            try:
...                                    with gzip.open(os.path.join(root, a), 'rb') as f:
...                                            j = json.load(f)
...                            except:
...                                    print 'not gzip', os.path.join(root, a)
...                                    continue
...                    imgs = nested_lookup.nested_lookup('imageUrl', j)
...                    for i in imgs:
...                            assets.append(i)
...                            urls.append('https://{}'.format(os.path.join(root, i)))
...
>>> len(urls)
1285432
>>> len(set(urls))
1285432
>>> len(assets)
1285432
>>> len(set(assets))
275454
</nowiki>

Revision as of 19:19, 17 December 2017

Lytro manufactured "light field" cameras, and offered free hosting for the exported "living picture" images on their pictures.lytro.com service. They recently discontinued the hosting service, breaking all embeds. See e.g. https://www.theverge.com/2017/12/6/16742314/lytro-focus-photos-support-cameras-illum

For an example of broken embeds, see e.g. https://www.theverge.com/2014/7/30/5949913/lytro-illum-review

Lytro had announced at one point plans to open-source their viewer, but that never happened: https://www.lytro.com/press/releases/lytro-unleashes-interactive-power-of-living-pictures-to-the-web-with-new-lytro-webgl-player

The Internet Archive Wayback Machine captured many Lytro embeds and galleries, but none of these currently work. The embedded Lytro web player references several JSON files, which IA captured, but did not parse, so all the URLs referenced in the JSON were not retrieved.

https://archive.org/details/lytro-hosted-partial-missing-files contains three sets of WARCs which capture many of the missing files.

The lfes-not-in-ia-4.txt file in that item contains a list of ~1.2M URLs which are the image assets referenced in the JSON files.


Worklog follows:

pictures.lytro.com and lfe-cdn.lytro.com were downloaded from the Wayback Machine based on the date prior to the shutdown:

$ ~/.gem/ruby/2.0.0/bin/wayback_machine_downloader https://lfe-cdn.lytro.com/ -t 20171129
$ ~/.gem/ruby/2.0.0/bin/wayback_machine_downloader https://pictures.lytro.com/ -t 20171129

A regex search inside of all of the pictures.lytro.com files to find lfe-cdn references:

["'\(]https?:\/\/lfe-cdn\.lytro\.com.*?["'\)]

Downloaded all the URLs from that list which weren't already present:

>>> with open('lfe-urls-in-pictures.txt', 'r') as f:
...     lfes = f.readlines()
... 
>>> len(lfes)
47045
>>> fresh = []
>>> for a in lfes:
...     b = a.strip("\"'()\n")
...     if not os.path.exists(b.split('https://')[1]):
...             fresh.append(b)
... 
>>> len(fresh)
39798
>>> with open('lfes-not-in-ia-1.txt', 'w') as f:
...     f.write('\n'.join(fresh))
... 
>>> 

$ ~/bin/wget --ca-certificate=$HOME/Downloads/curl-7.57.0/lib/ca-bundle.crt -x --warc-file=lfes-not-in-ia-1 --warc-cdx --wait=1 --random-wait -i lfes-not-in-ia-1.txt 

That list wasn't reduplicated, there were ~10k duplicate URLs out of ~40k. Also, a lot of 500 errors off the CDN, mostly for URLs without a "v2" in the URL. e.g. these fail:

https://lfe-cdn.lytro.com/lfe/2020944a-25a4-11e3-abd1-22000a8914f9/carousel_preview.jpg
https://lfe-cdn.lytro.com/lfe/2516a14c-25a4-11e3-9166-1231393ff52e/output.html5.json

but if I rewrite the second one to:

https://lfe-cdn.lytro.com/lfe/2516a14c-25a4-11e3-9166-1231393ff52e/v2/output.html5.json

it exists.

The v2 (I guess) player is their "new" WebGL-based one, which asks for these paths in the JS, e.g.:

LYT.PICTURE_ORIGINAL_URL = "https://lfe-cdn.lytro.com/lfe/8112c254-17f8-11e4-8fa7-22000a0d84a4/v2/output.html5.json";
LYT.PICTURE_NORMAL_URL = "https://lfe-cdn.lytro.com/lfe/8112c254-17f8-11e4-8fa7-22000a0d84a4/v2/output.html5_normal.json";
LYT.PICTURE_SMALL_URL = "https://lfe-cdn.lytro.com/lfe/8112c254-17f8-11e4-8fa7-22000a0d84a4/v2/output.html5_small.json";
LYT.PICTURE_TINY_URL = "https://lfe-cdn.lytro.com/lfe/8112c254-17f8-11e4-8fa7-22000a0d84a4/v2/output.html5_tiny.json";
LYT.PREVIEW_URL = "https://lfe-cdn.lytro.com/lfe/8112c254-17f8-11e4-8fa7-22000a0d84a4/v2/player_preview.jpeg";

Went through all the URLs and pulled out all the UUID keys and see if there are those additional paths still to be fetched.

>>> with open('lfe-urls-in-pictures.txt', 'r') as f:
...     lfes = f.readlines()
... 
>>> len(lfes)
47045
>>> v2s = []
>>> uuids = []
>>> for a in lfes:
...     b = a.strip("\"'()\n").split('/')
...     if len(b[4]) == 36:
...             uuids.append(b[4])
...     elif b[4] == 'announce':
...             uuids.append(b[5])
...     elif b[4] == 'players':
...             pass
...     else:
...             print b
... 
>>> len(uuids)
47007
>>> uuids = set(uuids)
>>> len(uuids)
10866
>>> import os
>>> for a in uuids:
...     for b in ['output.html5.json', 'output.html5_normal.json', 'output.html5_small.json', 'output.html5_tiny.json', 'player_preview.jpeg']:
...             if not os.path.exists('lfe-cdn.lytro.com/lfe/{}/v2/{}'.format(a, b)):
...                     v2s.append('https://lfe-cdn.lytro.com/lfe/{}/v2/{}'.format(a, b))
... 
>>> len(v2s)
30438
>>> len(set(v2s))
30438
>>> 

$ ~/bin/wget --ca-certificate=/Users/vitorio/Downloads/curl-7.57.0/lib/ca-bundle.crt -x --warc-file=lfes-not-in-ia-2 --warc-cdx --warc-max-size=1G --wait=1 --random-wait -i lfes-not-in-ia-2.txt

Could also back up schema.lytro.com, an S3 bucket which stores all the JSON schemas for the JSON files.

Note that the JSON files include an `asset_base` URL reference to lfe-cdn.lytro.com, but that isn't actually part of the JSON schema, and it doesn't appear to be checked by the last version of the player JS, so no need to rewrite it.

Now that we have all the UUIDs, let's check all the folders in the CDN directory to make sure we haven't missed any JSON downloads.

>>> assets = os.listdir('lfe-cdn.lytro.com/assets')
>>> lfe = os.listdir('lfe-cdn.lytro.com/lfe')
>>> len(lfe)
10896
>>> len(assets)
34
>>> uuids = assets + lfe
>>> len(uuids)
10930
>>> uuids = [x for x in uuids if len(x) == 36]
>>> len(uuids)
10927
>>> uuids = set(uuids)
>>> len(uuids)
10894
>>> v2s = []
>>> for a in uuids:
...     for b in ['output.html5.json', 'output.html5_normal.json', 'output.html5_small.json', 'output.html5_tiny.json', 'player_preview.jpeg']:
...             if not os.path.exists('lfe-cdn.lytro.com/lfe/{}/v2/{}'.format(a, b)):
...                     v2s.append('https://lfe-cdn.lytro.com/lfe/{}/v2/{}'.format(a, b))
... 
>>> len(v2s)
165
>>> with open('lfes-not-in-ia-3.txt', 'w') as f:
...     f.write('\n'.join(v2s))
... 
>>>

$ ~/bin/wget --ca-certificate=/Users/vitorio/Downloads/curl-7.57.0/lib/ca-bundle.crt -x --warc-file=lfes-not-in-ia-3 --warc-cdx --warc-max-size=1G --wait=1 --random-wait -i lfes-not-in-ia-3.txt

At least some of the JSON files were served by IA erroneously without being gzip decompressed:

lfe-cdn.lytro.com/lfe/02c39bca-1146-11e4-a2b8-22000ab80aeb/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/0f16b1d2-1128-11e4-9af7-22000a8a0b0d/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/175c8c20-1206-11e4-970d-22000ab80aeb/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/1e4f739e-1206-11e4-970d-22000ab80aeb/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/28ad3d90-112e-11e4-be98-22000a2d8f66/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/36066900-1131-11e4-8bd7-22000a2d8f66/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/493f6fc8-1201-11e4-bfe5-22000a2b9e9d/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/8a4b167c-1132-11e4-95dc-22000a2d8f66/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/8bb527ea-1133-11e4-8bd7-22000a2d8f66/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/90a1a35a-1200-11e4-9a06-22000a2b9e9d/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/90d88df2-1200-11e4-bfe5-22000a2b9e9d/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/91427488-1200-11e4-a093-22000ab80aeb/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/91f617fe-1133-11e4-9af7-22000a8a0b0d/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/926eb8c0-1139-11e4-84d3-22000a2b9e9d/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/92bb1d64-1139-11e4-b404-22000a2b9e9d/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/9664811e-1204-11e4-9573-22000a2b9e9d/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/96a5e7ee-1204-11e4-bfe5-22000a2b9e9d/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/970dea7e-1204-11e4-b34f-22000a2b9e9d/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/9728bc6e-1204-11e4-b8c1-22000ab80aeb/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/9b00b186-112b-11e4-966b-22000a2d8f66/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/a1ea6af6-11fc-11e4-b258-22000ab80aeb/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/db00af6a-0ed9-11e4-8a95-22000a4184f0/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/f1c74502-0ed8-11e4-967c-12313b04e812/v2/output.html5_small.json
lfe-cdn.lytro.com/lfe/f2509528-0ed8-11e4-8023-12313b04e812/v2/output.html5_small.json

The filenames generated by Lytro's processes seem to be fairly unique per embed, so no point in generating URLs not represented in the JSON files.

Parse all the JSON files and generate the list of image dependencies:

>>> import os, nested_lookup, json, gzip
>>> urls = []
>>> assets = []
>>> for root, dirs, files in os.walk('lfe-cdn.lytro.com/lfe'):
...     for a in files:
...             if os.path.splitext(a)[1] == '.json':
...                     try:
...                             with open(os.path.join(root, a), 'r') as f:
...                                     j = json.load(f)
...                     except:
...                             print os.path.join(root, a)
...                             try:
...                                     with gzip.open(os.path.join(root, a), 'rb') as f:
...                                             j = json.load(f)
...                             except:
...                                     print 'not gzip', os.path.join(root, a)
...                                     continue
...                     imgs = nested_lookup.nested_lookup('imageUrl', j)
...                     for i in imgs:
...                             assets.append(i)
...                             urls.append('https://{}'.format(os.path.join(root, i)))
... 
>>> len(urls)
1285432
>>> len(set(urls))
1285432
>>> len(assets)
1285432
>>> len(set(assets))
275454