~boringcactus/arewe1.0yet

a234c2c992835bd74abdd6dbd8022250e5002ead — Melody Horn 1 year, 2 months ago 5d92bd3
don't make assumptions about tarball order
1 files changed, 4 insertions(+), 3 deletions(-)

M update.py
M update.py => update.py +4 -3
@@ 1,3 1,4 @@
from collections import defaultdict
import csv
from dataclasses import dataclass
import datetime


@@ 83,7 84,7 @@ if not dump_tarball.exists():

csv.field_size_limit(69696969)
dump = tarfile.open(dump_tarball)
crates = dict()
crates = defaultdict(lambda: Crate('', -1))
metadata = None
for item in dump:
    if item.name.endswith('metadata.json'):


@@ 91,9 92,9 @@ for item in dump:
    elif item.name.endswith('crates.csv'):
        reader = csv.DictReader(io.TextIOWrapper(dump.extractfile(item), 'UTF-8'))
        for crate in reader:
            crates[crate['id']] = Crate(crate['name'], int(crate['downloads']))
            crates[crate['id']].name = crate['name']
            crates[crate['id']].downloads = int(crate['downloads'])
    elif item.name.endswith('versions.csv'):
        assert len(crates) > 0, "versions read before crates!"
        reader = csv.DictReader(io.TextIOWrapper(dump.extractfile(item), 'UTF-8'))
        for version in reader:
            if version['yanked'] == 't':