De-identify a dataset and measure what each method leaves behind

Verify

python3 - <<'PY'
import csv, os, stat, collections
rd = lambda p: list(csv.DictReader(open(p)))
orig = rd('/tmp/deid/people.csv')
cards = {r['card'] for r in orig}
masked = open('/tmp/deid/masked.csv').read()
tok = open('/tmp/deid/tokenised.csv').read()
print('full card numbers left in masked   :', sum(c in masked for c in cards))
print('full card numbers left in tokenised:', sum(c in tok for c in cards))
assert not any(c in masked for c in cards), 'masking left a full card number'
assert not any(c in tok for c in cards), 'tokenisation left a full card number'
vault = dict(csv.reader(open('/tmp/deid/vault/vault.csv')))
assert all(r['card'] in vault for r in rd('/tmp/deid/tokenised.csv')), 'a token the vault cannot resolve'
mode = stat.S_IMODE(os.stat('/tmp/deid/vault/vault.csv').st_mode)
print('vault permissions: %o' % mode)
assert mode & 0o077 == 0, 'the vault is readable by other accounts'
tr = rd('/tmp/deid/transposed.csv')
assert sorted(r['salary'] for r in tr) == sorted(r['salary'] for r in orig), 'transposition changed the values'
same = sum(a['salary'] == b['salary'] for a, b in zip(orig, tr)) / len(orig)
print('rows still holding their own salary: %.0f%%' % (same * 100))
assert same < 0.2, 'the salary column was not really shuffled'
def unique(rows):
    key = lambda r: (r['postcode'], r['birth_date'], r['sex'])
    c = collections.Counter(key(r) for r in rows)
    return sum(1 for r in rows if c[key(r)] == 1)
before, after = unique(orig), unique(rd('/tmp/deid/deid.csv'))
print('people unique on postcode + birth date + sex: before', before, 'after', after)
assert after < before, 'generalising did not reduce re-identification risk'
PY

Every assertion must pass. Read the printed numbers as well as the verdict: transposition passes precisely because every real salary is still in the file, and the uniqueness count after de-identification is rarely zero. Those two lines are the residue the lesson says each method leaves behind.