Find Near-Duplicate Rows

import json
import re

def normalize(value):
    return re.sub(r'\s+', ' ', str(value or '').strip()).lower()

def levenshtein(a, b):
    previous = list(range(len(b) + 1))
    for i, ca in enumerate(a, 1):
        current = [i]
        for j, cb in enumerate(b, 1):
            current.append(min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (ca != cb)))
        previous = current
    return previous[-1]

def similarity(a, b):
    longest = max(len(a), len(b))
    return 1.0 if longest == 0 else 1 - levenshtein(a, b) / longest

def find_fuzzy_duplicates(rows, key_columns, threshold=0.9):
    keys = [None if all(row.get(c) is None for c in key_columns)
            else '␟'.join(normalize(row.get(c)) for c in key_columns) for row in rows]
    groups = {}
    for key in [k for k in keys if k is not None]:
        if key in groups:
            continue
        match = next((g for g in groups if similarity(g, key) >= threshold), None)
        groups[key] = groups[match] if match else len(groups)
    sizes = {}
    for key in keys:
        if key is not None:
            sizes[groups[key]] = sizes.get(groups[key], 0) + 1
    numbers = {}
    result = []
    for key in keys:
        if key is None or sizes[groups[key]] < 2:
            result.append(None)
            continue
        group = groups[key]
        numbers.setdefault(group, len(numbers) + 1)
        result.append(numbers[group])
    return result

rows = [
    {'name': 'สมชาย ใจดี'},
    {'name': 'สมชาย  ใจดี '},
    {'name': 'Somchai Jaidee'},
    {'name': None},
]
print(json.dumps(find_fuzzy_duplicates(rows, ['name']), ensure_ascii=False))