What is vector quantisation and how does pgvector support binary quantisation?

-- Binary quantisation: convert float vectors to binary (0/1 per dimension)
-- Each dimension becomes 1 bit instead of 32 bits = 32x compression!

-- Store binary quantised vectors:
CREATE TABLE items_binary (
    id              BIGSERIAL   PRIMARY KEY,
    content         TEXT,
    embedding       VECTOR(1536),    -- keep original for reranking
    embedding_bin   BIT(1536)        -- binary quantised for fast coarse search
);

-- Quantise: positive values -> 1, negative/zero -> 0
UPDATE items_binary
SET embedding_bin = (
    SELECT string_agg(
        CASE WHEN val > 0 THEN '1' ELSE '0' END, ''
        ORDER BY ordinality
    )::bit(1536)
    FROM unnest(embedding::float4[]) WITH ORDINALITY AS t(val, ordinality)
);

-- Two-stage retrieval with binary quantisation:
-- Stage 1: Fast coarse search on binary vectors (Hamming distance)
CREATE INDEX ON items_binary USING hnsw (embedding_bin bit_hamming_ops);

WITH candidates AS (
    SELECT id, embedding_bin <~> '[...]'::bit(1536) AS hamming_dist
    FROM items_binary
    ORDER BY hamming_dist
    LIMIT 100   -- get 100 candidates quickly from binary index
)
-- Stage 2: Rerank top candidates using full-precision cosine distance
SELECT i.id, i.content,
       i.embedding <=> '[...]' AS cosine_dist
FROM items_binary i
JOIN candidates c ON i.id = c.id
ORDER BY cosine_dist
LIMIT 5;  -- return final top-5 after precise reranking