22 lines
727 B
SQL
22 lines
727 B
SQL
ALTER TABLE crawl_runs
|
|
ADD COLUMN IF NOT EXISTS skipped_count INTEGER NOT NULL DEFAULT 0;
|
|
|
|
CREATE TABLE IF NOT EXISTS parse_resource_cache (
|
|
id SERIAL PRIMARY KEY,
|
|
profile_key VARCHAR(255) NOT NULL,
|
|
resource_key VARCHAR(255) NOT NULL,
|
|
method VARCHAR(16) NOT NULL,
|
|
url TEXT NOT NULL,
|
|
request_fingerprint VARCHAR(64) NOT NULL,
|
|
etag TEXT,
|
|
last_modified TEXT,
|
|
body_hash VARCHAR(64) NOT NULL,
|
|
body_snapshot BYTEA NOT NULL,
|
|
parser_version VARCHAR(32),
|
|
fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
CONSTRAINT uq_parse_resource_cache_resource UNIQUE (profile_key, resource_key, request_fingerprint)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS ix_parse_resource_cache_profile_key
|
|
ON parse_resource_cache (profile_key);
|