feat: adds crawl resource cache
This commit is contained in:
21
migrations/005_parse_resource_cache.sql
Normal file
21
migrations/005_parse_resource_cache.sql
Normal file
@@ -0,0 +1,21 @@
|
||||
ALTER TABLE crawl_runs
|
||||
ADD COLUMN IF NOT EXISTS skipped_count INTEGER NOT NULL DEFAULT 0;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS parse_resource_cache (
|
||||
id SERIAL PRIMARY KEY,
|
||||
profile_key VARCHAR(255) NOT NULL,
|
||||
resource_key VARCHAR(255) NOT NULL,
|
||||
method VARCHAR(16) NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
request_fingerprint VARCHAR(64) NOT NULL,
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
body_hash VARCHAR(64) NOT NULL,
|
||||
body_snapshot BYTEA NOT NULL,
|
||||
parser_version VARCHAR(32),
|
||||
fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT uq_parse_resource_cache_resource UNIQUE (profile_key, resource_key, request_fingerprint)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ix_parse_resource_cache_profile_key
|
||||
ON parse_resource_cache (profile_key);
|
||||
Reference in New Issue
Block a user