Initial parser admin implementation
This commit is contained in:
Your Name
@@ -0,0 +1,194 @@
|
||||
CREATE TABLE IF NOT EXISTS admin_users (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
login TEXT NOT NULL UNIQUE,
|
||||
password_hash TEXT NOT NULL,
|
||||
role TEXT NOT NULL DEFAULT 'admin' CHECK (role IN ('admin','editor','viewer')),
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS admin_sessions (
|
||||
token_hash TEXT PRIMARY KEY,
|
||||
user_id BIGINT NOT NULL REFERENCES admin_users(id) ON DELETE CASCADE,
|
||||
csrf_token TEXT NOT NULL,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_admin_sessions_user_id ON admin_sessions(user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_admin_sessions_expires_at ON admin_sessions(expires_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sources (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
platform TEXT NOT NULL DEFAULT 'vk',
|
||||
name TEXT NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
external_id TEXT,
|
||||
external_owner_id BIGINT,
|
||||
active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
status TEXT NOT NULL DEFAULT 'new',
|
||||
status_msg TEXT,
|
||||
parse_from TIMESTAMPTZ,
|
||||
last_checked_at TIMESTAMPTZ,
|
||||
last_parsed_at TIMESTAMPTZ,
|
||||
priority INTEGER NOT NULL DEFAULT 100,
|
||||
tags TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
|
||||
settings_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
archived_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS ux_sources_platform_url_active
|
||||
ON sources(platform, lower(url))
|
||||
WHERE archived_at IS NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sources_active_platform ON sources(active, platform, priority, id)
|
||||
WHERE archived_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS raw_posts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
source_id BIGINT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
||||
platform TEXT NOT NULL,
|
||||
external_post_id TEXT NOT NULL,
|
||||
external_owner_id BIGINT,
|
||||
original_url TEXT NOT NULL,
|
||||
storage_owner_id BIGINT,
|
||||
storage_post_id BIGINT,
|
||||
storage_post_url TEXT,
|
||||
raw_text TEXT NOT NULL DEFAULT '',
|
||||
raw_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
text_hash TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
posted_at TIMESTAMPTZ,
|
||||
copied_at TIMESTAMPTZ,
|
||||
status TEXT NOT NULL DEFAULT 'raw_saved',
|
||||
skip_reason TEXT,
|
||||
error_reason TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE(source_id, external_post_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_status_created ON raw_posts(status, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_source_posted ON raw_posts(source_id, posted_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_posts_content_hash ON raw_posts(content_hash);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS raw_post_media (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
raw_post_id BIGINT NOT NULL REFERENCES raw_posts(id) ON DELETE CASCADE,
|
||||
platform TEXT NOT NULL,
|
||||
media_type TEXT NOT NULL CHECK (media_type IN ('photo','video','link','doc','unknown')),
|
||||
original_url TEXT,
|
||||
original_attachment_id TEXT,
|
||||
storage_attachment_id TEXT,
|
||||
storage_url TEXT,
|
||||
preview_url TEXT,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
duration_sec INTEGER,
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
error TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_post_media_post ON raw_post_media(raw_post_id, sort_order, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_post_media_status ON raw_post_media(status);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS jobs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
type TEXT NOT NULL,
|
||||
entity_type TEXT NOT NULL,
|
||||
entity_id BIGINT NOT NULL,
|
||||
payload_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
max_attempts INTEGER NOT NULL DEFAULT 5,
|
||||
next_run_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
locked_by TEXT,
|
||||
locked_at TIMESTAMPTZ,
|
||||
last_error TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS ux_jobs_open_entity_type
|
||||
ON jobs(type, entity_type, entity_id)
|
||||
WHERE status IN ('pending','retry','in_progress');
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_jobs_claim ON jobs(type, status, next_run_at, id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS worker_controls (
|
||||
name TEXT PRIMARY KEY,
|
||||
enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
settings_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
updated_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS worker_heartbeats (
|
||||
name TEXT PRIMARY KEY,
|
||||
heartbeat_at TIMESTAMPTZ NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'running',
|
||||
current_job_id BIGINT,
|
||||
meta_json JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS app_settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value_json JSONB NOT NULL,
|
||||
value_type TEXT NOT NULL DEFAULT 'str',
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
category TEXT NOT NULL DEFAULT 'General',
|
||||
updated_by BIGINT REFERENCES admin_users(id) ON DELETE SET NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS audit_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
actor_id BIGINT REFERENCES admin_users(id) ON DELETE SET NULL,
|
||||
action TEXT NOT NULL,
|
||||
entity_type TEXT NOT NULL,
|
||||
entity_id BIGINT,
|
||||
before_json JSONB,
|
||||
after_json JSONB,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
INSERT INTO worker_controls(name, enabled, settings_json)
|
||||
VALUES
|
||||
('vk-parser', TRUE, '{}'::jsonb),
|
||||
('vk-storage-uploader', TRUE, '{}'::jsonb)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
INSERT INTO app_settings(key, value_json, value_type, title, description, category)
|
||||
VALUES
|
||||
('parser_interval_sec', '300'::jsonb, 'int', 'Parser interval, sec', 'Pause between parser cycles.', 'Parser'),
|
||||
('parser_new_source_lookback_days', '14'::jsonb, 'int', 'New source lookback, days', 'How far back to read a new source.', 'Parser'),
|
||||
('parser_reparse_overlap_minutes', '120'::jsonb, 'int', 'Reparse overlap, min', 'Overlap window for known sources.', 'Parser'),
|
||||
('parser_min_text_length', '50'::jsonb, 'int', 'Minimum text length', 'Posts with shorter text are treated as junk.', 'Parser'),
|
||||
('parser_skip_reposts', 'true'::jsonb, 'bool', 'Skip reposts', 'Do not save reposts from VK copy_history.', 'Parser'),
|
||||
('parser_skip_empty_text', 'true'::jsonb, 'bool', 'Skip empty text posts', 'Treat posts without text as junk.', 'Parser'),
|
||||
('parser_skip_no_media', 'true'::jsonb, 'bool', 'Skip posts without media', 'Treat posts without media as junk.', 'Parser'),
|
||||
('parser_skip_text_too_short', 'true'::jsonb, 'bool', 'Skip short text posts', 'Treat posts shorter than parser_min_text_length as junk.', 'Parser'),
|
||||
('parser_store_skipped_posts', 'false'::jsonb, 'bool', 'Store skipped posts', 'Keep junk posts in raw_posts instead of dropping them.', 'Parser'),
|
||||
('parser_dedupe_content_hash', 'true'::jsonb, 'bool', 'Dedupe by content hash', 'Skip posts whose text and media hash already exists in raw_posts.', 'Parser'),
|
||||
('parser_source_pause_sec', '0'::jsonb, 'float', 'Source pause, sec', 'Pause between VK sources inside one parser cycle.', 'Parser'),
|
||||
('vk_requests_per_second', '3'::jsonb, 'int', 'VK requests per second', 'Global VK API rate limit per worker.', 'VK'),
|
||||
('vk_wall_page_size', '50'::jsonb, 'int', 'VK wall page size', 'wall.get count per page.', 'VK'),
|
||||
('vk_api_timeout_total_sec', '15'::jsonb, 'int', 'VK total timeout, sec', 'Total timeout for one VK API request.', 'VK'),
|
||||
('vk_api_timeout_connect_sec', '5'::jsonb, 'int', 'VK connect timeout, sec', 'Connection timeout for VK API.', 'VK'),
|
||||
('vk_rate_limit_sleep_sec', '1'::jsonb, 'float', 'VK rate limit sleep, sec', 'Sleep after VK API rate limit error code 6.', 'VK'),
|
||||
('vk_api_retry_attempts', '3'::jsonb, 'int', 'VK retry attempts', 'Attempts for temporary VK API/network errors.', 'VK'),
|
||||
('vk_api_retry_min_delay_sec', '2'::jsonb, 'float', 'VK retry min delay, sec', 'Initial retry delay for temporary VK errors.', 'VK'),
|
||||
('vk_api_retry_max_delay_sec', '10'::jsonb, 'float', 'VK retry max delay, sec', 'Maximum retry delay for temporary VK errors.', 'VK'),
|
||||
('uploader_interval_sec', '5'::jsonb, 'int', 'Uploader idle interval, sec', 'Pause when no jobs are available.', 'Uploader'),
|
||||
('uploader_download_timeout_sec', '45'::jsonb, 'int', 'Download timeout, sec', 'Timeout for source media download.', 'Uploader')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
Reference in New Issue
Block a user