Configuration Reference

Laredo uses HOCON configuration. Config is resolved in order (later overrides earlier):

1. Built-in defaults
2. Config file (--config flag or LAREDO_CONFIG env var)
3. Config directory (/etc/laredo/conf.d/*.conf, alphabetical)
4. Environment variables (LAREDO_SOURCES_PG_MAIN_CONNECTION)
5. CLI flags (--set key=value)

Sources

sources {
  <source_id> {
    type = postgresql | s3-kinesis | archive

    # PostgreSQL
    connection = "postgresql://user:pass@host:5432/dbname"
    slot_mode = ephemeral | stateful         # default: ephemeral
    slot_name = "laredo_01"                  # required for stateful
    always_baseline = false                  # default: false; force a full COPY
                                             # every startup even in stateful mode
                                             # (use for non-durable/in-memory targets)

    publication {
      name = "laredo_01_pub"                 # default: {slot_name}_pub
      create = false                         # default: false
      publish = "insert, update, delete, truncate"
      tables {
        "schema.table" {
          where = "status = 'active'"        # PG 15+ row filter
          columns = [col1, col2]             # PG 15+ column list
        }
      }
    }

    reconnect {
      max_attempts = 10                      # default: 10
      initial_backoff = 1s                   # default: 1s
      max_backoff = 60s                      # default: 60s
      backoff_multiplier = 2.0               # default: 2.0
    }

    # S3 + Kinesis
    baseline_bucket = "s3://bucket/prefix/"
    baseline_format = jsonl
    stream_arn = "arn:aws:kinesis:..."
    consumer_group = "laredo-01"
    checkpoint_table = "dynamodb://table"
    region = us-east-1

    # Archive (type = archive | file) — replay a snapshotter archive from disk
    # instead of a database. One archive source serves one table (a manifest is
    # per-table). The store/store_config/format/key_prefix shape mirrors the
    # snapshotter and a fan-out target's archive block.
    store = local | s3                       # required
    store_config { path = "/var/lib/laredo/archive/events" }  # local
    # store_config { bucket = ..., prefix = ..., region = ... } # s3 (ambient creds)
    format = jsonl                           # jsonl | protobuf; or a list, tried in order
    key_prefix = "public.events/"            # MUST match the archive's write prefix
    key_fields = [id]                        # primary key the archive was written with
    follow = false                           # keep watching for appended diffs and
                                             # wholesale replacement (re-baselines)
    poll_interval = 5s                       # manifest re-read interval while following
    state_path = "/var/lib/laredo/archive/events.ack"  # persist position → resume on
                                             # restart; omit to re-baseline each start

    # group = true expands ONE archive block into one source per table that
    # references it, deriving each table's key_prefix from "<schema>.<table>/".
    # Omit key_prefix; store_config.path is the archive ROOT, and state_path (if
    # set) is treated as a directory holding per-table "<schema>.<table>.pos" files.
    group = false
  }
}

Multi-table archive with group:

sources {
  seed { type = archive, group = true, store = local
         store_config { path = "/var/lib/laredo/archive" }, format = jsonl, follow = true }
}
tables = [
  { source = seed, schema = public, table = events, targets = [ { type = indexed-memory } ] }
  { source = seed, schema = public, table = users,  targets = [ { type = indexed-memory } ] }
]

This registers two sources — seed/public.events (prefix public.events/) and seed/public.users (prefix public.users/) — reading from the shared archive root. The synthesized per-table ids are the handles for pause/resume/reload.

Tables & Pipelines

tables = [
  {
    source = <source_id>
    schema = public
    table = config_document

    targets = [
      {
        type = indexed-memory | compiled-memory | http-sync | replication-fanout

        # indexed-memory
        lookup_fields = [field1, field2]
        additional_indexes = [
          { name = idx_name, fields = [field], unique = false }
        ]

        # compiled-memory
        compiler = "compiler_name"
        key_fields = [field1, field2]
        filter { field = key, prefix = "prefix/" }

        # http-sync
        base_url = "https://..."
        batch_size = 500                     # default: 500
        flush_interval = 200ms               # default: 200ms
        timeout_ms = 5000                    # default: 5000
        retry_count = 3                      # default: 3
        auth_header = "Bearer ..."
        headers { X-Custom = value }

        # replication-fanout (served on the engine-global fan-out listener; see
        # the top-level `fanout` block below and ADR-007)
        journal { max_entries = 1000000, max_age = 24h }
        snapshot { interval = 5m, retention { keep_count = 5, max_age = 1h } }  # in-memory
        max_clients = 500
        client_buffer { max_size = 50000, policy = drop_disconnect }
        heartbeat_interval = 5s
        # optional cold-tier archive (EDR-0005). store = local | s3; key_prefix
        # MUST match the laredo-snapshotter prefix for this table. s3 uses
        # ambient AWS credentials.
        archive {
          store = local                     # or: s3
          store_config { path = "/var/lib/laredo/archive/events" }   # s3: bucket, prefix, region (no credentials — ambient AWS only)
          format = jsonl                    # jsonl | protobuf; default jsonl; may be a list [jsonl, protobuf]
          key_prefix = "laredo/public.events/"
        }

        # Common
        buffer { max_size = 10000, policy = block }
        error_handling {
          max_retries = 5
          retry_backoff_ms = 1000
          retry_backoff_max_ms = 30000
          on_persistent_failure = isolate    # isolate | stop_source | stop_all
          dead_letter { enabled = false, store = s3, config { ... } }
        }
        filters = [{ type = field-equals, field = f, value = v }]
        transforms = [{ type = drop-fields, fields = [f1, f2] }]
      }
    ]

    ttl { mode = field, field = expires_at, check_interval = 30s }
    validation {
      post_baseline_row_count = true
      periodic_row_count { enabled = true, interval = 1h }
      on_mismatch = warn                     # warn | re_baseline | error
    }
  }
]

Snapshots

snapshot {
  enabled = true
  store = s3 | local
  store_config { bucket = ..., prefix = ..., region = ..., path = ... }
  serializer = jsonl
  schedule = "every 6h"
  on_shutdown = true
  retention { keep_count = 10, max_age = 7d }
  user_meta { key = value }
  on_restore_mismatch = warn                 # warn | reject
}

gRPC Server

grpc {
  port = 4001
  tls { enabled = false, cert_path = "", key_path = "" }
}

Fan-out replication listener

The replication protocol for replication-fanout targets is served by a single engine-global listener on its own port, distinct from the OAM/Query grpc port above. laredo-server starts it automatically when any replication-fanout target is configured; this block only overrides the default port (4002). See ADR-007.

fanout {
  grpc { port = 4002 }
}

Health

health {
  http_port = 8080
  readiness_path = "/health/ready"
  liveness_path = "/health/live"
}

Observability

observability {
  metrics { type = prometheus, port = 9090, path = "/metrics" }
  logging { type = structured, level = info }
}