Logging at scale, clustering OpenObserve for ingestion.

2025-09-01 10:09:34 - No Comments

We look into OpenObserve, how to set it up in a cluster and ingest some data. Talking about what it could be used for and what is it.

sudo apt install git curl protobuf-compiler nodejs postgresql
curl -sL https://deb.nodesource.com/setup_22.x -o /tmp/nodesource_setup.sh
vi /tmp/nodesource_setup.sh
sudo bash /tmp/nodesource_setup.sh
sudo apt-get install nodejs -y

curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh

git clone https://github.com/openobserve/openobserve.git
cd openobserve
git checkout v0.15.0-rc5
cd web
npm install
npm run build
cd ..
cargo build -r
sudo cp target/release/openobserve /usr/local/bin

sudo mkdir /opt/openobserve
sudo groupadd openobserve
sudo useradd -g openobserve openobserve
sudo chown openobserve:openobserve -R /opt/openobserve

sudo -u postgres psql

CREATE USER openobserve WITH LOGIN PASSWORD 'change_me_now';
CREATE DATABASE openobserve OWNER openobserve TEMPLATE template0 ENCODING 'UTF8';
GRANT USAGE, CREATE ON SCHEMA public TO openobserve;

ZO_S3_SERVER_URL="http://ceph-node:8888/openobserve-record"
ZO_S3_ACCESS_KEY=access_key
ZO_S3_SECRET_KEY=secret_key
ZO_S3_BUCKET_NAME=openobserve-record
ZO_S3_FEATURE_FORCE_HOSTED_STYLE=true
ZO_META_STORE=postgres
ZO_META_POSTGRES_DSN="postgres://openobserve:change_me_now@localhost:5432/openobserve"
ZO_LOCAL_MODE=false
ZO_ROOT_USER_EMAIL="root@example.com"
ZO_ROOT_USER_PASSWORD="qwerty"
ZO_NATS_ADDR=localhost:4222
ZO_CLUSTER_COORDINATOR=nats

[Unit]
Description=OpenObserve service
Wants=network-online.target

[Install]
WantedBy=multi-user.target

[Service]
WorkingDirectory=/opt/openobserve
EnvironmentFile=/etc/systemd/system/openobserve.env
Type=simple
User=openobserve
Group=openobserve
Delegate=yes
TimeoutStartSec=0
Restart=always
RestartSec=5s
ExecStart=/usr/local/bin/openobserve

sudo systemctl daemon-reload
sudo systemctl start openobserve
sudo systemctl status openobserve

sudo apt-get install ntp

sudo apt install wget
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
echo "deb https://artifacts.elastic.co/packages/9.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-9.x.list
sudo apt-get update && sudo apt-get install filebeat apt-transport-https
sudo mv /etc/filebeat/filebeat.yml /etc/filebeat/filebeat.yml.old
sudo vi /etc/filebeat/filebeat.yml

setup.ilm.enabled: false
setup.template.enabled: false

filebeat.inputs:
- type: filestream
  enabled: true
  id: ceph-logs
  paths:
    - /var/log/*.log

output.elasticsearch:
  hosts: ["http://localhost:5080"]
  timeout: 10
  path: "/api/default/"
  index: "default"
  username: "root@example.com"
  password: "qwerty"

sudo systemctl daemon-reload
sudo systemctl restart filebeat
sudo systemctl status filebeat