标签：readers   err   lockfile   efault   ddr   nec   摘要   结果   rgs   

上面这张架构图可以看出ELSA从架构上分为三层：

日志接收器，由syslog-ng完成负责接收来自本地、网络以及导入的日志文件
日志存储索引，存储由MySQL数据库完成，索引由sphinx完成。
Web前端 。
ELSA利用syslog-ng的pattern-db解析器进行有效的日志规范化，并利用Sphinx全文索引进行日志搜索。系统内部API将查询结果汇总后，发送给客户端，整个系统是异步执行，可以跑多个查询。接收器syslog-ng在接收日志时并没有进行归一化处理（类比OSSIM-Agent插件），所以对日志的正则表达式计算量不大，可以在syslog-ng中保持高效的日志接收率，系统大部分有Perl脚本组成，MySQL每秒可插入100K行数据。Sphinx在索引中为新插入的行建立索引，每个2小时会重新建立一次永久索引。整个系统最大效率发挥时每秒钟可以处理100K条日志。

如果你具备ELK实战经验的话，可以把ELSA理解为简版的ELK系统，结构简单，速度快。安装（感兴趣的朋友可以在基于Debian（包括Ubuntu）的OS上测试，在ELSA Google Code主页上获取安装tar包）比较简单就不介绍了，下面直接切入正题。

1.采集Windows服务器日志

我们可以采用Eventlog-to-Syslog工具将Windows平台的日志发送到ELSA服务器
方法：
将evtsys.exe和evtsys.dll复制到系统目录下输入下面命令
evtsys.exe -i -h ELSA服务器的IP
志将使用syslog协议发送到您的ELSA服务器，在该服务器中，日志将被解析为“ WINDOWS”类

2.采集Linux系统及相关服务的日志

Linux/Unix系统都有rsyslog 或 Syslogd进程，在其配置文件中加入下面的配置即可

. @ELSA服务器IP

3.配置文件

ELSA的主要配置文件是/etc/elsa_node.conf

{

本地数据库连接信息

    "database" : {
            "db": "syslog",
            "data_db": "syslog_data",
            "dsn" : "dbi:mysql:database=syslog",
            "username" : "elsa",
            "password" : "biglog"
    },

// 系统协调锁的目录
"lockfile_dir": "/opt/elsa/node/tmp/locks",

    "num_indexes": 200,

//如果要归档日志，请保留此项
"archive": {

Uncomment to establish a retention period in days for archive logs

            #"days": 90,
            "percentage": 33,
            "table_size": 10000000
    },
    //日志大小限制+索引大小。设置为磁盘总空间的95-90%。
    "log_size_limit" : 8000000000,
    "sphinx" : {

            "indexer": "/usr/bin/indexer",

            "allowed_temp_percent" : 40,

            "allowed_mem_percent": 25
            "host" : "127.0.0.1",
            "port" : 9312,
    "mysql_port" : 9306,

            "config_file" : "/etc/sphinxsearch/sphinx.conf",

            "index_path" : "/nsm/elsa/data/sphinx",

            "index_interval" : 60,

            "perm_index_size" : 10000000,
            # Where the optional stopwords file is
            "stopwords": {
                    "file": "/etc/sphinxsearch/sphinx_stopwords.txt",
                    "top_n": 0,
                    "interval": 0,
                    "whitelist": []
            },

            "pid_file": "/var/run/sphinxsearch/searchd.pid"
    },

    "logdir" : "/nsm/elsa/data/elsa/log",
"mysql_dir": "/nsm/elsa/data/elsa/mysql",

    "num_log_readers" : 1,
   #调试跟踪级别
    "debug_level" : "TRACE",

    "buffer_dir" : "/nsm/elsa/data/elsa/tmp/buffers/",

    "log_parse_errors": 1,

    "stats" : {
            "retention_days": 365
    },

    "min_expected_hosts": 2

}
ELSA的Web配置文件 /etc/elsa_web.conf

{
#定义API密钥
"apikeys": {
"elsa": "b7292980d34c99e2581d36681831667b"
},
"version": {
"Author": "mcholste",
"Date": "2014-07-17 15:12:58 -0700 (Thu, 17 Jul 2014)",
"Rev": "1205",
"Sphinx": "Sphinx 2.1.9"
},
"peers": {
"127.0.0.1": {
"url": "http://127.0.0.1:3154/",
"username": "elsa",
"apikey": "b7292980d34c99e2581d36681831667b"
}
},
"admin_email_address": "root@localhost",
"connectors": {
},
"dashboards": {
},
"datasources": {
},
"transforms": {
"whois": {
"known_subnets": {
"10.0.0.0": {
"end": "10.255.255.255",
"org": "MyOrg"
},
"192.168.0.0": {
"end": "192.168.255.255",
"org": "MyOrg"
},
"172.16.0.0": {
"end": "172.31.255.255",
"org": "MyOrg"
}
},
"known_orgs": {
"MyOrg": {
"name": "MyOrg",
"org": "MyOrg",
"descr": "MyOrg",
"cc": "US",
"country": "United States",
"city": "Anytown",
"state": "Somestate"
}
}
},
"parse": {
"tld": [
{
"field": "domain",
"pattern": "\.([a-zA-Z]+)$",
"extractions": [
"tld"
]
},
{
"field": "site",
"pattern": "\.([a-zA-Z]+)$",
"extractions": [
"tld"
]
},
{
"field": "uri",
"pattern": "\.([a-zA-Z]+)(:|/|$)",
"extractions": [
"tld"
]
}
],
"url": [
{
"field": "uri",
"pattern": "(?:(?<proto>[a-zA-Z]+)://)?(?:(?<username>[^/]+):(?<password>[^/]+)@)?(?<domain>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|[^/]+\.(?<tld>[a-zA-Z]+))(?::(?<port>\d+))?(?<resource>/[^?])?(?:\?(?<query_string>.))?$",
"extractions": [
"proto",
"username",
"password",
"domain",
"tld",
"port",
"resource",
"querystring"
]
}
],
"mimetype": [
{
"field": "msg",
"pattern": "[\"‘\(\[\s\|;:](?<mime>(?<type>application|audio|chemical|image|message|model|multipart|text|video)/(?<subtype>[\w-]+))[\"‘\)\]\s\|;:]",
"extractions": [
"mime",
"type",
"subtype"
]
}
]
}
},
"plugins": {
"SNORT": "Info::Snort",
"WINDOWS": "Info::Windows",
"URL": "Info::Url",
"BRO_NOTICE": "Info::Bro"
},
"info": {
"snort": {
"url_templates": [
"http://doc.emergingthreats.net/bin/view/Main/%d"
]
},
"url": {
"url_templates": [
"http://whois.domaintools.com/%s"
]
},
"windows": {
"url_templates": [
"http://www.ultimatewindowssecurity.com/securitylog/encyclopedia/event.aspx?eventid=%d"
]
}
},
"max_concurrent_archive_queries": 4,
"schedule_interval": 60,
"node_info_cache_timeout": 60,
"email": {
"display_address": "noreply-elsa@example.com",
"base_url": "http://elsa/",
"subject": "ELSA Alert"
},
"link_key": "secret",
"yui": {
"local": "inc"
},
"data_db": {
"db": "syslog",
"username": "elsa",
"password": "biglog"
},
"meta_db": {
"dsn": "dbi:mysql:database=elsa_web",
"username": "elsa",
"password": "biglog"
},
"auth": {
"method": "security_onion"
},
"admin_groups": [
"system",
"admin"
],
"auth_db": {
"dsn": "dbi:mysql:database=securityonion_db",
"username": "root",
"password": "",
"auth_statement": "SELECT PASSWORD(password) FROM user_info WHERE username=?",
"email_statement": "SELECT email FROM user_info WHERE username=?"
},
"peer_id_multiplier": 1000000000000,
"query_timeout": 55,
"pcap_url": "/capme",
"logdir": "/nsm/elsa/data/elsa/log",
"buffer_dir": "/nsm/elsa/data/elsa/tmp/buffers",
"debug_level": "TRACE",
"default_start_time_offset": 2,
"livetail": {
"poll_interval": 5,
"time_limit": 3600
}
}

4.典型应用场景（截图）

着重对ELSA软件的几个重点功能进行展示。

1.连接数 Top N

2.动态仪表盘展示

动态展示单位时间内处理日志的数量、查询量、采集主机的地址以及日志类型等参数。

3.查询日志详细信息

我们在Field Summary（字段摘要）中发现这些日志有15个字段（主机IP、进程名称、源地址、源端口、目的地址、目的端口、协议类型、输入字节数量、服务类型、持续时间、输出字节、输入数据包数量、输出数据包数量、国家代码等），每个字段后面是出现的次数，各个字段之间通过“|”符号分割。

4.查询ossec日志信息

5.侦测到针对MySQL 3306端口扫描报警日志信息

6.端口扫描报警日志信息

7.Ping报警日志信息

有关日志分析的相关话题大家可以阅读畅销书《Unix/Linux网络日志分析与流量监控》。

ELSA企业日志归档查询系统

标签：readers   err   lockfile   efault   ddr   nec   摘要   结果   rgs   

原文地址：https://blog.51cto.com/chenguang/2562538