[syslog-ng] SYSLOG-NG issue with ES 6.X
Marco Mignone
info at marcomignone.com
Wed Jan 10 14:53:46 UTC 2018
Hi Laszlo,
For my configuration I am using the docker image through a docker compose file:
—docker-compose file---
version: "3"
services:
syslog-ng:
container_name: syslog-ng
#depends_on:
# - "elasticsearch"
image: balabit/syslog-ng:latest
ports:
- "0.0.0.0:514:514/udp"
entrypoint: /usr/sbin/syslog-ng -Fedv
volumes:
- ~/Projects/Volumes/TST/var/log/syslog-ng:/var/log
- ~/Projects/Volumes/TST/etc/syslog-ng/syslog-ng.conf:/etc/syslog-ng/syslog-ng.conf
- es_lib:/jarfiles
networks:
- ESK
networks:
ESK:
volumes:
es_lib:
---Syslog configuration—
#############################################################################
# Default syslog-ng.conf file which collects all local logs into a
# single file called /var/log/messages.
#
@version: 3.13
@include "scl.conf"
@module mod-java
options {
time-zone(Europe/London);
# use-dns(yes);
# keep-hostname(yes);
# chain-hostnames(yes);
#create-dirs(yes);
};
source s_net {
udp(
ip(0.0.0.0),port(514),flags(no-parse)
);
};
destination d_file {
file("/var/log/${HOST}-${LEVEL}.log");
};
destination d_elasticsearch {
elasticsearch2(
client-lib-dir("/jarfiles/")
#client-lib-dir("/jarfiles/*.jar:/usr/lib/syslog-ng/3.13/java-modules/elastic-jest-client/*.jar:/usr/lib/syslog-ng/3.13/java-modules/")
index("index-${MONTH}")
type("syslog")
#time-zone("UTC")
client_mode("http")
cluster("docker-cluster")
#cluster_url("http://192.168.32.100:9200")
cluster_url("http://elasticsearch:9200")
#template(t_test)
flush-limit("1")
);
log {
source(s_net);
destination(d_elasticsearch);
destination(d_file);
};
This trigger the error.
If I comment the first client-lib and uncomment the second one all works fine.
Here is the final part of the startup debug messages:
syslog-ng | [2018-01-10T14:52:49.287682] Add path to classpath: /jarfiles/jopt-simple-5.0.2.jar;
syslog-ng | [2018-01-10T14:52:49.287860] Add path to classpath: /jarfiles/lucene-queries-7.0.1.jar;
syslog-ng | [2018-01-10T14:52:49.288053] Add path to classpath: /jarfiles/jackson-core-2.8.6.jar;
syslog-ng | [2018-01-10T14:52:49.288237] Add path to classpath: /jarfiles/hppc-0.7.1.jar;
syslog-ng | [2018-01-10T14:52:49.288399] Add path to classpath: /jarfiles/lucene-join-7.0.1.jar;
syslog-ng | [2018-01-10T14:52:49.288589] Add path to classpath: /jarfiles/HdrHistogram-2.1.9.jar;
syslog-ng | [2018-01-10T14:52:49.288769] Add path to classpath: /jarfiles/lucene-memory-7.0.1.jar;
syslog-ng | [2018-01-10T14:52:49.288970] Add path to classpath: /jarfiles/log4j-api-2.9.1.jar;
syslog-ng | [2018-01-10T14:52:49.289142] Add path to classpath: /jarfiles/lucene-highlighter-7.0.1.jar;
syslog-ng | [2018-01-10T14:52:49.289346] Add path to classpath: /jarfiles/log4j-core-2.9.1.jar;
syslog-ng | [2018-01-10T14:52:49.289533] Add path to classpath: /jarfiles/java-version-checker-6.0.0.jar;
syslog-ng | [2018-01-10T14:52:49.289734] Add path to classpath: /jarfiles/snakeyaml-1.15.jar;
syslog-ng | [2018-01-10T14:52:49.289899] Add path to classpath: /jarfiles/jackson-dataformat-cbor-2.8.6.jar;
syslog-ng | [2018-01-10T14:52:49.290089] Add path to classpath: /jarfiles/lucene-suggest-7.0.1.jar;
syslog-ng | [2018-01-10T14:52:49.290267] Add path to classpath: /jarfiles/lucene-spatial-extras-7.0.1.jar;
syslog-ng | [2018-01-10T14:52:49.290457] Add path to classpath: /jarfiles/t-digest-3.0.jar;
syslog-ng | [2018-01-10T14:52:49.290608] Add path to classpath: /jarfiles/lucene-queryparser-7.0.1.jar;
syslog-ng | [2018-01-10T14:52:49.290780] Add path to classpath: /jarfiles/lucene-core-7.0.1.jar;
syslog-ng | [2018-01-10T14:52:49.290956] Add path to classpath: /jarfiles/spatial4j-0.6.jar;
syslog-ng | [2018-01-10T14:52:49.291090] Add path to classpath: /jarfiles/securesm-1.1.jar;
syslog-ng | [2018-01-10T14:52:49.291225] Add path to classpath: /jarfiles/jts-1.13.jar;
syslog-ng | [2018-01-10T14:52:49.291353] Add path to classpath: /jarfiles/lucene-sandbox-7.0.1.jar;
syslog-ng | [2018-01-10T14:52:49.291499] Add path to classpath: /jarfiles/lucene-spatial-7.0.1.jar;
syslog-ng | [2018-01-10T14:52:49.291647] Add path to classpath: /jarfiles/joda-time-2.9.5.jar;
syslog-ng | [2018-01-10T14:52:49.291783] Add path to classpath: /jarfiles/jackson-dataformat-yaml-2.8.6.jar;
syslog-ng | [2018-01-10T14:52:49.291939] Add path to classpath: /jarfiles/lucene-backward-codecs-7.0.1.jar;
syslog-ng | [2018-01-10T14:52:49.292100] Add path to classpath: /jarfiles/lucene-analyzers-common-7.0.1.jar;
syslog-ng | [2018-01-10T14:52:49.292260] Add path to classpath: /jarfiles/jna-4.4.0-1.jar;
syslog-ng | [2018-01-10T14:52:49.313789] Exception: org.syslog_ng.elasticsearch_v2.ElasticSearchDestination;
syslog-ng | java.lang.ClassNotFoundException: org.syslog_ng.elasticsearch_v2.ElasticSearchDestination
syslog-ng | at java.net.URLClassLoader$1.run(URLClassLoader.java:359)
syslog-ng | at java.net.URLClassLoader$1.run(URLClassLoader.java:348)
syslog-ng | at java.security.AccessController.doPrivileged(Native Method)
syslog-ng | at java.net.URLClassLoader.findClass(URLClassLoader.java:347)
syslog-ng | at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
syslog-ng | at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:312)
syslog-ng | at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
syslog-ng | at java.lang.Class.forName0(Native Method)
syslog-ng | at java.lang.Class.forName(Class.java:278)
syslog-ng | at org.syslog_ng.SyslogNgClassLoader.loadClass(SyslogNgClassLoader.java:67)
syslog-ng | [2018-01-10T14:52:49.315737] Can't find class; class_name='org.syslog_ng.elasticsearch_v2.ElasticSearchDestination'
syslog-ng | [2018-01-10T14:52:49.315753] Java machine free;
syslog-ng | [2018-01-10T14:52:49.316367] Error initializing message pipeline; plugin name='java', location='#buffer:2:3'
syslog-ng exited with code 2
Hope this helps.
Marco
> On 5 Jan 2018, at 17:44, Budai, László <laszlo.budai at balabit.com> wrote:
>
> Hi,
>
> this seems like a bug (I guess resolving the `java-module-dir` in the scl file failes somehow and this is why you have to set the classpath manually).
> Could you share the content of your etc/scl.conf?
>
> regards,
> Laszlo Budai
>
> On Tue, Jan 2, 2018 at 9:47 PM, Marco Mignone <info at marcomignone.com <mailto:info at marcomignone.com>> wrote:
> I have forgot to say that I was using the syslog-ng Docker image -> balabit/syslog-ng:latest
>
> Thanks,
> Marco
>
>> On 2 Jan 2018, at 15:41, Marco Mignone <info at marcomignone.com <mailto:info at marcomignone.com>> wrote:
>>
>> Hi All,
>> That worked for me too but I have few questions:
>>
>> - Is this the expected behaviour?
>> - Do we still need to add the *.jar library files from the ES distribution?
>> - The client-lib-dir function seems to need *.jar when multiple paths are specified, apart from the last path in the line - is this correct?
>>
>>
>> My path in the ES destination:
>>
>> client-lib-dir(“/esjarfiles/*.jar:/usr/lib/syslog-ng/3.13/java-modules/elastic-jest-client/*.jar:/usr/lib/syslog-ng/3.13/java-modules/“)
>>
>> Thanks,
>> Marco
>>
>>> On 14 Dec 2017, at 23:08, hari ram <hariram at hotmail.com <mailto:hariram at hotmail.com>> wrote:
>>>
>>> root at ES6:/etc/syslog-ng# more syslog-ng.conf
>>> @version:3.13
>>> @module mod-java
>>> @include "scl.conf"
>>> options {
>>> flush_lines(0);
>>> keep_hostname(yes);
>>> normalize_hostnames(yes);
>>> threaded(yes);
>>> };
>>> source s_local { system(); internal(); };
>>> source s_network { syslog(transport(tcp)); };
>>> destination d_all { file ("/var/log/all.log"); };
>>> destination d_elastic {
>>> elasticsearch2(
>>> client-lib-dir("/usr/lib/syslog-ng/3.13/java-modules/elastic-jest-client/*.jar:/usr/share/elasticsearch/lib/:/usr/lib/syslog-ng/3.13/java-modules/") --- adding path
>>> client_mode("http")
>>> cluster_url("http://192.168.1.75:9200 <http://192.168.1.75:9200/>")
>>> index("syslog-ng_${YEAR}.${MONTH}.${DAY}")
>>> type("syslog")
>>> cluster("test")
>>> flush-limit("1000")
>>> template("$(format-json --scope rfc5424 --scope nv-pairs --exclude DATE --key ISODATE)")
>>> time-zone("UTC")
>>> );
>>> };
>>> log { source(s_network); destination(d_elastic); };
>>> log { source(s_local); destination(d_all); };
>>>
>>>
>>>
>>> fix the error, but i will test and come back.
>>>
>>> R!
>>> From: syslog-ng <syslog-ng-bounces at lists.balabit.hu <mailto:syslog-ng-bounces at lists.balabit.hu>> on behalf of hari ram <hariram at hotmail.com <mailto:hariram at hotmail.com>>
>>> Sent: 14 December 2017 23:04
>>> To: syslog-ng at lists.balabit.hu <mailto:syslog-ng at lists.balabit.hu>
>>> Subject: [syslog-ng] SYSLOG-NG issue with ES 6.X
>>>
>>> Hi
>>>
>>> I have installed SYSLOG-NG 3.13.2 on ubunutu, try to send logs to ES 6.0 i failed to do so, here is my inputs.
>>>
>>> root at ES6:/etc/syslog-ng# syslog-ng -V
>>> syslog-ng 3 (3.13.2)
>>> Config version: 3.13
>>> Installer-Version: 3.13.2
>>> Revision: 3.13.2-1
>>> Compile-Date: Dec 5 2017 13:24:07
>>> Module-Directory: /usr/lib/syslog-ng/3.13
>>> Module-Path: /usr/lib/syslog-ng/3.13
>>> Available-Modules: afuser,mod-python,afstomp,http,afsql,disk-buffer,mod-java,cef,pseudofile,sdjournal,kvformat,xml,csvparser,snmptrapd-parser,appmodel,confgen,pacctformat,linux-kmsg-format,dbparser,system-source,map-value-pairs,add-contextual-data,date,syslogformat,afamqp,geoip2-plugin,tfgetent,graphite,afmongodb,cryptofuncs,geoip-plugin,afsmtp,afsocket,redis,affile,stardate,basicfuncs,riemann,json-plugin,tags-parser,afprog
>>> Enable-Debug: off
>>> Enable-GProf: off
>>> Enable-Memtrace: off
>>> Enable-IPv6: on
>>> Enable-Spoof-Source: on
>>> Enable-TCP-Wrapper: on
>>> Enable-Linux-Caps: on
>>> Enable-Systemd: on
>>>
>>> ===
>>>
>>> root at ES6:/etc/syslog-ng# more syslog-ng.conf
>>> @version:3.13
>>> @module mod-java
>>> @include "scl.conf"
>>> options {
>>> flush_lines(0);
>>> keep_hostname(yes);
>>> normalize_hostnames(yes);
>>> threaded(yes);
>>> };
>>> source s_local { system(); internal(); };
>>> source s_network { syslog(transport(tcp)); };
>>> destination d_all { file ("/var/log/all.log"); };
>>> destination d_elastic {
>>> elasticsearch2(
>>> client-lib-dir("/usr/share/elasticsearch/lib/:/usr/lib/syslog-ng/3.13/java-modules/")
>>> client_mode("http")
>>> cluster_url("http://192.168.1.75:9200 <http://192.168.1.75:9200/>")
>>> index("syslog-ng_${YEAR}.${MONTH}.${DAY}")
>>> type("syslog")
>>> cluster("test")
>>> flush-limit("1000")
>>> template("$(format-json --scope rfc5424 --scope nv-pairs --exclude DATE --key ISODATE)")
>>> time-zone("UTC")
>>> );
>>> };
>>> log { source(s_network); destination(d_elastic); };
>>> log { source(s_local); destination(d_all); };
>>>
>>>
>>> ===
>>>
>>> root at ES6:/etc/syslog-ng# ls /usr/share/elasticsearch/lib/
>>> elasticsearch-6.0.1.jar jackson-dataformat-smile-2.8.6.jar jopt-simple-5.0.2.jar lucene-analyzers-common-7.0.1.jar lucene-join-7.0.1.jar lucene-sandbox-7.0.1.jar plugin-cli-6.0.1.jar
>>> HdrHistogram-2.1.9.jar jackson-dataformat-yaml-2.8.6.jar jts-1.13.jar lucene-backward-codecs-7.0.1.jar lucene-memory-7.0.1.jar lucene-spatial3d-7.0.1.jar securesm-1.2.jar
>>> hppc-0.7.1.jar java-version-checker-6.0.1.jar log4j-1.2-api-2.9.1.jar lucene-core-7.0.1.jar lucene-misc-7.0.1.jar lucene-spatial-7.0.1.jar snakeyaml-1.15.jar
>>> jackson-core-2.8.6.jar jna-4.4.0-1.jar log4j-api-2.9.1.jar lucene-grouping-7.0.1.jar lucene-queries-7.0.1.jar lucene-spatial-extras-7.0.1.jar spatial4j-0.6.jar
>>> jackson-dataformat-cbor-2.8.6.jar joda-time-2.9.5.jar log4j-core-2.9.1.jar lucene-highlighter-7.0.1.jar lucene-queryparser-7.0.1.jar lucene-suggest-7.0.1.jar t-digest-3.0.jar
>>>
>>>
>>> ====
>>>
>>> root at ES6:/etc/syslog-ng# ls /usr/lib/syslog-ng/3.13/java-modules/
>>> elastic.jar elastic-jest-client elastic-v2.jar hdfs.jar http.jar kafka.jar log4j-1.2.16.jar syslog-ng-common.jar syslog-ng-core.jar
>>>
>>> ==
>>>
>>> root at ES6:/etc/syslog-ng# syslog-ng -Fevd
>>>
>>> [2017-12-14T23:04:21.552408] Compiling #unnamed sequence [log] at [source generator system:14:12]
>>> [2017-12-14T23:04:21.552510] Compiling #unnamed single [log] at [/etc/syslog-ng/syslog-ng.conf:10:35]
>>> [2017-12-14T23:04:21.552632] Compiling d_all reference [destination] at [/etc/syslog-ng/syslog-ng.conf:27:24]
>>> [2017-12-14T23:04:21.552715] Compiling d_all sequence [destination] at [/etc/syslog-ng/syslog-ng.conf:12:1]
>>> [2017-12-14T23:04:21.552781] Compiling #unnamed junction [log] at [/etc/syslog-ng/syslog-ng.conf:12:20]
>>> [2017-12-14T23:04:21.552884] Compiling #unnamed single [log] at [/etc/syslog-ng/syslog-ng.conf:12:21]
>>> [2017-12-14T23:04:21.553211] Module loaded and initialized successfully; module='syslogformat'
>>> [2017-12-14T23:04:21.553425] Processing the time zone file (32bit part); filename='/usr/share/zoneinfo/UTC'
>>> [2017-12-14T23:04:21.671696] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/syslog-ng-core.jar;
>>> [2017-12-14T23:04:21.672418] Add path to classpath: /usr/share/elasticsearch/lib/;
>>> [2017-12-14T23:04:21.673641] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/syslog-ng-core.jar;
>>> [2017-12-14T23:04:21.673912] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/syslog-ng-common.jar;
>>> [2017-12-14T23:04:21.674218] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/hdfs.jar;
>>> [2017-12-14T23:04:21.674704] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/http.jar;
>>> [2017-12-14T23:04:21.675858] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/kafka.jar;
>>> [2017-12-14T23:04:21.676116] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/log4j-1.2.16.jar;
>>> [2017-12-14T23:04:21.676322] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/elastic-v2.jar;
>>> [2017-12-14T23:04:21.676484] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/elastic.jar;
>>> [2017-12-14T23:04:21.741649] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/syslog-ng-core.jar;
>>> [2017-12-14T23:04:21.746168] Error initializing message pipeline; plugin name='java', location='#buffer:2:3'
>>>
>>>
>>> Any suggestions ?
>>>
>>> R!
>>>
>>> ______________________________________________________________________________
>>> Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng <https://lists.balabit.hu/mailman/listinfo/syslog-ng>
>>> Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng <http://www.balabit.com/support/documentation/?product=syslog-ng>
>>> FAQ: http://www.balabit.com/wiki/syslog-ng-faq <http://www.balabit.com/wiki/syslog-ng-faq>
>> ______________________________________________________________________________
>> Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng <https://lists.balabit.hu/mailman/listinfo/syslog-ng>
>> Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng <http://www.balabit.com/support/documentation/?product=syslog-ng>
>> FAQ: http://www.balabit.com/wiki/syslog-ng-faq <http://www.balabit.com/wiki/syslog-ng-faq>
>>
>
>
> ______________________________________________________________________________
> Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng <https://lists.balabit.hu/mailman/listinfo/syslog-ng>
> Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng <http://www.balabit.com/support/documentation/?product=syslog-ng>
> FAQ: http://www.balabit.com/wiki/syslog-ng-faq <http://www.balabit.com/wiki/syslog-ng-faq>
>
>
>
> ______________________________________________________________________________
> Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
> Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
> FAQ: http://www.balabit.com/wiki/syslog-ng-faq
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.balabit.hu/pipermail/syslog-ng/attachments/20180110/6de05a86/attachment-0001.html>
More information about the syslog-ng
mailing list