On 5 Jan 2018, at 17:44, Budai, László <laszlo.budai@balabit.com> wrote:

Hi,

this seems like a bug (I guess resolving the `java-module-dir` in the scl file failes somehow and this is why you have to set the classpath manually).

Could you share the content of your etc/scl.conf?

regards,

Laszlo Budai

On Tue, Jan 2, 2018 at 9:47 PM, Marco Mignone <info@marcomignone.com> wrote:

I have forgot to say that I was using the syslog-ng Docker image -> balabit/syslog-ng:latest

Thanks,

Marco

On 2 Jan 2018, at 15:41, Marco Mignone <info@marcomignone.com> wrote:

Hi All,

That worked for me too but I have few questions:

- Is this the expected behaviour?

- Do we still need to add the *.jar library files from the ES distribution?

- The client-lib-dir function seems to need *.jar when multiple paths are specified, apart from the last path in the line - is this correct?

My path in the ES destination:

client-lib-dir(“/esjarfiles/*.jar:/usr/lib/syslog-ng/3.13/java-modules/elastic-jest-client/*.jar:/usr/lib/syslog-ng/3.13/java-modules/“)

Thanks,

Marco

On 14 Dec 2017, at 23:08, hari ram <hariram@hotmail.com> wrote:

root@ES6:/etc/syslog-ng# more syslog-ng.conf

@version:3.13

@module mod-java

@include "scl.conf"

options {

    flush_lines(0);

    keep_hostname(yes);

    normalize_hostnames(yes);

    threaded(yes);

};

source      s_local   { system(); internal();   };

source      s_network { syslog(transport(tcp)); };

destination d_all { file ("/var/log/all.log"); };

destination d_elastic {

  elasticsearch2(

    client-lib-dir("/usr/lib/syslog-ng/3.13/java-modules/elastic-jest-client/*.jar:/usr/share/elasticsearch/lib/:/usr/lib/syslog-ng/3.13/java-modules/")   --- adding path 

    client_mode("http")

    cluster_url("http://192.168.1.75:9200")

    index("syslog-ng_${YEAR}.${MONTH}.${DAY}")

    type("syslog")

    cluster("test")

    flush-limit("1000")

    template("$(format-json --scope rfc5424 --scope nv-pairs --exclude DATE --key ISODATE)")

    time-zone("UTC")

  );

};

log { source(s_network); destination(d_elastic); };

log { source(s_local); destination(d_all); };

fix the error, but i will test and come back.

R!

---

From: syslog-ng <syslog-ng-bounces@lists.balabit.hu> on behalf of hari ram <hariram@hotmail.com>
Sent: 14 December 2017 23:04
To: syslog-ng@lists.balabit.hu
Subject: [syslog-ng] SYSLOG-NG issue with ES 6.X

 

Hi

I have installed SYSLOG-NG 3.13.2 on ubunutu, try to send logs to ES 6.0 i failed to do so, here is my inputs.

root@ES6:/etc/syslog-ng# syslog-ng -V

syslog-ng 3 (3.13.2)

Config version: 3.13

Installer-Version: 3.13.2

Revision: 3.13.2-1

Compile-Date: Dec  5 2017 13:24:07

Module-Directory: /usr/lib/syslog-ng/3.13

Module-Path: /usr/lib/syslog-ng/3.13

Available-Modules: afuser,mod-python,afstomp,http,afsql,disk-buffer,mod-java,cef,pseudofile,sdjournal,kvformat,xml,csvparser,snmptrapd-parser,appmodel,confgen,pacctformat,linux-kmsg-format,dbparser,system-source,map-value-pairs,add-contextual-data,date,syslogformat,afamqp,geoip2-plugin,tfgetent,graphite,afmongodb,cryptofuncs,geoip-plugin,afsmtp,afsocket,redis,affile,stardate,basicfuncs,riemann,json-plugin,tags-parser,afprog

Enable-Debug: off

Enable-GProf: off

Enable-Memtrace: off

Enable-IPv6: on

Enable-Spoof-Source: on

Enable-TCP-Wrapper: on

Enable-Linux-Caps: on

Enable-Systemd: on

===

root@ES6:/etc/syslog-ng# more syslog-ng.conf

@version:3.13

@module mod-java

@include "scl.conf"

options {

    flush_lines(0);

    keep_hostname(yes);

    normalize_hostnames(yes);

    threaded(yes);

};

source      s_local   { system(); internal();   };

source      s_network { syslog(transport(tcp)); };

destination d_all { file ("/var/log/all.log"); };

destination d_elastic {

  elasticsearch2(

    client-lib-dir("/usr/share/elasticsearch/lib/:/usr/lib/syslog-ng/3.13/java-modules/")

    client_mode("http")

    cluster_url("http://192.168.1.75:9200")

    index("syslog-ng_${YEAR}.${MONTH}.${DAY}")

    type("syslog")

    cluster("test")

    flush-limit("1000")

    template("$(format-json --scope rfc5424 --scope nv-pairs --exclude DATE --key ISODATE)")

    time-zone("UTC")

  );

};

log { source(s_network); destination(d_elastic); };

log { source(s_local); destination(d_all); };

===

root@ES6:/etc/syslog-ng# ls /usr/share/elasticsearch/lib/

elasticsearch-6.0.1.jar            jackson-dataformat-smile-2.8.6.jar  jopt-simple-5.0.2.jar    lucene-analyzers-common-7.0.1.jar  lucene-join-7.0.1.jar         lucene-sandbox-7.0.1.jar         plugin-cli-6.0.1.jar

HdrHistogram-2.1.9.jar             jackson-dataformat-yaml-2.8.6.jar   jts-1.13.jar             lucene-backward-codecs-7.0.1.jar   lucene-memory-7.0.1.jar       lucene-spatial3d-7.0.1.jar       securesm-1.2.jar

hppc-0.7.1.jar                     java-version-checker-6.0.1.jar      log4j-1.2-api-2.9.1.jar  lucene-core-7.0.1.jar              lucene-misc-7.0.1.jar         lucene-spatial-7.0.1.jar         snakeyaml-1.15.jar

jackson-core-2.8.6.jar             jna-4.4.0-1.jar                     log4j-api-2.9.1.jar      lucene-grouping-7.0.1.jar          lucene-queries-7.0.1.jar      lucene-spatial-extras-7.0.1.jar  spatial4j-0.6.jar

jackson-dataformat-cbor-2.8.6.jar  joda-time-2.9.5.jar                 log4j-core-2.9.1.jar     lucene-highlighter-7.0.1.jar       lucene-queryparser-7.0.1.jar  lucene-suggest-7.0.1.jar         t-digest-3.0.jar

====

root@ES6:/etc/syslog-ng# ls /usr/lib/syslog-ng/3.13/java-modules/

elastic.jar  elastic-jest-client  elastic-v2.jar  hdfs.jar  http.jar  kafka.jar  log4j-1.2.16.jar  syslog-ng-common.jar  syslog-ng-core.jar

==

root@ES6:/etc/syslog-ng# syslog-ng -Fevd

[2017-12-14T23:04:21.552408]                 Compiling #unnamed sequence [log] at [source generator system:14:12]

[2017-12-14T23:04:21.552510]         Compiling #unnamed single [log] at [/etc/syslog-ng/syslog-ng.conf:10:35]

[2017-12-14T23:04:21.552632]   Compiling d_all reference [destination] at [/etc/syslog-ng/syslog-ng.conf:27:24]

[2017-12-14T23:04:21.552715]     Compiling d_all sequence [destination] at [/etc/syslog-ng/syslog-ng.conf:12:1]

[2017-12-14T23:04:21.552781]       Compiling #unnamed junction [log] at [/etc/syslog-ng/syslog-ng.conf:12:20]

[2017-12-14T23:04:21.552884]         Compiling #unnamed single [log] at [/etc/syslog-ng/syslog-ng.conf:12:21]

[2017-12-14T23:04:21.553211] Module loaded and initialized successfully; module='syslogformat'

[2017-12-14T23:04:21.553425] Processing the time zone file (32bit part); filename='/usr/share/zoneinfo/UTC'

[2017-12-14T23:04:21.671696] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/syslog-ng-core.jar;

[2017-12-14T23:04:21.672418] Add path to classpath: /usr/share/elasticsearch/lib/;

[2017-12-14T23:04:21.673641] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/syslog-ng-core.jar;

[2017-12-14T23:04:21.673912] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/syslog-ng-common.jar;

[2017-12-14T23:04:21.674218] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/hdfs.jar;

[2017-12-14T23:04:21.674704] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/http.jar;

[2017-12-14T23:04:21.675858] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/kafka.jar;

[2017-12-14T23:04:21.676116] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/log4j-1.2.16.jar;

[2017-12-14T23:04:21.676322] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/elastic-v2.jar;

[2017-12-14T23:04:21.676484] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/elastic.jar;

[2017-12-14T23:04:21.741649] Add path to classpath: /usr/lib/syslog-ng/3.13/java-modules/syslog-ng-core.jar;

[2017-12-14T23:04:21.746168] Error initializing message pipeline; plugin name='java', location='#buffer:2:3'

Any suggestions ?

R!

______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq

______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq

______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq

______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq