Commit d1a85982 authored by koennecke's avatar koennecke
Browse files

Modified the mongodb logging code to write and query on a single DB and...

Modified the mongodb logging code to write and query on a single DB and instrument specific collections
This is still with the sinqTestDB. I still need to change for production
parent b162ad86
......@@ -5,6 +5,11 @@
copyright: see file COPYRIGHT
Mark Koennecke, February 2016
Modified to write GELF format messages and for the system to have one database but per instrument
collections.
Mark Koennecke, May 2017
*/
#include <stdio.h>
......@@ -17,6 +22,7 @@
static char mongoURL[1024];
static char inst[132];
static char logDB[] = "sinqTestDB";
static mongoc_client_t *client;
static mongoc_collection_t *collection;
......@@ -62,11 +68,14 @@ static void MongoCallback(unsigned int severity, const char *timeStamp,
bson_oid_init (&oid, NULL);
BSON_APPEND_OID (doc, "_id", &oid);
BSON_APPEND_DOUBLE(doc,"timestamp",tVal);
BSON_APPEND_INT32(doc,"severity",severity);
BSON_APPEND_UTF8(doc,"timetext",timeStamp);
BSON_APPEND_UTF8(doc,"sub",subsystem);
BSON_APPEND_UTF8(doc,"message",message);
BSON_APPEND_INT32(doc,"level",severity);
BSON_APPEND_UTF8(doc,"_timetext",timeStamp);
BSON_APPEND_UTF8(doc,"_sub",subsystem);
BSON_APPEND_UTF8(doc,"short_message",message);
BSON_APPEND_UTF8(doc,"version","1.1");
BSON_APPEND_UTF8(doc,"host",inst);
if (!mongoc_collection_insert (collection, MONGOC_INSERT_NONE, doc, NULL, &error)) {
fprintf (stdout, "%s\n", error.message);
MongoClose(NULL);
......@@ -80,6 +89,7 @@ static int MongoConfigAction(SConnection * pCon, SicsInterp * pSics,
void *pData, int argc, char *argv[])
{
int status;
char collName[256];
if(argc < 2) {
SCWrite(pCon,"ERROR: need keyword", eError);
......@@ -100,7 +110,8 @@ static int MongoConfigAction(SConnection * pCon, SicsInterp * pSics,
SCWrite(pCon,"ERROR: failed to connect to mongodb, bad URL?", eError);
return 0;
}
collection = mongoc_client_get_collection (client, inst, "log");
snprintf(collName,sizeof(collName),"log_%s",inst);
collection = mongoc_client_get_collection (client, logDB, collName);
if(!collection){
SCWrite(pCon,"ERROR: failed to connect link to mongo DB", eError);
return 0;
......@@ -114,7 +125,8 @@ static int MongoConfigAction(SConnection * pCon, SicsInterp * pSics,
SCWrite(pCon,"ERROR: failed to connect to mongodb, bad URL?", eError);
return 0;
}
collection = mongoc_client_get_collection (client, inst, "log");
snprintf(collName,sizeof(collName),"log_%s",inst);
collection = mongoc_client_get_collection (client, logDB, collName);
if(!collection){
SCWrite(pCon,"ERROR: failed to connect link to mongo DB", eError);
return 0;
......@@ -149,21 +161,21 @@ static void QueryCallback(const bson_t *doc, void *userData)
char sevBuf[20];
bson_iter_init(&iter,doc);
bson_iter_find(&iter,"timetext");
bson_iter_find(&iter,"_timetext");
timeText = bson_iter_utf8(&iter,&length);
DynStringConcat(result,(char *)timeText);
DynStringConcatChar(result,' ');
bson_iter_init(&iter,doc);
bson_iter_find(&iter,"severity");
bson_iter_find(&iter,"level");
severity = bson_iter_int32(&iter);
formatSeverity(severity,sevBuf,sizeof(sevBuf));
DynStringConcat(result,(char *)sevBuf);
DynStringConcatChar(result,' ');
bson_iter_find(&iter,"sub");
bson_iter_find(&iter,"_sub");
sub = bson_iter_utf8(&iter,&length);
DynStringConcat(result,(char *)sub);
DynStringConcatChar(result,' ');
bson_iter_find(&iter,"message");
bson_iter_find(&iter,"short_message");
message = bson_iter_utf8(&iter,&length);
DynStringConcat(result,(char *)message);
DynStringConcatChar(result,'\n');
......
......@@ -4,6 +4,10 @@
* COPYRIGHT GPL
*
* Mark Koennecke, February 2016
*
* Modified to allow for GELF standrad logging and the new organisation with one DB and multiple collections
*
* Mark Koennecke, May 2017
*/
#include <sicslogquery.h>
#include <unistd.h>
......@@ -14,6 +18,7 @@
static char lastError[1024];
static char mongoURL[1024];
static char inst[132];
static char logDB[] = "sinqTestDB";
static mongoc_client_t *client;
static mongoc_collection_t *collection;
......@@ -75,7 +80,7 @@ int sicslogQuery(int argc, char *argv[], ResultCallback_t func, void *userData)
char *exp = NULL;
char *instLocal = NULL;
char *tmp, *pPtr;
char jsonQuery[2024], subQuery[132];
char jsonQuery[2024], subQuery[132], collName[256];
struct timeval tv;
mongoc_collection_t *collection = NULL;
bson_t *query;
......@@ -159,14 +164,14 @@ int sicslogQuery(int argc, char *argv[], ResultCallback_t func, void *userData)
/*
build the query
*/
snprintf(jsonQuery, sizeof(jsonQuery),"{ \"timestamp\" : {\"$gt\": %ld, \"$lt\": %ld}, \"severity\": {\"$lte\": %d}",
snprintf(jsonQuery, sizeof(jsonQuery),"{ \"timestamp\" : {\"$gt\": %ld, \"$lt\": %ld}, \"level\": {\"$lte\": %d}",
from,to,severity);
if(sub != NULL){
snprintf(subQuery,sizeof(subQuery),", \"sub\" : \"%s\"", sub);
snprintf(subQuery,sizeof(subQuery),", \"_sub\" : \"%s\"", sub);
strncat(jsonQuery,subQuery,sizeof(jsonQuery));
}
if(exp != NULL){
snprintf(subQuery,sizeof(subQuery),", \"message\" : {\"$regex\": \"%s\" }", exp);
snprintf(subQuery,sizeof(subQuery),", \"short_message\" : {\"$regex\": \"%s\" }", exp);
strncat(jsonQuery,subQuery,sizeof(jsonQuery));
}
strncat(jsonQuery,"}",sizeof(jsonQuery));
......@@ -174,7 +179,8 @@ int sicslogQuery(int argc, char *argv[], ResultCallback_t func, void *userData)
/*
now execute the query
*/
collection = mongoc_client_get_collection(client,instLocal,"log");
snprintf(collName,sizeof(collName),"log_%s",instLocal);
collection = mongoc_client_get_collection(client,logDB,collName);
if(collection == NULL){
snprintf(lastError,sizeof(lastError),"Failed to find log for %s", instLocal);
status = 1;
......
......@@ -4,6 +4,10 @@
* copyright: GPL
*
* Mark Koennecke, February 2016
*
* modified for the new GELF conforming log messages and the new organisation of the log
*
* Mark Koennecke, May 2017
*/
#include <sicslogquery.h>
#include <bson.h>
......@@ -36,15 +40,15 @@ static void ResultPrint(const bson_t *doc, void *userData)
char sevBuf[20];
bson_iter_init(&iter,doc);
bson_iter_find(&iter,"timetext");
bson_iter_find(&iter,"_timetext");
timeText = bson_iter_utf8(&iter,&length);
bson_iter_init(&iter,doc);
bson_iter_find(&iter,"severity");
bson_iter_find(&iter,"level");
severity = bson_iter_int32(&iter);
formatSeverity(severity,sevBuf,sizeof(sevBuf));
bson_iter_find(&iter,"sub");
bson_iter_find(&iter,"_sub");
sub = bson_iter_utf8(&iter,&length);
bson_iter_find(&iter,"message");
bson_iter_find(&iter,"short_message");
message = bson_iter_utf8(&iter,&length);
fprintf(stdout,"%s %s %s %s\n",timeText,sevBuf, sub, message);
......@@ -56,7 +60,7 @@ int main(int argc, char *argv[])
int status;
char *error;
sicslogSetup("mongodb://logwriter:sinqsics@mpc1965:27017/?authSource=admin",NULL);
sicslogSetup("mongodb://userRW:2yfbt4AwBw4RY7j@mgdbaas001:27017,mgdbaas002:27017,mgdbaas003:27017/testDB?replicaSet=rs_prod-DataScience-01&connectTimeoutMS=300&authSource=sinqTestDB",NULL);
status = sicslogQuery(argc,argv,ResultPrint,NULL);
if(status != 0){
error = sicslogGetError();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment