Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Open sidebar
sinqdev
sicspsi
Commits
ffaa887e
Commit
ffaa887e
authored
Jun 16, 2017
by
koennecke
Browse files
Adapted remaining pymongo scripts to the log server
parent
5347d3d5
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
51 additions
and
22 deletions
+51
-22
utils/pymongo/errorstatistics
utils/pymongo/errorstatistics
+18
-9
utils/pymongo/errorsummary
utils/pymongo/errorsummary
+15
-4
utils/pymongo/messagestatistics
utils/pymongo/messagestatistics
+18
-9
No files found.
utils/pymongo/errorstatistics
View file @
ffaa887e
#!/usr/bin/python
# virtualenv activation for the best and brightest pymongo version
# This version works on rhel7
activate_this
=
'/afs/psi.ch/project/sinqdev/mksics/sicspsi/utils/pymongo/rhel7/bin/activate_this.py'
execfile
(
activate_this
,
dict
(
__file__
=
activate_this
))
import
pymongo
import
datetime
import
time
...
...
@@ -14,21 +20,24 @@ if len(sys.argv) < 2:
inst
=
sys
.
argv
[
1
]
mongodb
=
MongoClient
(
'mongodb://logwriter:sinqsics@mpc1965:27017/?authSource=admin'
)
mongodb
=
MongoClient
(
'mongodb://sinqUserRW:8Mh7]YFUE6JP]dQ7@mgdbaas001:27017,mgdbaas002:27017,mgdbaas003:27017/sinqProdDB01?replicaSet=rs_prod-DataScience-01&connectTimeoutMS=300&authSource=sinqProdDB01'
)
db
=
mongodb
[
'sinqProdDB01'
]
totime
=
datetime
.
datetime
.
today
()
totime
=
datetime
.
datetime
(
totime
.
year
,
totime
.
month
,
totime
.
day
)
fromtime
=
totime
-
timedelta
(
1
)
messagedict
=
{}
db
=
mongodb
[
inst
]
for
entry
in
db
.
log
.
find
({
"timestamp"
:
{
"$gt"
:
time
.
mktime
(
fromtime
.
timetuple
()),
"$lt"
:
time
.
mktime
(
totime
.
timetuple
())},
"severity"
:
{
"$lte"
:
2
}
}):
mes
=
entry
[
'message'
]
if
messagedict
.
has_key
(
mes
):
messagedict
[
mes
]
=
messagedict
[
mes
]
+
1
else
:
messagedict
[
mes
]
=
1
query
=
'cursor = db.log_%s.find({"timestamp" : {"$gt": time.mktime(fromtime.timetuple()),"$lt" : time.mktime(totime.timetuple())}, "level": {"$lte": 2} })'
%
inst
#print(query)
exec
query
for
entry
in
cursor
:
mes
=
entry
[
'short_message'
]
if
messagedict
.
has_key
(
mes
):
messagedict
[
mes
]
=
messagedict
[
mes
]
+
1
else
:
messagedict
[
mes
]
=
1
for
mes
in
sorted
(
messagedict
,
key
=
messagedict
.
get
):
...
...
utils/pymongo/errorsummary
View file @
ffaa887e
#!/usr/bin/python
# virtualenv activation for the best and brightest pymongo version
# This version works on rhel7
activate_this
=
'/afs/psi.ch/project/sinqdev/mksics/sicspsi/utils/pymongo/rhel7/bin/activate_this.py'
execfile
(
activate_this
,
dict
(
__file__
=
activate_this
))
import
pymongo
import
datetime
import
time
...
...
@@ -7,7 +12,9 @@ import time
from
pymongo
import
MongoClient
from
datetime
import
timedelta
mongodb
=
MongoClient
(
'mongodb://logwriter:sinqsics@mpc1965:27017/?authSource=admin'
)
mongodb
=
MongoClient
(
'mongodb://sinqUserRW:8Mh7]YFUE6JP]dQ7@mgdbaas001:27017,mgdbaas002:27017,mgdbaas003:27017/sinqProdDB01?replicaSet=rs_prod-DataScience-01&connectTimeoutMS=300&authSource=sinqProdDB01'
)
dblist
=
[
'amor'
,
'boa'
,
'dmc'
,
'eiger'
,
'focus'
,
'hrpt'
,
'morpheus'
,
'narziss'
,
'orion'
,
'poldi'
,
'rita2'
,
'sans'
,
'sans2'
,
'tasp'
,
'zebra'
]
...
...
@@ -16,9 +23,13 @@ totime = datetime.datetime.today()
totime
=
datetime
.
datetime
(
totime
.
year
,
totime
.
month
,
totime
.
day
)
fromtime
=
totime
-
timedelta
(
1
)
db
=
mongodb
[
'sinqProdDB01'
]
print
(
'Error counts between '
+
str
(
fromtime
)
+
' to '
+
str
(
totime
)
)
for
inst
in
dblist
:
db
=
mongodb
[
inst
]
ct
=
db
.
log
.
find
({
"timestamp"
:
{
"$gt"
:
time
.
mktime
(
fromtime
.
timetuple
()),
"$lt"
:
time
.
mktime
(
totime
.
timetuple
())},
"severity"
:
{
"$lte"
:
2
}
}).
count
()
com
=
'ct = db.log_%s.find({"timestamp" : {"$gt": time.mktime(fromtime.timetuple()), "$lt" : time.mktime(totime.timetuple())}, "level": {"$lte": 2} }).count()'
%
(
inst
)
#
print(com)
exec
com
print
(
inst
+
','
+
str
(
ct
))
utils/pymongo/messagestatistics
View file @
ffaa887e
#!/usr/bin/python
# virtualenv activation for the best and brightest pymongo version
# This version works on rhel7
activate_this
=
'/afs/psi.ch/project/sinqdev/mksics/sicspsi/utils/pymongo/rhel7/bin/activate_this.py'
execfile
(
activate_this
,
dict
(
__file__
=
activate_this
))
import
pymongo
import
datetime
import
time
...
...
@@ -14,21 +19,25 @@ if len(sys.argv) < 2:
inst
=
sys
.
argv
[
1
]
mongodb
=
MongoClient
(
'mongodb://logwriter:sinqsics@mpc1965:27017/?authSource=admin'
)
mongodb
=
MongoClient
(
'mongodb://sinqUserRW:8Mh7]YFUE6JP]dQ7@mgdbaas001:27017,mgdbaas002:27017,mgdbaas003:27017/sinqProdDB01?replicaSet=rs_prod-DataScience-01&connectTimeoutMS=300&authSource=sinqProdDB01'
)
db
=
mongodb
[
'sinqProdDB01'
]
totime
=
datetime
.
datetime
.
today
()
totime
=
datetime
.
datetime
(
totime
.
year
,
totime
.
month
,
totime
.
day
)
fromtime
=
totime
-
timedelta
(
1
)
messagedict
=
{}
db
=
mongodb
[
inst
]
for
entry
in
db
.
log
.
find
({
"timestamp"
:
{
"$gt"
:
time
.
mktime
(
fromtime
.
timetuple
()),
"$lt"
:
time
.
mktime
(
totime
.
timetuple
())}
}):
mes
=
entry
[
'message'
]
if
messagedict
.
has_key
(
mes
):
messagedict
[
mes
]
=
messagedict
[
mes
]
+
1
else
:
messagedict
[
mes
]
=
1
query
=
'cursor = db.log_%s.find({"timestamp" : {"$gt": time.mktime(fromtime.timetuple()),"$lt" : time.mktime(totime.timetuple())}})'
%
inst
#print(query)
exec
query
for
entry
in
cursor
:
mes
=
entry
[
'short_message'
]
if
messagedict
.
has_key
(
mes
):
messagedict
[
mes
]
=
messagedict
[
mes
]
+
1
else
:
messagedict
[
mes
]
=
1
for
mes
in
sorted
(
messagedict
,
key
=
messagedict
.
get
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment