Skip to content

Commit 2387f82

Browse files
committed
Merge branch 'pluginRefactory'
2 parents 1e70925 + 2ba5976 commit 2387f82

File tree

12 files changed

+520
-115
lines changed

12 files changed

+520
-115
lines changed

.travis.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,13 @@ services: mongodb
1616
install:
1717
- "pip install pep8 --use-mirrors"
1818
- "pip install pyflakes --use-mirrors"
19-
- "pip install pymongo --use-mirrors"
19+
- "pip install boto --use-mirrors"
20+
- "pip install pymongo sqlalchemy MySQL-python --use-mirrors"
2021
- "python setup.py install"
2122
# - "pip install -r requirements.txt --use-mirrors"
2223
# # command to run tests
2324
before_script:
2425
- "pep8 . --exclude test,docs,examples"
2526
- "pyflakes ."
27+
- mysql -e 'create database poulet;'
2628
script: nosetests

config/database.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
sqlite:
2+
adapter: sqlite3
3+
database: "/tmp/reportdb.sql"
4+
timeout: 500
5+
mysql:
6+
adapter: mysql2
7+
database: poulet
8+
username:
9+
encoding: utf8
10+

docs/index.rst

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,9 @@ libnmap is a python toolkit for manipulating nmap. It currently offers the follo
1414
- plugins: enables you to support datastores for your scan results directly in the "NmapReport" object from report module
1515

1616
- mongodb: only plugin implemented so far, ultra basic, for POC purpose only
17+
- sqlalchemy: Allow to store/retreive NmapReport to sqlite/mysql/... all engine supported by sqlalchemy
18+
- rabbitMQ : todo
1719
- couchdb: todo
18-
- sqlalchemy: todo
1920
- elastic search: todo
2021
- csv: todo
2122

@@ -35,6 +36,7 @@ The different modules are documented below:
3536
parser
3637
objects
3738
diff
39+
plugins_s3
3840

3941
Indices and tables
4042
==================

docs/plugins_s3.rst

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
libnmap.plugins.s3.NmapS3Plugin
2+
===============================
3+
4+
Using libnmap.plugins.s3
5+
------------------------
6+
7+
This modules enables the user to directly use S3 buckets to store and retrieve NmapReports.
8+
9+
NmapS3Plugin methods
10+
--------------------
11+
12+
.. automodule:: libnmap.plugins.s3
13+
.. autoclass:: NmapS3Plugin
14+
:members:

libnmap/objects.py

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -627,7 +627,7 @@ def save(self, backend):
627627
628628
The primary key of the stored object is returned.
629629
630-
:return: integer
630+
:return: str
631631
"""
632632
if backend is not None:
633633
#do stuff
@@ -877,6 +877,42 @@ def id(self):
877877
"""
878878
return hash(1)
879879

880+
def __eq__(self, other):
881+
"""
882+
Compare eq NmapReport based on :
883+
884+
- create a diff obj and check the result
885+
report are equal if added&changed&removed are empty
886+
887+
:return: boolean
888+
"""
889+
rval = False
890+
if(self.__class__ == other.__class__ and self.id == other.id):
891+
diffobj = self.diff(other)
892+
rval = (len(diffobj.changed()) == 0 and
893+
len(diffobj.added()) == 0 and
894+
len(diffobj.removed()) == 0
895+
)
896+
return rval
897+
898+
def __ne__(self, other):
899+
"""
900+
Compare ne NmapReport based on:
901+
902+
- create a diff obj and check the result
903+
report are ne if added|changed|removed are not empty
904+
905+
:return: boolean
906+
"""
907+
rval = True
908+
if(self.__class__ == other.__class__ and self.id == other.id):
909+
diffobj = self.diff(other)
910+
rval = (len(diffobj.changed()) != 0 or
911+
len(diffobj.added()) != 0 or
912+
len(diffobj.removed()) != 0
913+
)
914+
return rval
915+
880916
def __repr__(self):
881917
return "{0}: started at {1} hosts up {2}/{3}".format(
882918
self.__class__.__name__,

libnmap/plugins/backendplugin.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ def insert(self, NmapReport):
1414
"""
1515
insert NmapReport in the backend
1616
:param NmapReport:
17-
:return: the ident of the object in the backend for future usage
17+
:return: str the ident of the object in the backend for
18+
future usage
1819
or None
1920
"""
2021
raise NotImplementedError
@@ -34,9 +35,9 @@ def get(self, id):
3435
"""
3536
raise NotImplementedError
3637

37-
def getall(self):
38+
def getall(self, filter):
3839
"""
39-
:return: collection of NmapReport
40+
:return: collection of tuple (id,NmapReport)
4041
:param filter: Nice to have implement a filter capability
4142
"""
4243
raise NotImplementedError

libnmap/plugins/backendpluginFactory.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ class BackendPluginFactory(object):
1010
ie : mybackend = BackendPluginFactory.create()
1111
"""
1212
@classmethod
13-
def create(self, plugin_name="mongodb", **kwargs):
13+
def create(cls, plugin_name="mongodb", **kwargs):
1414
"""Import the needed lib and return an object NmapBackendPlugin
1515
representing the backend of your desire.
1616
NmapBackendPlugin is an abstract class, to know what argument
@@ -25,5 +25,8 @@ def create(self, plugin_name="mongodb", **kwargs):
2525
pluginclasses = inspect.getmembers(pluginobj, inspect.isclass)
2626
for classname, classobj in pluginclasses:
2727
if inspect.getmodule(classobj).__name__.find(plugin_path) == 0:
28-
backendplugin = classobj(**kwargs)
28+
try:
29+
backendplugin = classobj(**kwargs)
30+
except Exception as error:
31+
print "Cannot create Backend: %s" % (error)
2932
return backendplugin

libnmap/plugins/mongodb.py

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,15 @@
88
from libnmap.plugins.backendplugin import NmapBackendPlugin
99

1010

11-
class NmapMongoPlugin(NmapBackendPlugin):
11+
class NmapMongodbPlugin(NmapBackendPlugin):
12+
"""
13+
This class handle the persistence of NmapRepport object in mongodb
14+
Implementation is made using pymongo
15+
Object of this class must be create via the
16+
BackendPluginFactory.create(**url) where url is a named dict like
17+
{'plugin_name': "mongodb"} this dict may reeive all the param
18+
MongoClient() support
19+
"""
1220
def __init__(self, dbname=None, store=None, **kwargs):
1321
NmapBackendPlugin.__init__(self)
1422
if dbname is not None:
@@ -19,17 +27,23 @@ def __init__(self, dbname=None, store=None, **kwargs):
1927
self.collection = self.dbclient[self.dbname][self.store]
2028

2129
def insert(self, report):
22-
# create a json object from an NmapReport instance
30+
"""
31+
create a json object from an NmapReport instance
32+
:param NmapReport: obj to insert
33+
:return: str id
34+
"""
2335
j = json.dumps(report, cls=ReportEncoder)
2436
try:
25-
id = self.collection.insert(json.loads(j))
37+
oid = self.collection.insert(json.loads(j))
2638
except:
2739
print "MONGODB cannot insert"
2840
raise
29-
return id
41+
return str(oid)
3042

3143
def get(self, str_report_id=None):
32-
"""get return a NmapReport object
44+
""" select a NmapReport by Id
45+
:param str: id
46+
:return: NmapReport object
3347
"""
3448
rid = str_report_id
3549
nmapreport = None
@@ -38,29 +52,35 @@ def get(self, str_report_id=None):
3852

3953
if isinstance(rid, ObjectId):
4054
#get a specific report by mongo's id
41-
r = self.collection.find({'_id': rid})
42-
if r is not None:
55+
resultset = self.collection.find({'_id': rid})
56+
if resultset.count() == 1:
4357
#search by id means only one in the iterator
44-
record = r[0]
45-
#remove mongo's id
58+
record = resultset[0]
59+
#remove mongo's id to recreate the NmapReport Obj
4660
del record['_id']
4761
nmapreport = NmapParser.parse_fromdict(record)
4862
return nmapreport
4963

5064
def getall(self, dict_filter=None):
51-
"""return a list of all NmapReport saved in the backend
65+
"""return a list of tuple (id,NmapReport) saved in the backend
5266
TODO : add a filter capability
5367
"""
54-
nmapreportList = []
55-
r = self.collection.find()
56-
for report in r:
68+
nmapreportlist = []
69+
resultset = self.collection.find()
70+
for report in resultset:
71+
oid = report['_id']
5772
del report['_id']
5873
nmapreport = NmapParser.parse_fromdict(report)
59-
nmapreportList.append(nmapreport)
60-
return nmapreportList
74+
nmapreportlist.append((oid, nmapreport))
75+
return nmapreportlist
6176

6277
def delete(self, report_id=None):
78+
"""
79+
delete an obj from the backend
80+
:param str: id
81+
:return: dict document with result or None
82+
"""
6383
if report_id is not None and isinstance(report_id, str):
64-
self.collection.remove({'_id': ObjectId(report_id)})
84+
return self.collection.remove({'_id': ObjectId(report_id)})
6585
else:
66-
self.collection.remove({'_id': report_id})
86+
return self.collection.remove({'_id': report_id})

libnmap/plugins/s3.py

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
"""
2+
:mod:`libnmap.plugin.s3` -- S3 Backend Plugin
3+
===================================
4+
5+
.. module:: libnmap.plugin.s3
6+
:platform: Linux
7+
:synopsis: a plugin is representation of a S3 backend using boto
8+
.. moduleauthor:: Ronald Bister
9+
.. moduleauthor:: Mike Boutillier
10+
"""
11+
12+
#!/usr/bin/env python
13+
import json
14+
from bson.objectid import ObjectId
15+
from boto.s3.connection import S3Connection, OrdinaryCallingFormat
16+
from boto.s3.key import Key
17+
from boto.s3.bucketlistresultset import bucket_lister
18+
from boto.exception import S3ResponseError
19+
from libnmap.reportjson import ReportEncoder
20+
from libnmap.parser import NmapParser
21+
from libnmap.plugins.backendplugin import NmapBackendPlugin
22+
23+
24+
class NmapS3Plugin(NmapBackendPlugin):
25+
"""
26+
This plugin save the reports on S3 and compatible.
27+
"""
28+
def __init__(self, **kwargs):
29+
"""
30+
- create the conn object
31+
- create the bucket (if it doesn't exist)
32+
- if not given, awsaccessKey_nmapreport
33+
- may raise exception (ie in case of conflict bucket name)
34+
- sample :
35+
To connect to walrus:
36+
from libnmap.plugins.backendpluginFactory import
37+
BackendPluginFactory
38+
walrusBackend =
39+
BackendPluginFactory.create(
40+
plugin_name='s3',
41+
host="walrus.ecc.eucalyptus.com",
42+
path="/services/Walrus",port=8773,
43+
is_secure=False,
44+
aws_access_key_id='UU72FLVJCAYRATLXI70YH',
45+
aws_secret_access_key=
46+
'wFg7gP5YFHjVlxakw1g1uCC8UR2xVW5ax9ErZCut')
47+
To connect to S3:
48+
mybackend_S3 =
49+
BackendPluginFactory.create(
50+
plugin_name='s3',
51+
is_secure=True,
52+
aws_access_key_id='MYACCESSKEY',
53+
aws_secret_access_key='MYSECRET')
54+
"""
55+
NmapBackendPlugin.__init__(self)
56+
try:
57+
calling_format = OrdinaryCallingFormat()
58+
if 'bucket' not in kwargs:
59+
self.bucket_name = ''.join(
60+
[kwargs['aws_access_key_id'].lower(),
61+
"_nmapreport"])
62+
else:
63+
self.bucket_name = kwargs['bucket']
64+
del kwargs['bucket']
65+
kwargs['calling_format'] = calling_format
66+
self.conn = S3Connection(**kwargs)
67+
self.bucket = self.conn.lookup(self.bucket_name)
68+
if self.bucket is None:
69+
self.bucket = self.conn.create_bucket(self.bucket_name)
70+
except:
71+
raise
72+
73+
def insert(self, report):
74+
"""
75+
create a json string from an NmapReport instance
76+
and push it to S3 bucket.
77+
78+
:param NmapReport: obj to insert
79+
:rtype: string
80+
:return: str id
81+
:todo: Add tagging option
82+
"""
83+
try:
84+
oid = ObjectId()
85+
mykey = Key(self.bucket)
86+
mykey.key = str(oid)
87+
strjsonnmapreport = json.dumps(report, cls=ReportEncoder)
88+
mykey.set_contents_from_string(strjsonnmapreport)
89+
except:
90+
print "Bucket cannot insert"
91+
raise
92+
return str(oid)
93+
94+
def get(self, str_report_id=None):
95+
"""
96+
select a NmapReport by Id.
97+
98+
:param str: id
99+
:rtype: NmapReport
100+
:return: NmapReport object
101+
"""
102+
nmapreport = None
103+
if str_report_id is not None and isinstance(str_report_id, str):
104+
try:
105+
mykey = Key(self.bucket)
106+
mykey.key = str_report_id
107+
nmapreportjson = json.loads(mykey.get_contents_as_string())
108+
nmapreport = NmapParser.parse_fromdict(nmapreportjson)
109+
except S3ResponseError:
110+
print "Not Found"
111+
return nmapreport
112+
113+
def getall(self, dict_filter=None):
114+
"""
115+
:rtype: List of tuple
116+
:return: list of key/report
117+
:todo: add a filter capability
118+
"""
119+
nmapreportlist = []
120+
for key in bucket_lister(self.bucket):
121+
if isinstance(key, Key):
122+
nmapreportjson = json.loads(key.get_contents_as_string())
123+
nmapreport = NmapParser.parse_fromdict(nmapreportjson)
124+
nmapreportlist.append((key.key, nmapreport))
125+
return nmapreportlist
126+
127+
def delete(self, report_id=None):
128+
"""
129+
delete an obj from the backend
130+
131+
:param str: id
132+
:return: dict document with result or None
133+
"""
134+
rcode = None
135+
if report_id is not None and isinstance(report_id, str):
136+
rcode = self.bucket.delete_key(report_id)
137+
return rcode

0 commit comments

Comments
 (0)