8000 Add mongosource by miraculixx · Pull Request #16 · omegaml/minibatch · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -109,3 +109,4 @@ dmypy.json
.pyre/

# End of https://www.gitignore.io/api/python
/config.yml
9 changes: 5 additions & 4 deletions minibatch/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@


def streaming(name, interval=None, size=None, emitter=None,
relaxed=True, keep=False, url=None, sink=None, **kwargs):
relaxed=True, keep=False, url=None, sink=None,
queue=None, **kwargs):
"""
make and call a streaming function

Expand Down Expand Up @@ -57,7 +58,7 @@ def myproc(window):
def inner(fn):
em = make_emitter(name, fn, interval=interval, size=size,
emitter=emitter, relaxed=relaxed, keep=keep,
url=url, sink=sink, **kwargs)
url=url, sink=sink, queue=queue, **kwargs)
em.persist(keep)
em.run()

Expand All @@ -74,7 +75,7 @@ class IntegrityError(Exception):


def make_emitter(name, emitfn, interval=None, size=None, relaxed=False,
url=None, sink=None, emitter=None, keep=False, **kwargs):
url=None, sink=None, emitter=None, keep=False, queue=None, **kwargs):
from minibatch.window import RelaxedTimeWindow, FixedTimeWindow, CountWindow

if interval is None and size is None:
Expand All @@ -84,7 +85,7 @@ def make_emitter(name, emitfn, interval=None, size=None, relaxed=False,

emitfn._count = 0
stream = Stream.get_or_create(name, interval=interval or size, url=url)
kwargs.update(stream=stream, emitfn=emitfn, forwardfn=forwardfn)
kwargs.update(stream=stream, emitfn=emitfn, forwardfn=forwardfn, queue=queue)
if interval and emitter is None:
if relaxed:
em = RelaxedTimeWindow(name, interval=interval, **kwargs)
Expand Down
3 changes: 2 additions & 1 deletion minibatch/contrib/kafka.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from kafka import KafkaConsumer, KafkaProducer
from json import loads, dumps

from kafka import KafkaConsumer, KafkaProducer


class KafkaSource:
"""
Expand Down
121 changes: 121 additions & 0 deletions minibatch/contrib/mongodb.py
10BC8
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
import pymongo
from time import sleep

from minibatch import logger


class MongoSource:
"""
A mongodb collection source

Usage:
# start consuming from mongo collection
stream = mb.stream('test')
source = MongoSource(collection)
stream.attach(source)

# stream to a python callable
streaming('test')(lambda v: print(v))

Args:
collection (pymongo.Collection): a mongo collection
size (int): the number of new documents to fetch for each stream.append
defaults to 1
idcol (str): the name of the id column, defaults to _id
delay (float): the wait time in seconds between change queries, defaults
to .1

Notes:
* the collection must have a key column that is naturally ordered (ascending),
that is a new records' key must be compare greater as any previous key

* by default MongoSources uses the object id (_id column) as the sort order,
because it is naturally increasing for each new inserted object. Specify
as idcol='column name'

* MongoSource implements a polling change observer that is executed on
once every delay seconds, for every query it retrieves at most N=size
messages. All messages are appended to the stream one by one.
If you know that new messages arrive more or less frequently change
either size or delay to optimize polling behavior. For example if
messages arrive more frequently than .1 seconds but processing them
in steps of > .1 seconds is ok, specify size=number of messages in each
interval. If messages arrive less frequently than every .1 seconds,
considering specifying a delay > .1 seconds to reduce the polling load
on the database.

* For a Mongo replicaset, use the MongoReplicasetSource instead. It uses
the MongoDB native change stream instead of a polling change observer
which is more efficient.
"""

def __init__(self, collection, size=1, idcol=None, delay=.1):
self.collection = collection
self._cancel = None
self._lastid = None
self._size = size
self._idcol = '_id'
self._delay = delay

def changes(self, N=1):
latest_id = None
while True:
if self._cancel:
break
sortkey = {
'sort': [('_id', pymongo.ASCENDING)],
}
query = {}
if latest_id is not None:
query[self._idcol] = {
'$gt': latest_id
}
docs = self.collection.find(query, **sortkey).limit(N)
for doc in docs:
latest_id = doc['_id']
yield doc
sleep(self._delay)

def stream(self, stream):
self._cancel = False
while not self._cancel:
for doc in self.changes(N=self._size):
if self._cancel:
break
stream.append(doc)
logger.debug("stream done")

def cancel(self):
self._cancel = True


class MongoSink:
"""
A mongodb collection sink
"""
def __init__(self, collection):
self.collection = collection

def put(self, messages):
if isinstance(messages, dict):
messages = [messages]
return self.collection.insert_many(messages)


class MongoReplicasetSource(MongoSource):
def changes(self, N=1):
criteria = [
{'$match': {
'operationType': 'insert'
}}
]
self._cancel = False
docs = []
with self.collection.watch(criteria) as changes:
for doc in changes:
if self._cancel:
break
docs.append(doc)
if len(docs) >= N:
yield docs
docs = []
51 changes: 51 additions & 0 deletions minibatch/example/mongodb/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
from multiprocessing import Process

from datetime import datetime
from time import sleep

from minibatch import connectdb, streaming, stream
from minibatch.contrib.mongodb import MongoSource, MongoSink
from minibatch.contrib.mqtt import MQTTSource, MQTTSink
from minibatch.example.util import clean


def consumer():
@streaming('test', size=1, keep=True)
def myprocess(window):
try:
db = connectdb(alias='consumer')
print("consuming ... {}".format(window.data))
db.processed.insert_one({'data': window.data or {}})
except Exception as e:
print(e)
return window


def main():
print("setting up")
clean()
# setup mqtt source and producer
url = 'mongodb://localhost/test'
db = connectdb(url=url)
source_coll = db['source']
sink_coll = db['processed']
source = MongoSource(source_coll)
producer = MongoSink(sink_coll)
# attach to the stream
s = stream('test')
s.attach(source)
# set up a streaming function
emitp = Process(target=consumer)
emitp.start()
# publish some messages
print("publishing messages")
for i in range(10):
producer.put(dict(foo='bar', time=datetime.now().isoformat()))
sleep(.1)
# check we got the messages
print("wait to receive all messages")
sleep(3)
docs = list(doc for doc in sink_coll.find())
print("processed items:", len(docs))
print(docs)
emitp.terminate()
4 changes: 4 additions & 0 deletions minibatch/example/mongodb/__main__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
from minibatch.example.mongodb import main

if __name__ == '__main__':
main()
10 changes: 6 additions & 4 deletions minibatch/tests/test_kafka.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from multiprocessing import Process
from multiprocessing import Process, Queue
from unittest import TestCase

from threading import Thread
Expand Down Expand Up @@ -28,17 +28,19 @@ def test_consumer(self):
s = stream('test', url=self.url)
s.attach(source)

def consumer():
def consumer(q):
url = str(self.url)

@streaming('test', executor=LocalExecutor(), url=url)
@streaming('test', executor=LocalExecutor(), url=url, queue=q)
def process(window):
db = connectdb(url=url)
db.processed.insert(window.data)

p = Process(target=consumer)
q = Queue()
p = Process(target=consumer, args=(q,))
p.start()
sleep(1)
q.put(True)
p.terminate()

docs = list(self.db.processed.find())
Expand Down
32 changes: 20 additions & 12 deletions minibatch/tests/test_minibatch.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from multiprocessing import Process
from multiprocessing import Process, Queue
from unittest import TestCase

import sys
Expand Down Expand Up @@ -46,13 +46,13 @@ def test_fixed_size(self):
"""
from minibatch import streaming

def consumer():
def consumer(q):
logger.debug("starting consumer on {self.url}".format(**locals()))
url = str(self.url)

# note the stream decorator blocks the consumer and runs the decorated
# function asynchronously upon the window criteria is satisfied
@streaming('test', size=2, keep=True, url=self.url)
@streaming('test', size=2, keep=True, url=self.url, queue=q)
def myprocess(window):
logger.debug("*** processing")
try:
Expand All @@ -63,7 +63,8 @@ def myprocess(window):
raise

# start stream consumer
proc = Process(target=consumer)
q = Queue()
proc = Process(target=consumer, args=(q,))
proc.start()
# fill stream
stream = Stream.get_or_create('test', url=self.url)
Expand All @@ -72,6 +73,7 @@ def myprocess(window):
# give it some time to process
logger.debug("waiting")
self.sleep(10)
q.put(True)
proc.terminate()
# expect 5 entries, each of length 2
data = list(doc for doc in self.db.processed.find())
Expand All @@ -85,12 +87,12 @@ def test_timed_window(self):
"""
from minibatch import streaming

def consumer():
def consumer(q):
# note the stream decorator blocks the consumer and runs the decorated
# function asynchronously upon the window criteria is satisfied
url = str(self.url)

@streaming('test', interval=1, keep=True)
@streaming('test', interval=1, keep=True, queue=q)
def myprocess(window):
try:
db = connectdb(url=url)
Expand All @@ -100,7 +102,8 @@ def myprocess(window):
return window

# start stream consumer
proc = Process(target=consumer)
q = Queue()
proc = Process(target=consumer, args=(q,))
proc.start()
# fill stream
stream = Stream.get_or_create('test', url=self.url)
Expand All @@ -109,6 +112,7 @@ def myprocess(window):
self.sleep(.5)
# give it some time to process
self.sleep(5)
q.put(True)
proc.terminate()
# expect at least 5 entries (10 x .5 = 5 seconds), each of length 1-2
data = list(doc for doc in self.db.processed.find())
Expand All @@ -122,7 +126,7 @@ def test_timed_window_relaxed(self):
"""
from minibatch import streaming

def consumer():
def consumer(q):
# note the stream decorator blocks the consumer and runs the decorated
# function asynchronously upon the window criteria is satisfied
url = str(self.url)
Expand All @@ -137,7 +141,8 @@ def myprocess(window):
return window

# start stream consumer
proc = Process(target=consumer)
q = Queue()
proc = Process(target=consumer, args=(q,))
proc.start()
# fill stream
stream = Stream.get_or_create('test', url=self.url)
Expand All @@ -146,6 +151,7 @@ def myprocess(window):
self.sleep(.5)
# give it some time to process
self.sleep(5)
q.put(True)
proc.terminate()
# expect at least 5 entries (10 x .5 = 5 seconds), each of length 1-2
data = list(doc for doc in self.db.processed.find())
Expand All @@ -162,13 +168,13 @@ def _do_test_slow_emitfn(self, workers=None, expect_fail=None, timeout=None):
MiniBatchTests._do_test_slow_emitfn.__doc__ = MiniBatchTests._do_test_slow_emitfn.__doc__.format(
workers=workers)

def consumer(workers):
def consumer(workers, q):
logger.debug("starting consumer on={self.url} workers={workers}".format(**locals()))
url = str(self.url)

# note the stream decorator blocks the consumer and runs the decorated
# function asynchronously upon the window criteria is satisfied
@streaming('test', size=2, keep=True, url=self.url, max_workers=workers)
@streaming('test', size=2, keep=True, url=self.url, max_workers=workers, queue=q)
def myprocess(window):
logger.debug("*** processing {}".format(window.data))
from minibatch import connectdb
Expand All @@ -191,7 +197,8 @@ def check():

# start stream consumer
# -- use just one worker, we expect to fail
proc = Process(target=consumer, args=(workers,))
q = Queue()
proc = Process(target=consumer, args=(workers, q))
proc.start()
# fill stream
stream = Stream.get_or_create('test', url=self.url)
Expand All @@ -202,6 +209,7 @@ def check():
# note it takes at least 25 seconds using 1 worker (5 windows, 5 seconds)
# so we expect to fail
self.sleep(12)
q.put(True)
proc.terminate()
if expect_fail:
with self.assertRaises(AssertionError):
Expand Down
Loading
0