Tasks and coroutines in python 3.5

The following code demonstrates the using of python 3.5  coroutines in order to implement async IO

import asyncio
from threading import Thread, current_thread
import datetime
print (current_thread())
async def compute(x, y):
    print("Compute %s + %s ... Current thread:%s" % (x, y , current_thread() ))
  
    await asyncio.sleep(1.0)
    return x + y
async def print_sum(x, y):
    print (current_thread())
    result = await compute(x, y)
    print("%s + %s = %s" % (x, y, result))
loop = asyncio.get_event_loop()
theTask1 = loop.create_task(print_sum(1, 2))
theTask2 = loop.create_task(print_sum(4, 2))
theTask3 = loop.create_task(print_sum(14, 2))
tasks = [theTask1,theTask2,theTask3]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()

And the result:
<_MainThread(MainThread, started 1696)>
<_MainThread(MainThread, started 1696)>
Compute 1 + 2 ... Current thread:<_MainThread(MainThread, started 1696)>
<_MainThread(MainThread, started 1696)>
Compute 4 + 2 ... Current thread:<_MainThread(MainThread, started 1696)>
<_MainThread(MainThread, started 1696)>
Compute 14 + 2 ... Current thread:<_MainThread(MainThread, started 1696)>
1 + 2 = 3
4 + 2 = 6
14 + 2 = 16

References
https://docs.python.org/3/library/asyncio-task.html
http://wla.berkeley.edu/~cs61a/fa11/lectures/streams.html#coroutines

 

functools partial

The functools.partial allow to  wrapped a method with anew method that has only  partial of the parameters of the wrapped function .
In constract to using lambda for this ‘the functools.partial keep the parameters as reference that can be changed after declaring the wrapper function
Example:

import functools
fullName = lambda pName , pFamily : pName + " "+ pFamily
n = "Zvika"
GetName = lambda y: fullName(n, y)
GetName2 = functools.partial(fullName, n)
print ("n set before creating the lambda")
print (GetName("Peer"), GetName2("Peer"))
print ("n set after creating the lambda")
n = "Alon"
print (GetName("Peer"), GetName2("Peer"))

And the results: 
 n set before creating the lambda
Zvika Peer Zvika Peer
n set after creating the lambda
Alon Peer Zvika Peer

Resources:
http://stackoverflow.com/questions/3252228/python-why-is-functools-partial-necessary

functools @wraps

The functools @wrap allow to keep attributed function original  name
For an example:

from functools import wraps
def myAttribute(func):
    @wraps(func)
    def WrapprintName (*args, **kwargs):
        print ("The method is: " + func.__name__ )
        return func(*args, **kwargs)
    return WrapprintName
@myAttribute
def printName (pName ):
    """Print my name """
    print (pName)
print (printName.__name__)  # prints 'f'
print (printName.__doc__)   # prints 'does some math'
printName ("Zvika")

And the result:
printName
Print my name
The method isprintName
Zvika

The printName method name was kept although the printName method was annotated with @myAttribute

functools.total_ordering

Example of using the  @functools.total_ordering (based on the sample from http://pymotw.com/2/functools/)

import functools
import inspect
from pprint import pprint
@functools.total_ordering
class MyObject(object):
    def __init__(self, val):
        self.val = val
    def __eq__(self, other):
        print ('  testing __eq__(%s, %s)' % (self.val, other.val))
        return len ( self.val)  == len ( other.val)
    def __gt__(self, other):
        print ('  testing __gt__(%s, %s)' % (self.val, other.val))
        return len (self.val) > len (other.val)
    def TestMethod(self):
        pass
print ('Methods:\n')
pprint(inspect.getmembers(MyObject))
a = MyObject("Long name bla bal ")
b = MyObject("Short mame")
print ('\nComparisons:')
for expr in [ 'a < b','a > b', 'a <= b', 'a == b', 'a >= b' ]:
    print ('\n%-6s:' % expr)
    result = eval(expr)
    print ('  result of %s: %s' % (expr, result))

The Result:
Methods:
[('TestMethod', <function MyObject.TestMethod at 0x03146270>),
('__class__', <class 'type'>),
('__delattr__', <slot wrapper '__delattr__' of 'object' objects>),
('__dict__',
  mappingproxy({'__module__': '__main__', '__le__': <function total_ordering.<locals>.<lambda> at 0x031464B0>, '__dict__': <attribute '__dict__' of 'MyObject' objects>, '__eq__': <function MyObject.__eq__ at 0x031461E0>, '__doc__': None, '__hash__': None, '__lt__': <function total_ordering.<locals>.<lambda> at 0x03146420>, '__gt__': <function MyObject.__gt__ at 0x03146228>, '__init__': <function MyObject.__init__ at 0x03132A08>, 'TestMethod': <function MyObject.TestMethod at 0x03146270>, '__weakref__': <attribute '__weakref__' of 'MyObject' objects>, '__ge__': <function total_ordering.<locals>.<lambda> at 0x03146468>})),
('__dir__', <method '__dir__' of 'object' objects>),
('__doc__', None),
('__eq__', <function MyObject.__eq__ at 0x031461E0>),
('__format__', <method '__format__' of 'object' objects>),
('__ge__', <function total_ordering.<locals>.<lambda> at 0x03146468>),
('__getattribute__', <slot wrapper '__getattribute__' of 'object' objects>),
('__gt__', <function MyObject.__gt__ at 0x03146228>),
('__hash__', None),
('__init__', <function MyObject.__init__ at 0x03132A08>),
('__le__', <function total_ordering.<locals>.<lambda> at 0x031464B0>),
('__lt__', <function total_ordering.<locals>.<lambda> at 0x03146420>),
('__module__', '__main__'),
('__ne__', <slot wrapper '__ne__' of 'object' objects>),
('__new__', <built-in method __new__ of type object at 0x5E774EA8>),
('__reduce__', <method '__reduce__' of 'object' objects>),
('__reduce_ex__', <method '__reduce_ex__' of 'object' objects>),
('__repr__', <slot wrapper '__repr__' of 'object' objects>),
('__setattr__', <slot wrapper '__setattr__' of 'object' objects>),
('__sizeof__', <method '__sizeof__' of 'object' objects>),
('__str__', <slot wrapper '__str__' of 'object' objects>),
('__subclasshook__',
  <built-in method __subclasshook__ of type object at 0x03135E30>),
('__weakref__', <attribute '__weakref__' of 'MyObject' objects>)]

Comparisons:
a < b :

  testing __gt__(Long name bla bal , Short mame)

  result of a < b: False

a > b :

  testing __gt__(Long name bla bal , Short mame)

  result of a > b: True

a <= b:

  testing __gt__(Long name bla bal , Short mame)

  result of a <= b: False

a == b:

  testing __eq__(Long name bla bal , Short mame)

  result of a == b: False

a >= b:

  testing __gt__(Long name bla bal , Short mame)

  result of a >= b: True

Notes
1.The @functools.total_ordering annotation add implementation of the <= and the => compressions sign  based on the gt and eq methods (without the annotation an exception will be raised ) .
2.The pretty print module handle the iterable collection printing (adding the line feed and cr after printing each member of the list)
3.The inspect module is responsible for getting classes meta data .
4.The  __gt__ and the   __eq__ must be override when using  @functools.total_ordering

Check if tcp port is open using scapy

The simple way to scan the open port
sudo nmap -sS -O 192.168.0.1

The following python script  checks if a port is open using scapy .

import logging
import sys
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
from scapy.all import *
dst_ip = "192.168.0.1"
src_port = 400
dst_port=80
 
tcp_connect_scan_resp = sr1(IP(dst=dst_ip)/TCP(sport=src_port,dport=dst_port,flags="S"),timeout=3)
if(tcp_connect_scan_resp is None):
    print ("The port is Closed")
    sys.exit()
     
print ("The flags:" + str (tcp_connect_scan_resp.getlayer(TCP).flags))    
    
if(tcp_connect_scan_resp.haslayer(TCP)):
    if(tcp_connect_scan_resp.getlayer(TCP).flags == 0x12):
        #send_rst = sr(IP(dst=dst_ip)/TCP(sport=src_port,dport=dst_port,flags="AR"),timeout=3)
        print ("The port is Open")
        sys.exit ();
            
#(tcp_connect_scan_resp.getlayer(TCP).flags == 0x14):
print ("The port is Closed ")

notes:
Currently scapy supports only Python 2.7 .
Needed sudo privilege in order to execute script with scapy .

The results:
zvika@ubuntu:~/myStaff/myCode$ sudo python PortsScan.py
Begin emission:
..Finished to send 1 packets.
*
Received 3 packets, got 1 answers, remaining 0 packets
The flags:18
The port is Open

The code is based on :
http://resources.infosecinstitute.com/port-scanning-using-scapy/

Other good references : 
http://theitgeekchronicles.files.wordpress.com/2012/05/scapyguide1.pdf
http://thesprawl.org/research/scapy/
http://thepacketgeek.com/scapy-p-06-sending-and-receiving-with-scapy/

Outer join in cyper

The following code :

  1: def handle_row(row):
  2:     print ("cypher query result:")
  3:     print ( len (row))
  4:     for nextNode in row:
  5:         print (nextNode)
  6:         print (type(nextNode))
  7: 
  8: 
  9: theCyperCode = "START n=node(1) ,other=node(2, 3)  RETURN  n,other"
 10: 
 11: cypher.execute(graph_db, theCyperCode, row_handler=handle_row)

returns
cypher query result:
2
(1 {"name":"Bruce Willis"})
<class 'py2neo.neo4j.Node'>
(2 {"name":"John McClane"})
<class 'py2neo.neo4j.Node'>
cypher query result:
2
(1 {"name":"Bruce Willis"})
<class 'py2neo.neo4j.Node'>
(3 {"name":"Alan Rickman"})
<class 'py2neo.neo4j.Node'>
Start

Making a flat list out of list of lists in Python

 

I have found the following ways to flatten a list in python

  1: import timeit 
  2: import itertools
  3: 
  4: def reduce(function, iterable, initializer=None):
  5:     it = iter(iterable)
  6:     if initializer is None:
  7:         try:
  8:             initializer = next(it)
  9:         except StopIteration:
 10:             raise TypeError('reduce() of empty sequence with no initial value')
 11:     accum_value = initializer
 12:     for x in it:
 13:         accum_value = function(accum_value, x)
 14:     return accum_value
 15: 
 16: 
 17: myList =[['a',2,3],[4,5,6], ['zvika'], [(1,5),8,9]]*2
 18: 
 19: Way1 = [item for sublist in myList for item in sublist]
 20: 
 21: print ( Way1 )
 22: 
 23: Way2 = sum(myList, [])
 24: 
 25: print ( Way2 )
 26: 
 27: Way3 = reduce(lambda x,y: x+y,myList)
 28: 
 29: print ( Way3 )
 31: Way4 = list(itertools.chain.from_iterable(myList))
 32: 
 33: print ( Way4)
 34: 
 35: TimeTest =
 30: 
 timeit.Timer(
 36:         'sum(l, [])',
 37:         'l=[[1, 2, 3], [4, 5, 6, 7, 8], [1, 2, 3, 4, 5, 6, 7]]'
 38:     ).timeit()
 39: 
 40: print ( TimeTest )

The first way list inside a list.
The second based on a simple explaining :
Just contact one list to the other. 

  1: l = [['Add my'],['and me']]
  2: 
  3: Way2 = sum(l, [])
  4: 
  5: print (Way2)

The 3 way is base one the python reduce function (similar to the sum)
The 4 use the itertools chain method
Note the timeit is a util that helps to measure performance of a code
Resources:
http://stackoverflow.com/questions/952914/making-a-flat-list-out-of-list-of-lists-in-python

Pulling sub document from a document

The following code demonstrates the removing of sub document from a document using the pull command .

  1: import pymongo
  2: from datetime import *
  3: client = pymongo.MongoClient("localhost", 27017)
  4: 
  5: db = client['test-database']
  6: 
  7: userCollection = db['userCollection']
  8: 
  9: userCollection.remove ({"Name":"Uzi"} ,safe=True)
 10: 
 11: new_user = {"Name":"Uzi",
 12:     "Age":90,
 13:     "Childs":["fstChild","scnChild"],
 14:     "dateofbirth" : datetime(1970, 10, 25),
 15:     "email" : "loveme42@hotmail.com",
 16:     "RunningNo":nextNo ,
 17:     "Blog address":"http://zvikastechnologiesblog.blogspot.com",
 18:     "BankAcounts":[{"Name":"Leumi" ,"No":"123"},{"Name":"apoalim" ,"No":"456"}]}
 19: 
 20: userCollection.save (new_user)
 21: 
 22: userCollection.update({"Name":"Uzi"},
 23: {"$pull":{"BankAcounts":{"No":{"$ne":"123"}}}}, safe=True)
 24: 
 25: users = db.userCollection.find({"Name":"Uzi"},limit=1)
 26: 
 27: for user in users:
 28:     print ( type ( user))
 29:     print (  user)

And the result:
{'Age': 90, 'dateofbirth': datetime.datetime(1970, 10, 25, 0, 0), 'Blog address': 'http://zvikastechnologiesblog.blogspot.com', '_id': ObjectId('51b5c0fddef1c125e8218723'), 'RunningNo': 175, 'email': 'loveme42@hotmail.com', 'Name': 'Uzi', 'BankAcounts': [{'Name': 'Leumi', 'No': '123'}], 'Childs': ['fstChild', 'scnChild']}

The Mongo Query removes all the sub documents of docuemnts where the sub docuemnt "No" value is not equal to "123"
Note:There is a deiffrent between 123 and “123”.

python properties setter getter and deleter

Property attribute allows to declare read only attribute using the @property attribute.
For an example:

  1: class myCls():
  2:     def __init__(self):
  3:         pass 
  4:         self.myPropValue = 164
  5:     @property
  6:     def ProoValue(self):
  7:         return self.myPropValue
  8: 
  9: theCls = myCls()
 10: 
 11: print (theCls.ProoValue)
 12: 
 13: print ("Done")

Declaring a full property in python is done using properties setter and getter  and deleter attributes.
Note that the property should be declare using the @property attribute in front  of the setter and getter declarations.

  1: class myCls():
  2:     def __init__(self):
  3:         pass 
  4:         self.myPropValue = 164
  5:     @property
  6:     def ProoValue(self):
  7:         return self.myPropValue
  8: 
  9:     @ProoValue.setter
 10:     def ProoValue(self, value):
 11:         self.myPropValue = value
 12: 
 13:     @ProoValue.deleter
 14:     def ProoValue(self):
 15:         del self.myPropValue
 16: 
 17: theCls = myCls()
 18: 
 19: theCls.ProoValue = 6
 20: 
 21: print (theCls.ProoValue)
 22: 
 23: del theCls.ProoValue
 24: 
 25: print ("Done")

Play with Payton collections

Derived my class from dict and overriding the __missing__ method used to return value in case the key is not in the collection.

  1: class myTestCls (dict):
  2:     def __missing__ (self, key):
  3:         return "Not here sory"
  4: theCls = myTestCls();
  5: 
  6: print ( theCls["TestValue"])
  7: 
  8: print (" -- done --") 
  9: 
 10: 

Check if the new type collection is equal to base collection with the same values

  1: class myTestCls (dict):
  2:     def __missing__ (self, key):
  3:         return "Not here sory"
  4: 
  5: theCls1 = myTestCls(cat=1, dog=2, pig=3)
  6: 
  7: theCls2 = {'cat': 1, 'dog': 2, 'pig': 3} 
  8: 
  9: print ( theCls1 == theCls2)
 10: 
 11: print ( type ( theCls1))
 12: 
 13: print ( type ( theCls2))
 14: 
 15: print (" -- done --") 
 16: 

The results
True
<class '__main__.myTestCls'>
<class 'dict'>

Override the __eq__ method in order to check that the types of the objects are the same.

  1: class myTestCls (dict):
  2:     def __missing__ (self, key):
  3:         return "Not here sory"
  4:     def __eq__(self, other):
  5:         if type (other) == type( self):
  6:             return super(myTestCls, self).__eq__( other)
  7:         return False
  8:         
  9:     def __ne__(self, other):
 10:         return NotImplemented
 11: 
 12: theCls1 = myTestCls(cat=1, dog=2, pig=3)
 13: 
 14: theCls2 = {'cat': 1, 'dog': 2, 'pig': 3} 
 15: 
 16: theCls3 = myTestCls(zip (['cat', 'dog', 'pig'], [1, 2, 3]))
 17: 
 18: print ( theCls1 == theCls2)
 19: 
 20: print ( theCls1 == theCls3)
 21: 
 22: print ( type ( theCls1))
 23: 
 24: print ( type ( theCls2))
 25: 
 26: print ( type ( theCls3))
 27: 
 28: print (" -- done –")

returns
False
True
<class '__main__.myTestCls'>
<class 'dict'>
<class '__main__.myTestCls'>

http://docs.python.org/3.3/library/stdtypes.html#set

MongoDB Cont

Deleting documents from collection .
The following code deletes all documents that the property RunningNo value is 9

  1: import pymongo
  2: from datetime import *
  3: client = pymongo.MongoClient("localhost", 27017)
  4: #Create of get the DB  
  5: db = client['test-database']
  6: 
  7: print (db.name)
  8: 
  9: #create user collection
 10: 
 11: userCollection = db['userCollection']
 12: 
 13: for nextNo in range (1,10,2):
 14: 
 15:     new_user = {"Name":"zvika",
 16:                 "Age":42,
 17:                 "Childs":["Lior","Gal","Shahf"],
 18:                 "dateofbirth" : datetime(1970, 10, 25),
 19:                 "email" : "loveme42@hotmail.com",
 20:                 "RunningNo":nextNo ,
 21:                 "Blog address":"http://zvikastechnologiesblog.blogspot.com"}
 22: 
 23:     userCollection.save (new_user)
 24: 
 25: userCollection.remove ({"RunningNo":9} ,safe=True)
 26: 
 27: users = db.userCollection.find({"Name":"zvika"},{"RunningNo":1,"email":2},sort=[("RunningNo", pymongo.DESCENDING)] ,limit=24)
 28: 
 29: for user in users:
 30:     print ( type ( user))
 31:     print (  user)
 

using sub documents

  1: new_user = {"Name":"zvika",
  2:     "Age":42,
  3:     "Childs":["Lior","Gal","Shahf"],
  4:     "dateofbirth" : datetime(1970, 10, 25),
  5:     "email" : "loveme42@hotmail.com",
  6:     "RunningNo":nextNo ,
  7:     "Blog address":"http://zvikastechnologiesblog.blogspot.com",
  8:     "BankAcounts":[{"Name":"Leumi" ,"No":"123"},{"Name":"apoalim" ,"No":"456"}]}
  9: 
 10: userCollection.save (new_user)
 11: 
 12: users = db.userCollection.find({"BankAcounts.Name":"Leumi"},limit=1)

Note: the code above returns that all  documents that match the query

Setting anew Property

  1: db.userCollection.update({"BankAcounts.Name":"Leumi"},
  2: {"$set":{"NewField":"NewFieldValue"}}, safe=True)

The Result:
{'email': 'loveme42@hotmail.com', 'Name': 'zvika', 'BankAcounts': [{'No': '123', 'Name': 'Leumi'}, {'No': '456', 'Name': 'apoalim'}], 'Childs': ['Lior', 'Gal', 'Shahf'], 'dateofbirth': datetime.datetime(1970, 10, 25, 0, 0), 'Blog address': 'http://zvikastechnologiesblog.blogspot.com', 'RunningNo': 175, 'NewField': 'NewFieldValue', '_id': ObjectId('51a4bcb4def1c124d490d385'), 'Age': 42}

Insert new property to the collection

  1: db.userCollection.update({"BankAcounts.Name":"Leumi"},
  2:     {"$set":{"BankAcounts.$.NewField":"NewFieldValue"}}, safe=True)

note the $ sign is used to represent a collection
The Result:
{'RunningNo': 175, 'BankAcounts': [{'NewField': 'NewFieldValue', 'No': '123', 'Name': 'Leumi'}, {'No': '456', 'Name': 'apoalim'}], '_id': ObjectId('51a4bd67def1c1143c53f8a4'), 'email': 'loveme42@hotmail.com', 'Name': 'zvika', 'Blog address': 'http://zvikastechnologiesblog.blogspot.com', 'Childs': ['Lior', 'Gal', 'Shahf'], 'Age': 42, 'dateofbirth': datetime.datetime(1970, 10, 25, 0, 0)}

Cypher Optional relationships

The cypher optional relationships is equivalent to the SQL outer join .
It is declared by using the [?] sign and it is used in the case where  if there is no relationship or node found we wants the query to return a null node instead of not returning a result at all .

The sample DB used in this post :
from py2neo import neo4j
from py2neo import node, rel
from py2neo import  cypher

graph_db = neo4j.GraphDatabaseService("http://localhost:7474/db/data/")

die_hard = graph_db.create(
    node(name="Bruce Willis"),
    node(name="John McClane"),
    node(name="Alan Rickman"),
    node(name="Hans Gruber"),
    node(name="Nakatomi Plaza"),
    rel(0, "PLAYS", 1),
    rel(2, "PLAYS", 3),
    rel(1, "VISITS", 4),
    rel(3, "STEALS_FROM", 4),
    rel(1, "KILLS", 3),
)

Running the following example :

theCyperCode = "START Bruce=node(1) MATCH Bruce-->ConnectedNode-[?]->NodeConectedToConnectedNode RETURN ConnectedNode, NodeConectedToConnectedNode"

cypher.execute(graph_db, theCyperCode, row_handler=handle_row)
produce the following results :
cypher query result:
(2 {"name":"John McClane"})
(5 {"name":"Nakatomi Plaza"})
cypher query result:
(2 {"name":"John McClane"})
(4 {"name":"Hans Gruber"})

node 2 has 2  connections
Try a case where there is no results :

theCyperCode = "START Hans=node(4) MATCH Hans-->ConnectedNode-[?]->NodeConectedToConnectedNode RETURN ConnectedNode, NodeConectedToConnectedNode"
 
cypher.execute(graph_db, theCyperCode, row_handler=handle_row)

produce the following results :

cypher query result:
(5 {"name":"Nakatomi Plaza"})
None


Node 5 has no connection going out from it. therefore the optional relation chip return null as the connected node.

In case where the optional relationship is not used :

theCyperCode = "START Hans=node(4) MATCH Hans-->ConnectedNode-->NodeConectedToConnectedNode RETURN ConnectedNode, NodeConectedToConnectedNode"
     
cypher.execute(graph_db, theCyperCode, row_handler=handle_row)
No results will return at all.