Ruby and Set

  1: require 'set'
  2: 
  3: mySet = Set.new ["Orange","Banana","Kivy"]
  4: 
  5: mySet.each {|nextFruit| puts nextFruit}
  6: 
  7: puts mySet.inspect
  8: 
  9: puts "Done"

Returns:
Orange
Banana
Kivy
#<Set: {"Orange", "Banana", "Kivy"}>
Done
Dividing the set into smaller set by condition

  1: require 'set'
  2: 
  3: mySet = Set.new ["Orange","Banana","Kivy","Limon","Mango"]
  4: 
  5: dsets = mySet.divide { |fr1,fr2| fr1.length == fr2.length }
  6: 
  7: dsets.each {|nextSet| puts nextSet.inspect}
  8: 
  9: puts "Done"

Returns
I:\Learn\Ruby>myRuby7
#<Set: {"Orange", "Banana"}>
#<Set: {"Kivy"}>
#<Set: {"Limon", "Mango"}>
Done

Hbase

A great hello world tutorial explaining about how to start with Hbase and Hadoop can be found Here.
This is my summery and notes about the post :
Installing the SSH server:
sudo apt-get install openssh-server

Create the Hadoop user:
sudo addgroup hadoop
sudo adduser --ingroup hadoop huser

Generate the user public keys:
#login as hadoop user
sudo -i -u huser 
#Create the hadoop user public key
ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa
#Copy the generated public key onto the ssh/authorized_keys
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys 

Setting Up HDFS
#Create a directory used to contain the HDFS file
mkdir /home/huser/my_hdfs_folder

Update the hadoop config with the hdfs directrory
Note:hadoop.tmp.dir is used as the base for temporary directories locally, and also in HDFS.
The following configuration set the created directory as the HDFS directory.

  1: <?xml version=”1.0”?>
  2:  <?xml-stylesheet type=”text/xsl” href=”configuration.xsl”?>
  3:  <configuration>
  4:  <property>
  5:  <name>hadoop.tmp.dir</name>
  6:  <value>/home/huser/my_hdfs_folder</value>
  7:  </property>
  8:  <property>
  9:  <name>fs.default.name</name>
 10:  <value>hdfs://ubuntu:8020</value>
 11:  </property>
 12:  </configuration>

#Format the HDFS
/usr/local/hadoop/bin/hadoop namenode –format
#start the hadoop single instance
/usr/local/hadoop/bin/start-all.sh

View the lifeness of the hdoop instance in the following url:
http://ubuntu:50070/dfshealth.jsp

Setting up HBase
HBase need a directory inside of the HDFS
We create it using the HDFS fs –mkdir command for example
/usr/local/hadoop/bin/hadoop fs -mkdir myHbase
The new hdfs directoy should be point out in the Hbase site configuration file :
hbase-site.xml.

  1: configuration> 
  2:  <property> 
  3:  <name>hbase.rootdir</name> 
  4:  <value>hdfs://ubuntu:8020/user/huser/myHbase</value> 
  5:  <description> 
  6:  </description> 
  7:  </property> 
  8:  <property> 
  9:  <name>hbase.master</name> 
 10:  <value>ubuntu:60000</value> 
 11:  <description> 
 12:  </description> 
 13:  </property> 
 14:  </configuration>

Start the HBase DB
/usr/local/hbase/bin/start-hbase.sh
Monitor its lifeness
http://ubuntu:60010/master-status
Starting the Shell
/usr/local/hbase/bin/hbase shell

Create and Update a simple DB
#Create a new table named myBlogs along with a column family BlogText 
create ‘myBlogs','BlogText'
#insert some data

  1: put ‘myBlogs','Ruby','BlogText:1','About ruby bla bla.'
  2: 
  3: put ‘myBlogs','Ruby','BlogText:2','about {|X| bla bal.'
  4: 
  5: put ‘myBlogs','Ruby','BlogText:3','for loops.'
  6: 
  7: put ‘myBlogs','Python','BlogText:1','iter tools .'

The following code is used to query the created  hbase DB

  1: package my.learn.hbase;
  2: 
  3: import java.util.NavigableMap;
  4: import java.util.NavigableSet;
  5: 
  6: import org.apache.hadoop.conf.Configuration;
  7: import org.apache.hadoop.hbase.HBaseConfiguration;
  8: import org.apache.hadoop.hbase.client.HBaseAdmin;
  9: import org.apache.hadoop.hbase.client.HTableFactory;
 10: import org.apache.hadoop.hbase.client.HTableInterface;
 11: import org.apache.hadoop.hbase.client.Result;
 12: import org.apache.hadoop.hbase.client.ResultScanner;
 13: import org.apache.hadoop.hbase.client.Scan;
 14: import org.apache.hadoop.hbase.util.Bytes;
 15: 
 16: public class HBaseReadMyBlogsData  {
 17: 
 18:   public static final byte[] TablemyBlogs = Bytes.toBytes("myBlogs");
 19:   // The column family
 20:   public static final byte[] BlogText_FAMILY = Bytes.toBytes("BlogText");
 21: 
 22:   
 23:   private void ShowTheBlogsText() throws Exception {
 24: 
 25:     // Load's the hbase-site.xml config
 26:     Configuration config = HBaseConfiguration.create();
 27:     //Factory for creating HTable instances.
 28:     HTableFactory factory = new HTableFactory();
 29:     
 30:     HBaseAdmin.checkHBaseAvailable(config);
 31: 
 32:     // Link to table
 33:     HTableInterface table = factory.createHTableInterface(config,
 34:         TablemyBlogs);
 35: 
 36:     // Used to retrieve rows from the table
 37:     Scan scan = new Scan();
 38: 
 39:     // Scan through each row in the table
 40:     ResultScanner rs = table.getScanner(scan);
 41:     try {
 42:       // Loop through each retrieved row
 43:       for (Result r = rs.next(); r != null; r = rs.next()) {
 44:         //print out the row key
 45:         System.out.println("Key: " + new String(r.getRow()));
 46:       
 47:         //For each key loop over its qualifier for "ruby" key we will have 1 , 2 , 3 
 48:         
 49:         NavigableMap familyMap = r
 50:             .getFamilyMap(BlogText_FAMILY);
 51:         // This is a list of the qualifier keys
 52:         NavigableSet keySet = familyMap.navigableKeySet();
 53: 
 54:         // Print out each value within each qualifier
 55:         for (byte[] key : keySet) {
 56:           System.out.println("\t Definition: " + (new String(key))
 57:               + ", Value:"
 58:               + new String(r.getValue(BlogText_FAMILY, key)));
 59:         }
 60:       }
 61:     } catch (Exception e) {
 62:       throw e;
 63:     } finally {
 64:       rs.close();
 65:     }
 66: 
 67:   }
 68: }

Notes:
The HBaseAdmin provides an interface to manage HBase database table metadata + general administrative functions. like create, drop, list, enable and disable tables.
The HBaseAdmin can be used to add and drop table column families.

MODULE_USB_DRIVER macro part1

The module_usb_driver macro is an helper macro that wrap the module_init and module_exit macros for using to register usb module.
The input parameter is a structure of type struct usb_driver containing the interfaces for the driver.
The parameters:
const char *name;
The name of the usb driver module should be unique.

const struct usb_device_id *id_table;
The table of device id of this device. This value must be set in order to cause the kernel to call this device.
Atypical id_table looks like this:

  1:  static const struct usb_device_id usbduxsigma_usb_table[] = {
  2:          { USB_DEVICE(0x13d8, 0x0020) },
  3:          { USB_DEVICE(0x13d8, 0x0021) },
  4:          { USB_DEVICE(0x13d8, 0x0022) },
  5:          { }
  6:  };

Where the USB_DEVICE macro contains the vendor and the device id

int (*probe) (struct usb_interface *intf,const struct usb_device_id *id);
This callback return if a particular  device can be managed by this module.
If the module can handle the device the module initialization is done in this callback .
The device_id struct

  1:  struct usb_device_id {
  2:       /* which fields to match against? */
  3:       __u16           match_flags;
  4: 
  5:       /* Used for product specific matches; range is inclusive */
  6:       __u16           idVendor;
  7:       __u16           idProduct;
  8:       __u16           bcdDevice_lo;
  9:       __u16           bcdDevice_hi;
 10: 
 11:       /* Used for device class matches */
 12:       __u8            bDeviceClass;
 13:       __u8            bDeviceSubClass;
 14:       __u8            bDeviceProtocol;
 15: 
 16:       /* Used for interface class matches */
 17:       __u8            bInterfaceClass;
 18:       __u8            bInterfaceSubClass;
 19:       __u8            bInterfaceProtocol;
 20: 
 21:       /* Used for vendor-specific interface matches */
 22:       __u8            bInterfaceNumber;
 23: 
 24:       /* not matched against */
 25:       kernel_ulong_t  driver_info
 26:               __attribute__((aligned(sizeof(kernel_ulong_t))));
 27: 

Contains the idVendor and idProducts tfrom the id tables

struct usb_interface - what usb device drivers talk to

ActiveMQ topic sender

Starting ActiveMQ service

Use the following maven pom for my simple client test

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.mycompany</groupId>
  <artifactId>Client</artifactId>
  <version>1.0-SNAPSHOT</version>
  <packaging>jar</packaging>
  <name>Client</name>
  <url>http://maven.apache.org</url>
  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  </properties>
<repositories> 
    <repository>
      <id>repository.jboss.org-public</id>
      <name>JBoss.org Maven repository</name>
      <url>https://repository.jboss.org/nexus/content/groups/public</url>
    </repository>  
</repositories>
  <dependencies>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>3.8.1</version>
      <scope>test</scope>
    </dependency>
   <dependency>
	<groupId>javax.jms</groupId>
	<artifactId>jms</artifactId>
	<version>1.1</version>
</dependency>
<dependency>
  <groupId>org.apache.activemq</groupId>
  <artifactId>activemq-all</artifactId>
  <version>5.8.0</version>
</dependency>
  </dependencies>
</project>

The ActiveMQ monitor can be run  from the following url http://localhost:8161

The code for the demo

  1: package com.mycompany.client;
  2: 
  3: import javax.jms.*;
  4: import javax.naming.InitialContext;
  5: import javax.naming.NamingException;
  6: import java.util.Properties;
  7: import java.util.logging.Level;
  8: import java.util.logging.Logger;
  9:  
 10: public class MessageProducer {
 11:  private String topicName = "myTopic.Programming";
 12:  
 13:  private String initialContextFactory = "org.apache.activemq"
 14: +".jndi.ActiveMQInitialContextFactory";
 15:  private String connectionString = "tcp://localhost:61616";
 16:   
 17: 
 18:  public void publishWithTopicLookup() {
 19:      try {
 20:          Properties properties = new Properties();
 21:          TopicConnection topicConnection = null;
 22:          properties.put("java.naming.factory.initial", initialContextFactory);
 23:          properties.put("connectionfactory.QueueConnectionFactory",
 24:            connectionString);
 25:          properties.put("topic." + topicName, topicName);
 26:         
 27:          
 28:           // initialize
 29:           // the required connection factories
 30:           InitialContext ctx = new InitialContext(properties);
 31:           TopicConnectionFactory topicConnectionFactory = (TopicConnectionFactory) ctx
 32:                            .lookup("QueueConnectionFactory");
 33:           topicConnection = topicConnectionFactory.createTopicConnection();
 34:          
 35:           TopicSession topicSession = topicConnection.createTopicSession(
 36:              false, Session.AUTO_ACKNOWLEDGE);
 37:            Topic topic = (Topic) ctx.lookup(topicName);
 38: 
 39:            javax.jms.TopicPublisher topicPublisher = topicSession
 40:                        .createPublisher(topic);
 41:         
 42:            String msg = "This is a test message";
 43:            TextMessage textMessage = topicSession.createTextMessage(msg);
 44:         
 45:            topicPublisher.publish(textMessage);
 46:            System.out.println("Publishing message " +textMessage);
 47:            topicPublisher.close();
 48:            topicSession.close(); 
 49:            topicConnection.close();
 50:      } catch (NamingException ex) {
 51:          Logger.getLogger(MessageProducer.class.getName()).log(Level.SEVERE, null, ex);
 52:      }
 53:          catch (JMSException e) {
 54:    throw new RuntimeException("Error in initial context lookup", e);
 55:   }
 56:  }
 57: }

The connections:
http://localhost:8161/admin/connections.jsp

Note I set a break point before the connection closing in order to inspect the connection in the connections page

The sample use JNDI in order to generate the ActiveMQ objects .
The context factory that  is used to generate the objects is  ActiveMQInitialContextFactory.
It is set in the following command:
properties.put("java.naming.factory.initial", initialContextFactory);

The created topic in the topics page (http://localhost:8161/admin/topics.jsp)

Redis Simple

loading the Radis local server :
zvika@ubuntu:~$ redis-server

Check if the Redis Server is alive :
zvika@ubuntu:~$ redis-cli ping
PONG

Simple get set  commands to redis in redis client shell

zvika@ubuntu:~$ redis-cli
redis 127.0.0.1:6379> set myName Zvika
OK
redis 127.0.0.1:6379> get myName 
"Zvika"
redis 127.0.0.1:6379> 

Use the Radis RB as interface between Radis and Ruby
https://github.com/redis/redis-rb

Print Ruby default require path:
ruby -e 'puts $:' 

Simple Ruby program that’s interacts with Redis

require 'rubygems'
require 'redis'
require 'json'
r = Redis.new
#Check if Radis server is alive
puts  (r.ping)
#Simple Set Get operation 
r.set('Age','42')
puts (r.get('Age'))
#Use Json lib in order to convert form json to string 
#and the oposite direction 
r.set "Data", {'Family'=>'Peer'}.to_json
theJasonObject = JSON.load (r.get('Data'))
puts (theJasonObject['Family'])

The route way points table

Not long ago I had a mission to normalize a Routes Waypoint table .
The current RoutePoints  table should be split into 2 tables:Route table and a Waypoints table. 

The following  is used to create the table demo sample for the post

IF EXISTS ( SELECT * 
    FROM sys.tables 
    WHERE name = 'RoutesPoints' ) 
   DROP TABLE RoutesPoints; 
   
CREATE TABLE RoutesPoints
   ( RouteName VARCHAR(255), 
      WayPointName VARCHAR(255), 
      WayPointlocation GEOGRAPHY , 
      WayPointType VARCHAR(6),  
   ); 
INSERT INTO RoutesPoints
   (  RouteName, WayPointName,WayPointlocation, WayPointType ) 
VALUES 
(  'Route1', 'WayPoint1', geography::STGeomFromText('POINT(32.34900 36.65150)', 4326) , 'Start' ),
(  'Route2', 'WayPoint2', geography::STGeomFromText('POINT(32.34900 36.65260)', 4326) , 'Start' ),
(  'Route1', 'WayPoint3', geography::STGeomFromText('POINT(32.34900 36.65600)', 4326) , 'Middle' ),
(  'Route3', 'WayPoint4', geography::STGeomFromText('POINT(32.34900 36.65666)', 4326) , 'End' ),
(  'Route3', 'WayPoint5', geography::STGeomFromText('POINT(32.34800 36.65160)', 4326) , 'Middle' ),
(  'Route2', 'WayPoint6', geography::STGeomFromText('POINT(32.34980 36.65290)', 4326) , 'Start' ),
(  'Route1', 'WayPoint7', geography::STGeomFromText('POINT(32.34908 36.61100)', 4326) , 'Start' ),
(  'Route2', 'WayPoint8', geography::STGeomFromText('POINT(32.34950 36.65150)', 4326) , 'Middle' ),
(  'Route1', 'WayPoint9', geography::STGeomFromText('POINT(32.34900 36.65540)', 4326) , 'Start' ),
(  'Route2', 'WayPoint10', geography::STGeomFromText('POINT(32.34900 36.65550)', 4326) , 'Start' ),
(  'Route1', 'WayPoint11', geography::STGeomFromText('POINT(32.34900 36.65230)', 4326) , 'End' ),
(  'Route2', 'WayPoint12', geography::STGeomFromText('POINT(32.34940 36.65150)', 4326) , 'Middle' ),
(  'Route1', 'WayPoint13', geography::STGeomFromText('POINT(32.34600 36.65270)', 4326) , 'Start' ),
(  'Route4', 'WayPoint16', geography::STGeomFromText('POINT(32.34500 36.65340)', 4326) , 'Start' ),
(  'Route1', 'WayPoint14', geography::STGeomFromText('POINT(32.34500 36.65160)', 4326) , 'Middle' ),
(  'Route2', 'WayPoint17', geography::STGeomFromText('POINT(32.34400 36.65150)', 4326) , 'Middle' ),
(  'Route3', 'WayPoint18', geography::STGeomFromText('POINT(32.34200 36.65140)', 4326) , 'Start' ),
(  'Route3', 'WayPoint19', geography::STGeomFromText('POINT(32.34200 36.65150)', 4326) , 'Start' ),
(  'Route1', 'WayPoint20', geography::STGeomFromText('POINT(32.34100 36.65140)', 4326) , 'Middle' ),
(  'Route2', 'WayPoint21', geography::STGeomFromText('POINT(32.34100 36.65150)', 4326) , 'End' ),
(  'Route1', 'WayPoint22', geography::STGeomFromText('POINT(32.34910 36.65160)', 4326) , 'Middle' ),
(  'Route2', 'WayPoint23', geography::STGeomFromText('POINT(32.34430 36.65100)', 4326) , 'Start' ),
(  'Route2', 'WayPoint24', geography::STGeomFromText('POINT(32.34440 36.65180)', 4326) , 'Start' ),
(  'Route2', 'WayPoint25', geography::STGeomFromText('POINT(32.34530 36.65140)', 4326) , 'Middle' ),
(  'Route1', 'WayPoint26', geography::STGeomFromText('POINT(32.34340 36.65800)', 4326) , 'Start' ),
(  'Route3', 'WayPoint27', geography::STGeomFromText('POINT(32.34670 36.65700)', 4326) , 'Start' ),
(  'Route2', 'WayPoint28', geography::STGeomFromText('POINT(32.34540 36.65600)', 4326) , 'End' ),
(  'Route3', 'WayPoint29', geography::STGeomFromText('POINT(32.34650 36.65500)', 4326) , 'Middle' ),
(  'Route1', 'WayPoint30', geography::STGeomFromText('POINT(32.34345 36.65140)', 4326) , 'Start' ),
(  'Route4', 'WayPoint31', geography::STGeomFromText('POINT(32.34584 36.65130)', 4326) , 'Start' ),
(  'Route5', 'WayPoint31', geography::STGeomFromText('POINT(32.34888 36.65200)', 4326) , 'Middle' ),
(  'Route4', 'WayPoint32', geography::STGeomFromText('POINT(32.34900 36.65100)', 4326) , 'Middle' )
 

The rolls for the normalization process  :
Every route should contains one start waypoint , one end way point and 0..n middle waypoints. if no start or end drop the points drop the route data if more then one end or start points exists to the same route take the first occurrence.

The T-SQL for the normalized tables

 IF EXISTS ( SELECT * 
    FROM sys.tables 
    WHERE name = 'RoutePoints' ) 
   DROP TABLE RoutePoints; 
   
CREATE TABLE RoutePoints
   ( RouteID int, 
      WayPointName VARCHAR(255), 
      WayPointlocation GEOGRAPHY , 
      WayPointType VARCHAR(6), 	 
   ); 
 IF EXISTS ( SELECT * 
    FROM sys.tables 
    WHERE name = 'Routes' ) 
   DROP TABLE Routes; 
   
CREATE TABLE Routes
   ( RouteID int , 
      [Name] VARCHAR(255), 
	 CONSTRAINT PK_Routes_RoutesID PRIMARY KEY CLUSTERED (RouteID)   
   ); 
go 
ALTER TABLE RoutePoints
  ADD CONSTRAINT RouteKey FOREIGN KEY (RouteID)
      REFERENCES Routes (RouteID)
      ON DELETE CASCADE
GO

The resolution for this problem is based on this article.