יום שישי, 18 באפריל 2014

Linux block device base code

I found this article about simple block device very useful as a base for creating block device. I have copy the code from the article into here (In case the article will be vanished for some reason ): (open source gnu license )

/*
 * A sample, extra-simple block driver. Updated for kernel 2.6.31.
 *
 * (C) 2003 Eklektix, Inc.
 * (C) 2010 Pat Patterson <pat at superpat dot com>
 * Redistributable under the terms of the GNU GPL.
 */
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/init.h>
#include <linux/kernel.h> /* printk() */
#include <linux/fs.h>     /* everything... */
#include <linux/errno.h>  /* error codes */
#include <linux/types.h>  /* size_t */
#include <linux/vmalloc.h>
#include <linux/genhd.h>
#include <linux/blkdev.h>
#include <linux/hdreg.h>
MODULE_LICENSE("Dual BSD/GPL");
static char *Version = "1.4";
static int major_num = 0;
module_param(major_num, int, 0);
static int logical_block_size = 512;
module_param(logical_block_size, int, 0);
static int nsectors = 1024; /* How big the drive is */
module_param(nsectors, int, 0);
/*
 * We can tweak our hardware sector size, but the kernel talks to us
 * in terms of small sectors, always.
 */
#define KERNEL_SECTOR_SIZE 512
/*
 * Our request queue.
 */
static struct request_queue *Queue;
/*
 * The internal representation of our device.
 */
static struct sbd_device {
	unsigned long size;
	spinlock_t lock;
	u8 *data;
	struct gendisk *gd;
} Device;
/*
 * Handle an I/O request.
 */
static void sbd_transfer(struct sbd_device *dev, sector_t sector,
		unsigned long nsect, char *buffer, int write) {
	unsigned long offset = sector * logical_block_size;
	unsigned long nbytes = nsect * logical_block_size;
	if ((offset + nbytes) > dev->size) {
		printk (KERN_NOTICE "sbd: Beyond-end write (%ld %ld)\n", offset, nbytes);
		return;
	}
	if (write)
		memcpy(dev->data + offset, buffer, nbytes);
	else
		memcpy(buffer, dev->data + offset, nbytes);
}
static void sbd_request(struct request_queue *q) {
	struct request *req;
	req = blk_fetch_request(q);
	while (req != NULL) {
		// blk_fs_request() was removed in 2.6.36 - many thanks to
		// Christian Paro for the heads up and fix...
		//if (!blk_fs_request(req)) {
		if (req == NULL || (req->cmd_type != REQ_TYPE_FS)) {
			printk (KERN_NOTICE "Skip non-CMD request\n");
			__blk_end_request_all(req, -EIO);
			continue;
		}
		sbd_transfer(&Device, blk_rq_pos(req), blk_rq_cur_sectors(req),
				req->buffer, rq_data_dir(req));
		if ( ! __blk_end_request_cur(req, 0) ) {
			req = blk_fetch_request(q);
		}
	}
}
/*
 * The HDIO_GETGEO ioctl is handled in blkdev_ioctl(), which
 * calls this. We need to implement getgeo, since we can't
 * use tools such as fdisk to partition the drive otherwise.
 */
int sbd_getgeo(struct block_device * block_device, struct hd_geometry * geo) {
	long size;
	/* We have no real geometry, of course, so make something up. */
	size = Device.size * (logical_block_size / KERNEL_SECTOR_SIZE);
	geo->cylinders = (size & ~0x3f) >> 6;
	geo->heads = 4;
	geo->sectors = 16;
	geo->start = 0;
	return 0;
}
/*
 * The device operations structure.
 */
static struct block_device_operations sbd_ops = {
		.owner  = THIS_MODULE,
		.getgeo = sbd_getgeo
};
static int __init sbd_init(void) {
	/*
	 * Set up our internal device.
	 */
	Device.size = nsectors * logical_block_size;
	spin_lock_init(&Device.lock);
	Device.data = vmalloc(Device.size);
	if (Device.data == NULL)
		return -ENOMEM;
	/*
	 * Get a request queue.
	 */
	Queue = blk_init_queue(sbd_request, &Device.lock);
	if (Queue == NULL)
		goto out;
	blk_queue_logical_block_size(Queue, logical_block_size);
	/*
	 * Get registered.
	 */
	major_num = register_blkdev(major_num, "sbd");
	if (major_num < 0) {
		printk(KERN_WARNING "sbd: unable to get major number\n");
		goto out;
	}
	/*
	 * And the gendisk structure.
	 */
	Device.gd = alloc_disk(16);
	if (!Device.gd)
		goto out_unregister;
	Device.gd->major = major_num;
	Device.gd->first_minor = 0;
	Device.gd->fops = &sbd_ops;
	Device.gd->private_data = &Device;
	strcpy(Device.gd->disk_name, "sbd0");
	set_capacity(Device.gd, nsectors);
	Device.gd->queue = Queue;
	add_disk(Device.gd);
	return 0;
out_unregister:
	unregister_blkdev(major_num, "sbd");
out:
	vfree(Device.data);
	return -ENOMEM;
}
static void __exit sbd_exit(void)
{
	del_gendisk(Device.gd);
	put_disk(Device.gd);
	unregister_blkdev(major_num, "sbd");
	blk_cleanup_queue(Queue);
	vfree(Device.data);
}
module_init(sbd_init);
module_exit(sbd_exit);

and the make file


obj-m := sbd.o
KDIR := /lib/modules/$(shell uname -r)/build
PWD := $(shell pwd)
default:
	$(MAKE) -C $(KDIR) SUBDIRS=$(PWD) modules

In order to run and test the code :

1.After compilation insmode the block device .
2.Cat the available block devices
zvika@zvika-Linux:/proc$ cat partitions
Results:

major minor  #blocks  name

   8        0  156290904 sda
   8        1   39952318 sda1
   8        2          1 sda2
   8        5   76077056 sda5
   8        6    2085888 sda6
   8        7   38172672 sda7
  11        0    1048575 sr0
251        0        512 sbd0
2. Partion the sdb0 device:
zvika@zvika-Linux:/proc$ sudo mke2fs /dev/sbd0

Results:
mke2fs 1.42 (29-Nov-2011)

Filesystem label=
OS type: Linux
Block size=1024 (log=0)
Fragment size=1024 (log=0)
Stride=0 blocks, Stripe width=0 blocks
64 inodes, 512 blocks
25 blocks (4.88%) reserved for the super user
First data block=1
Maximum filesystem blocks=524288
1 block group
8192 blocks per group, 8192 fragments per group
64 inodes per group
Allocating group tables: done                            
Writing inode tables: done                            
Writing superblocks and filesystem accounting information: done

Mount the block device:
z
vika@zvika-Linux:/$ sudo mkdir myFileSystem
sudo mount /dev/sbd0 /myFileSystem

Test by coping a file to the folder of the block device

sudo cp deleteme.txt /myFileSystem

In order to free the device :

sudo umount /dev/sbd0
sudo rmmod hello-1.ko

And the copied files disappeared 

References :

http://www.ae.iitm.ac.in/pipermail/ilugc/2006-April/025329.html
http://blog.superpat.com/2010/05/04/a-simple-block-driver-for-linux-kernel-2-6-31/comment-page-1/#comment-16760

יום חמישי, 17 באפריל 2014

Running camel-example-cxf-tomcat

1.Install the last Tomcat server change the credentials in the tomcat-users.xml file to :

<tomcat-users> 
  <role rolename="manager"/>
  <role rolename="manager-gui"/>
  <role rolename="manager-script"/>
  <role rolename="manager-jmx"/>
  <role rolename="manager-status"/>
  <role rolename="admin-gui"/>
  <role rolename="admin-script"/>
  <user username="admin" password="admin" roles="admin ,manager ,manager-gui, manager-script, manager-jmx, manager-status, admin-gui, admin-script"/>
</tomcat-users>

2.Add in the maven configuration file (H:\Program Files\NetBeans 8.0\java\maven\conf\settings.xml) in the servers section the following xml


<server>
	<id>TomcatServer</id>
	<username>admin</username>
	<password>admin</password>
</server>

3.Change in the pom  file the name of the server to :
  <server>TomcatServer</server>
4.Run the tomcat server
5.Execute the tomcat:: deploy goal
(After the first deployment we can use the "tomcat:undeploy", "tomcat:redeploy" goals )

Capture31


6.Check that the deployment succeeded in the http://localhost:8080/manager
Capture32
7.Lets Test is with SoapUI
Note : the Wsdl location is :
http://localhost:8080/camel-example-cxf-tomcat/webservices/incident?wsdl 
Capture33

Splitting the reducer output to different Cfs

In one of my projects ,I needed to split the result of the reducer to 3  different Cassandra CFs according to score ranges calculation in the reducer .
The following code is an example of how we did it :

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
import org.apache.cassandra.db.IColumn;
import org.apache.cassandra.thrift.*;
import org.apache.cassandra.hadoop.*;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.apache.hadoop.util.*;
public class ScoresBenchMark extends Configured implements Tool {
   
    public static void main(String[] args) throws Exception {
        ToolRunner.run(new Configuration(), new ScoresBenchMark(), args);
        System.exit(0);
    }
  
    public int run(String[] args) throws Exception {
  ConfigHelper.setRangeBatchSize(getConf(), 99);
    
        final Job job = new Job(getConf(), "ScoresBenchMark");
        final Configuration conf = job.getConfiguration();
   
        job.setJarByClass(ScoresBenchMark.class);
        
  job.setMapperClass(Map.class);
  
  //For testing set only one reducer 
        job.setNumReduceTasks(1);
     
        //Handle cassandra input output tables 
  ConfigHelper.setInputRpcPort(conf, "9160");
        
  ConfigHelper.setInputInitialAddress(conf, "127.0.0.1"); //for test using the local host cassandra 
  
        ConfigHelper.setInputPartitioner(conf, "org.apache.cassandra.dht.RandomPartitioner");
        
  ConfigHelper.setInputColumnFamily(conf, "TestDataKS", "RowDataCF");
        
  //get all records
        SlicePredicate predicate = new SlicePredicate().setSlice_range(new SliceRange(ByteBufferUtil.bytes(""), ByteBufferUtil.bytes(""), false, Integer.MAX_VALUE));
        
  ConfigHelper.setInputSlicePredicate(conf, predicate);
    
        ConfigHelper.setOutputInitialAddress(conf, "127.0.0.1");
        
  ConfigHelper.setOutputRpcPort(conf, "9160");
        
  ConfigHelper.setOutputPartitioner(conf, "org.apache.cassandra.dht.RandomPartitioner");
        
  ConfigHelper.setOutputKeyspace(conf, "TestDataKS");
  //Set the 3 tests output CF by the calculation result 
  MultipleOutputs.addNamedOutput(job, "TestScoresCFLow", ColumnFamilyOutputFormat.class, ByteBuffer.class, List.class);
        
  MultipleOutputs.addNamedOutput(job, "TestScoresCFMid", ColumnFamilyOutputFormat.class, ByteBuffer.class, List.class);
  
  MultipleOutputs.addNamedOutput(job, "TestScoresCFHi", ColumnFamilyOutputFormat.class, ByteBuffer.class, List.class);
  
        //set up input as cassandra cf        
        job.setInputFormatClass(ColumnFamilyInputFormat.class);
        
  //Handle the output as cassandra cf 
        
        job.setReducerClass(Reduce.class);
        job.setOutputFormatClass(ColumnFamilyOutputFormat.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);
        job.setOutputKeyClass(ByteBuffer.class);
        job.setOutputValueClass(List.class);
    
        job.waitForCompletion(true);
        return 0;
    }
  
    public static class Map extends Mapper<ByteBuffer, SortedMap<ByteBuffer, IColumn>, Text, LongWritable> {  
    
        public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context) throws IOException, InterruptedException { 
      
   //Loop over the coloumn data and map key -> value 
            for (IColumn nextCol : columns.values()){
             context.write(new Text(ByteBufferUtil.string(nextCol.name())),
         new LongWritable(ByteBufferUtil.toLong(nextCol.value())));
   }
        }   
    }
  
    public static class Reduce extends Reducer<Text, LongWritable, ByteBuffer, List<Mutation>> {
 
        private MultipleOutputs _output;
        public void setup(Context context) {
            _output = new MultipleOutputs(context);
        }
        public void cleanup(Context context) throws IOException, InterruptedException {
            _output.close();
        }
    
        public void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
   for ( ICalculator nextCalcolator in mCalculators )
   {
    float theScore = nextCalcolator.Calc (values) ;
    String theCalculatorName = nextCalcolator.Class.ToString ();
    Column c = new Column();
            
    c.setName(ByteBufferUtil.bytes(theCalculatorName));
    c.setValue(ByteBufferUtil.bytes(theScore);
    c.setTimestamp(System.currentTimeMillis());
    Mutation m = new Mutation();
    m.setColumn_or_supercolumn(new ColumnOrSuperColumn());
    m.column_or_supercolumn.setColumn(c);
    
    if ( theScore < 30 )
    {
     TestScoresCFMid_output.write("TestScoresCFLow", ByteBufferUtil.bytes(key.toString()), Collections.singletonList(m));
    } 
    else if ( theScore > 74 )
    {
     TestScoresCFMid_output.write("TestScoresCFHi", ByteBufferUtil.bytes(key.toString()), Collections.singletonList(m));
    }
    else 
    {
     TestScoresCFMid_output.write("TestScoresCFMid", ByteBufferUtil.bytes(key.toString()), Collections.singletonList(m));
    }
    
   }
        }
    }
}

References :
https://gist.githubusercontent.com/rstrickland/3763728/raw/7bb0b6d9211bc99e58ca222161e8b31db799a2d5/mo_example.java
http://modular.math.washington.edu/home/wstein/www/home/was/tmp/apache-cassandra-1.1.3/javadoc/org/apache/cassandra/thrift/Mutation.html
http://ng911dev1.cs.columbia.edu/docs/cassandra/org/apache/cassandra/thrift/SlicePredicate.html
http://hadoop.apache.org/docs/r2.3.0/api/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.html
http://svn.apache.org/repos/asf/cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyOutputFormat.java

יום רביעי, 16 באפריל 2014

Play with Scala actors

The following code is a little demonstration for Scala actors:

package myScalaProject
import scala.actors.Actor
import scala.actors.Actor._
import java.lang.Float
class ActorTester extends Actor {
def act() {
       println("Actor Started.")
       println ("The actor thread id :" +  Thread.currentThread().getId())
              loop {
                     receive {
                     case i: Int => println("Int: " + i.toString)
                     case s: String => println("String: " + s)      
                     case f: Float => println("Float: " + f) 
                     case 5=> println("Five")  
                     case _ => println("Default.")
                     }
              }
                 println("Actor ended.") //doesn’t called
       }
}     
object PlayWithActors {
 println ("The main thread thread id :" +  Thread.currentThread().getId())
 val myActorTester = new ActorTester
 myActorTester.start
 
 //Sending data to the actor
 myActorTester ! "Hello actor"
 myActorTester ! 5
 myActorTester !  164.1
 myActorTester ! 15
}

And the Result :


The main thread thread id :1
Actor Started.
The actor thread id :12
String: Hello actor
Int: 5
Default.
Int: 15

הבלוג שלי במיקרוסופט הוחזר לחיים

הכתובת שלו הינה :
הבלוג של צביקה פאר