#! /usr/bin/env python
#@+leo-ver=4
#@+node:@file stasher.py
#@@first
"""
A simple implementation of the
U{Kademlia<http://www.infoanarchy.org/wiki/wiki.pl?Kademlia>}
P2P distributed storage and retrieval protocol, designed to
utilise the U{I2P<http://www.i2p.net>} stealth network as its transport.

Most application developers will only need to know about the L{KNode} class
"""

# I strongly recommend that when editing this file, you use the Leo
# outlining and literate programming editor - http://leo.sf.net
# If Leo doesn't agree with your religion, please try to leave the markups intact

#@+others
#@+node:explanatory comments
#@+at
# Tech overview:
#     - this implementation creates each Node ID as an SHA1 hash of
#       the node's 'destination' - the string which constitutes its
#       address as an I2P endpoint.
# 
# Datagram formats:
#     - each datagram sent from one node to another is a python dict object,
#       encoded and decoded with the 'bencode' object serialisation module.
#     - we use bencode because regular Python pickle is highly insecure,
#       allowing crackers to create malformed pickles which can have all
#       manner of detrimental effects, including execution of arbitrary code.
#     - the possible messages are listed below, along with their consituent
#       dictionary keys:
#           1. ping:
#               - msgId - a message identifier guaranteed to be unique
#                 with respect to the sending node
#           2. findNode:
#               - msgId - unique message identifier
#               - hash - the hash we're looking for
#               - initiator - True/False, according to whether this node
#                 should initiate/perform the findNode, or whether this
#                 rpc is coming from another seeking node
#           3. findData:
#               - msgId - unique message identifier
#               - hash - the exact key hash of the data we want to retrieve
#               - initiator - True/False, according to whether this node
#                 should initiate/perform the findNode, or whether this
#                 rpc is coming from another seeking node
#           4. store:
#               - msgId - unique message identifier
#               - hash - the exact key hash of the data we want to store
#               - data - the data we want to store
#           5. reply:
#               - msgId - the original msgId we're replying to
#              The other items in a reply message depend on what kind
#              of message we're replying to, listed below:
#                     1. ping - no additional data
#                     2. findNode:
#                         - nodes - a list of dests nearest the given hash
#                     3. findData:
#                         - nodes - as for findNode, OR
#                         - data - the retrieved data, or None if not found
#                     4. store:
#                         - status - True or False according to whether
#                           the store operation was successful
#@-at
#@-node:explanatory comments
#@+node:imports
import sys, os, types, sha, random, threading, thread, traceback, Queue
import time, math, random, pickle, getopt, re
import signal

# some windows-specifics (yggghh)
if sys.platform == 'win32':
    try:
        import win32api
        import win32process
        import _winreg
    except:
        print "Python win32 extensions not installed."
        print "Please go to http://sourceforge.net/project/showfiles.php?group_id=78018"
        print "and download/install the file pywin32-202.win32-py%s.%s.exe" % \
            sys.version_info[:2]
        sys.exit(1)

from StringIO import StringIO
from pdb import set_trace

try:
    import bencode
except:
    print "The bencode module is missing from your python installation."
    print "Are you sure you installed Stasher correctly?"
    sys.exit(1)

try:
    import i2p.socket
    import i2p.select
    import i2p.pylib
    SocketServer = i2p.pylib.SocketServer
    socket = i2p.pylib.socket
except:
    print "You don't appear to have the I2P Python modules installed."
    print "Not good. Stasher totally needs them."
    print "Please to to i2p/apps/sam/python in your I2P cvs tree, and"
    print "install the core I2P python modules first"
    sys.exit(1)

#@-node:imports
#@+node:constants

# --------------------------------------------
# START USER-CONFIGURABLE CONSTANTS
# --------------------------------------------

# host:port to connect to I2P SAM Bridge
samAddr = i2p.socket.samaddr

# host:port to listen on for command line client
clientAddr = "127.0.0.1:7659"

defaultNodename = "0" # will be prefixed by 'stashernode'

# maximum size of each stored item
maxValueSize = 30000

# maximum number of noderefs that can be stored in a bucket
# (refer spec section 2.1, first paragraph)
maxBucketSize = 20

# number of peers to return from a search
numSearchPeers = 3

# maximum number of concurrent queries per findnode/finddata rpc
maxConcurrentQueries = 10

# number of peers to store onto
numStorePeers = 10

# Logger settings
logFile = None
logVerbosity = 2

# data directory location - set to a path to override the default
# which is the user's home dir
dataDir = None

# whether a node, on startup, should do a findnode on itself to
# locate its closest neighbours
greetPeersOnStartup = False
#greetPeersOnStartup = True

# multi-purpose testing flag
testing = False
#testing = True

tunnelDepth = 0

# set to True to enable single handler thread that manages all nodes,
# or False to make each node run its own handler thread
#runCore = False
runCore = True

# timeouts - calibrate as needed
timeout = {
    'ping' : 120,
    'findNode' : 120,
    'findData' : 120,
    'store' : 120,
    }

logToSocket = None

desperatelyDebugging = False

if desperatelyDebugging:
    runCoreInBackground = False
else:
    runCoreInBackground = True

# --------------------------------------------
# END OF USER-CONFIGURABLE CONSTANTS
# --------------------------------------------

# ----------------------------------------------
# hack anything below this line at your own risk

#@-node:constants
#@+node:globals
# keep a dict of existing nodes, so we can prevent
# client progs from creating 2 nodes of the same name
_nodes = {}

version = "0.0.1"

#@-node:globals
#@+node:Exceptions
# define our exceptions

class KValueTooLarge(Exception):
    """
    Trying to insert a value of excessive size into the network.
    Maximum key size is L{maxValueSize}
    """

class KBadHash(Exception):
    """
    Invalid hash string
    """

class KNotImplemented(Exception):
    """
    A required method was not implemented
    """

class KBadNode(Exception):
    """
    Invalid Node object
    """

class KBadPeer(Exception):
    """
    Invalid Peer object - should be a KPeer
    """

class KBadDest(Exception):
    """Invalid I2P Node Dest"""

#@-node:Exceptions
#@+node:Mixins
#@+node:class KBase
class KBase:
    """
    A mixin which adds a class-specific logger
    """
    def log(self, verbosity, msg):
        
        log(verbosity, msg, 1, self.__class__.__name__)

    def logexc(self, verbosity, msg):

        logexc(verbosity, msg, 1, self.__class__.__name__)

#@-node:class KBase
#@-node:Mixins
#@+node:Main Engine
#@+node:class KCore
class KCore(KBase):
    """
    Singleton class which performs all the needed background processing.
    
    By scheduling all processing through this object, we eliminate the
    need to create threads on a per-node basis, and also make this thing
    far easier to debug.

    The core launches only two background threads:
        - L{threadRxPackets} - listen for incoming packets bound for
          any node running within a single process
        - L{threadHousekeeping} - periodically invoke maintenance methods
          of each node, so the node can check for timeout conditions and
          other untoward happenings

    These threads start up when the first node in this process is created,
    and stop when the last node ceases to exist.

    Upon first import, the L{stasher} module creates one instance of this
    class. Upon creation, L{KNode} objects register themselves with this core.
    """
    #@    @+others
    #@+node:attributes
    #@-node:attributes
    #@+node:__init__
    def __init__(self, bg=True):
        """
        Creates the I2P Kademlia core object
        """
        self.bg = bg
        self.fg = False
    
        # subscribed nodes
        self.nodes = []
        #self.nodesLock = threading.Lock()
        
        self.isRunning = False
        self.isRunning_rx = False
    
    #@-node:__init__
    #@+node:subscribe
    def subscribe(self, node):
        """
        Called by a node to 'subscribe' for background processing
        If this is the first node, starts the handler thread
        """
        #self.nodesLock.acquire()
        try:
            nodes = self.nodes
            
            if node in nodes:
                self.log(2, "duhhh! node already subscribed" % repr(node))
                return
        
            nodes.append(node)
            
            if not self.isRunning:
                self.isRunning = True
                if self.bg and not self.fg:
                    self.log(3, "First node subscribing, launching threads")
                    thread.start_new_thread(self.threadRxPackets, ())
                    thread.start_new_thread(self.threadHousekeeping, ())
        except:
            traceback.print_exc()
            self.log(2, "exception")
    
        #self.nodesLock.release()
    
    #@-node:subscribe
    #@+node:unsubscribe
    def unsubscribe(self, node):
        """
        Unsubscribes a node from the core
        
        If this was the last node, stops the handler thread
        """
        #self.nodesLock.acquire()
        try:
            nodes = self.nodes
            
            if node not in nodes:
                self.log(4, "duhhh! node %s was not subscribed" % repr(node))
                return
    
            self.log(2, "trying to unsubscribe node %s" % node.name)
            nodes.remove(node)
    
            if len(nodes) == 0:
                self.isRunning = False
        except:
            traceback.print_exc()
            self.log(2, "exception")
    
        #self.nodesLock.release()
    
    #@-node:unsubscribe
    #@+node:threadRxPackets
    def threadRxPackets(self):
        """
        Sits on a select() loop, processing incoming datagrams
        and actioning them appropriately.
        """
        self.isRunning_rx = True
        self.log(3, "KCore packet receiver thread running")
        try:
            while self.isRunning:
                socks = [node.sock for node in self.nodes]
                if desperatelyDebugging:
                    set_trace()
                try:
                    inlist, outlist, errlist = self.select(socks, [], [], 1)
                except KeyboardInterrupt:
                    self.isRunning = 0
                    return
    
                self.log(5, "\ninlist=%s" % repr(inlist))
                if inlist:
                    self.log(5, "got one or more sockets with inbound data")
                    #self.nodesLock.acquire()
                    for sock in inlist:
                        node = self.nodeWhichOwnsSock(sock)
                        if node != None:
                            node._doRx()
                    #self.nodesLock.release()
    
                elif self.fg:
                    return
    
                else:
                    time.sleep(0.1)
        except:
            #self.nodesLock.release()
            traceback.print_exc()
            self.log(1, "core handler thread crashed")
        self.isRunning_rx = False
        self.log(3, "core handler thread terminated")
    
    #@-node:threadRxPackets
    #@+node:threadHousekeeping
    def threadHousekeeping(self):
        """
        Periodically invoke nodes' housekeeping
        """
        self.log(3, "\nnode housekeeping thread running")
        try:
            while self.isRunning:
                #self.log(4, "calling nodes' housekeeping methods")
                #self.nodesLock.acquire()
                for node in self.nodes:
                    node._doHousekeeping()
                #self.nodesLock.release()
                time.sleep(1)
            self.log(3, "\nnode housekeeping thread terminated")
        except:
            #self.nodesLock.release()
            traceback.print_exc()
            self.log(1, "\nnode housekeeping thread crashed")
    
    #@-node:threadHousekeeping
    #@+node:nodeWhichOwnsSock
    def nodeWhichOwnsSock(self, sock):
        """
        returns ref to node which owns a socket
        """
        for node in self.nodes:
            if node.sock == sock:
                return node
        return None
    #@-node:nodeWhichOwnsSock
    #@+node:cycle
    def cycle(self):
    
        self.fg = True
        self.threadRxPackets()
    
    #@-node:cycle
    #@+node:run
    def run(self, func=None):
        """
        Runs the core in foreground, with the client func in background
        """
        if func==None:
            func = test
    
        self.bg = False
    
        thread.start_new_thread(self.runClient, (func,))
        
        set_trace()
    
        self.threadRxPackets()
    
    #@-node:run
    #@+node:stop
    def stop(self):
        self.isRunning = False
    
    #@-node:stop
    #@+node:runClient
    def runClient(self, func):
    
        self.log(3, "Core: running client func")
        try:
            func()
        except:
            traceback.print_exc()
        self.log(3, "Core: client func exited")
        self.stop()
    #@-node:runClient
    #@+node:select
    def select(self, inlist, outlist, errlist, timeout):
        
        return i2p.select.select(inlist, outlist, errlist, timeout)
    
    #@-node:select
    #@-others

#@-node:class KCore
#@+node:create instance
# create an instance of _KCore
core = KCore()

#@-node:create instance
#@-node:Main Engine
#@+node:Basic Classes
#@+node:Node-local Storage
#@+node:class KStorageBase
class KStorageBase(KBase):
    """
    Base class for node storage objects

    This needs to be overridden by implementation-specific
    solutions.
    """
    #@    @+others
    #@+node:__init__
    def __init__(self, node, *args, **kw):
        """
        Override this method
        
        First argument should be a node instance
        """
        raise KNotImplemented
    
    #@-node:__init__
    #@+node:putRefs
    def putRefs(self, *refs):
        """
        Saves one or more noderefs
        
        Arguments:
            - zero or more KPeer objects, or lists or tuples of objects
        """
        raise KNotImplemented
    #@-node:putRefs
    #@+node:getRefs
    def getRefs(self):
        """
        Returns a list of KPeer objects, comprising refs
        of peers known to this node
        """
        raise KNotImplemented
    
    #@-node:getRefs
    #@+node:putKey
    def putKey(self, key, value):
        """
        Stores value, a string, into the local storage
        under key 'key'
        """
        raise KNotImplemented
    
    #@-node:putKey
    #@+node:getKey
    def getKey(self, key):
        """
        Attempts to retrieve item from node's local, which was
        stored with key 'key'.
        
        Returns value as a string if found, or None if not present
        """
        raise KNotImplemented
    #@-node:getKey
    #@+node:private methods
    #@+others
    #@+node:_expandRefsList
    def _expandRefsList(self, args, lst=None):
        """
        Takes a sequence of args, each of which can be a KPeer
        object, or a list or tuple of KPeer objects, and expands
        this into a flat list
        """
        if lst == None:
            lst = []
        for item in args:
            if type(item) in [type(()), type([])]:
                self._expandRefsList(item, lst)
            else:
                lst.append(item)
        return lst
    
    #@-node:_expandRefsList
    #@-others
    #@-node:private methods
    #@-others
#@-node:class KStorageBase
#@+node:class KStorageFile
class KStorageFile(KStorageBase):
    """
    Implements node-local storage, using the local filesystem,
    with the following hierarchy:
        
        - HOME ( ~ in linux, some other shit for windows)
           - .i2pkademlia
               - <nodename>
                   - noderefs
                       - <node1 base64 hash>
                           - contains node dest, and other shit
                       - ...
                   - keys
                       - <keyname1>
                           - contains raw key value
                       - ...

    This is one ugly sukka, perhaps a db4, mysql etc implementation
    would be better.
    """
    #@    @+others
    #@+node:__init__
    def __init__(self, node, storeDir=None):
        """
        Creates a persistent storage object for node
        'nodeName', based at directory 'storeDir' (default
        is nodeDir
        """
        self.node = node
        self.nodeName = node.name
    
        if storeDir == None:
            # work out local directory
            self.topDir = userI2PDir()
    
        # add node dir and subdirs
        self.nodeDir = userI2PDir(self.nodeName)
        
        self.refsDir = os.path.join(self.nodeDir, "noderefs")
        if not os.path.isdir(self.refsDir):
            os.makedirs(self.refsDir)
    
        self.keysDir = os.path.join(self.nodeDir, "keys")
        if not os.path.isdir(self.keysDir):
            os.makedirs(self.keysDir)
    
    #@-node:__init__
    #@+node:putRefs
    def putRefs(self, *args):
        """
        Saves one or more noderefs into filesystem
        
        Arguments:
            - zero or more KPeer objects, or lists or tuples of objects
        """
        lst = self._expandRefsList(args)
        for item in lst:
            b64hash = shahash(item.dest)
            itemPath = os.path.join(self.refsDir, b64hash)
            itemDict = {'dest':item.dest} # might need to expand later
            itemPickle = bencode.bencode(itemDict)
            file(itemPath, "wb").write(itemPickle)
        pass
    #@-node:putRefs
    #@+node:getRefs
    def getRefs(self):
        """
        Returns a list of KPeer objects, comprising refs
        of peers known to this node
    
        These are read from the directory self.refsDir.
        Any that can't be unpickled and instantiated are dropped, but logged
        """
        peers = []
        for f in os.listdir(self.refsDir):
    
            path = os.path.join(self.refsDir, f)
            pickled = file(path, "rb").read()
            try:
                d = bencode.bdecode(pickled)
            except:
                self.log(3, "node %s, bad pickle ref file %s" % (
                    self.nodeName, f))
                continue
            
            # instantiate a peer object
            try:
                peer = KPeer(self.node, d['dest'])
            except:
                self.log(3, "node %s, bad unpickled ref file %s" % (
                    self.nodeName, f))
                continue
    
            # success
            peers.append(peer)
    
        return peers
    
    #@-node:getRefs
    #@+node:putKey
    def putKey(self, key, val, keyIsHashed=False):
        """
        Stores a string into this storage under the key 'key'
    
        Returns True if key was saved successfully, False if not
        """
        try:
            if keyIsHashed:
                keyHashed = key
            else:
                keyHashed = shahash(key)
            keyHashed = keyHashed.lower()
            keyPath = os.path.join(self.keysDir, keyHashed)
            file(keyPath, "wb").write(val)
            self.log(4, "stored key: '%s'\nunder hash '%s'\n(keyIsHashed=%s)" % (
                key, keyHashed, keyIsHashed))
            return True
        except:
            traceback.print_exc()
            self.log(3, "failed to store key")
            return False
    
    #@-node:putKey
    #@+node:getKey
    def getKey(self, key, keyIsHashed=False):
        """
        Attempts to retrieve item from node's local file storage, which was
        stored with key 'key'.
        
        Returns value as a string if found, or None if not present
        """
        try:
            if keyIsHashed:
                keyHashed = key
            else:
                keyHashed = shahash(key)
            
            keyHashed = keyHashed.lower()
            self.log(4, "key=%s, keyHashed=%s, keyIsHashed=%s" % (key, keyHashed, keyIsHashed))
    
            keyPath = os.path.join(self.keysDir, keyHashed)
        
            if os.path.isfile(keyPath):
                return file(keyPath, "rb").read()
            else:
                return None
        except:
            traceback.print_exc()
            self.log(3, "error retrieving key '%s'" % key)
            return None
    
    #@-node:getKey
    #@-others
#@-node:class KStorageFile
#@-node:Node-local Storage
#@+node:class KHash
class KHash(KBase):
    """
    Wraps 160-bit hashes as abstract objects, on which
    operations such as xor, <, >, etc can be performed.
    
    Kademlia node ids and keys are held as objects
    of this class.

    Internally, hashes are stored as python long ints
    """
    #@    @+others
    #@+node:__init__
    def __init__(self, val=None, **kw):
        """
        Create a new hash object.
        
        val can be one of the following:
            - None (default) - a random value will be created
            - long int - this will be used as the raw hash
            - string - the string will be hashed and stored
            - another KHash object - its value will be taken
            - a KNode or KPeer object - its hash will be taken
    
        If val is not given, a raw hash value can be passed in
        with the keyword 'raw'. Such value must be a python long int
        or a 20-char string
        """
        self.value = 0L
        if val:
            if isinstance(val, KHash):
                self.value = val.value
            elif type(val) in [type(0), type(0L)]:
                self.value = long(val)
            elif isinstance(val, KNode) or isinstance(val, KPeer):
                self.value = val.id.value
            else:
                raw = self.raw = shahash(val, bin=1)
                for c in raw:
                    self.value = self.value * 256 + ord(c)
        else:
            rawval = kw.get('raw', None)
            if rawval == None:
                # generate random
                random.seed()
                for i in range(20):
                    self.value = self.value * 256 + random.randint(0, 256)
            elif type(rawval) in [type(0), type(0L)]:
                self.value = long(rawval)
            elif type(rawval) == type(""):
                if len(rawval) == 20:
                    for i in rawval:
                        self.value = self.value * 256 + ord(i)
                elif len(rawval) == 40:
                    try:
                        self.value = long(rawval, 16)
                    except:
                        raise KBadHash(rawval)
                else:
                    raise KBadHash(rawval)
            else:
                print "rawval=%s %s %s" % (type(rawval), rawval.__class__, repr(rawval))
                raise KBadHash(rawval)
    
    #@-node:__init__
    #@+node:__str__
    def __str__(self):
        return "<KHash: 0x%x>" % self.value
    
    def __repr__(self):
        return str(self)
    
    #@-node:__str__
    #@+node:asHex
    def asHex(self):
        return ("%040x" % self.value).lower()
    
    #@-node:asHex
    #@+node:distance
    def distance(self, other):
        """
        calculates the 'distance' between this hash and another hash,
        and returns it as i (where distance = 2^i, and 0 <= i < 160)
        """
    
        #log(4, "comparing: %s\nwith %s" % (self.value, other.value))
    
        rawdistance = self.value ^ other.value
        if not rawdistance:
            return 0
    
        return int(math.log(rawdistance, 2))
    
    #@-node:distance
    #@+node:rawdistance
    def rawdistance(self, other):
        """
        calculates the 'distance' between this hash and another hash,
        and returns it raw as this xor other
        """
        return self.value ^ other.value
    
    #@-node:rawdistance
    #@+node:operators
    def __eq__(self, other):
        #log(2, "KHash: comparing %s to %s" % (self, other))
        res = self.value == getattr(other, 'value', None)
        #self.log(2, "KHash: res = %s" % repr(res))
        return res
    
    def __ne__(self, other):
        return not (self == other)
    
    def __lt__(self, other):
        return self.value < other.value
    
    def __gt__(self, other):
        return self.value > other.value
        
    def __le__(self, other):
        return self.value <= other.value
    
    def __ge__(self, other):
        return self.value >= other.value
    
    def __ne__(self, other):
        return self.value != other.value
    
    def __xor__(self, other):
        return self.value ^ other.value
    
    #@-node:operators
    #@-others
#@-node:class KHash
#@+node:class KBucket
class KBucket(KBase):
    """
    Implements the 'k-bucket' object as required in Kademlia spec
    """
    #@    @+others
    #@+node:__init__
    def __init__(self):
        """
        Creates a single k-bucket
        """
        # list of known nodes
        # order is least recently seen at head, most recently seen at tail
        self.nodes = []
    
        # list of death-row records
        # refer spec section 2.1, paragraph 2
        # we deviate a little:
        #   when we hear from a new peer, and the bucket is full,
        #   we temporarily displace the old peer, and stick the new
        #   peer at end of list, then send out a ping
        #   If we hear from the old peer within a reasonable time,
        #   the new peer gets evicted and replaced with the old peer
        #
        # this list holds 2-tuples (oldpeer, newpeer), where
        # oldpeer is the least-recently-seen peer that we displaced, and
        # newpeer is the new peer we just heard from.
        self.deathrow = []
    
    #@-node:__init__
    #@+node:justSeenPeer
    def justSeenPeer(self, peer):
        """
        Tells the bucket that we've just seen a given node
        """
        nodes = self.nodes
    
        if not isinstance(peer, KPeer):
            raise KBadNode
    
        try:
            idx = nodes.index(peer)
        except:
            idx = -1    
        if idx >= 0:
            del nodes[idx]
            nodes.append(peer)
        else:
            nodes.append(peer)
            
        # might at some time need to implement death-row logic 
        # when we set a bucket size limit - refer __init__
    #@-node:justSeenPeer
    #@+node:__iter__
    def __iter__(self):
        return iter(self.nodes)
    #@-node:__iter__
    #@-others
#@-node:class KBucket
#@+node:class KPeer
class KPeer(KBase):
    """
    Encapsulates a peer node of a L{KNode},
    storing its ID and contact info
    """
    #@    @+others
    #@+node:__init__
    def __init__(self, node, dest):
        """
        Create a ref to a kademlia peer node
        
        Arguments:
            - node - reference to node which has the relationship
              to this peer
            - dest - the peer's I2P destination, as base64
        """
        if not isinstance(node, KNode):
            raise KBadNode(node)
        if not isinstance(dest, str):
            raise KBadDest(dest)
    
        self.node = node
        self.dest = dest
        self.id = KHash(dest)
    
        self.justSeen()
    
    #@-node:__init__
    #@+node:send_ping
    def send_ping(self, **kw):
        """
        Sends a ping to remote peer
        """
        self.send_raw(type="ping", **kw)
    #@-node:send_ping
    #@+node:send_store
    def send_store(self, **kw):
        """
        sends a store command to peer
        """
        self.log(4, "\npeer %s\ndest %s...\nsending store cmd: %s" % (self, self.dest[:12], repr(kw)))
    
        self.send_raw(type="store", **kw)
    
    #@-node:send_store
    #@+node:send_findNode
    def send_findNode(self, hash, **kw):
        """
        sends a findNode command to peer
        """
        if not isinstance(hash, KHash):
            raise KBadHash
    
        self.log(5, "\nquerying peer %s\ntarget hash %s" % (self, hash))
    
        self.send_raw(type="findNode", hash=hash.value, **kw)
    
    #@-node:send_findNode
    #@+node:send_findData
    def send_findData(self, hash, **kw):
        """
        sends a findData command to peer
        """
        if not isinstance(hash, KHash):
            raise KBadHash
    
        self.log(5, "\nquerying peer %s\ntarget hash %s" % (self, hash))
    
        self.send_raw(type="findData", hash=hash.value, **kw)
    
    #@-node:send_findData
    #@+node:send_reply
    def send_reply(self, **kw):
        """
        Sends an RPC reply back to upstream peer
        """
        self.log(5, "\nnode %s\nreplying to peer %s:\n%s" % (
            self.node, self, kw))
        self.send_raw(type="reply", **kw)
    
    #@-node:send_reply
    #@+node:send_raw
    def send_raw(self, **kw):
        """
        Sends a raw datagram to peer
        
        No arguments - just keywords, all of which must be strings or
        other objects which can be bencoded
        """
        self.node._sendRaw(self, **kw)
    #@-node:send_raw
    #@+node:justSeen
    def justSeen(self):
        self.timeLastSeen = time.time()
    
    #@-node:justSeen
    #@+node:lowlevel
    #@+others
    #@+node:__str__
    def __str__(self):
    
        return "<KPeer:%s=>0x%s... dest %s...>" % (
            self.node.name, ("%x" % self.id.value)[:8], self.dest[:8])
    
    #@-node:__str__
    #@+node:__repr__
    def __repr__(self):
        
        return str(self)
    
    #@-node:__repr__
    #@+node:__eq__
    def __eq__(self, other):
    
        #self.log(2, "KPeer: comparing %s to %s (%s to %s)" % (self, other, self.__class__, other.__class__))
        res = self.id == getattr(other, 'id', None)
        #self.log(2, "KPeer: res=%s" % res)
        return res
    
    #@-node:__eq__
    #@+node:__ne__
    def __ne__(self, other):
        return not (self == other)
    #@-node:__ne__
    #@-others
    #@-node:lowlevel
    #@-others
#@-node:class KPeer
#@-node:Basic Classes
#@+node:RPC Classes
#@+node:class KRpc
class KRpc(KBase):
    """
    Base class for RPCs between nodes.
    Refer subclasses
    """
    #@    @+others
    #@+node:attribs
    type = 'unknown' # override in subclass
    
    #@-node:attribs
    #@+node:__init__
    def __init__(self, localNode, client=None, **kw):
        """
        Holds all the information for an RPC
    
        Arguments:
            - localNode - the node from which this RPC is being driven
            - client - a representation of who is initiating this rpc, one of:
                - None - an API caller, which is to be blocked until the RPC completes
                  or times out
                - (upstreamPeer, upstreamMsgId) - an upstream peer
                - callable object - something which requires a callback upon completion
                  in which case the callable will be invoked with the RPC results as the
                  first argument
    
        Keywords:
            - cbArgs - optional - if given, and if client is a callback, the callback
              will be invoked with the results as first argument, and this object as
              second argument
        """
        self.localNode = localNode
    
        if client == None:
            # an api client
            self.isLocal = True
            self.queue = Queue.Queue()
            self.callback = None
        elif callable(client):
            self.isLocal = False
            self.callback = client
        elif isinstance(client, tuple):
            # we're doing the RPC on behalf of an upstream peer
            upstreamPeer, upstreamMsgId = client
            upstreamPeer = localNode._normalisePeer(upstreamPeer)
            self.isLocal = False
            self.upstreamPeer = upstreamPeer
            self.upstreamMsgId = upstreamMsgId
            self.callback = None
    
        # save keywords
        self.__dict__.update(kw)
    
        # set time for receiving a tick.
        # if this is set to an int absolute time value, the on_tick method
        # will be called as soon as possible after that time
        self.nextTickTime = None
    
        # and register with node as a pending command
        self.localNode.rpcPending.append(self)
    
        # now start up the request
        self.start()
    
    #@-node:__init__
    #@+node:__del__
    def __del__(self):
    
        #self.log(4, "\nRPC %s getting the chop" % (str(self)))
        pass
    
    #@-node:__del__
    #@+node:__str__
    def __str__(self):
    
        return "<%s on node %s>" % (self.__class__.__name__, self.localNode.name)
    
    #@-node:__str__
    #@+node:__repr__
    def __repr__(self):
        return str(self)
    #@-node:__repr__
    #@+node:bindPeerReply
    def bindPeerReply(self, peer, msgId):
        """
        Sets up the node to give us a callback when a reply
        comes in from downstream peer 'peer' with msg id 'msgId'
        """
        self.localNode.rpcBindings[(peer.dest, msgId)] = (self, peer)
    
    #@-node:bindPeerReply
    #@+node:unbindPeerReply
    def unbindPeerReply(self, peer, msgId):
        """
        Disables the callback from node for replies
        from peer 'peer' with msgId 'msgId'
        """
        bindings = self.localNode.rpcBindings
        peerdest = peer.dest
        if bindings.has_key((peerdest, msgId)):
            del bindings[(peerdest, msgId)]
    
    #@-node:unbindPeerReply
    #@+node:unbindAll
    def unbindAll(self):
        """
        Remove all reply bindings
        """
        bindings = self.localNode.rpcBindings
        self.log(5, "node bindings before: %s" % bindings)
        for k,v in bindings.items():
            if v[0] == self:
                del bindings[k]
        self.log(5, "node bindings after: %s" % bindings)
    
    #@-node:unbindAll
    #@+node:start
    def start(self):
        """
        Start the RPC running.
        Override this in subclasses
        """
        raise KNotImplemented
    
    #@-node:start
    #@+node:execute
    def execute(self):
        """
        Only for synchronous (application-level) execution.
        Wait for the RPC to complete (or time out) and return
        whatever it came up with
        """
        if core.fg:
            print "servicing background thread"
            while self.queue.empty():
                core.cycle()
    
        return self.queue.get()
    
    #@-node:execute
    #@+node:terminate
    def terminate(self):
        """
        Clean up after ourselves.
        Mainly involves removing ourself from local node
        """
        self.unbindAll()
        try:
            self.localNode.rpcPending.remove(self)
        except:
            #traceback.print_exc()
            pass
    
    #@-node:terminate
    #@+node:returnValue
    def returnValue(self, res=None, **kw):
        """
        Passes a return value back to the original caller, be it
        the local application, or an upstream peer
        
        Arguments:
            - just one - a result object to pass back, if this RPC
              was instigated by a local application call.
              Note that if this RPC was instigated by an upstream
              peer, this will be ignored.
        
        Keywords:
            - the items to return, in the case that this RPC was
              instigated by an upstream peer. Ignored if this
              RPC was instigated by a local application call.
              Note - the RPC invocation/reply dict keys are
              listed at the top of this source file.
        """
        self.terminate()
        if self.callback:
            if hasattr(self, 'cbArgs'):
                self.callback(res, self.cbArgs)
            else:
                self.callback(res)
        elif self.isLocal:
            self.queue.put(res)
        else:
            self.upstreamPeer.send_reply(msgId=self.upstreamMsgId,
                                         **kw)
    #@-node:returnValue
    #@+node:on_reply
    def on_reply(self, peer, msgId, **details):
        """
        Callback which fires when a downstream peer replies
        
        Override this in subclasses
        """
        raise KNotImplemented
    
    #@-node:on_reply
    #@+node:on_tick
    def on_tick(self):
        """
        Callback which fires if the whole RPC times out, in which
        case the RPC should return whatever it can
        
        Override in subclasses
        """
        self.localNode.rpcPending.remove(self)
    
    #@-node:on_tick
    #@-others
#@-node:class KRpc
#@+node:PING
#@+node:class KRpcPing
class KRpcPing(KRpc):
    """
    Implements the PING rpc as per Kademlia spec
    """
    #@    @+others
    #@+node:attribs
    type = 'ping'
    
    #@-node:attribs
    #@+node:__init__
    def __init__(self, localNode, client=None, **kw):
        """
        Creates and performs a PING RPC
    
        Arguments:
            - localNode - the node performing this RPC
            - upstreamPeer - if given, the peer wanting a reply
            - upstreamMsgId - if upstreamPeer is given, this is the msgId
              of the RPC message from the upstream peer
    
        Keywords:
            - peer - the peer to ping - default is local node
        """
        peer = kw.get('peer', None)
        if peer != None:
            peer = localNode._normalisePeer(peer)
        self.peerToPing = peer
    
        if kw.has_key('cbArgs'):
            KRpc.__init__(self, localNode, client, cbArgs=kw['cbArgs'])
        else:
            KRpc.__init__(self, localNode, client)
    
    #@-node:__init__
    #@+node:start
    def start(self):
        """
        Sends out the ping
        """
        peer = self.peerToPing
    
        # are we ourselves being pinged?
        if peer == None:
            # yes, just reply
            self.returnValue(True)
            return
    
        # no - we need to ping a peer
        thisNode = self.localNode
    
        msgId = thisNode.msgId = thisNode._msgIdAlloc()
    
        # bind for peer response
        self.bindPeerReply(peer, msgId)
    
        # and send it off
        self.log(3, "node %s sending ping" % self.localNode.name)
        peer.send_ping(msgId=msgId)
    
        # and set a reply timeout
        self.nextTickTime = time.time() + timeout['ping']
    
    #@-node:start
    #@+node:on_reply
    def on_reply(self, peer, msgId, **details):
        """
        Callback for PING reply
        """
        self.log(3, "got ping reply from %s" % peer)
        self.returnValue(True)
    
    #@-node:on_reply
    #@+node:on_tick
    def on_tick(self):
        """
        'tick' handler.
        
        For PING RPC, the only time we should get a tick is when the ping
        has timed out
        """
        self.log(3, "timeout awaiting ping reply from %s" % self.peerToPing)
        self.returnValue(False)
    
    #@-node:on_tick
    #@-others
#@-node:class KRpcPing
#@-node:PING
#@+node:FIND_NODE
#@+node:class KPeerQueryRecord
class KPeerQueryRecord(KBase):
    """
    Keeps state information regarding a peer we're quering
    """
    #@    @+others
    #@+node:__init__
    def __init__(self, peer, table, state=None, **kw):
    
        self.peer = peer
        self.dest = peer.dest
        self.deadline = time.time() + timeout['findNode']
        self.table = table
    
        # state is always one of:
        #  - 'start'        - have not yet sent query to peer
        #  - 'recommended'  - peer was recommended by another peer, no query sent
        #  - 'queried'      - sent query, awaiting reply or timeout
        #  - 'replied'      - this peer has replied to our query
        #  - 'timeout'      - timed out waiting for peer reply
        #  - 'toofar'       - too far away to be of interest
        #  - 'closest'      - this peer is one of the closest so far
    
        if state == None:
            state = 'start'
        if not isinstance(state, str):
            raise Exception("Invalid state %s" % state)
    
        self.state = state
    
        self.__dict__.update(kw)
    
    #@-node:__init__
    #@+node:hasTimedOut
    def hasTimedOut(self, now=None):
        if now == None:
            now = time.time()
        return self.state == 'queried' and now > self.deadline
    
    #@-node:hasTimedOut
    #@+node:__cmp__
    def __cmp__(self, other):
    
        return cmp(self.peer.id.rawdistance(self.table.sorthash),
                   other.peer.id.rawdistance(self.table.sorthash))
    
    #@-node:__cmp__
    #@+node:__lt__ etc
    def __lt__(self, other):
        return (cmp(self, other) < 0)
    
    def __le__(self, other):
        return (cmp(self, other) <= 0)
    
    def __gt__(self, other):
        return (cmp(self, other) > 0)
    
    def __ge__(self, other):
        return (cmp(self, other) <= 0)
    
    #@-node:__lt__ etc
    #@+node:isCloserThanAllOf
    def isCloserThanAllOf(self, tab):
        """
        returns True if this peerRec is closer to the desired hash
        than all of the peerRecs in table 'tab'
        """
        if not isinstance(tab, KPeerQueryTable):
            self.log(2, "invalid qtable %s" % repr(tab))
            raise Exception("invalid qtable %s" % repr(tab))
        
        for rec in tab:
            if self > rec:
                return False
        return True
    
    #@-node:isCloserThanAllOf
    #@+node:isCloserThanOneOf
    def isCloserThanOneOf(self, tab):
        """
        returns True if this peerRec is closer to the desired hash
        than one or more of of the peerRecs in table 'tab'
        """
        if not isinstance(tab, KPeerQueryTable):
            self.log(2, "invalid qtable %s" % repr(tab))
            raise Exception("invalid qtable %s" % repr(tab))
        
        for rec in tab:
            if self < rec:
                return True
        return False
    
    #@-node:isCloserThanOneOf
    #@-others
#@-node:class KPeerQueryRecord
#@+node:class KPeerQueryTable
class KPeerQueryTable(KBase):
    """
    Holds zero or more instances of KPeerQuery and
    presents/sorts table in different forms
    """
    #@    @+others
    #@+node:__init__
    def __init__(self, lst=None, sorthash=None, state=None, **kw):
        self.peers = []
        if lst == None:
            lst = []
        else:
            self.setlist(lst, state, **kw)
        self.sorthash = sorthash
    
    #@-node:__init__
    #@+node:setlist
    def setlist(self, lst, state=None, **kw):
        for item in lst:
            self.append(item, state, **kw)
    
    #@-node:setlist
    #@+node:getExpired
    def getExpired(self):
        """
        return a list of peers which have expired
        """
        return KPeerQueryTable(
                filter(lambda item: item.hasTimedOut(), self.peers),
                self.sorthash
                )
    
    #@-node:getExpired
    #@+node:purgeExpired
    def purgeExpired(self):
        """
        Eliminate peers which have expired
        """
        for peer in self.peers:
            if peer.hasTimedOut():
                self.peers.remove(peer)
    
    #@-node:purgeExpired
    #@+node:sort
    def sort(self):
        """
        Sort the table in order of increasing distance from self.sorthash
        """
        self.peers.sort()
    
    #@-node:sort
    #@+node:select
    def select(self, criterion):
        """
        Returns a table of items for which criterion(item) returns True
        Otherwise, if 'criterion' is a string, returns the items whose
        state == criterion.
        Otherwise, if 'criterion' is a list or tuple, return the items
        whose state is one of the elements in criterion
        """
        if callable(criterion):
            func = criterion
        elif type(criterion) in [type(()), type([])]:
            func = lambda p: p.state in criterion
        else:
            func = lambda p: p.state == criterion
    
        recs = []
        for peerRec in self.peers:
            if func(peerRec):
                recs.append(peerRec)
        return self.newtable(recs)
    
    #@-node:select
    #@+node:count
    def count(self, *args):
        """
        returns the number of records whose state is one of args
        """
        count = 0
        for rec in self.peers:
            if rec.state in args:
                count += 1
        return count
    
    #@-node:count
    #@+node:changeState
    def changeState(self, oldstate, newstate):
        """
        for all recs of state 'oldstate', change their
        state to 'newstate'
        """
        for p in self.peers:
            if p.state == oldstate:
                p.state = newstate
    #@-node:changeState
    #@+node:filter
    def filter(self, func):
        """
        Eliminate, in place, all items where func(item) returns False
        """
        for peerRec in self.peers:
            if not func(peerRec):
                self.peers.remove(peerRec)
    
    #@-node:filter
    #@+node:purge
    def purge(self, func):
        """
        Eliminate, in place, all items where func(item) returns True
        """
        if 0 and desperatelyDebugging:
            set_trace()
        for peerRec in self.peers:
            if func(peerRec):
                self.peers.remove(peerRec)
    
    #@-node:purge
    #@+node:chooseN
    def chooseN(self, n):
        """
        Randomly select n peer query records
        """
        candidates = self.peers[:]
    
        self.log(3, "candidates = %s" % repr(candidates))
    
        chosen = []
        i = 0
    
        if len(candidates) <= n:
            chosen = candidates
        else:
            while i < n:
                try:
                    peer = random.choice(candidates)
                except:
                    self.log(2, "failed to choose one of %s" % repr(candidates))
                    raise
                chosen.append(peer)
                candidates.remove(peer)
                i += 1
    
        return self.newtable(chosen)
    
    #@-node:chooseN
    #@+node:__str__
    def __str__(self):
        return "<KPeerQueryTable: %d peers>" % len(self) #.peers)
    
    def __repr__(self):
        return str(self)
    
    #@-node:__str__
    #@+node:newtable
    def newtable(self, items, state=None, **kw):
        """
        Returns a new KPeerQueryTable object, based on this
        one, but containing 'items'
        """
        tab = KPeerQueryTable(items, sorthash=self.sorthash, state=state, **kw)
        return tab
    
    #@-node:newtable
    #@+node:dump
    def dump(self):
    
        c = self.count
        self.log(2,
                "PeerQueryTable stats:\n"
                "start:       %s\n"
                "recommended: %s\n"
                "queried:     %s\n"
                "replied:     %s\n"
                "timeout:     %s\n"
                "closest:     %s\n"
                "toofar:      %s\n"
                "TOTAL:       %s\n" % (
                    c('start'),
                    c('recommended'),
                    c('queried'),
                    c('replied'),
                    c('timeout'),
                    c('closest'),
                    c('toofar'),
                    len(self.peers)))
                    
        #states = [p.state for p in self.peers]
        #self.log(3, "PeerQueryTable states:\n%s" % states)
    
    #@-node:dump
    #@+node:list-like methods
    #@+node:extend
    def extend(self, items, state, **kw):
        for item in items:
            self.append(item, state, **kw)
    
    #@-node:extend
    #@+node:append
    def append(self, item, state=None, **kw):
    
        if isinstance(item, KPeerQueryRecord):
            self.log(5, "adding a KPeerQueryRecord, state=%s" % state)
            if state != None:
                item.state = state
            item.__dict__.update(kw)
            peerRec = item
    
        elif isinstance(item, KPeer):
            self.log(5, "adding a KPeer")
            peerRec = KPeerQueryRecord(item, self, state, **kw)
    
        else:
            self.log(2, "bad peer %s" % repr(item))
            raise KBadPeer
    
        if peerRec not in self:
            self.log(5, "peerRec=%s list=%s" % (peerRec, self.peers))
            self.peers.append(peerRec)
        else:
            self.log(5, "trying to append duplicate peer???")
    
    #@-node:append
    #@+node:remove
    def remove(self, item):
        self.peers.remove(item)
    
    #@-node:remove
    #@+node:__getitem__
    def __getitem__(self, idx):
        """
        Allow the table to be indexed by any of:
            - KPeerQueryRecord
            - integer index
            - long string - treated as dest
            - short string - treated as peer id hash string
            - KHash - finds peer with that id
            - KPeer - returns peer with that peer
        """
        if type(idx) == type(0):
            return self.peers[idx]
        elif isinstance(idx, KPeer):
            for peer in self.peers:
                if peer.peer == idx:
                    return peer
            raise IndexError("Query table has no peer %s" % idx)
        elif isinstance(idx, str):
            if len(str) > 512:
                for peer in self.peers:
                    if peer.peer.dest == idx:
                        return peer
                raise IndexError("No peer with dest %s" % idx)
            else:
                for peer in self.peers:
                    if peer.peer.id.value == idx:
                        return peer
                raise IndexError("No peer with dest hash %s" % idx)
        elif isinstance(idx, KHash):
            for peer in self.peers:
                if peer.peer.id == idx:
                    return peer
                raise IndexError("No peer with id %s" % idx)
        else:
            raise IndexError("Invalid selector %s" % repr(idx))
    
    #@-node:__getitem__
    #@+node:__len__
    def __len__(self):
        return len(self.peers)
    
    #@-node:__len__
    #@+node:__getslice__
    def __getslice__(self, fromidx, toidx):
        return KPeerQueryTable(self.peers[fromidx:toidx], self.sorthash)
    
    #@-node:__getslice__
    #@+node:__iter__
    def __iter__(self):
        return iter(self.peers)
    
    #@-node:__iter__
    #@+node:__add__
    def __add__(self, other):
        self.extend(other)
    
    #@-node:__add__
    #@+node:__contains__
    def __contains__(self, other):
        self.log(5, "testing if %s is in %s" % (other, self.peers))
        for peerRec in self.peers:
            if peerRec.peer.dest == other.peer.dest:
                return True
        return False
    
    #@-node:__contains__
    #@-node:list-like methods
    #@-others
#@-node:class KPeerQueryTable
#@+node:class KRpcFindNode
class KRpcFindNode(KRpc):
    """
    Implements the FIND_NODE rpc as per Kademlia spec
    """
    #@    @+others
    #@+node:spec info comments
    #@+at
    # Verbatim extract from original Kademlia paper follows:
    # 
    # The lookup initiator starts by picking x nodes from its closest
    # non-empty k-bucket (or, if that bucket has fewer than x
    # entries, it just takes the closest x nodes it knows of).
    # 
    # The initiator then sends parallel, asynchronous
    # FIND NODE RPCs to the x nodes it has chosen.
    # x is a system-wide concurrency parameter, such as 3.
    # 
    # In the recursive step, the initiator resends the
    # FIND NODE to nodes it has learned about from previous RPCs.
    # 
    # [Paraphrased - in the recursive step, the initiator sends a FIND_NODE to
    # each of the nodes that were returned as results of these previous
    # FIND_NODE RPCs.]
    # 
    # (This recursion can begin before all of the previous RPCs have
    # returned).
    # 
    # Of the k nodes the initiator has heard of closest to
    # the target, it picks x that it has not yet queried and resends
    # the FIND_NODE RPC to them.
    # 
    # Nodes that fail to respond quickly are removed from consideration
    # until and unless they do respond.
    # 
    # If a round of FIND_NODEs fails to return a node any closer
    # than the closest already seen, the initiator resends
    # the FIND NODE to all of the k closest nodes it has
    # not already queried.
    # 
    # The lookup terminates when the initiator has queried and gotten
    # responses from the k closest nodes it has seen.
    #@-at
    #@-node:spec info comments
    #@+node:attribs
    type = 'findNode'
    #@-node:attribs
    #@+node:__init__
    def __init__(self, localNode, client=None, **kw):
        """
        Creates and launches the findNode rpc
        
        Arguments:
            - localNode - the node performing this RPC
            - client - see KRpc.__init__
    
        Keywords:
            - hash - a string, long int or KHash object representing
              what we're looking for. treatment depends on type:
                  - KHash object - used as is
                  - string - gets wrapped into a KHash object
                  - long int - wrapped into a KHash object
              refer KHash.__init__
            - raw - whether 'hash' is already a hash, default True
            - local - True/False - whether to only search local store,
              or pass on the query to the network, default True
        """
        kw = dict(kw)
        if kw.get('raw', False):
            h = kw['hash']
            del kw['hash']
            kw['raw'] = h
            self.hashWanted = KHash(**kw)
        else:
            self.hashWanted = KHash(kw['hash'], **kw)
        self.isLocalOnly = kw.get('local', True)
    
        self.numQueriesPending = 0
    
        self.numRounds = 0   # count number of rounds
        self.numReplies = 0  # number of query replies received
        self.numQueriesSent = 0
        self.numPeersRecommended = 0
    
        # whichever mode we're called from, we gotta find the k closest peers
        self.localNode = localNode
        self.peerTab = self.findClosestPeersInitial()
    
        self.log(4, "KRpcFindNode: isLocalOnly=%s" % self.isLocalOnly)
    
        if kw.has_key('cbArgs'):
            KRpc.__init__(self, localNode, client, cbArgs=kw['cbArgs'])
        else:
            KRpc.__init__(self, localNode, client)
    
    #@-node:__init__
    #@+node:start
    def start(self):
        """
        Kicks off this RPC
        """
        # if we're being called by an upstream initiator, just return the peer list
        if self.isLocalOnly:
            peerDests = [peer.dest for peer in self.peerTab]
            self.log(5, "findNode: local only: returning to upstream with %s" % repr(peerDests))
            self.returnValue(peerDests)
            return
    
        # just return nothing if we don't have any peers
        if len(self.peerTab) == 0:
            self.returnValue([])
            return
    
        # send off first round of queries
        self.sendSomeQueries()
    
        return
    
    #@-node:start
    #@+node:sendSomeQueries
    def sendSomeQueries(self, **kw):
        """
        First step of findNode
        
        Select alpha nodes that we haven't yet queried, and send them queries
        """
        # bail if too busy
        if self.numQueriesPending >= maxConcurrentQueries:
            return
    
        # shorthand
        localNode = self.localNode
        hashWanted = self.hashWanted
    
        # randomly choose some peers
        #somePeerRecs = self.peerTab.chooseN(numSearchPeers)
        somePeerRecs = self.peerTab.select('start')
        
        # start our ticker
        self.nextTickTime = time.time() + timeout['findNode']
    
        numQueriesSent = 0
    
        # and send them findNode queries
        if len(somePeerRecs) > 0:
            for peerRec in somePeerRecs:
                self.log(3, "querying %s" % peerRec)
                if self.numQueriesPending < maxConcurrentQueries:
                    self.sendOneQuery(peerRec)
                    numQueriesSent += 1
                else:
                    break
            self.log(3, "%s queries sent, awaiting reply" % numQueriesSent)
        else:
            self.log(3, "no peer recs???")
            for peerRec in self.peerTab:
                self.log(4, "%s state=%s, dest=%s..." % (peerRec, peerRec.state, peerRec.dest[:12]))
    
    #@-node:sendSomeQueries
    #@+node:sendOneQuery
    def sendOneQuery(self, peerRec):
        """
        Sends off a query to a single peer
        """
        if peerRec.state != 'start':
            self.log(2, "duh!! peer state %s:\n%s" % (peerRec.state, peerRec))
            return 
    
        msgId = self.localNode._msgIdAlloc()
        self.bindPeerReply(peerRec.peer, msgId)
        peerRec.msgId = msgId
    
        if self.type == 'findData':
            peerRec.peer.send_findData(hash=self.hashWanted, msgId=msgId)
        else:
            peerRec.peer.send_findNode(hash=self.hashWanted, msgId=msgId)
    
        peerRec.state = 'queried'
    
        self.numQueriesPending += 1
    
        self.numQueriesSent += 1
    
    #@-node:sendOneQuery
    #@+node:findClosestPeersInitial
    def findClosestPeersInitial(self):
        """
        Searches our k-buckets, and returns a table of k of
        peers closest to wanted hash into self.closestPeersInitial
        """
        hashobj = self.hashWanted
    
        lst = []
        buckets = self.localNode.buckets
        for bucket in buckets:
            for peer in bucket:
                lst.append(peer)
    
        table = KPeerQueryTable(lst, self.hashWanted, 'start')
        table.sort()
    
        return table[:maxBucketSize]
    
    #@-node:findClosestPeersInitial
    #@+node:addPeerIfCloser
    def addPeerIfCloser(self, peer):
        """
        Maintains the private .peersToQuery array.
        If the array is not yet maxed (ie, length < maxBucketSize),
        the peer is simply added.
        However, if the array is maxed, it finds the least-close peer,
        and replaces it with the given peer if closer.
        """
    #@-node:addPeerIfCloser
    #@+node:isCloserThanQueried
    def isCloserThanQueried(self, peer):
        """
        Test function which returns True if argument 'peer'
        is closer than all the peers in self.peersAlreadyQueried,
        or False if not
        """
        for p in self.peersAlreadyQueried:
            if p.id.rawdistance(self.hashWanted) < peer.id.rawdistance(self.hashWanted):
                return False
        return True
    
    #@-node:isCloserThanQueried
    #@+node:on_reply
    def on_reply(self, peer, msgId, **details):
        """
        Callback for FIND_NODE reply
        """
        # shorthand
        peerTab = self.peerTab
    
        self.numReplies += 1
    
        # ------------------------------------------------------------
        # determine who replied, and get the raw dests sent back
        try:
            peerRec = peerTab[peer]
        except:
            traceback.print_exc()
            self.log(3, "discarding findNode reply from unknown peer %s %s, discarding" % (
                peer, details))
            return
    
        # one less query to wait for
        self.numQueriesPending -= 1
    
        # ----------------------------------------------------------
        # peerRec is the peer that replied
        # peers is a list of raw dests
    
        # save ref to this peer, it's seemingly good
        self.localNode.addref(peerRec.peer)
    
        # mark it as having replied
        if peerRec.state != 'queried':
            self.log(2, "too weird - got a reply from a peer we didn't query")
        peerRec.state = 'replied'
    
        # wrap the returned peers as KPeer objects
        peersReturned = details.get('result', [])
        peersReturned = [self.localNode._normalisePeer(p) for p in peersReturned]
    
        self.numPeersRecommended += len(peersReturned)
    
        # and add them to table in state 'recommended'
        for p in peersReturned:
            peerTab.append(p, 'recommended')
    
        # try to fire off more queries
        self.sendSomeQueries()
    
        # and check for and action possible end of query round
        self.checkEndOfRound()
    
    
    #@-node:on_reply
    #@+node:on_tick
    def on_tick(self):
        """
        Callback for FIND_NODE reply timeout
        """
        # check for timeouts, and update offending peers
        now = time.time()
        for peerRec in self.peerTab:
            if peerRec.hasTimedOut(now):
                peerRec.state = 'timeout'
    
                # makes room for more queries
                self.sendSomeQueries()
    
        # possible end of round
        self.checkEndOfRound()
    
        # schedule next tick
        self.nextTickTime = time.time() + 5
    
    #@-node:on_tick
    #@+node:checkEndOfRound
    def checkEndOfRound(self):
        """
        Checks if we've hit the end of a query round.
        If so, and if either:
            - we've got some closer peers, OR
            - we've heard from less than maxBucketSize peers,
        fire off more queries
        
        Otherwise, return the best available
        """
        peerTab = self.peerTab
    
        if core.fg:
            set_trace()
    
        # has this query round ended?
        if peerTab.count('start', 'queried') > 0:
            # not yet
            return
    
        self.log(2, "********** query round ended")
    
        # ------------------------------------
        # end of round processing
    
        self.numRounds += 1
    
        # did we get any closer to required hash?
        if self.type == 'findData' \
                or self.gotAnyCloser() \
                or peerTab.count('closest') < maxBucketSize:
    
            # yes - save these query results
            self.log(4, "starting another round")
            peerTab.changeState('replied', 'closest')
            peerTab.changeState('recommended', 'start')
    
            # cull the shortlist
            self.log(2, "culling to k peers")
            if peerTab.count('closest') > maxBucketSize:
                peerTab.sort()
                excess = peerTab.select('closest')[maxBucketSize:]
                excess.changeState('closest', 'toofar')
                pass
    
            # and start up another round
            self.sendSomeQueries()
    
        # did anything launch?
        if peerTab.count('start', 'queried') == 0:
            # no - we're screwed
            self.returnTheBestWeGot()
            
        # done for now
        return
    
    #@-node:checkEndOfRound
    #@+node:gotAnyCloser
    def gotAnyCloser(self):
        """
        Tests if any peer records in state 'recommended' or 'replied'
        are nearer than the records in state 'closest'
        """
        peerTab = self.peerTab
    
        # get current closest peers
        closest = peerTab.select('closest')
    
        # if none yet, then this was just end of first round
        if len(closest) == 0:
            return True
    
        # get the peers we're considering
        #candidates = peerTab.select(('recommended', 'replied'))
        candidates = peerTab.select('recommended')
    
        # now test them
        gotOneCloser = False
        for c in candidates:
            #if c.isCloserThanOneOf(closest):
            if c.isCloserThanAllOf(closest):
                return True
    
        # none were closer
        return False
    
    #@-node:gotAnyCloser
    #@+node:returnTheBestWeGot
    def returnTheBestWeGot(self):
        """
        Returns the k closest nodes to the wanted hash that we have
        actually heard from
        """
        # pick the peers which have replied to us
        closest = self.peerTab.select('closest')
        
        self.peerTab.dump()
    
        # add ourself to the list - we could easily be one of the best
        localNode = self.localNode
        selfDest = localNode._normalisePeer(localNode.dest) 
        closest.append(selfDest, state='closest')
        
        # sort in order of least distance first
        closest.sort()
    
        # pick the best k of these
        #peersHeardFrom = peersHeardFrom[:maxBucketSize]
        #peersHeardFrom = peersHeardFrom[:numSearchPeers]
    
        # extract their dest strings
        peers = [p.peer.dest for p in closest]
    
        # pass these back
        self.returnValue(peers)
    
        # and we're done
        return
    
    #@-node:returnTheBestWeGot
    #@+node:returnValue
    def returnValue(self, items):
        """
        override with a nicer call sig
        """
        # a hack for testing - save this RPC object into the node
        # so we can introspect it
        self.localNode.lastrpc = self
    
        items = items[:maxBucketSize]
    
        self.reportStats()
    
        KRpc.returnValue(self, items, result=items)
    
    
    #@-node:returnValue
    #@+node:reportStats
    def reportStats(self):
        """
        Logs a stat dump of query outcome
        """
        if self.isLocalOnly:
            return
        self.log(2,
            "query terminated after %s rounds, %s queries, %s replies, %s recommendations" % (
               (self.numRounds+1),
               self.numQueriesSent,
               (self.numReplies+1),
               self.numPeersRecommended
               )
            )
    #@-node:reportStats
    #@-others
#@-node:class KRpcFindNode
#@-node:FIND_NODE
#@+node:FIND_DATA
#@+node:class KRpcFindData
class KRpcFindData(KRpcFindNode):
    """
    variant of KRpcFindNode which returns key value if found
    """
    #@    @+others
    #@+node:attribs
    type = 'findData'
    #@-node:attribs
    #@+node:start
    def start(self):
        """
        Kicks off the RPC.
        If requested key is stored locally, simply returns it.
        Otherwise, falls back on parent method
        """
        # if we posses the data, just return the data
        value = self.localNode.storage.getKey(self.hashWanted.asHex(), keyIsHashed=True)
        if value != None:
            self.log(4, "Found required value in local storage")
            self.log(4, "VALUE='%s'" % value)
            self.on_gotValue(value, self.hashWanted.asHex())
            return
    
        # no such luck - pass on to parent
        KRpcFindNode.start(self)
    
    #@-node:start
    #@+node:on_reply
    def on_reply(self, peer, msgId, **details):
        """
        Callback for FIND_NODE reply
        """
        res = details.get('result', None)
        if isinstance(res, str):
            self.on_gotValue(res, self.hashWanted.asHex())
        else:
            KRpcFindNode.on_reply(self, peer, msgId, **details)
    
    #@-node:on_reply
    #@+node:on_gotValue
    def on_gotValue(self, value, hash=None):
        """
        Callback which fires when we get the value stored under a key
        
        Value is either the real value, or a splitfile manifest
        If a real value, just return it.
        If a splitfile manifest, launch nested findValue RPCs to get each chunk
        """
        nchunks = 0
        try:
            firstline, rest = value.split("\n", 1)
            firstline = firstline.strip()
            kwd, str_nchunks = firstline.split(":")
            if kwd != 'chunks':
                raise hell
            nchunks = int(nchunks)
            value = rest
        except:
            pass # in this case, hell hath no fury at all
        
        if nchunks == 0:
            self.returnValue(value)
            return
    
        # now we get to the hard bit - we have to set up nested findData RPCs to
        # get all the chunks and reassemble them
        hashes = rest.strip().split("\n")
        
        # do sanity checks
        hashesAllValid = [len(h) == 40 for h in hashes]
        if len(hashes) != nchunks:
            self.log(
                2,
                "Splitfile retrieval failure\nmanifest contains %s hashes, should have been %s" % (
                    len(hashes), nchunks))
            self.returnValue(None)
        if False in hashesAllValid:
            self.log(2, "Splitfile retrieval failure - one or more invalid hashes")
    
        # now this is a bit weird - we need to bind each chunk to its hash, so we create a
        # class which produces callables which fire our on_gotChunk callback
        class ChunkNotifier:
            def __init__(me, h, cb):
                me.h = h
                me.cb = cb
            def __call__(me, val):
                me.cb(me.h, val)
        
        # now launch the chunk retrieval RPCs
        # result is that for each retrieved chunk, our on_gotChunk callback will
        # be invoked with the arguments (hash, value), so we can tick them off
        self.numChunks = nchunks
        self.numChunksReceived = 0
        self.chunkHashes = hashes
        self.chunks = dict.fromkeys(hashes)
        for h in hashes:
            KRpcFindData(self.localNode, h, ChunkNotifier(h, self.on_gotChunk))
    
        # now, we can sit back and receive the chunks
    
    #@-node:on_gotValue
    #@+node:on_gotChunk
    def on_gotChunk(self, hexhash, value):
        """
        Callback which fires when a nested chunk findNode returns
        """
        if value == None:
            self.log(2, "Chunk retrieval failed, fatal to this findData")
            self.returnValue(None)
            return
        
        # got a value - vet it against hash
        if shahash(value) != hexhash:
            self.log(2, "Got a chunk, but it doesn't hash right - fatal to this findData")
            self.returnValue(None)
            return
    
        # it's valid - stash it
        self.chunks[hexhash] = value
        self.numChunksReceived += 1
    
        # have we finished yet?
        if self.numChunksReceived <= self.numChunks:
            # no
            self.log(4, "Received chunk %s of %s" % (self.numChunksReceived, self.numChunks))
            return
    
        # maybe we have
        self.log(4, "We appear to have all chunks, checking further")
    
        # sanity check
        if None in self.chunks.values():
            self.log(2, "Fatal - reached chunk count, but chunks still missing")
            self.returnValue(None)
            return
                
        # finally done - got all chunks, hashes are valid, reassemble in order
        allChunks = [self.chunks[h] for h in self.chunkHashes]
        reassembled = "".join(allChunks)
        self.log(4, "Reassembled all %s chunks, SUCCESS" % self.numChunks)
        self.returnValue(reassembled)
    
    #@-node:on_gotChunk
    #@+node:returnValue
    def returnValue(self, items):
        """
        override with a nicer call sig
        """
        # a hack for testing - save this RPC object into the node
        # so we can introspect it
        self.localNode.lastrpc = self
    
        # another debugging hack
        self.reportStats()
    
        KRpc.returnValue(self, items, result=items)
    
    #@-node:returnValue
    #@-others

#@-node:class KRpcFindData
#@-node:FIND_DATA
#@+node:STORE
#@+node:class KRpcStore
class KRpcStore(KRpc):
    """
    Implements key storage
    """
    #@    @+others
    #@+node:attribs
    type = 'store'
    #@-node:attribs
    #@+node:__init__
    def __init__(self, localNode, client=None, **kw):
        """
        Creates and launches a STORE rpc
        
        Arguments:
            - localNode - the node performing this RPC
            - client - see KRpc.__init__
    
        Keywords:
            - key - the key under which we wish to save the data
            - value - the value we wish to save
            - local - True/False:
                - if True, only save in local store
                - if False, do a findNode to find the nodes to save the
                  key to, and tell them to save it
              default is True
        """
        self.key = kw['key']
        #self.keyHashed = shahash(self.key)
        self.keyHashed = self.key
        self.value = kw['value']
        self.isLocalOnly = kw.get('local', True)
    
        # set 'splitting' flag to indicate if we need to insert as splitfiles
        self.splitting = len(self.value) > maxValueSize
    
        self.log(4, "isLocalOnly=%s" % self.isLocalOnly)
    
        if kw.has_key('cbArgs'):
            KRpc.__init__(self, localNode, client, cbArgs=kw['cbArgs'])
        else:
            KRpc.__init__(self, localNode, client)
    
    #@-node:__init__
    #@+node:start
    def start(self):
        """
        Kicks off this RPC
        """
        # if too big, then break up into <30k chunks
        if self.splitting:
            self.storeSplit()
            return
    
        # not too big - prefix a 0 chunk count, and go ahead as a single entity
        self.value = "chunks:0\n" + self.value
            
        # if local only, or no peers, just save locally
        if self.isLocalOnly or len(self.localNode.peers) == 0:
            result = self.localNode.storage.putKey(self.keyHashed, self.value, keyIsHashed=True)
            if result:
                result = 1
            else:
                result = 0
            self.returnValue(result)
            return
    
        # no - se have to find peers to store the key to, and tell them to
        # store the key
        
        # launch a findNode rpc, continue in our callback
        KRpcFindNode(self.localNode, self.on_doneFindNode,
                     hash=self.keyHashed, raw=True, local=False)
        return
    
    
    #@-node:start
    #@+node:storeSplit
    def storeSplit(self):
        """
        Gets called if we're splitting a big file into smaller chunks
        
        Here, we:
            - break the file up into chunks
            - build a manifest
            - launch store RPCs to store each chunk, where the key is SHA(chunk)
            - launch a store RPC to store the 'manifest' (noting that if the manifest
              is too big, it'll get recursively inserted as a splitfile as well
        """
        # break up into chunks
        chunks = []
        hashes = []
        size = len(self.value)
        i = 0
        self.nchunks = 0
        while i < size:
            chunks.append(self.value[i:i+maxValueSize])
            hashes.append(shahash(chunks[-1]))
            i += maxValueSize
            self.nchunks += 1
    
        # build the manifest
        manifest = "chunks:%s\n%s\n" % (self.nchunks, "\n".join(hashes))
        
        # set progress attributes
        self.chunkManifestInserted = False
        self.chunksInserted = 0
    
        # launch nested Store RPCs for manifest, and each chunk
        KRpcStore(self.localNode, self.on_doneChunkManifest,
                  local=self.isLocalOnly,
                  key=self.key,
                  value=manifest)
        i = 0
        while i < self.nchunks:
            KRpcStore(self.localNode, self.on_doneChunk,
                      local=self.isLocalOnly,
                      key=hashes[i],
                      value=chunks[i])
            i += 1
    
        # now sit back and wait for the callbacks
    #@-node:storeSplit
    #@+node:on_doneChunkManifest
    def on_doneChunkManifest(self, result):
        """
        Callback which fires when a manifest insert succeeds/fails
        """
        # the chunk callback handles all
        self.on_doneChunk(result, isManifest=True)
    #@-node:on_doneChunkManifest
    #@+node:on_doneChunk
    def on_doneChunk(self, result, isManifest=False):
        """
        Callback which fires when a single chunk insert succeeds/fails
        """
        # a failure either way means the whole RPC has failed
        if not result:
            # one huge fuck-up
            self.returnValue(False)
            return
    
        # update our tally
        if isManifest:
            self.chunkManifestInserted = True
        else:
            self.chunksInserted += 1
    
        # finished?
        if self.chunkManifestInserted and (self.chunksInserted == self.nchunks):
            # yep = success
            self.returnValue(True)
    
    #@-node:on_doneChunk
    #@+node:returnValue
    def returnValue(self, result):
        """
        an override with a nicer call sig
        """
        # a hack for testing - save this RPC object into the node
        # so we can introspect it
        self.localNode.lastrpc = self
    
        try:
            KRpc.returnValue(self, result, status=result)
        except:
            traceback.print_exc()
            self.log(3, "Failed to return %s" % repr(result))
            KRpc.returnValue(self, 0, status=0)
    
    #@-node:returnValue
    #@+node:on_doneFindNode
    def on_doneFindNode(self, lst):
        """
        Receive a callback from findNode
        
        Send STORE command to each node that comes back
        """
        localNode = self.localNode
    
        # normalise results
        normalisePeer = localNode._normalisePeer
        peers = [normalisePeer(p) for p in lst] # wrap in KPeer objects
    
        self.log(2, "STORE RPC findNode - got peers %s" % repr(peers))
    
        i = 0
    
        self.numPeersSucceeded = 0
        self.numPeersFailed = 0
        self.numPeersFinished = 0
    
        # and fire off store messages for each peer
        for peer in peers:
    
            if peer.dest == localNode.dest:
                self.log(3, "storing to ourself")
                localNode.storage.putKey(self.keyHashed, self.value, keyIsHashed=True)
                self.numPeersSucceeded += 1
                self.numPeersFinished += 1
            else:
                msgId = self.localNode._msgIdAlloc()
                self.log(4, "forwarding store cmd to peer:\npeer=%s\nmsgId=%s" % (peer, msgId))
                self.bindPeerReply(peer, msgId)
                peer.send_store(key=self.keyHashed, value=self.value, msgId=msgId)
            i += 1
            if i >= numStorePeers:
                break
    
        self.nextTickTime = time.time() + timeout['store']
    
        self.log(2, "Sent store cmd to %s peers, awaiting responses" % i)
    
        self.numPeersToStore = i
    
    
    #@-node:on_doneFindNode
    #@+node:on_reply
    def on_reply(self, peer, msgId, **details):
        """
        callback which fires when we get a reply from a STORE we sent to a
        peer
        """
        self.numPeersSucceeded += 1
        self.numPeersFinished += 1
        
        if self.numPeersFinished == self.numPeersToStore:
            # rpc is finished
            self.returnValue(True)
    
    #@-node:on_reply
    #@+node:on_tick
    def on_tick(self):
    
        self.log(3, "Timeout awaiting store reply from %d out of %d peers" % (
            self.numPeersToStore - self.numPeersSucceeded, self.numPeersToStore))
    
        if self.numPeersSucceeded == 0:
            self.log(3, "Store timeout - no peers replied, storing locally")
            self.localNode.storage.putKey(self.keyHashed, self.value, keyIsHashed=True)
    
        self.returnValue(True)
    
    #@-node:on_tick
    #@-others
#@-node:class KRpcStore
#@-node:STORE
#@+node:PINGALL
#@+node:class KRpcPingAll
class KRpcPingAll(KRpc):
    """
    Pings all peers
    """
    #@    @+others
    #@+node:attribs
    type = 'pingall'
    #@-node:attribs
    #@+node:__init__
    def __init__(self, localNode, client=None, **kw):
        """
        Creates and launches a PINGALL rpc
    
        Arguments:
            - localNode - the node performing this RPC
            - client - see KRpc.__init__
    
        Keywords: none
        """
        if kw.has_key('cbArgs'):
            KRpc.__init__(self, localNode, client, cbArgs=kw['cbArgs'])
        else:
            KRpc.__init__(self, localNode, client)
    
    #@-node:__init__
    #@+node:start
    def start(self):
        """
        Kicks off this RPC
        """
        # launch a findNode rpc against each of our peers
        peers = self.localNode.peers
        self.numSent = self.numPending = len(peers)
        self.numReplied = self.numFailed = 0
        for peer in peers:
            KRpcPing(self.localNode, self.on_reply, peer=peer)
        return
    
    #@-node:start
    #@+node:on_reply
    def on_reply(self, result):
        """
        callback which fires when we get a reply from a STORE we sent to a
        peer
        """
        log(3, "got %s" % repr(result))
    
        if result:
            self.numReplied += 1
        else:
            self.numFailed += 1
        self.numPending -= 1
    
        if self.numPending <= 0:
            res = "pinged:%s replied:%s timeout:%s" % (
                    self.numSent, self.numReplied, self.numFailed)
            self.log(3, res)
            self.returnValue(res)
    
    #@-node:on_reply
    #@+node:on_tick
    def on_tick(self):
    
        self.log(3, "this shouldn't have happened")
        self.returnValue(False)
    
    #@-node:on_tick
    #@+node:returnValue
    def returnValue(self, result):
        """
        an override with a nicer call sig
        """
        # a hack for testing - save this RPC object into the node
        # so we can introspect it
        self.localNode.lastrpc = self
    
        try:
            KRpc.returnValue(self, result, status=result)
        except:
            traceback.print_exc()
            self.log(3, "Failed to return %s" % repr(result))
            KRpc.returnValue(self, 0, status=0)
    
    #@-node:returnValue
    #@-others
#@-node:class KRpcPingAll
#@-node:PINGALL
#@-node:RPC Classes
#@+node:Node Socket Server
#@+node:class KNodeServer
class KNodeServer(KBase, SocketServer.ThreadingMixIn, SocketServer.TCPServer):
    """
    Listens for incoming socket connections
    """
    #@    @+others
    #@+node:__init__
    def __init__(self, node, addr=None):
    
        if addr == None:
            addr = clientAddr
    
        self.isRunning = True
    
        self.node = node
    
        listenHost, listenPort = addr.split(":")
        listenPort = int(listenPort)
        self.listenPort = listenPort
        SocketServer.TCPServer.__init__(self, (listenHost, listenPort), KNodeReqHandler)
    
    #@-node:__init__
    #@+node:serve_forever
    def serve_forever(self):
    
        print "awaiting client connections on port %s" % self.listenPort
        while self.isRunning:
            self.handle_request()
    
    #@-node:serve_forever
    #@-others
#@-node:class KNodeServer
#@+node:class KNodeReqHandler
class KNodeReqHandler(KBase, SocketServer.StreamRequestHandler):
    """
    Manages a single client connection
    """
    #@    @+others
    #@+node:handle
    def handle(self):
        """
        Conducts all conversation for a single req
        """
        req = self.request
        client = self.client_address
        server = self.server
        node = self.server.node
    
        read = self.rfile.read
        readline = self.rfile.readline
        write = self.wfile.write
        flush = self.wfile.flush
        
        finish = self.finish
        
        # start with a greeting
        write("Stasher version %s ready\n" % version)
        
        # get the command
        line = readline().strip()
    
        try:
            cmd, args = re.split("\\s+", line, 1)
        except:
            cmd = line
            args = ''
    
        self.log(3, "cmd=%s args=%s" % (repr(cmd), repr(args)))
        
        if cmd in ["get", "getlocal"]:
            isLocal = cmd == "getlocal"
            value = node.get(args, local=isLocal)
            if value == None:
                write("notfound\n")
            else:
                write("ok\n%s\n%s" % (len(value), value))
            flush()
            time.sleep(2)
            finish()
            return
        
        elif cmd in ["put", "putlocal"]:
            isLocal = cmd == "putlocal"
            try:
                size = int(readline())
                value = read(size)
                res = node.put(args, value, local=isLocal)
                if res:
                    write("ok\n")
                else:
                    write("failed\n")
                flush()
            except:
                traceback.print_exc()
                write("exception\n")
            finish()
            return
    
        elif cmd == 'addref':
            try:
                res = node.addref(args, True)
                if res:
                    write("ok\n")
                else:
                    write("failed\n")
                flush()
            except:
                traceback.print_exc()
                write("exception\n")
            finish()
            return
    
        elif cmd == 'getref':
            res = node.dest
            write("ok\n")
            write("%s\n" % res)
            flush()
            time.sleep(1)
            finish()
            return
    
        elif cmd == 'pingall':
            res = node._pingall()
            write(res+"\n")
            finish()
            return
    
        elif cmd == "die":
            server.isRunning = False
            write("server terminated\n")
            finish()
    
        else:
            write("unrecognisedcommand\n")
            finish()
            return
    
    #@-node:handle
    #@+node:finish
    def finish(self):
    
        SocketServer.StreamRequestHandler.finish(self)
    
    #@-node:finish
    #@-others
#@-node:class KNodeReqHandler
#@+node:class KNodeClient
class KNodeClient(KBase):
    """
    Talks to a KNodeServer over a socket
    
    Subclass this to implement Stasher clients in Python
    """
    #@    @+others
    #@+node:__init__
    def __init__(self, address=clientAddr):
    
        if type(address) in [type(()), type([])]:
            self.host, self.port = clientAddr
        else:
            self.host, self.port = clientAddr.split(":")
            self.port = int(self.port)
    
        self.hello()
    
    #@-node:__init__
    #@+node:hello
    def hello(self):
        
        self.connect()
        self.close()
    #@-node:hello
    #@+node:connect
    def connect(self):
    
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.connect((self.host, self.port))
    
        self.rfile = self.sock.makefile("rb")
        self.read = self.rfile.read
        self.readline = self.rfile.readline
        self.wfile = self.sock.makefile("wb")
        self.write = self.wfile.write
        self.flush = self.wfile.flush
    
        # read greeting
        greeting = self.readline()
        parts = re.split("\\s+", greeting)
        if parts[0] != "Stasher":
            self.close()
            raise Exception("Not connected to valid stasher interface")
    
    #@-node:connect
    #@+node:close
    def close(self):
        
        self.rfile.close()
        #self.wfile.close()
        self.sock.close()
    
    #@-node:close
    #@+node:get
    def get(self, key, **kw):
        """
        sends a get command to stasher socket, and retrieves
        and interprets result
        
        Arguments:
            - key - key to retrieve
        
        Keywords:
            - local - default False - if True, only looks in local storage
        
        Returns key's value if found, or None if key not found
        """
        if kw.get('local', False):
            cmd = 'getlocal'
        else:
            cmd = 'get'
    
        self.connect()
        
        self.write("%s %s\n" % (cmd, key))
        self.flush()
    
        #print "waiting for resp line"
        res = self.readline().strip()
    
        if res == "ok":
            size = int(self.readline())
            val = self.read(size)
            self.close()
            return val
        else:
            self.close()
            return None
    
    #@-node:get
    #@+node:put
    def put(self, key, val, **kw):
        """
        Tells remote stasher port to insert a file into the network
    
        Arguments:
            - key - key to insert under
            - val - value to insert under this key
        
        Keywords:
            - local - default False - if True, only looks in local storage
        
        """
        if kw.get('local', False):
            cmd = 'putlocal'
        else:
            cmd = 'put'
    
        self.connect()
        self.write("%s %s\n" % (cmd, key))
        self.write("%s\n" % len(val))
        self.write(val)
        self.flush()
        
        res = self.readline().strip()
    
        self.close()
    
        if res == "ok":
            return True
        else:
            print repr(res)
            return False
    
    #@-node:put
    #@+node:addref
    def addref(self, ref):
        """
        Passes a new noderef to node
        """
        self.connect()
        self.write("addref %s\n" % ref)
        self.flush()
        
        res = self.readline().strip()
    
        self.close()
    
        if res == "ok":
            return True
        else:
            print repr(res)
            return False
    
    #@-node:addref
    #@+node:getref
    def getref(self):
        """
        Uplifts node's own ref
        """
        self.connect()
        self.write("getref\n")
        self.flush()
        
        res = self.readline().strip()
    
        if res == "ok":
            ref = self.readline().strip()
            self.close()
            return ref
        else:
            self.close()
            return "failed"
    
    #@-node:getref
    #@+node:pingall
    def pingall(self):
        """
        Uplifts node's own ref
        """
        self.connect()
        self.write("pingall\n")
        self.flush()
        
        res = self.readline().strip()
    
        self.close()
    
        return res
    
    
    #@-node:pingall
    #@+node:kill
    def kill(self):
        """
        Tells remote server to fall on its sword
        """
        try:
            while 1:
                self.connect()
                self.write("die\n")
                self.flush()
                self.close()
        except:
            pass
    
    
    #@-node:kill
    #@+node:__getitem__
    def __getitem__(self, item):
        
        return self.get(item)
    
    #@-node:__getitem__
    #@+node:__setitem__
    def __setitem__(self, item, val):
        
        if not self.put(item, val):
            raise Exception("Failed to insert")
    
    #@-node:__setitem__
    #@-others

#@-node:class KNodeClient
#@-node:Node Socket Server
#@+node:NODE
#@+node:class KNode
class KNode(KBase):
    """
    B{Public API to this Kademlia implementation}
    
    You should not normally need to use, or even be aware of, 
    any of the other classes

    And in this class, the only methods you need to worry about are:
        - L{start} - starts the node running
        - L{stop} - stops the node
        - L{get} - retrieve a key value
        - L{put} - stores a key value
        - L{addref} - imports a noderef

    This class implements a single kademlia node.
    Within a single process, you can create as many nodes as you like.
    """
    #@    @+others
    #@+node:attributes
    SocketFactory = None # defaults to I2P socket
    
    #@-node:attributes
    #@+node:__init__
    def __init__(self, name, **kw):
        """
        Creates a kademlia node of name 'name'.
    
        Name is mandatory, because each name is permanently written
        to the SAM bridge's store
        
        I thought of supporting random name generation, but went off this
        idea because names get permanently stored to SAM bridge's file
    
        Arguments:
            - name - mandatory - a short text name for the node, should
              be alphanumerics, '-', '.', '_'
              This name is used for the SAM socket session.
        
        Keywords:
            - storage - optional - an instance of L{KStorageBase} or one of
              its subclasses. If not given, default action is to instantiate
              a L{KStorageFile} object against the given node name
        """
        # remember who we are
        self.name = name
    
        # not running yet, will launch when explicitly started, or implicitly
        # when the first operation gets done
        self.isRunning = False
    
        # create socket and get its dest, and determine our node id
        self.id = KHash("<NONE>")
        self.log(5, "creating socket for node %s" % name)
        self.log(5, "socket for node %s created" % name)
        if self.SocketFactory == None:
            self.SocketFactory = i2p.socket.socket
        self.sock = self.SocketFactory(
            "stashernode-"+name,
            i2p.socket.SOCK_DGRAM,
            samaddr=samAddr,
            **kw)
        #self.sockLock = threading.Lock() # prevents socket API reentrance
        self.sock.setblocking(0)
        self.dest = self.sock.dest
        self.id = KHash(self.dest)
    
        # create our buckets
        self.buckets = []
        for i in range(160):
            self.buckets.append(KBucket())
        
        # create our storage object, default to new instance of KStorageFile
        self.storage = kw.get('storage', KStorageFile(self))
    
        # dig out all previously known nodes
        self.peers = self.storage.getRefs()
    
        # set up dict of callers awaiting replies
        # keys are (peerobj, msgId) tuples, values are Queue.Queue objects
        self.pendingPings = {}
    
        # mapping of (peer, msgId) to RPC object, so when RPC replies come in,
        # they can be passed directly to the RPC object concerned
        self.rpcBindings = {}
    
        # KRpc objects waiting for peer replies - used for checking for timeouts
        self.rpcPending = []
        
        # miscellaneous shit
        self._msgIdNext = 0
        #self._msgIdLock = threading.Lock()
    
        # register in global map
        _nodes[name] = self
    
    
    #@-node:__init__
    #@+node:__del__
    def __del__(self):
        """
        Cleanup
        """
    
    #@-node:__del__
    #@+node:application-level
    #@+node:start
    def start(self, doPings=True):
        """
        Starts the node running
        """
        # barf if already running
        if self.isRunning:
            self.log(3, "node %s is already running!" % self.name)
            return
    
        self.log(3, "starting node %s" % self.name)
    
        # first step - ping all our peers
        if doPings:
            for peer in self.peers:
                self.log(3, "doing initial ping\n%s\n%s" % (self, peer))
                KRpcPing(self, peer=peer)
    
        # first step - do a findNode against our own node id, and ping our
        # neighbours
        if greetPeersOnStartup:
            neighbours = KRpcFindNode(self, hash=self.id).execute()
            self.log(3, "neighbours=%s" % repr([n[:10] for n in neighbours]))
            for n in neighbours:
                n = self._normalisePeer(n)
                KRpcPing(self, peer=n)
    
        # note now that we're running
        self.isRunning = True
    
        # and enlist with the core
        if runCore:
            core.subscribe(self)
        else:
            # central core disabled, run our own receiver thread instead
            thread.start_new_thread(self._threadRx, ())
    #@-node:start
    #@+node:stop
    def stop(self):
        """
        Shuts down the node
        """
        self.isRunning = 0
        if runCore:
            try:
                core.unsubscribe(self)
            except:
                pass
    #@-node:stop
    #@+node:get
    def get(self, item, callback=None, **kw):
        """
        Attempts to retrieve data from the network
        
        Arguments:
            - item - the key we desire
            - callback - optional - if given, the get will be performed
              asynchronously, and callback will be invoked upon completion, with
              the result as first argument
        Keywords:
            - local - optional - if True, limits this search to this local node
              default is False
        
        Returns:
            - if no callback - the item value if the item was found, or None if not
            - if callback, None is returned
        """
        def processResult(r):
            if isinstance(r, str):
                return r
            return None
    
        if callback:
            # create a func to process callback result
            def onCallback(res):
                callback(processResult(res))
        
            self._finddata(item, onCallback, **kw)
        else:
            return processResult(self._finddata(item, **kw))
    
    #@-node:get
    #@+node:put
    def put(self, key, value, callback=None, **kw):
        """
        Inserts a named key into the network
    
        Arguments:
            - key - one of:
                - None - a secure key will be generated and used
                - a KHash object
                - a raw string which will be hashed into a KHash object
            - val - a string, the value associated with the key
    
        Keywords:
            - local - default False - if True, limits the insert to the
              local node
    
        If the value is larger than L{maxValueSize}, a L{KValueTooLarge}
        exception will occur.
        """
        return self._store(key, value, callback, **kw)
    
    #@-node:put
    #@+node:addref
    def addref(self, peer, doPing=False):
        """
        Given a peer node's destination, add it to our
        buckets and internal data store
        
        Arguments:
            - peer - one of:
                - the I2P destination of the peer node, as
                  a base64 string
                - a KNode object
                - a KPeer object
            - doPing - ping this node automatically (default False)
        """
        peer = self._normalisePeer(peer)
    
        # remember peer if not already known
        if peer.dest == self.dest:
            self.log(3, "node %s, trying to add ref to ourself???" % self.name)
            return peer
        elif not self._findPeer(peer.dest):
            self.peers.append(peer)
            self.storage.putRefs(peer)
        else:
            self.log(4, "node %s, trying to add duplicate noderef %s" % (
                self.name, peer))
            return peer
    
        # update our KBucket
        dist = self.id.distance(peer.id)
        self.buckets[dist].justSeenPeer(peer)
    
        if doPing:
            self.log(4, "doing initial ping\n%s\n%s" % (self, peer))
            KRpcPing(self, peer=peer)
    
        return peer
    
    #@-node:addref
    #@+node:__getitem__
    def __getitem__(self, item):
        """
        Allows dict-like accesses on the node object
        """
        return self.get(item)
    #@-node:__getitem__
    #@+node:__setitem__
    def __setitem__(self, item, val):
        """
        Allows dict-like key setting on the node object
        """
        self.put(item, val)
    
    #@-node:__setitem__
    #@-node:application-level
    #@+node:peer/rpc methods
    #@+node:_ping
    def _ping(self, peer=None, callback=None, **kw):
        """
        Sends a ping to remote peer, and awaits response
        
        Not of much real use to application level, except
        perhaps for testing
    
        If the argument 'peer' is not given, the effect is to 'ping the
        local node', which I guess might be a bit silly
    
        The second argument 'callback' is a callable, which if given, makes this
        an asynchronous (non-blocking) call, in which case the callback will be
        invoked upon completion (or timeout).
        
        If the keyword 'cbArgs' is given in addition to the callback, the callback
        will fire with the results as first argument and this value as second arg
        """
        if callback:
            KRpcPing(self, callback, peer=peer, **kw)
        else:
            return KRpcPing(self, peer=peer).execute()
    
    #@-node:_ping
    #@+node:_pingall
    def _pingall(self, callback=None):
        """
        Sends a ping to all peers, returns text string on replies/failures
        """
        if callback:
            KRpcPingAll(self, callback, **kw)
        else:
            return KRpcPingAll(self).execute()
    
       
    #@-node:_pingall
    #@+node:_findnode
    def _findnode(self, something=None, callback=None, **kw):
        """
        Mainly for testing - does a findNode query on the network
        
        Arguments:
            - something - one of:
                - plain string - the string gets hashed and used for the search
                - int or long int - this gets used as the raw hash
                - a KHash object - that's what gets used
                - None - the value of the 'raw' keyword will be used instead
            - callback - optional - if given, a callable object which will be
              called upon completion, with the result as argument
    
        Keywords:
            - local - optional - if True, only returns the closest peers known to
              node. if False, causes node to query other nodes.
              default is False
            - raw - one of:
                - 20-byte string - this gets used as a binary hash
                - 40-byte string - this gets used as a hex hash
        """
        if not kw.has_key('local'):
            kw = dict(kw)
            kw['local'] = False
    
        self.log(3, "about to instantiate findnode rpc")
        if callback:
            KRpcFindNode(self, callback, hash=something, **kw)
            self.log(3, "asynchronously invoked findnode, expecting callback")
        else:
            lst = KRpcFindNode(self, hash=something, **kw).execute()
            self.log(3, "back from findnode rpc")
            res = [self._normalisePeer(p) for p in lst] # wrap in KPeer objects
            return res
    
    #@-node:_findnode
    #@+node:_finddata
    def _finddata(self, something=None, callback=None, **kw):
        """
        As for findnode, but if data is found, return the data instead
        """
        if not kw.has_key('local'):
            kw = dict(kw)
            kw['local'] = False
    
        self.log(3, "about to instantiate finddata rpc")
        if callback:
            KRpcFindData(self, callback, hash=something, **kw)
            self.log(3, "asynchronously invoked finddata, expecting callback")
        else:
            res = KRpcFindData(self, hash=something, **kw).execute()
            self.log(3, "back from finddata rpc")
            if not isinstance(res, str):
                self.log(4, "findData RPC returned %s" % repr(res))
                res = [self._normalisePeer(p) for p in res] # wrap in KPeer objects
            return res
    
    #@-node:_finddata
    #@+node:_store
    def _store(self, key, value, callback=None, **kw):
        """
        Performs a STORE rpc
        
        Arguments:
            - key - string - text name of key
            - value - string - value to store
    
        Keywords:
            - local - if given and true, only store value onto local store
        """
        if not kw.has_key('local'):
            kw = dict(kw)
            kw['local'] = False
    
        key = shahash(key)
        if callback:
            KRpcStore(self, callback, key=key, value=value, **kw)
            self.log(3, "asynchronously invoked findnode, expecting callback")
        else:
            res = KRpcStore(self, key=key, value=value, **kw).execute()
            return res
    
    #@-node:_store
    #@+node:_findPeer
    def _findPeer(self, dest):
        """
        Look up our table of current peers for a given dest.
        
        If dest is found, return its object, otherwise return None
        """
        for peerObj in self.peers:
            if peerObj.dest == dest:
                return peerObj
        return None
    
    #@-node:_findPeer
    #@-node:peer/rpc methods
    #@+node:comms methods
    #@+node:_sendRaw
    def _sendRaw(self, peer, **kw):
        """
        Serialises keywords passed, and sends this as a datagram
        to node 'peer'
        """
        # update our KBucket
        dist = self.id.distance(peer.id)
        self.buckets[dist].justSeenPeer(peer)
    
        # save ref to this peer
        self.addref(peer)
    
        params = dict(kw)
        msgId = params.get('msgId', None)
        if msgId == None:
            msgId = params['msgId'] = self._msgIdAlloc()
    
        objenc = messageEncode(params)
        self.log(5, "node %s waiting for send lock" % self.name)
        #self.sockLock.acquire()
        self.log(5, "node %s got send lock" % self.name)
        try:
            self.sock.sendto(objenc, 0, peer.dest)
        except:
            traceback.print_exc()
        #self.sockLock.release()
        self.log(5, "node %s released send lock" % self.name)
    
        self.log(4, "node %s sent %s to peer %s" % (self.name, params, peer.dest))
        return msgId
    
    #@-node:_sendRaw
    #@-node:comms methods
    #@+node:engine
    #@+node:_threadRx
    def _threadRx(self):
        """
        Thread which listens for incoming datagrams and actions
        accordingly
        """
        self.log(3, "starting receiver thread for node %s" % self.name)
    
        try:
            # loop to drive the node
            while self.isRunning:
                self._doChug()
        except:
            traceback.print_exc()
            self.log(3, "node %s - THREAD CRASH!" % self.name)
    
        self.log(3, "receiver thread for node %s terminated" % self.name)
    
    #@-node:_threadRx
    #@+node:_doChug
    def _doChug(self):
        """
        Do what's needed to drive the node.
        Handle incoming packets
        Check on and action timeouts
        """
        # handle all available packets
        while self._doRx():
            pass
    
        # do maintenance - eg processing timeouts
        self._doHousekeeping()
    
    #@-node:_doChug
    #@+node:_doRx
    def _doRx(self):
        """
        Receives and handles one incoming packet
        
        Returns True if a packet got handled, or False if timeout
        """
        # get next packet
        self.log(5, "%s seeking socket lock" % self.name)
        #self.sockLock.acquire()
        self.log(5, "%s got socket lock" % self.name)
        try:
            item = self.sock.recvfrom(-1)
        except i2p.socket.BlockError:
            #self.sockLock.release()
            self.log(5, "%s released socket lock after timeout" % self.name)
            if not runCore:
                time.sleep(0.1)
            return False
        except:
            traceback.print_exc()
            self.log(5, "%s released socket lock after exception" % self.name)
            #self.sockLock.release()
            return True
        #self.sockLock.release()
        self.log(5, "%s released socket lock normally" % self.name)
    
        try:
            (data, dest) = item
        except ValueError:
            self.log(3, "node %s: recvfrom returned no dest, possible spoof" \
                % self.name)
            data = item[0]
            dest = None
    
        # try to decode
        try:
            d = messageDecode(data)
        except:
            traceback.print_exc()
            self.log(3, "failed to unpickle incoming data for node %s" % \
                self.name)
            return True
    
        # ditch if not a dict
        if type(d) != type({}):
            self.log(3, "node %s: decoded packet is not a dict" % self.name)
            return True
    
        # temporary workaround for sam socket bug    
        if dest == None:
            if hasattr(d, 'has_key') and d.has_key('dest'):
                dest = d['dest']
    
        # try to find it in our store
        peerObj = self._findPeer(dest)
        if peerObj == None:
            # previously unknown peer - add it to our store
            peerObj = self.addref(dest)
        else:
            peerObj.justSeen() # already exists - refresh its timestamp
            self.addref(peerObj.dest)
    
        # drop packet if no msgId
        msgId = d.get('msgId', None)
        if msgId == None:
            self.log(3, "no msgId, dropping")
            return True
        del d['msgId']
    
        msgType = d.get('type', 'unknown')
    
        if desperatelyDebugging:
            pass
            #set_trace()
    
        # if a local RPC is awaiting this message, fire its callback
        item = self.rpcBindings.get((peerObj.dest, msgId), None)
        if item:
            rpc, peer = item
            try:
                rpc.unbindPeerReply(peerObj, msgId)
                if desperatelyDebugging:
                    set_trace()
                rpc.on_reply(peerObj, msgId, **d)
    
            except:
                traceback.print_exc()
                self.log(2, "unhandled exception in RPC on_reply")
        else:
            # find a handler, fallback on 'unknown'
            self.log(5, "\nnode %s\ngot msg id %s type %s:\n%s" % (
                self.name, msgId, msgType, d))
            hdlrName = d.get('type', 'unknown')
            hdlr = getattr(self, "_on_"+hdlrName)
            try:
                if desperatelyDebugging:
                    set_trace()
                hdlr(peerObj, msgId, **d)
            except:
                traceback.print_exc()
                self.log(2, "unhandled exception in unbound packet handler %s" % hdlrName)
    
        return True
    
    #@-node:_doRx
    #@+node:_doHousekeeping
    def _doHousekeeping(self):
        """
        Performs periodical housekeeping on this node.
        
        Activities include:
            - checking pending records for timeouts
        """
        now = time.time()
    
        # DEPRECATED - SWITCH TO RPC-based
        # check for expired pings
        for msgId, (dest, q, pingDeadline) in self.pendingPings.items():
    
            if pingDeadline > now:
                # not timed out, leave in pending
                continue
    
            # ping has timed out
            del self.pendingPings[msgId]
            q.put(False)
    
        # check for timed-out RPCs
        for rpc in self.rpcPending[:]:
            if rpc.nextTickTime != None and now >= rpc.nextTickTime:
                try:
                    rpc.on_tick()
                except:
                    traceback.print_exc()
                    self.log(2, "unhandled exception in RPC on_tick")
    
    #@-node:_doHousekeeping
    #@-node:engine
    #@+node:event handling
    #@+others
    #@+node:_on_ping
    def _on_ping(self, peer, msgId, **kw):
        """
        Handler for ping received events
        """
        KRpcPing(self, (peer, msgId), local=True, **kw)
        return
        
        # old stuff
    
        self.log(3, "\nnode %s\nfrom %s\nreceived:\n%s" % (self.name, peer, kw))
    
        # randomly cause ping timeouts if testing
        if testing:
            howlong = random.randint(0, 5)
            self.log(3, "deliberately pausing for %s seconds" % howlong)
            time.sleep(howlong)
    
        # pong back to node
        peer.send_reply(msgId=msgId)
    
        
    #@nonl
    #@-node:_on_ping
    #@+node:_on_findNode
    def _on_findNode(self, peer, msgId, **kw):
        """
        Handles incoming findNode command
        """
        KRpcFindNode(self, (peer, msgId), local=True, **kw)
    
    #@-node:_on_findNode
    #@+node:_on_findData
    def _on_findData(self, peer, msgId, **kw):
        """
        Handles incoming findData command
        """
        KRpcFindData(self, (peer, msgId), local=True, **kw)
    
    #@-node:_on_findData
    #@+node:_on_store
    def _on_store(self, peer, msgId, **kw):
        """
        Handles incoming STORE command
        """
        self.log(4, "got STORE rpc from upstream:\npeer=%s\nmsgId=%s\nkw=%s" % (peer, msgId, kw))
    
        KRpcStore(self, (peer, msgId), local=True, **kw)
    
    #@-node:_on_store
    #@+node:_on_reply
    def _on_reply(self, peer, msgId, **kw):
        """
        This should never happen
        """
        self.log(4, "got unhandled reply:\npeer=%s\nmsgId=%s\nkw=%s" % (
            peer, msgId, kw))
    
    #@-node:_on_reply
    #@+node:_on_unknown
    def _on_unknown(self, peer, msgId, **kw):
        """
        Handler for unknown events
        """
        self.log(3, "node %s from %s received msgId=%s:\n%s" % (
            self.name, peer, msgId, kw))
    
    #@-node:_on_unknown
    #@-others
    #@-node:event handling
    #@+node:Socket Client Server
    #@+node:serve
    def serve(self):
        """
        makes this node listen on socket for incoming client
        connections, and services these connections
        """
        server = KNodeServer(self)
        server.serve_forever()
    
    #@-node:serve
    #@-node:Socket Client Server
    #@+node:lowlevel stuff
    #@+others
    #@+node:__str__
    def __str__(self):
        return "<KNode:%s=0x%s...>" % (
            self.name,
            ("%x" % self.id.value)[:8],
            )
    #@-node:__str__
    #@+node:__repr__
    def __repr__(self):
        return str(self)
    
    #@-node:__repr__
    #@+node:_msgIdAlloc
    def _msgIdAlloc(self):
        """
        issue a new and unique message id
        """
        #self._msgIdLock.acquire()
        msgId = self._msgIdNext
        self._msgIdNext += 1
        #self._msgIdLock.release()
        return msgId
    #@-node:_msgIdAlloc
    #@+node:_normalisePeer
    def _normalisePeer(self, peer):
        """
        Takes either a b64 dest string, a KPeer object or a KNode object,
        and returns a KPeer object
        """
        # act according to whatever type we're given
        if isinstance(peer, KPeer):
            return peer # already desired format
        elif isinstance(peer, KNode):
            return KPeer(self, peer.dest)
        elif isinstance(peer, str) and len(peer) > 256:
            return KPeer(self, peer)
        else:
            self.log(3, "node %s, trying to add invalid noderef %s" % (
                self.name, peer))
            raise KBadNode(peer)
    
    #@-node:_normalisePeer
    #@+node:__del__
    def __del__(self):
        """
        Clean up on delete
        """
        self.log(3, "node dying: %s" % self.name)
    
        try:
            del _nodes[self.name]
        except:
            pass
    
        self.stop()
    
    #@-node:__del__
    #@-others
    #@-node:lowlevel stuff
    #@-others
#@-node:class KNode
#@-node:NODE
#@+node:funcs
#@+others
#@+node:userI2PDir
def userI2PDir(nodeName=None):
    """
    Returns a directory under user's home dir into which
    stasher files can be written

    If nodename is given, a subdirectory will be found/created
    
    Return value is toplevel storage dir if nodename not given, 
    otherwise absolute path including node dir
    """
    if dataDir != None:
        if not os.path.isdir(dataDir):
            os.makedirs(dataDir)
        return dataDir

    if sys.platform == 'win32':
        home = os.getenv("APPDATA")
        if home:
            topDir = os.path.join(home, "stasher")
        else:
            topDir = os.path.join(os.getcwd(), "stasher")
    else:
        #return os.path.dirname(__file__)
        topDir = os.path.join(os.path.expanduser('~'), ".stasher")

    if not os.path.isdir(topDir):
        os.makedirs(topDir)
    if nodeName == None:
        return topDir
    else:
        nodeDir = os.path.join(topDir, nodeName)
        if not os.path.isdir(nodeDir):
            os.makedirs(nodeDir)
        return nodeDir

#@-node:userI2PDir
#@+node:nodePidfile
def nodePidfile(nodename):
    return os.path.join(userI2PDir(nodename), "node.pid")

#@-node:nodePidfile
#@+node:messageEncode
def messageEncode(params):
    """
    Serialise the dict 'params' for sending
    
    Temporarily using bencode - replace later with a more
    efficient struct-based impl.
    """
    try:
        return bencode.bencode(params)
    except:
        log(1, "encoder failed to encode: %s" % repr(params))
        raise

#@-node:messageEncode
#@+node:messageDecode
def messageDecode(raw):
    return bencode.bdecode(raw)
#@-node:messageDecode
#@+node:shahash
def shahash(somestr, bin=False):
    shaobj = sha.new(somestr)
    if bin:
        return shaobj.digest()
    else:
        return shaobj.hexdigest()

#@-node:shahash
#@+node:log
logLock = threading.Lock()

def log(verbosity, msg, nPrev=0, clsname=None):

    global logToSocket, logFile

    # create logfile if not exists
    if logFile == None:
        logFile = os.path.join(userI2PDir(), "stasher.log")

    # rip the stack
    caller = traceback.extract_stack()[-(2+nPrev)]
    path, line, func = caller[:3]
    path = os.path.split(path)[1]

    #print "func is type %s, val %s" % (type(func), repr(func))

    #if hasattr(func, "im_class"):
    #    func = 

    if clsname:
        func = clsname + "." + func

    #msg = "%s:%s:%s():  %s" % (
    #    path,
    #    line,
    #    func,
    #    msg.replace("\n", "\n   + "))

    msg = "%s():%s:  %s" % (
        func,
        line,
        msg.replace("\n", "\n   + "))

    # do better logging later
    if verbosity > logVerbosity:
        return

    if logToSocket:
        try:
            if isinstance(logToSocket, int):
                portnum = logToSocket
                logToSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                connected = 0
                while 1:
                    try:
                        logToSocket.connect(("localhost", portnum))
                        break
                    except socket.error:
                        print "Please open an xterm/nc listening on %s" % logToSocket
                        time.sleep(1)

            logToSocket.send(msg+"\n")
        except:
            traceback.print_exc()
    else:
        print msg

    logLock.acquire()
    file(logFile, "a+").write(msg + "\n")
    logLock.release()
#@-node:log
#@+node:logexc
def logexc(verbosity, msg, nPrev=0, clsname=None):

    fd = StringIO("%s\n" % msg)
    traceback.print_exc(file=fd)
    log(verbosity, fd.getvalue(), nPrev, clsname)

#@-node:logexc
#@+node:spawnproc
def spawnproc(*args, **kw):
    """
    Spawns a process and returns its PID
    
    VOMIT!
    
    I have to do a pile of odious for the win32 side
    
    Returns a usable PID
    
    Keywords:
        - priority - priority at which to spawn - default 20 (highest)
    """
    # get priority, convert to a unix 'nice' value
    priority = 20 - kw.get('priority', 20)
    
    if sys.platform != 'win32':
        # *nix - easy
        #print "spawnproc: launching %s" % repr(args)
        
        # insert nice invocation
        args = ['/usr/bin/nice', '-n', str(priority)] + list(args)
        return os.spawnv(os.P_NOWAIT, args[0], args)

    else:
        # just close your eyes here and pretend this abomination isn't happening! :((
        args = list(args)
        args.insert(0, sys.executable)
        cmd = " ".join(args)
        #print "spawnproc: launching %s" % repr(cmd)

        if 0:
            try:
                c =  _winreg.ConnectRegistry(None, _winreg.HKEY_LOCAL_MACHINE)
                c1 = _winreg.OpenKey(c, "SOFTWARE")
                c2 = _winreg.OpenKey(c1, "Microsoft")
                c3 = _winreg.OpenKey(c2, "Windows NT")
                c4 = _winreg.OpenKey(c3, "CurrentVersion")
                supportsBelowNormalPriority = 1
            except:
                supportsBelowNormalPriority = 0
        else:
            if sys.getwindowsversion()[3] != 2:
                supportsBelowNormalPriority = 0
            else:
                supportsBelowNormalPriority = 1
    
        # frig the priority into a windows value
        if supportsBelowNormalPriority:
            if priority < 7:
                pri = win32process.IDLE_PRIORITY_CLASS
            elif priority < 14:
                pri = 0x4000
            else:
                pri = win32process.NORMAL_PRIORITY_CLASS
        else:
            if priority < 11:
                pri = win32process.IDLE_PRIORITY_CLASS
            else:
                pri = win32process.NORMAL_PRIORITY_CLASS
    
        print "spawnproc: launching %s" % repr(args)
        si = win32process.STARTUPINFO()
        hdl = win32process.CreateProcess(
            sys.executable, # lpApplicationName
            cmd,  # lpCommandLine
            None,  # lpProcessAttributes
            None, # lpThreadAttributes
            0,    # bInheritHandles
            0,    # dwCreationFlags
            None, # lpEnvironment
            None, # lpCurrentDirectory
            si,   # lpStartupInfo
            )
        pid = hdl[2]
        #print "spawnproc: pid=%s" % pid
        return pid
#@-node:spawnproc
#@+node:killproc
def killproc(pid):
    if sys.platform == 'win32':
        print repr(pid)
        handle = win32api.OpenProcess(1, 0, pid)
        print "pid %s -> %s" % (pid, repr(handle))
        #return (0 != win32api.TerminateProcess(handle, 0))
        win32process.TerminateProcess(handle, 0)
    else:
        return os.kill(pid, signal.SIGKILL)
#@-node:killproc
#@+node:i2psocket
def i2psocket(self, *args, **kw):
    return i2p.socket.socket(*args, **kw)

#@-node:i2psocket
#@+node:usage
def usage(detailed=False, ret=0):
    
    print "Usage: %s <options> [<command> [<ars>...]]" % sys.argv[0]
    if not detailed:
        print "Type %s -h for help" % sys.argv[0]
        sys.exit(ret)

    print "This is stasher, distributed file storage network that runs"
    print "atop the anonymising I2P network (http://www.i2p.net)"
    print "Written by aum - August 2004"
    print
    print "Options:"
    print "  -h, --help              - display this help"
    print "  -v, --version           - print program version"
    print "  -V, --verbosity=n       - verbosity, default 1, 1=quiet ... 4=noisy"
    print "  -S, --samaddr=host:port - host:port of I2P SAM port, "
    print "                            default %s" % i2p.socket.samaddr
    print "  -C, --clientaddr=host:port - host:port for socket interface to listen on"
    print "                            for clients, default %s" % clientAddr
    print "  -d, --datadir=dir       - directory in which stasher files get written"
    print "                            default is ~/.stasher"
    print "  -f, --foreground        - only valid for 'start' cmd - runs the node"
    print "                            in foreground without spawning - for debugging"
    print "  -l, --localonly         - only valid for get/put - restricts the get/put"
    print "                            operation to the local node only"
    print
    print "Commands:"
    print "  start [<nodename>]"
    print "    - launches a single node, which forks off and runs in background"
    print "      nodename is a short unique nodename, default is '%s'" % defaultNodename
    print "  stop [<nodename>]"
    print "    - terminates running node <nodename>"
    print "  get <keyname> [<file>]"
    print "    - attempts to retrieve key <keyname> from the network, saving"
    print "      to file <file> if given, or to stdout if not"
    print "  put <keyname> [<file>]"
    print "    - inserts key <keyname> into the network, taking its content"
    print "      from file <file> if given, otherwise reads content from stdin"
    print "  addref <file>"
    print "    - adds a new noderef to the node, taking the base64 noderef"
    print "      from file <file> if given, or from stdin"
    print "      (if you don't have any refs, visit http://stasher.i2p, or use"
    print "      the dest in the file aum.stasher in cvs)"
    print "  getref <file>"
    print "    - uplifts the running node's dest as base64, writing it to file"
    print "      <file> if given, or to stdout"
    print "  hello"
    print "    - checks that local node is running"
    print "  pingall"
    print "    - diagnostic tool - pings all peers, waits for replies or timeouts,"
    print "      reports results"
    print "  help"
    print "    - display this help"
    print

    sys.exit(0)

#@-node:usage
#@+node:err
def err(msg):
    sys.stderr.write(msg+"\n")
#@-node:err
#@+node:main
def main():
    """
    Command line interface
    """
    global samAddr, clientAddr, logVerbosity, dataDir

    argv = sys.argv
    argc = len(argv)

    try:
        opts, args = getopt.getopt(sys.argv[1:],
                                   "h?vV:S:C:sd:fl",
                                   ['help', 'version', 'samaddr=', 'clientaddr=',
                                    'verbosity=', 'status', 'datadir=', 'foreground',
                                    'shortversion', 'localonly',
                                    ])
    except:
        traceback.print_exc(file=sys.stdout)
        usage("You entered an invalid option")

    daemonise = True
    verbosity = 2
    debug = False
    foreground = False
    localOnly = False

    for opt, val in opts:
        
        if opt in ['-h', '-?', '--help']:
            usage(True)

        elif opt in ['-v', '--version']:
            print "Stasher version %s" % version
            sys.exit(0)

        elif opt in ['-V', '--verbosity']:
            logVerbosity = int(val)

        elif opt in ['-f', '--foreground']:
            foreground = True
        
        elif opt in ['-S', '--samaddr']:
            samAddr = val

        elif opt in ['-C', '--clientaddr']:
            clientAddr = val
        
        elif opt in ['-s', '--status']:
            dumpStatus()

        elif opt in ['-d', '--datadir']:
            dataDir = val

        elif opt == '--shortversion':
            sys.stdout.write("%s" % version)
            sys.stdout.flush()
            sys.exit(0)

        elif opt in ['-l', '--localonly']:
            localOnly = True

    #print "Debug - bailing"
    #print repr(opts)
    #print repr(args)
    #sys.exit(0)

    # Barf if no command given
    if len(args) == 0:
        err("No command given")
        usage(0, 1)

    cmd = args.pop(0)
    argc = len(args)

    #print "cmd=%s, args=%s" % (repr(cmd), repr(args))
    
    if cmd not in ['help', '_start', 'start', 'stop',
                   'hello', 'get', 'put', 'addref', 'getref',
                   'pingall']:
        err("Illegal command '%s'" % cmd)
        usage(0, 1)

    if cmd == 'help':
        usage()

    # dirty hack
    if foreground and cmd == 'start':
        cmd = '_start'

    # magic undocumented command name - starts node, launches its client server,
    # this should only happen if we're spawned from a 'start' command
    if cmd == '_start':
        if argc not in [0, 1]:
            err("start: bad argument count")
            usage()
        if argc == 0:
            nodeName = defaultNodename
        else:
            nodeName = args[0]
        
        # create and serve a node
        #set_trace()
        node = KNode(nodeName)
        node.start()
        log(3, "Node %s launched, dest = %s" % (node.name, node.dest))
        node.serve()
        sys.exit(0)
    
    if cmd == 'start':
        if argc not in [0, 1]:
            err("start: bad argument count")
            usage()
        if argc == 0:
            nodeName = defaultNodename
        else:
            nodeName = args[0]
        pidFile = nodePidfile(nodeName)

        if os.path.exists(pidFile):
            err(("Stasher node '%s' seems to be already running. If you are\n" % nodeName)
                +"absolutely sure it's not running, please remove its pidfile:\n"
                +pidFile+"\n")
            sys.exit(1)

        # spawn off a node
        import stasher
        pid = spawnproc(sys.argv[0], "-S", samAddr, "-C", clientAddr, "_start", nodeName)
        file(pidFile, "wb").write("%s" % pid)
        print "Launched stasher node as pid %s" % pid
        print "Pidfile is %s" % pidFile
        sys.exit(0)

    if cmd == 'stop':
        if argc not in [0, 1]:
            err("stop: bad argument count")
            usage()
        if argc == 0:
            nodeName = defaultNodename
        else:
            nodename = args[0]

        pidFile = nodePidfile(nodeName)

        if not os.path.isfile(pidFile):
            err("Stasher node '%s' is not running - cannot kill\n" % nodeName)
            sys.exit(1)

        pid = int(file(pidFile, "rb").read())
        try:
            killproc(pid)
            print "Killed stasher node (pid %s)" % pid
        except:
            print "Failed to kill node (pid %s)" % pid
        os.unlink(pidFile)
        sys.exit(0)

    try:
        client = KNodeClient()
    except:
        traceback.print_exc()
        err("Node doesn't seem to be up, or reachable on %s" % clientAddr)
        return


    if cmd == 'hello':
        err("Node seems fine")
        sys.exit(0)
    
    elif cmd == 'get':
        if argc not in [1, 2]:
            err("get: bad argument count")
            usage()

        key = args[0]

        if argc == 2:
            # try to open output file
            path = args[1]
            try:
                outfile = file(path, "wb")
            except:
                err("Cannot open output file %s" % repr(path))
                usage(0, 1)
        else:
            outfile = sys.stdout

        if logVerbosity >= 3:
            sys.stderr.write("Searching for key - may take up to %s seconds or more\n" % (
                timeout['findData']))
        res = client.get(key, local=localOnly)
        if res == None:
            err("Failed to retrieve '%s'" % key)
            sys.exit(1)
        else:
            outfile.write(res)
            outfile.flush()
            outfile.close()
            sys.exit(0)

    elif cmd == 'put':
        if argc not in [1, 2]:
            err("put: bad argument count")
            usage()

        key = args[0]

        if argc == 2:
            # try to open input file
            path = args[1]
            try:
                infile = file(path, "rb")
            except:
                err("Cannot open input file %s" % repr(path))
                usage(0, 1)
        else:
            infile = sys.stdin

        val = infile.read()
        if len(val) > maxValueSize:
            err("File is too big - please trim to %s" % maxValueSize)

        if logVerbosity >= 3:
            sys.stderr.write("Inserting key - may take up to %s seconds\n" % (
                timeout['findNode'] + timeout['store']))
        res = client.put(key, val, local=localOnly)
        if res == None:
            err("Failed to insert '%s'" % key)
            sys.exit(1)
        else:
            sys.exit(0)

    elif cmd == 'addref':
        if argc not in [0, 1]:
            err("addref: bad argument count")
            usage()

        if argc == 1:
            # try to open input file
            path = args[0]
            try:
                infile = file(path, "rb")
            except:
                err("Cannot open input file %s" % repr(path))
                usage(0, 1)
        else:
            infile = sys.stdin

        ref = infile.read()

        res = client.addref(ref)
        if res == None:
            err("Failed to add ref")
            sys.exit(1)
        else:
            sys.exit(0)

    elif cmd == 'getref':
        if argc not in [0, 1]:
            err("getref: bad argument count")
            usage()

        res = client.getref()

        if argc == 1:
            # try to open output file
            path = args[0]
            try:
                outfile = file(path, "wb")
            except:
                err("Cannot open output file %s" % repr(path))
                usage(0, 1)
        else:
            outfile = sys.stdout

        if res == None:
            err("Failed to retrieve node ref")
            sys.exit(1)
        else:
            outfile.write(res)
            outfile.flush()
            outfile.close()
            sys.exit(0)

    elif cmd == 'pingall':
        if logVerbosity > 2:
            print "Pinging all peers, waiting %s seconds for results" % timeout['ping']
        res = client.pingall()
        print res
        sys.exit(0)

#@-node:main
#@-others
#@-node:funcs
#@+node:MAINLINE
#@+others
#@+node:mainline
if __name__ == '__main__':

    main()

#@-node:mainline
#@-others
#@-node:MAINLINE
#@-others

#@-node:@file stasher.py
#@-leo
