Changeset 203

Show
Ignore:
Timestamp:
09/19/08 18:29:29 (4 months ago)
Author:
pierre
Message:

In the io.py file, add methods to save/load pickle data. Much more faster for handling big SpikeLists. Some names have been changed, could be discussed, but the whole package is much more user friendly. Need to continue to work on the AnalogSignal? classes

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • branches/cleanup/src/io.py

    r202 r203  
    1 import numpy, os, logging 
    2  
     1import numpy, os, logging, cPickle 
    32DEFAULT_BUFFER_SIZE = 10000 
    43 
    54 
    6 class TxtDataLoader(object): 
     5class DataIO(object): 
    76    """ 
    8     Convenient object to load txt file and read data 
     7    Convenient object to load txt file and read data. Data could be in text files or in  
     8    pickled files, dumped via Python. This is a much more faster way to read/write data. 
     9    Nevertheless, they can't be directly accessible. 
    910    """ 
    1011    def __init__(self, filename): 
    …… 
    1314            raise Exception("The file %s does not exists !" %filename) 
    1415     
    15     def load(self, sepchar = "\t", skipchar = "#"): 
     16    def pickle_load(self): 
     17        return cPickle.load(file(self.filename,"r")) 
     18     
     19    def pickle_write(self, object, filename): 
     20        return cPickle.dump(object, file(filename, "w")) 
     21     
     22    def txt_load(self, sepchar = "\t", skipchar = "#"): 
    1623        myfile = open(self.filename, "r", DEFAULT_BUFFER_SIZE) 
    1724        contents = myfile.readlines() 
    …… 
    2532                    data.append([int(float(items[1])), float(items[0])]) 
    2633        return(data) 
     34     
     35    def txt_write(self, data, filename): 
     36        # The simplest way to do that, for the moment, is to supposed that you have 
     37        # numpy 1.0.4 :-) 
     38        numpy.savetxt(filename, data) 
    2739 
    2840 
    29 class TxtSpikeListLoader(object): 
     41 
     42class SpikeListIO(object): 
    3043    """ 
    3144    Convenient object to load txt file and process them in order 
    …… 
    3447    def __init__(self, filename): 
    3548        self.filename = filename 
    36         self.loader   = TxtDataLoader(self.filename) 
     49        self.data_io  = DataIO(self.filename) 
    3750        self.header   = {} 
    3851     
    …… 
    5669        f.close() 
    5770     
    58     def write_header(self, outputfile): 
    59         f = open(outputfile,"w") 
    60         for key, value in self.header.items(): 
    61             f.write("# %s = %s" %(key, value)) 
     71    def write_header(self, spikelist, filename): 
     72        f = open(filename,"w") 
     73        f.write("# dimensions = %s\n" % spikelist.dimensions) 
     74        f.write("# first_id = %d\n" % numpy.min(spikelist.id_list)+1) 
     75        f.write("# last_id = %d\n" % numpy.max(spikelist.id_list)+1) 
     76        f.write("# dt = %g\n" % spikelist.dt) 
    6277        f.close() 
    63      
    64     #def write_data(self, SpikeList, outputfile): 
     78         
    6579     
    6680    def get_spikelist(self, id_list=None, dt=None, t_start=None, t_stop=None, dims=None, label=None): 
    67         self.parse_header() 
    6881         
    69         if dt is None and 'dt' in self.header: 
    70             logging.debug("dt is infered from the file header") 
    71             dt = self.header['dt'] 
     82        # First, we try to see if the data are stored in a pickled way (MUCH FASTER) 
     83        try : 
     84            res = self.data_io.pickle_load() 
     85            return res 
     86        except Exception: 
     87        # Otherwise we try the text mode.... 
     88            self.parse_header() 
     89            if dt is None and 'dt' in self.header: 
     90                logging.debug("dt is infered from the file header") 
     91                dt = self.header['dt'] 
     92            if (id_list is None) and (('first_id' in self.header) and ('last_id' in self.header)): 
     93                logging.debug("id_list is infered from the file header") 
     94                n_cells = int(self.header['last_id']) - int(self.header['first_id']) + 1 
     95                id_list = n_cells 
     96            else: 
     97                raise Exception("id_list not provided and cannot be inferred from file header.") 
     98            if dims is None and 'dimensions' in self.header: 
     99                dims = self.header['dimensions'] 
     100            if label is None: 
     101                label = self.data_io.filename 
     102            if isinstance(id_list, int): # allows to just specify the number of neurons 
     103                id_list = range(id_list) 
     104            from NeuroTools import signals 
     105            return signals.SpikeList(self.data_io.txt_load(), id_list, dt, t_start, t_stop, dims, label) 
    72106 
    73         if (id_list is None) and (('first_id' in self.header) and ('last_id' in self.header)): 
    74             logging.debug("id_list is infered from the file header") 
    75             n_cells = int(self.header['last_id']) - int(self.header['first_id']) + 1 
    76             id_list = n_cells 
    77         else: 
    78             raise Exception("id_list not provided and cannot be inferred from file header.") 
     107    def save_spikelist_pickle(self, spikelist, filename): 
     108        self.data_io.pickle_write(spikelist, filename) 
    79109     
    80         if dims is None and 'dimensions' in self.header: 
    81             dims = self.header['dimensions'] 
    82          
    83         if label is None: 
    84             label = self.loader.filename 
    85  
    86         if isinstance(id_list, int): # allows to just specify the number of neurons 
    87             id_list = range(id_list) 
    88          
    89         from NeuroTools import signals 
    90         return signals.SpikeList(self.loader.load(), id_list, dt, t_start, t_stop, dims, label) 
    91  
     110    def save_spikelist_txt(self, spikelist, filename): 
     111        self.write_header(spikelist, filename) 
     112        data = numpy.array(spikelist.convert(format="[times, ids]")) 
     113        data = data.reshape(data.shape[1],data.shape[0]) 
     114        self.data_io.txt_write(data, filename+".tmp") 
     115        os.system("cat %s >> %s" %(filename+".tmp", filename)) 
     116        os.system("rm %s" %(filename+".tmp")) 
    92117 
    93118#class HDF5DataLoader(object): 
    …… 
    146171 
    147172 
     173 
     174############################################## 
     175### WORK IN PROGRESS 
     176############################################## 
     177 
     178 
     179 
     180 
    148181def load_conductancetracelist(filename, id_list, dt = None, t_start=None, t_stop=None): 
    149182    if isinstance(id_list,int): # allows to just specify the number of neurons