bml_load_continuous

 BML_LOAD_CONTINUOUS loads continuous raw from one or more files

 Use as 
   raw = bml_load_continuous(cfg)
   raw = bml_load_continuous(cfg.roi)
   [raw, file_raw_map] = bml_load_continuous(cfg)
   [raw, file_raw_map] = bml_load_continuous(cfg.roi)

 cfg is configuratin struct
   cfg.roi - ROI table with regions of interest to load
   cfg.channel - cellstr with channels to be selected
   cfg.electrode - ANNOT table with electrodes information. Should contain
                   variables 'channel' and 'electrode'. Optional.
   cfg.folder - overwrites info in cfg.roi
   cfg.chantype - string, overwrites info in cfg.roi
   cfg.filetype - overwrites info in cfg.roi
   cfg.Fs - overwrites info in cfg.roi
   cfg.timetol - double: time tolerance in seconds per sample. Defaults to 1e-6
   cfg.dryrun - logical: should a dry-run test be performed? Defaults to false
   cfg.ft_feedback - string: default to 'no'. Defines verbosity of fieldtrip
           functions 
   cfg.discontinuous - string or logical: 
           * true or 'allow' to allow discontinous files to be loaded filling 
           the gap with zero-padding, if possible within timetol.
           * false or 'no' to issue an error if discontinous files are found
           * 'warn' to allow with a warning
   cfg.match_labels - logical, indicates if labels of all files should be
           the same. Defaulst to true. If false a warning will be issued
           when concatenating files with different labels. 
   cfg.padval - value with which to pad if discontinuous files are loaded.
           Defualts to zero

 returns a continuous FT_DATATYPE_RAW and optionally a file-raw mapping

 -------------------------------------------------------------------------

 This function loads a fieldtrip data structure representing a raw continuous 
 in the form of a channels by times matrix (see FT_DATATYPE_RAW). The main
 argument is cfg.roi, ROI table with the synchronization file coordinates.

0001 function [raw, file_raw_map] = bml_load_continuous(cfg)
0002 
0003 % BML_LOAD_CONTINUOUS loads continuous raw from one or more files
0004 %
0005 % Use as
0006 %   raw = bml_load_continuous(cfg)
0007 %   raw = bml_load_continuous(cfg.roi)
0008 %   [raw, file_raw_map] = bml_load_continuous(cfg)
0009 %   [raw, file_raw_map] = bml_load_continuous(cfg.roi)
0010 %
0011 % cfg is configuratin struct
0012 %   cfg.roi - ROI table with regions of interest to load
0013 %   cfg.channel - cellstr with channels to be selected
0014 %   cfg.electrode - ANNOT table with electrodes information. Should contain
0015 %                   variables 'channel' and 'electrode'. Optional.
0016 %   cfg.folder - overwrites info in cfg.roi
0017 %   cfg.chantype - string, overwrites info in cfg.roi
0018 %   cfg.filetype - overwrites info in cfg.roi
0019 %   cfg.Fs - overwrites info in cfg.roi
0020 %   cfg.timetol - double: time tolerance in seconds per sample. Defaults to 1e-6
0021 %   cfg.dryrun - logical: should a dry-run test be performed? Defaults to false
0022 %   cfg.ft_feedback - string: default to 'no'. Defines verbosity of fieldtrip
0023 %           functions
0024 %   cfg.discontinuous - string or logical:
0025 %           * true or 'allow' to allow discontinous files to be loaded filling
0026 %           the gap with zero-padding, if possible within timetol.
0027 %           * false or 'no' to issue an error if discontinous files are found
0028 %           * 'warn' to allow with a warning
0029 %   cfg.match_labels - logical, indicates if labels of all files should be
0030 %           the same. Defaulst to true. If false a warning will be issued
0031 %           when concatenating files with different labels.
0032 %   cfg.padval - value with which to pad if discontinuous files are loaded.
0033 %           Defualts to zero
0034 %
0035 % returns a continuous FT_DATATYPE_RAW and optionally a file-raw mapping
0036 %
0037 % -------------------------------------------------------------------------
0038 %
0039 % This function loads a fieldtrip data structure representing a raw continuous
0040 % in the form of a channels by times matrix (see FT_DATATYPE_RAW). The main
0041 % argument is cfg.roi, ROI table with the synchronization file coordinates.
0042 %
0043 
0044 %% READING ARGUMENTS
0045 file_raw_map_vars = {'starts','ends','s1','t1','s2','t2','folder','name','nSamples','Fs'};
0046 
0047 if istable(cfg)
0048   cfg = struct('roi',cfg);
0049 end
0050 
0051 folder        = bml_getopt(cfg,'folder');
0052 channel       = bml_getopt(cfg,'channel');
0053 chantype      = bml_getopt(cfg,'chantype');
0054 filetype      = bml_getopt(cfg,'filetype');
0055 Fs            = bml_getopt(cfg,'Fs');
0056 roi           = bml_annot_table(bml_getopt(cfg,'roi'),'roi');
0057 timetol       = bml_getopt(cfg,'timetol',1e-5);
0058 dryrun        = bml_getopt(cfg,'dryrun',false);
0059 ft_feedback   = bml_getopt_single(cfg,'ft_feedback','no');
0060 discontinuous = bml_getopt(cfg,'discontinuous','warn');
0061 padval        = bml_getopt(cfg,'padval',0);
0062 electrode     = bml_annot_table(bml_getopt(cfg,'electrode'),'electrode');
0063 match_labels  = bml_getopt(cfg,'match_labels',true);
0064 
0065 if isempty(roi)
0066   raw=[];
0067   return  
0068 end
0069 roi = bml_roi_table(roi);
0070 
0071 if islogical(discontinuous)
0072   if discontinuous
0073     discontinuous = {'allow'};
0074   else
0075     discontinuous = {'no'};
0076   end
0077 end
0078 
0079 %consolidating chunks of same file when possible
0080 roi = bml_sync_consolidate(roi);
0081 
0082 %removing zero length rois
0083 roi = roi(roi.duration > 0,:);
0084 
0085 if ~isempty(folder)
0086   %ToDo: combine cfg.folder with roi.folder in a smart way
0087   roi.folder = repmat({folder},height(roi),1);
0088 end
0089 
0090 %using roi parameters if none specified in call
0091 if isempty(chantype) && ismember('chantype',roi.Properties.VariableNames)
0092   chantype  = cellstr(unique(roi.chantype));
0093 end
0094 if isempty(filetype) && ismember('filetype',roi.Properties.VariableNames)
0095   filetype  = cellstr(unique(roi.filetype));
0096 end
0097 if isempty(Fs) && ismember('Fs',roi.Properties.VariableNames)
0098   Fs        = unique(roi.Fs);
0099 end
0100 assert(length(filetype)==1,'unique filetype required: %s',strjoin(filetype));
0101 assert(length(chantype)==1,'unique chantype required: %s',strjoin(chantype));
0102 assert(length(Fs)==1,'unique Fs required: %s',strjoin(string(num2str(Fs))));
0103 
0104 %dealing with skip factors
0105 skipFactor=1;
0106 chantype_split=strsplit(chantype{1},':');
0107 if numel(chantype_split) == 2
0108   skipFactor=str2double(chantype_split{2});
0109 elseif numel(chantype_split) > 2
0110   ft_error('Use '':'' to specify skipfactor, e.g. analog:10')
0111 end
0112 
0113 %selecting electrodes
0114 if ~isempty(electrode)
0115   assert(ismember('channel',electrode.Properties.VariableNames),"'channel' variable required in cfg.electrode");
0116   assert(ismember('electrode',electrode.Properties.VariableNames),"'electrode' variable required in cfg.electrode");
0117   %checking electrodes for time, filetype and chantype
0118   cfg1=[];
0119   cfg1.keep='x';
0120   cfg1.warn=false;
0121   electrode = bml_annot_intersect(cfg1,electrode,...
0122                   bml_annot_table(table(min(roi.starts),max(roi.ends))));
0123   assert(height(electrode)>0,"no electrode for roi time");
0124   if ismember('filetype',electrode.Properties.VariableNames)
0125     electrode = electrode(strcmp(electrode.filetype,filetype),:);
0126     assert(height(electrode)>0,"incorrect filetype for cfg.electrode");
0127   end
0128   if ismember('chantype',electrode.Properties.VariableNames)
0129     electrode = electrode(...  
0130       contains(electrode.chantype,{chantype{1}},'IgnoreCase',true)|...
0131       strcmp(electrode.chantype,'all')|...
0132       strcmp(electrode.chantype,'any')|...
0133       strcmp(electrode.chantype,'NA')|...
0134       strcmp(electrode.chantype,''),:);
0135     assert(height(electrode)>0,"incorrect chantype for cfg.electrode");
0136   else
0137     fprintf("electrode table has no chantype variable.\n")
0138   end
0139   
0140   assert(numel(electrode.channel)==numel(unique(electrode.channel)),...
0141       "repeated electrode entries for time/filetype/chantype");
0142 
0143   if isempty(channel)
0144       channel = electrode.channel;
0145   end
0146 end
0147 
0148 %optimizing channel selection for trellis files
0149 if ~isempty(channel) && contains(filetype,"trellis")
0150   chantype = {['^(',strjoin(channel,'|'),')$']};
0151   if skipFactor > 1
0152     chantype{1} = [chantype{1},':',num2str(skipFactor)];
0153   end
0154   channel=[];
0155 end
0156 
0157 %% LOAD FIRST FILE
0158 
0159 %saving mapping between raw and files
0160 file_raw_map=roi(1,file_raw_map_vars);
0161 [s,e]=bml_crop_idx_valid(roi(1,:));
0162 file_raw_map.s1=s;
0163 file_raw_map.s2=e;
0164 file_raw_map.t1=bml_idx2time(roi(1,:),s);
0165 file_raw_map.t2=bml_idx2time(roi(1,:),e);
0166 file_raw_map.raw1=1;
0167 file_raw_map.raw2=floor((e-s+1)/skipFactor);
0168 file_raw_map.skipFactor = skipFactor;
0169 
0170 %loading first raw
0171 cfg=[]; 
0172 cfg.chantype=chantype;
0173 cfg.trl = [ceil(s/skipFactor), floor(e/skipFactor), 0];
0174 cfg.dataset=fullfile(roi.folder{1},roi.name{1});
0175 cfg.feedback=ft_feedback;
0176 hdr = ft_read_header(cfg.dataset,'chantype',cfg.chantype);
0177 
0178 %checking nomimal sampling frequencies
0179 assert(hdr.Fs*skipFactor==roi.Fs(1),...
0180   'File %s chantype %s has Fs %f, not %f as defined in cfg.roi',...
0181   roi.name{1},strjoin(cfg.chantype),hdr.Fs,roi.Fs(1));
0182 
0183 if ~dryrun
0184   raw = ft_preprocessing(cfg);
0185 else
0186   raw = hdr;
0187   assert(raw.nSamples>=floor(e/skipFactor),'index overflow s2=%i but nSample=%i',floor(e/skipFactor),raw.nSamples);
0188 end
0189 
0190 %selecting channels
0191 if ~isempty(channel)
0192   channel_selected=ft_channelselection(channel,raw.label);
0193   if numel(channel_selected)==0
0194     error('%s not present in raw %s \nAvailable channels are: %s',strjoin(channel),cfg.dataset,strjoin(raw.label));
0195   elseif ~dryrun
0196     cfg=[]; cfg.channel=channel; cfg.feedback=ft_feedback;
0197     raw = ft_selectdata(cfg,raw);
0198   end
0199 end
0200 
0201 % time = raw.time{1}+bml_idx2time(roi(1,:),s);
0202 time = bml_idx2time(roi(1,:),ceil(s/skipFactor):floor(e/skipFactor),skipFactor);
0203 raw.time{1} = time;
0204 if abs(time(end)-bml_idx2time(roi(1,:),floor(e/skipFactor),skipFactor)) > timetol
0205   error('timetol violated')
0206 end
0207 
0208 %% LOAD OTHER FILES (if present)
0209 for i=2:height(roi)
0210   
0211   %saving mapping between raw and files
0212   row=roi(i,file_raw_map_vars);
0213   [s,e]=bml_crop_idx_valid(roi(i,:));
0214   row.s1=s;
0215   row.s2=e;
0216   row.t1=bml_idx2time(roi(i,:),s);
0217   row.t2=bml_idx2time(roi(i,:),e);
0218   row.raw1=max(file_raw_map.raw2)+1;
0219   row.raw2=floor((e-s)/skipFactor)+row.raw1;
0220   row.skipFactor = skipFactor;
0221   
0222   %loading next raw
0223   cfg=[]; 
0224   cfg.chantype=chantype;
0225   cfg.trl = [ceil(s/skipFactor), floor(e/skipFactor), 0];
0226   cfg.dataset=fullfile(roi.folder{i},roi.name{i});
0227   cfg.feedback=ft_feedback;
0228   if ~dryrun
0229     next_raw = ft_preprocessing(cfg);
0230   else
0231     next_raw = ft_read_header(cfg.dataset,'chantype',cfg.chantype);
0232     assert(next_raw.nSamples>=floor(e/skipFactor),'index overflow s2=%i but nSample=%i',floor(e/skipFactor),next_raw.nSamples);
0233   end
0234   
0235   if ~isempty(channel)
0236     if isstring(channel); channel = {char(channel)}; end
0237     if ~dryrun
0238       cfg=[]; cfg.channel=channel; cfg.feedback=ft_feedback;
0239       next_raw = ft_selectdata(cfg,next_raw);
0240     end
0241   end
0242     
0243   %next_time = next_raw.time{1}+bml_idx2time(roi(i,:),s);
0244   next_time = bml_idx2time(roi(i,:),ceil(s/skipFactor):floor(e/skipFactor),skipFactor);
0245   next_raw.time{1} = next_time;
0246   assert(abs(next_time(end)-bml_idx2time(roi(i,:),floor(e/skipFactor),skipFactor)) < timetol, 'timetol violated');
0247  
0248   %verifying contiguity
0249   delta_t = next_time(1) - time(end) - skipFactor/Fs;
0250   if abs(delta_t) > timetol %non contiguous files
0251     if ~ismember(discontinuous,{'allow','warn'})
0252       roi
0253       error("To concateneting discontinuous rois use cfg.discontinuous='allow'");
0254     end  
0255     
0256     delta_s = delta_t*Fs/skipFactor;
0257     delta_s_int = round(delta_s);
0258     assert(delta_s>0,"rois overlap by %f > tolerance = %f correct or increase tolerance",delta_t,timetol);
0259 
0260     if abs(delta_s_int - delta_s) < timetol*Fs/skipFactor
0261       if ismember(discontinuous,{'warn'})
0262         warning("concatenating discontinous files %i samples added",delta_s_int);
0263       end
0264       if ~dryrun             
0265         %zero padding the end of the previous raw
0266         starts = raw.time{1}(1) - 0.5*skipFactor/Fs;
0267         ends = next_time(1) + 0.5*skipFactor/Fs;
0268         [raw, ~, post] = bml_pad(raw,starts,ends,padval);
0269         
0270         %correcting raw mapping due to zero padding
0271         %the starts of next file in raw coordinates is delayed by post
0272         row.raw1=max(file_raw_map.raw2)+1+post;
0273         row.raw2=floor((e-s)/skipFactor)+row.raw1;
0274       else
0275         time = [time,time(end)+(1:delta_s_int)*skipFactor/Fs];
0276       end
0277     else
0278         roi
0279       error("can't concatenate discontinuous files with sample snap error of %f seconds within timetol of %f seconds",...
0280         abs(delta_s_int - delta_s)*skipFactor/Fs, timetol);
0281     end 
0282   end
0283     
0284   if ~dryrun
0285     cfg1=[]; 
0286     cfg1.timetol = timetol; 
0287     cfg1.timeref = 'common';
0288     cfg1.match_labels = match_labels;
0289     raw = bml_hstack(cfg1, raw, next_raw);
0290     time = raw.time{1};
0291   else
0292     time = [time next_time];
0293   end
0294   
0295   file_raw_map = [file_raw_map;row];
0296 end
0297 
0298 if dryrun
0299   raw = [];
0300 elseif ~isempty(electrode)
0301   %changing labels from channels to electrodes
0302     raw.label = bml_map(raw.label,electrode.channel,electrode.electrode);
0303 end
0304 
0305 file_raw_map = bml_roi_table(file_raw_map);
0306 
0307 
0308 
0309