Skip to content

Commit

Permalink
switched to append_common
Browse files Browse the repository at this point in the history
  • Loading branch information
@robertoostenveld
robertoostenveld committed Apr 24, 2017
1 parent 42131f1 commit e9cd4d8
Show file tree
Hide file tree
Showing 4 changed files with 335 additions and 1,211 deletions.
290 changes: 15 additions & 275 deletions ft_appenddata.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,289 +83,29 @@

% set the defaults
cfg.appendsens = ft_getopt(cfg, 'appendsens', 'no');

% ensure consistent input data
for i=2:length(varargin)
if isfield(varargin{1}, 'topo')
assert(isequaln(varargin{1}.topo, varargin{i}.topo), 'the input has inconsistent topo fields')
end
if isfield(varargin{1}, 'topolabel')
assert(isequaln(varargin{1}.topolabel, varargin{i}.topolabel), 'the input has inconsistent topolabel fields')
end
if isfield(varargin{1}, 'unmixing')
assert(isequaln(varargin{1}.unmixing, varargin{i}.unmixing), 'the input has inconsistent unmixing fields')
end
end

Nchan = zeros(size(varargin));
Ntrial = zeros(size(varargin));
label = {};
for i=1:length(varargin)
Nchan(i) = length(varargin{i}.label);
Ntrial(i) = length(varargin{i}.trial);
fprintf('input dataset %d, %d channels, %d trials\n', i, Nchan(i), Ntrial(i));
label = cat(1, label(:), varargin{i}.label(:));
end

% try to locate the trial definition (trl) in the nested configuration and
% check whether the input data contains trialinfo
% this is DEPRECATED - don't look in cfg-tree for stuff anymore
% hastrialinfo = 0;
% trl = cell(1, length(varargin));
% for i=1:length(varargin)
% if isfield(varargin{i}, 'cfg')
% trl{i} = ft_findcfg(varargin{i}.cfg, 'trl');
% else
% trl{i} = [];
% end
% if isempty(trl{i})
% % a trial definition is expected in each continuous data set
% warning('could not locate the trial definition ''trl'' in data structure %d', i);
% end
% hastrialinfo = isfield(varargin{i}, 'trialinfo') + hastrialinfo;
% end
% hastrialinfo = hastrialinfo==length(varargin);

hastrialinfo = 0;
hassampleinfo = 0;
sampleinfo = cell(size(varargin));
for i=1:length(varargin)
if isfield(varargin{i}, 'sampleinfo')
sampleinfo{i} = varargin{i}.sampleinfo;
else
sampleinfo{i} = [];
end

% the function should behave properly even if no sampleinfo is present,
% hence the warning seems inappropriate (ES, 24-apr-2014)
% if isempty(sampleinfo{i})
% % a sample definition is expected in each data set
% warning('no ''sampleinfo'' field in data structure %d', i);
% end

hassampleinfo = isfield(varargin{i}, 'sampleinfo') + hassampleinfo;
hastrialinfo = isfield(varargin{i}, 'trialinfo') + hastrialinfo;
end
hassampleinfo = hassampleinfo==length(varargin);
hastrialinfo = hastrialinfo==length(varargin);

% check the consistency of the labels across the input-structures
alllabel = unique(label, 'first');
order = zeros(length(alllabel),length(varargin));
for j=1:length(varargin)
tmplabel = varargin{j}.label;
[ix,iy] = match_str(alllabel, tmplabel);
order(ix,j) = iy;
end

% check whether the data are obtained from the same datafile in case either
% (1) we have sampleinfos and they are not identical or (2) we don't have
% sampleinfos
removesampleinfo = 0;
removetrialinfo = 0;
try
origfile1 = ft_findcfg(varargin{1}.cfg, 'datafile');
for j=2:length(varargin)
hassampleinfos = isfield(varargin{1}, 'sampleinfo') &&...
isfield(varargin{j}, 'sampleinfo');

if ((hassampleinfos &&...
~isequal(varargin{1}.sampleinfo, varargin{j}.sampleinfo)) ||...
~hassampleinfos) &&...
~isempty(origfile1) && ~strcmp(origfile1, ft_findcfg(varargin{j}.cfg, 'datafile'))
removesampleinfo = 1;
warning('input data comes from different datafiles; removing sampleinfo field');
break;
end
end
catch err
if strcmp(err.identifier, 'MATLAB:nonExistentField')
% this means no data.cfg is present; should not be treated as a fatal error
fprintf('cannot determine from which datafiles the data is taken\n');
else
% not sure which error, probably a bigger problem
throw(err);
end
end

catlabel = all(sum(order~=0,2)==1);
cattrial = any(sum(order~=0,2)==length(varargin));
shuflabel = cattrial && ~all(all(order-repmat(order(:,1),[1 length(varargin)])==0));
prunelabel = cattrial && sum(sum(order~=0,2)==length(varargin))<length(alllabel);

if shuflabel
fprintf('the channel order in the input-structures is not consistent, reordering\n');
if prunelabel
fprintf('not all input-structures contain the same channels, pruning the input prior to concatenating over trials\n');
selall = find(sum(order~=0,2)==length(varargin));
alllabel = alllabel(selall);
order = order(selall,:);
end
for i=1:length(varargin)
varargin{i}.label = varargin{i}.label(order(:,i));
for j=1:length(varargin{i}.trial)
varargin{i}.trial{j} = varargin{i}.trial{j}(order(:,i),:);
end
end
end

if cattrial && catlabel
error('cannot determine how the data should be concatenated');

elseif cattrial
fprintf('concatenating the trials over all datasets\n');

data = [];
data.label = varargin{1}.label;
data.trial = {};
data.time = {};
if hassampleinfo, data.sampleinfo = []; end
if hastrialinfo, data.trialinfo = []; end;

for i=1:length(varargin)
data.trial = cat(2, data.trial, varargin{i}.trial(:)');
data.time = cat(2, data.time, varargin{i}.time(:)');
% check if all datasets to merge have the sampleinfo field
if hassampleinfo, data.sampleinfo = cat(1, data.sampleinfo, varargin{i}.sampleinfo); end
if hastrialinfo, data.trialinfo = cat(1, data.trialinfo, varargin{i}.trialinfo); end
% FIXME is not entirely robust if the different inputs have different number of columns in trialinfo
end

elseif catlabel
fprintf('concatenating the channels within each trial\n');

if ~all(diff(Ntrial)==0)
error('not all datasets have the same number of trials');
else
Ntrial = Ntrial(1);
end

data = [];
data.label = varargin{1}.label;
data.trial = varargin{1}.trial;
data.time = varargin{1}.time;
if hassampleinfo, data.sampleinfo=varargin{i}.sampleinfo; end
if hastrialinfo, data.trialinfo =varargin{i}.trialinfo; end

for i=2:length(varargin)
% concatenate the labels
data.label = cat(1, data.label(:), varargin{i}.label(:));

% check whether the trialinfo and sampleinfo fields are consistent
if hassampleinfo && ~isequaln(data.sampleinfo, varargin{i}.sampleinfo)
removesampleinfo = 1;
end
if hastrialinfo && ~isequaln(data.trialinfo, varargin{i}.trialinfo)
removetrialinfo = 1;
end
end

if ~isfield(data, 'fsample')
fsample = 1/mean(diff(data.time{1}));
cfg.appenddim = ft_getopt(cfg, 'appenddim', []);
cfg.parameter = {'trial'}; % this is hard-coded, it is used for consistency with ft_appendtimelock and ft_appendfreq

if isempty(cfg.appenddim) || strcmp(cfg.appenddim, 'auto')
if checkchan(varargin{:}, 'identical') && checktime(varargin{:}, 'identical', cfg.tolerance)
cfg.appenddim = 'rpt';
elseif checkchan(varargin{:}, 'unique')
cfg.appenddim = 'chan';
% elseif checktime(varargin{:}, 'unique', cfg.tolerance)
% cfg.appenddim = 'time';
else
fsample = data.fsample;
end

for j=1:Ntrial
%pre-allocate memory for this trial
data.trial{j} = [data.trial{j}; zeros(sum(Nchan(2:end)), size(data.trial{j},2))];

%fill this trial with data
endchan = Nchan(1);
%allow some jitter for irregular sample frequencies
tolerance = 0.01*(1/fsample);
for i=2:length(varargin)
if ~all(data.time{j}-varargin{i}.time{j}<tolerance)
error('there is a difference in the time axes of the input data');
end
begchan = endchan+1;
endchan = endchan+Nchan(i);
data.trial{j}(begchan:endchan,:) = varargin{i}.trial{j};
end
error('cfg.appenddim should be specified');
end

else
% labels are inconsistent, cannot determine how to concatenate the data
error('cannot determine how the data should be concatenated');
end
fprintf('concatenating over the "%s" dimension\n', cfg.appenddim);

% copy some fields from the input over to the output
data = copyfields(varargin{1}, data, {'topo', 'topolabel', 'unmixing', 'fsample'});

% unshuffle the channels again to match the order of the first input data-structure
if shuflabel
fprintf('reordering the channels back to the original input order\n');
[dum, reorder] = sort(order(order(:,1)~=0,1));
for i=1:length(data.trial)
data.trial{i} = data.trial{i}(reorder,:);
end
data.label = data.label(reorder);
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% the following section is shared with ft_appenddata, ft_appendtimelock and ft_appendfreq
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
hasgrad = false;
haselec = false;
hasopto = false;
for i=1:length(varargin)
hasgrad = hasgrad || isfield(varargin{i}, 'grad');
haselec = haselec || isfield(varargin{i}, 'elec');
hasopto = hasopto || isfield(varargin{i}, 'opto');
end
if hasgrad || haselec || hasopto
% collect the sensor definitions from all inputs
grad = cell(size(varargin));
elec = cell(size(varargin));
opto = cell(size(varargin));
for j=1:length(varargin)
if isfield(varargin{j}, 'elec')
elec{j} = varargin{j}.elec;
end
if isfield(varargin{j}, 'grad')
grad{j} = varargin{j}.grad;
end
if isfield(varargin{j}, 'opto')
opto{j} = varargin{j}.opto;
end
end
% see test_pull393.m for a description of the expected behavior
if strcmp(cfg.appendsens, 'yes')
fprintf('concatenating sensor information across input arguments\n');
% append the sensor descriptions, skip the empty ones
if hasgrad, data.grad = ft_appendsens([], grad{~cellfun(@isempty, grad)}); end
if haselec, data.elec = ft_appendsens([], elec{~cellfun(@isempty, elec)}); end
if hasopto, data.opto = ft_appendsens([], opto{~cellfun(@isempty, opto)}); end
else
% discard sensor information when it is inconsistent across the input arguments
removegrad = any(cellfun(@isempty, grad));
removeelec = any(cellfun(@isempty, elec));
removeopto = any(cellfun(@isempty, opto));
for j=2:length(varargin)
removegrad = removegrad || ~isequaln(grad{j}, grad{1});
removeelec = removeelec || ~isequaln(elec{j}, elec{1});
removeopto = removeopto || ~isequaln(opto{j}, opto{1});
end
if hasgrad && ~removegrad, data.grad = grad{1}; end
if haselec && ~removeelec, data.elec = elec{1}; end
if hasopto && ~removeopto, data.opto = opto{1}; end
end
end

if removesampleinfo
fprintf('removing sampleinfo field from output\n');
if isfield(data, 'sampleinfo'), data = rmfield(data, 'sampleinfo'); end
end

if removetrialinfo
fprintf('removing trialinfo field from output\n');
if isfield(data, 'trialinfo'), data = rmfield(data, 'trialinfo'); end
end
% use a low-level function that is shared with the other ft_appendxxx functions
data = append_common(cfg, varargin{:});

This comment has been minimized.

Sign in to view

Copy link
@jens-k

jens-k Jul 17, 2017

Contributor

Calling append_common here leads to an error when combining normal and ica component data (s. below). This may happen, e.g. when you want to calculate coherence between components and ECG or eye channels.

Therefore, this example fails:
http:https://www.fieldtriptoolbox.org/example/use_independent_component_analysis_ica_to_remove_ecg_artifacts

Is there an easy workaround?

Error using ft_notification (line 314)
input data should be of the same datatype

Error in ft_error (line 39)
ft_notification(varargin{:});

Error in ft_selectdata (line 103)
if ~ok, ft_error('input data should be of the same datatype'); end

Error in append_common (line 94)
[varargin{:}] = ft_selectdata(tmpcfg, varargin{:});

Error in ft_appenddata (line 123)
data = append_common(cfg, dummy{:});

This comment has been minimized.

Sign in to view

Copy link
@schoffelen

schoffelen Aug 11, 2017

Contributor

Confirmed. Sorry it took so long to look into this. I think I provided a fix in ft_appenddata, which removes any component-structure specific info (topo/unmixing/topolabel). This results in the data not to be recognized as 'comp' anymore, avoiding the error in append_common. I think that this change is no problem, because append_common throws out the topo/unmixing/topolabel anyway a.t.m.

This comment has been minimized.

Sign in to view

Copy link
@schoffelen

schoffelen Aug 11, 2017

Contributor

Will push the fix in a minute.

This comment has been minimized.

Sign in to view

Copy link
@jens-k

jens-k Aug 15, 2017

Contributor

Great, thanks!

% do the general cleanup and bookkeeping at the end of the function
ft_postamble debug
ft_postamble trackconfig
ft_postamble provenance
ft_postamble previous varargin
ft_postamble provenance data
ft_postamble history data
ft_postamble savevar data
Loading

0 comments on commit e9cd4d8

Please sign in to comment.