E = logical(eye(size(table1))); % create a mask for the two different rules
% rule 1: diagonal elements first
table2 = zeros(size(table1)); % create result matrix
[~,jj] = sort(table1(E));
[~,ii] = sort(jj);
table2(E) = ii; % assign rank of diagonal elements
% rule 2: rest of the matrix
E = ~E;
B = reshape(table1(E),size(table1,1)-1,size(table1,2))'; % B is the matrix of table1 without diagonal elements
[~,jj] = sort(B,2); % sort along column dimension,
[~,ii] = sort(jj,2);
table2 = table2'; % matlab is column-major, so you have to transpose the dest matrix before putting in the elements
table2(E) = reshape(ii',[],1);
table2 = table2'; % transpose back, done.
% treat zeros apart: 0 has rank 0
table2(table1==0) = 0;