Sometimes just writing things out brute force can shed some light. I think you have express $M$ and $L$ in terms of their factors. To be honest,I'm perfectly happy using FFTW. I haven't rolled my own DFTs for a very long time.
So I wrote a little Matlab script for the 12 point transform in terms of 4 point and 3 points
with input
intxt =
{'x[0]'} {'x[1]'} {'x[2]'} {'x[3]'} {'x[4]'} {'x[5]'} {'x[6]'} {'x[7]'} {'x[8]'} {'x[9]'} {'x[10]'} {'x[11]'}
where we form the 2D matrix
{'x[0]'} {'x[1]' } {'x[2]' }
{'x[3]'} {'x[4]' } {'x[5]' }
{'x[6]'} {'x[7]' } {'x[8]' }
{'x[9]'} {'x[10]'} {'x[11]'}
clear all
M=12;
in=[0:(M-1)];
intxt=num2cell(in);
intxt=cellfun(@(x) ['x[',int2str(x),']'],intxt,'UniformOutput',false)
zzz=DFT2D(intxt,3,4)
%%
function y=txtdft(x) % 1d dft
y=[];
n=length(x);
for k=1:n
yy=[];
for i=1:n
if i < n
yy=[yy strcat(['W_{',int2str(n),'}^{',...
int2str(i-1),',',int2str(k-1),'}'],strcat(x{i},'+'))];
else
yy=[yy strcat(['W_{',int2str(n),'}^{',int2str(i-1),',...
',int2str(k-1),'}'],strcat(x{i}))];
end
% yy=['(',yy,')'];
end
y=[y {yy}];
end
end
%%
function [y]=twiddle(x)
[m,n]=size(x);
y=cell(m,n);
D=m*n;
for i=1:m
for k=1:n
twid= ['W_{',int2str(D),'}^{', ...
int2str(i-1),',',int2str(k-1),'}'];
y(i,k)={[ twid, '(',x{i,k},')']};
end
end
end
%%
function [y]=DFT2D( x, mm,nn )
y=cell(size(x));
length(x)
assert(length(x) == mm*nn,'not proper');
x12d=reshape(x,mm,nn)' % mm = number colums nn=number rows
y_out=cell(size(x12d));
for i=1:mm
x11=x12d(:,i);
y_out(:,i)= txtdft(x11)';
end
zz=twiddle(y_out)';
y_out2=cell(size(zz));
for i=1:nn
x11=zz(:,i);
y_out2(:,i)= txtdft(x11)';
end
y=reshape(y_out2,length(x),1); % read out rowszzz
end
And Pretty Printing the terms:
$$
\begin{multline}
Y[0]=W_{3}^{0,0}W_{12}^{0,0}(W_{4}^{0,0}x[0]+W_{4}^{1,0}x[3]+W_{4}^{2,0}x[6]+W_{4}^{3,0}x[9])+W_{3}^{1,0}W_{12}^{0,1}(W_{4}^{0,0}x[1]+W_{4}^{1,0}x[4]+W_{4}^{2,0}x[7]+W_{4}^{3,0}x[10])+W_{3}^{2,0}W_{12}^{0,2}(W_{4}^{0,0}x[2]+W_{4}^{1,0}x[5]+W_{4}^{2,0}x[8]+W_{4}^{3,0}x[11]) \end{multline}
$$
$$
\begin{multline}
Y[1]=W_{3}^{0,1}W_{12}^{0,0}(W_{4}^{0,0}x[0]+W_{4}^{1,0}x[3]+W_{4}^{2,0}x[6]+W_{4}^{3,0}x[9])+W_{3}^{1,1}W_{12}^{0,1}(W_{4}^{0,0}x[1]+W_{4}^{1,0}x[4]+W_{4}^{2,0}x[7]+W_{4}^{3,0}x[10])+W_{3}^{2,1}W_{12}^{0,2}(W_{4}^{0,0}x[2]+W_{4}^{1,0}x[5]+W_{4}^{2,0}x[8]+W_{4}^{3,0}x[11])
\end{multline}
$$
$$
\begin{multline}
Y[2]=W_{3}^{0,2}W_{12}^{0,0}(W_{4}^{0,0}x[0]+W_{4}^{1,0}x[3]+W_{4}^{2,0}x[6]+W_{4}^{3,0}x[9])+W_{3}^{1,2}W_{12}^{0,1}(W_{4}^{0,0}x[1]+W_{4}^{1,0}x[4]+W_{4}^{2,0}x[7]+W_{4}^{3,0}x[10])+W_{3}^{2,2}W_{12}^{0,2}(W_{4}^{0,0}x[2]+W_{4}^{1,0}x[5]+W_{4}^{2,0}x[8]+W_{4}^{3,0}x[11])
\end{multline}
$$
$$
\begin{multline}
Y[3]= W_{3}^{0,0}W_{12}^{1,0}(W_{4}^{0,1}x[0]+W_{4}^{1,1}x[3]+W_{4}^{2,1}x[6]+W_{4}^{3,1}x[9])+W_{3}^{1,0}W_{12}^{1,1}(W_{4}^{0,1}x[1]+W_{4}^{1,1}x[4]+W_{4}^{2,1}x[7]+W_{4}^{3,1}x[10])+W_{3}^{2,0}W_{12}^{1,2}(W_{4}^{0,1}x[2]+W_{4}^{1,1}x[5]+W_{4}^{2,1}x[8]+W_{4}^{3,1}x[11]) \end{multline}
$$
$$
\begin{multline}
Y[4]=W_{3}^{0,1}W_{12}^{1,0}(W_{4}^{0,1}x[0]+W_{4}^{1,1}x[3]+W_{4}^{2,1}x[6]+W_{4}^{3,1}x[9])+W_{3}^{1,1}W_{12}^{1,1}(W_{4}^{0,1}x[1]+W_{4}^{1,1}x[4]+W_{4}^{2,1}x[7]+W_{4}^{3,1}x[10])+W_{3}^{2,1}W_{12}^{1,2}(W_{4}^{0,1}x[2]+W_{4}^{1,1}x[5]+W_{4}^{2,1}x[8]+W_{4}^{3,1}x[11])
\end{multline}
$$
$$
\begin{multline}
Y[5]=W_{3}^{0,2}W_{12}^{1,0}(W_{4}^{0,1}x[0]+W_{4}^{1,1}x[3]+W_{4}^{2,1}x[6]+W_{4}^{3,1}x[9])+W_{3}^{1,2}W_{12}^{1,1}(W_{4}^{0,1}x[1]+W_{4}^{1,1}x[4]+W_{4}^{2,1}x[7]+W_{4}^{3,1}x[10])+W_{3}^{2,2}W_{12}^{1,2}(W_{4}^{0,1}x[2]+W_{4}^{1,1}x[5]+W_{4}^{2,1}x[8]+W_{4}^{3,1}x[11])
\end{multline}
$$
$$
\begin{multline}
Y[6]= W_{3}^{0,0}W_{12}^{2,0}(W_{4}^{0,2}x[0]+W_{4}^{1,2}x[3]+W_{4}^{2,2}x[6]+W_{4}^{3,2}x[9])+W_{3}^{1,0}W_{12}^{2,1}(W_{4}^{0,2}x[1]+W_{4}^{1,2}x[4]+W_{4}^{2,2}x[7]+W_{4}^{3,2}x[10])+W_{3}^{2,0}W_{12}^{2,2}(W_{4}^{0,2}x[2]+W_{4}^{1,2}x[5]+W_{4}^{2,2}x[8]+W_{4}^{3,2}x[11])
\end{multline}
$$
$$
\begin{multline}
Y[8]= W_{3}^{0,1}W_{12}^{2,0}(W_{4}^{0,2}x[0]+W_{4}^{1,2}x[3]+W_{4}^{2,2}x[6]+W_{4}^{3,2}x[9])+W_{3}^{1,1}W_{12}^{2,1}(W_{4}^{0,2}x[1]+W_{4}^{1,2}x[4]+W_{4}^{2,2}x[7]+W_{4}^{3,2}x[10])+W_{3}^{2,1}W_{12}^{2,2}(W_{4}^{0,2}x[2]+W_{4}^{1,2}x[5]+W_{4}^{2,2}x[8]+W_{4}^{3,2}x[11])
\end{multline}
$$
$$
\begin{multline}
Y[8]=W_{3}^{0,2}W_{12}^{2,0}(W_{4}^{0,2}x[0]+W_{4}^{1,2}x[3]+W_{4}^{2,2}x[6]+W_{4}^{3,2}x[9])+W_{3}^{1,2}W_{12}^{2,1}(W_{4}^{0,2}x[1]+W_{4}^{1,2}x[4]+W_{4}^{2,2}x[7]+W_{4}^{3,2}x[10])+W_{3}^{2,2}W_{12}^{2,2}(W_{4}^{0,2}x[2]+W_{4}^{1,2}x[5]+W_{4}^{2,2}x[8]+W_{4}^{3,2}x[11])
\end{multline}
$$
$$
\begin{multline}
Y[9]=W_{3}^{0,0}W_{12}^{3,0}(W_{4}^{0,3}x[0]+W_{4}^{1,3}x[3]+W_{4}^{2,3}x[6]+W_{4}^{3,3}x[9])+W_{3}^{1,0}W_{12}^{3,1}(W_{4}^{0,3}x[1]+W_{4}^{1,3}x[4]+W_{4}^{2,3}x[7]+W_{4}^{3,3}x[10])+W_{3}^{2,0}W_{12}^{3,2}(W_{4}^{0,3}x[2]+W_{4}^{1,3}x[5]+W_{4}^{2,3}x[8]+W_{4}^{3,3}x[11])
\end{multline}
$$
$$
\begin{multline}
Y[10]=W_{3}^{0,1}W_{12}^{3,0}(W_{4}^{0,3}x[0]+W_{4}^{1,3}x[3]+W_{4}^{2,3}x[6]+W_{4}^{3,3}x[9])+W_{3}^{1,1}W_{12}^{3,1}(W_{4}^{0,3}x[1]+W_{4}^{1,3}x[4]+W_{4}^{2,3}x[7]+W_{4}^{3,3}x[10])+W_{3}^{2,1}W_{12}^{3,2}(W_{4}^{0,3}x[2]+W_{4}^{1,3}x[5]+W_{4}^{2,3}x[8]+W_{4}^{3,3}x[11])
\end{multline}
$$
$$
\begin{multline}
Y[11]= W_{3}^{0,2}W_{12}^{3,0}(W_{4}^{0,3}x[0]+W_{4}^{1,3}x[3]+W_{4}^{2,3}x[6]+W_{4}^{3,3}x[9])+W_{3}^{1,2}W_{12}^{3,1}(W_{4}^{0,3}x[1]+W_{4}^{1,3}x[4]+W_{4}^{2,3}x[7]+W_{4}^{3,3}x[10])+W_{3}^{2,2}W_{12}^{3,2}(W_{4}^{0,3}x[2]+W_{4}^{1,3}x[5]+W_{4}^{2,3}x[8]+W_{4}^{3,3}x[11])\end{multline}
$$
For 60 points the 2 d matrix is
{'x[0]' } {'x[1]' } {'x[2]' } {'x[3]' } {'x[4]' }
{'x[5]' } {'x[6]' } {'x[7]' } {'x[8]' } {'x[9]' }
{'x[10]'} {'x[11]'} {'x[12]'} {'x[13]'} {'x[14]'}
{'x[15]'} {'x[16]'} {'x[17]'} {'x[18]'} {'x[19]'}
{'x[20]'} {'x[21]'} {'x[22]'} {'x[23]'} {'x[24]'}
{'x[25]'} {'x[26]'} {'x[27]'} {'x[28]'} {'x[29]'}
{'x[30]'} {'x[31]'} {'x[32]'} {'x[33]'} {'x[34]'}
{'x[35]'} {'x[36]'} {'x[37]'} {'x[38]'} {'x[39]'}
{'x[40]'} {'x[41]'} {'x[42]'} {'x[43]'} {'x[44]'}
{'x[45]'} {'x[46]'} {'x[47]'} {'x[48]'} {'x[49]'}
{'x[50]'} {'x[51]'} {'x[52]'} {'x[53]'} {'x[54]'}
{'x[55]'} {'x[56]'} {'x[57]'} {'x[58]'} {'x[59]'}
and we shove the first column into the script. the first column will look like:
$$
\begin{multline}
Y[0,0]=W_{3}^{0,0}W_{12}^{0,0}(W_{4}^{0,0}x[0]+W_{4}^{1,0}x[15]+W_{4}^{2,0}x[30]+W_{4}^{3,0}x[45])+W_{3}^{1,0}W_{12}^{0,1}(W_{4}^{0,0}x[5]+W_{4}^{1,0}x[20]+W_{4}^{2,0}x[35]+W_{4}^{3,0}x[50])+W_{3}^{2,0}W_{12}^{0,2}(W_{4}^{0,0}x[10]+W_{4}^{1,0}x[25]+W_{4}^{2,0}x[40]+W_{4}^{3,0}x[55])
\end{multline}
$$
I'm not going to go through the tedium of going through each column DFT,
doing the twiddle one the 12 by 5 matrix, and 12 row 5 point DFTs,
but your welcome to modify it yourself