% this script is for calculating late-miss cross-validation skill(f=0.1)

%##########################################################################################    
%##########################################################################################    
%###########################       NH mean         ########################################    
%##########################################################################################    
%##########################################################################################    

clear all; close all;

load c:\scozztemann3\newtemp\nhhinfxxxhad % for NH CRU/HAD series
load c:\scozztemann3\newtemp\nhlinfxxxhad
act1=nhhinfxxxhad+nhlinfxxxhad;
act1l=nhlinfxxxhad;
act1h=nhhinfxxxhad;

%%------------- smooth instrumental data series---------------------
stdkeepers=act1;
    cutoff=10; %years
	lowlim=1./(cutoff./2)
%%%%%%%%%%%%%%
[nrows,ncols]=size(stdkeepers) 
gotstart=zeros(1,ncols);
gotstop=zeros(1,ncols);
start=zeros(1,ncols);
stop=zeros(1,ncols);
stop(:)=nrows;
[lowa,lowb]=butter(5,lowlim,'low');

%%%%%%%%%%%%%%  
%%%% This section finds the starting and stopping year for each column. It assumes that
%%%%  there are no missing values between start and stop, otherwise filtering chokes.

for i=1:nrows
	for j=1:ncols
		if ~isnan(stdkeepers(i,j)) & gotstart(j)~=1
			start(j)=i;
			gotstart(j)=1;
		elseif isnan(stdkeepers(i,j)) & gotstart(j)==1 & gotstop(j)~=1
			stop(j)=i-1;
			gotstop(j)=1;	
		end
	end
end
%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%
%%%%	Apply the zero-phase butterworth filter but only to the proxies
	highf=zeros(nrows,ncols);
	lowf=zeros(nrows,ncols); 
	lowf=lowf./lowf;
	highf=highf./highf;
	for k=1:ncols
		clear y y2
		data=stdkeepers(start(k):stop(k),k);
%		lowf(start(k):stop(k),k)=lowpass(data,0.05,2,2);
        frequency = 0.1;
        [smoothed0,icb,ice,mse0] = lowpassmin(data,frequency);
        lowf(start(k):stop(k),k)= smoothed0;
%		lowf(start(k):stop(k),k)=filtfilt(lowa,lowb,data);
		highf(start(k):stop(k),k)=data-lowf(start(k):stop(k),k);
	end
    %%
  act1=  lowf;
%%-----------------------------------------------------------------
% do loop calculation
veridat=[];

for istep=1:19
%istep=1

in1=101; in2=146; % removed rows of global mean temp.

t1=load(strcat('c:\scozztemann3\z1308\veri\nhrech\recon',num2str(istep)));
t2=load(strcat('c:\scozztemann3\z1308\veri\nhrecl\recon',num2str(istep)));
    [nrows,ncols]=size(t1)
    
inf1=t1(nrows-145:nrows,1)+t2(nrows-145:nrows,1); % infilled (NaNs are estimated) global mean temp.
inf1h=t1(nrows-145:nrows,1);
inf1l=t2(nrows-145:nrows,1);

%%------------- smooth instrumental data series---------------------
stdkeepers=inf1;
    cutoff=10; %years
	lowlim=1./(cutoff./2)
%%%%%%%%%%%%%%
[nrows,ncols]=size(stdkeepers) 
gotstart=zeros(1,ncols);
gotstop=zeros(1,ncols);
start=zeros(1,ncols);
stop=zeros(1,ncols);
stop(:)=nrows;
[lowa,lowb]=butter(5,lowlim,'low');

%%%%%%%%%%%%%%  
%%%% This section finds the starting and stopping year for each column. It assumes that
%%%%  there are no missing values between start and stop, otherwise filtering chokes.

for i=1:nrows
	for j=1:ncols
		if ~isnan(stdkeepers(i,j)) & gotstart(j)~=1
			start(j)=i;
			gotstart(j)=1;
		elseif isnan(stdkeepers(i,j)) & gotstart(j)==1 & gotstop(j)~=1
			stop(j)=i-1;
			gotstop(j)=1;	
		end
	end
end
%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%
%%%%	Apply the zero-phase butterworth filter but only to the proxies
	highf=zeros(nrows,ncols);
	lowf=zeros(nrows,ncols); 
	lowf=lowf./lowf;
	highf=highf./highf;
	for k=1:ncols
		clear y y2
		data=stdkeepers(start(k):stop(k),k);
%		lowf(start(k):stop(k),k)=lowpass(data,0.05,2,2);
        frequency = 0.1;
        [smoothed0,icb,ice,mse0] = lowpassmin(data,frequency);
        lowf(start(k):stop(k),k)= smoothed0;
%		lowf(start(k):stop(k),k)=filtfilt(lowa,lowb,data);
		highf(start(k):stop(k),k)=data-lowf(start(k):stop(k),k);
	end
    %%
  inf1=  lowf;
%%-----------------------------------------------------------------

%inf1=t1; % infilled (NaNs are estimated) global mean temp.
%inf1=act1;

% put back NaNs for reconstruction


%---- calculate skill(beta) of reconstructed NH mean------
glb=888
clear aa bb aare aace cc ccw aaw rmulti rm RE CE

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
aare=nanmean(nanmean(act1(1:100,1)));
aace=nanmean(nanmean(act1(101:146,1)));
aareh=nanmean(nanmean(act1h(1:100,1)));
aaceh=nanmean(nanmean(act1h(101:146,1)));
aarel=nanmean(nanmean(act1l(1:100,1)));
aacel=nanmean(nanmean(act1l(101:146,1)));

aa=act1(101:146,1); bb=inf1(101:146,1);
aah=act1h(101:146,1); bbh=inf1h(101:146,1);
aal=act1l(101:146,1); bbl=inf1l(101:146,1);

cc=aa-bb;
cch=aah-bbh;
ccl=aal-bbl;

ccw=cc'*cc;
aaw=aa'*aa;
betamulti=1-ccw/aaw;
ccwh=cch'*cch;
aawh=aah'*aah;
betamultih=1-ccwh/aawh;
ccwl=ccl'*ccl;
aawl=aal'*aal;
betamultil=1-ccwl/aawl;

rmulti=corrcoef(aa,bb);
rm=rmulti(1,2)^2
rmultih=corrcoef(aah,bbh);
rmh=rmultih(1,2)^2
rmultil=corrcoef(aal,bbl);
rml=rmultil(1,2)^2

%---- calculate multivariate RE of reconstruction------
aaw=(aa-aare)'*(aa-aare);
RE=1-ccw/aaw
aawh=(aah-aareh)'*(aah-aareh);
REh=1-ccwh/aawh
aawl=(aal-aarel)'*(aal-aarel);
REl=1-ccwl/aawl

%---- calculate multivariate CE of reconstruction------
aaw=(aa-aace)'*(aa-aace);
CE=1-ccw/aaw
aawh=(aah-aaceh)'*(aah-aaceh);
CEh=1-ccwh/aawh
aawl=(aal-aacel)'*(aal-aacel);
CEl=1-ccwl/aawl

aan=aa; bbn=bb;
veridat(istep,1:9)=[RE CE rm REh CEh rmh REl CEl rml];

end %loop end

save('c:\scozztemann3\z1308\veridat1','veridat','-ascii')

%##########################################################################################    
%##########################################################################################    
%###########################       SH mean         ########################################    
%##########################################################################################    
%##########################################################################################    

clear all; close all;

load c:\scozztemann3\newtemp\shhinfxxxhad % for SH CRU/HAD series
load c:\scozztemann3\newtemp\shlinfxxxhad
act1=shhinfxxxhad+shlinfxxxhad;
act1l=shlinfxxxhad;
act1h=shhinfxxxhad;

%%------------- smooth instrumental data series---------------------
stdkeepers=act1;
    cutoff=10; %years
	lowlim=1./(cutoff./2)
%%%%%%%%%%%%%%
[nrows,ncols]=size(stdkeepers) 
gotstart=zeros(1,ncols);
gotstop=zeros(1,ncols);
start=zeros(1,ncols);
stop=zeros(1,ncols);
stop(:)=nrows;
[lowa,lowb]=butter(5,lowlim,'low');

%%%%%%%%%%%%%%  
%%%% This section finds the starting and stopping year for each column. It assumes that
%%%%  there are no missing values between start and stop, otherwise filtering chokes.

for i=1:nrows
	for j=1:ncols
		if ~isnan(stdkeepers(i,j)) & gotstart(j)~=1
			start(j)=i;
			gotstart(j)=1;
		elseif isnan(stdkeepers(i,j)) & gotstart(j)==1 & gotstop(j)~=1
			stop(j)=i-1;
			gotstop(j)=1;	
		end
	end
end
%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%
%%%%	Apply the zero-phase butterworth filter but only to the proxies
	highf=zeros(nrows,ncols);
	lowf=zeros(nrows,ncols); 
	lowf=lowf./lowf;
	highf=highf./highf;
	for k=1:ncols
		clear y y2
		data=stdkeepers(start(k):stop(k),k);
%		lowf(start(k):stop(k),k)=lowpass(data,0.05,2,2);
        frequency = 0.1;
        [smoothed0,icb,ice,mse0] = lowpassmin(data,frequency);
        lowf(start(k):stop(k),k)= smoothed0;
%		lowf(start(k):stop(k),k)=filtfilt(lowa,lowb,data);
		highf(start(k):stop(k),k)=data-lowf(start(k):stop(k),k);
	end
    %%
  act1=  lowf;
%%-----------------------------------------------------------------
% do loop calculation
veridat=[];

for istep=1:19
%istep=1

in1=101; in2=146; % removed rows of global mean temp.

t1=load(strcat('c:\scozztemann3\z1308\veri\shrech\recon',num2str(istep)));
t2=load(strcat('c:\scozztemann3\z1308\veri\shrecl\recon',num2str(istep)));
    [nrows,ncols]=size(t1)
    
inf1=t1(nrows-145:nrows,1)+t2(nrows-145:nrows,1); % infilled (NaNs are estimated) global mean temp.
inf1h=t1(nrows-145:nrows,1);
inf1l=t2(nrows-145:nrows,1);

%%------------- smooth instrumental data series---------------------
stdkeepers=inf1;
    cutoff=10; %years
	lowlim=1./(cutoff./2)
%%%%%%%%%%%%%%
[nrows,ncols]=size(stdkeepers) 
gotstart=zeros(1,ncols);
gotstop=zeros(1,ncols);
start=zeros(1,ncols);
stop=zeros(1,ncols);
stop(:)=nrows;
[lowa,lowb]=butter(5,lowlim,'low');

%%%%%%%%%%%%%%  
%%%% This section finds the starting and stopping year for each column. It assumes that
%%%%  there are no missing values between start and stop, otherwise filtering chokes.

for i=1:nrows
	for j=1:ncols
		if ~isnan(stdkeepers(i,j)) & gotstart(j)~=1
			start(j)=i;
			gotstart(j)=1;
		elseif isnan(stdkeepers(i,j)) & gotstart(j)==1 & gotstop(j)~=1
			stop(j)=i-1;
			gotstop(j)=1;	
		end
	end
end
%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%
%%%%	Apply the zero-phase butterworth filter but only to the proxies
	highf=zeros(nrows,ncols);
	lowf=zeros(nrows,ncols); 
	lowf=lowf./lowf;
	highf=highf./highf;
	for k=1:ncols
		clear y y2
		data=stdkeepers(start(k):stop(k),k);
%		lowf(start(k):stop(k),k)=lowpass(data,0.05,2,2);
        frequency = 0.1;
        [smoothed0,icb,ice,mse0] = lowpassmin(data,frequency);
        lowf(start(k):stop(k),k)= smoothed0;
%		lowf(start(k):stop(k),k)=filtfilt(lowa,lowb,data);
		highf(start(k):stop(k),k)=data-lowf(start(k):stop(k),k);
	end
    %%
  inf1=  lowf;
%%-----------------------------------------------------------------

%inf1=t1; % infilled (NaNs are estimated) global mean temp.
%inf1=act1;

% put back NaNs for reconstruction


%---- calculate skill(beta) of reconstructed sh mean------
glb=888
clear aa bb aare aace cc ccw aaw rmulti rm RE CE

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
aare=nanmean(nanmean(act1(1:100,1)));
aace=nanmean(nanmean(act1(101:146,1)));
aareh=nanmean(nanmean(act1h(1:100,1)));
aaceh=nanmean(nanmean(act1h(101:146,1)));
aarel=nanmean(nanmean(act1l(1:100,1)));
aacel=nanmean(nanmean(act1l(101:146,1)));

aa=act1(101:146,1); bb=inf1(101:146,1);
aah=act1h(101:146,1); bbh=inf1h(101:146,1);
aal=act1l(101:146,1); bbl=inf1l(101:146,1);

cc=aa-bb;
cch=aah-bbh;
ccl=aal-bbl;

ccw=cc'*cc;
aaw=aa'*aa;
betamulti=1-ccw/aaw;
ccwh=cch'*cch;
aawh=aah'*aah;
betamultih=1-ccwh/aawh;
ccwl=ccl'*ccl;
aawl=aal'*aal;
betamultil=1-ccwl/aawl;

rmulti=corrcoef(aa,bb);
rm=rmulti(1,2)^2
rmultih=corrcoef(aah,bbh);
rmh=rmultih(1,2)^2
rmultil=corrcoef(aal,bbl);
rml=rmultil(1,2)^2

%---- calculate multivariate RE of reconstruction------
aaw=(aa-aare)'*(aa-aare);
RE=1-ccw/aaw
aawh=(aah-aareh)'*(aah-aareh);
REh=1-ccwh/aawh
aawl=(aal-aarel)'*(aal-aarel);
REl=1-ccwl/aawl

%---- calculate multivariate CE of reconstruction------
aaw=(aa-aace)'*(aa-aace);
CE=1-ccw/aaw
aawh=(aah-aaceh)'*(aah-aaceh);
CEh=1-ccwh/aawh
aawl=(aal-aacel)'*(aal-aacel);
CEl=1-ccwl/aawl

aan=aa; bbn=bb;
veridat(istep,1:9)=[RE CE rm REh CEh rmh REl CEl rml];

end %loop end

save('c:\scozztemann3\z1308\veridat2','veridat','-ascii')


